concourse-technician 0.3.4 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e124b3ec79ff6952ad28547f53e6ab42271d509a
-  data.tar.gz: 2a816dbb8ee2c1ec2cc32b7ec095c019af487994
+  metadata.gz: 4c97c608157d2c56434e208f79d548d58fd591c7
+  data.tar.gz: 5717fe2194475f37fa01e47ff57d0c9af67eefbb
 SHA512:
-  metadata.gz: 9967df2118470c437ceb6c788b2fc9f8c7eabffd03577d4db874f8bd43d0999bdc40733a156dd1591a79532cf85072d4c044717f2588c1492590153c1d3f70f9
-  data.tar.gz: deea7e620f9b8fba9b213397fbe7d951c38aef8aa4c0d1f58268825aada852d05fb3632319f83e5e2654d67766f57c9953f0ffc401fc85ef783e1b3008377460
+  metadata.gz: 194f576c50f1459eb1b1a48b5387f9281256e8efee93877c928e7fd684202a7cdcc325aaed2a16091d298c65440d530f857ebc6fca08df446c5d8cca31a67dce
+  data.tar.gz: 94af80ccab86ad6dc1dd36b941453f086aa86c518f00e9bc75d8c03b7f670e807e94204f54fab295438c564c8a67b293a8cb9ac11f3ae007f494f50e31c8019e

data/README.org

@@ -11,23 +11,56 @@
     gem install concourse-technician
   #+END_SRC
 
+* Configuration
+
+  ~concourse-technician~ needs to be pointed at your Concourse database. It
+  looks for a configuration file at =~/.config/concourse-technician.yaml=. To
+  tell it to look somewhere else, set =CONCOURSE_TECHNICIAN_CONFIG= in the
+  environment.
+
+** Example
+
+   #+BEGIN_SRC yaml
+     ---
+     adapter: postgres
+     database: my-awesome-database
+     host: localhost
+     password: MyVerySecurePasswordStoredUnencryptedOnDisk
+     port: 5432
+     user: me
+
+     volumes_root: /concourse/volumes
+   #+END_SRC
+
 * Usage
 
   To get a list of abandoned volumes...
 
   #+BEGIN_SRC shell
-    technician abandoned_volumes
+    technician database abandoned_volumes
   #+END_SRC
 
   To clean up all abandoned volumes...
 
   #+BEGIN_SRC shell
-    technician delete_abandoned_volumes
+    technician database delete_abandoned_volumes
+  #+END_SRC
+
+  To check if a worker is failing due to issues with volume reaping...
+
+  #+BEGIN_SRC shell
+    technician volume_reaper damaged
+  #+END_SRC
+
+  To resolve issues with volume reaping...
+
+  #+BEGIN_SRC shell
+  technician volume_reaper repair
   #+END_SRC
 
 * License
 
-  ~dbviz~ is available under the [[https://tldrlegal.com/license/mit-license][MIT License]]. See ~LICENSE.txt~ for the full text.
+  ~concourse-technician~ is available under the [[https://tldrlegal.com/license/mit-license][MIT License]]. See ~LICENSE.txt~ for the full text.
 
 * Contributors
 

data/bin/technician

@@ -3,9 +3,13 @@
 require 'json'
 require 'concourse-technician/cli'
 require 'concourse-technician/database'
+require 'concourse-technician/volume_reaper'
 
-ConcourseTechnician::Database.new.tap do |database|
-  ConcourseTechnician::CLI.new(database: database).execute(*ARGV).tap do |out|
-    puts JSON.dump(out.respond_to?(:to_i) ? out.to_i : out.to_a) if out
-  end
+commands = {
+  database: ConcourseTechnician::Database.new,
+  volume_reaper: ConcourseTechnician::VolumeReaper.new
+}
+
+ConcourseTechnician::CLI.new(commands).execute(*ARGV).tap do |out|
+  puts JSON.dump(out.respond_to?(:to_i) ? out.to_i : out.to_a) if out
 end

data/concourse-technician.gemspec

@@ -12,7 +12,9 @@ Gem::Specification.new do |gem|
   gem.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
   gem.require_paths = ['lib']
 
+  gem.add_runtime_dependency 'btrfs', '~> 0.1', '>= 0.1.5'
   gem.add_runtime_dependency 'contracts', '~> 0.14', '>= 0.14.0'
   gem.add_runtime_dependency 'instacli', '~> 1.1', '>= 1.1.2'
   gem.add_runtime_dependency 'sequel', '~> 4.36', '>= 4.36.0'
+  gem.add_runtime_dependency 'systemized', '~> 0.2', '>= 0.2.4'
 end

data/lib/concourse-technician.rb

@@ -2,3 +2,4 @@
 require_relative 'concourse-technician/cli'
 require_relative 'concourse-technician/database'
 require_relative 'concourse-technician/settings'
+require_relative 'concourse-technician/volume_reaper'

data/lib/concourse-technician/volume_reaper.rb

@@ -0,0 +1,133 @@
+require 'btrfs'
+require 'contracts'
+require 'fileutils'
+require 'systemized'
+require 'time'
+require_relative 'settings'
+
+module ConcourseTechnician
+  class VolumeReaper
+    include ::Contracts::Core
+    include ::Contracts::Builtin
+    include Settings
+
+    Contract None => ArrayOf[Time]
+    def timestamps
+      @timestamps ||= worker.journal.read(1000).map do |entry|
+        Time.parse entry.split.first
+      end.sort
+    end
+
+    Contract None => ArrayOf[Hash]
+    def recent_events
+      @recent_events ||= logs.read(100).map do |event|
+        begin
+          JSON.load event
+        rescue JSON::ParserError
+          nil
+        end
+      end.compact
+    end
+
+    Contract None => Any
+    def damaged
+      detected? ? exit : abort
+    end
+
+    Contract None => Any
+    def repair
+      detected ? repair! : abort
+    end
+
+    private
+
+    Contract RespondTo[:to_s] => Bool
+    def env?(key)
+      if ![nil, 'nil', 'false'].include? ENV[key.to_s.upcase]
+        true
+      else
+        block_given? ? yield : false
+      end
+    end
+
+    Contract None => Bool
+    def stagnant_logs?
+      env?(:stagnant_logs) do
+        (Time.now - timestamps.last) > (timestamps.last - timestamps.first)
+      end
+    end
+
+    Contract None => Bool
+    def reaping_failure?
+      env?(:reaping_failure) do
+        recent_events.any? do |event|
+          event['message'] == 'baggageclaim.tick.failed-to-reap'
+        end
+      end
+    end
+
+    Contract RespondTo[:to_s] => nil
+    def report(message)
+      STDERR.puts message.to_s unless env?(:QUIET)
+    end
+
+    Contract None => Bool
+    def detected?
+      stagnant_logs?.tap do |status|
+        report "Most recent log entry @ #{timestamps.last}"
+        report "Logs seem stagnant? #{status}"
+      end && reaping_failure?.tap do |status|
+        report "Recently volume reaping failure? #{status}"
+      end
+    end
+
+    Contract None => ::Systemized::Service
+    def worker
+      @worker = ::Systemized::Service.new('concourse-worker')
+    end
+
+    Contract None => ::Systemized::Journal
+    def logs
+      @logs ||= ::Systemized::Journal.new('concourse-worker', output: 'cat')
+    end
+
+    Contract None => String
+    def volumes_root
+      @volumes_root ||= ENV.fetch('VOLUMES_ROOT') do
+        settings.fetch('volumes_root') { '/concourse/volumes' }
+      end
+    end
+
+    Contract None => ArrayOf[::Btrfs::Subvolume]
+    def subvolumes
+      @subvolumes ||= ::Btrfs::Volume.new(volumes_root).subvolumes
+    end
+
+    Contract None => Maybe[Bool]
+    def reap_subvolumes
+      report "Reaping #{subvolumes.size} subvolumes..."
+      subvolumes.each(&:delete).all?(&:deleted?) unless env?(:NOOP)
+    end
+
+    Contract None => ArrayOf[String]
+    def dead_volumes
+      Dir.glob("#{volumes_root}/dead/*")
+    end
+
+    Contract None => Any
+    def reap_dead_volumes
+      report "Reaping #{dead_volumes.size} dead volumes..."
+      dead_volumes.each do |volume|
+        FileUtils.rmtree volume, verbose: env?(:QUIET), noop: env?(:NOOP)
+      end
+    end
+
+    Contract None => Any
+    def repair!
+      worker.stop if worker.active?
+      reap_subvolumes
+      reap_dead_volumes
+      worker.start
+    end
+  end
+end

metadata

@@ -1,15 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: concourse-technician
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.2
 platform: ruby
 authors:
 - Chris Olstrom
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-18 00:00:00.000000000 Z
+date: 2016-09-15 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: btrfs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.5
 - !ruby/object:Gem::Dependency
   name: contracts
   requirement: !ruby/object:Gem::Requirement
@@ -70,6 +90,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 4.36.0
+- !ruby/object:Gem::Dependency
+  name: systemized
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.4
 description: 
 email: chris@olstrom.com
 executables:
@@ -86,6 +126,7 @@ files:
 - lib/concourse-technician/cli.rb
 - lib/concourse-technician/database.rb
 - lib/concourse-technician/settings.rb
+- lib/concourse-technician/volume_reaper.rb
 homepage: https://github.com/colstrom/concourse-technician
 licenses:
 - MIT
@@ -111,4 +152,3 @@ signing_key:
 specification_version: 4
 summary: Troubleshoot Concourse without fly
 test_files: []
-has_rdoc: