tinyci 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5531654099be53cbad1500af1c180555e4453396
-  data.tar.gz: 4f90146f6f2aaf50b78fa162a363a8d80666f810
+  metadata.gz: 39e61578a53b2ca58444388af0f0b1b3653c2896
+  data.tar.gz: 86be8e7dbf6d742a8814323ccf75d5c1ce645c22
 SHA512:
-  metadata.gz: 697dbb6e5e32e9bede86917e1466093180c87a0a327946ffcc12fab7fcc7c06aad91be64d11004cccef9f127a96dd892bdfbaf1783dce2ca233eb5b779efd355
-  data.tar.gz: 48097b1dc7213415c2d462dc055beb50ab77b6467f71f488dd12e34870ec59f7e2ce4abb41e00d5a135b2c8a0eafff37b69221c409c39aef2058448c67f9d17b
+  metadata.gz: 864c7040005a7a6f778bb5392c580847e35024e160150063148d52a0ddcf059b5ab0f5f56f756efc8907eb16a74c0aea071585b1a399413dde3611a83dca92e0
+  data.tar.gz: b84f48399d21f174c485c2b3626fb35ca3b05d89f0b5547e06388848caaa5b2672e1c55ac8bdd4adf73ea291e710af44f3bfb03894289a651b002e34839a956d

data/.tinyci.yml

@@ -0,0 +1,8 @@
+builder:
+  class: ScriptBuilder
+  config:
+    command: docker build -t tinyci .
+tester:
+  class: ScriptTester
+  config:
+    command: docker run tinyci bundle exec rspec --format documentation

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+0.3.0 - October 18, 2018
+  Add compactor
+  Fixes to hooks
 0.2.0 - October 13, 2018
   Add hooks
 0.1.2 - August 16, 2018

data/Dockerfile

@@ -0,0 +1,38 @@
+FROM alpine:3.8
+
+RUN mkdir -p /etc \
+  && { \
+    echo 'install: --no-document'; \
+    echo 'update: --no-document'; \
+  } >> /etc/gemrc
+
+RUN apk add --no-cache -u \
+  ruby \
+  ruby-dev \
+  ruby-irb \
+  ruby-etc \
+  libffi-dev \
+  build-base \
+  git
+
+RUN git config --global user.email "you@example.com"
+RUN git config --global user.name "Your Name"
+
+# RUN ln -sf /usr/share/zoneinfo/Europe/London /etc/localtime
+
+RUN gem install bundler
+RUN bundle config --global silence_root_warning 1
+
+WORKDIR /tmp 
+ADD Gemfile Gemfile
+ADD Gemfile.lock Gemfile.lock
+ADD tinyci.gemspec tinyci.gemspec
+ADD lib/tinyci/version.rb lib/tinyci/version.rb
+ADD lib/tinyci/logo.txt lib/tinyci/logo.txt
+RUN bundle install 
+
+ADD . /tinyci
+
+WORKDIR /tinyci
+
+CMD bin/tinyci

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tinyci (0.2.0)
+    tinyci (0.3.0)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -107,6 +107,19 @@ hooker:
     after_build: ./after_build.sh
 ```
 
+#### Compactor
+
+With continued use, the `builds` directory will grow ever larger. TinyCI provides the `compact` command to deal with this. It compresses old builds into `.tar.gz` files.
+
+"Old" in this context is defined using two options to the `tinyci compact` command:
+
+* `--num-builds-to-leave` - How many build directories to leave in place, starting from the newest. Defaults to `1`.
+* `--builds-to-leave` - A comma-separated list of specific build directories to leave in place.
+
+The latter option is intended for use in an automated deployment system, to allow the script to run without removing builds that are being used somewhere else in the system. For a demonstration of this, see the [example project](#Example_Project).
+
+To use it, simply run `tinyci compact` from the root of the repo you wish to compact.
+
 #### Logging/Output
 
 TinyCI is executed in a `post-update` git hook. As such, the output is shown to the user as part of the local call to `git push`. Once the TinyCI hook is running, the local git process can be freely killed if the user does not wish to watch the output - this will not affect the remote execution of TinyCI.
@@ -123,8 +136,6 @@ Instead, this second TinyCI execution should begin tailing the output of the fir
 
 * It would be desirable to add a command to the TinyCI binary to SSH into the remote server and tail the the log output of a currently running TinyCI process, enabling functionality similar to that described in above, but without making a new commit.
 
-* The exported copies of each build are left on the disk forever. This will obviously eat disk space. One could handle deleting/compressing them with a cronjob, however it would be desirable to integrate this functionality into TinyCI and to automate it.
-
 * More configuration options should be added, eg. specification of the export path.
 
 * In general, more local functionality for the TinyCI binary would be desirable, some way to retrieve the results of tests and query them from the local clone for example, or some way to show them in `git log` output.
@@ -160,85 +171,9 @@ Now, commit the configuration file and push it to your remote repository. As dis
 
 #### Example Project
 
-Here we will demonstrate cloning the [tinyci-example](https://github.com/JonnieCache/tinyci-example) project, creating a bare clone of it to simulate our server where the tests will run, and pushing a commit to see TinyCI in action.
-
-The example project has very simple build and test scripts written in bash.
-
-First, create a directory to store both clones:
-
-    $ mkdir tinyci-test
-    $ cd tinyci-test
-
-Clone the example project from github:
-
-    $ git clone https://github.com/JonnieCache/tinyci-example.git
-    
-    Cloning into 'tinyci-example'...
-    remote: Counting objects: 8, done.
-    remote: Compressing objects: 100% (6/6), done.
-    remote: Total 8 (delta 0), reused 8 (delta 0), pack-reused 0
-    Unpacking objects: 100% (8/8), done.
-    
-Clone it again into a bare repository:
-
-    $ git clone --bare tinyci-example tinyci-example-bare
-    
-    Cloning into bare repository 'tinyci-example-bare'...
-    done.
-    
-Install tinyci:
-
-    $ gem install tinyci
-    
-Install the tinyci hook into our bare clone:
-
-    $ cd tinyci-example-bare
-    $ tinyci install
-    
-    [09:48:42] tinyci post-update hook installed sucessfully
-    
-Go into our initial clone and make a change:
-
-    $ cd ../tinyci-example
-    $ echo "foo" > bar
-    $ git add bar
-    $ git commit -m 'foobar'
-    
-    [master cebb4ec] foobar
-     1 file changed, 1 insertion(+)
-     create mode 100644 bar
-    
-Add our bare repo as a remote:
-
-    $ git remote add test ../tinyci-example-bare
-    
-Push the commit and watch tinyci in action:
-
-    $ git push test master
-    
-    Counting objects: 3, done.
-    Delta compression using up to 4 threads.
-    Compressing objects: 100% (2/2), done.
-    Writing objects: 100% (3/3), 268 bytes | 0 bytes/s, done.
-    Total 3 (delta 1), reused 0 (delta 0)
-    remote: [09:49:53] Commit: 22c36d0c1213361d06b385d2b55648985347e1f4
-    remote: [09:49:53] Cleaning...
-    remote: [09:49:53] Exporting...
-    remote: [09:49:53] Building...
-    remote: [09:49:53] Testing...
-    remote: [09:49:53] foo bar
-    remote: [09:49:53] Finished 22c36d0c1213361d06b385d2b55648985347e1f4
-    remote: [09:49:53] Commit: cebb4ec18b89f093c7cdeb909c2c340708052575
-    remote: [09:49:53] Cleaning...
-    remote: [09:49:53] Exporting...
-    remote: [09:49:53] Building...
-    remote: [09:49:53] Testing...
-    remote: [09:49:53] foo bar
-    remote: [09:49:53] Finished cebb4ec18b89f093c7cdeb909c2c340708052575
-    To ../tinyci-example-bare
-       22c36d0..cebb4ec  master -> master
+There is an example project available at https://github.com/JonnieCache/tinyci-example.
     
-Here we are seeing TinyCI testing both the initial commit that is present in the github repo, an then the new commit that we just made. Breaking the test to see the failure output is left as an exercise for the reader.
+See that repo for a walkthrough demonstrating various TinyCI features.
     
 ### Contributing
 

data/lib/tinyci/cli.rb

@@ -1,8 +1,10 @@
 require 'tinyci/version'
 require 'tinyci/scheduler'
 require 'tinyci/installer'
+require 'tinyci/compactor'
 require 'tinyci/git_utils'
 require 'optparse'
+require 'pidfile'
 
 module TinyCI
   # Defines the CLI interface. Uses OptionParser.
@@ -29,6 +31,12 @@ module TinyCI
         'install' => OptionParser.new do |o|
           o.banner = "Usage: install [options]"
           o.on("-q", "--[no-]quiet", "quietly run") {|v| opts[:quiet] = v}
+        end,
+        'compact' => OptionParser.new do |o|
+          o.banner = "Usage: compact [options]"
+          o.on("-n", "--num-builds-to-leave <NUM>", "number of builds to leave in place, starting from the most recent") {|n| opts[:num_builds_to_leave] = n}
+          o.on("-b", "--builds-to-leave <BUILDS>", "specific build directories to leave in place, comma-separated") {|b| opts[:builds_to_leave] = b.split(',')}
+          o.on("-q", "--[no-]quiet", "quietly run") {|v| opts[:quiet] = v}
         end
       }
       
@@ -40,6 +48,7 @@ Global options:
 Available commands:
     run      build and test the repo
     install  install the git hook into the current repository
+    compact  compress old build artifacts
     version  print the TinyCI version number
 TXT
       if argv[0] == '--help'
@@ -95,6 +104,12 @@ TXT
       TinyCI::Installer.new(logger: logger, working_dir: opts[:dir]).write!
     end
     
+    def self.do_compact(opts)
+      logger = MultiLogger.new(quiet: opts[:quiet])
+      
+      TinyCI::Compactor.new(logger: logger, working_dir: opts[:dir], num_builds_to_leave: opts[:num_builds_to_leave], builds_to_leave: opts[:builds_to_leave]).compact!
+    end
+    
     def do_version(opts)
       puts TinyCI::VERSION
       

data/lib/tinyci/compactor.rb

@@ -0,0 +1,99 @@
+require 'fileutils'
+require 'tinyci/git_utils'
+require 'zlib'
+require 'rubygems/package'
+require 'find'
+
+module TinyCI
+  
+  # Tool for compressing old builds into .tar.gz files
+  class Compactor
+    include GitUtils
+    include Subprocesses
+    
+    BLOCKSIZE_TO_READ = 1024 * 1000
+    
+    # Constructor
+    # 
+    # @param [String] working_dir The directory from which to run.
+    # @param [Integer] num_builds_to_leave How many builds not to compact, starting from the newest
+    # @param [String] builds_to_leave Comma-separated list of build directory names not to compact
+    # @param [Logger] logger Logger object
+    def initialize(working_dir: nil, num_builds_to_leave: nil, builds_to_leave: nil, logger: nil)
+      @logger = logger
+      @working_dir = working_dir || repo_root
+      @num_builds_to_leave = (num_builds_to_leave || 1).to_i
+      @builds_to_leave = builds_to_leave || []
+    end
+    
+    # Compress and delete the build directories
+    def compact!
+      unless inside_repository?
+        log_error "not currently in a git repository"
+        return false
+      end
+      
+      directories_to_compact.each do |dir|
+        compress_directory dir
+        FileUtils.rm_rf builds_dir(dir)
+        
+        log_info "Compacted #{archive_path(dir)}"
+      end
+    end
+    
+    private
+    
+    # Build the list of directories to compact according to the options
+    def directories_to_compact
+      builds = Dir.entries builds_dir
+      builds.select! {|e| File.directory? builds_dir(e) }
+      builds.reject! {|e| %w{. ..}.include? e }
+      builds.sort!
+      
+      builds = builds[0..-(@num_builds_to_leave+1)]
+      builds.reject! {|e| @builds_to_leave.include?(e) || @builds_to_leave.include?(builds_dir(e, 'export'))}
+
+      builds
+    end
+    
+    # Get the location of the builds directory
+    def builds_dir(*path_segments)
+      File.join @working_dir, 'builds/', *path_segments
+    end
+    
+    # Build the path for a compressed archive
+    def archive_path(dir)
+      File.join(builds_dir, dir+".tar.gz")
+    end
+    
+    # Create a .tar.gz file from a directory
+    # Done in pure ruby to ensure portability
+    def compress_directory(dir)
+      File.open archive_path(dir), 'wb' do |oarchive_path|
+        Zlib::GzipWriter.wrap oarchive_path do |gz|
+          Gem::Package::TarWriter.new gz do |tar|
+            Find.find "#{builds_dir}/"+dir do |f|
+              relative_path = f.sub "#{builds_dir}/", ""
+              mode = File.stat(f).mode
+              size = File.stat(f).size
+              
+              if File.directory? f
+                tar.mkdir relative_path, mode
+              else
+                tar.add_file_simple relative_path, mode, size do |tio|
+                  File.open f, 'rb' do |rio|
+                    while buffer = rio.read(BLOCKSIZE_TO_READ)
+                      tio.write buffer
+                    end
+                  end
+                end
+              end
+              
+            end
+          end
+        end
+      end
+      
+    end
+  end
+end

data/lib/tinyci/runner.rb

@@ -83,7 +83,7 @@ module TinyCI
           raise e if ENV['TINYCI_ENV'] == 'test'
           
           log_error e
-          log_error e.backtrace
+          log_debug e.backtrace
           
           return false
         ensure
@@ -101,7 +101,7 @@ module TinyCI
           raise e if ENV['TINYCI_ENV'] == 'test'
           
           log_error e
-          log_error e.backtrace
+          log_debug e.backtrace
           
           return false
         ensure
@@ -116,7 +116,7 @@ module TinyCI
         raise e if ENV['TINYCI_ENV'] == 'test'
         
         log_error e
-        log_error e.backtrace
+        log_debug e.backtrace
         return false
       end
       
@@ -182,10 +182,12 @@ module TinyCI
       Time.at execute(git_cmd('show', '-s', '--format=%ct', @commit)).to_i
     end
 
+    # Ensure a path exists
     def ensure_path(path)
       execute 'mkdir', '-p', path
     end
     
+    # Delete the export path
     def clean
       FileUtils.rm_rf export_path
     end

data/lib/tinyci/scheduler.rb

@@ -43,7 +43,7 @@ module TinyCI
       pid = PidFile.new(pidfile: 'tinyci.pid', piddir: @working_dir)
       
       result = if @commit
-        run_commit @commit
+        run_commit get_commit @commit
       else
         run_all_commits
       end
@@ -60,16 +60,15 @@ module TinyCI
     # @return [Array<String>] the sha1 hashes in reverse order of creation time
     def get_commits
       log = execute(git_cmd('log', '--notes=tinyci*', '--format=%H %ct %N§§§', '--reverse'))
-      
       lines = log.split("§§§")
-      lines.map do |line|
-        parts = line.split(' ')
-        {
-          sha: parts[0],
-          time: parts[1],
-          result: parts[2]
-        }
-      end.select {|c| c[:result].nil?}
+      
+      lines.map {|l| format_commit_data(l)}.select {|c| c[:result].nil?}
+    end
+    
+    def get_commit(sha)
+      data = execute(git_cmd('show', '--quiet', '--notes=tinyci*', "--format=%H %ct", sha))
+      
+      format_commit_data(data)
     end
     
     # Instantiates {Runner} for a given git object, runs it, and stores the result
@@ -84,6 +83,15 @@ module TinyCI
       set_result(commit, result)
     end
     
+    def format_commit_data(data)
+      parts = data.split(' ')
+      {
+        sha: parts[0],
+        time: parts[1],
+        result: parts[2]
+      }
+    end
+    
     # Repeatedly gets the list of eligable commits and runs TinyCI against them until there are no more remaining
     def run_all_commits
       commits = get_commits

data/lib/tinyci/subprocesses.rb

@@ -21,7 +21,7 @@ module TinyCI
       
       unless status.success?
         log_error output
-        raise SubprocessError.new(label, command.join(' '), status.to_i)
+        raise SubprocessError.new(label, command.join(' '), status)
       end
       
       output.chomp
@@ -43,7 +43,7 @@ module TinyCI
         status = waiter.value
         unless status.success?
           log_error output
-          raise SubprocessError.new(label, commands[i].join(' '), status.to_i)
+          raise SubprocessError.new(label, commands[i].join(' '), status)
         end
       end
       

data/lib/tinyci/version.rb

@@ -1,3 +1,3 @@
 module TinyCI
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tinyci
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Jonathan Davies
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-13 00:00:00.000000000 Z
+date: 2018-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -233,8 +233,10 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".ruby-version"
+- ".tinyci.yml"
 - ".yardopts"
 - CHANGELOG.md
+- Dockerfile
 - Gemfile
 - Gemfile.lock
 - Guardfile
@@ -247,6 +249,7 @@ files:
 - lib/tinyci/builders/script_builder.rb
 - lib/tinyci/builders/test_builder.rb
 - lib/tinyci/cli.rb
+- lib/tinyci/compactor.rb
 - lib/tinyci/config.rb
 - lib/tinyci/executor.rb
 - lib/tinyci/git_utils.rb
@@ -271,7 +274,7 @@ metadata:
   allowed_push_host: https://rubygems.org
 post_install_message: " _____ _               _____  _____\n/__   (_)_ __  _   _  /
   ___/ /_  _/\n   | || | '_ \\| | | |/ /     / /\n   | || | | | | |_| / /___/\\/ /_
-  \ \n   |_||_|_| |_|\\__, \\____/\\____/  0.2.0\n               |___/\n\n"
+  \ \n   |_||_|_| |_|\\__, \\____/\\____/  0.3.0\n               |___/\n\n"
 rdoc_options: []
 require_paths:
 - lib