deplomat 0.1.13 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85c420483a9680a16bcffb390c89a1e65ff92734ec191844ddeb93103cfc24c1
-  data.tar.gz: 47632dd964956da6aeb1bfaf7a2ecf953f5254f43a34143f20c4623159661c66
+  metadata.gz: c91e9845ccf8392c7fff51614f56677df22aa87f00d29a1147fcf510c54ef88a
+  data.tar.gz: 4a35bfdfa7ae33d72234e4ef46d6b612b0bc5743285d3055662c44efb5ba62e2
 SHA512:
-  metadata.gz: 11c1ccfc9525ed5bc32a064c88a52c9a861c0186a130271dc165933b0efa1e3faa7f868b83f42104c23d74fda21b2693e96ee3a5050a21fd98f3773451c5e6bd
-  data.tar.gz: 2ceb5111bc322aaab57d84e499eb99e8c6f4856bff2ace60a89151c7fc6ef7374090409f96c9ee7031b5ac2ff396ea99968e4edee41f3bb24ef3e3f2cc24e4c0
+  metadata.gz: 4192a3789c9f630cb1640830ec417d3e05c4331a37c371ab265d6691ec788812a9606bd1a950d2971611a682f43344a88e060c9e27c163daca4cf73cbe033196
+  data.tar.gz: 0e1044ee65039255401f375057b61cea4d55f121c3d851f789bab155f90e0f70a4cc80e109e70da0f1f46566298305636b5a24ffed2766f1c16bb2b96239745a

data/README.rdoc

@@ -1,3 +1,262 @@
-= deplomat
+# Deplomat - a stack agnostic deployment system
+Deplomat is a stack agnostic deployment system that uses bash and ssh commands.
+The purpose of Deplomat is to be a suitable deployment system for all and easily scale from
+a one-man operation to big teams and multiple servers.
 
-Stack agnostic deployment system that uses bash and ssh commands.
+How does it work?
+-----------------
+  * It uses SSH to send commands to remote servers and can also execute scripts on a local machine.
+  * The SSH connection is opened in such a way that it is persistent, so it's fast to execute multiple commands over it;
+  * The deployment script is a simple ruby script. You just create your own methods and then call them.
+
+Let's take a look at an example script, step by step. For the purposes of this tutorial, we'll simplify things.
+The process will resemble a typical web-app deployment, but won't be too specific or complicated.
+
+    #!/usr/bin/env ruby
+    require 'rubygems'
+    require 'deplomat'
+
+    $env          = ARGV[0] || "staging"
+    $local        = Deplomat::LocalNode.new(path: "/home/user/myproject")
+    $server       = Deplomat::RemoteNode.new host: "myproject.com", port: 22, user: "deploy"
+    $branch       = ARGV[1] || $env
+    $app_name     = "myproject"
+    $project_dir  = "/var/www/#{$app_name}"
+    $release_dir  = Time.now.to_i
+
+We've defined a bunch of global variables here. They're global to easily distinguish them from unimportant things, but they might as well
+have been regular local variables. We have also created a `LocalNode` object - it is used to run commands on the local machine; and a `RemoteNode`
+object, which, you guessed it, is used to run commands on a remote machine. You can theoretically have as many different remote nodes
+as you want, but in our example we'll just have one.
+
+Ok, now let's write the actual deployment code.
+
+    # Create the release dir
+    $server.create_dir("#{$project_dir}/#$env/releases/#{$release_dir}")
+
+    # Upload to the remote dir
+    $server.upload("#{$local.current_path}/*", "#{$project_dir}/#$env/releases/#{$release_dir}/")
+    
+    # cd to the release dir we've just created and uploaded, we'll do things inside it.
+    $server.cd("#{$project_dir}/#$env/releases/#{$release_dir}")
+
+Here, we used standard `Deplomat::RemoteNode` methods. First, we used `Deplomat::RemoteNode#create_dir` which
+ran a `"mkdir -p /home/user/myproject/staging/releases/[timestamp]"` command on the server for us. Then we used the 
+`Deplomat::RemoteNode#upload` which uploaded all the files from our project directory to the server. And, finally,
+we've changed current directory on the server to the one we've just created. So far so good, but we now need
+to do a few things on the server before we can restart our webapp:
+
+    $server.create_symlink("#{$project_dir}/#$env/shared/config/database.yml", "config/")
+    $server.create_symlink("#{$project_dir}/#$env/shared/config/secrets.yml", "config/")
+    $server.create_symlink("#{$project_dir}/#$env/shared/config/cable.yml", "config/")
+    $server.create_symlink("#{$project_dir}/#$env/shared/config/redis.yml", "config/")
+    $server.create_symlink("#{$project_dir}/#$env/shared/log", "./")
+    $server.create_symlink("#{$project_dir}/#$env/shared/public/uploads", "public/")
+
+Here, we created symlinks to the files and directories that persist across deployments. For example, the files users
+upload should not evaporate with each new release and so we put them in the `/var/www/myproject/staging/shared/public/uploads`
+directory and then symlink them to the release directory.
+
+For the final steps, we need to migrate the database, instruct the server to restart and symlink the release directory:
+
+    # Migrate DB. Our migration script requires a login shell to work properly,
+    # so we instruct deplomat to run it using a login shell.
+    $server.execute("bin/migrate_db #{$env}", login_shell: true)
+
+    # Restart the server
+    $server.execute("mkdir -p tmp")
+    $server.touch("tmp/restart.txt")
+
+    if $server.file_exists?("#{$project_dir}/#$env/current")
+      $server.mv("#{$project_dir}/#$env/current", "#{$project_dir}/#$env/previous")
+    end
+    $server.create_symlink("#{$project_dir}/#$env/releases/#{$release_dir}", "#{$project_dir}/#$env/current")
+
+You can see how we used `#file_exists?`, `#touch`, `#mv` methods here. Those are also standard methods of `Deplomat::Node`.
+Notice how we checked if `"#{$project_dir}/#$env/current"` exists first, because it might not exist on our first deployment.
+However if it does exist, it's wise to rename this symlink into `previous` so we can later undo the deployment easily by renaming that
+symlink back to `current`.
+
+Our script is ready, now you can run `./deploy` (assuming it's in the root dir of your project and you've made it executable).
+
+
+Adding more structure with methods
+----------------------------------
+Our script above is ok, however as your project grows you'll discover you'd want to add more structure to it. It
+makes sense to group some actions into methods, so you can have something like this:
+
+    #!/usr/bin/env ruby
+    require 'rubygems'
+    require 'deplomat'
+    require 'deployment_steps' # << THIS IS WHERE WE PUT OUR METHODS THAT WE USE BELOW
+
+    $env          = ARGV[0] || "staging"
+    $local        = Deplomat::LocalNode.new(path: "/home/user/myproject")
+    $server       = Deplomat::RemoteNode.new host: "myproject.com", port: 22, user: "deploy"
+    $branch       = ARGV[1] || $env
+    $app_name     = "myproject"
+    $project_dir  = "/var/www/#{$app_name}"
+    $release_dir  = Time.now.to_i
+
+    create_release_dir_and_upload!
+    create_symlinks!
+    migrate_db!
+    restart_server!
+
+This script looks much nicer. We moved deployment code into separate methods into a file we called
+`deployment_steps.rb` and it looks like this:
+
+    def create_release_dir_and_upload!
+      # Create the release dir
+      $server.create_dir("#{$project_dir}/#$env/releases/#{$release_dir}")
+      # Upload to the remote dir
+      $server.upload("#{$local.current_path}/*", "#{$project_dir}/#$env/releases/#{$release_dir}/")
+      # cd to the release dir we've just created and uploaded, we'll do things inside it.
+      $server.cd("#{$project_dir}/#$env/releases/#{$release_dir}")
+    end
+
+    def create_symlinks!
+      $server.create_symlink("#{$project_dir}/#$env/shared/config/database.yml", "config/")
+      $server.create_symlink("#{$project_dir}/#$env/shared/config/secrets.yml", "config/")
+      $server.create_symlink("#{$project_dir}/#$env/shared/config/cable.yml", "config/")
+      $server.create_symlink("#{$project_dir}/#$env/shared/config/redis.yml", "config/")
+      $server.create_symlink("#{$project_dir}/#$env/shared/log", "./")
+      $server.create_symlink("#{$project_dir}/#$env/shared/public/uploads", "public/")
+    end
+
+    def migrate_db!
+      # Migrate DB. Our migration script requires a login shell to work properly,
+      # so we instruct deplomat to run it using a login shell.
+      $server.execute("bin/migrate_db #{$env}", login_shell: true)
+    end
+
+    def restart_server!
+      # Restart the server
+      $server.execute("mkdir -p tmp")
+      $server.touch("tmp/restart.txt")
+
+      if $server.file_exists?("#{$project_dir}/#$env/current")
+        $server.mv("#{$project_dir}/#$env/current", "#{$project_dir}/#$env/previous")
+      end
+      $server.create_symlink("#{$project_dir}/#$env/releases/#{$release_dir}", "#{$project_dir}/#$env/current")
+    end
+
+Notice, we were able to use the same `$server`, `$release_dir`, `$env` and some other variables inside that file precisely
+because we made them global. While global vars are not great for large systems, deployment scripts such as this
+one can take advantage of them without complicating things too much.
+
+Deployment requisites
+---------------------
+Sometimes you need to run a piece of code while deploying the project, but you only need to run it once - that is,
+on one deployment after which it will never be run again. Much like Ruby On Rails runs migrations once and then only runs
+new migrations when necessary.
+
+An example would be when you need to add something to the config file. While you can do it manually by logging into your server
+and editing the config file, it's much more desirable to automate that process, because then you won't forget to do it.
+
+Deplomat has a special feature called Deployment requisites, which allow you to
+
+  * Write special ruby scripts called "requisites", which have access to all the variables and features your script has access to;
+  * Enumarate the tasks that are being run on each deployment and keep track of them by using a special requisite counter (a file created on the server);
+  * Assign each task to be run before or after a particular deployment method in your script;
+
+Let's see how we do that. The first step would be to replace method calls with a call to `#add_task` in your deployment script:
+
+    #!/usr/bin/env ruby
+    require 'rubygems'
+    require 'deplomat'
+    require 'deployment_steps' # << THIS IS WHERE WE PUT OUR METHODS THAT WE USE BELOW
+
+    $env          = ARGV[0] || "staging"
+    $local        = Deplomat::LocalNode.new(path: "/home/user/myproject")
+    $server       = Deplomat::RemoteNode.new host: "myproject.com", port: 22, user: "deploy"
+    $branch       = ARGV[1] || $env
+    $app_name     = "myproject"
+    $project_dir  = "/var/www/#{$app_name}"
+    $release_dir  = Time.now.to_i
+
+    add_task :create_release_dir_and_upload!
+    add_task :create_symlinks!
+    add_task :migrate_db!
+    add_task :restart_server!
+
+    execute_tasks!
+
+This script's behavior is equivalent to the one we had previously. We can make it shorter though by writing:
+
+    # --- top part with requires and global var settings ommited ---
+
+    add_task :create_release_dir_and_upload!, :create_symlinks!, :migrate_db!, :restart_server!
+    execute_tasks!
+
+
+We'll add two lines of code that will read the current requisite number from the server and then load the requisites from
+a local directory called `./deployment_requisites/`. You can change the defaults by passing an additional `path:` argument
+to the `load_requisites!` method, but we're not going to do it here.
+
+    # --- top part with requires and global var settings ommited ---
+
+    # This method is kind of a callback, it's called automatically after every #before_task
+    # or #after_task call. We need to define it manually here,
+    # otherwise requisite number will be stuck on the same number
+    # and never updated.
+    def update_requisite_number!(n)
+      $server.update_requisite_number!(n)
+    end
+
+    # read current requisite number from the server
+    req_n = $server.current_requisite_number
+    
+    # load requisites
+    load_requisites!(req_n)
+
+    add_task :create_release_dir_and_upload!, :create_symlinks!, :migrate_db!, :restart_server!
+    execute_tasks!
+
+The `#execute_tasks!` method will now not only run your methods, but also run the requisites associated with each task. Now
+you might ask, where's the actual code for the requisites? Let's create two files in the `./deployment_requisites/` dir on
+your local machine:
+
+    # ./deployment_requisites/1_add_var1_to_config_file.rb
+    before_task(:migrate_db, 1) do
+      $server.execute("cat 'var1: true' >> config/secrets.yml")
+    end
+
+    # ./deployment_requisites/2_add_var2_to_config_file.rb
+    after_task(:create_release_dir_and_upload!, 2) do
+      $server.execute("cat 'var2: true' >> config/secrets.yml")
+    end
+
+Notice two things here:
+
+  * Filenames start with a number. That's very important: each new requisite file should get a number that's larger than the previous number;
+  * When calling `before_task` or `after_task` the second argument should always be the number that equals that
+    consecutive number in the file name;
+  * You can have only one call to `before_task` or `after_task` per requisite file.
+
+When you deploy, that's what's going to happen:
+
+  * Deplomat will check for the requisite number in the counter file at `"#{@current_path}/.deployment_requisites_counter"` (that's
+    the default location, can be changed by passing an additional argument to `Deplomat::Node#current_requisite_number` and 
+    `Deplomat::Node#update_requisite_number`, see sources);
+
+  * Use that fetched counter number to run only requisites with the numbers that are higher;
+
+  * Update that number upon each requisite script completion (so if there's an error somehwere, we're still left at the exact requisite number
+    that ran successfully). The update is done by calling the `#update_requisite_number!` we defined in our deployment script.
+
+
+Where can I find the list of all methods?
+----------------------------------------
+For now, because this is a relatively small library, you're better off browsing the sources, specifically:
+
+  * [Deplomat::Node](lib/deplomat/node.rb)
+  * [Deplomat::LocalNode](lib/deplomat/local_node.rb)
+  * [Deplomat::RemoteNode](lib/deplomat/remotenode.rb)
+
+TODO
+-----
+
+  * `Deplomat::Node` methods to read and update .yml files easily (usecase: update config files in requisite scripts).
+  * Include `git-archive-all` script
+  * Include scripts for default Ruby On Rails deployments (perhaps as a separate repo)

data/VERSION

@@ -1 +1 @@
-0.1.13
+0.2.0

data/deplomat.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: deplomat 0.1.13 ruby lib
+# stub: deplomat 0.2.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "deplomat".freeze
-  s.version = "0.1.13"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Roman Snitko".freeze]
-  s.date = "2018-11-17"
+  s.date = "2018-12-17"
   s.description = "Stack agnostic deployment system that uses bash and ssh commands".freeze
   s.email = "roman.snitko@gmail.com".freeze
   s.extra_rdoc_files = [
@@ -35,7 +35,11 @@ Gem::Specification.new do |s|
     "lib/deplomat/local_node.rb",
     "lib/deplomat/node.rb",
     "lib/deplomat/remote_node.rb",
+    "spec/deployment_requisites/1_req.rb",
+    "spec/deployment_requisites/2_req.rb",
+    "spec/deployment_requisites/not_a_task.rb",
     "spec/directives_spec.rb",
+    "spec/fixtures/.deployment_requisites_counter",
     "spec/fixtures/cleaning/.keep",
     "spec/fixtures/dir1/file1",
     "spec/fixtures/dir1/file2",

data/lib/deplomat/directives.rb

@@ -1,4 +1,6 @@
-$partials = {}
+$partials                   = {}
+$deplomat_tasks             = []
+$deplomat_tasks_callbacks   = {}
 
 def execute_in(env:)
   yield if $env == env
@@ -23,3 +25,58 @@ def print_to_terminal(message, color: nil, newline: true)
   message = message.send(color) if color
   $stdout.print message
 end
+
+# This wrapper allows us to insert external before/after
+# tasks into every method executed by deplomat.
+# Very useful for the one-time-tasks implementation.
+def add_task(*tasks)
+  $deplomat_tasks += tasks
+end
+alias :add_tasks :add_task
+
+def before_task(task_name, requisite_number, &block)
+  $deplomat_tasks_callbacks[task_name] ||= {}
+  $deplomat_tasks_callbacks[task_name][:before] ||= []
+  $deplomat_tasks_callbacks[task_name][:before] << lambda { block.call; update_requisite_number!(requisite_number) }
+end
+
+def after_task(task_name, requisite_number, &block)
+  $deplomat_tasks_callbacks[task_name] ||= {}
+  $deplomat_tasks_callbacks[task_name][:after] ||= []
+  $deplomat_tasks_callbacks[task_name][:after] << lambda { block.call; update_requisite_number!(requisite_number) }
+end
+
+def execute_tasks!
+  $deplomat_tasks.each do |t|
+    $deplomat_tasks_callbacks[t][:before].each { |block| block.call } if $deplomat_tasks_callbacks[t] && $deplomat_tasks_callbacks[t][:before]
+    self.send(t)
+    $deplomat_tasks_callbacks[t][:after].each { |block| block.call } if $deplomat_tasks_callbacks[t] && $deplomat_tasks_callbacks[t][:after]
+  end
+end
+
+# This should be redefined in the deployment script.
+def update_requisite_number!(n)
+  # call Node#update_requisite_number! here
+  # or do something else, more complicated.
+end
+
+def load_requisites!(counter, requisites_path: "./deployment_requisites")
+
+  files = Dir["#{requisites_path}/*.rb"].map do |fn|
+    number = fn.split("/").last.match(/\A\d+_/).to_a.first&.chomp("_")
+    number ? [number, fn] : nil
+  end
+    .compact                            # ignore files that don't start with a number
+    .sort { |x,y| x.first <=> y.first } # Sort files according to the number in front of their names
+    .map { |fn| fn.last }               # lose the folded Array we used to store the consecutive requisite number
+    .slice(counter..-1)                 # lose files whose consecutive requisite number is below or equals the counter
+
+  log = []
+  files.each do |fn|
+    log << "Loading requisite: #{fn.split("/").last }"
+    require fn
+  end
+
+  return { counter: counter + files.length, log: log }
+
+end

data/lib/deplomat/exceptions.rb

@@ -1,4 +1,5 @@
 module Deplomat
   class ExecutionError  < Exception;end
   class NoSuchPathError < Exception;end
+  class RequisitesNumberError < Exception; end
 end

data/lib/deplomat/node.rb

@@ -125,6 +125,25 @@ module Deplomat
       execute("git checkout #{target}")
     end
 
+    def update_requisite_number!(n, counter_file_path: "#{@current_path}/.deployment_requisites_counter")
+      current_number = current_requisite_number(counter_file_path)
+      if n <= current_number
+        raise Deplomat::RequisitesNumberError, message: "New requisite number (#{n}) is below or equals the current one (#{current_number}). " + 
+                                                        "Something must have gone wrong."
+      else
+        File.open(counter_file_path, "w") { |f| f.puts n }
+      end
+    end
+
+    def current_requisite_number(fn="#{@current_path}/.deployment_requisites_counter")
+      if file_exists?(fn)
+        return File.readlines(fn).first.to_i
+      else
+        raise Deplomat::RequisitesNumberError, message: "Requisite counter file `#{@current_path}/.deployment_requisites_counter` doesn't exist. " +
+                                                        "Please create it manually and symlink it in the deployment script if necessary."
+      end
+    end
+
     def clean(path: @current_path, except: [], leave: [0, :last])
       # Gets us all entries sorted by date, most recent ones first
       entries_by_date = execute("ls -t", path, log_command: false)[:out].split("\n")

data/spec/deployment_requisites/1_req.rb

@@ -0,0 +1,3 @@
+before_task(:task1, 1) do
+  $deployment_requisites_test_file1 = true
+end

data/spec/deployment_requisites/2_req.rb

@@ -0,0 +1,3 @@
+before_task(:task2, 2) do
+  $deployment_requisites_test_file2 = true
+end

data/spec/directives_spec.rb

@@ -2,6 +2,10 @@ require 'spec_helper'
 
 describe "directives" do
 
+  before(:each) do
+    $deplomat_tasks = []
+  end
+
   it "executes arbitrary ruby code in a particular environment" do
     $env = 'staging'
     expect(self).to receive(:hello).exactly(1).times
@@ -20,6 +24,41 @@ describe "directives" do
     execute_partial "hello_partial", x: 'hello'
   end
 
-  it "passes before and after messages"
+  it "executes tasks consecutively, running before and after callbacks" do
+
+    $out = ""
+    def task1; $out += "task1"; end
+    def task2; $out += "task2"; end
+    def task3; $out += ",task3"; end
+    
+    before_task(:task1, 1) { $out += "before_task1:" }
+    after_task(:task1,  2) { $out += ":after_task1,"  }
+    before_task(:task2, 3) { $out += "before_task2:" }
+    after_task(:task2,  4) { $out += ":after_task2"  }
+    before_task(:task4, 5) { $out += "before_task4:" }
+    after_task(:task4,  6) { $out += ":after_task4"  }
+
+    add_task :task1, :task2, :task3
+
+    execute_tasks!
+    expect($out).to eq("before_task1:task1:after_task1,before_task2:task2:after_task2,task3")
+
+  end
+
+  it "loads and runs requisites" do
+
+    def task1;end
+    def task2;end
+
+    add_task :task1, :task2
+
+    result = load_requisites!(1, requisites_path: "#{File.expand_path(File.dirname(__FILE__))}/deployment_requisites")
+    expect(result[:counter]).to eq(2)
+    expect(result[:log]).to eq(["Loading requisite: 2_req.rb"])
+    execute_tasks!
+    expect($deployment_requisites_test_file1).to be_nil
+    expect($deployment_requisites_test_file2).to be_truthy
+
+  end
 
 end

data/spec/fixtures/.deployment_requisites_counter

@@ -0,0 +1 @@
+1

data/spec/node_spec.rb

@@ -44,6 +44,21 @@ describe Deplomat::Node do
       @node.execute("ls #{@fixtures_dir}/dir1", message: ["hello", "bye"])
     end
 
+    it "updates requisite counter" do
+
+      requisites_counter_fn = "#{Dir.pwd}/spec/.deployment_requisites_counter"
+      @node.cd Dir.pwd + "/spec"
+
+      File.open(requisites_counter_fn, "w") { |f| f.puts 1 }
+      @node.update_requisite_number!(2)
+      expect(File.readlines(requisites_counter_fn).first.to_i).to eq(2)
+
+      expect(-> { @node.update_requisite_number!(1) }).to raise_exception(Deplomat::RequisitesNumberError)
+
+      File.unlink(requisites_counter_fn)
+      expect(-> { @node.update_requisite_number!(3) }).to raise_exception(Deplomat::RequisitesNumberError)
+    end
+
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deplomat
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.2.0
 platform: ruby
 authors:
 - Roman Snitko
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-17 00:00:00.000000000 Z
+date: 2018-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sys-proctable
@@ -104,7 +104,11 @@ files:
 - lib/deplomat/local_node.rb
 - lib/deplomat/node.rb
 - lib/deplomat/remote_node.rb
+- spec/deployment_requisites/1_req.rb
+- spec/deployment_requisites/2_req.rb
+- spec/deployment_requisites/not_a_task.rb
 - spec/directives_spec.rb
+- spec/fixtures/.deployment_requisites_counter
 - spec/fixtures/cleaning/.keep
 - spec/fixtures/dir1/file1
 - spec/fixtures/dir1/file2