larrow-runner 0.0.1

data/lib/larrow/runner/manifest/configuration.rb

@@ -0,0 +1,126 @@
+module Larrow::Runner::Manifest
+  # The top of manifest model which store Steps information 
+  class Configuration
+    DEFINED_GROUPS = {
+      all:[
+        :init,
+        :source_sync, #inner step
+        :prepare, 
+        :compile, :unit_test,
+        :before_install, #inner_step
+        :install, :functional_test, 
+        :before_start, #inner_step
+        :start, :integration_test,
+        :after_start, :complete #inner_step
+      ],
+      custom: [
+        :init,
+        :prepare, 
+        :compile, :unit_test,
+        :install, :functional_test, 
+        :start, :integration_test,
+      ],
+      deploy: [
+        :init,:source_sync,:prepare, 
+        :compile,:before_install,:install,
+        :before_start,:start,:after_start,
+        :complete
+      ],
+      image: [:init]
+    }
+   
+    attr_accessor :steps, :image, :source_dir
+    def initialize
+      self.steps = {}
+      self.source_dir = '$HOME/source'
+    end
+
+    def put_to_step title, *scripts
+      steps[title] ||= CmdStep.new(nil, title)
+      steps[title].scripts += scripts.flatten
+      self
+    end
+
+    def insert_to_step title, *scripts
+      steps[title] ||= CmdStep.new(nil, title)
+      steps[title].scripts.unshift *scripts.flatten
+      self
+    end
+
+    def add_source_sync source_accessor
+      steps[:source_sync] = FunctionStep.new(:source_sync) do |node|
+        source_accessor.update_source node,source_dir
+      end
+    end
+
+    def steps_for type
+      groups = DEFINED_GROUPS[type]
+      # ignore init when image id is specified
+      groups = groups - [:init] if image
+      groups.each do |title|
+        yield steps[title] if steps[title]
+      end
+    end
+
+    def dump
+      data = DEFINED_GROUPS[:all].reduce({}) do |sum,title|
+        next sum if steps[title].nil?
+        scripts_data = steps[title].scripts.map(&:dump).compact
+        sum.update title.to_s => scripts_data
+      end
+      YAML.dump data
+    end
+  end
+
+  # Describe a set of scripts to accomplish a specific goal
+  class CmdStep
+    attr_accessor :scripts, :title
+    def initialize scripts, title
+      self.scripts = scripts || []
+      self.title = title
+    end
+
+    def run_on node
+      scripts.each do |script|
+        node.execute script.actual_command, base_dir: script.base_dir
+      end
+    end
+  end
+
+  # An abstract step which bind business logic with block
+  # This class designed for some typically service,eg:
+  #   * local file folder sync
+  #   * some service invoke
+  class FunctionStep
+    attr_accessor :block, :title
+    def initialize title, &block
+      self.title = title
+      self.block = block
+    end
+    
+    def run_on node
+      block.call node
+    end
+  end
+
+  # store the real command line
+  #   :cannt_fail used to declare `non-zero retcode of current script will be fail`
+  class Script
+    attr_accessor :cmd, :base_dir, :args, :cannt_fail
+    def initialize cmd, base_dir:nil, args:{}, cannt_fail: true
+      self.cmd = cmd
+      self.args = args
+      self.cannt_fail = cannt_fail
+      self.base_dir = base_dir
+    end
+
+    def actual_command
+      sprintf(cmd, args)
+    end
+
+    def dump
+      return nil if cmd.empty?
+      cmd
+    end
+  end
+end

data/lib/larrow/runner/manifest.rb

@@ -0,0 +1,48 @@
+require 'larrow/runner/manifest/base_loader'
+require 'larrow/runner/manifest/configuration'
+
+module Larrow
+  module Runner
+    # support multiple manifest style, such as travis, larrow, etc...
+    module Manifest
+      # Adapters is a set of class to adapt different manifest style.
+      # There isn't Adapter module, these classes are under Manifest module.
+      autoload :Travis, 'larrow/runner/manifest/adapter/travis'
+      autoload :Larrow, 'larrow/runner/manifest/adapter/larrow'
+      autoload :Blank,  'larrow/runner/manifest/adapter/blank'
+
+      extend self
+
+      def configuration source_accessor
+        [ Larrow, Travis, Blank ].each do |clazz|
+          configuration = clazz.new(source_accessor).load
+          return configuration if configuration
+        end
+      end
+
+      def add_base_scripts configuration,source_accessor
+        configuration.add_source_sync source_accessor
+        unless configuration.image
+          lines = <<-EOF
+#{package_update}
+#{bashrc_cleanup}
+          EOF
+          scripts = lines.split(/\n/).map{|s| Script.new s}
+          configuration.insert_to_step :init, scripts
+        end
+      end
+
+      def package_update
+        <<-EOF
+apt-get update -qq
+apt-get install git libssl-dev build-essential curl libncurses5-dev -y -qq
+        EOF
+      end
+
+      # remove PS1 check, for user to make ssh connection without tty
+      def bashrc_cleanup
+        "sed '/$PS1/ d' -i /root/.bashrc"
+      end
+    end
+  end
+end

data/lib/larrow/runner/model/app.rb

@@ -0,0 +1,62 @@
+module Larrow::Runner
+  module Model
+    class App
+
+      attr_accessor :vcs, :node, :configuration
+      def initialize vcs, attributes={}
+        self.vcs = vcs
+        self.configuration = vcs.configuration
+        self.assign attributes unless attributes.empty?
+      end
+
+      def assign arg
+        arg.each_pair do |k,v|
+          self.send "#{k}=".to_sym, v
+        end
+      end
+
+      def action group
+        configuration.steps_for(group) do |a_step|
+          RunLogger.title "[#{a_step.title}]"
+          begin_at = Time.new
+          a_step.run_on node
+          during = sprintf('%.2f',Time.new - begin_at)
+          RunLogger.level(1).detail "#{a_step.title} complete (#{during}s)"
+        end
+      end
+
+      def allocate
+        RunLogger.title 'allocate resource'
+        begin_at = Time.new
+        option = {image_id: configuration.image}
+        self.node = Node.new(*Cloud.create(option).first)
+        during = sprintf('%.2f', Time.new - begin_at)
+        RunLogger.level(1).detail "allocated(#{during}s)"
+      end
+
+      def build_image
+        action :image
+        node.stop
+        new_image = Cloud.create_image node.instance.id
+        RunLogger.level(1).detail "New Image Id: #{new_image.id}"
+        [
+          "To reduce the system setup, you might want to change larrow.yml.",
+          "  You can replace init step with the follow contents:",
+          "  image: #{new_image.id}"
+        ].each{|s| RunLogger.level(1).detail s}
+
+        new_image
+      end
+
+      def deploy
+        action :deploy
+        RunLogger.level(1).detail "application is deploy on: #{node.host}"
+        node
+      end
+
+      def dump
+        {nodes:[node.dump]}
+      end
+    end
+  end
+end

data/lib/larrow/runner/model/node.rb

@@ -0,0 +1,73 @@
+module Larrow::Runner
+  module Model
+  class Node
+    include Larrow::Runner::Service
+    include Larrow::Qingcloud
+    attr_accessor :instance, :eip
+    attr_accessor :user,:host
+    def initialize instance, eip, user='root'
+      self.instance = instance
+      self.eip = eip
+      self.host = eip.address
+      self.user = user
+      @executor = Executor.new host, user, nil, nil
+    end
+
+    def execute command, base_dir:nil
+      block = if block_given?
+                -> (data) { yield data }
+              else
+                -> (data) {
+                  data.split(/\r?\n/).each do |msg|
+                    RunLogger.level(1).info msg 
+                  end
+                }
+              end
+      @executor.execute command, base_dir: base_dir, &block
+    end
+
+    def stop
+      self.instance = instance.stop
+    end
+
+    def destroy
+      instance.destroy.force
+      eip.destroy.force
+      self
+    end
+
+    def dump
+      {
+        instance:{id: instance.id},
+        eip:{id:eip.id, address:eip.address}
+      }
+    end
+
+    def self.show resources, level=0
+      resources.map do |hash|
+        node = load_obj hash
+        RunLogger.level(level).info "instance: #{node.instance.id}"
+        RunLogger.level(level).info "eip:"
+        RunLogger.level(level+1).info "id: #{node.eip.id}"
+        RunLogger.level(level+1).info "address: #{node.eip.address}"
+      end
+    end
+
+    def self.cleanup resources
+      resources.map do |hash|
+        node = load_obj hash
+        future{node.destroy}
+      end.map do |instance|
+        RunLogger.detail "node cleaned: #{instance.address}"
+      end
+    end
+
+    def self.load_obj data
+        instance = Instance.new data[:instance][:id]
+        eip = Eip.new data[:eip][:id],address:data[:eip][:address]
+        new instance,eip
+    end
+
+  end
+  end
+end

data/lib/larrow/runner/service/cloud.rb

@@ -0,0 +1,54 @@
+require 'larrow/qingcloud'
+
+module Larrow
+  module Runner
+    module Service
+      class Cloud
+        include Qingcloud
+        def initialize args={}
+          Qingcloud.remove_connection
+          access_id   = args[:qy_access_key_id]
+          secret_key  = args[:qy_secret_access_key]
+          zone_id     = args[:zone_id]
+          @keypair_id = args[:keypair_id]
+          Qingcloud.establish_connection access_id,secret_key,zone_id
+        end
+
+        # return: Array< [ instance,eip ] >
+        # WARN: eips contains promise object, so it should be force
+        def create image_id:nil,count:1
+          RunLogger.level(1).detail "assign node"
+          instances = Instance.create(image_id: image_id||'trustysrvx64c',
+                                                 count:count,
+                                                 login_mode:'keypair',
+                                                 keypair_id: @keypair_id
+                                                )
+
+          eips = Eip.create(count:count)
+          
+          (0...count).map do |i|
+            RunLogger.level(1).detail "bind ip: #{eips[i].address}"
+            eips[i] = eips[i].associate instances[i].id
+            [ instances[i], eips[i] ]
+          end
+        end
+
+        # return image future
+        def create_image instance_id
+          Image.create instance_id
+        end
+
+        def image? image_id
+          Image.list(:self, ids: [image_id]).size == 1
+        end
+
+        def check_available
+          KeyPair.list
+        rescue
+          Qingcloud.remove_connection
+          raise $!
+        end
+      end
+    end
+  end
+end

data/lib/larrow/runner/service/executor.rb

@@ -0,0 +1,56 @@
+require 'net/ssh'
+require 'net/scp'
+
+module Larrow
+  module Runner
+    module Service
+      class Executor
+        attr_accessor :ip, :user, :port, :password
+        def initialize ip, user, port, password
+          self.ip       = ip
+          self.user     = user
+          self.port     = port
+          self.password = password
+          @canceling = nil
+          @dlogger = RunLogger #::Logger.new "#{ip}_cmd.log"
+        end
+
+        def execute cmd, base_dir:nil
+          connection.open_channel do |ch|
+            RunLogger.level(1).detail "# #{cmd}"
+            cmd = "cd #{base_dir}; #{cmd}" unless base_dir.nil?
+            errmsg = ''
+            ch.exec cmd do |ch,success|
+              if RunOption.key? :debug
+                ch.on_data{ |c, data| yield data }
+                ch.on_extended_data{ |c, type, data| yield data }
+              else
+                ch.on_extended_data{ |c, type, data| errmsg << data }
+              end
+              ch.on_request('exit-status') do |c,data|
+                status = data.read_long
+                if status != 0
+                  fail ExecutionError,{cmd:cmd,
+                                       errmsg: errmsg, 
+                                       status: status}
+                end
+              end
+            end
+          end
+          trap("INT") { @canceling = true }
+          connection.loop(0.1) do
+            not (@canceling || connection.channels.empty?)
+          end
+        end
+
+        def scp local_file_path, remote_file_path
+          raise 'not completed.'
+        end
+
+        def connection
+          @connection ||= Net::SSH.start(ip,user)
+        end
+      end
+    end
+  end
+end

data/lib/larrow/runner/service.rb

@@ -0,0 +1,8 @@
+module Larrow
+  module Runner
+    module Service
+      autoload :Cloud,    'larrow/runner/service/cloud'
+      autoload :Executor, 'larrow/runner/service/executor'
+    end
+  end
+end

data/lib/larrow/runner/session.rb

@@ -0,0 +1,64 @@
+require 'yaml'
+module Larrow
+  module Runner
+    module Session
+      extend self
+
+      FILE = "#{ENV['HOME']}/.larrow"
+      def login
+        return unless check_file
+        puts "The larrow config will be generated at #{FILE}."
+        data = nil
+        loop do
+          data = [:qy_access_key_id,
+                  :qy_secret_access_key,
+                  :zone_id,
+                  :keypair_id].
+                  reduce({}){|s,key| s.update key => value_for(key)}
+
+          cloud = Service::Cloud.new data
+          begin
+            cloud.check_available
+            RunLogger.info "login success! write to ~/.larrow"
+            break
+          rescue Exception => e
+            RunLogger.info "login fail: #{e.message}"
+            return unless ask "try again"
+          end
+        end
+        content={'qingcloud' => data}
+        File.write FILE, YAML.dump(content)
+      end
+
+      def load_cloud
+        args = begin
+                 YAML.
+                   load(File.read FILE).
+                   with_indifferent_access[:qingcloud]
+               rescue
+                 nil
+               end
+        Service::Cloud.new args if args
+      end
+
+      def value_for name
+        print sprintf("%25s: ", name)
+        v = $stdin.gets.strip
+        v.empty? ? nil : v
+      end
+
+      def check_file
+        return true unless File.exist?(FILE)
+        puts "#{FILE} does exist: "
+        puts File.read(FILE)
+        RunOption[:force] || ask("overwrite #{FILE}")
+      end
+
+      def ask title
+        print "#{title} ? (yes/[no]) "
+        v = $stdin.gets.strip
+        ['yes','y','Y','Yes','YES'].include? v
+      end
+    end
+  end
+end

data/lib/larrow/runner/vcs/base.rb

@@ -0,0 +1,17 @@
+module Larrow::Runner::Vcs
+  class Base
+    attr_accessor :larrow_file
+    include Larrow::Runner
+    def configuration merge=true
+      configuration = Manifest.configuration(self)
+      if merge
+        Manifest.add_base_scripts configuration,self
+      end
+      configuration
+    end
+
+    def formatted_url
+      raise 'not implement yet'
+    end
+  end
+end

data/lib/larrow/runner/vcs/file_system.rb

@@ -0,0 +1,58 @@
+module Larrow::Runner
+  module Vcs
+    class FileSystem < Base
+      # path: absulute path of LarrowFile
+      attr_accessor :project_folder
+      def initialize path
+        if File.file? path
+          path  = File.absolute_path path
+          self.larrow_file    = File.basename path
+          self.project_folder = File.dirname path 
+        else # directory
+          self.project_folder = File.absolute_path path
+        end
+      end
+
+      def formatted_url
+        self.project_folder
+      end
+
+      def get filename
+        file_path = "#{project_folder}/#{filename}"
+        return nil unless File.exist? file_path
+
+        File.read(file_path)
+      end
+
+      def update_source node, target_dir
+        command = rsync_command node.user, node.host,target_dir
+        invoke command
+        invoke "ssh-keygen -R #{node.host} 2>&1"
+      end
+
+      def rsync_command user, host, target_dir
+        ssh_path = '%s@%s:%s' % [user, host, target_dir]
+
+        excludes = (get('.gitignore')||'').  # rsync exclude according .gitignore
+          split(/[\r\n]/).                   # 
+          select{|s| s =~ /^[^#]/}.          # not commented
+          compact.                           # not blank
+          unshift('.git').                   # .git itself is ignored
+          map{|s| "--exclude '#{s}'" }       # build rsync exclude arguments
+
+        ssh_options = "-e 'ssh -o StrictHostKeyChecking=no'"
+
+        rsync_options = "-az #{ssh_options} #{excludes.join ' '}"
+        rsync_options += ' -v' if RunOption.key? :debug
+
+        "rsync #{rsync_options} #{project_folder}/ '#{ssh_path}' 2>&1"
+      end
+      def invoke command
+        `#{command}`.split(/\r?\n/).each do |msg|
+          RunLogger.level(1).info msg
+        end
+      end
+
+    end
+  end
+end

data/lib/larrow/runner/vcs/github.rb

@@ -0,0 +1,48 @@
+require 'faraday'
+
+module Larrow::Runner::Vcs
+  class Github < Base
+    URL_TEMPLATE='https://raw.githubusercontent.com/%s/%s/%s/%s'
+    attr_accessor :organize, :name, :branch
+    # url sample:
+    # git@github.com:fsword/larrow-qingcloud.git
+    # https://github.com/fsword/larrow-qingcloud.git
+    def initialize url
+      self.branch = 'master'
+      case url
+      when /git@github\.com:(.+)\/(.+)\.git/
+        self.organize = $1
+        self.name     = $2
+      when /http.:\/\/github.com\/(.+)\/(.+)\.git/
+        self.organize = $1
+        self.name     = $2
+      end
+    end
+
+    def formatted_url
+      'git@github.com:%s/%s.git' % [organize, name]
+    end
+
+    def get filename
+      url = URL_TEMPLATE % [organize, name, branch, filename]
+      resp = Faraday.get(url)
+      case resp.status
+      when 200
+        resp.body
+      when 404
+        nil
+      else
+        raise resp.body
+      end
+    end
+
+    def update_source node, target_dir
+      template = ["git clone ",
+                  "--depth 1",
+                  "http://github.com/%s/%s.git",
+                  "-b %s %s"].join(' ')
+      cmd = template % [organize, name, branch, target_dir]
+      node.execute cmd
+    end
+  end
+end

data/lib/larrow/runner/vcs.rb

@@ -0,0 +1,20 @@
+require 'larrow/runner/vcs/base'
+module Larrow
+  module Runner
+    # Access source code from Version Control System
+    # eg: Subversion, Github, LocalStore
+    module Vcs
+      autoload :Github,    'larrow/runner/vcs/github'
+      autoload :FileSystem,'larrow/runner/vcs/file_system'
+      def self.detect url
+        case url
+        when /github\.com.+\.git$/
+          Github.new(url)
+        else # local file/folder
+          fail "cannot recognized: #{url}" unless File.exist? url
+          FileSystem.new url
+        end
+      end
+    end
+  end
+end

data/lib/larrow/runner/version.rb

@@ -0,0 +1,5 @@
+module Larrow
+  module Runner
+    VERSION = "0.0.1"
+  end
+end

data/lib/larrow/runner.rb

@@ -0,0 +1,34 @@
+require 'active_support/deprecation'
+require 'active_support/core_ext/hash'
+
+require "larrow/runner/version"
+require 'larrow/runner/logger'
+require 'larrow/runner/service'
+require 'larrow/runner/session'
+
+require 'larrow/runner/errors'
+
+module Larrow
+  module Runner
+    # default runtime logger
+    RunLogger = if ENV['RUN_AS']
+                  Logger.new "#{ENV['RUN_AS']}.log"
+                else
+                  Logger.new $stdout
+                end
+    # global options
+    RunOption = {}.with_indifferent_access
+    # cloud wrapper
+    Cloud = Session.load_cloud
+  end
+end
+
+require 'larrow/runner/vcs'
+require 'larrow/runner/manifest'
+require 'larrow/runner/helper'
+
+require 'larrow/runner/manager'
+require 'larrow/runner/cli'
+require 'larrow/runner/model/app'
+require 'larrow/runner/model/node'
+

data/spec/fixtures/travis_erlang.yml

@@ -0,0 +1,8 @@
+language: erlang
+before_script:
+  - erl -verion
+script:
+  - ./rebar compile ct
+otp_release:
+  - R16B
+