panoramix 0.5.0

data/lib/panoramix/aws.rb

@@ -0,0 +1,13 @@
+require "panoramix/instances/aws"
+
+module Panoramix
+
+  module AWS
+    module_function
+
+    def config(config)
+      AwsInstance.new(config.getHash)
+    end
+  end
+
+end

data/lib/panoramix/connections/base_connection.rb

@@ -0,0 +1,49 @@
+require 'open3'
+
+module Panoramix
+  class BaseConnection
+
+    def shell(cmd, silent, env)
+
+      env.select{|k,v| v.nil?}.each{ |k, v| env[k]=Panoramix.ip_host(Panoramix::Link[k]) }
+      puts "Running #{cmd}".magenta if ENV["VERBOSE"]
+
+      @err = ""
+      @out = ""
+      Open3.popen3(env, cmd) do |stdin, stdout, stderr, wait_thr|
+        stdin.close
+        stdout.each_line do |line|
+          puts line unless silent
+          @out << "#{line}"
+        end
+        stderr.each_line do |line|
+          puts line.red unless silent
+          @err << "#{line}"
+        end
+        @exit_status = wait_thr.value
+      end
+
+      if ENV["VERBOSE"] && !@out.empty?
+        puts "\tout => \n\t\t#{@out.gsub("\n","\n\t\t")}".green
+      end
+      if ENV["VERBOSE"] && !@err.empty?
+        if @exit_status.success?
+          puts "\terr => \n\t\t#{@err.gsub("\n","\n\t\t")}".brown
+        else
+          puts "\terr => \n\t\t#{@err.gsub("\n","\n\t\t")}".red
+        end
+      end
+      puts "\texit_status => #{@exit_status}\n".blue if ENV["VERBOSE"]
+
+      return {
+        :out => @out,
+        :err => @err,
+        :exit_status => @exit_status
+      }
+    end
+
+    def close_connection
+    end
+
+  end
+end

data/lib/panoramix/connections/docker_connection.rb

@@ -0,0 +1,85 @@
+require 'time'
+require 'net/ssh/gateway'
+require 'panoramix/connections/base_connection'
+
+
+module Panoramix
+  class DockerConnection < BaseConnection
+
+    def initialize(host, user)
+      @host = host
+      @user = user
+      @connected = false
+    end
+
+    def connected?
+      @connected
+    end
+
+    def connect
+      return if @connected
+      return unless @host && @user
+      ready?
+    end
+
+    # Returns true if connection is enabled
+    def ready?
+      return false unless @host && @user
+      ! open_tunnel.nil?
+    end
+
+    def shell(cmd, silent=false, env=Hash.new)
+      raise "No host unless" unless @host && @user
+      #open_tunnel unless @gateway
+      # Run passing tunneled port
+      env['DOCKER_HOST'] = "tcp://127.0.0.1:#{@port}"
+      super(cmd, silent, env)
+    end
+
+    def close_connection
+      if @connected
+        @gateway.close(@port)
+        @connected = false
+        @gateway = nil
+        @port = nil
+      end
+    end
+
+    private
+
+    def open_tunnel
+      unless @gateway
+        # Open a new ssh tunnel
+        print "Connecting remote host..."
+        # Wait until docker starts
+        Timeout::timeout(600) do
+          begin
+            @gateway = Net::SSH::Gateway.new(@host, @user)
+            3.times do
+              begin
+                @port = @gateway.open('127.0.0.1', 666, rand(63000)+2000)
+                break
+              rescue
+                puts "Port already in use"
+              end
+            end
+            #Check this connection
+            Timeout::timeout(10) do
+              res = shell("docker ps", true)
+              raise "Servide docker not available" unless res[:exit_status].success?
+            end
+          rescue Exception => err
+            sleep(10)
+            print "."
+            @gateway.close(@port) if @gateway
+            retry
+          end
+        end
+        @connected = true
+        puts " done."
+      end
+      @gateway
+    end
+
+  end
+end

data/lib/panoramix/connections/local_connection.rb

@@ -0,0 +1,15 @@
+require 'panoramix/connections/base_connection'
+
+module Panoramix
+
+  class LocalConnection < BaseConnection
+    def shell(cmd, silent, env )
+      res = super(cmd, silent, env)
+    end
+
+    def ready?
+      return true
+    end
+  end
+
+end

data/lib/panoramix/connections.rb

@@ -0,0 +1,28 @@
+require 'panoramix/connections/local_connection'
+require 'panoramix/connections/docker_connection'
+
+module Panoramix
+
+  class Connections
+    class << self
+        attr_reader :connections
+
+      	def initialize
+      		@connections=Hash.new
+      		@connections["local"] = LocalConnection.new
+      	end
+
+      	def register(host, config)
+          @connections[host] = DockerConnection.new(config[:host], config[:user])
+      	end
+
+        def connect key
+          @connections[key].connect unless @connections[key].connected?
+        end
+
+      	def [](key)
+          	@connections[key]
+        end
+      end
+  end
+end

data/lib/panoramix/dsl.rb

@@ -0,0 +1,11 @@
+module Rake
+	module DSL
+
+		def load file
+			task=Rake::Task.tasks.any? { |task| task.name == file }
+			Rake::Task[file].invoke if task
+			super file
+		end
+		
+	end
+end

data/lib/panoramix/external.rb

@@ -0,0 +1,39 @@
+require "panoramix/plugin/wget"
+require "panoramix/plugin/s3"
+require "panoramix/plugin/git"
+require "panoramix/plugin/docker_image"
+require "panoramix/mpi"
+
+def external(dep, &block)
+  # Provided value can be either array or hash, so we need to make sure the final value is an array
+  value = [dep.values[0]].flatten
+
+  # Select the hash with params
+  params = value.select { |d| d.is_a? Hash }.first
+
+  # Handle with MPI
+  handled_dep = Panoramix::MPI.handle_external(dep.keys[0], params)
+
+  raise "Only a single external depdency supported (received: #{dep.keys})" if handled_dep.length != 1
+
+  # Select provided dependencies
+  prerequisites = value.select { |d| ! d.is_a? Hash }
+
+  case handled_dep.values[0].keys[0]
+  when :s3
+    s3(handled_dep, prerequisites, block)
+  when :wget
+    wget(handled_dep, prerequisites, block)
+  when :git
+    git(handled_dep, prerequisites, block)
+  when :dockerimage
+    external_docker_image(handled_dep, block)
+  when :docker
+    puts "Warning using obsolete external dependency key
+      \t => external <image_name> {#{"docker".bold}: <docker_image_name> }.\n\tchange to:
+      \t => external <image_name> {#{"dockerimage".bold}: <docker_image_name> }.".red
+    external_docker_image(handled_dep, block)
+  else
+    raise "Unknown external protocol '#{dep.values[0].keys[0]}'"
+  end
+end

data/lib/panoramix/hosts.rb

@@ -0,0 +1,58 @@
+require "panoramix/instances/local"
+
+module Panoramix
+  class Hosts
+    class << self
+
+      attr_reader :hosts
+
+    	def initialize
+    		@hosts=Hash.new
+    		register("local",  LocalInstance.new)
+    	end
+
+    	def register(name, instance)
+        if instance.class.ancestors.include? BaseInstance
+          instance.host(name)
+          @hosts[name] = {instance: instance, tasks: {}}
+          define_tasks(name)
+        else
+          puts "Warning using obsolete instance on host key
+            \t => host <host_name>, #{"<environment_class>".bold} \n\tchange to:
+            \t => host <host_name>, #{"Panoramix::<Cloud>.config(<environment_class>)".bold}
+            \t where Cloud must be AWS or Metal".red
+          exit(1)
+        end
+    	end
+
+      def define_tasks(name)
+        Panoramix::Tasks::Actions.each do |act|
+          block_act = custom_host_action(act.action_name, act.order_class, name)
+          descriptions =(name != 'local') ? I18n.t('host') : Hash.new
+          Panoramix.define_task("#{name}:#{act.action_name}", [], descriptions[act.action_name.to_sym], block_act)
+        end
+      end
+
+      def custom_host_action(action, class_order, host_name)
+        Proc.new do |t|
+          tasks = Panoramix::Hosts[host_name][:tasks]
+          class_order.each do |p|
+            tasks[p].each do |task_name|
+              host_name2 =(host_name != 'local') ? "#{host_name}:" : ""
+              Rake::Task["#{host_name2}#{task_name}:#{action}"].invoke
+            end  if tasks[p]
+          end
+        end
+      end
+
+    	def [](key)
+        	@hosts[key]
+      end
+
+      def to_s
+        @hosts.pretty_inspect
+      end
+
+    end
+  end
+end

data/lib/panoramix/instances/aws.rb

@@ -0,0 +1,74 @@
+require "raws"
+require "panoramix/instances/base"
+
+module Panoramix
+
+  class AwsInstance < BaseInstance
+
+    attr_reader :machine
+
+    def initialize(config)
+      @tunnel=nil
+      @machine = Raws::EC2::Machine.new(config[:name], config[:region])
+      @machine.config = config
+      @record = Raws::Route53::Record.new(config[:subdomain_id], config[:region])
+    end
+
+    def host(host)
+      @id=host
+      @machine.config[:name] = host
+      @record.name = host
+    end
+
+    def status
+      puts @machine.status
+    end
+
+    def deployed?
+      @machine.deployed?
+    end
+
+    def timestamp
+      @machine.timestamp
+    end
+
+    def run_instance args = {}
+      @machine.config.merge!(args)
+      if (@machine.config[:create_dns_record])
+        @record.ips.push(@machine.public_ip) #TODO: Wrong if private dns in aws => change for ip
+        @record.save
+      end
+      @machine.save
+      clear_known_hosts_by_hostid
+    end
+
+    def terminate_instance args=nil
+      @machine.terminate_instance
+      @record.delete if (@machine.config[:create_dns_record])
+    end
+
+    def self.deployable?
+      true
+    end
+
+    def connection_params
+      {:user => "ubuntu", :host => ip}
+    end
+
+    def ip
+      if @machine.config[:use_public_ip]
+        @ip = @machine.public_ip unless @ip
+      else
+        @ip = @machine.private_ip unless @ip
+      end
+      @ip
+    end
+    private
+
+    def clear_known_hosts_by_hostid
+      result = `ssh-keygen -R #{ip} 2>&1`
+      raie ("Failed to update known_host file. \n#{result}") if !result.include? "updated"
+    end
+
+  end
+end

data/lib/panoramix/instances/base.rb

@@ -0,0 +1,13 @@
+module Panoramix
+  class BaseInstance
+
+    def host(host)
+      @host = host
+    end
+
+    def ip
+      @host
+    end
+
+  end
+end

data/lib/panoramix/instances/local.rb

@@ -0,0 +1,24 @@
+require "panoramix/instances/base"
+
+module Panoramix
+
+  class LocalInstance < BaseInstance
+    def deployed?
+      true
+    end
+
+    def status
+      ""
+    end
+
+    def ip
+      "127.0.0.1"
+    end
+
+    def self.deployable?
+      false
+    end
+
+  end
+
+end

data/lib/panoramix/instances/metal.rb

@@ -0,0 +1,26 @@
+require "panoramix/instances/base"
+
+module Panoramix
+
+  class MetalInstance < BaseInstance
+    def initialize(user)
+      @tunnel = nil
+      @user = user
+    end
+
+    def connection_params
+        {:user => @user, :host => @host}
+    end
+
+    def deployed?
+      `ping -c 1 #{@host}`
+       $?.exitstatus == 0
+    end
+
+    def self.deployable?
+      false
+    end
+
+  end
+
+end

data/lib/panoramix/locale.rb

@@ -0,0 +1,4 @@
+require "i18n"
+
+I18n.load_path << File.expand_path(File.join(File.dirname(__FILE__),"../../locales/en.yml"))
+I18n.enforce_available_locales = true

data/lib/panoramix/metal.rb

@@ -0,0 +1,13 @@
+require "panoramix/instances/metal"
+
+module Panoramix
+
+  module Metal
+    module_function
+
+    def config(user)
+      MetalInstance.new(user)
+    end
+  end
+
+end

data/lib/panoramix/mpi.rb

@@ -0,0 +1,97 @@
+require "json"
+require 'pry'
+
+module Panoramix
+
+  class MPI
+
+    class << self
+
+    	def initialize
+        @mpi_path = "MPI.rb"
+      	@mpi_file = {}
+        @project_mpi = {}
+        @final_mpi = {}
+
+        # Read MPI.rb if exists
+        if File.exists?(@mpi_path)
+          load @mpi_path
+          @mpi_file = Panoramix::MPI::Deps
+        end
+    	end
+
+      def handle_external(key, value)
+        # Get current scope
+        scope = Panoramix::current_host_name
+
+        # Save dependency into @project_mpi
+        @project_mpi[scope] = {} if @project_mpi[scope].nil?
+        @project_mpi[scope].merge!({key => value})
+
+        @final_mpi[scope] = {} if @final_mpi[scope].nil?
+        @final_mpi[scope].merge!({key => value})
+
+        # Check whether dependencies can be overriden by an MPI file
+        if (@mpi_file && @mpi_file[scope] && @mpi_file[scope][key])
+
+          # Generate the new value from MPI
+          mpi_value =  @mpi_file[scope][key]
+
+          # Save overriden value to @final_mpi
+          @final_mpi[scope] = {} if @final_mpi[scope].nil?
+          @final_mpi[scope].merge!({key => mpi_value})
+
+          return {key => mpi_value}
+        end
+
+        {key => value}
+      end
+
+      def print_final_mpi
+        puts @final_mpi.pretty_inspect
+      end
+
+      def print_mpi
+         puts @project_mpi.pretty_inspect
+      end
+
+      def diff_sh
+         system("bash -c \"diff <(echo '#{@final_mpi.pretty_inspect}') <(echo '#{@project_mpi.pretty_inspect}')| colordiff\"")
+      end
+
+      def generate
+        # Check whether the file already exists
+        if File.exists? @mpi_path
+          puts "Error: mpi.rb already exists in this project. Please, remove it before in order to create a newest version.".red
+          exit 0
+        end
+
+        # Get directory containing MPI.rb
+        dir = Rake.application.original_dir
+        
+        # Replace each key path with the variable current_dir
+        new_file = {} 
+        @project_mpi.each do |host ,deps|
+          new_hash = {} 
+          deps.each do |k, v| 
+            key = k.gsub(dir, '#{current_dir}')
+            new_hash[key] = v
+          end
+          new_file[host] = new_hash
+        end
+
+        header = 
+%&current_dir = File.expand_path(File.dirname(__FILE__))
+
+Panoramix::MPI::Deps =& 
+
+        File.open(@mpi_path,"w") do |f|
+          f.write(header)
+          f.write(new_file.pretty_inspect.gsub('\#','#'))
+        end
+      end
+
+    end
+
+  end
+end

data/lib/panoramix/panoramix_core.rb

@@ -0,0 +1,109 @@
+require "time"
+
+module Panoramix
+	module_function
+
+	def global_timestamps(prerequisites, scope)
+		timestamps = prerequisites.map do |p|
+			any_task = Rake::Task.tasks.any? { |t| t.name == p }
+			if( any_task && (Rake::Task[p].class == Rake::FileTask))
+				Rake::Task[p].timestamp
+			else
+				Panoramix::TIMESTAMP["#{scope}:#{p}"]
+			end
+		end
+
+		# Remove nil values (fe: Directory tasks)
+		timestamps.compact
+	end
+
+	def run_task(task, instance, block=nil)
+		if instance.needed? global_timestamps(task.prerequisites, task.scope.first)
+			begin
+				block.call(instance) if block
+				instance.run_default unless block
+			rescue Panoramix::Plugin::DockerUpExceptionError => e
+				puts "Error: #{e.message}".red
+				puts "Exiting program...".red
+				exit(0)
+			end
+		else
+			puts "task #{task} not needed"
+		end
+	end
+
+	def run_custom_task(instance, name)
+		method = instance.method(name.to_sym)
+		begin
+			method.call
+		rescue Panoramix::Plugin::ValidationError => e
+			puts "Validation error: #{e.message}".red
+		end
+	end
+
+	def define_action_task(name, desctiption, instance, operation)
+		if instance.respond_to? operation
+			Rake.application.last_description = desctiption
+			Rake::Task.define_task "#{name}:#{operation.to_s}" do |t|
+				run_custom_task(instance, operation)
+			end
+		end
+	end
+
+	def define_task(name, prerequisites, description, block=nil)
+		Rake.application.last_description = description
+		Rake::Task.define_task({name => prerequisites}, &block)
+	end
+
+	def define_tasks(name, descriptions, instance, prerequisites, block=nil)
+		fill_current_host_tasks(name, instance.class)
+		action = Proc.new do |t|
+			run_task(t, instance, block) if block
+			run_task(t, instance) unless block
+			Panoramix::TIMESTAMP["#{t.scope.first}:#{name}"] = instance.timestamp
+		end
+
+		define_task(name, prerequisites, descriptions[:main], action)
+		Panoramix::Tasks::Actions.each do |act|
+			define_action_task(name, descriptions[act.action_name.to_sym], instance, act.action_name)
+		end
+	end
+
+	def resolve_host_dependencies
+		current_host.class.deployable? ? :host : nil
+	end
+
+	def connection host_name
+		unless Panoramix::Connections[host_name]
+			host = Panoramix::Hosts[host_name][:instance]
+			Panoramix::Connections.register(host_name, host.connection_params)
+			Panoramix::Connections.connect host_name
+		end
+		Panoramix::Connections[host_name]
+	end
+
+	def current_host
+		Panoramix::Hosts[current_host_name][:instance]
+	end
+
+	def fill_current_host_tasks(name, type)
+		unless Panoramix::Hosts[current_host_name][:tasks][type]
+			 Panoramix::Hosts[current_host_name][:tasks][type] = []
+		end
+		Panoramix::Hosts[current_host_name][:tasks][type].push(name)
+	end
+
+	def current_host_name
+		Rake.application.current_scope.first || "local"
+	end
+
+	def validation_error(instance, error)
+		puts "#{instance.class.name.split("::").last} validation error \n\t#{error.gsub("\n", "\n\t").red}"
+	end
+
+	def ip_host host_name
+		host = Panoramix::Hosts[host_name]
+		return !host.nil? ? host[:instance].ip : nil
+	end
+
+end