claudius 0.0.4 → 0.0.5

data/README.md

@@ -1,6 +1,6 @@
 # Claudius
 
-Claudius is a easy-to-use domain specific language for clouds experiments. Language is build on [fog.io](http://fog.io), which enables flexible and powerfull way to manage virtual machine instances on various cloude poviders. Connections with virtual machines is based on ssh. To provide information experiment flow, DSL generates readable graph of execution.
+Claudius is an easy-to-use domain specific language for cloud experiments. It has been designed to speed up process of building distributed experiments and highly reduce time of remote machines configuration. To avoid vendor lock-in, Claudius was build on top of fog.io library, which enables flexible and powerful way to manage machine instances at various cloud providers. Remote commands execution is based on  SSH protocol (SSH-2). DLS allow users to generate readable execution graph, which is extremely useful for experiment flow verification and help avoid wasting money.
 
 ## Installation
 
@@ -13,14 +13,14 @@ If you want export execution tree to image you need [Graphivz](http://www.graphv
 
 ## Keywords
 
-* **experiment** - central part of claudius. Method defines a new experiment.  Parameters: 
-    * **experiment_name**
-    * **body** - block of code describing performed experiment
+* **experiment** -- main part of Claudius DLS, which defines a new experiment. After command, you are obligated to provide 2 parameters:
+    * **experiment_name** - string, which is the name of the experiment
+    * **body** - block of code describing designed experiment
     
-    You may call the following methods on returned experiment object:
+    You are allowed to call following methods on returned experiment object:
 
-    * **run** - method starts previously defined experiment, 
-    * **export_tree(path = 'execution_tree_path')** - method creates an execution graph of experiment and save it as an image
+    * **run** - method starts previously designed experiment.
+    * **export_tree(path = 'execution_tree_path')** - method generates readable execution graph. Graph is saved as an image and in .dot file format (graph description language) - for further user processing.
 
 * **define_providers** - method takes as a parameter description of machines used in experiment. In experiments You are allowed to use 2 different types of machines
 
@@ -54,8 +54,8 @@ In order to authenticate in AWS services (at other cloud providers also) you are
    "aws_access_key_id" : "XXXXXXXXXXXXXXXXXXXX",
    "aws_secret_access_key" : "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
    "key_name" : "My-irleand-key",
-   "groups" : "My-irleand-group"
-   "path_to_pem_file" : "./pems/my-irleand-key.pem"
+   "groups" : ["My-irleand-group"]
+   "private_key_path" : "./pems/my-irleand-key.pem"
 }
 ```
 

data/claudius.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "claudius"
-  spec.version       = "0.0.4"
+  spec.version       = "0.0.5"
   spec.authors       = ["radk0s", "blost"]
   spec.email         = ["rachamot@gmail.com", "piotr.skibiak@gmail.com"]
   spec.description   = 'Claudius is an easy-to-use domain specific language for cloud experiments. ' +

data/examples/ping.rb

@@ -1,11 +1,12 @@
-
 # Ping 
 # ---------
 # This example shows how to conduct simple experiment with ping command.
 #
 
 require 'claudius'
+
 config = load_config('./user_config.json')
+
 execution_tree = experiment 'Hello' do
   define_providers do
     cloud('aws',  :provider => config['provider'],
@@ -13,29 +14,27 @@ execution_tree = experiment 'Hello' do
           :endpoint => 'https://ec2.eu-west-1.amazonaws.com/',
           :aws_access_key_id => config['aws_access_key_id'],
           :aws_secret_access_key => config['aws_secret_access_key'])
-    .create_instances(['t2.micro=>in1'],
+    .create_instances(['t1.micro=>instance1', 't1.micro=>instance2', 't1.micro=>instance3'],
                       :username => 'ubuntu',
-                      :private_key_path =>config['path_to_pem_file'],
+                      :private_key_path =>config['private_key_path'],
                       :key_name => config['key_name'],
-                      :image_id => config['image_id'],
                       :groups => config['groups'])
   end
-  foreach ['in1'] do |instance_name|
-    concurrent do
-      on 'localhost' do
-        execute do
-          ssh "ping #{getInstance(instance_name).host} -c 3"
-        end
+  foreach ['instance1', 'instance2', 'instance3'], concurrently do |instance_name|
+    on 'localhost' do
+      execute do
+        ssh "ping #{getInstance(instance_name).host} -c 3"
       end
-      on instance_name do
-        execute do
-          ssh "ping #{getInstance(instance_name).host} -c 3"
-          ssh "echo Hello World! > TestFile"
-        end
+    end
+    on instance_name do
+      execute do
+        ssh "ping #{getInstance(instance_name).host} -c 3"
+        ssh "echo Hello World! > TestFile"
       end
     end
   end
 end
 
 ap execution_tree.run
-execution_tree.export_tree('tree')
+execution_tree.export_tree('tree')
+execution_tree.destroy_machines

data/examples/simple_program.rb

@@ -22,4 +22,4 @@ end
 # execute tree and print result time using *awesome_print*
 ap execution_tree.run
 # export tree to tree.png and tree.dot
-execution_tree.export_tree('tree')
+execution_tree.export_tree('tree')

data/examples/simple_program_2.rb

@@ -0,0 +1,41 @@
+require 'claudius'
+config = load_config('./user_config.json')
+
+execution_tree = experiment 'Hello' do
+  	define_providers do
+    	cloud('aws',  :provider => config['provider'],
+	          :region =>config['region'],
+	          :aws_access_key_id => config['aws_access_key_id'],
+	          :aws_secret_access_key => config['aws_secret_access_key'])
+      	.create_instances(['t1.micro=>instance1','t1.micro=>instance2','t1.micro=>instance3'],
+              :username => 'ubuntu',
+              :private_key_path =>config['private_key_path'],
+              :key_name => config['key_name'],
+              :groups => config['groups'])
+	end
+	foreach ['instance1','instance2','instance3'], concurrently do |instance|
+	    on instance do
+	        before do
+	          ssh "mkdir test"
+	        end
+	        concurrent do
+	            foreach 1..3 do |m|
+	            	execute do
+	                	ssh "cd test"
+	               		ssh "touch testFile#{m}"
+	               	end
+	            end
+	            execute do
+	                ssh "ls"
+	            end
+	        end
+			after do
+	          ssh "rm -rf test"
+	        end
+	    end
+	end
+end
+
+ap execution_tree.run
+execution_tree.export_tree('tree')
+execution_tree.destroy_machines

data/lib/remote_execution/cloud_provider.rb

@@ -11,17 +11,26 @@ class CloudProvider
     @instances = Array.new
   end
 
+  def create_instance(instance_type, parameters)
+    puts 'creating machine: ' + instance_type
+    parameters.store(:flavor_id, instance_type.split('=>').first)
+    name = ("#{@provider_name}:" + instance_type).split('=>')
+    machine = @provider.servers.bootstrap(parameters)
+    @instances.push(machine)
+    machine_id = @provider.servers.get(machine.id)
+    vm = VirtualMachine.new(machine_id.public_ip_address, parameters[:username], :keys => parameters[:private_key_path])
+    @virtual_machines.store(name.last, vm)
+  end
+
   def create_instances(instances_types, *parameters)
-    parameters = parameters[0]
+    threads = []
     instances_types.each do |instance_type|
-      puts 'creating machine: ' + instance_type
-      parameters.store(:flavor_id, instance_type.split('=>').first)
-      name = ("#{@provider_name}:" + instance_type).split('=>')
-      machine = @provider.servers.bootstrap(parameters)
-      @instances.push(machine)
-      machine_id = @provider.servers.get(machine.id)
-      vm = VirtualMachine.new(machine_id.public_ip_address, parameters[:username], :keys => parameters[:private_key_path])
-      @virtual_machines.store(name.last, vm)
+      threads.push(Thread.fork do
+        create_instance(instance_type, parameters[0])
+      end)
+    end
+    threads.each do |thread|
+      thread.join
     end
   end
 

data/lib/remote_execution/provider.rb

@@ -22,8 +22,8 @@ class MachineManager
     end
   end
 
-  def manual(name, host, username, *args)
-    vm = VirtualMachine.new(host, username, *args)
+  def manual(name, ip, username, *args)
+    vm = VirtualMachine.new(ip, username, *args)
     @virtual_machines.store(name, vm)
   end
 

data/lib/remote_execution/virtual_machine.rb

@@ -8,10 +8,10 @@ def s_ssh(session, commands)
 end
 
 class VirtualMachine
-  attr_accessor :host, :username, :args, :log_file, :active
+  attr_accessor :ip_address, :username, :args, :log_file, :active
 
-  def initialize(host, username, *args)
-    @host = host
+  def initialize(ip_address, username, *args)
+    @ip_address = ip_address
     @username = username
     args[0].store(:timeout, 5)
     @args = args
@@ -19,19 +19,19 @@ class VirtualMachine
   end
 
   def to_s
-    "[host: #{host}, user: #{username}]"
+    "[IP: #{ip_address}, user: #{username}]"
   end
 
   def ssh_test(session,command)
     res = session.exec!(command)
     puts res
-    if (res!='' and res!="attributes.json\n")
+    if res!='' and res!="attributes.json\n"
       @active = true
     end
   end
 
   def invoke(commands)
-    Net::SSH.start(host, username, *args) do |ssh|
+    Net::SSH.start(ip_address, username, *args) do |ssh|
       commands.each do |command|
         print command , "\n"
         s_ssh ssh, command

data/lib/tree_building/claudius.rb

@@ -1,105 +1,124 @@
 require 'concurrent_node.rb'
 require 'on_node.rb'
+require 'foreach_node.rb'
 require 'execution_node.rb'
 
 def experiment(name, &block)
-  experiment = Experiment.new(name, &block)
+    Experiment.new(name, &block)
 end
 
 class Experiment
-  def initialize(name, &block)
-    @child_node
-    @current_node = nil
-    @tmp_asynchronously = false
-    @tmp_safely = false
-    @tmp_execution_place = 'localhost'
-    @in_after_scope = false
-    @in_before_scope = false
-    @root = Node.new(nil,block)
-    @current_node = @root
-    @root.name = name
-    instance_eval(&block) if block
-  end
-
-  def asynchronously
-     @child_node = ConcurrentNode.new(@current_node, nil)
-     @current_node.node_list.push(@child_node)
-     @current_node = @child_node
-  end
-
-  def safely
-       puts 'should be safely'
-  end
-
-  def before(*args, &block)
-    @in_before_scope = true
-    yield
-    @in_before_scope = false
-  end
-
-  def after(*args, &block)
-      @in_after_scope = true
-      yield
-      @in_after_scope = false
-  end
-
-  def foreach(parameters, *args, &block)
-    puts @current_node.name
-    parameters.each do |parameter|
-      @child_node = Node.new(@current_node, block)
-      @child_node.is_safely = @tmp_safely
-      @child_node.is_asynchronously = @tmp_asynchronously
-      @current_node.node_list.push(@child_node)
-      @current_node = @child_node
-      block.call parameter
-      @current_node = @current_node.parent
+    def initialize(name, &block)
+        @child_node
+        @current_node = nil
+        @tmp_concurrently = false
+        @tmp_safely = false
+        @tmp_execution_place = 'localhost'
+        @in_after_scope = false
+        @in_before_scope = false
+        @root = Node.new(nil, block)
+        @current_node = @root
+        @root.name = name
+        instance_eval(&block) if block
     end
-  end
-
-  def on(instance, *args, &block)
-    @child_node = OnNode.new(@current_node, instance, block)
-
-    @current_node.node_list.push(@child_node)
-    @current_node = @child_node
-    block.call
-    @current_node = @current_node.parent
-  end
-
-  def concurrent(*args, &block)
-    @child_node = ConcurrentNode.new(@current_node, block)
-    @current_node.node_list.push(@child_node)
-    @current_node = @child_node
-    block.call
-    @current_node = @current_node.parent
-  end
-
-  def execute(*args, &block)
-    @child_node = ExecutionNode.new(@current_node, block)
-    @child_node.is_safely = @tmp_safely
-    @child_node.is_asynchronously = @tmp_asynchronously
-    @current_node.node_list.push(@child_node)
-    @current_node = @child_node
-    block.call
-    @current_node = @current_node.parent
-  end
-
-  def ssh(command)
-    if @in_before_scope
-      @current_node.before_list.push(command)
-    elsif $in_after_scope
-      @current_node.after_list.push(command)
-    else
-      @current_node.commands.push(command)
+
+    def concurrently
+        @tmp_concurrently = true
+    end
+
+    def safely
+        @tmp_safely = true
+    end
+
+    def concurrent(*args, &block)
+        @child_node = ConcurrentNode.new(@current_node, block)
+        @current_node.node_list.push(@child_node)
+        @current_node = @child_node
+        block.call
+        @current_node = @current_node.parent
+    end
+
+    def before(*args, &block)
+        @in_before_scope = true
+        yield
+        @in_before_scope = false
+    end
+
+    def after(*args, &block)
+        @in_after_scope = true
+        yield
+        @in_after_scope = false
+    end
+
+    def foreach(parameters, *args, &block)
+
+        @current_node = ForeachNode.new(@current_node, block)
+
+        @current_node.is_safely = @tmp_safely
+        @tmp_safely = false
+
+        @current_node.is_concurrently = @tmp_concurrently
+        @tmp_concurrently = false
+
+        @current_node.parent.node_list.push(@current_node)
+
+        parameters.each do |parameter|
+            @child_node = Node.new(@current_node, block)
+            @child_node.name = "Parameter: #{parameter}"
+            @current_node.node_list.push(@child_node)
+            @current_node = @child_node
+            block.call parameter
+            @current_node = @current_node.parent
+        end
+
+        @current_node = @current_node.parent
+    end
+
+    def on(instance, *args, &block)
+        @child_node = OnNode.new(@current_node, instance, block)
+
+        @current_node.node_list.push(@child_node)
+        @current_node = @child_node
+        block.call
+        @current_node = @current_node.parent
+    end
+
+    def execute(*args, &block)
+        @child_node = ExecutionNode.new(@current_node, block)
+        @child_node.is_safely = @tmp_safely
+        @child_node.is_concurrently = @tmp_concurrently
+        @current_node.node_list.push(@child_node)
+        @current_node = @child_node
+        block.call
+        @current_node = @current_node.parent
+    end
+
+    def ssh(command)
+        if @in_before_scope
+            @current_node.before_list.push(command)
+        elsif @in_after_scope
+            @current_node.after_list.push(command)
+        else
+            @current_node.commands.push(command)
+        end
     end
-  end
 
-  def run
-    @root.run(nil)
-    # $vms_manager.destroy_machines
-  end
+    def run
+        begin
+            @root.run(nil)
+        rescue
+            '{error : true}'
+        end
+    end
 
-  def export_tree(path = 'execution_tree')
-    @root.print_tree(path)
-  end
+    def destroy_machines
+        if $vms_manager
+            $vms_manager.destroy_machines
+        end
+    end
+
+    def export_tree(path = 'execution_tree')
+        @root.print_tree(path)
+    end
 
 end

data/lib/tree_building/concurrent_node.rb

@@ -23,7 +23,7 @@ class ConcurrentNode < Node
         pids.each{|pid| Process.waitpid(pid)}
 
         after_exec(instance, @total_time)
-        return @total_time
+        @total_time
     end
 
     def draw_block(graph)

data/lib/tree_building/execution_node.rb

@@ -42,7 +42,7 @@ class ExecutionNode < Node
     finish = Time.now
 
     @total_time[:total] = finish - start
-    return @total_time
+    @total_time
   end
 
 end

data/lib/tree_building/foreach_node.rb

@@ -0,0 +1,40 @@
+require 'node.rb'
+
+class ForeachNode < Node
+    def initialize (parent, block)
+        super(parent, block)
+        self.name = "[Foreach]\n"
+    end
+
+    def run(instance)
+        init_time
+        before_exec(instance, @total_time)
+        begin
+            if is_concurrently
+                pids = []
+                node_list.each do |node|
+                    pids << fork do
+                        @total_time[:exec].push(node.run(instance))
+                    end
+                end
+                pids.each{|pid| Process.waitpid(pid)}
+            else
+                node_list.each do |node|
+                    @total_time[:exec].push(node.run(@instance))
+                end
+            end
+        rescue
+            if is_safely
+                raise "An error has occured"
+            end
+        end
+        after_exec(@instance, @total_time)
+        @total_time
+    end
+
+    def draw_block(graph)
+        graph.hexagon << graph.node(id)
+        graph.node(id).label(name + 'Concurrently: ' + is_concurrently.to_s + '\nSafely: ' + is_safely.to_s)
+    end
+
+end

data/lib/tree_building/node.rb

@@ -3,7 +3,7 @@ require 'awesome_print'
 
 class Node
   attr_accessor :parent, :code_block, :is_safely, :node_list,
-                :before_list, :after_list, :is_asynchronously,
+                :before_list, :after_list, :is_concurrently, :is_safely,
                 :exec_block, :commands, :name, :id
 
   def initialize(parent, block)
@@ -13,7 +13,7 @@ class Node
     self.before_list = Array.new
     self.after_list = Array.new
     self.is_safely = false
-    self.is_asynchronously = false
+    self.is_concurrently = false
     self.commands = Array.new
     self.id = self.to_s
     self.name = '[Node]'
@@ -72,7 +72,7 @@ class Node
 
   def draw_before_blocks(node, graph)
     if node.before_list.length > 0
-      id = Random.rand(100000000000000000000000)
+      id = Random.rand(2**64)
       name = "[BEFORE]\n"
       node.before_list.each do |before|
         name += ">#{before}\n"
@@ -85,7 +85,7 @@ class Node
 
   def draw_after_blocks(node, graph)
     if node.after_list.length > 0
-      id = Random.rand(100000000000000000000000)
+      id = Random.rand(2**64)
       name = "[AFTER]\n"
       node.after_list.each do |after|
         name += ">#{after}\n"

data/lib/tree_building/on_node.rb

@@ -5,7 +5,7 @@ class OnNode < Node
 	def initialize (parent, instance, block)
 		super(parent, block)
 		self.instance = instance
-    self.name = "[OnNode]\n" + instance
+    self.name = "[OnInstance]\n" + instance
   end
 
   def run(instance)
@@ -16,14 +16,12 @@ class OnNode < Node
       puts @instance
       @total_time[:exec].push(node.run(@instance))
     end
-
     after_exec(@instance, @total_time)
-
-    return @total_time
+    @total_time
   end
 
   def draw_block(graph)
-    graph.hexagon << graph.node(id)
+    graph.tripleoctagon << graph.node(id)
     graph.node(id).label(name)
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: claudius
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-17 00:00:00.000000000 Z
+date: 2015-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -136,14 +136,15 @@ files:
 - examples/on_instance.rb
 - examples/ping.rb
 - examples/simple_program.rb
+- examples/simple_program_2.rb
 - examples/specify_instances.rb
 - lib/remote_execution/cloud_provider.rb
 - lib/remote_execution/provider.rb
 - lib/remote_execution/virtual_machine.rb
 - lib/tree_building/claudius.rb
 - lib/tree_building/concurrent_node.rb
-- lib/tree_building/event_machine_play.rb
 - lib/tree_building/execution_node.rb
+- lib/tree_building/foreach_node.rb
 - lib/tree_building/node.rb
 - lib/tree_building/on_node.rb
 - test/execution_test.rb

data/lib/tree_building/event_machine_play.rb

@@ -1,20 +0,0 @@
-#!/usr/bin/ruby
-
-require 'rubygems'
-require 'eventmachine'
-
-class Server < EventMachine::Connection
-  attr_accessor :options, :status
-
-  def receive_data(data)
-    puts "#{@status} -- #{data}"
-    send_data("helo\n")
-  end
-end
-
-EM.run do
-  EM.start_server 'localhost', 8080, Server do |conn|
-    conn.options = {:my => 'options'}
-    conn.status = :OK
-  end
-end