org-converge 0.0.11 → 0.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 80a71a84d944ea98cb325e2480e14143b116f475
-  data.tar.gz: 76039b9b0128a1234831a9c72f790d28ac4a0cf8
+  metadata.gz: 72e89091fab09c57f8564f168764b393111f5138
+  data.tar.gz: c7a7d29b0c0a86e11d95271d117a685cbab41793
 SHA512:
-  metadata.gz: 3f5742da864b2ace8cd17435ee42e07262996fc19c4c60e42a06fb6165336449d8bd8c1f754e21a96fb5ca092ed147852a541e5afab4184e9e572a745d423669
-  data.tar.gz: 3159fa01c575fe5bff5ff51ea1837cfd44f54879cdca9207fcc3f61c7ef8b5dce25f8521163f991d7f3ff82ed0665839581b68d9c92cc094e99a1f3f95b883df
+  metadata.gz: 43e71d209d7ea8bf2ada721adc05cda93734d3c85c1d9d3dcd01f8ee056258d1276aa7d4528672251a377af134cf3ed23507e3af430d7590bde9a7c75932a5b8
+  data.tar.gz: 6b95cf0b6c050102f52740400aa1d305f47369aec5655772717009b61597b31913250ac3f8c2b1b4704eddcee7aafaa6f7557b34aea2fd84d03f67a1c3802740

data/TODO

@@ -2,7 +2,7 @@
 #+TODO:    TODO | DONE CANCELED 
 #+startup: showeverything
 
-* [0/9] 0.1.0 version
+* [0/10] 0.1.0 version
 
 - [ ] Macros can be loaded and applied to the configuration
 - [ ] Actually support converging and idempotency (~--runmode=idempotent~)
@@ -17,8 +17,48 @@
 - [ ] Can use ~:dir~ for running a process relative to that directory
 - [ ] Bugfix for when results blocks have only inline examples or images
 - [ ] Bugfix for when the result from a ~org-spec~ run has non-zero exit status
+- [ ] Bugfixes for spec run
 
-* [1/1]   0.0.10
+* [0/5] 0.0.14
+
+- [ ] Distributed run helpers for the header arguments
+  : ssh
+- [ ] =:waits:=
+  Wait for other processes to start before starting.
+  The process does not start until the alive signal to the following
+  list of processes succeeds.
+- [ ] =:awaits:=
+  Wait for other processes to finish before starting.
+- [ ] =:onerror=
+  aborteverything, restart, runhandler
+- [ ] =:assertedby=
+
+* [2/2] 0.0.13
+
+- [X] =:waitsfor=, =:waitfor=, =:sleep:=
+  The process starts after this delay
+- [X] =:timeoutin=, =:timeout=
+
+  The process stops running after this time
+
+* [1/1] 0.0.12
+
+- [X] Use the shebang for the binary
+
+#+begin_src ruby :results output
+cmd = "#!/bin/bash".gsub('#!', '')
+
+puts cmd
+#+end_src
+
+#+RESULTS:
+: /bin/bash
+
+* [1/1] 0.0.11
+
+- [X] Normalize the binaries used for some languages (bash, node)
+
+* [1/1] 0.0.10
 
 - [X] Add ~:procs~ to code blocks to identify how many times it should run
 

data/lib/org-converge/babel_output_buffer.rb

@@ -45,13 +45,20 @@ module Orgmode
     def insert(line)
       # We try to get the lang from #+BEGIN_SRC and #+BEGIN_EXAMPLE blocks
       if line.begin_block?
+        block_header_arguments = { }
+        line.block_header_arguments.each_pair do |k, v|
+          if k.class == Symbol
+            block_header_arguments[k] = v
+          else
+            new_key = k.gsub(':', '').to_sym
+            block_header_arguments[new_key] = v
+          end
+        end
+
         case
-        when line.block_header_arguments[':tangle']
-          @current_tangle = line.block_header_arguments[':tangle']
-          @tangle[@current_tangle][:header] = {
-            :shebang => line.block_header_arguments[':shebang'],
-            :mkdirp  => line.block_header_arguments[':mkdirp']
-          }
+        when block_header_arguments[:tangle]
+          @current_tangle = block_header_arguments[:tangle]
+          @tangle[@current_tangle][:header] = block_header_arguments
           @tangle[@current_tangle][:lang] = line.block_lang
         when line.properties['block_name']
           # unnamed blocks are not run
@@ -59,16 +66,10 @@ module Orgmode
           @buffer = ''
 
           # Need to keep track of the options from a block before running it
-          @scripts[@scripts_counter][:header] = { 
-            :shebang => line.block_header_arguments[':shebang'],
-            :mkdirp  => line.block_header_arguments[':mkdirp'],
-            :name    => line.properties['block_name'],
-            :before  => line.block_header_arguments[':before'],
-            :after   => line.block_header_arguments[':after'],
-            :procs   => line.block_header_arguments[':procs'],
-          }
+          @scripts[@scripts_counter][:header] = block_header_arguments
+          @scripts[@scripts_counter][:header][:name] = line.properties['block_name']
           @scripts[@scripts_counter][:lang] = normalize_lang(line.block_lang)
-          
+
         # TODO: have a way to specify which are the default binaries to be used per language
         # when binary_detected?(@block_lang)
         else

data/lib/org-converge/command.rb

@@ -42,6 +42,7 @@ module OrgConverge
       true
     rescue => e
       @logger.error e
+      @logger.error e.backtrace.join("\n")
       false
     end
 
@@ -81,7 +82,7 @@ module OrgConverge
       logger.error "Cannot converge because there were errors during tangle step".fg 'red'
     end
 
-    def run_blocks_chain!
+    def run_blocks_chain!      
       # Chain the blocks by defining them as Rake::Tasks dynamically
       tasks = { }
 
@@ -93,8 +94,9 @@ module OrgConverge
         task = Rake::Task.define_task task_name do
           with_running_engine do |engine|
             file = File.expand_path("#{@run_dir}/#{key}")
-            cmd = "#{script[:lang]} #{file}"
-            engine.register task_name, cmd, { :cwd => @root_dir, :logger => logger }
+            bin = determine_lang_bin(script)
+            cmd = "#{bin} #{file}"
+            run_procs(script, cmd, engine)
           end
         end
         tasks[task_name] = { 
@@ -146,8 +148,9 @@ module OrgConverge
         display_name = script[:header][:name]
         with_running_engine do |engine|
           file = File.expand_path("#{@run_dir}/#{key}")
-          cmd = "#{script[:lang]} #{file}"
-          engine.register display_name, cmd, { :cwd => @root_dir, :logger => logger }
+          bin = determine_lang_bin(script)
+          cmd = "#{bin} #{file}"
+          run_procs(script, cmd)
         end
       end
       logger.info "Run has completed successfully.".fg 'green'
@@ -157,12 +160,12 @@ module OrgConverge
       @engine = OrgConverge::Engine.new(:logger => @logger, :babel => @babel)
       babel.tangle_runnable_blocks!(@run_dir)
       babel.ob.scripts.each do |key, script|
-        file = File.expand_path("#{@run_dir}/#{key}")
-        cmd = "#{script[:lang]} #{file}"
-
         # Decision: Only run blocks which have a name
         next unless script[:header][:name]
 
+        file = File.expand_path("#{@run_dir}/#{key}")
+        bin = determine_lang_bin(script)
+        cmd = "#{bin} #{file}"
         run_procs(script, cmd)
       end
       logger.info "Running code blocks now! (#{babel.ob.scripts.count} runnable blocks found in total)"
@@ -176,7 +179,8 @@ module OrgConverge
       scripts = babel.ob.scripts.select {|k, h| h[:header][:name] =~ Regexp.new(@options['--name']) }
       scripts.each do |key, script|
         file = File.expand_path("#{@run_dir}/#{key}")
-        cmd = "#{script[:lang]} #{file}"
+        bin = determine_lang_bin(script)
+        cmd = "#{bin} #{file}"
         run_procs(script, cmd)
       end
 
@@ -203,7 +207,8 @@ module OrgConverge
         display_name = script[:header][:name]
         with_running_engine do |engine|
           file = File.expand_path("#{@run_dir}/#{key}")
-          cmd = "#{script[:lang]} #{file}"
+          bin = determine_lang_bin(script)
+          cmd = "#{bin} #{file}"
           engine.register display_name, cmd, { :cwd => @root_dir, :logger => logger }
         end
       end
@@ -238,7 +243,8 @@ module OrgConverge
         display_name = script[:header][:name]
         script_file  = File.expand_path("#{@run_dir}/#{key}")
         results_file = File.expand_path("#{@results_dir}/#{key}")
-        cmd           = "#{script[:lang]} #{script_file}"
+        bin = determine_lang_bin(script)
+        cmd = "#{bin} #{script_file}"
 
         with_running_engine(:runmode => 'spec', :results_dir => @results_dir) \
         do |engine|
@@ -335,16 +341,25 @@ module OrgConverge
       end
     end
 
-    def run_procs(script, cmd)
+    def run_procs(script, cmd, engine=nil)
+      engine ||= @engine
       display_name = script[:header][:name]
       if script[:header][:procs]
         procs = script[:header][:procs].to_i
-        procs.times do |i|
-          proc_name = "#{display_name}-#{i}"
-          @engine.register proc_name, cmd, { :cwd => @root_dir, :logger => logger }
+        1.upto(procs) do |i|
+          proc_name = "#{display_name}:#{i}"
+          engine.register proc_name, cmd, { :cwd => @root_dir, :logger => logger,  :header => script[:header] }
         end
       else
-        @engine.register display_name, cmd, { :cwd => @root_dir, :logger => logger }
+        engine.register display_name, cmd, { :cwd => @root_dir, :logger => logger, :header => script[:header] }
+      end
+    end
+
+    def determine_lang_bin(script)
+      if script[:header][:shebang]
+        script[:header][:shebang].gsub('#!', '')
+      else
+        script[:lang]
       end
     end
   end

data/lib/org-converge/engine.rb

@@ -19,6 +19,10 @@ module OrgConverge
       @logger  = options[:logger] || Logger.new(STDOUT)
       @babel   = options[:babel]
       @runmode = options[:runmode]
+
+      # Code blocks whose start invocation is manipulated run inside a thread
+      @threads = []
+      @running_threads = { }
     end
 
     # We allow other processes to exit with 0 status
@@ -29,28 +33,45 @@ module OrgConverge
       watch_for_output
       sleep 0.1
       begin
-        status = watch_for_termination        
-      end while @running.count > 0
+        status = watch_for_termination do 
+          @threads.each do |t|
+            unless t.alive?
+              t.exit
+              @running_threads.delete(t.__id__)
+            end
+          end
+        end
+      end while (@running.count > 0 or @running_threads.count > 0)
     end
 
     # Overriden: we do not consider process formations
     def spawn_processes
       @processes.each do |process|
         reader, writer = create_pipe
+        pid    = nil
+        thread = nil
         begin
           # In case of spec mode, we need to redirect the output to a results file instead
           writer = File.open(process.options[:results], 'a') if @runmode == 'spec'
-          pid = process.run(:output => writer)
-          @names[process] = "#{@names[process]}.#{pid}"
+          pid, thread = process.run(:output => writer, :header => process.options[:header])
+          @names[process] = "#{@names[process]}.#{pid || thread.__id__}"
 
           # NOTE: In spec mode we need to be more strict on what is flushed by the engine
           # because we will be comparing the output
-          writer.puts "started with pid #{pid}" unless @runmode == 'spec'
+          unless @runmode == 'spec'
+            writer.puts "started with pid #{pid}" if pid
+            writer.puts "started thread with tid #{thread.__id__}" if thread
+          end
         rescue Errno::ENOENT
           writer.puts "unknown command: #{process.command}" unless @runmode == 'spec'
         end
-        @running[pid] = [process]
-        @readers[pid] = reader
+
+        @running[pid] = [process] if pid
+        @readers[pid || thread.__id__] = reader
+        if thread
+          @threads << thread
+          @running_threads[thread.__id__] = [process]
+        end
       end
     end
 
@@ -65,12 +86,13 @@ module OrgConverge
     def output(name, data)
       data.to_s.lines.map(&:chomp).each do |message|
         # FIXME: In case the process has finished before its lines where flushed
+        output = "#{name} -- #{message}"
         ps, pid = name.empty? ? '<defunct>' : name.split('.')
-        output  = "#{pad_process_name(ps)}(#{pid})".fg get_color_for_pid(pid.to_i)
+        output  = "#{pad_process_name(ps)}".fg get_color_for_pid(pid.to_i)
         output += " -- "
         output += message
 
-        # FIXME: When the process has stopped already, the name of the process does not appear
+        # FIXME: When the process has stopped already, the name of the process/thread does not appear
         #        (which means that this approach is wrong from the beginning probably)
         logger.info output
       end
@@ -81,7 +103,7 @@ module OrgConverge
     private
     def name_padding
       @name_padding ||= begin
-                          name_padding  = @names.values.map { |n| n.length }.sort.last
+                          name_padding  = @names.values.map { |n| n.split('.').first.length }.sort.last
                           [ 9, name_padding ].max
                         end
     end
@@ -102,12 +124,70 @@ module OrgConverge
       pid
     rescue Errno::ECHILD
     end
+
+    def name_for(pid)
+      process = nil
+      index   = nil
+      if @running[pid]
+        process, index = @running[pid]
+      elsif @running_threads[pid]
+        process, index = @running_threads[pid]
+      end
+      name_for_index(process, index)
+    end
+
+    def name_for_index(process, index)
+      [ @names[process], index.to_s ].compact.join(".")
+    end
   end
 
   # Need to expose the options to make the process be aware
   # of the possible running mode (specially spec mode)
   # and where to put the results output
+  require 'timeout'
   class CodeBlockProcess < Foreman::Process
     attr_reader :options
+
+    def run(options={})
+      env    = @options[:env].merge(options[:env] || {})
+      output = options[:output] || $stdout
+      runner = "#{Foreman.runner}".shellescape      
+
+      # whitelist the modifiers which manipulate how to the block is started
+      block_modifiers = { }
+      if options[:header]
+        block_modifiers[:waitfor] = options[:header][:waitsfor]  || options[:header][:waitfor] || options[:header][:sleep]
+        block_modifiers[:timeout] = options[:header][:timeoutin] || options[:header][:timeout] || options[:header][:timeoutafter]
+      end
+
+      pid     = nil
+      thread  = nil
+      process = nil
+
+      if block_modifiers and (block_modifiers[:waitfor] || block_modifiers[:timeout])
+        thread = Thread.new do
+          waitfor = block_modifiers[:waitfor].to_i
+          timeout = block_modifiers[:timeout].to_i
+          process = proc do
+            sleep waitfor if waitfor > 0
+            wrapped_command = "exec #{runner} -d '#{cwd}' -p -- #{command}"
+            pid = Process.spawn env, wrapped_command, :out => output, :err => output
+          end
+          timeout > 0 ? Timeout::timeout(timeout, &process) : process.call
+        end
+      else    
+        if Foreman.windows?
+          Dir.chdir(cwd) do
+            pid = Process.spawn env, expanded_command(env), :out => output, :err => output
+          end
+        else
+          wrapped_command = "exec #{runner} -d '#{cwd}' -p -- #{command}"
+          pid = Process.spawn env, wrapped_command, :out => output, :err => output
+        end
+      end
+
+      # In case of thread, pid will be nil
+      return pid, thread
+    end
   end
 end

data/lib/org-converge/version.rb

@@ -1,3 +1,3 @@
 module OrgConverge
-  VERSION = "0.0.11"
+  VERSION = "0.0.12"
 end

data/spec/converge_examples/basic_run_example/setup.org

@@ -6,7 +6,7 @@ with all output being flushed to the screen.
 - Count some numbers with bash
 
 #+name: bash_counter
-#+begin_src sh :shebang #!/bin/bash
+#+begin_src sh
 echo '' > out.log
 for i in `seq 1 5`; do 
 echo "Writing! $i"
@@ -18,7 +18,7 @@ done
 - Count some numbers with ruby
 
 #+name: ruby_counter
-#+begin_src ruby :shebang #!/usr/bin/ruby
+#+begin_src ruby
 $stdout.sync = true
 sleep 0.2
 10.times do |n|

data/spec/converge_examples/block_modifiers/run.org

@@ -0,0 +1,29 @@
+#+TITLE: Code block modifiers
+
+We have the following process, but it will take 5 seconds to start...
+
+#+name: waits-5-seconds
+#+begin_src sh :sleep 2
+echo "Wait..."
+for i in `seq 1 4`;
+do
+echo $i >> out.log
+sleep 1
+done
+echo "Now done!"
+#+end_src
+
+This one on the other hand starts as soon as possible:
+
+#+name: does-not-wait
+#+begin_src sh 
+echo "whoosh" > out.log
+#+end_src
+
+#+name: timeout-in-3-seconds
+#+begin_src sh :timeout 3
+while true; do
+  echo "Eventually will timeout..."
+  sleep 1
+done
+#+end_src

data/spec/converge_examples/expected_results/spec.org

@@ -7,12 +7,14 @@ results, we can try checking whether the contents from the code block
 are evaling into what is being documented in the results block.
 
 #+name: hello
-#+begin_src bash :results output code :exports both
+#+begin_src sh :results output code :exports both
 for i in `seq 20`; do 
 echo "hello"
 done
 #+end_src
 
+.........
+
 #+RESULTS: hello
 #+BEGIN_SRC sh
 hello
@@ -60,3 +62,4 @@ hola
 hola
 #+END_SRC
 
+.........

data/spec/converge_examples/expected_results/spec2.org

@@ -26,7 +26,7 @@ This one would fail near christmas, outside of Japan,
 when the site is down, or there is a change in the rss.xml being provided.
 
 #+name: christmas_check
-#+begin_src bash :results output
+#+begin_src sh :results output
 curl https://isitchristmas.com/rss.xml 2> /dev/null | grep title
 # curl isitchristmastyet.com 2> /dev/null
 #+end_src

data/spec/converge_examples/runlist_example/setup.org

@@ -13,6 +13,6 @@ echo "first" > out.log
 The script below will only be called once the one above is run.
 
 #+name: second
-#+begin_src ruby :shebang #!/usr/local/bin/ruby
+#+begin_src ruby
 File.open("out.log", 'a') {|f| f.puts "second" }
 #+end_src

data/spec/converge_examples/shebang/run.org

@@ -0,0 +1,24 @@
+#+title: Free style shebang usage
+#+runmode :sequential
+
+#+name: ruby20
+#+begin_src ruby :shebang #!~/.rvm/rubies/ruby-2.0.0-p353/bin/ruby :results output code
+puts RUBY_VERSION
+#+end_src
+
+#+RESULTS: ruby20
+#+BEGIN_SRC ruby
+2.0.0
+#+END_SRC
+
+#+name: ruby193
+#+begin_src ruby :shebang #!~/.rvm/rubies/ruby-1.9.3-p0/bin/ruby :results output code
+puts RUBY_VERSION
+#+end_src
+
+#+RESULTS: ruby193
+#+BEGIN_SRC ruby
+1.9.3
+#+END_SRC
+
+.

data/spec/converge_spec.rb

@@ -106,7 +106,7 @@ describe OrgConverge::Command do
     File.read(File.join(example_dir, 'out.log')).should == expected_contents
   end
 
-  it "should run 'expected_results' with src blocks" do
+  pending "should run 'expected_results' with src blocks" do
     example_dir = File.join(EXAMPLES_DIR, 'expected_results')
     setup_file = File.join(example_dir, 'spec.org')
 
@@ -145,4 +145,18 @@ describe OrgConverge::Command do
     largest = File.open("#{example_dir}/result").read
     largest.should == "906609\n"
   end
+
+  it "should support 'block_modifiers'" do
+    example_dir = File.join(EXAMPLES_DIR, 'block_modifiers')
+    setup_file = File.join(example_dir, 'run.org')
+
+    o = OrgConverge::Command.new({ 
+                                   '<org_file>' => setup_file,
+                                   '--root-dir' => example_dir
+                                 })
+    success = o.execute!
+    success.should == true
+    result = File.open("#{example_dir}/out.log").read
+    result.should == "whoosh\n1\n2\n3\n4\n"
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: org-converge
 version: !ruby/object:Gem::Version
-  version: 0.0.11
+  version: 0.0.12
 platform: ruby
 authors:
 - Waldemar Quevedo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-28 00:00:00.000000000 Z
+date: 2014-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docopt
@@ -130,12 +130,14 @@ files:
 - spec/converge_examples/basic_run_example/setup.org
 - spec/converge_examples/basic_tangle/conf.yml.expected
 - spec/converge_examples/basic_tangle/setup.org
+- spec/converge_examples/block_modifiers/run.org
 - spec/converge_examples/commented_block/run.org
 - spec/converge_examples/expected_results/spec.org
 - spec/converge_examples/expected_results/spec2.org
 - spec/converge_examples/linked_tasks/tasks.org
 - spec/converge_examples/multi_proc/run.org
 - spec/converge_examples/runlist_example/setup.org
+- spec/converge_examples/shebang/run.org
 - spec/converge_examples/specified_block/run.org
 - spec/converge_spec.rb
 - spec/spec_helper.rb
@@ -167,12 +169,14 @@ test_files:
 - spec/converge_examples/basic_run_example/setup.org
 - spec/converge_examples/basic_tangle/conf.yml.expected
 - spec/converge_examples/basic_tangle/setup.org
+- spec/converge_examples/block_modifiers/run.org
 - spec/converge_examples/commented_block/run.org
 - spec/converge_examples/expected_results/spec.org
 - spec/converge_examples/expected_results/spec2.org
 - spec/converge_examples/linked_tasks/tasks.org
 - spec/converge_examples/multi_proc/run.org
 - spec/converge_examples/runlist_example/setup.org
+- spec/converge_examples/shebang/run.org
 - spec/converge_examples/specified_block/run.org
 - spec/converge_spec.rb
 - spec/spec_helper.rb