pipa 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 694998b5a4309d4faacf4183ecf7cb8541a40c86
+  data.tar.gz: b3285ca9af59c771d9a676540dd3a2c09003d058
+SHA512:
+  metadata.gz: 074fae7374f8ac5c280b103bb91e1c3de53de15c8791f3aa405a1edc55d32343f269dc45fb1a8fc8328b5ce5e85a180332ad392b968ea5b1f03cb3a773a25624
+  data.tar.gz: c397d29ab6c7ae786478fa2e0c2bc589c8403556e5fc0a08b026dd12020b987cc8667619ffcc15f7a4ed99d1892f905808023cdd3bbb909a0177da2d4d8a0109

data/bin/pipa

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require 'pipa'
+require 'yaml'
+
+config = YAML.load(IO.read(ARGV[0]))
+
+pipa = Pipa.new(config["stages"])
+pipa.execute
+pipa.wait

data/lib/pipa.rb

@@ -0,0 +1,127 @@
+require 'set'
+require 'open3'
+require 'colorize'
+
+class Pipa
+	def initialize(stages)
+		@stages = stages
+		@to_be_executed = Set.new( stages.map{|k,v| k} )
+		@executed = Set.new
+		@log = {}
+		@ret = {}
+
+		@stages.each do |name, attributes|
+			attributes["dependencies"] ||= []
+			attributes["dependencies_set"] = Set.new(attributes["dependencies"])
+		end
+
+		@threads = []
+
+		@success = true
+	end
+
+	def execute
+		@to_be_executed.delete_if do |stage|
+			if resolved_dependencies? stage
+				execute_stage(stage)
+				true
+			else
+				false
+			end
+		end
+	end
+
+	def wait
+		@threads.map(&:join)
+		unless @to_be_executed.empty?
+			log_error "Some stages couldn't be executed: #{@to_be_executed.to_a}"
+		end
+		@success
+	end
+
+	private
+
+	def log_info(msg)
+		puts
+		puts "-------- [#{Time.now.utc}] --------".green
+		puts msg.green
+		puts "-------------------------------------------".green
+	end
+
+	def log_error(msg)
+		puts
+		puts "-------- [#{Time.now.utc}] --------".red
+		puts msg.red
+		puts "-------------------------------------------".red
+	end
+
+	def resolved_dependencies?(stage)
+		@stages[stage]["dependencies_set"].subset?(@executed)
+	end
+
+	def execute_stage(stage)
+		@threads << Thread.new do
+			t = Time.now
+
+			reader, writer = IO.pipe
+
+			mode = ["bash","ruby","node","http"].find {|m| !@stages[stage][m].nil?}
+			input = @stages[stage]["dependencies"].map{|d| "___deserialize(#{@ret[d].dump})"}.join(',')
+
+			cmd = case mode
+				when "bash"
+					["bash", "-e", "-c", "bash -e -c '#{@stages[stage][mode]}' | ruby -e \"require 'json'; print ARGF.read.to_json\" | tee >(cat >&3)"]
+				when "ruby"
+					["ruby", "-e", %Q(
+						require 'json'
+						#{@stages[stage][mode]};
+						___deserialize = JSON.instance_method(:parse)
+						___ret = main(#{input});
+						___ret_fd = IO.open(3, 'w')
+						___ret_fd.write(___ret.to_json);
+						___ret_fd.close
+					)]
+				when "node"
+					["node", "-e", %Q(
+						___deserialize = JSON.parse
+						#{@stages[stage][mode]};
+						const ___ret = main(#{input});
+						fs.writeSync(3, JSON.stringify(___ret));
+					)]
+				when "http"
+					["ruby", "-e", %Q(
+						require 'httpclient'
+						require 'json'
+						puts ___ret = HTTPClient.get_content("#{@stages[stage][mode]}")
+						___ret_fd = IO.open(3, 'w')
+						___ret_fd.write(___ret.to_json);
+						___ret_fd.close
+					)]
+			end
+
+			@log[stage] = ""
+			@ret[stage] = ""
+
+			Open3.popen2e(*cmd, 3 => writer.fileno) do |stdin, stdout_err, wait_thr|
+				while line = stdout_err.gets
+					puts line
+					@log[stage] << line
+				end
+
+				exit_status = wait_thr.value
+				if exit_status.success?
+					writer.close
+					@ret[stage] = reader.read
+					reader.close
+					@executed.add(stage)
+					log_info "Stage '#{stage}' took #{Time.now - t}s."
+				else
+					log_error "Stage '#{stage}' failed with status #{exit_status.exitstatus} after #{Time.now - t}s. Error msg:\n#{@log[stage]}"
+					@success = false
+				end
+
+				execute
+			end
+		end
+	end
+end

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification
+name: pipa
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Miguel Cantón Cortés
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-05-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+description: Define and execute pipelines
+email: miwelc@gmail.com
+executables:
+- pipa
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/pipa
+- lib/pipa.rb
+homepage: http://rubygems.org/gems/pipa
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Pipelines, easy
+test_files: []