madcat 1.0.7

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.swp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in madcat.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,25 @@
+Copyright (C) 2012, moe@busyloop.net
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, pulverize, distribute,
+synergize, compost, defenestrate, sublicense, and/or sell copies of the
+Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+If the Author of the Software (the "Author") needs a place to crash and
+you have a sofa available, you should maybe give the Author a break and
+let him sleep on your couch.
+
+If you are caught in a dire situation wherein you only have enough time
+to save one person out of a group, and the Author is a member of that
+group, you must save the Author.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO BLAH BLAH BLAH ISN'T IT FUNNY
+HOW UPPER-CASE MAKES IT SOUND LIKE THE LICENSE IS ANGRY AND SHOUTING AT YOU.
+

data/README.md

@@ -0,0 +1,10 @@
+# Madcat
+
+Experimental JSON pipeline.
+
+
+## Installation
+
+    $ gem install madcat
+
+

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/bin/madcat

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'madcat'
+
+MadCat::Cli.run!(ARGV)
+

data/lib/madcat/helpers.rb

@@ -0,0 +1,79 @@
+module MadCat
+  # Common Helpers
+  module Helpers
+    module ClassMethods
+      #
+      require 'open4'
+      require 'hashie/mash'
+      # Run argv via Open4, optionally passing it stdin.
+      #
+      # @param [Array] argv Command
+      # @param [String] stdin Stdin data
+      def exec!(argv, stdin='')
+        stdout, stderr = '', ''
+        status = Open4::spawn argv, {'quiet' => true, 'raise' => true, 'stdin' => stdin, 'stdout' => stdout, 'stderr' => stderr}
+        Hashie::Mash.new({ :status => status.exitstatus, :stdin => stdin, :stdout => stdout, :stderr => stderr, :cmd => argv })
+      end
+
+      # Read stream line-wise, yield line.split(delim)
+      #
+      # @param [IO] stream Stream to read from
+      # @param [String] delim Field delimiter
+      def stream_as_array(stream=STDIN, delim="\t")
+        stream.each do |line|
+          yield line.split(delim)
+        end
+      end
+
+      # preamble code by Starr Horne;
+      # https://github.com/starrhorne/preamble/
+      require 'yaml'
+      def yaml_preamble(text)
+        preamble_lines = []
+        body_lines = []
+
+        state = :before_preamble
+
+        text.split("\n").each do |line|
+          stripped = line.strip
+
+          case state
+            when :before_preamble
+              new_state = case stripped
+                when "---" 
+                  :preamble
+                when "" 
+                  :before_preamble
+                else 
+                  raise "First line must begin with ---"
+              end
+
+            when :preamble
+              new_state = case stripped
+                when "---" 
+                  :after_preamble
+                else 
+                  preamble_lines << line
+                  :preamble
+              end
+
+            when :after_preamble
+              new_state = :after_preamble
+              body_lines << line
+
+            else raise "Invalid State: #{ state }"
+          end
+
+          state = new_state
+        end
+        return [YAML::load(preamble_lines.join), body_lines.join]
+      end
+    end
+
+    def self.included base
+      base.extend ClassMethods
+    end
+
+    self.extend ClassMethods
+  end
+end

data/lib/madcat/version.rb

@@ -0,0 +1,3 @@
+module MadCat
+  VERSION = "1.0.7"
+end

data/lib/madcat.rb

@@ -0,0 +1,345 @@
+require "madcat/version"
+require "madcat/helpers"
+
+require "logger"
+require "optix"
+require "json"
+require "yaml"
+require "deep_merge"
+require "chronic_duration"
+
+module MadCat
+CAT = <<EOCAT
+                          ,
+   ,-.       _,---._ __  / \\
+  /  )    .-'       `./ /   \\
+ (  (   ,'            `/    /|
+  \\  `-"             \\'\\   / |
+   `.              ,  \\ \\ /  |
+    /`.          ,'-`----Y   |
+   (            ;        |   '
+   |  ,-.    ,-'         |  /
+   |  | (   | #{("v"+MadCat::VERSION).rjust(10)} | /
+   )  |  \\  `.___________|/
+   `--'   `--'
+EOCAT
+  module Cli
+    Optix::command do
+      params "[filename] [key=value] [...]"
+      #text "FIXME: docs needed"
+
+      CAT.split("\n").each { |line| text line }
+
+      # global options
+      opt :path, "Search path for pipeline executables", :default => '$MADCAT_PATH || $PATH'
+      opt :meta_key, "Insert metadata (read-only) on this key (OFF = no meta)", :default => '__pipeline__'
+      opt :key, "Pipeline key", :default => 'default'
+      opt :trace, "Trace all i/o", :default => false
+      opt :log_level, "Loglevel (fatal|error|warn|info|debug)", :default => 'warn', :short => :L
+      opt :ls, "List available pipelines to stdout and exit", :short => :l
+      opt :quiet_stdout, "Print leaf stdout to stdout", :default => false, :short => :Q
+      opt :quiet_stderr, "Do not print stderr to stderr", :default => false
+      opt :marker, "Insert informative marker above forwarded stdout/stderr", :default => false
+      opt :version, "Print version and exit"
+
+      trigger :version do
+        STDERR.puts "madcat v#{VERSION}"
+      end
+
+      exec do |cmd, opts, argv|
+        start_time = Time.now
+
+        logger = Logger.new(STDERR)
+        logger.level = Logger.const_get(opts[:log_level].upcase)
+        logger.level = Logger::DEBUG if opts[:trace]
+
+        # read input from ARGV[0] or STDIN
+        if ARGV[0] and !ARGV[0].include? '=' and File.readable? ARGV[0]
+          blob = File.read(ARGV.shift)
+          if blob[0..1] == '#!'
+            # remove shebang from input, if present
+            blob = blob.lines.to_a[1..-1].join
+          end
+        else
+          blob = STDIN.read()
+        end
+
+        # accept initial input as YAML or JSON
+        if blob[0..2] == '---'
+          blob = YAML.load(blob)
+        else
+          blob = JSON.parse(blob)
+        end
+
+        # the key that contains our pipeline must be called
+        # pipeline_$KEY (where $KEY defaults to 'default')
+        pipeline_key = "pipeline_#{opts[:key]}"
+
+        unless blob.include? pipeline_key
+          raise RuntimeError, "Pipeline key '#{pipeline_key}' not found in input"
+        end
+
+        unless blob[pipeline_key].is_a? Array
+          raise RuntimeError, "Value of pipeline key '#{pipeline_key}' is not an Array"
+        end
+
+        if blob["#{pipeline_key}_path"].is_a? String
+          opts[:path] = blob["#{pipeline_key}_path"]
+        end
+
+        if opts[:ls]
+          blob.keys.each do |key|
+            if key =~ /^pipeline_/ and not key =~ /^pipeline_.*_path$/
+              puts key.sub(/^pipeline_/,'')
+            end
+          end
+          exit 0
+        end
+
+        opts[:path] = ENV['MADCAT_PATH'] || ENV['PATH'] if opts[:path] == '$MADCAT_PATH || $PATH'
+        bin_path = opts[:path].split(':')
+
+        meta = { 'id' => pipeline_key,
+                 'current' => blob["pipeline_#{opts[:key]}"][0],
+                 'todo' => blob["pipeline_#{opts[:key]}"],
+                 'done' => [],
+                 'path' => bin_path }
+
+        # add read-only meta-data to the json?
+        meta_key = nil
+        if opts[:meta_key] != 'OFF'
+          meta_key = opts[:meta_key]
+          blob[meta_key] = meta
+        end
+
+        # key=value from ARGV
+        ARGV.each do |arg|
+          unless arg.include? '='
+            STDERR.puts "Error: Positional arguments must be given as key=value"
+            STDERR.puts "Invalid argument: #{arg}"
+            exit 2
+          end
+          k,v = arg.split('=', 2)
+
+          kk = k.split('.')
+          p = blob
+          kk.each_with_index do |e,i|
+            p = p[e] = i == kk.length-1 ? v : (p[e].is_a?(Hash) ? p[e] : {})
+          end
+        end
+
+        first_id = "#{meta['todo'][0].split(' ')[0]}:0"
+        results = MadCat::LitterBox.new(logger).run(blob.to_json, first_id, opts[:trace], 0, meta, meta_key)
+        end_time = Time.now
+
+        sum_jobs_duration = 0
+        error = false
+        results.each do |r|
+          indent = '  ' * (r[:depth] || 0)
+          line = "#{indent}#{r['id']} exit:#{r[:exit]} time:#{r['duration'].round(2)} stdin:#{r[:stdin].length} stdout:#{r[:stdout].length} stderr:#{r[:stderr].length}"
+          sum_jobs_duration += r['duration']
+
+          if r[:exit] != 0 or r.include? :exception_type
+            if r[:exception_type] == 'PipelineStop'
+              if 0 == r[:todo].length
+                line += " FIN"
+              else
+                line += " BREAK:#{r[:todo].length}"
+              end
+              logger.info line
+            else
+              line += " ERROR"
+              logger.error line
+              # be verbose, hoomins love that!
+              logger.fatal "ERROR: job '#{r['id']}' #{r[:exception_type]}, dumping..."
+              logger.fatal "-- DUMP for job #{r['id']} ------------------"
+
+              r.to_yaml.split("\n").each do |line|
+                next if line[0..2] == '---'
+                logger.fatal line
+              end
+              error = true
+            end
+          else
+            if 0 == r[:todo].length and 0 < r[:stdout].length
+              line += " FIN+"
+              unless opts[:quiet_stdout]
+                STDOUT.puts "-- STDOUT by #{r['id']} ---------------" if opts[:marker]
+                STDOUT.puts r[:stdout]
+              end
+            end
+            logger.info line
+          end
+
+          if opts[:quiet_stderr] == false and 0 < r[:stderr].length
+            STDERR.puts "-- STDERR by #{r['id']} ---------------" if opts[:marker]
+            STDERR.puts r[:stderr]
+          end
+        end
+
+        logger.info "pipeline_#{opts[:key]} finished, #{results.length} jobs"
+
+        total_duration = end_time - start_time
+        total_duration_text = ChronicDuration.output(total_duration.round(2), :format => :chrono)
+
+        avg_per_job = ChronicDuration.output((total_duration / results.length).round(2), :format => :chrono)
+
+        in_madcat = total_duration - sum_jobs_duration
+        in_madcat_text = ChronicDuration.output(in_madcat.round(2), :format => :chrono)
+        mad_prc = (in_madcat / total_duration * 100).round(2)
+        logger.info "time total: #{total_duration_text}, avg: #{avg_per_job}, in madcat: #{in_madcat_text} (#{mad_prc}%)"
+
+        exit 1 if error
+        exit 0
+      end
+    end
+
+    def self.run!(argv=ARGV)
+      Optix::invoke!(argv)
+    end
+  end
+
+  class LitterBox
+    class ErrorFromJSON < RuntimeError; end
+    class NonZeroExit < RuntimeError; end
+    class MalformedInput < RuntimeError; end
+    class MalformedOutput < RuntimeError; end
+    class PipelineStop < RuntimeError; end
+
+    def initialize(logger)
+      @logger = logger
+    end
+
+    def run(blob, id=:unknown, trace=false, depth=0, meta_in=nil, meta_key=nil)
+      begin
+        # scaffold the results-object as early as possible
+        # because this will also be returned and augmented
+        # when an exception occurs.
+        results = [{'id' => id, :exit => -1, :stdin => blob,
+                    'duration' => -1, :stdout => '', :stderr => ''}]
+
+        blob = JSON.parse(blob)
+
+        meta = Marshal.load(Marshal.dump(meta_in)) # deep_copy
+
+        todo = meta['todo']
+        raise RuntimeError, 'Pipeline has no jobs defined?!' if todo.length == 0
+
+        pipeline_id = meta['id']
+        job = todo.shift
+        cmd = job.split(' ')
+        job = cmd[0]
+
+        # find our executable
+        bin_path = meta['path']
+        bin = nil
+        bin_path.each do |e|
+          bin = File.join(e, cmd[0])
+          break if File.executable? bin
+          bin = nil
+        end
+
+        if bin.nil?
+          raise RuntimeError, "Executable #{job} not found in #{bin_path.join(':')}"
+        end
+
+        # update meta
+        meta['current'] = job
+
+        # add read-only metadata
+        if meta_key
+          blob[meta_key] = meta
+        end
+
+      
+        # run our executable
+        blob['*'] = cmd[0]
+        stdin = blob.to_json
+        results[0][:stdin] = stdin # we modified it above
+        start_time = Time.now
+        p = MadCat::Helpers::exec!([bin] + cmd[1..-1], stdin)
+        duration = (Time.now - start_time).to_f
+        results[0].merge!({:bin => bin, :exit => p.status, 'duration' => duration,
+                           :stdout => p.stdout, :stderr => p.stderr, :todo => todo,
+                           :depth => depth })
+
+        if trace
+          d=0
+          meta['done'].each do |node|
+            l.debug "## #{'  ' * d}#{node.keys[0]}"
+            d+=1
+          end
+          l.debug "## #{'  ' * d}#{id} <--"
+
+          l.debug "#" * 70
+          # zomg HACK: append a newline to stdin/stdout/stderr to convince
+          # the yaml serializer to print them as block literals.
+          # a bit evil, but it makes the trace-output much more readable...
+          [:stdin, :stdout, :stderr].each do |key|
+            if 0 < results[0][key].length && results[0][key][-1] != "\n"
+              results[0][key] += "\n"
+            end
+          end
+
+          results[0].to_yaml.split("\n").each do |line|
+            next if line[0..2] == '---'
+            l.debug line
+          end
+          l.debug "#" * 70
+        end
+ 
+        # alert on exit code != 0
+        raise NonZeroExit, "#{p.status}" if p.status != 0
+
+        # no output?  game over.
+        raise PipelineStop if p.stdout == ''
+
+        # parse output and wrap JSON errors
+        begin
+          blobs = JSON.parse(p.stdout)
+        rescue => e
+          raise MalformedOutput, "Could not parse JSON: #{e.message}" if 0 < todo.length
+        end
+        blobs = [blobs] unless blobs.is_a? Array
+
+        # raise on json-key 'err'
+        blobs.each do |blob|
+          if blob.include? 'err'
+            raise ErrorFromJSON, blob['err']
+          end
+        end
+
+        # oob mode
+        meta['done'] << { results[-1]['id'] => results[-1].select{|k,v| ['duration', :exit].include? k} }
+
+        # feed blob to next stage of the pipeline
+        i=0
+        blobs.each do |new_inner_blob|
+          break if 0 == todo.length 
+
+          # store output blob under jobid
+          blob_ = Marshal.load(Marshal.dump(blob)) # deep_copy
+          blob_[job] ||= {}
+          blob_[job].deep_merge! new_inner_blob
+
+          got = run(blob_.to_json, "#{todo[0].split(' ')[0]}:#{i}", trace, depth+1, meta, meta_key)
+          results += got
+
+          i+=1
+        end
+        results
+      rescue => e
+        results[-1][:exception_type] = e.class.name.split('::')[-1]
+        results[-1][:exception_message] = e.message
+        results[-1][:exception_backtrace] = e.backtrace
+
+        # bubble up
+        results
+      end
+    end
+
+    def l
+      @logger
+    end
+  end
+end

data/madcat.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/madcat/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Moe"]
+  gem.email         = ["moe@busyloop.net"]
+  gem.description   = %q{Katzeklo, Katzeklo...}
+  gem.summary       = %q{...ja das macht die Katze froh}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "madcat"
+  gem.require_paths = ["lib"]
+  gem.version       = MadCat::VERSION
+
+  gem.add_dependency 'optix'
+  gem.add_dependency 'open4'
+  gem.add_dependency 'hashie'
+  gem.add_dependency 'json'
+  gem.add_dependency 'chronic_duration'
+  gem.add_dependency 'deep_merge'
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: madcat
+version: !ruby/object:Gem::Version
+  version: 1.0.7
+  prerelease: 
+platform: ruby
+authors:
+- Moe
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-13 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: optix
+  requirement: &22553000 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *22553000
+- !ruby/object:Gem::Dependency
+  name: open4
+  requirement: &22552320 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *22552320
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: &22551900 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *22551900
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &22551400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *22551400
+- !ruby/object:Gem::Dependency
+  name: chronic_duration
+  requirement: &22550760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *22550760
+- !ruby/object:Gem::Dependency
+  name: deep_merge
+  requirement: &22550140 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *22550140
+description: Katzeklo, Katzeklo...
+email:
+- moe@busyloop.net
+executables:
+- madcat
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/madcat
+- lib/madcat.rb
+- lib/madcat/helpers.rb
+- lib/madcat/version.rb
+- madcat.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: ! '...ja das macht die Katze froh'
+test_files: []
+has_rdoc: