RubyGems - bahuvrihi-tap - Versions diffs - 0.10.2 → 0.10.3 - Mend

bahuvrihi-tap 0.10.2 → 0.10.3

Files changed (22) hide show

data/cgi/run.rb +36 -0
data/cmd/run.rb +4 -3
data/cmd/server.rb +36 -0
data/lib/tap/app.rb +1 -1
data/lib/tap/constants.rb +1 -1
data/lib/tap/env.rb +8 -2
data/lib/tap/exe.rb +1 -57
data/lib/tap/support/declarations.rb +73 -51
data/lib/tap/support/executable.rb +81 -2
data/lib/tap/support/gems/rack.rb +151 -0
data/lib/tap/support/parsers/base.rb +81 -0
data/lib/tap/support/parsers/command_line.rb +90 -0
data/lib/tap/support/parsers/server.rb +83 -0
data/lib/tap/task.rb +327 -7
data/lib/tap/workflow.rb +9 -48
data/template/404.erb +12 -0
data/template/index.erb +41 -0
data/vendor/url_encoded_pair_parser.rb +93 -0
metadata +10 -4
data/lib/tap/support/command_line/parser.rb +0 -121
data/lib/tap/support/framework.rb +0 -83
data/lib/tap/support/framework_class.rb +0 -182

data/lib/tap/support/gems/rack.rb ADDED Viewed

@@ -0,0 +1,151 @@
+require 'rack'
+require 'cgi'
+module Tap
+  module Support
+    module Gems
+      Tap::Env.manifest(:public_paths, "public") do |search_path|
+        Dir.glob(File.join(search_path, "**/*")).collect do |path|
+          ["/" + Tap::Root.relative_filepath(search_path, path), path]
+        end
+      end
+      Tap::Env.manifest(:templates, "template") do |search_path|
+        Dir.glob(File.join(search_path, "**/*.erb")).collect do |path|
+          ["/" + Tap::Root.relative_filepath(search_path, path), path]
+        end
+      end
+      Tap::Env.manifest(:cgis, "cgi") do |search_path|
+        Dir.glob(File.join(search_path, "**/*.rb")).collect do |path|
+          ["/" + Tap::Root.relative_filepath(search_path, path), path]
+        end
+      end
+      module Rack
+        def call(env)
+          path = env['PATH_INFO']
+          case
+          when public_page = search(:public_paths, path)
+            # serve named static pages
+            response(env) { File.read(public_page) }
+          when cgi_page = search(:cgis, path)
+            # serve cgis relative to a cgi path
+            run_cgi(cgi_page, env)
+          when path == "/" || path == "/index"
+            # serve up the homepage
+            if env["QUERY_STRING"] == "refresh=true"
+              reset(:cgis) do |key, path|
+                Support::Lazydoc[path].resolved = false
+              end
+            end
+            template_response('index', env)
+          else
+            # handle all other requests as errors
+            template_response('404', env)
+          end
+        end
+        #--
+        # Runs a cgi and returns an array as demanded by rack.
+        def run_cgi(cgi_path, env)
+          current_output = $>
+          cgi_output = StringIO.new("")
+          begin
+            $> = cgi_output
+            with_env(env) { load(cgi_path) }
+            # collect the headers and body from the cgi output
+            headers, body = cgi_output.string.split(/\r?\n\r?\n/, 2)
+            raise "missing headers from: #{cgi_path}" if headers == nil
+            body = "" if body == nil
+            headers = headers.split(/\r?\n/).inject({}) do |hash, line|
+              key, value = line.split(/:/, 2)
+              hash[key] = value
+              hash
+            end
+            [headers.delete('Status') || 200, headers, body]
+          rescue(Exception)
+            # when an error occurs, return a standard cgi error with backtrace
+            [500, {'Content-Type' => 'text/plain'}, %Q{#{$!.class}: #{$!.message}\n#{$!.backtrace.join("\n")}}]
+          ensure
+            $> = current_output
+          end
+        end
+        # Executes block with ENV set to the specified hash.  Non-string env variables are not set.
+        def with_env(env)
+          current_env = {}
+          ENV.each_pair {|key, value| current_env[key] = value }
+          begin
+            ENV.clear
+            env.each_pair {|key, value| ENV[key] = value if value.kind_of?(String)}
+            yield
+          ensure
+            ENV.clear
+            current_env.each_pair {|key, value| ENV[key] = value }
+          end
+        end
+        DEFAULT_ERROR_TEMPLATE = %Q{
+<html>
+<body>
+# Error handling request: <%= error.message %></br>
+# <%= error.backtrace.join("<br/># ") %>
+<code><pre>
+<%= cgi.to_yaml %>
+<%= rack.to_yaml %>
+</pre></code>
+</body>
+</html>
+}
+        def response(env)
+          ::Rack::Response.new.finish do |res|
+            res.write begin
+              yield(res)
+            rescue
+              template(DEFAULT_ERROR_TEMPLATE, env, :error => $!)
+            end
+          end
+        end
+        def template(template, env, attributes={})
+          # partition and sort the env variables into
+          # cgi and rack variables.
+          rack, cgi = env.to_a.partition do |(key, value)|
+            key =~ /^rack/
+          end.collect do |part|
+            part.sort_by do |key, value|
+              key
+            end.inject({}) do |hash, (key,value)|
+              hash[key] = value
+              hash
+            end
+          end
+          Templater.new( template , {:server => self, :env => env, :cgi => cgi, :rack => rack}.merge(attributes) ).build
+        end
+        def template_response(name, env)
+          path = search(:templates, name)
+          response(env) { template(File.read(path), env) }
+        end
+      end
+    end
+  end
+end

data/lib/tap/support/parsers/base.rb ADDED Viewed

@@ -0,0 +1,81 @@
+module Tap
+  module Support
+    module Parsers
+      class Base
+        class << self
+          # Parses the input string as YAML, if the string matches the YAML document
+          # specifier (ie it begins with "---\s*\n").  Otherwise returns the string.
+          #
+          #   str = {'key' => 'value'}.to_yaml       # => "--- \nkey: value\n"
+          #   Tap::Script.parse_yaml(str)            # => {'key' => 'value'}
+          #   Tap::Script.parse_yaml("str")          # => "str"
+          def parse_yaml(str)
+            str =~ /\A---\s*\n/ ? YAML.load(str) : str
+          end
+        end
+        attr_reader :argvs
+        attr_reader :rounds
+        attr_reader :sequences
+        attr_reader :forks
+        attr_reader :merges
+        attr_reader :sync_merges
+        def build(env, app)
+          # attempt lookup and instantiate the task class
+          task_declarations = argvs.collect do |argv|
+            pattern = argv.shift
+            const = env.search(:tasks, pattern) or raise ArgumentError, "unknown task: #{pattern}"
+            task_class = const.constantize or raise ArgumentError, "unknown task: #{pattern}"
+            task_class.instantiate(argv, app)
+          end
+          # remove tasks used by the workflow
+          tasks = targets.collect do |index|
+            task, args = task_declarations[index]
+            unless args.empty?
+              raise ArgumentError, "workflow target receives args: #{task} [#{args.join(', ')}]"
+            end
+            tasks[index] = nil
+            task
+          end
+          # build the workflow
+          [:sequence, :fork, :merge, :sync_merge].each do |type|
+            send("#{type}s").each do |source, targets|
+              source.send(type, *targets.collect {|t| tasks[t] })
+            end
+          end
+          # build queues
+          queues = rounds.collect do |round|
+            round.each do |index|
+              task, args = task_declarations[index]
+              task.enq(*args) if task
+            end
+            app.queue.clear
+          end
+          queues.delete_if {|queue| queue.empty? }
+          queues
+        end
+        protected
+        def targets
+          results = sequences.collect {|source, targets| targets } +
+          forks.collect {|source, targets| targets } +
+          merges.collect {|target, sources| target } +
+          sync_merges.collect {|target, sources| target }
+          results.flatten.uniq.sort
+        end
+      end
+    end
+  end
+end

data/lib/tap/support/parsers/command_line.rb ADDED Viewed

@@ -0,0 +1,90 @@
+require 'tap/support/parsers/base'
+module Tap
+  module Support
+    module Parsers
+      class CommandLine < Base
+        class << self
+          def parse_sequence(str, count=0)
+             seq = []
+             seq << count if str[0] == ?:
+             str.split(/:+/).each do |n|
+               seq << n.to_i unless n.empty?
+             end
+             seq << count + 1 if str[-1] == ?:
+             [seq.shift, seq]
+          end
+          def pairs_regexp(l, r)
+            /\A--(\d*)#{Regexp.escape(l)}([\d,]*)#{Regexp.escape(r)}\z/
+          end
+          def parse_pairs(lead, str, count=0)
+             bracket = []
+             str.split(/,+/).each do |n|
+               bracket << n.to_i unless n.empty?
+             end
+             [lead.empty? ? count : lead.to_i, bracket]
+          end
+        end
+        ROUND = /\A--(\+(\d+)|\+*)\z/
+        SEQUENCE = /\A--(\d*(:\d*)+)\z/
+        FORK = pairs_regexp("[", "]")
+        MERGE = pairs_regexp("{", "}")
+        SYNC_MERGE = pairs_regexp("(", ")")
+        INVALID =  /\A--(\z|[^A-Za-z])/
+        def initialize(argv)
+          @sequences = []
+          @forks = []
+          @merges = []
+          @sync_merges = []
+          current = []
+          current_round = []
+          @argvs = []
+          @rounds = [current_round]
+          argv.each do |arg|
+            unless arg =~ INVALID
+              current << arg
+              next
+            end
+            # for peformance split to match
+            # most arguments just once.
+            unless current.empty?
+              current_round << @argvs.length
+              @argvs << current
+              current = []
+            end
+            case arg
+            when ROUND
+              current_round = (@rounds[$2 ? $2.to_i : $1.length] ||= [])
+            when SEQUENCE
+              @sequences << CommandLine.parse_sequence($1, @argvs.length-1)
+            when FORK
+              @forks << CommandLine.parse_pairs($1, $2, @argvs.length-1)
+            when MERGE
+              @merges << CommandLine.parse_pairs($1, $2, @argvs.length-1)
+            when SYNC_MERGE
+              @sync_merges << CommandLine.parse_pairs($1, $2, @argvs.length-1)
+            else
+              raise ArgumentError, "invalid argument: #{arg}"
+            end
+          end
+          unless current.empty?
+            current_round << @argvs.length
+            @argvs << current
+          end
+          @rounds.delete_if {|round| round.nil? || round.empty? }
+        end
+      end
+    end
+  end
+end

data/lib/tap/support/parsers/server.rb ADDED Viewed

@@ -0,0 +1,83 @@
+require 'tap/support/parsers/base'
+module Tap
+  module Support
+    module Parsers
+      # rounds syntax
+      # 0[task]=dump&0[config][key]=value&0[input][]=a&0[input][]=b
+      # sequence[1]=1,2,3
+      # fork[1]=2,3
+      # round[0]=1,2,3
+      # ....
+      class Server < Base
+        class << self
+          def parse_argv(hash)
+            raise ArgumentError, "no task specified" unless hash.has_key?('task')
+            # parse task
+            argv = [hash['task']]
+            # parse configs
+            configs = hash['config']
+            configs = YAML.load(configs) if configs.kind_of?(String)
+            case configs
+            when Hash
+              configs.each_pair do |key, value|
+                argv << "--#{key}"
+                argv << value
+              end
+            when nil
+            else raise ArgumentError, "non-hash configs specified: #{configs}"
+            end
+            # parse inputs
+            inputs = hash['inputs']
+            inputs = YAML.load(inputs) if inputs.kind_of?(String)
+            case inputs
+            when Array then argv.concat(inputs)
+            when nil
+            else raise ArgumentError, "non-array inputs specified: #{inputs}"
+            end
+            argv
+          end
+          def parse_pairs(values)
+            [*values].collect do |value|
+              value.split(',').collect {|i| i.to_i }
+            end.collect do |split|
+              next if split.empty?
+              [split.shift, split]
+            end.compact
+          end
+        end
+        INDEX = /\A\d+\z/
+        def initialize(argh)
+          @argvs = []
+          argh.each_pair do |key, value|
+            case key
+            when INDEX
+              argvs[key.to_i] = Server.parse_argv(value)
+            else
+              instance_variable_set("@#{key}s", Server.parse_pairs(value))
+            end
+          end
+          @rounds ||= []
+          @sequences ||= []
+          @forks ||= []
+          @merges ||= []
+          @sync_merges ||= []
+        end
+      end
+    end
+  end
+end

data/lib/tap/task.rb CHANGED Viewed

@@ -1,4 +1,6 @@
-require 'tap/support/framework'
+require 'tap/support/batchable'
+require 'tap/support/executable'
+require 'tap/support/command_line'
 module Tap
@@ -154,19 +156,309 @@ module Tap
   #   t1.array == t2.array                         # => true
   #   t1.array.object_id == t2.array.object_id     # => false
   #
-  class Task
+  class Task
+    include Support::Batchable
+    include Support::Configurable
     include Support::Executable
-    include Support::Framework
-    attr_reader :task_block
+    class << self
+      # Returns the default name for the class: to_s.underscore
+      attr_accessor :default_name
+      # Returns class dependencies
+      attr_reader :dependencies
+      def inherited(child)
+        unless child.instance_variable_defined?(:@source_file)
+          caller.first =~ Support::Lazydoc::CALLER_REGEXP
+          child.instance_variable_set(:@source_file, File.expand_path($1))
+        end
+        child.instance_variable_set(:@default_name, child.to_s.underscore)
+        child.instance_variable_set(:@dependencies, dependencies.dup)
+        super
+      end
+      def instance
+        @instance ||= new
+      end
+      # Generates or updates the specified subclass of self.
+      def subclass(const_name, configs={}, dependencies=[], options={}, &block)
+        #
+        # Lookup or create the subclass constant.
+        #
+        current, constants = const_name.to_s.constants_split
+        subclass = if constants.empty?
+          # The constant exists; validate the constant is a subclass of self.
+          unless current.kind_of?(Class) && current.ancestors.include?(self)
+            raise ArgumentError, "#{current} is already defined and is not a subclass of #{self}!"
+          end
+          current
+        else
+          # Generate the nesting module
+          subclass_const = constants.pop
+          constants.each {|const| current = current.const_set(const, Module.new)}
+          # Create and set the subclass constant
+          current.const_set(subclass_const, Class.new(self))
+        end
+        #
+        # Define the subclass
+        #
+        subclass.define_configurations(configs)
+        subclass.define_dependencies(dependencies)
+        subclass.define_process(block) if block_given?
+        #
+        # Register documentation
+        #
+        const_name = current == Object ? subclass_const : "#{current}::#{subclass_const}"
+        caller.each_with_index do |line, index|
+          case line
+          when /\/tap\/support\/declarations.rb/ then next
+          when Support::Lazydoc::CALLER_REGEXP
+            subclass.source_file = File.expand_path($1)
+            lzd = subclass.lazydoc(false)
+            lzd[const_name, false]['manifest'] = lzd.register($3.to_i - 1)
+            break
+          end
+        end
+        arity = options[:arity] || (block_given? ? block.arity : -1)
+        comment = Support::Comment.new
+        comment.subject = case
+        when arity > 0
+          Array.new(arity, "INPUT").join(' ')
+        when arity < 0
+          array = Array.new(-1 * arity - 1, "INPUT")
+          array << "INPUTS..."
+          array.join(' ')
+        else ""
+        end
+        subclass.lazydoc(false)[const_name, false]['args'] ||= comment
+        subclass.default_name = const_name.underscore
+        subclass
+      end
+      def instantiate(argv, app=Tap::App.instance) # => instance, argv
+        opts = OptionParser.new
+        # Add configurations
+        config = {}
+        unless configurations.empty?
+          opts.separator ""
+          opts.separator "configurations:"
+        end
+        configurations.each do |receiver, key, configuration|
+          opts.on(*Support::CommandLine.configv(configuration)) do |value|
+            config[key] = value
+          end
+        end
+        # Add options on_tail, giving priority to configurations
+        opts.separator ""
+        opts.separator "options:"
+        opts.on_tail("-h", "--help", "Print this help") do
+          opts.banner = "#{help}usage: tap run -- #{to_s.underscore} #{args.subject}"
+          puts opts
+          exit
+        end
+        # Add option for name
+        name = default_name
+        opts.on_tail('--name NAME', /^[^-].*/, 'Specify a name') do |value|
+          name = value
+        end
+        # Add option to add args
+        use_args = []
+        opts.on_tail('--use FILE', /^[^-].*/, 'Loads inputs from file') do |value|
+          obj = YAML.load_file(value)
+          case obj
+          when Hash
+            obj.values.each do |array|
+              # error if value isn't an array
+              use_args.concat(array)
+            end
+          when Array
+            use_args.concat(obj)
+          else
+            use_args << obj
+          end
+        end
+        opts.parse!(argv)
+        obj = new({}, name, app)
+        path_configs = load_config(app.config_filepath(name))
+        if path_configs.kind_of?(Array)
+          path_configs.each_with_index do |path_config, i|
+            obj.initialize_batch_obj(path_config, "#{name}_#{i}") unless i == 0
+          end
+          path_configs = path_configs[0]
+        end
+        argv = (argv + use_args).collect {|str| str =~ /\A---\s*\n/ ? YAML.load(str) : str }
+        [obj.reconfigure(path_configs).reconfigure(config), argv]
+      end
+      def lazydoc(resolve=true)
+        lazydoc = super(false)
+        lazydoc.register_method_pattern('args', :process) unless lazydoc.resolved?
+        super
+      end
+      DEFAULT_HELP_TEMPLATE = %Q{<% manifest = task_class.manifest %>
+<%= task_class %><%= manifest.subject.to_s.strip.empty? ? '' : ' -- ' %><%= manifest.subject %>
+<% unless manifest.empty? %>
+<%= '-' * 80 %>
+<% manifest.wrap(77, 2, nil).each do |line| %>
+  <%= line %>
+<% end %>
+<%= '-' * 80 %>
+<% end %>
+}
+      def help
+        Tap::Support::Templater.new(DEFAULT_HELP_TEMPLATE, :task_class => self).build
+      end
+      def depends_on(dependency_class, *args)
+        unless dependency_class.respond_to?(:instance)
+          raise ArgumentError, "dependency_class does not respond to instance: #{dependency_class}"
+        end
+        (dependencies << [dependency_class, args]).uniq!
+        self
+      end
+      protected
+      def dependency(name, dependency_class, *args)
+        depends_on(dependency_class, *args)
+        define_method(name) do
+          index = Support::Executable.index(dependency_class.instance, args)
+          Support::Executable.results[index]._current
+        end
+        public(name)
+      end
+      def define(name, klass=Tap::Task, &block)
+        instance_var = "@#{name}".to_sym
+        define_method(name) do |*args|
+          raise ArgumentError, "wrong number of arguments (#{args.length} for 1)" if args.length > 1
+          instance_name = args[0] || name
+          instance_variable_set(instance_var, {}) unless instance_variable_defined?(instance_var)
+          instance_variable_get(instance_var)[instance_name] ||= config_task(instance_name, klass, &block)
+        end
+        define_method("#{name}=") do |input|
+          input = {name => input} unless input.kind_of?(Hash)
+          instance_variable_set(instance_var, input)
+        end
+        public(name, "#{name}=")
+      end
+      def define_configurations(configs)
+        case configs
+        when Hash
+          # hash configs are simply added as default configurations
+          attr_accessor(*configs.keys)
+          configs.each_pair do |key, value|
+            configurations.add(key, value)
+          end
+          public(*configs.keys)
+        when Array
+          # array configs define configuration methods
+          configs.each do |method, key, value, opts, config_block|
+            send(method, key, value, opts, &config_block)
+          end
+        else
+          raise ArgumentError, "cannot define configurations from: #{configs}"
+        end
+      end
+      def define_dependencies(dependencies)
+        dependencies.each do |name, dependency_class, *args|
+          dependency(name, dependency_class, *args)
+        end if dependencies
+      end
+      def define_process(block)
+        send(:define_method, :process, &block)
+      end
+    end
+    instance_variable_set(:@source_file, __FILE__)
+    instance_variable_set(:@default_name, 'tap/task')
+    instance_variable_set(:@dependencies, [])
+    lazy_attr :manifest
+    lazy_attr :args
+    # The application used to load config_file templates
+    # (and hence, to initialize batched objects).
+    attr_reader :app
+    # The name of self.
+    #--
+    # Currently names may be any object.  Audit makes use of name
+    # via to_s, as does app when figuring configuration filepaths.
+    attr_accessor :name
+    # The task block provided during initialization.
+    attr_reader :task_block
+    # Initializes a new instance and associated batch objects.  Batch
+    # objects will be initialized for each configuration template
+    # specified by app.each_config_template(config_file) where
+    # config_file = app.config_filepath(name).
     def initialize(config={}, name=nil, app=App.instance, &task_block)
-      super(config, name, app)
+      super()
+      @app = app
+      @name = name || self.class.default_name
       @task_block = (task_block == nil ? default_task_block : task_block)
+      @_method_name = :execute
       @multithread = false
       @on_complete_block = nil
-      @_method_name = :execute
+      @dependencies = []
+      case config
+      when Support::InstanceConfiguration
+        @config = config
+        config.bind(self)
+      else
+        initialize_config(config)
+      end
+      self.class.dependencies.each do |task_class, args|
+        depends_on(task_class.instance, *args)
+      end
+    end
+    # Creates a new batched object and adds the object to batch. The batched object
+    # will be a duplicate of the current object but with a new name and/or
+    # configurations.
+    def initialize_batch_obj(overrides={}, name=nil)
+      obj = super().reconfigure(overrides)
+      obj.name = name if name
+      obj
     end
     # Enqueues self and self.batch to app with the inputs.
@@ -175,7 +467,7 @@ module Tap
     def enq(*inputs)
       app.queue.enq(self, inputs)
     end
     batch_function :enq, :multithread=
     batch_function(:on_complete) {}
@@ -237,6 +529,26 @@ module Tap
       task_block.call(*inputs)
     end
+    # Logs the inputs to the application logger (via app.log)
+    def log(action, msg="", level=Logger::INFO)
+      # TODO - add a task identifier?
+      app.log(action, msg, level)
+    end
+    # Raises a TerminateError if app.state == State::TERMINATE.
+    # check_terminate may be called at any time to provide a
+    # breakpoint in long-running processes.
+    def check_terminate
+      if app.state == App::State::TERMINATE
+        raise App::TerminateError.new
+      end
+    end
+    # Returns self.name
+    def to_s
+      name.to_s
+    end
     protected
     # Hook to set a default task block.  By default, nil.
@@ -255,5 +567,13 @@ module Tap
     def on_execute_error(err)
       raise err
     end
+    private
+    def config_task(name, klass=Tap::Task, &block)
+      configs = config[name] || {}
+      raise ArgumentError, "config '#{name}' is not a hash" unless configs.kind_of?(Hash)
+      klass.new(configs, name, &block)
+    end
   end
 end