datalackeytools 0.3.4

data/bin/datalackey-shell

@@ -0,0 +1,1505 @@
+#!/usr/bin/env ruby
+
+# Copyright 2019 Ismo Kärkkäinen
+# Licensed under Universal Permissive License. See LICENSE.txt.
+
+# Argument handling and checking
+# datalackey process
+# Presenting command output
+# Notification presenters
+# Command handlers
+# Run command handlers
+
+require 'optparse'
+require 'readline'
+require 'shellwords'
+require 'json'
+require 'yaml'
+require 'set'
+require 'pathname'
+require 'io/console'
+require_relative '../lib/datalackeylib'
+
+$version = 1
+history_file_basename = '.datalackey-shell.history'
+history_file_max_lines = 2000
+$echo_lackey_output = false
+$echo_lackey_input = false
+
+# Argument handling and checking
+
+class Arguments
+  attr_reader :directory, :memory, :lackey, :permissions
+
+  def initialize
+    @directory = nil
+    @memory = nil
+    @lackey = nil
+    @permissions = nil
+  end
+
+  def parse(args)
+    parser = OptionParser.new do |opts|
+      opts.summary_indent = '  '
+      opts.summary_width = 26
+      opts.banner = "Usage: datalackey-shell [options]"
+      opts.separator ""
+      opts.separator "Options:"
+      opts.on("-m", "--memory", "Store data in memory.") do
+        @memory = true
+      end
+      opts.on("-d", "--directory [DIR]",
+          "Store data under (working) directory.") do |d|
+        @directory = d || ''
+      end
+      opts.on_tail("-h", "--help", "Print this help and exit.") do
+        puts opts
+        exit 0
+      end
+      opts.on("-l", "--lackey PROGRAM",
+          "Use specified datalackey executable.") do |e|
+        @lackey = e
+      end
+      opts.on("-p", "--permissions MODE", [:user, :group, :other],
+          "File permissions cover (user, group, other).") do |p|
+        @permissions = { :user => "600", :group => "660", :other => "666" }[p]
+      end
+      opts.on('-e', '--echo', 'Turn datalackey command and reply echo on.') do
+        $echo_lackey_output = true
+        $echo_lackey_input = true
+      end
+    end
+    parser.parse!(args)
+    begin
+      @directory, @permissions, @memory =
+        DatalackeyProcess.verify_directory_permissions_memory(
+          @directory, @permissions, @memory)
+    rescue ArgumentError => e
+      STDERR.puts e.to_s
+      exit 1
+    end
+  end
+end
+
+arguments = Arguments.new
+arguments.parse(ARGV)
+
+# Output when multiple threads and possibly prompt are present.
+
+$prompt = '> '
+$in_prompt = false
+$output_mutex = Mutex.new
+
+def print_lines(messages)
+  return false if not (messages.is_a? Array or messages.is_a? String) or messages.empty?
+  messages = [ messages ] unless messages.is_a? Array
+  rows, cols = IO.console.winsize
+  cut = []
+  messages.each do |m|
+    if m.length < cols or m.include? '\n'
+      cut.push m
+      next
+    end
+    pieces = m.split
+    line = pieces.first
+    for k in 1...pieces.length
+      if line.length + 1 + pieces[k].length <= cols
+        line << ' ' << pieces[k]
+        next
+      end
+      cut.push line
+      line = '  ' + pieces[k]
+    end
+    cut.push line
+  end
+  in_prompt = $in_prompt
+  $output_mutex.synchronize do
+    puts() if in_prompt
+    cut.each { |m| puts m }
+    print($prompt) if in_prompt
+    $stdout.flush
+    Readline.redisplay() if in_prompt
+  end
+  return true
+end
+
+# datalackey process
+
+def handle_notifcation(category, action, message, vars)
+  case category
+  when :internal
+    case action
+    when :stored then print_lines("Stored: #{vars.first}")
+    when :deleted then print_lines("Deleted: #{vars.first}")
+    when :data_error then print_lines("DATA ERROR: #{vars.first}")
+    when :started
+    when :ended
+    when :error_format then print_lines("FORMAT ERROR: #{vars.first}")
+    end
+  when :internal_error
+    case action
+    when :user_id
+      print_lines("Bad identifier in run command? #{msg.join(' ')}")
+    else
+      print_lines("INTERNAL ERROR: #{msg.join(' ')}")
+    end
+  end
+end
+
+begin
+  $lackey_proc = DatalackeyProcess.new(arguments.lackey, arguments.directory, arguments.permissions, arguments.memory)
+rescue ArgumentError => e
+  puts e.to_s
+  exit 1
+end
+stderr_discarder = DiscardReader.new($lackey_proc.stderr)
+$lackey = DatalackeyIO.new($lackey_proc.stdin, $lackey_proc.stdout,
+  Proc.new do |category, action, message, vars|
+    handle_notifcation(category, action, message, vars)
+  end,
+  Proc.new { |json| print_lines(json) if $echo_lackey_input },
+  Proc.new { |json| print_lines(json) if $echo_lackey_output })
+
+# Command handlers
+
+$handler = Hash.new(nil)
+
+class Handler
+
+  def syntax_array_requirements(arr)
+    return [ arr, arr.length, false ] unless arr.first.is_a? Integer
+    alternatives = false
+    required = arr.first
+    arr = arr[1...arr.length]
+    if required < 0
+      required = arr.length
+      alternatives = true
+    end
+    return arr, required, alternatives
+  end
+
+  def construct_usages(result, value, seen = [])
+    if value.is_a? Array
+      value, required, alternatives = syntax_array_requirements(value)
+      prev = nil
+      value.each_index do |k|
+        alt = value[k]
+        if alternatives and 0 < k
+          result.push(prev.is_a?(Array) ? ' || ' : '|')
+        elsif prev.is_a? String
+          result.push ' '
+        elsif prev.is_a? Symbol
+          result.push(' ') unless result.last == '[' or result.last == '|'
+        end
+        result.push('[') if 0 < required and k == required
+        prev = alt
+        construct_usages(result, alt, seen)
+      end
+      result.pop() if result.last == '|'
+      result.push(']') if 0 < required and required < value.length
+      return result
+    end
+    if seen.include? value
+      result.push '...'
+      return result
+    end
+    if @syntax.has_key? value
+      seen.push value
+      construct_usages(result, @syntax[value], seen)
+      seen.pop
+      return result
+    end
+    result.push value.to_s
+    return result
+  end
+
+  def usages
+    top = @syntax[:root]
+    return { :root => [ top ] } if top.is_a? String # Parameterless command.
+    results = { }
+    if top.first.is_a? Integer and top[1].is_a? String or top.first.is_a? String
+      results[:root] = construct_usages([], :root)
+    else # Command with variations. Expect array of arrays of symbol(s).
+      top = top[1...top.length] if top.first.is_a? Integer
+      top.each { |sym| results[sym] = construct_usages([], sym) }
+    end
+    return results
+  end
+
+  def print_help
+    msgs = []
+    usages.each_pair do |key, value|
+      m = "#{value.join}"
+      m.concat(" : #{@syntax[:help][key]}") if @syntax[:help].include? key
+      msgs.push m
+    end
+    print_lines(msgs.sort)
+  end
+
+  def completion_search(suggestions, parts, part_idx, key, item)
+    return true if part_idx >= parts.length
+    last = part_idx + 1 == parts.length
+    if item.is_a? Symbol # These either validate part or add suggestions.
+      cands = nil
+      case item
+      when :command
+        cands = $handler.keys
+      when :label
+        cands = $lackey.data.keys
+      when :process_id
+        cands = $lackey.process.keys
+      when :history_process_id
+        cands = [] # Run command identifiers from history.
+        scan_history([ Regexp.new('^run[[:blank:]]+') ]).each do |h|
+          pieces = h.shellsplit
+          cands.push(pieces[1]) if 1 < pieces.length
+        end
+      when :executable
+        return File.executable?(parts[part_idx]) unless last
+        cands = Dir[parts.last + '*'].grep(/^#{Regexp.escape(parts.last)}/)
+      when :string
+        return true # Anything goes.
+      when :int
+        begin
+          Integer(parts[part_idx])
+          return true # Any integer is fine.
+        rescue ArgumentError
+          return false
+        end
+      when :null
+        # Requires that :null is before :string in allowed values.
+        return $null_handler.null == parts[part_idx] unless last
+        cands = [ $null_handler.null ]
+      when :varname
+        return not(parts[part_idx].include?('=')) unless last
+        cands = []
+      else
+        return completion_search(suggestions, parts, part_idx, item, @syntax[item])
+      end
+      return false if cands.nil?
+      return cands.include?(parts[part_idx]) unless last
+      suggestions.concat cands
+      return true
+    elsif item.is_a? String
+      return item == parts[part_idx] unless last
+      suggestions.push item
+      return true
+    elsif item.is_a? Array
+      item, required, ored = syntax_array_requirements(item)
+      ok = true
+      item.each_index do |k|
+        sub = item[k]
+        res = completion_search(suggestions, parts, part_idx, key, sub)
+        ok = (ored ? (ok or res) : (required <= k ? ok : res))
+        break unless ok or ored
+        part_idx = part_idx + 1 unless ored
+      end
+      return ok
+    end
+    raise ArgumentError.new('Item other than nil, Symbol, String or Array.')
+  end
+
+  def completion_candidates(parts, str)
+    suggestions = []
+    parts.push('') if str.empty?
+    completion_search(suggestions, parts, 0, :root, @syntax[:root])
+    return suggestions.grep(/^#{Regexp.escape(str)}/).sort.uniq
+  end
+
+  def pop_false(match, condition)
+    match.pop() unless condition
+    return condition
+  end
+
+  def verify_search(match, parts, key, item)
+    return nil if match.length == parts.length
+    part = parts[match.length]
+    last = match.length + 1 == parts.length
+    if item.is_a? Symbol # These either validate part or add suggestions.
+      match.push item
+      case item
+      when :command
+        return pop_false(match, $handler.has_key?(part))
+      when :label
+        return pop_false(match, $lackey.data.has_key?(part))
+      when :process_id
+        return pop_false(match, $lackey.process.has_key?(part))
+      when :history_process_id
+        scan_history([ Regexp.new('^run[[:blank:]]+') ]).each do |h|
+          pieces = h.shellsplit
+          return true if pieces.length >= 2 and pieces[1] == part
+        end
+        return pop_false(match, false)
+      when :executable
+        return pop_false(match, File.executable?(part))
+      when :string
+        return true # Anything goes.
+      when :int
+        begin
+          Integer(part)
+          return true # Any integer is fine.
+        rescue ArgumentError
+          return pop_false(match, false)
+        end
+      when :null
+        nv = $null_handler.null
+        return pop_false(match, nv == part)
+      when :varname
+        return pop_false(match, not(part.include?('=')))
+      end
+      match.pop
+      return verify_search(match, parts, item, @syntax[item])
+    elsif item.is_a? String
+      match.push item
+      return pop_false(match, item == part)
+    elsif item.is_a? Array
+      item, required, alternatives = syntax_array_requirements(item)
+      required = 1 if alternatives
+      return false if parts.length < match.length + required
+      orig_count = match.length
+      item.each_index do |k|
+        sub = item[k]
+        ok = verify_search(match, parts, key, sub)
+        return nil if ok.nil? or (ok and match.length == parts.length)
+        if alternatives
+          return true if ok # One match is enough.
+          next
+        elsif not ok
+          if k < required # Looking for required consecutive matches.
+            match.pop(match.length - orig_count)
+            return false
+          elsif required == 0
+            return false # Optional repeating element not matched. Don't recurse.
+          end
+        end
+      end
+      return true unless alternatives
+      match.pop(match.length - orig_count)
+      return false
+    end
+    raise ArgumentError.new('Item other than nil, Symbol, String or Array.')
+  end
+
+  def verify(parts)
+    top = @syntax[:root]
+    return (parts.length == 1 and parts[0] == top) if top.is_a? String
+    match = []
+    return false unless verify_search(match, parts, :root, top).nil?
+    # Convert according to match information as needed.
+    parts.each_index do |k|
+      case match[k]
+      when :null
+        parts[k] = nil
+      when :int
+        parts[k] = Integer(parts[k])
+      end
+    end
+    return true
+  end
+
+  def register_search
+    # Currently it holds that the strings are at top level first or second
+    # level first items.
+    top = @syntax[:root]
+    return [ top ] if top.is_a? String
+    raise ArgumentError.new('Not String or Array.') unless top.is_a? Array
+    items, required, search_children = syntax_array_requirements(top)
+    return [ items.first ] unless search_children
+    cmds = []
+    items.each do |item|
+      sub = @syntax[item]
+      if sub.is_a? String
+        cmds.push sub
+        next
+      end
+      sub, required, wrong_assumption = syntax_array_requirements(sub)
+      raise ArgumentError.new('Alternatives at level 2') if wrong_assumption
+      raise ArgumentError.new('Symbol at level 2') unless sub.first.is_a? String
+      cmds.push sub.first
+    end
+    return cmds.uniq
+  end
+
+  def register
+    register_search.each do |cmd|
+      raise KeyError.new("Command in handlers: #{cmd}") if $handler.has_key? cmd
+      $handler[cmd] = self
+    end
+  end
+
+  def send(patt_act, message, user_id = false)
+    $lackey.send(patt_act, message, user_id)
+  end
+
+  def dump(json_as_string)
+    $lackey.dump(json_as_string)
+  end
+end
+
+class HelpHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ 1, 'help', :cmds ],
+      :cmds => [ 0, :command, :cmds ],
+      :help => { :root => "Print command help." }
+    }
+    register
+  end
+
+  def handle(parts)
+    printed = Set.new
+    keys = (parts.length > 1) ? parts[1...parts.length] : $handler.keys
+    keys.sort.each do |key|
+      if $handler.include? key
+        object = $handler[key]
+        next if printed.include? object
+        printed.add object
+        object.print_help
+      else
+        print_lines("Unknown command: #{key}")
+      end
+    end
+  end
+end
+#$handler['help'] = HelpHandler.new
+HelpHandler.new
+
+class ExitHandler < Handler
+  attr_reader :code, :exiting
+
+  def initialize
+    @syntax = {
+      :root => [ 1, 'exit', :int ],
+      :help => { :root => "Waits for datalackey to finish, exits with code." }
+    }
+    @code = 0
+    @exiting = false
+    register
+  end
+
+  def handle(parts)
+    @exiting = true
+    @code = parts[1] if parts.length == 2
+  end
+
+  def command_name
+    return @syntax[:root][1]
+  end
+end
+$exit_handler = ExitHandler.new
+
+class NullHandler < Handler
+  attr_reader :null
+
+  def initialize
+    @syntax = {
+      :root => [ -1, :nullcmd, :nullcmdwhat ],
+      :nullcmd => [ 'null', :string ],
+      :nullcmdwhat => 'null?',
+      :help => {
+        :nullcmd => "Sets string that is interpreted as null when null is allowed.",
+        :nullcmdwhat => "Prints out string that is interpreted as null."
+      }
+    }
+    @null = 'null'
+    register
+  end
+
+  def handle(parts)
+    if parts.length == 2
+      @null = parts.last
+    else
+      print_lines("String interpreted as null is: '#{@null}'")
+    end
+  end
+end
+$null_handler = NullHandler.new
+
+class ListHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ 1, 'ls', :regexps ],
+      :regexps => [ 0, :string, :regexps ],
+      :help => { :root => "Prints out the list of data labels that match optional regular expressions." }
+    }
+    register
+  end
+
+  def handle(parts)
+    exps = []
+    parts[1...parts.length].sort.uniq.each { |e| exps.push Regexp.new(e) }
+    matching = []
+    $lackey.data.keys.sort.each do |label|
+      match = exps.length == 0
+      exps.each do |e|
+        match = e.match(label)
+        break if match
+      end
+      matching.push(label) if match
+    end
+    print_lines(matching)
+  end
+end
+ListHandler.new
+
+class SetHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ 'set', :string, :string ],
+      :help => {
+        :root => "Sets label to JSON-encoded value (use ' or \" as needed)."
+      }
+    }
+    register
+  end
+
+  def handle(parts)
+    k = JSON.generate parts[1]
+    dump "{#{k}:#{parts.last}}"
+  end
+end
+SetHandler.new
+
+class GetHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ 2, 'get', :label, :labels ],
+      :labels => [ 0, :label, :labels ],
+      :help => { :root => "Gets labels." }
+    }
+    pretty_print = Proc.new do |category, action, message, vars|
+      m = nil
+      case category
+      when :print then m = JSON.pretty_generate(vars.first)
+      when :error then m = "Get failure: #{vars.join(' ')}"
+      when :note then m = "Missing: #{vars.join(' ')}"
+      end
+      print_lines m
+    end
+    @actions = PatternAction.new([{
+      :print => [ { :get => [ '@', 'get', '', '?' ] } ],
+      :error => [ { :fail => [ '@', 'get', 'failed', '*' ] } ],
+      :note => [ { :missing => [ '@', 'get', 'missing', '*' ] } ]
+    }], [ pretty_print ])
+    register
+  end
+
+  def handle(parts)
+    send(@actions, [ 'get' ].concat(parts[1...parts.length]))
+  end
+end
+GetHandler.new
+
+class RenameHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ 2, 'mv', :label_pair, :label_pairs ],
+      :label_pairs => [ 0, :label_pair, :label_pairs ],
+      :label_pair => [ :label, :string ],
+      :help => { :root => "Re-labels values. Takes label and new name pairs." }
+    }
+    message_proc = Proc.new do |category, action, message, vars|
+      case category
+      when :note then print_lines "Missing: #{vars.join(' ')}"
+      else false
+      end
+    end
+    @actions = PatternAction.new([{
+      :note => [ { :missing => [ '@', 'rename', 'missing', '*' ] } ]
+    }], [ message_proc ])
+    register
+  end
+
+  def handle(parts)
+    send(@actions, [ 'rename' ].concat(parts[1...parts.length]))
+  end
+end
+RenameHandler.new
+
+class DeleteHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ 2, 'rm', :label, :labels ],
+      :labels => [ 0, :label, :labels ],
+      :help => { :root => "Removes labels from storage." }
+    }
+    message_proc = Proc.new do |category, action, message, vars|
+      case category
+      when :note then print_lines "Missing: #{vars.join(' ')}"
+      else nil
+      end
+    end
+    @actions = PatternAction.new([{
+      :note => [ { :missing => [ '@', 'delete', 'missing', '*' ] } ]
+    }], [ message_proc ])
+    register
+  end
+
+  def handle(parts)
+    send(@actions, [ 'delete' ].concat(parts[1...parts.length]))
+  end
+end
+DeleteHandler.new
+
+class StorageInfoHandler < Handler
+  def initialize
+    @syntax = {
+      :root => 'info',
+      :help => { :root => "Print information about storage." }
+    }
+    present = Proc.new do |category, action, message, vars|
+      out = []
+      if category == :return and action == :storage_info
+        vars.first.each_pair do |label, info|
+          description = "#{label} : "
+          fmts = []
+          info.each_pair { |format, size| fmts.push "#{format}: #{size}" }
+          fmts.sort!
+          out.push(description + fmts.join(', '))
+        end
+        out.sort!
+      end
+      print_lines out
+    end
+    @actions = PatternAction.new([{
+      :return => [ { :storage_info => [ '@', 'storage-info', '', '?' ] } ]
+    }], [ present ])
+    register
+  end
+
+  def handle(parts)
+    send(@actions, [ 'storage-info' ])
+  end
+end
+StorageInfoHandler.new
+
+class PingHandler < Handler
+  def initialize
+    @syntax = {
+      :root => 'ping',
+      :help => { :root => "Check that datalackey responds." }
+    }
+    @actions = PatternAction.new([]) # Normal internal done is enough.
+    register
+  end
+
+  def handle(parts)
+    tracker = send(@actions, [ 'no-op' ])
+    if tracker.nil?
+      print_lines("No connection to datalackey.")
+      return
+    end
+    print_lines("Datalackey responded.") if tracker.status
+  end
+end
+PingHandler.new
+
+class VersionHandler < Handler
+  def initialize
+    @syntax = {
+      :root => 'version',
+      :help => { :root => "Print version information." }
+    }
+    register
+  end
+
+  def handle(parts)
+    vs = [ "shell: #{$version}" ]
+    $lackey.version.each_pair { |k, v| vs.push "#{k}: #{v}" }
+    vs.sort!
+    print_lines vs
+  end
+end
+VersionHandler.new
+
+class EchoHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ -1, :lackey, :lackeywhat ],
+      :lackey => [ 'echo', :inout, :onoff ],
+      :lackeywhat => 'echo?',
+      :inout => [ -1, 'command', 'reply', 'all' ],
+      :onoff => [ -1, 'on', 'off' ],
+      :help => {
+        :lackey => "Set the printing of communication with datalackey on or off.",
+        :lackeywhat => "Print datalackey command/reply printing status."
+      }
+    }
+    register
+  end
+
+  def handle(parts)
+    if parts.length > 1
+      what = parts[1]
+      on = parts[2] == 'on'
+      $echo_lackey_input = on if what == 'command' or what == 'all'
+      $echo_lackey_output = on if what == 'reply'or what == 'all'
+    else
+      print_lines([ "command: #{$echo_lackey_input ? 'on' : 'off'}",
+        "reply  : #{$echo_lackey_output ? 'on' : 'off'}" ])
+    end
+  end
+end
+EchoHandler.new
+
+def scan_history(regexps)
+  matched = Array.new
+  Readline::HISTORY.take(Readline::HISTORY.length - 1).each do |h|
+    match = regexps.length == 0
+    regexps.each do |e|
+      match = e.match(h)
+      break if match
+    end
+    matched.push(h) if match
+  end
+  return matched
+end
+
+class HistoryHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ 1, 'history', :regexps ],
+      :regexps => [ 0, :string, :regexps ],
+      :help => { :root => "Print commands that match any regex, or all." }
+    }
+    register
+  end
+
+  def handle(parts)
+    exps = []
+    parts[1...parts.length].each { |e| exps.push Regexp.new(e) }
+    print_lines scan_history(exps)
+  end
+end
+HistoryHandler.new
+
+class RecallHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ 'recall', :history_process_id ],
+      :help => { :root => "Fetch last \"run identifier\" related settings from history. Finds the output, env, arg, channel, and notify settings that were in effect when last run with given identifier was done. If no identifier is given, last run command identifier is used. Prints the lines found from history." }
+    }
+    register
+  end
+
+  def scan_until_first(new_to_old, exps)
+    regexps = []
+    exps.each { |e| regexps.push Regexp.new(e) }
+    matches = []
+    new_to_old.each do |c|
+      for k in 0...regexps.length
+        next unless regexps[k].match(c)
+        matches.push(c)
+        return matches if k == 0
+        break
+      end
+    end
+    return matches
+  end
+
+  def print_until_first(new_to_old, exps)
+    print_lines scan_until_first(new_to_old, exps).reverse
+  end
+
+  def handle(parts)
+    id_given = parts.length == 2
+    es = [ '^output[+]?[[:blank:]]+',
+      '^arg[+]?[[:blank:]]+', '^env[+]?[[:blank:]]+', '^env-clear$',
+      '^channel[[:blank:]]+', "^#{$exit_handler.command_name}",
+      '^notify[[:blank:]]+' ]
+    es.push('^run[[:blank:]]+' + (id_given ? parts[1] + '[[:blank:]]+' : ''))
+    exps = []
+    es.each { |e| exps.push Regexp.new(e) }
+    candidates = scan_history(exps)
+    # Drop everything after the last matching run command.
+    while not candidates.empty? and not candidates.last.start_with? 'run'
+      candidates.pop
+    end
+    # Pick commands before last exit. Reverses order.
+    session = []
+    while not candidates.empty? and not candidates.last.start_with? $exit_handler.command_name
+      session.push candidates.pop
+    end
+    print_until_first(session, [ '^notify[[:blank:]]+data[[:blank:]]' ])
+    print_until_first(session, [ '^notify[[:blank:]]+process[[:blank:]]' ])
+    print_until_first(session, [ '^channel[[:blank:]]+in[[:blank:]]' ])
+    print_until_first(session, [ '^channel[[:blank:]]+out[[:blank:]]' ])
+    print_until_first(session, [ '^channel[[:blank:]]+err[[:blank:]]' ])
+    print_until_first(session,
+      [ '^env[[:blank:]]', '^env[+][[:blank:]]', '^env-clear$' ])
+    print_until_first(session, [ '^arg[[:blank:]]', '^arg[+][[:blank:]]' ])
+    print_until_first(session, [ '^output[[:blank:]]', '^output[+][[:blank:]]', '^output-naming[[:blank:]]' ])
+  end
+end
+RecallHandler.new
+
+# Run command handlers
+
+class NotifyHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ -1, :notify, :notifywhat ],
+      :notify => [ 'notify', :procdata, :onoff ],
+      :notifywhat => 'notify?',
+      :procdata => [ -1, 'data', 'process' ],
+      :onoff => [ -1, 'on', 'off' ],
+      :help => {
+        :notify => "Turn process or data notifications on or off.",
+        :notifywhat => "Print data/process notification status."
+      }
+    }
+    @notify = { 'process' => 'off', 'data' => 'off' }
+    register
+  end
+
+  def handle(parts)
+    if parts.length > 1
+      @notify[parts[1]] = parts[2]
+    else
+      msgs = []
+      @notify.keys.sort.each { |key| msgs.push "#{key} : #{@notify[key]}" }
+      print_lines msgs
+    end
+  end
+
+  def command_part
+    command = []
+    @notify.each_pair do |key, value|
+      command.concat(['notify', key]) if value == 'on'
+    end
+    return command
+  end
+end
+$notify_handler = NotifyHandler.new
+
+class InputHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ -1, :input, :inputadd, :inputwhat ],
+      :input => [ 1, 'input', :args ],
+      :inputadd => [ 2, 'input+', :a2b, :args ],
+      :inputwhat => 'input?',
+      :args => [ 0, :a2b, :args ],
+      :a2b => [ -1, [ :string, ':', :label ], [ :string, '=', :value ] ],
+      :value => [ -1, :null, :int, :string ],
+      :help => {
+        :input => "Clears input mapping and passes label as name, or value of name as given directly.",
+        :inputadd => "Adds to current input mapping.",
+        :inputwhat => "Prints current input mapping."
+      }
+    }
+    @order = []
+    @mapping = { }
+    register
+  end
+
+  def handle(parts)
+    if parts[0] == 'input?'
+      msgs = []
+      @order.each do |name|
+        direct, val = @mapping[name]
+        t = direct ? "=" : ":"
+        msgs.push "#{name} #{t} #{val}"
+      end
+      print_lines msgs
+      return
+    end
+    clear = parts[0] == 'input'
+    order_in = []
+    map_in = { }
+    idx = 1
+    while idx < parts.length
+      name = parts[idx]
+      direct = parts[idx + 1] == '='
+      src = parts[idx + 2]
+      idx += 3
+      if map_in.has_key?(name) or not clear and @mapping.has_key?(name)
+        print_lines "Error: #{name} already in mapping."
+        return
+      end
+      order_in.push name
+      map_in[name] = [direct, src]
+    end
+    if clear
+      @order.clear
+      @mapping.clear
+    end
+    @order.concat order_in
+    @mapping.merge! map_in
+  end
+
+  def command_part
+    command = []
+    @order.each do |name|
+      direct, val = @mapping[name]
+      command.concat [ direct ? 'direct' : 'input', val, name ]
+    end
+    return command
+  end
+end
+$input_handler = InputHandler.new
+
+class OutputHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ -1, :output, :outputadd, :outputwhat, :outputnaming ],
+      :output => [ 1, 'output', :maps ],
+      :outputadd => [ 2, 'output+', :map, :maps ],
+      :outputwhat => 'output?',
+      :outputnaming => [ 'output-naming', :string, :string ],
+      :maps => [ 0, :map, :maps ],
+      :map => [ -1, [ :string, :string ], [ :string, :null ] ],
+      :help => {
+        :output => "Clears output mapping and maps name to label. Using null as label discards that name.",
+        :outputadd => "Adds to current output mapping.",
+        :outputwhat => "Prints current output mapping.",
+        :outputnaming => "Sets prefix and postfix to unmapped names."
+      }
+    }
+    @mapping = Hash.new
+    @prefix = nil
+    @postfix = nil
+    register
+  end
+
+  def handle(parts)
+    if parts[0] == 'output?'
+      msgs = []
+      msgs.push("Prefix: #{@prefix}") unless @prefix.nil?
+      msgs.push("Postfix: #{@postfix}") unless @postfix.nil?
+      @mapping.keys.sort.each do |name|
+        label = @mapping[name]
+        msgs.push "\"#{name}\" \"#{label}\""
+      end
+      print_lines msgs
+      return
+    elsif parts[0] == 'output-naming'
+      @prefix = parts[1].length ? parts[1] : nil
+      @postfix = parts[2].length ? parts[2] : nil
+      return
+    elsif parts[0] == 'output'
+      @mapping.clear
+      @prefix = nil
+      @postfix = nil
+    end
+    msgs = []
+    idx = 1
+    while idx < parts.length
+      name = parts[idx]
+      src = parts[idx + 1]
+      idx += 2
+      msgs.push("#{name} : #{@mapping[name]} -> #{src}") if @mapping.has_key? name
+      @mapping[name] = src
+    end
+    print_lines msgs
+  end
+
+  def command_part
+    command = []
+    command.concat([ 'output-prefix', @prefix ]) unless @prefix.nil?
+    command.concat([ 'output-postfix', @postfix ]) unless @postfix.nil?
+    @mapping.each_pair { |name, val| command.concat [ 'output', name, val ] }
+    return command
+  end
+end
+$output_handler = OutputHandler.new
+
+class EnvHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ -1, :env, :envadd, :envwhat, :envclear ],
+      :env => [ 1, 'env', :varvalues ],
+      :envadd => [ 2, 'env+', :varval, :varvalues ],
+      :envwhat => 'env?',
+      :envclear => 'env-clear',
+      :varvalues => [ 0, :varval, :varvalues ],
+      :varval => [ :varname, :string ],
+      :help => {
+        :env => "Clears environment variable mapping and sets var to value.",
+        :envadd => "Sets var to value in environment variable mapping.",
+        :envwhat => "Prints out current mapping.",
+        :envclear => "Ignores environment variables outside current mapping."
+      }
+    }
+    @mapping = { }
+    @clear = false
+    register
+  end
+
+  def handle(parts)
+    if parts[0] == 'env?'
+      msgs = []
+      msgs.push("Restrict environment to given values.") if @clear == true
+      @mapping.keys.sort.each do |name|
+        value = @mapping[name]
+        msgs.push "\"#{name}\" \"#{value}\""
+      end
+      print_lines msgs
+      return
+    elsif parts[0] == 'env-clear'
+      @clear = true
+      return
+    elsif parts[0] == 'env'
+      @mapping.clear
+      @clear = false
+    end
+    msgs = []
+    idx = 1
+    while idx < parts.length
+      name = parts[idx]
+      value = parts[idx + 1]
+      idx += 2
+      msgs.push("#{name} : #{@mapping[name]} -> #{value}") if @mapping.has_key? name
+      @mapping[name] = value
+    end
+    print_lines msgs
+  end
+
+  def command_part
+    command = []
+    command.push('env-clear') if @clear
+    @mapping.each_pair { |name, val| command.concat [ 'env', name, val ] }
+    return command
+  end
+end
+$env_handler = EnvHandler.new
+
+class ArgHandler < Handler
+  attr_reader :command_part
+
+  def initialize
+    @syntax = {
+      :root => [ -1, :arg, :argadd, :argwhat ],
+      :arg => [ 1, 'arg', :values ],
+      :argadd => [ 2, 'arg+', :string, :values ],
+      :argwhat => 'arg?',
+      :values => [ 0, :string, :values ],
+      :help => {
+        :arg => "Clear argument list first and add to argument list.",
+        :argadd => "Add to argument list.",
+        :argwhat => "Print current program argument list."
+      }
+    }
+    @command_part = []
+    register
+  end
+
+  def handle(parts)
+    if parts[0] == 'arg?'
+      msgs = []
+      @command_part.each { |arg| msgs.push "\"#{arg}\"" }
+      print_lines msgs
+      return
+    end
+    @command_part.clear if parts[0] == 'arg'
+    @command_part.concat parts[1...parts.length]
+  end
+end
+$arg_handler = ArgHandler.new
+
+class ChannelHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ -1, :channelin, :channelout, :channelwhat ],
+      :channelin => [ 'channel', 'in', :informat ],
+      :channelout => [ 'channel', :outchannel, :outformat ],
+      :channelwhat => 'channel?',
+      :informat => [ -1, 'json', 'none' ],
+      :outchannel => [ -1, 'out', 'err' ],
+      :outformat => [ -1, 'json', 'bytes', 'none' ],
+      :help => {
+        :channelin => "Set program stdin format to JSON for data etc. input or closed if none.",
+        :channelout => "Set program stdout/stderr format. Bytes is passed on as JSON array and none means to ignore any output.",
+        :channelwhat => "Print current channel setting."
+      }
+    }
+    @channels = { 'in' => 'json', 'out' => 'json', 'err' => 'bytes' }
+    register
+  end
+
+  def handle(parts)
+    if parts[0] == 'channel?'
+      msgs = []
+      @channels.each_pair { |c, fmt| msgs.push "#{c} #{fmt}" }
+      print_lines msgs.sort
+      return
+    end
+    @channels[parts[1]] = parts[2]
+  end
+
+  def command_part
+    command = []
+    @channels.each_pair do |c, f|
+      next if f == 'none'
+      command.push((c == 'in') ? 'in' : 'out')
+      command.push((f == 'json') ? 'JSON' : f)
+      command.push "std#{c}"
+    end
+    return command
+  end
+end
+$channel_handler = ChannelHandler.new
+
+$actions_run_common = %q(
+---
+error:
+- args_missing: [ "@", run, error, missing, "*" ]
+- command_error: [ "@", run, error, identifier, in-use ]
+- syntax: [ "@", run, error, "?", argument, unknown ]
+- syntax: [ "@", run, error, "?", duplicate, "?" ]
+- command_error: [ "@", run, error, change-directory, "?", "?" ]
+- command_error: [ "@", run, error, env, argument, duplicate, "?" ]
+- command_error: [ "@", run, error, env, argument, invalid, "?" ]
+- command_error: [ "@", run, error, in, missing ]
+- command_error: [ "@", run, error, in, multiple ]
+- command_error: [ "@", run, error, notify, no-input ]
+- command_error: [ "@", run, error, out, duplicate ]
+- command_error: [ "@", run, error, out, missing ]
+- command_error: [ "@", run, error, output, duplicate, "?" ]
+- command_error: [ "@", run, error, program, "*" ]
+- run_internal: [ "@", run, error, exception ]
+- run_internal: [ "@", run, error, no-memory ]
+- run_internal: [ "@", run, error, no-processes ]
+- run_internal: [ "@", run, error, no-thread ]
+- run_internal: [ "@", run, error, pipe ]
+- syntax: [ "@", "?", error, argument, invalid ]
+- syntax: [ "@", "?", error, argument, not-integer ]
+- syntax: [ "@", "?", missing, "*" ]
+note:
+- run_error_input_failed: [ "@", run, error, input, failed ]
+- run_child_error_output_format: [ "@", run, error, format ]
+- run_child_error_output_format: [ "@", error, format ]
+- run_terminated: [ "@", run, terminated, "?" ]
+- run_exit: [ "@", run, exit, "?" ]
+- run_signal: [ "@", run, signal, "?" ]
+- run_stop: [ "@", run, stopped, "?" ]
+- run_continue: [ "@", run, continued ]
+bytes:
+- bytes: [ "@", run, bytes, "?", "*" ]
+)
+
+$actions_bgrun_only = %q(
+---
+return:
+- run_running: [ "@", run, running, "?" ]
+note:
+- run_closed: [ "@", run, input, closed ]
+)
+
+class RunHandler < Handler
+  @@actions_run_common = YAML.load($actions_run_common)
+  @@actions_bgrun_only = YAML.load($actions_bgrun_only)
+
+  def initialize
+    @syntax = {
+      :root => [ 3, 'run', :identifier, :executable, '&' ],
+      :identifier => [ -1, :null, :int, :string ],
+      :help => { :root => "Runs executable as identifier. Uses current input, output, channel, env, and arg settings." }
+    }
+    run_common = Proc.new do |category, action, message, vars|
+      m = nil
+      case category.to_s
+      when 'error' then m = "ERROR: #{action} : #{message.join(' ')}"
+      when 'note'
+        case action.to_s
+        when 'run_error_input_failed'
+          m = "Input failed: #{message.first.to_s}"
+        when 'run_child_error_output_format'
+          m = "Output format error: #{message.first}"
+        end
+      when 'bytes' then m = ''.concat(*vars)
+      end
+      print_lines m
+    end
+    run_only = Proc.new do |category, action, message, vars|
+      m = nil
+      case category.to_s
+      when 'note'
+        case action.to_s
+        when 'run_terminated' then m = "Terminated."
+        when 'run_exit' then m = "Exit: #{vars.first}" if 0 < vars.first
+        when 'run_signal' then m = "Signal: #{vars.first}"
+        when 'run_stop' then m = "Stopped by signal: #{vars.first}"
+        when 'run_continue' then m = "Continued."
+        end
+      end
+      print_lines m
+    end
+    @actions = PatternAction.new([ @@actions_run_common ],
+      [ run_only, run_common ])
+    bgrun_only = Proc.new do |category, action, message, vars|
+      m = nil
+      case category.to_s
+      when 'return'
+        case action.to_s
+        when 'run_running' then m = "PID: #{vars.first}"
+        end
+      when 'note'
+        case action.to_s
+        when 'run_terminated' then m = "Terminated: #{message.first}"
+        when 'run_exit'
+          m = "Exit #{message.first}: #{vars.first}" if 0 < vars.first
+        when 'run_signal' then m = "Signal #{message.first}: #{vars.first}"
+        when 'run_stop' then m = "Stopped #{message.first}: #{vars.first}"
+        when 'run_continue' then m = "Continued: #{message.first}"
+        end
+      end
+      print_lines m
+    end
+    @bgactions = PatternAction.new(
+      [ @@actions_bgrun_only, @@actions_run_common ],
+      [ bgrun_only, run_common ])
+    register
+  end
+
+  def handle(parts)
+    command = [ parts[1], 'run' ]
+    command.concat $notify_handler.command_part
+    command.concat $input_handler.command_part
+    command.concat $output_handler.command_part
+    command.concat $channel_handler.command_part
+    command.concat $env_handler.command_part
+    command.concat [ 'program', parts[2] ]
+    command.concat $arg_handler.command_part
+    send(parts.last == '&' ? @bgactions : @actions, command, true)
+  end
+end
+RunHandler.new
+
+class PsHandler < Handler
+  def initialize
+    @syntax = {
+      :root => 'ps',
+      :help => { :root => "List running proceses." }
+    }
+    register
+  end
+
+  def handle(parts)
+    report = [ " PID\t: Identifier" ]
+    procs = $lackey.process
+    procs.keys.sort.each { |id| report.push " #{procs[id]}\t: #{id}" }
+    print_lines report
+  end
+end
+PsHandler.new
+
+class KillHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ 2, 'kill', :process_id, :identifiers ],
+      :identifiers => [ 0, :process_id, :identifiers ],
+      :help => { :root => "Terminates processes." }
+    }
+    message_proc = Proc.new do |category, action, message, vars|
+      case category
+      when :note then print_lines "Missing: #{vars.join(' ')}"
+      else nil
+      end
+    end
+    @actions = PatternAction.new([{
+      :note => [ { :missing => [ '@', 'terminate', 'missing', '*' ] } ]
+    }], [ message_proc ])
+    register
+  end
+
+  def handle(parts)
+    send(@actions, [ 'terminate' ].concat(parts[1...parts.length]))
+  end
+end
+KillHandler.new
+
+$actions_feed = %q(
+---
+error:
+- args_missing: [ "@", feed, error, missing, "*" ]
+- argument_unknown: [ "@", feed, error, "?", argument, unknown ]
+- duplicate: [ "@", feed, error, "?", duplicate, "?" ]
+- feed_closed: [ "@", feed, error, closed ]
+- feed_process: [ "@", feed, error, not-found ]
+)
+
+class FeedHandler < Handler
+  @@actions_feed = YAML.load($actions_feed)
+
+  def initialize
+    @syntax = {
+      :root => [ 2, 'feed', :process_id, :identifiers ],
+      :identifiers => [ 0, :process_id, :identifiers ],
+      :help => { :root => "Pass current input set to processes." }
+    }
+    message_proc = Proc.new do |category, action, message, vars|
+      m = nil
+      case category.to_s
+      when 'error'
+        case action.to_s
+        when 'args_missing' then m = "Missing: #{vars.join(' ')}"
+        when 'argument_unknown' then m = "Unknown: #{vars.join(' ')}"
+        when 'duplicate' then m = "Duplicate: #{vars.join(' ')}"
+        when 'feed_closed' then m = "Closed."
+        when 'feed_process' then m = "No process."
+        end
+      end
+      print_lines m
+    end
+    @actions = PatternAction.new([ @@actions_feed ], [ message_proc ])
+    register
+  end
+
+  def handle(parts)
+    c = $handler['input'].command_part
+    parts[1...parts.length].each do |procid|
+      command = [ 'feed', procid ]
+      command.concat(c) unless c.empty? # Empty feed sends empty JSON object.
+      send(@actions, command)
+    end
+  end
+end
+FeedHandler.new
+
+class EndFeedHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ 2, 'close', :process_id, :identifiers ],
+      :identifiers => [ 0, :process_id, :identifiers ],
+      :help => { :root => "Closes process input." }
+    }
+    notify = Proc.new do |category, action, message, vars|
+      m = nil
+      case category
+      when :note
+        case action
+        when :not_open then m = "Closed already: #{vars.join(' ')}"
+        when :missing then m = "Missing: #{vars.join(' ')}"
+        end
+      end
+      print_lines m
+    end
+    @actions = PatternAction.new([{
+      :note => [ { :not_open => [ '@', 'end-feed', 'not-open', '*' ],
+        :missing => [ '@', 'end-feed', 'missing', '*' ] } ]
+    }], [ notify ])
+    register
+  end
+
+  def handle(parts)
+    send(@actions, [ 'end-feed' ].concat(parts[1...parts.length]))
+  end
+end
+EndFeedHandler.new
+
+class RawHandler < Handler
+  def initialize
+    @syntax = {
+      :root => [ 'raw', :string ],
+      :help => {
+        :root => "Pass unchecked JSON input as single string to datalackey." }
+    }
+    register
+  end
+
+  def handle(parts)
+    dump parts[1]
+  end
+end
+RawHandler.new
+
+
+def completer(str)
+  # See if we can get the command that is being typed.
+  begin
+    full = Readline.line_buffer.clone
+  rescue NotImplementedError
+    # No context available so just pick matching words.
+    cands = $handler.keys.grep(/^#{Regexp.escape(str)}/)
+    cands.concat $lackey.data.keys.grep(/^#{Regexp.escape(str)}/)
+    cands.concat $lackey.process.keys.grep(/^#{Regexp.escape(str)}/)
+    # Should add various words from syntax. Basically scan for strings.
+    cands.sort!
+    return cands
+  end
+  begin
+    parts = Shellwords.shellsplit(full)
+  rescue ArgumentError
+    begin
+      parts = Shellwords.shellsplit(full + '"')
+    rescue ArgumentError
+      parts = Shellwords.shellsplit(full + "'")
+    end
+  end
+  if parts.length < 2 and not full.end_with? ' '
+    return $handler.keys.sort.grep(/^#{Regexp.escape(str)}/)
+  end
+  h = $handler[parts[0]]
+  return (h.nil? ? [] : h.completion_candidates(parts, str))
+end
+Readline.completion_proc = Proc.new { |str| completer(str) }
+
+
+# Load history.
+history_dir = arguments.directory.nil? ? Dir.pwd : arguments.directory
+history_file = File.join(history_dir, history_file_basename)
+if File.exist? history_file
+  begin
+    fp = File.new history_file
+    fp.readlines.each { |line| Readline::HISTORY << line.chomp  }
+  rescue StandardError
+    STDERR.puts "Failed to read history file: #{history_file}"
+  end
+end
+
+while not $exit_handler.exiting
+  $output_mutex.synchronize { } # So we do not print prompt during other output.
+  $in_prompt = true
+  line = Readline.readline($prompt, false)
+  $in_prompt = false
+  break if line.nil?
+  line.strip!
+  begin
+    parts = line.shellsplit
+  rescue ArgumentError => e
+    print_lines e.to_s
+    next
+  end
+  next if parts.length == 0
+  h = $handler[parts[0]]
+  if h.nil?
+    print_lines "Unknown command: #{parts[0]}"
+    next
+  end
+  if h.verify(parts)
+    if $stdin.tty? and (Readline::HISTORY.empty? or Readline::HISTORY[Readline::HISTORY.length - 1] != line)
+      Readline::HISTORY.push line
+    end
+    h.handle(parts)
+  else
+    print_lines "Command verify failed."
+  end
+end
+
+# For recall to work, ensure there is exit in case ctrl-D was pressed.
+if Readline::HISTORY.empty? or not Readline::HISTORY[Readline::HISTORY.length - 1].start_with? $exit_handler.command_name
+  Readline::HISTORY.push $exit_handler.command_name
+end
+
+# Save history.
+if $stdin.tty?
+  begin
+    fp = Pathname.new history_file
+    fp.open('w') do |f|
+      Readline::HISTORY.to_a.last(history_file_max_lines).each { |l| f.puts l }
+    end
+  rescue StandardError
+    STDERR.puts "Failed to write history file: #{history_file}"
+  end
+end
+
+$lackey_proc.finish
+$lackey.close
+stderr_discarder.close
+$lackey.finish
+puts() unless $exit_handler.exiting
+if $lackey_proc.exit_code != 0 and not $lackey_proc.exit_code.nil?
+  puts("datalackey exit: #{$lackey_proc.exit_code}")
+end
+exit $exit_handler.code