datalackeytools 0.3.4

data/bin/datalackey-state

@@ -0,0 +1,1005 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Copyright © 2019-2021 Ismo Kärkkäinen
+# Licensed under Universal Permissive License. See LICENSE.txt.
+
+require 'optparse'
+require 'yaml'
+require 'tempfile'
+require_relative '../lib/datalackeylib'
+
+$CMDOUT = $stdout
+$USEROUT = $stderr
+$OVERWRITE_ACTION = nil
+$FOLLOW = 0
+$QUIET = false
+# For the case when we run datalackey.
+$LACKEY = nil
+$MEMORY = nil
+$DIRECTORY = nil
+$PERMISSIONS = nil
+$ECHO = false
+
+$root_dir = Dir.pwd
+
+parser = OptionParser.new do |opts|
+  opts.summary_indent = '  '
+  opts.summary_width = 26
+  opts.banner = 'Usage: datalackey-state [options] state-files...'
+  opts.separator ''
+  opts.separator 'Options:'
+  opts.on('-o', '--stdout', 'Use stdout as command output, stderr for user.') do
+    $CMDOUT = $stdout
+    $USEROUT = $stderr
+  end
+  opts.on('-e', '--stderr', 'Use stderr as command output, stdout for user.') do
+    $CMDOUT = $stderr
+    $USEROUT = $stdout
+  end
+  opts.on('-q', '--quiet', 'Suppress normal command output.') do
+    $QUIET = true
+  end
+  opts.on('-h', '--help', 'Print this help and exit.') do
+    $USEROUT.puts opts
+    exit 0
+  end
+  opts.separator 'Options for error checking and tracking execution:'
+  opts.on('--warn', 'Print warning when state is overwritten.') do
+    $OVERWRITE_ACTION = :warn
+  end
+  opts.on('--error', 'State overwriting is an error.') do
+    $OVERWRITE_ACTION = :error
+  end
+  opts.on('-f', '--follow [LEVEL]', Integer, 'Print state/signal/command.') do |level|
+    $FOLLOW = level
+  end
+end
+
+DatalackeyProcess.options_for_OptionParser(parser, true,
+  proc { |arg| $LACKEY = arg },
+  proc { |arg| $MEMORY = arg },
+  proc { |arg| $DIRECTORY = arg },
+  proc { |arg| $PERMISSIONS = arg },
+  proc { |arg| $ECHO = arg })
+parser.parse!
+
+if ARGV.empty?
+  $USEROUT.puts "No state files."
+  exit 1
+end
+
+$userout_mutex = Mutex.new
+def userout(message)
+  return false if message.is_a?(Array) && message.empty?
+  message = [ message ] unless message.is_a? Array
+  $userout_mutex.synchronize do
+    message.each { |m| $USEROUT.puts m }
+  end
+  true
+end
+
+class Machine
+  attr_accessor :variables
+  attr_reader :had_error, :state, :state_name, :machine
+
+  def initialize
+    @commands = Hash.new(nil)
+    @variables = Hash.new(nil)
+    @skip_expand = { }
+    @signal_mutex = Mutex.new
+    @signal = nil
+    @forced_state = nil
+    @state_mutex = Mutex.new
+    @label2signal = { }
+    @state = nil
+    @state_name = nil
+    @machine = nil
+  end
+
+  def register(name, object, expand_skip_count = 0)
+    raise ArgumentError, 'Command name is not a string.' unless name.is_a? String
+    @commands[name] = object
+    @skip_expand[name] = expand_skip_count if expand_skip_count > 0
+  end
+
+  def get_binding
+    binding
+  end
+
+  def expand(item, info, seen = [])
+    if item.is_a? Array
+      info.push '['
+      result = []
+      item.each { |v| result.push expand(v, info, seen) }
+      info.push ']'
+      return result
+    end
+    if item.is_a? Hash
+      info.push '{'
+      result = { }
+      item.each_pair { |k, v| result[k] = expand(v, info, seen) }
+      info.push '}'
+      return result
+    end
+    info.concat([ ',', item ]) unless info.last == item
+    if item.is_a?(String) && item.start_with?('$') && ENV.key?(item[1...item.size])
+      item = ENV[item[1...item.size]]
+      info.concat [ '=>', item ]
+      return expand(item, info, seen)
+    end
+    if @variables.key? item
+      s = seen.clone
+      s.push item
+      raise "Loop: #{item} via #{s.join(' => ')}." if seen.include? item
+      item = @variables[item]
+      info.concat [ '=>', item ]
+      return expand(item, info, s)
+    end
+    item
+  end
+
+  def set_signal(sig, target_state = nil)
+    @signal_mutex.synchronize do
+      if @signal.nil? || sig == 'error'
+        userout "Signal: #{sig}" if $FOLLOW > 0
+        @signal = sig
+        @had_error = @signal == 'error'
+        @forced_state = target_state
+        return true
+      end
+    end
+    false
+  end
+
+  def set_error(message)
+    userout message
+    set_signal('error')
+  end
+
+  def get_signal
+    @signal_mutex.synchronize { @signal }
+  end
+
+  def label2signal
+    @state_mutex.synchronize { @label2signal }
+  end
+
+  def state_source
+    @state.nil? ? nil : @state.fetch(:load, { }).fetch(:source, nil)
+  end
+
+  def run_state(state, name)
+    userout "State: #{name}" if $FOLLOW > 0
+    state[:commands].each do |cmd|
+      info = []
+      begin
+        s_cmd = cmd.to_s
+        userout "Command: #{s_cmd}" if $FOLLOW > 2
+        if cmd.is_a? Hash
+          expanded = expand(cmd, info)
+        else
+          skip = @skip_expand.fetch(cmd.first, 0)
+          skip = cmd.size if cmd.size < skip
+          expanded = cmd[0...skip]
+          rest = cmd[skip...cmd.size]
+          rest = [ rest ] unless rest.is_a? Array
+          expanded.concat expand(rest, info)
+        end
+        s_exp = expanded.to_s
+        userout "Expanded: #{s_exp}" if $FOLLOW > 3 && s_cmd != s_exp
+      rescue RuntimeError => e
+        set_error e.to_s
+        return
+      end
+      handler = nil
+      name = nil
+      if expanded.is_a? Hash
+        @commands.each_key do |key|
+          next unless cmd.key? key # Registered names ought to be unique.
+          name = key
+          break
+        end
+        if name.nil?
+          set_error('Match for dictionary keys not found.')
+          break
+        end
+      else
+        if expanded.is_a? String
+          name = expanded
+        elsif expanded.is_a? Array
+          name = expanded.first
+        else
+          set_error("Unexpected type for command: #{expanded.class.name}")
+          break
+        end
+      end
+      handler = @commands[name]
+      set_error("Unknown command: #{name}") if handler.nil?
+      break if handler.nil? || !get_signal.nil?
+      userout "Command: #{name}" if $FOLLOW == 2
+      handler.handle(expanded)
+      break unless get_signal.nil? # Prevents following output for next command.
+    end
+  end
+
+  def run(initial_state_name, states)
+    @state_mutex.synchronize { @machine = states }
+    last_state_name = '__terminate_all'
+    finalized = false
+    name = initial_state_name
+    signal2state = { } # Only changes when state has new global mapping.
+    loop do
+      userout($lackey_stderr.getlines) unless $QUIET
+      finalized = true if name == last_state_name
+      unless states.key? name
+        userout "Transition to unknown state: #{name}"
+        break
+      end
+      state = states[name]
+      signal2state = state.fetch('global_signal2state', signal2state)
+      @state_mutex.synchronize do
+        @label2signal = state.fetch('label2signal', { })
+        @state = state # Command may need e.g. the location of the state file.
+        @state_name = name
+      end
+      run_state(state, name)
+      @state_mutex.synchronize do
+        @label2signal = { }
+        @state = nil
+        @state_name = nil
+      end
+      sig = nil
+      name = nil
+      @signal_mutex.synchronize do
+        sig = @signal
+        @signal = nil
+        name = @forced_state
+        @forced_state = nil
+      end
+      name = nil unless sig == '__jump' # Jump may have been "interrupted".
+      if name.nil?
+        name = state.fetch('signal2state', { }).fetch(sig, nil)
+        name = signal2state.fetch(sig, nil) if name.nil?
+        break if name.nil?
+      end
+    end
+    return unless !finalized && states.key?(last_state_name)
+    run_state(states[last_state_name], last_state_name)
+  end
+end
+$machine = Machine.new
+
+$signal_sender = proc do |action, message, vars|
+  case action.first
+  when :error
+    $machine.set_error(message.join(' '))
+    true
+  when :exitcode
+    $machine.set_error(message.join(' ')) if vars.first != 0
+    true
+  else false
+  end
+end
+
+class Command
+  attr_reader :action
+
+  def initialize(name, expand_skip = 0)
+    $machine.register(name, self, expand_skip)
+    @name = name
+  end
+
+  def handle(cmd)
+    cmd.flatten!
+    userout(cmd.to_s) if $FOLLOW > 4
+    $lackey.send(@action, cmd)
+  end
+
+  def assert(cmd, minlen, maxlen = nil)
+    if cmd.size < minlen
+      $machine.set_error "Command #{@name} too short: #{cmd.size} < #{minlen}"
+      return false
+    end
+    return true if maxlen.nil?
+    if maxlen < cmd.size
+      $machine.set_error "Command #{@name} too long: #{maxlen} < #{cmd.size}"
+      return false
+    end
+    true
+  end
+end
+
+class CommentCommand < Command
+  def initialize
+    super('comment', 1000000)
+  end
+
+  def handle(cmd)
+  end
+end
+CommentCommand.new
+
+class SetCommand < Command
+  def initialize
+    super('set', 2)
+  end
+
+  def handle(cmd)
+    return unless assert(cmd, 3)
+    $machine.variables[cmd[1]] = cmd.size == 3 ? cmd[2] : cmd[2...cmd.size]
+  end
+end
+SetCommand.new
+
+class DefaultCommand < Command
+  def initialize
+    super('default', 2)
+  end
+
+  def handle(cmd)
+    return unless assert(cmd, 3)
+    v = cmd[1]
+    return if $machine.variables.key? v
+    $machine.variables[v] = cmd.size == 3 ? cmd[2] : cmd[2...cmd.size]
+  end
+end
+DefaultCommand.new
+
+class UnsetCommand < Command
+  def initialize
+    super('unset', 1000000)
+  end
+
+  def handle(cmd)
+    cmd.flatten!
+    cmd.shift
+    cmd.each { |name| $machine.variables.delete name }
+  end
+end
+UnsetCommand.new
+
+class AssertVarCommand < Command
+  def initialize
+    super('assert_var', 1000000)
+  end
+
+  def handle(cmd)
+    cmd.flatten!
+    cmd.shift
+    unset = []
+    cmd.each { |name| unset.push(name) unless $machine.variables.key? name }
+    $machine.set_error("Not set: #{unset.join(' ')}") unless unset.empty?
+  end
+end
+AssertVarCommand.new
+
+class AssertNotVar < Command
+  def initialize
+    super('assert_notvar', 1000000)
+  end
+
+  def handle(cmd)
+    cmd.flatten!
+    cmd.shift
+    set = []
+    cmd.each { |name| set.push(name) if $machine.variables.key? name }
+    $machine.set_error("Set: #{set.join(' ')}") unless set.empty?
+  end
+end
+AssertNotVar.new
+
+class AssertDataCommand < Command
+  def initialize
+    super('assert_data')
+  end
+
+  def handle(cmd)
+    cmd.flatten!
+    cmd.shift
+    data = $lackey.data
+    missing = []
+    cmd.each { |name| missing.push(name) unless data.key? name }
+    $machine.set_error("Missing: #{missing.join(' ')}") unless missing.empty?
+  end
+end
+AssertDataCommand.new
+
+class PushCommand < Command
+  def initialize
+    super('push', 2)
+  end
+
+  def handle(cmd)
+    return unless assert(cmd, 3)
+    # Creates a stack if not present, or turns scalar into stack.
+    stack = $machine.variables.fetch(cmd[1], [])
+    stack = [ stack ] unless stack.is_a? Array
+    stack.push(3 < cmd.size ? cmd[2...cmd.size] : cmd[2])
+    $machine.variables[cmd[1]] = stack
+  end
+end
+PushCommand.new
+
+class PopCommand < Command
+  def initialize
+    super('pop', 3)
+  end
+
+  def handle(cmd)
+    return unless assert(cmd, 3, 3)
+    unless $machine.variables.key? cmd[1]
+      $machine.set_error("Stack not found: #{cmd[1]}")
+      return
+    end
+    # Interprets scalas as stack. Empty array treated as stack.
+    stack = $machine.variables[cmd[1]]
+    stack = [ stack ] unless stack.is_a? Array
+    return $machine.set_error("Stack empty: #{cmd[1]}") if stack.empty?
+    $machine.variables[cmd[2]] = stack.pop
+    $machine.variables.delete(cmd[1]) if stack.empty?
+  end
+end
+PopCommand.new
+
+class RunBase < Command
+  @@actions_run_common = {
+    error: {
+      syntax: [ 'run', 'error', '?', 'argument', 'unknown' ],
+      command_error: [ 'run', 'error', '*' ],
+      run_internal: [ [ 'run', 'error', 'exception' ],
+        [ 'run', 'error', 'no-memory' ],
+        [ 'run', 'error', 'no-processes' ],
+        [ 'run', 'error', 'no-thread' ],
+        [ 'run', 'error', 'pipe' ] ]
+    },
+    note: {
+      run_error_input_failed: [ 'run', 'error', 'input', 'failed' ],
+      run_child_error_output_format: [
+        [ 'run', 'error', 'format' ], [ 'error', 'format' ] ]
+    },
+    exitcode: [ 'run', 'exit', '?' ],
+    bytes: [ 'run', 'bytes', '?', '*' ]
+  }
+  @@prev_bytes_mutex = Mutex.new
+  @@prev_bytes_id = nil
+
+  def initialize(name)
+    @msgmaker = proc do |action, message, vars|
+      message_maker(action, message, vars)
+    end
+    super(name)
+  end
+
+  def message_maker(action, message, vars)
+    out = []
+    case action.first
+    when :note
+      # Should signal error, presumably.
+      case action.last
+      when :run_error_input_failed
+        out.push "Output from #{message[0]} failed."
+      when :run_child_error_output_format
+        out.push "Output to #{message[0]} failed."
+      end
+    when :bytes
+      id = message[0]
+      @@prev_bytes_mutex.synchronize do
+        if id != @@prev_bytes_id
+          out.push "#{id}:"
+          @prev_bytes_id = id
+        end
+      end
+      out.push ''.concat(*vars)
+    end
+    userout(out) unless $QUIET
+    !out.empty?
+  end
+
+  def handle(cmd)
+    return unless assert(cmd, 4)
+    t = cmd[0] # Swap command and user-given identifier.
+    cmd[0] = cmd[1]
+    cmd[1] = t
+    # Locate executable unless absolute path.
+    idx = cmd.index 'program'
+    unless idx.nil? || idx + 1 == cmd.size
+      cmd[idx + 1] = DatalackeyProcess.locate_executable(cmd[idx + 1],
+        [ Dir.pwd, $root_dir, File.dirname($machine.state_source) ])
+    end # Missing program will be found normally and causes error.
+    userout(cmd.to_s) if $FOLLOW > 4
+    $lackey.send(@action, cmd, true)
+  end
+end
+
+class RunCommand < RunBase
+  def initialize
+    super('run')
+    @action = PatternAction.new([
+      { return: [ 'run', 'running', '?' ],
+        error: [ 'run', 'terminated', '?' ] },
+      @@actions_run_common ], [ @msgmaker, $signal_sender ])
+  end
+
+  def handle(cmd)
+    cmd.flatten!
+    super(cmd)
+  end
+end
+RunCommand.new
+
+class ProcessCommand < Command
+  def initialize(name)
+    @action = nil
+    super(name)
+  end
+
+  def handle(cmd)
+    cmd.flatten!
+    cmd.concat $lackey.launched.keys if cmd.size == 1
+    return if cmd.size == 1
+    super(cmd)
+  end
+end
+
+class CloseCommand < ProcessCommand
+  def initialize
+    super('close')
+  end
+
+  def handle(cmd)
+    cmd.flatten!
+    cmd[0] = 'end-feed'
+    super(cmd)
+  end
+end
+CloseCommand.new
+
+class TerminateCommand < ProcessCommand
+  def initialize
+    super('terminate')
+  end
+end
+TerminateCommand.new
+
+class FeedCommand < Command
+  def initialize
+    super('feed')
+    @action = PatternAction.new(
+      [ { error: [ 'error', '*' ] } ], [ $signal_sender ])
+  end
+end
+FeedCommand.new
+
+class WaitProcessCommand < Command
+  def initialize
+    super('wait_process')
+  end
+
+  def handle(cmd)
+    cmd.flatten!
+    return unless assert(cmd, 2)
+    begin
+      deadline = Time.new + Float(cmd[1])
+    rescue ArgumentError
+      $machine.set_error "Not a number: #{cmd[1]}"
+      return
+    end
+    waited = 2 < cmd.size ? $lackey.launched.keys : cmd[2...cmd.size]
+    while (Time.new <=> deadline) == -1
+      sleep(0.2)
+      remains = false
+      current = $lackey.process.keys
+      waited.each { |id| remains ||= current.include? id }
+      return unless remains
+    end
+  end
+end
+WaitProcessCommand.new
+
+class WaitDataCommand < Command
+  def initialize
+    super('wait_data')
+  end
+
+  def handle(cmd)
+    cmd.flatten!
+    unseen = cmd[1...cmd.size]
+    loop do
+      data = $lackey.data
+      unseen.delete_if { |u| data.key? u }
+      break if unseen.empty?
+      sleep(0.2)
+    end
+  end
+end
+WaitDataCommand.new
+
+class WaitSignalCommand < Command
+  def initialize
+    super('wait_signal')
+  end
+
+  def handle(cmd)
+    while $machine.get_signal.nil?
+      sleep(0.2)
+    end
+  end
+end
+WaitSignalCommand.new
+
+class SignalCommand < Command
+  def initialize
+    super('signal')
+  end
+
+  def handle(cmd)
+    cmd.flatten!
+    return unless assert(cmd, 2, 2)
+    $machine.set_signal(cmd[1])
+  end
+end
+SignalCommand.new
+
+class JumpCommand < Command
+  def initialize
+    super('jump')
+  end
+
+  def handle(cmd)
+    cmd.flatten!
+    return unless assert(cmd, 2, 2)
+    $machine.set_signal('__jump', cmd[1])
+  end
+end
+JumpCommand.new
+
+class DeleteCommand < Command
+  def initialize
+    super('delete')
+    @action = nil
+  end
+end
+DeleteCommand.new
+
+class RenameCommand < Command
+  def initialize
+    super('rename')
+    @action = nil
+  end
+end
+RenameCommand.new
+
+class PrintCommand < Command
+  def initialize
+    super('print')
+  end
+
+  def handle(cmd)
+    return if $QUIET
+    # Array containing arrays will be treated as multi-line output.
+    cmd.shift
+    multi = false
+    cmd.each_index do |k|
+      next unless cmd[k].is_a? Array
+      multi = true
+      cmd[k] = cmd[k].flatten.join(' ')
+    end
+    userout cmd.join(multi ? "\n" : ' ')
+  end
+end
+PrintCommand.new
+
+class ValueCommand < Command
+  def handle_with_map(cmd, map)
+    return if $QUIET
+    cmd.flatten!
+    cmd.shift
+    out = []
+    cmd.each { |id| out.push "#{id} : #{map.fetch(id, 'NOT FOUND')}" }
+    userout out
+  end
+end
+
+class SerialCommand < ValueCommand
+  def initialize
+    super('serial')
+  end
+
+  def handle(cmd)
+    handle_with_map(cmd, $lackey.data)
+  end
+end
+SerialCommand.new
+
+class PIDCommand < ValueCommand
+  def initialize
+    super('pid')
+  end
+
+  def handle(cmd)
+    handle_with_map(cmd, $lackey.process)
+  end
+end
+PIDCommand.new
+
+class ShellCommand < Command
+  def initialize
+    @rule = { 'argv' => '*', 'script' => '*', 'stdout' => '?', 'stderr' => '?', '*' => '*' }
+    super('script')
+  end
+
+  def handle(cmd)
+    # Get interpreter and check that it exists.
+    cmd.delete 'comment'
+    argv = cmd.fetch('argv', [])
+    cmd.delete 'argv'
+    script = cmd['script'] # Pre-requisite for getting this far, exists.
+    cmd.delete 'script'
+    vout = cmd.fetch('stdout', nil)
+    cmd.delete 'stdout'
+    verr = cmd.fetch('stderr', nil)
+    cmd.delete 'stderr'
+    vin = cmd.fetch('stdin', nil)
+    cmd.delete 'stdin'
+    if cmd.size != 1
+      $machine.set_error "Multiple interpreters: #{cmd.keys.sort.join(' ')}"
+      return
+    end
+    interpreter = cmd.keys.first
+    hashbangargs = cmd[cmd.keys.first]
+    hashbangargs = hashbangargs.join(' ') if hashbangargs.is_a? Array
+    unless File.exist?(interpreter) && File.executable?(interpreter)
+      $machine.set_error "Not found or not an executable: #{interpreter}"
+      return
+    end
+    # Write script into a temporary file.
+    temp = Tempfile.new('state', Dir.tmpdir)
+    begin
+      temp.puts "#!#{interpreter} #{hashbangargs}"
+      temp.puts script
+      temp.chmod(0o700)
+      temp.close
+      argv.prepend temp.path
+    rescue StandardError => e
+      $machine.set_error("Shell script prepare exception:\n#{e}")
+      temp.close!
+      return
+    end
+    wait = nil
+    stdin = nil
+    rd_out = nil
+    rd_err = nil
+    begin
+      # Run script with arguments.
+      stdin, stdout, stderr, wait = Open3.popen3(*argv)
+      rd_out = vout.nil? ? DiscardReader.new(stdout) : StoringReader.new(stdout)
+      rd_err = verr.nil? ? DiscardReader.new(stderr) : StoringReader.new(stderr)
+      stdin.puts(vin) unless vin.nil?
+      stdin.close
+      wait.join
+      $machine.variables[vout] = rd_out.getlines.join("\n") unless vout.nil?
+      $machine.variables[verr] = rd_err.getlines.join("\n") unless verr.nil?
+      exitcode = wait.value.exitstatus
+      wait = nil
+      $machine.set_error("Shell exit: #{exitcode}") if exitcode != 0
+    rescue StandardError => e
+      $machine.set_error("Shell run exception:\n#{e}")
+    ensure
+      stdin.close unless stdin.nil?
+      rd_out.close unless rd_out.nil?
+      rd_err.close unless rd_err.nil?
+      wait.join unless wait.nil?
+      temp.close!
+    end
+  end
+end
+ShellCommand.new
+
+class RubyCommand < Command
+  def initialize
+    super('ruby')
+  end
+
+  def handle(cmd)
+    # Access to variables via @variables in binding.
+    script = cmd['ruby']
+    script = script.join("\n") if script.is_a? Array
+    begin
+      eval(script, $machine.get_binding)
+    rescue StandardError => e
+      $machine.set_error "Eval failed: #{e}"
+    end
+  end
+end
+RubyCommand.new
+
+# Load and parse all state files here.
+
+$overwrites = []
+
+def add_states(states, new_states)
+  new_states.each_pair do |name, state|
+    $overwrites.push({ state: name }.merge(state[:load])) if states.key? name
+    states[name] = state
+  end
+end
+
+$built_ins = { '__setup' => %q(
+---
+- __terminate_all:
+  - close
+  - wait_process terminate_delay
+  - terminate
+  comment: "Default termination, close feeds, terminate all after 5 seconds."
+- __setup:
+  - set pipes in JSON stdin out JSON stdout out bytes stderr
+  - set pipes_noerr in JSON stdin out JSON stdout
+  - set notify_all notify data notify process
+  - [ set, terminate_delay, 5 ]
+  global_signal2state:
+    error: __terminate_all
+  comment: |
+    First state to run. Default variable set-up. Program will add first state
+    in the first item of the first file with null key unless you set it when
+    replacing this. Several states in the first item results in random pick.
+) }
+
+$first_state = nil
+
+def load_statefile(filename, current_directory, parent_includes = [])
+  if $built_ins.key? filename
+    fullname = filename
+    contents = YAML.safe_load($built_ins[filename])
+    directory = '(built-in)' # Built-in can include only another built-in.
+  else
+    begin
+      fullname = File.realpath(filename, current_directory)
+      contents = YAML.safe_load(File.read(fullname))
+      directory = File.dirname(fullname)
+    rescue Errno::ENOENT
+      userout "Could not find #{filename} from #{current_directory}"
+      return nil
+    rescue StandardError => e
+      userout e.to_s
+      userout "Failed to read #{filename} from #{current_directory}"
+      return nil
+    end
+  end
+  if parent_includes.include? fullname
+    userout "Include loop: #{filename} from #{current_directory}"
+    return nil
+  end
+  unless contents.is_a? Array
+    userout "Not a top level array: #{filename} from #{current_directory}"
+    return nil
+  end
+  states = { }
+  contents.each_index do |index|
+    item = contents[index]
+    common = { load: { source: fullname, index: index } }
+    begin
+      item.delete 'comment'
+      [ 'global_signal2state', 'signal2state', 'label2signal' ].each do |shared|
+        next unless item.key? shared
+        common[shared] = item[shared]
+        item.delete shared
+      end
+    rescue NoMethodError
+      userout "Item is not a mapping: #{fullname} : #{index}"
+      return nil
+    end
+    include_name = item.fetch('include', nil)
+    if include_name.nil?
+      # Separate each state while keeping common parts.
+      item.each_pair do |state, commands|
+        if $first_state.nil?
+          $first_state = state
+          if item.size != 1
+            userout "First state mapping with multiple states is ambiguous: #{fullname} : #{index}"
+            return nil
+          end
+        end
+        commands.each_index do |k|
+          commands[k] = commands[k].split(' ') if commands[k].is_a? String
+        end
+        separated = common.clone
+        separated[:commands] = commands
+        add_states(states, { state => separated })
+      end
+    else
+      parent_includes.push fullname
+      included = load_statefile(include_name, directory, parent_includes)
+      parent_includes.pop
+      if included.nil?
+        userout " .. #{fullname} : #{index}"
+        return nil
+      end
+      add_states(states, included)
+    end
+  end
+  states
+end
+
+states = load_statefile('__setup', $root_dir)
+$first_state = nil
+ARGV.each do |filename|
+  new_states = load_statefile(filename, $root_dir)
+  exit(2) if new_states.nil?
+  add_states(states, new_states)
+end
+
+if !$OVERWRITE_ACTION.nil? && !$overwrites.empty?
+  $overwrites.each do |ow|
+    userout "#{ow[:state]} replaced by #{ow[:source]} : #{ow[:index]}"
+  end
+  exit(4) if $OVERWRITE_ACTION == :error
+end
+
+unless states['__setup'].key? 'signal2state'
+  states['__setup']['signal2state'] = { }
+end
+unless states['__setup']['signal2state'].key? nil
+  states['__setup']['signal2state'][nil] = $first_state
+end
+
+def label_signal_check(action, message, vars)
+  case action.first
+  when :stored
+    l2s = $machine.label2signal
+    $machine.set_signal(l2s[vars.first]) if l2s.key? vars.first
+  when :data_error then $machine.set_error(message.to_s)
+  when :error then $machine.set_error(message.to_s)
+  end
+end
+sig_check = proc { |act, msg, vs| label_signal_check(act, msg, vs) }
+
+if $LACKEY.nil? && $MEMORY.nil? && $DIRECTORY.nil?
+  # Seemingly running under datalackey.
+  unless $PERMISSIONS.nil?
+    userout 'Cannot give --permissions/-p unless running datalackey.'
+    exit 1
+  end
+  if $stdin.tty?
+    userout 'Not running under datalackey, turning on memory storage.'
+    $MEMORY = true
+  end
+end
+
+if $LACKEY.nil? && $MEMORY.nil? && $DIRECTORY.nil?
+  $lackey_proc = DatalackeyParentProcess.new($CMDOUT, $stdin)
+  $lackey_stderr = DiscardReader.new($lackey_proc.stderr)
+  $lackey = DatalackeyIO.new(
+    $lackey_proc.stdin, $lackey_proc.stdout, sig_check)
+else
+  begin
+    $DIRECTORY, $PERMISSIONS, $MEMORY =
+      DatalackeyProcess.verify_directory_permissions_memory(
+        $DIRECTORY, $PERMISSIONS, $MEMORY)
+    $lackey_proc = DatalackeyProcess.new(
+      $LACKEY, $DIRECTORY, $PERMISSIONS, $MEMORY)
+  rescue ArgumentError => e
+    userout e.to_s
+    exit 1
+  end
+  $lackey_stderr = StoringReader.new($lackey_proc.stderr)
+  echo = $ECHO ? proc { |json| userout json } : nil
+  $lackey = DatalackeyIO.new(
+    $lackey_proc.stdin, $lackey_proc.stdout, sig_check, echo, echo)
+end
+
+$machine.run('__setup', states)
+
+$lackey_proc.finish
+$lackey.close
+if $LACKEY.nil? && $MEMORY.nil? && $DIRECTORY.nil?
+  $lackey_proc.stdout.close
+  $stderr.close
+end
+$lackey.finish
+$lackey_stderr.close
+userout $lackey_stderr.getlines
+if !$lackey_proc.exit_code.zero? && !$lackey_proc.exit_code.nil?
+  userout("datalackey exit: #{$lackey_proc.exit_code}")
+end
+exit $machine.variables.fetch('__exitcode', $machine.had_error ? 1 : 0)