mbt-gen 0.0.3 → 0.0.5

data/lib/mbt-gen.rb

@@ -1,1958 +1,2058 @@
-#!/usr/bin/ruby
-
-require 'date'
-require 'fileutils'
-require 'tmpdir'
-require 'yaml'
-require 'time'
-
-require 'nokogiri'
-
-require_relative 'solver-lib'
-require_relative 'progress'
-require_relative 'regexp-to-smtlib'
-
-
-CHUNK_SIZE = 10000
-DEFAULT_Z3_PATH = "z3"
-
-CONFIG_FILE = "config.yml"
-OUTPUT_FILE = "result.xml"
-CONTINUATION_FILE = "continuation.log"
-COMBINATION_LOG = "combinations.log"
-VARS_LOG = "vars.log"
-
-OUTPUT_DIR = "result"
-LISTS_DIR = "lists"
-QUERY_DIR = "queries"
-PARAMETERIZED_QUERY_DIR = "parameterized_queries"
-XML_TEMPLATE = ""
-
-DEFAULT_CONFIG = {
-  :z3path => "z3",
-  :generation_system_mode => "Production",
-  :generation_system => "Production"
-}
-
-USAGE = <<EOF
-usage:
-
-GENERATING A SINGLE MESSAGE:
-
-  ruby #{$PROGRAM_NAME} -list-validation-rules
-  - outputs a list of all validation rule names that have been specified in #{$PROGRAM_NAME}.
-
-  ruby #{$PROGRAM_NAME} <options>
-  - generates a message of the given type, which
-    - is valid XML
-    - adheres to the schema, and
-    - satisfies all validation rules specified for messages of this type.
-  where <options> is a combination of
-    -negate <validation_rule> - negates the named validation rule. This causes the resulting message to still satisfy all other validation rules, but to violate this one.
-  NOTE: the result can be found in #{OUTPUT_FILE}
-  
-GENERATING MESSAGES FOR ALL COMBINATIONS:
-
-  ruby #{$PROGRAM_NAME} -list-keys
-  - outputs a list of all keys that have been used in #{$PROGRAM_NAME} to mark fields or structures.
-
-  ruby #{$PROGRAM_NAME} -list <key>
-  - outputs a list of all fields and structures that have been marked with <key> in #{$PROGRAM_NAME}.
-
-  ruby #{$PROGRAM_NAME} -count-docs-for-key <key>
-  - calculates the number of combinations of all fields and structures marked with <key>
-
-  ruby #{$PROGRAM_NAME} -generate-docs-for-key <key> <options>
-  - generates all valid messages of the given type that can be derived from combinations of all the fields and structures marked with <key>.
-  where <options> is a combination of
-    -continue - continues a previously interrupted generation process. Must use the same <key> the the interrupted generation.ARGF
-    -max-num-docs <limit> - stops generation process after generating <limit> documents.
-  NOTE: enumerating large combination-spaces may take a long time.
-        - Use -count-docs-for-key to measure the size of the combination-space beforehand.
-        - Also: you can interrupt the generation process at any time using Ctrl+C. The intermediate result is stored in the directory #{OUTPUT_DIR} and the generation process can
-          be restarted later by adding -continue to the command line.
-  NOTE: do not expect the number of generated documents to equal the number of combinations as combinations may not have any valid message instances (for instance when they contradict the validation rules).
-  NOTE: the result can be found in the directory #{OUTPUT_DIR}
-
-VALIDATING THE MODEL:
-
-  ruby #{$PROGRAM_NAME} -validate <file.xml>
-  - reads in the given XML file and validates it against the model - that is, it checks whether it conforms to the specified XML schema and to the validation rules.
-EOF
-
-class FatalError < RuntimeError
-end
-
-class NormalProgramTermination < RuntimeError
-end
-
-def raw_field(xpath)
-  path_str = "base"
-  path_str = xpath.gsub("/", "-") if xpath
-  "field-#{path_str}"
-end
-
-def filled_var_for_field(xpath)
-  "#{raw_field(xpath)}-filled"
-end
-
-def value_var_for_field(xpath)
-  "#{raw_field(xpath)}-value"
-end
-
-def exists_var_for_structure(xpath)
-  struct_str = "base"
-  struct_str = xpath.gsub("/", "-") if xpath
-  "struct-#{struct_str}-exists"
-end
-
-def size_var_for_list(xpath)
-  "#{raw_field(xpath)}-size"
-end
-
-def value_var_for_list(xpath, index)
-  "#{raw_field(xpath)}-index-#{index}-value"
-end
-
-def translate_datatype_to_SMTLIB(datatype)
-  case datatype
-  when :string
-    return "String"
-  when :int
-    return "Int"
-  when :bool
-    return "Bool"
-  when :date
-    return "Int"
-  when :timestamp
-    return "Int"
-  when :enum
-    return "String"
-  end
-end
-
-def translate_value_to_SMTLIB(value)
-  if value.is_a?(Integer) then
-    return value.to_s
-  elsif value.is_a?(String) then
-    return value.inspect
-  elsif value.is_a?(Date) then
-    return (value - (Time.at(0).to_date)).to_i  # days since 1970.01.01
-  elsif value.is_a?(Time) then
-    return value.to_i
-  else
-    raise RuntimeError.new("cannot translate value to SMTLIB: #{value.inspect}")
-  end
-end
-
-def min_constraint_for_expr(expr, lower_bound)
-  "(>= #{expr} #{lower_bound})"
-end
-
-def max_constraint_for_expr(expr, upper_bound)
-  "(<= #{expr} #{upper_bound})"
-end
-
-def regex_constraint_for_expr(expr, regex)
-  "(str.in.re #{expr} #{Regexp_to_SMTLIB.translate_regexp_to_SMTLIB(regex)})"
-end
-
-def maxLength_contraint_for_expr(expr, maxLength)
-  "(<= (str.len #{expr}) #{maxLength})"
-end
-
-def oneOf_contraint_for_expr(expr, values)
-  list = values.map do |val|
-    "(= #{expr} \"#{val}\")"
-  end
-  "(or #{list.join(" ")})"
-end
-
-def enum_constraint_for_expr(expr, values)
-  "(or #{values.map{|x| "(= #{expr} #{translate_value_to_SMTLIB(x)})"}.join(" ")})"
-end
-
-def translate_field_def_to_SMTLIB_constraints(progress, solver, fielddef, xpath)
-  varname = raw_field(xpath)
-  if fielddef[:optional] == nil || fielddef[:optional] == false then
-    # field is required
-    assertionID = "required-#{varname}"
-    info = {
-      :assertion_type => :constraint,
-      :xpath => xpath,
-      :constraint_type => :required
-    }
-    solver.associateAssertionIDWith(assertionID, info)
-    solver.to_solver(progress, "(assert (! #{varname}-filled :named #{assertionID}))")
-  end
-  datatype = fielddef[:datatype]
-  case datatype
-  when :int
-    if fielddef[:min] then
-        assertionID = "min-#{varname}"
-        info = {
-          :assertion_type => :constraint,
-          :xpath => xpath,
-          :datatype => :int,
-          :value => fielddef[:min],
-          :constraint_type => :min
-        }
-        solver.associateAssertionIDWith(assertionID, info)
-        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{min_constraint_for_expr("#{varname}-value", translate_value_to_SMTLIB(fielddef[:min]))}) :named #{assertionID}))") 
-    end
-    if fielddef[:max] then
-        assertionID = "max-#{varname}"
-        info = {
-          :assertion_type => :constraint,
-          :xpath => xpath,
-          :datatype => :int,
-          :value => fielddef[:max],
-          :constraint_type => :max
-        }
-        solver.associateAssertionIDWith(assertionID, info)
-        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{max_constraint_for_expr("#{varname}-value", translate_value_to_SMTLIB(fielddef[:max]))}) :named #{assertionID}))")
-    end
-  when :string
-    if fielddef[:regex] then
-        assertionID = "regex-#{varname}"
-        info = {
-          :assertion_type => :constraint,
-          :xpath => xpath,
-          :datatype => :string,
-          :pattern => fielddef[:regex],
-          :constraint_type => :regex
-        }
-        solver.associateAssertionIDWith(assertionID, info)
-        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{regex_constraint_for_expr("#{varname}-value", fielddef[:regex])}) :named #{assertionID}))") 
-    end 
-    # TODO: minLength?
-    if fielddef[:maxLength] then
-      assertionID = "maxLength-#{varname}"
-        info = {
-          :assertion_type => :constraint,
-          :xpath => xpath,
-          :datatype => :string,
-          :value => fielddef[:maxLength],
-          :constraint_type => :maxLength
-        }
-      solver.associateAssertionIDWith(assertionID, info)
-      solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{maxLength_contraint_for_expr("#{varname}-value", fielddef[:maxLength])}) :named #{assertionID}))") 
-    end
-    if fielddef[:from_list] then
-      assertionID = "oneOf-#{varname}"
-      list_filename = "#{LISTS_DIR}/#{fielddef[:from_list]}"
-      info = {
-        :assertion_type => :constraint,
-        :xpath => xpath,
-        :datatype => :string,
-        :filename => list_filename,
-        :constraint_type => :from_list
-      }
-      unless File.exist?(list_filename)
-        raise RuntimeError.new("could not find list file #{list_filename}.")
-      end
-      values = File.read(list_filename).split("\n").map{|v| v.chomp()}
-      solver.associateAssertionIDWith(assertionID, info)
-      solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{oneOf_contraint_for_expr("#{varname}-value", values)}) :named #{assertionID}))") 
-    end
-  when :enum
-    assertionID = "enum-#{varname}"
-    info = {
-      :assertion_type => :constraint,
-      :xpath => xpath,
-      :datatype => :enum,
-      :values => fielddef[:values],
-      :constraint_type => :values
-    }
-    solver.associateAssertionIDWith(assertionID, info)
-    solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{enum_constraint_for_expr("#{varname}-value", fielddef[:values])}) :named #{assertionID}))") 
-  end
-end
-
-def translate_list_def_to_SMTLIB_constraints(progress, solver, fielddef, varname)
-  datatype = fielddef[:datatype]
-  case datatype
-  when :int
-    if fielddef[:min] then
-        fielddef[:model_maxLength].times do |i|
-          solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{min_constraint_for_expr("#{varname}-index-#{i}-value", translate_value_to_SMTLIB(fielddef[:min]))}) :named list-min-#{varname}-list-index-#{i}))") 
-        end
-    end
-    if fielddef[:max] then
-        fielddef[:model_maxLength].times do |i|
-          solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{max_constraint_for_expr("#{varname}-index-#{i}-value", translate_value_to_SMTLIB(fielddef[:max]))}) :named list-max-#{varname}-list-index-#{i}))") 
-        end
-    end
-  when :string
-    if fielddef[:regex] then
-        fielddef[:model_maxLength].times do |i|
-          solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{regex_constraint_for_expr("#{varname}-index-#{i}-value", fielddef[:regex])}) :named list-regex-#{varname}-index-#{i}))") 
-        end
-    end 
-    # TODO: minLength?
-    if fielddef[:maxLength] then
-        fielddef[:model_maxLength].times do |i|
-          solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{maxLength_contraint_for_expr("#{varname}-index-#{i}-value", fielddef[:maxLength])}) :named list-maxLength-#{varname}-index-#{i}))") 
-        end
-    end 
-  when :enum
-    fielddef[:model_maxLength].times do |i|
-      solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{enum_constraint_for_expr("#{varname}-index-#{i}-value", fielddef[:values])}) :named list-enum-#{varname}-index-#{i}))") 
-    end
-  end
-end
-
-def default_value_for_fielddef(fielddef)
-  datatype = fielddef[:datatype]
-  case datatype
-  when :int
-    return "0"
-  when :string
-    return "".inspect
-  when :date
-    return "0"
-  when :timestamp
-    return "0"
-  when :enum
-    return "".inspect
-  else
-    raise RuntimeError.new("unknown datatype: #{datatype.inspect}")
-  end
-end
-
-def translate_structure_to_SMTLIB(progress, solver, struct, hash = {})
-  prefix = hash[:prefix]
-  config = hash[:config]
-  options = hash[:options]
-  translate_substructure_to_SMTLIB(progress, solver, struct, hash)
-  translate_validation_rules_to_SMTLIB(progress, solver, struct, prefix, config, options)
-  if options[:add_SMTLIB] then
-    solver.to_solver(progress, "; --- additional SMTLIB Code: ---")
-    solver.to_solver(progress, options[:add_SMTLIB])
-  end
-end
-
-def translate_substructure_to_SMTLIB(progress, solver, struct, hash = {})
-  prefix = hash[:prefix]
-  struct_varname = hash[:struct_varname]
-  config = hash[:config]
-  options = hash[:options]
-  solver.to_solver(progress, "; --- Structure: ---")
-  struct.each do |field, definition|
-    if field == :additional_smtlib then
-      solver.to_solver(progress, "; additional SMTLIB of structure #{prefix}")
-      solver.to_solver(progress, translate_validation_rule_to_SMTLIB(definition, struct, prefix, config, options, "<additional SMTLIB of structure #{prefix}>"))
-    elsif field == :validation_rules || field == :predicates || field == :parent_link || field == :struct_xpath_prefix
-      next 
-    else
-      solver.to_solver(progress, "; #{field.to_s}")
-      if definition[:type] == :field then
-        display_prefix = ""
-        display_prefix = "#{prefix}/" if prefix
-        xpath = field.to_s
-        xpath = "#{prefix}/#{xpath}" if prefix
-        solver.declare_const(progress, filled_var_for_field(xpath), "Bool", {:xpath => xpath})
-        solver.declare_const(progress, value_var_for_field(xpath), translate_datatype_to_SMTLIB(definition[:datatype]), {:xpath => xpath, :datatype => definition[:datatype]})
-        solver.to_solver(progress, "(assert (! (=> (not #{filled_var_for_field(xpath)}) (= #{value_var_for_field(xpath)} #{default_value_for_fielddef(definition)})) :named empty-field-has-no-value-for-#{raw_field(xpath)}))") 
-        if (struct_varname)
-          solver.to_solver(progress, "(assert (! (=> (not #{struct_varname}) (not #{filled_var_for_field(xpath)})) :named empty-struct-has-no-fields-for-#{raw_field(xpath)}))") 
-        end
-        translate_field_def_to_SMTLIB_constraints(progress, solver, definition, xpath)
-      elsif definition[:type] == :list then
-        display_prefix = ""
-        display_prefix = "#{prefix}/" if prefix
-        xpath = field.to_s
-        xpath = "#{prefix}/#{xpath}" if prefix
-        solver.declare_const(progress, filled_var_for_field(xpath), "Bool", {:xpath => xpath})
-        solver.declare_const(progress, size_var_for_list(xpath), "Int", {:xpath => xpath})
-        solver.to_solver(progress, "(assert (! (=> (not #{filled_var_for_field(xpath)}) (= #{size_var_for_list(xpath)} 0)) :named empty-list-field-has-zero-size-#{raw_field(xpath)}))")
-        definition[:model_maxLength].times do |i|
-          solver.declare_const(progress, value_var_for_list(xpath, i), translate_datatype_to_SMTLIB(definition[:datatype]), {:xpath => xpath, :xpath_element => definition[:xpath_element], :index => i, :datatype => definition[:datatype]})
-          solver.to_solver(progress, "(assert (! (=> (<= #{size_var_for_list(xpath)} #{i}) (= #{value_var_for_list(xpath, i)} #{default_value_for_fielddef(definition)})) :named empty-list-field-has-no-value-for-#{raw_field(xpath)}-index-#{i}))") 
-        end
-        if (struct_varname)
-          solver.to_solver(progress, "(assert (! (=> (not #{struct_varname}) (= #{size_var_for_list(xpath)} 0)) :named empty-struct-has-no-fields-for-#{raw_field(xpath)}))") 
-        end
-        translate_list_def_to_SMTLIB_constraints(progress, solver, definition, raw_field(xpath))
-      elsif definition[:type] == :structure then
-        xpath = field.to_s
-        xpath = "#{prefix}/#{xpath}" if prefix
-        solver.declare_const(progress, exists_var_for_structure(xpath), "Bool", {:xpath => xpath})
-        if (struct_varname)
-          solver.to_solver(progress, "(assert (! (=> (not #{struct_varname}) (not #{exists_var_for_structure(xpath)})) :named empty-struct-has-no-substructure-#{raw_field(xpath)}))") 
-        end
-        translate_substructure_to_SMTLIB(progress, solver, definition[:ref], :prefix => xpath, :struct_varname => exists_var_for_structure(xpath), :options => options, :config => config)
-      else
-        next
-      end
-    end
-  end
-end
-
-def resolve_field_path(path, relative_to_struct, prefix)
-  current = relative_to_struct
-  result = nil
-  path.split("/").each do |segment|
-    unless current then
-      raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect}: resolution worked up until #{result.inspect}, but then the last segment was a field or list.")
-    end
-    field_def = current[segment.to_sym]
-    unless field_def then
-      raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect}: structure #{prefix.inspect} does not have a member #{segment.to_sym.inspect}.")
-    end
-    if field_def[:type] == :structure then
-      unless field_def[:ref] then
-        raise RuntimeError.new("could not resolve field path: #{path.inspect}: substructure #{segment.inspect} of structure #{prefix.inspect} does not have a :ref.")
-      end
-      current = field_def[:ref]
-      if prefix then
-        prefix = "#{prefix}/#{segment}"
-      else
-        prefix = segment
-      end
-    elsif field_def[:type] == :field then
-      current = nil
-      result = segment
-      if prefix
-        result = "#{prefix}/#{result}"
-      end
-    elsif field_def[:type] == :list then
-      current = nil
-      result = segment
-      if prefix
-        result = "#{prefix}/#{result}"
-      end
-    end
-  end
-  return result
-end
-
-def resolve_field_def(path, relative_to_struct, prefix)
-  current = relative_to_struct
-  result = nil
-  path.split("/").each do |segment|
-    unless current then
-      raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect}: resolution worked up until #{result.inspect}, but then the last segment was a field or list.")
-    end
-    field_def = current[segment.to_sym]
-    unless field_def then
-      raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect}: structure #{prefix.inspect} does not have a member #{segment.to_sym.inspect}.")
-    end
-    if field_def[:type] == :structure then
-      unless field_def[:ref] then
-        raise RuntimeError.new("could not resolve field path: #{path.inspect}: substructure #{segment.inspect} of structure #{prefix.inspect} does not have a :ref.")
-      end
-      current = field_def[:ref]
-      if prefix then
-        prefix = "#{prefix}/#{segment}"
-      else
-        prefix = segment
-      end
-    elsif field_def[:type] == :field then
-      current = nil
-      result = field_def
-    elsif field_def[:type] == :list then
-      current = nil
-      result = field_def
-    end
-  end
-  return result
-end
-
-def resolve_struct_path(path, relative_to_struct, prefix)
-  current = relative_to_struct
-  result = nil
-  path.split("/").each do |segment|
-    unless current then
-      raise RuntimeError.new("could not resolve field path: #{path.inspect} relative to struct #{relative_to_struct.inspect}")
-    end
-    if segment == "." then
-      result = current[:struct_xpath_prefix]
-    elsif segment == ".." then
-      current = current[:parent_link]
-      if prefix then
-        prefix_segs = prefix.split("/")
-        prefix = prefix_segs[0,prefix_segs.size()-1].join("/")
-      else
-        prefix = current[:struct_xpath_prefix]
-      end
-    else
-      field_def = current[segment.to_sym]
-      unless field_def then
-        raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect} relative to struct #{relative_to_struct.inspect}")
-      end
-      if field_def[:type] == :structure then
-        current = field_def[:ref]
-        if prefix then
-          prefix = "#{prefix}/#{segment}"
-        else
-          prefix = segment
-        end
-        result = prefix
-      end
-    end
-  end
-  return result
-end
-
-def resolve_struct_def(path, relative_to_struct, prefix, context_desc = "")
-  current = relative_to_struct
-  result = nil
-  path.split("/").each do |segment|
-    unless current then
-      raise RuntimeError.new("could not resolve field path: #{path.inspect} relative to struct #{relative_to_struct.inspect} #{context_desc}")
-    end
-    if segment == "." then
-      result = current
-    elsif segment == ".." then
-      current = current[:parent_link]
-      if prefix then
-        prefix_segs = prefix.split("/")
-        prefix = prefix_segs[0,prefix_segs.size()-1].join("/")
-      else
-        prefix = current[:struct_xpath_prefix]
-      end
-    else
-      field_def = current[segment.to_sym]
-      unless field_def then
-        raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect} relative to struct #{relative_to_struct.inspect} #{context_desc}")
-      end
-      if field_def[:type] == :structure then
-        current = field_def[:ref]
-        if prefix then
-          prefix = "#{prefix}/#{segment}"
-        else
-          prefix = segment
-        end
-        result = current
-      end
-    end
-  end
-  return result
-end
-
-def translate_validation_rules_to_SMTLIB(progress, solver, struct, prefix, config, options)
-  solver.to_solver(progress, "; --- Validation Rules ---")
-  translate_validation_rules_for_structure_to_SMTLIB(progress, solver, struct, prefix, config, options)
-  struct.each_pair do |key, value|
-    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
-    if value[:type] == :structure then
-      if prefix then
-        new_prefix = "#{prefix.to_s}/#{key}"
-      else
-        new_prefix = key.to_s
-      end
-      translate_validation_rules_to_SMTLIB(progress, solver, value[:ref], new_prefix, config, options)
-    end
-  end
-end
-
-def translate_validation_rule_to_SMTLIB(rule, context_struct, prefix, config, options, rule_name)
-  date_now = options[:date_now]
-  timestamp_now = options[:timestamp_now]
-  return rule.gsub(/FILLED\[[^\]]*\]/) do |m|
-                    unless m =~ /FILLED\[([^\]]*)\]/ then
-                      raise RuntimeError.new("could not parse FILLED[...]: #{m.inspect} in validation rule #{rule_name}")
-                    end
-                    field = resolve_field_path($1, context_struct, prefix)
-                    filled_var_for_field(field)
-                  end
-                  .gsub(/VALUE\[[^\]]*\]/) do |m|
-                    unless m =~ /VALUE\[([^\]]*)\]/ then
-                      raise RuntimeError.new("could not parse VALUE[...]: #{m.inspect} in validation rule #{rule_name}")
-                    end
-                    field = resolve_field_path($1, context_struct, prefix)
-                    value_var_for_field(field)
-                  end
-                  .gsub(/EXIST\[[^\]]*\]/) do |m|
-                    unless m =~ /EXIST\[([^\]]*)\]/ then
-                      raise RuntimeError.new("could not parse EXIST[...]: #{m.inspect} in validation rule #{rule_name}")
-                    end
-                    s = resolve_struct_path($1, context_struct, prefix)
-                    exists_var_for_structure(s)
-                  end
-                  .gsub(/SIZE\[[^\]]*\]/) do |m|
-                    unless m =~ /SIZE\[([^\]]*)\]/ then
-                      raise RuntimeError.new("could not parse SIZE[...]: #{m.inspect} in validation rule #{rule_name}")
-                    end
-                    l = resolve_field_path($1, context_struct, prefix)
-                    size_var_for_list(l)
-                  end
-                  .gsub(/LIST_CONTAINS\[[^\]]*,[^\]]*\]/) do |m|
-                    unless m =~ /LIST_CONTAINS\[([^\]]*),([^\]]*)\]/ then
-                      raise RuntimeError.new("could not parse LIST_CONTAINS[...]: #{m.inspect} in validation rule #{rule_name}")
-                    end
-                    l = resolve_field_path($1, context_struct, prefix)
-                    l_def = resolve_field_def($1, context_struct, prefix)
-                    value = $2
-                    terms = []
-                    l_def[:model_maxLength].times do |i|
-                      terms << "(and (< #{i} #{size_var_for_list(l)}) (= #{value_var_for_list(l, i)} #{value}))"
-                    end
-                    "(or #{terms.join(" ")})"
-                  end
-                  .gsub(/LIST_VALUE_FIRST_ELEMENT\[[^\]]*\]/) do |m|
-                    unless m =~ /LIST_VALUE_FIRST_ELEMENT\[([^\]]*)\]/ then
-                      raise RuntimeError.new("could not parse LIST_VALUE_FIRST_ELEMENT[...]: #{m.inspect} in validation rule #{rule_name}")
-                    end
-                    xpath = $1
-                    l = resolve_field_path(xpath, context_struct, prefix)
-                    value_var_for_list(l, 0)
-                  end
-                  .gsub("[Date.now]", date_now.to_s)
-                  .gsub("[Timestamp.now]", timestamp_now.to_s)
-                  .gsub(/PREDICATE\[[^\]]*,[^\]]*\]/) do |m|
-                    unless m =~ /PREDICATE\[([^\]]*),([^\]]*)\]/ then
-                      raise RuntimeError.new("could not parse PREDICATE[...]: #{m.inspect} in validation rule #{rule_name}")
-                    end
-                    path = $1
-                    key = $2.strip().to_sym
-                    sdef = resolve_struct_def(path, context_struct, prefix, "in validation rule #{rule_name}")
-                    unless sdef
-                      raise RuntimeError.new("could not resolve struct path #{path.inspect} in validation rule #{rule_name}")
-                    end
-                    preds = sdef[:predicates]
-                    mdk = preds[key]
-                    if mdk then
-                      if mdk[:smtlib] then
-                        translate_validation_rule_to_SMTLIB(mdk[:smtlib], sdef, sdef[:struct_xpath_prefix], config, options, "#{rule_name}.predicate.#{key}[#{sdef[:struct_xpath_prefix]}]" )
-                      else
-                        raise RuntimeError.new("Predicate #{key} for structure #{sdef[:struct_xpath_prefix]} does not have an SMTLIB definition, but is referenced in validation rule #{rule_name}")
-                      end
-                    else
-                      raise RuntimeError.new("No predicate #{key.inspect} defined for structure #{sdef[:struct_xpath_prefix]}, but one referenced in validation rule #{rule_name}")
-                    end
-                  end
-                  .gsub(/CONFIG\[[^\]]*\]/) do |m|
-                    unless m =~ /CONFIG\[([^\]]*)\]/ then
-                      raise RuntimeError.new("could not parse LIST_CONTAINS[...]: #{m.inspect} in validation rule #{rule_name}")
-                    end
-                    key = $1.to_sym
-                    if config.has_key?(key) then
-                      "\"#{config[key]}\"" 
-                    else
-                      raise RuntimeError.new("unknown CONFIG-key #{key.inspect} encountered in validation rule #{rule_name}" )
-                    end
-                  end
-                  .gsub(/FIELD_IN_QUERY\[[^\]]*\]/) do |m|
-                    unless m =~ /FIELD_IN_QUERY\[([^\]]*),([^\]]*)\]/ then
-                      raise RuntimeError.new("could not parse FIELD_IN_QUERY[..,..]: #{m.inspect} in validation rule #{rule_name}")
-                    end
-                    xpath = $1
-                    query_filename = $2
-                    fpath = resolve_field_path(xpath, context_struct, prefix)
-                    fdef = resolve_field_def(xpath, context_struct, prefix)
-                    values = File.read("#{QUERY_DIR}/#{query_filename.strip()}").split("\n")
-                    if fdef[:datatype] == :string then
-                      terms = values.filter{|l| !l.start_with?("#")}.map do |val|
-                        "(= #{value_var_for_field(fpath)} \"#{val}\")"
-                      end
-                    elsif fdef[:datatype] == :int then
-                      terms = values.filter{|l| !l.start_with?("#")}.map do |val|
-                        "(= #{value_var_for_field(fpath)} #{Integer(val)})"
-                      end
-                    else
-                      raise RuntimeError.new("FIELD_IN_QUERY[] was applied to a field of datatype #{fdef[:datatype].inspect}. However, it can only be applied to fields of datatype string or int.")
-                    end
-                    "(or #{terms.join(" ")})"
-                  end
-                  .gsub(/FIELD_IN_SMTLIB_PARAMETERIZED_QUERY\[[^\]]*\]/) do |m|
-                    unless m =~ /FIELD_IN_SMTLIB_PARAMETERIZED_QUERY\[([^\]]*),([^\]]*),([^\]]*)\]/ then
-                      raise RuntimeError.new("could not parse FIELD_IN_SMTLIB_PARAMETERIZED_QUERY[..,..,..]: #{m.inspect} in validation rule #{rule_name}")
-                    end
-                    field_xpath = $1
-                    param_smtlib = $2
-                    query_filename = $3
-                    fpath = resolve_field_path(field_xpath.strip(), context_struct, prefix)
-                    fdef = resolve_field_def(field_xpath.strip(), context_struct, prefix)
-                    query_result = YAML.load_file("#{PARAMETERIZED_QUERY_DIR}/#{query_filename.strip()}")
-                    terms = []
-                    if fdef[:datatype] == :string then
-                      terms << "(or #{query_result.keys.map { |key| "(= #{param_smtlib} \"#{key}\")" }.join(" ")})"
-                      query_result.each_pair do |key, values|
-                        terms << "(=> (= #{param_smtlib} \"#{key}\") (or #{values.map{ |v| "(= #{value_var_for_field(fpath)} \"#{v}\")" }.join(" ")}))"
-                      end
-                    else
-                      raise RuntimeError.new("FIELD_IN_SMTLIB_PARAMETERIZED_QUERY[] was applied to a field of datatype #{fdef[:datatype].inspect}. However, it can only be applied to fields of datatype string.")
-                    end
-                    "(and #{terms.join(" ")})"
-                  end
-                  .gsub(/CONFIG_IN_QUERY\[[^\]]*\]/) do |m|
-                    unless m =~ /CONFIG_IN_QUERY\[([^\]]*),([^\]]*)\]/ then
-                      raise RuntimeError.new("could not parse CONFIG_IN_QUERY[..,..]: #{m.inspect} in validation rule #{rule_name}")
-                    end
-                    key = $1.to_sym
-                    unless config.has_key?(key) then
-                      raise RuntimeError.new("unknown CONFIG key #{key.inspect} encountered in validation rule #{rule_name}")
-                    end
-                    query_filename = $2
-                    values = File.read("#{QUERY_DIR}/#{query_filename.strip()}").split("\n")
-                    terms = values.filter{|l| !l.start_with?("#")}.map do |val|
-                      "(= \"#{config[key]}\" \"#{val}\")"
-                    end
-                    "(or #{terms.join(" ")})"
-                  end
-end
-
-def translate_validation_rules_for_structure_to_SMTLIB(progress, solver, struct, prefix, config, options)
-  return unless struct[:validation_rules]
-
-  solver.to_solver(progress, "; --- Validation Rules: ---")
-  negated_rules = options[:negated_validation_rules] || []
-  solver.to_solver(progress, "; negated rules: #{negated_rules.empty? ? "<none>" : negated_rules.join(", ")}")
-
-  rules = struct[:validation_rules]
-  rules.each do |rule_name, rule|
-    rule = rule[:smtlib]  # ignore the text representation
-    next unless rule # warning about this omission is the job of validate_model()
-    rule_SMTLIB = translate_validation_rule_to_SMTLIB(rule, struct, prefix, config, options, rule_name)
-    if negated_rules.include?(rule_name) then
-      rule_SMTLIB = "(not #{rule_SMTLIB})"
-    end
-    solver.to_solver(progress, "(assert (! #{rule_SMTLIB} :named validation_rule_#{rule_name}_instance_#{prefix.gsub("/", "_")}))")
-  end
-end
-
-def parse_field_value(fielddef, value)
-  value = value.gsub("\n", "")
-  datatype = fielddef[:datatype]
-  case datatype
-  when :int
-    return Integer(value)
-  when :string
-    return value
-  when :enum
-    return value
-  when :date
-    return Date.parse(value)
-  when :timestamp
-    return Time.new(value)
-  end
-end
-
-def solver_value_to_string(solver_value)
-  if solver_value =~ /\"(.*)\"/ then
-    return $1
-  else
-    return solver_value
-  end
-end
-
-def build_document_from_model(progress, struct, model)
-  doc = Nokogiri::XML(XML_TEMPLATE)
-  add_model_struct_to_doc(progress, struct, doc, model)
-  return doc
-end
-
-def add_model_struct_to_doc(progress, struct, doc, model, prefix = nil)
-  struct.each do |key, value|
-    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
-    if prefix then
-      xpath = "#{prefix}/#{key}"
-    else
-      xpath = key.to_s
-    end
-    model_value = model[xpath]
-    if value[:type] == :field then
-      unless model_value then
-        raise RuntimeError.new("xpath '#{xpath}' not contained in model.")
-      end
-      if model_value[:filled] == true || model_value[:filled] == "true" then
-        doc.add_child("<#{key}>#{solver_value_to_string(model_value[:value])}</#{key}>")
-      end
-    elsif value[:type] == :list then
-      node = doc.add_child("<#{key}></#{key}>")
-      if node.is_a?(Nokogiri::XML::NodeSet) then
-        node = node.first 
-      end
-      xpath_element = value[:xpath_element]
-      if Integer(model_value[:size]) > 0 then
-        model_value[:value].slice(0, Integer(model_value[:size])).each do |mvalue|
-          node.add_child("<#{xpath_element}>#{solver_value_to_string(mvalue)}</#{xpath_element}>")
-        end
-      end
-    elsif value[:type] == :structure then
-      unless value[:ref] then
-        raise RuntimeError.new("struct is broken: :structure #{key.inspect} does not have a :ref")
-      end
-      if model_value[:exists] == true || model_value[:exists] == "true" then
-        node = doc.add_child("<#{key}></#{key}>")
-        if node.is_a?(Nokogiri::XML::NodeSet) then
-          node = node.first 
-        end
-        add_model_struct_to_doc(progress, value[:ref], node, model, xpath)
-      end
-    else
-      raise RuntimeError.new("struct is broken: encountered unknown :type #{value[:type].inspect}")
-    end
-  end
-end
-
-def collect_validation_rule_names(struct)
-  result = []
-  struct.each do |key, value|
-    next if key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
-    if key == :validation_rules
-      value.each do |key, value|
-        result << key
-      end
-    else
-      if value[:type] == :structure
-        result += collect_validation_rule_names(value[:ref])
-      end
-    end
-  end
-  return result
-end
-
-def set_standard_options(progress, solver)
-  solver.to_solver(progress, "(set-option :produce-unsat-cores true) ; enable generation of unsat cores")
-  solver.to_solver(progress, "(set-option :smt.core.minimize true) ; ensure that unsat cores are minimal")
-  solver.to_solver(progress, "")
-end
-
-def generate_doc(progress, message, output_filename, config, options)
-  log = options[:log] || nil
-  log.puts "generating doc #{output_filename}" if log
-  options[:solverLog] = options[:solverLog] || "#{output_filename}.smt2"
-
-  z3path = config[:z3path] || DEFAULT_Z3_PATH
-  z3 = Z3Solver.new(progress, z3path)
-  begin
-    model = z3.query_model(progress, options) do |solver|
-      set_standard_options(progress, solver)
-      log.puts "translating structure" if log
-      translate_structure_to_SMTLIB(progress, solver, message, :options => options, :config => config)
-      solver.to_solver(progress, "")
-    end
-  ensure
-    z3.close()
-  end
-
-  if model then
-    result_doc = build_document_from_model(progress, message, model)
-    dir = File.dirname(output_filename)
-    FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
-    File.open(output_filename, "w") do |f|
-      f.puts result_doc.to_xml()
-    end
-    progress.print_line "DONE. Generated XML was written to #{output_filename}, SMTLIB log was written to #{options[:solverLog]}"
-    return true
-  else
-    progress.print_line "FAILED. No XML document was generated. For more details, please see the SMTLIB log in #{options[:solverLog]}"
-    return false
-  end
-end
-
-def z3ValueToRubyValue(value)
-  if value =~ /\"(.*)\"/ then # string
-    return $1
-  elsif value =~ /\d+/ then # int
-    return Integer(value)
-  elsif value == "true"
-    return true
-  elsif value == "false" then
-    return false 
-  else
-    raise RuntimeError.new("enountered unknown type of value in z3 model: #{value.inspect}")
-  end
-end
-
-def extract_key_field_values_from_model(key_fields, model)
-  result = {}
-  key_fields.each do |xpath, fieldDesc|
-    fieldInfo = model[xpath]
-    if fieldDesc[:type] == :field then
-      if fieldInfo[:filled] then
-        result[xpath] = z3ValueToRubyValue(fieldInfo[:value])
-      else
-        result[xpath] = nil
-      end
-    elsif fieldDesc[:type] == :struct || fieldDesc[:type] == :structure then
-      if fieldInfo.has_key?(:exists) then
-        result[xpath] = z3ValueToRubyValue(fieldInfo[:exists])
-      else
-        result[xpath] = false
-      end
-    else
-      raise RuntimeError.new("encountered unknown :type in solver model: #{fieldDesc[:type].inspect}")
-    end
-  end
-  return result
-end
-
-def all_values_for_field(field)
-  unless field[:type] == :field then
-    raise RuntimeError.new("something that is not a field definition was passed to all_values_for_field(): #{field.inspect}")
-  end
-  unless field[:datatype] == :enum then
-    raise RuntimeError.new("all_values_for_field() is only applicable to fields of datatype enum! not #{field[:datatype].inspect}.")
-  end
-  values = field[:values].dup
-  if field[:optional] == true then
-    values << nil
-  end
-  return values
-end
-
-def calc_combinations_for_key_fields(key_fields)
-  count = 1
-  key_fields.each do |field|
-    if field[1][:type] == :field then
-      if field[1][:datatype] == :enum then
-        num_values = field[1][:values].size
-        num_values += 1 if field[1][:optional]
-      elsif field[1][:datatype] == :int then
-        if field[1].has_key?(:min) and field[1].has_key?(:max) and field[1][:min] <= field[1][:max] then
-          if field[1][:optional] then
-            num_values = 2 + field[1][:max] - field[1][:min]
-          else
-            num_values = 1 + field[1][:max] - field[1][:min]
-          end
-        else
-          return "potentially infinite"
-        end
-      elsif field[1][:datatype] == :string then
-        return "potentially infinite"
-      end
-    elsif field[1][:type] == :structure then
-      num_values = 2
-    end
-    count *= num_values
-  end
-  return count
-end
-
-def calc_combinations(message, key)
-  key_fields = collect_key_fields(key, message)
-  return calc_combinations_for_key_fields(key_fields)
-end
-
-def read_docs_from_log_file(log_filename)
-  result = []
-  if File.exist?(log_filename) then
-    File.open(log_filename, "r") do |f|
-      f.each_line do |line|
-        doc = {}
-        if line =~ /(.*)\ <-\ \{(.*)\}/ then
-          filename = $1
-          combination = $2
-          combination.split(", ").each do |assignment|
-            if assignment =~ /(.*):\ (.*)/ then
-              key = $1
-              value = $2
-              if value =~ /\"(.*)\"/ then
-                value = $1
-              elsif value == "nil" then
-                value = nil
-              end
-              doc[key] = value
-            else
-              raise RuntimeError.new("could not parse logfile #{log_filename}: invalid assignment #{assignment.inspect}")
-            end
-          end
-        else
-          raise RuntimeError.new("could not parse logfile #{log_filename}: invalid line #{line.inspect}")
-        end
-        result << doc
-      end
-    end
-  end
-  return result
-end
-
-def prepare_output_doc_filename(number, output_dir)
-  chunk_number = number / 100000
-  dirname = "#{output_dir}/#{chunk_number}"
-  unless Dir.exist?(dirname) then
-    FileUtils.mkdir_p(dirname)
-  end
-  return "#{dirname}/doc#{number}.xml"
-end
-
-def values_for_key_field(field)
-  if field[:type] == :field then
-    if field[:datatype] == :enum then
-      if field[:optional] then
-        return field[:values] + [nil]
-      else
-        return field[:values]
-      end
-    elsif field[:datatype] == :int then
-      if field.has_key?(:min) and field.has_key?(:max) and field[:min] <= field[:max] then
-        return (field[:min]..field[:max]).to_a
-      else
-        raise RuntimeError.new("cannot determine values for unbounded :int field")
-      end
-    else
-      raise RuntimeError.new("cannot determine values for datatype #{field[:datatype].inspect}")
-    end
-  elsif field[:type] == :structure then
-    if field[:optional] then
-      return [true, false]
-    else
-      return [true]
-    end
-  else
-    raise RuntimeError.new("Encountered unknown :type in model: #{field[:type].inspect} - keys can only mark fields or structures.")
-  end
-end
-
-def collect_key_fields(fkey, struct, prefix = nil)
-  result = {}
-  struct.each do |key, value|
-    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
-    if value[:type] == :structure
-      if value.has_key?(:keys) and value[:keys].include?(fkey) then
-        xpath = key.to_s
-        xpath = "#{prefix}/#{xpath}" if prefix
-        result[xpath] = value.clone().filter{|k,v| [:type, :optional, :datatype, :values].include?(k)}
-      end
-      if prefix then
-        new_prefix = "#{prefix}/#{key}" 
-      else
-        new_prefix = key
-      end
-      result = result.merge(collect_key_fields(fkey, value[:ref], new_prefix))
-    elsif value[:type] == :field and value.has_key?(:keys) and value[:keys].include?(fkey) then
-      xpath = key.to_s
-      xpath = "#{prefix}/#{xpath}" if prefix
-      result[xpath] = value.clone().filter{|k,v| [:type, :optional, :datatype, :values, :min, :max].include?(k)}
-    end
-  end
-  return result
-end
-
-def collect_key_fields_and_strctures(fkey, struct, prefix = nil)
-  fields = []
-  structures = []
-  struct.each do |key, value|
-    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
-    if value[:type] == :structure
-      if value.has_key?(:keys) and value[:keys].include?(fkey) then
-        xpath = key.to_s
-        xpath = "#{prefix}/#{xpath}" if prefix
-        structures << xpath
-      end
-      if prefix then
-        new_prefix = "#{prefix}/#{key}" 
-      else
-        new_prefix = key
-      end
-      result_fields, result_structures = collect_key_fields_and_strctures(fkey, value[:ref], new_prefix)
-      fields += result_fields
-      structures += result_structures
-    elsif value[:type] == :field and value.has_key?(:keys) and value[:keys].include?(fkey) then
-      xpath = key.to_s
-      xpath = "#{prefix}/#{xpath}" if prefix
-      fields << xpath
-    end
-  end
-  return fields, structures
-end
-
-def collect_keys(struct)
-  result = []
-  struct.each do |key, value|
-    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
-    if value[:type] == :structure
-      if value.has_key?(:keys) then
-        result += value[:keys]
-      end
-      if value[:ref] then
-        result += collect_keys(value[:ref])
-      else
-        raise RuntimeError.new("Model error: encountered structure without :ref: key=#{key.inspect} value=#{value.inspect}")
-      end
-    elsif value[:type] == :field and value.has_key?(:keys) then
-      result += value[:keys]
-    end
-  end
-  result.uniq
-end
-
-def collect_validation_rules(struct)
-  result = []
-  struct.each do |key, value|
-    next if key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
-    if key == :validation_rules then
-      result += value.keys
-    else
-      if value[:type] == :structure
-        if value[:ref] then
-          result += collect_validation_rules(value[:ref])
-        else
-          raise RuntimeError.new("Model error: encountered structure without :ref: key=#{key.inspect} value=#{value.inspect}")
-        end
-      end
-    end
-  end
-  result.uniq
-end
-
-def list_keys(message)
-  keys = collect_keys(message)
-  puts "keys: #{keys.join(", ")}"
-end
-
-def list_fields_and_structures(message, fkey)
-  fields, structures = collect_key_fields_and_strctures(fkey, message)
-  puts "fields for key #{fkey}:"
-  fields.each { |f| puts "  #{f}" }
-  puts "structures for key #{fkey}:"
-  structures.each { |s| puts "  #{s}" }
-end
-
-def list_validation_rules(message)
-  rules = collect_validation_rules(message)
-  puts "Validation Rules:"
-  rules.each { |r| puts "  #{r}" }
-end
-  
-def value_of_type(value, type)
-  if type == :string or type == :enum then
-    "\"#{value}\""
-  elsif type == :int then
-    value.to_s
-  elsif type == :date then
-    (Date.parse(value) -  Time.at(0).to_date).to_i.to_s
-  elsif type == :timestamp then
-    (Time.new(value) - Time.at(0)).to_i.to_s
-  else
-    raise RuntimeError.new("encountered unknown :datatype #{type.inspect}")
-  end
-end
-
-class CoreEnumerationAlgo
-
-  def initialize()
-    @num_tries = 0
-    @num_docs = 0
-    @on_new_doc = nil
-    @on_unsat = nil
-  end
-
-
-  # fix values of all variables left from switched
-  def enumerate_fixing_clause(assignment, vars, switched)
-    clauses = []
-    switched.times do |index|
-      var_xpath = vars[index][0]
-      var_def = vars[index][1]
-      if var_def[:type] == :field then
-        if assignment[var_xpath] == nil then
-          clauses << "(assert (! (= #{filled_var_for_field(var_xpath)} false) :named fixing_clause_#{index}))"
-        else
-          clauses << "(assert (! (and (= #{filled_var_for_field(var_xpath)} true) (= #{value_var_for_field(var_xpath)} #{value_of_type(assignment[var_xpath], var_def[:datatype])})) :named fixing_clause_#{index}))"
-        end
-      elsif var_def[:type] == :structure then
-        clauses << "(assert (! (= #{exists_var_for_structure(var_xpath)} #{assignment[var_xpath].inspect}) :named fixing_clause_#{index}))"
-      end
-    end
-    "; fixing clauses:\n#{clauses.join("\n")}" 
-  end
-
-  def summarize_fixing_clause(assignment, vars, switched)
-    fixed = switched.times.to_a.map do |index|
-      var_xpath = vars[index][0]
-      "#{var_xpath}=#{assignment[var_xpath].inspect}"
-    end
-    "fixed(#{fixed.join(", ")})"
-  end
-  
-  # prevent the var from taking values that already have been enumerated exhaustively
-  def enumerate_blocking_clause(var_xpath, var_def, blocked)
-    clauses = []
-    num = 0
-    if var_def[:type] == :field then
-      blocked.each do |value|
-        if value == nil then
-          clauses << "(assert (! (not (= #{filled_var_for_field(var_xpath)} false)) :named blocking_clause_#{num}))"
-        else
-          clauses << "(assert (! (not (and (= #{filled_var_for_field(var_xpath)} true) (= #{value_var_for_field(var_xpath)} #{value_of_type(value, var_def[:datatype])}))) :named blocking_clause_#{num}))"
-        end
-        num += 1
-      end
-    elsif var_def[:type] == :structure then
-      blocked.each do |value|
-        clauses << "(assert (! (not (= #{exists_var_for_structure(var_xpath)} #{value.inspect})) :named blocking_clause_#{num}))"
-        num += 1
-      end
-    end
-    "; blocking clauses:\n#{clauses.join("\n")}" 
-  end
-
-  def summarize_blocking_clause(var_xpath, blocked)
-    "blocked(#{var_xpath})=[#{blocked.map{|v| v.inspect}.join(", ")}]"
-  end
-
-  def generate_sparse_doc(progress, message, key, config, options, code, output_doc_filename)
-    additional_options = {}
-    if code then
-        additional_options = { :add_SMTLIB => code }
-    end
-    return generate_doc_return_key_values(progress, message, key, output_doc_filename, config, options.merge(additional_options))
-  end
-
-  def log_filename_for_output_filename(output_filename)
-    "#{output_filename}.smt2"
-  end
-
-  def generate_doc_return_key_values(progress, message, key, output_filename, config, options)
-    log = options[:log] || nil
-    log.puts "generating doc #{output_filename}" if log
-    options[:solverLog] = log_filename_for_output_filename(output_filename)
-
-    key_fields = collect_key_fields(key, message)
-
-    z3path = config[:z3path] || DEFAULT_Z3_PATH
-    z3 = Z3Solver.new(progress, z3path)
-    begin
-      model = z3.query_model(progress, options) do |solver|
-        set_standard_options(progress, solver)
-        log.puts "translating structure" if log
-        translate_structure_to_SMTLIB(progress, solver, message, :options => options, :config => config)
-        solver.to_solver(progress, "")
-      end
-    ensure
-      z3.close()
-    end
-
-    if model then
-      result_doc = build_document_from_model(progress, message, model)
-      dir = File.dirname(output_filename)
-      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
-      File.open(output_filename, "w") do |f|
-        f.puts result_doc.to_xml(:save_with => Nokogiri::XML::Node::SaveOptions::AS_XML)
-      end
-      progress.print_line "DONE. Generated XML was written to #{output_filename}, SMTLIB log was written to #{options[:solverLog]}"
-      return extract_key_field_values_from_model(key_fields, model)
-    else
-      progress.print_line "FAILED. No XML document was generated. For more details, please see the SMTLIB log in #{options[:solverLog]}"
-      return nil
-    end
-  end
-
-  def store_continuation_point(doc_number, state, assignment, new_assignment, filename)
-    File.open(filename, "w") do |f|
-      f.puts("#{doc_number};#{@num_docs};state(#{state.map{|s| if s then "[#{s.map{|v| v.inspect}.join(",")}]" else "nil" end}.join("#")});assignment(#{assignment.to_a.map{|k,v| "#{k}:#{v.inspect}"}.join(",")});new_assignment(#{new_assignment.to_a.map{|k,v| "#{k}:#{v.inspect}"}.join(",")})")
-    end
-  end
-
-  def self.parse_state(state_str)
-    unless state_str =~ /state\((.*)\)/
-      return nil
-    end
-    split = $1.split("#")
-    split.map do |e|
-      if e == "nil" then
-        nil
-      elsif e =~ /\[(.*)\]/ then
-        $1.split(",").map do |i|
-          if i.strip == "nil" then
-            nil
-          elsif i.strip == "true" then
-            true
-          elsif i.strip == "false" then
-            false
-          elsif i.strip =~ /\"(.*)\"/ then
-            $1
-          end
-        end
-      end
-    end
-  end
-
-  def self.parse_assignment(str)
-    unless str =~ /assignment\((.*)\)/
-      return nil
-    end
-    split = $1.split(",")
-    result = {}
-    split.each do |e|
-      s2 = e.split(":")
-      key = s2[0]
-      value = s2[1]
-      if value == "nil" then
-        result[key] = nil
-      elsif value == "true" then
-        result[key] = true
-      elsif value == "false" then
-        result[key] = false
-      elsif value =~ /\"(.*)\"/
-        result[key] = $1
-      end
-    end
-    result
-  end
-
-  def self.parse_new_assignment(str)
-    unless str =~ /new_assignment\((.*)\)/
-      return nil
-    end
-    split = $1.split(",")
-    result = {}
-    split.each do |e|
-      s2 = e.split(":")
-      key = s2[0]
-      value = s2[1]
-      if value == "nil" then
-        result[key] = nil
-      elsif value == "true" then
-        result[key] = true
-      elsif value == "false" then
-        result[key] = false
-      elsif value =~ /\"(.*)\"/
-        result[key] = $1
-      end
-    end
-    result
-  end
-
-  def self.parse_continuation_point(line)
-    split = line.split(";")
-    doc_number = Integer(split[0])
-    num_docs = Integer(split[1])
-    state = CoreEnumerationAlgo.parse_state(split[2])
-    assignment = CoreEnumerationAlgo.parse_assignment(split[3])
-    new_assignment = CoreEnumerationAlgo.parse_assignment(split[4])
-    return [doc_number, num_docs, state, assignment, new_assignment]
-  end
-
-  # TODO: logging
-  
-  def display_combination_space(assignment, vars, switched, blocked)
-    # enumerate_fixing_clause(assignment, vars, switched) + enumerate_blocking_clause(vars[switched][0], vars[switched][1], blocked)
-
-    # => all vars left of switched are fixed to their assigned values (when the value is nil, then they are empty)
-    # => the switched var is prevented to have any of the blocked values
-    res = switched.times.to_a.map do |index|
-      var_xpath = vars[index][0]
-      "#{var_xpath}=#{assignment[var_xpath].inspect}"
-    end + ["#{vars[switched][0]}\{#{blocked.map{|v| v.inspect}.join(",")}}"]
-    "{#{res.join(", ")}}"
-  end
-
-  # core algo
-  def enumerate(progress, switched, vars, assignment, message, key, output_dir, config, options, state = [nil] * vars.size())
-    enumerate(progress, switched+1, vars, assignment, message, key, output_dir, config, options, state) if switched < vars.size()-1
-    is_finite_state = keys_are_finite_state(vars)
-    if state[switched] == nil then
-      blocked = [assignment[vars[switched][0]]]
-      state[switched] = blocked
-    else
-      blocked = state[switched] + [assignment[vars[switched][0]]]
-      state[switched] = blocked
-    end
-    found = true
-    while found do
-      found = false
-      # - fix all vars left of switched to their values in assignment
-      # - block switched from obtaining the values in blocked
-      # progress.print_line "DEBUG: #{summarize_fixing_clause(assignment, vars, switched)}, #{summarize_blocking_clause(vars[switched][0], blocked)}"
-      clauses = enumerate_fixing_clause(assignment, vars, switched) + enumerate_blocking_clause(vars[switched][0], vars[switched][1], blocked)
-      output_doc_filename = prepare_output_doc_filename(@num_tries, output_dir)
-      new_assignment = generate_sparse_doc(progress, message, key, config, options, clauses, output_doc_filename)
-      generated_doc_number = @num_tries
-      store_continuation_point(generated_doc_number, state, assignment, new_assignment, "#{output_dir}/#{CONTINUATION_FILE}")
-      if is_finite_state then
-        progress.report_progress(determine_progress(state, vars))
-      else
-        progress.report_progress(nil)
-      end
-      @num_tries += 1
-      if new_assignment then # sat
-        @num_docs += 1
-        @on_new_doc.call(generated_doc_number, output_doc_filename, new_assignment) if @on_new_doc
-        if switched < vars.size()-1 then
-          enumerate(progress, switched+1, vars, new_assignment, message, key, output_dir, config, options, state)
-        end
-        blocked << new_assignment[vars[switched][0]]
-        state[switched] = blocked
-        found = true
-      else # unsat
-        @on_unsat.call(generated_doc_number, log_filename_for_output_filename(output_doc_filename), display_combination_space(assignment, vars, switched, blocked)) if @on_unsat
-        found = false
-      end
-    end
-    state[switched] = nil
-  end
-  
-  # NOTE: can only be called when all key-fields are finite-state
-  def is_final_state(state, new_assignment, key_fields_list)
-    return state[1,state.size()] == [nil] * (state.size()-1) &&
-       state[0].to_set == values_for_key_field(key_fields_list[0][1]).to_set &&
-       new_assignment.empty?
-  end
-
-  # NOTE: can only be called when all key-fields are finite-state
-  def determine_progress(state, vars)
-    combinations_per_value = [nil] * vars.size()
-    comb = 1
-    vars.size().times.to_a.reverse_each do |i|
-      combinations_per_value[i] = comb
-      comb = values_for_key_field(vars[i][1]).size() * comb
-    end
-
-    blocked_combinations = 0
-    state.size().times do |i|
-      if state[i] then
-        blocked_combinations += state[i].size() * combinations_per_value[i]
-      end
-    end
-    blocked_combinations
-  end
-
-  def num_combinations(key_def)
-    if key_def[:type] == :field then
-      if key_def[:datatype] == :enum then
-        key_def[:values].size() + (key_def[:optional] ? 1 : 0)
-      elsif key_def[:datatype] == :int then
-        if key_def.has_key?(:min) and key_def.has_key?(:max) and key_def[:min] <= key_def[:max] then
-          1 + key_def[:max] - key_def[:min] + (key_def[:optional] ? 1 : 0)
-        else
-          nil
-        end
-      else
-        nil
-      end
-    elsif key_def[:type] == :structure then
-      2
-    end
-  end
-  
-  def keys_are_finite_state(list)
-    list.each do |field|
-      name = field[0]
-      if field[1][:type] == :field then
-        if field[1][:datatype] == :enum then
-          # enums are finite-state
-        elsif field[1][:datatype] == :int and field[1].has_key?(:min) and field[1].has_key?(:max) and field[1][:min] <= field[1][:max] then
-          # ints are finite-state when both bounds are given and :min <= :max
-        else
-          return false
-        end
-      end
-    end
-    return true
-  end
-
-  def generate_docs_for_key(message, key, output_dir, config, options)
-
-    number_of_combinations = calc_combinations(message, key)
-    if number_of_combinations.is_a?(String) then
-      progress = ProgressBar.new(100000000000000)
-    else
-      progress = ProgressBar.new(number_of_combinations)
-    end
-    begin
-    ensure
-    end
-    progress.report_progress(0.0)
-    progress.print_line "generating sparse documents efficiently..."
-    progress.print_line "there is a theoretical maximum of #{number_of_combinations} combinations."
-    progress.print_line "Note, however, that we are generating only those documents that are valid, which are usually much less."
-
-    progress.print_line "preparing output dir #{output_dir}..."
-    if options[:continue] then
-      unless Dir.exist?(output_dir) then
-        raise FatalError.new("FATAL: directory #{output_dir} not found. There seems to be nothing to continue from...")
-      end
-      log_mode = "a"
-    else
-      unless Dir.exist?(output_dir) then
-        FileUtils.mkdir_p(output_dir)
-      end
-      log_mode = "w"
-    end
-    log_filename = "#{output_dir}/#{COMBINATION_LOG}"
-
-    key_fields = collect_key_fields(key, message)
-    key_fields_list = key_fields.each_pair.to_a
-    key_fields_list.sort! {|a,b| num_combinations(b[1]) <=> num_combinations(a[1]) }
-
-    if options[:continue] then
-      stored_key_fields_str = File.read("#{output_dir}/#{VARS_LOG}")
-      unless key_fields_list.to_yaml == stored_key_fields_str then
-        raise FatalError.new("vars are differing! please specify the same key in order to use -continue!")
-      end
-      progress.print_line "continuing enumerating documents for key #{key}!"
-    else
-      # store vars
-      File.open("#{output_dir}/#{VARS_LOG}", "w") do |f|
-        f.puts(key_fields_list.to_yaml)
-      end
-    end
-
-    cont_filename = "#{output_dir}/#{CONTINUATION_FILE}"
-    if options[:continue] then
-      unless File.exist?(cont_filename) then
-        raise FatalError.new("FATAL: file #{cont_filename} not found. There seems to be no continuation point to continue from...")
-      end
-      doc_number, num_docs, state, assignment, new_assignment = CoreEnumerationAlgo.parse_continuation_point(File.read(cont_filename).chomp)
-      if is_final_state(state, new_assignment, key_fields_list) then
-        progress.report_progress(number_of_combinations)
-        raise FatalError.new("FATAL: The enumeration is finished. There is nothing left to do. Resuming the enumeration hence does not make sense.")
-      end
-      progress.print_line "RESUMING enumeration from the following continuation point:"
-      progress.print_line "|doc_number=#{doc_number}"
-      progress.print_line "|num_docs=#{num_docs}"
-      progress.print_line "|state=#{state.inspect}"
-      progress.print_line "|assignment=#{assignment.inspect}"
-      progress.print_line "|new_assignment=#{new_assignment.inspect}"
-      if keys_are_finite_state(key_fields_list) then
-        progress.report_progress(determine_progress(state, key_fields_list))
-      else
-        progress.report_progress(nil)
-      end
-
-      if new_assignment && !new_assignment.empty? then
-        assignment = new_assignment
-      end
-
-      # restore state at continuation point
-      @num_tries = doc_number+1 # we do not want to overwrite the last document written before the STOP
-      number_of_combinations += 1 # account for this in the safety mechanism
-      @num_docs = num_docs
-      doc_counter = 0
-      File.open(log_filename, log_mode) do |log|
-
-        @on_new_doc = Proc.new { |num_doc, doc_filename, combination|
-          # combination logging
-          log.puts "#{doc_filename} <- {#{combination.map{|k,v| "#{k}=#{v.inspect}"}.join(", ")}}"
-
-          doc_counter += 1
-          if options[:max_num_docs] && doc_counter >= options[:max_num_docs] then
-            raise NormalProgramTermination.new("-max-num-docs exceeded. Stopping!")
-          end
-
-          # safety mechanism
-          if @num_docs > number_of_combinations then
-            raise RuntimeError.new("This should never happen! Number of combinations exceeded. #{@num_docs} documents generated, but there are only #{number_of_combinations} possible combinations of the key vars.")
-          end
-        }
-        @on_unsat = Proc.new { |num_doc, log_filename, combination_str|
-              log.puts "CONTRADICTION (see: #{log_filename}) <- #{combination_str}"
-              progress.print_line "doc#{num_doc} NOT generated"
-              progress.print_line "No more valid combinations of the free vars could be found. -> Switching"
-        }
-
-        # core algo - resume from state
-        enumerate(progress, 0, key_fields_list, assignment, message, key, output_dir, config, options, state)
-      end
-    else
-      # init safety mechanism
-      @num_tries = 0
-      @num_docs = 0
-      doc_counter = 0
-      # first try - a freely generated document
-      File.open(log_filename, log_mode) do |log|
-        output_doc_filename = prepare_output_doc_filename(@num_tries, output_dir)
-        key_values = generate_doc_return_key_values(progress, message, key, output_doc_filename, config, options)
-        log.puts "#{output_doc_filename} <- {#{key_values.map{|k,v| "#{k}=#{v.inspect}"}.join(", ")}}"
-        doc_counter += 1
-        @num_tries += 1
-        @on_new_doc = Proc.new { |num_doc, doc_filename, combination|
-          # combination logging
-          log.puts "#{doc_filename} <- {#{combination.map{|k,v| "#{k}=#{v.inspect}"}.join(", ")}}"
-
-          doc_counter += 1
-          if options[:max_num_docs] && doc_counter >= options[:max_num_docs] then
-            raise NormalProgramTermination.new("-max-num-docs exceeded. Stopping!")
-          end
-
-          # safety mechanism
-          unless number_of_combinations.is_a?(String) then
-            if @num_docs > number_of_combinations then
-              raise RuntimeError.new("This should never happen! Number of combinations exceeded. #{@num_docs} documents generated, but there are only #{number_of_combinations} possible combinations of the key vars.")
-            end
-          end
-        }
-        @on_unsat = Proc.new { |num_doc, log_filename, combination_str|
-              log.puts "CONTRADICTION (see: #{log_filename}) <- #{combination_str}"
-              progress.print_line "doc#{num_doc} NOT generated"
-              progress.print_line "No more valid combinations of the free vars could be found. -> Switching"
-        }
-
-        # core algo - start enumeration
-        enumerate(progress, 0, key_fields_list, key_values, message, key, output_dir, config, options)
-      end
-      
-      progress.print_line "No more valid documents could be found. -> Exiting"
-      progress.report_progress(number_of_combinations)
-    end
-  end
-end
-
-def introduce_parent_links(message, parent = nil, prefix = nil)
-  if parent != nil then
-    message[:parent_link] = parent
-  end
-  if prefix != nil then
-    message[:struct_xpath_prefix] = prefix
-  end
-  message.each_pair do |key, value|
-    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
-    if value[:type] == :structure then
-      if prefix then
-        new_prefix = "#{prefix}/#{key}"
-      else
-        new_prefix = key
-      end
-      introduce_parent_links(value[:ref], message, new_prefix)
-    end
-  end
-end
-
-def validate_model(message, prefix=nil)
-
-  message.each_pair do |key, value|
-    if key == :validation_rules then
-      value.each_pair do |vr_key, vr_value|
-        unless vr_value[:text] && vr_value[:text] != "" then
-          puts "INFO: Model Problem: validation rule #{vr_key} does not have a textual representation."
-        end
-        unless vr_value[:smtlib] && vr_value[:smtlib] != "" then
-          puts "WARN: Model Problem: validation rule #{vr_key} does not have an SMTLIB representation. It will hence be omitted!"
-        end
-      end
-    elsif key == :predicates then
-      value.each_pair do |mkey, mvalue|
-        next unless mvalue
-        unless mvalue[:text] && mvalue[:text] != "" then
-            puts "INFO: Model Problem: structure #{prefix}'s predicate #{mkey} does not have a textual representation."
-        end
-        unless mvalue[:smtlib] && mvalue[:smtlib] != "" then
-            puts "WARN: Model Problem: structure #{prefix}'s predicate #{mkey} does not have an SMTLIB representation. It will be regarded as always true."
-        end
-      end
-    elsif key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
-      next
-    else
-      if value[:type] == :structure then
-        unless value[:ref] then
-          raise RuntimeError.new("Model Problem: encountered structure without a :ref (key=#{key.inspect}, value=#{value.inspect})")
-        end
-        if prefix then
-          new_prefix = "#{prefix}/#{key}"
-        else
-          new_prefix = key
-        end
-        validate_model(value[:ref], new_prefix)
-      elsif value[:type] == :field then
-        unless value[:datatype] then
-          raise RuntimeError.new("Model Problem: encountered field without a :datatype (key=#{key.inspect}, value=#{value.inspect})")
-        end
-        if value[:datatype] == :enum then
-          unless value[:values] then
-            raise RuntimeError.new("Model Problem: encountered enum field without a :values (key=#{key.inspect}, value=#{value.inspect})")
-          end
-        end
-      end
-    end  
-  end
-end
-
-def validate_doc_against_schema(struct, doc)
-  root = doc.root
-  sdef = struct[root.name.to_sym]
-  unless sdef[:type] == :structure then
-    puts "ERROR: schema validation failed: root element should be a structure"
-  end  
-  # first, check that each doc element is defined for the struct
-  validate_doc_struct_against_schema(sdef[:ref], root, root.name)
-  # second, check that all required elements from the struct are actually present in the doc
-  validate_schema_against_doc_struct(sdef[:ref], root, root.name)
-end
-
-def validate_doc_struct_against_schema(struct, doc, prefix = nil)
-  doc.children.to_a.filter{|x| x.is_a?(Nokogiri::XML::Element) }.each do |c|
-    fdef = struct[c.name.to_sym]
-    if prefix then
-      xpath = "#{prefix}/#{c.name}"
-    else
-      xpath = c.name
-    end
-    if fdef
-      if fdef[:type] == :structure then
-        return unless validate_doc_struct_against_schema(fdef[:ref], c, xpath)
-      end
-    else
-      if prefix then
-        xpath = "#{prefix}/#{c.name}"
-      else
-        xpath = c.name
-      end
-      raise FatalError.new("ERROR: schema validation failed: #{c.name} of #{xpath} is present in document, but not defined in schema.")
-    end
-  end
-end
-
-def validate_schema_against_doc_struct(struct, doc, prefix = nil)
-  struct.each do |key, value|
-    next if key == :additional_smtlib || key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix
-      if prefix then
-        xpath = "#{prefix}/#{key}"
-      else
-        xpath = key
-      end
-    if value[:type] == :structure then
-      node = doc.at_xpath(key.to_s)
-      if node then
-        validate_schema_against_doc_struct(value[:ref], node, xpath)
-      else
-        if value[:optional] == false then
-          raise FatalError.new("ERROR: schema validation failed: non-optional structure #{xpath} not found in doc.")
-        end
-      end
-    elsif value[:type] == :field then
-      if value[:optional] == false then
-        unless doc.at_xpath(key.to_s) then
-          raise FatalError.new("ERROR: schema validation failed: non-optional field #{xpath} not found in doc.")
-        end
-      end
-    elsif value[:type] == :list then
-      if value[:optional] == false then
-        unless doc.at_xpath(key.to_s) then
-          raise FatalError.new("ERROR: schema validation failed: non-optional list #{xpath} not found in doc.")
-        end
-      end
-      list = doc.at_xpath(key.to_s)
-      if list then
-        list.children.to_a.each do |c|
-          next unless c.is_a?(Nokogiri::XML::Element)
-          unless c.name == value[:xpath_element] then
-            raise FatalError.new("ERROR: schema validation failed: encountered element #{c.name} in list #{xpath}, which should only contain #{value[:xpath_element]} elements.")
-          end
-        end
-      end
-    else
-      raise RuntimeError.new("Model error: encountered element #{xpath} of unknown type: #{value[:type]}.")
-    end
-  end  
-end
-
-def translate_doc_to_SMTLIB(progress, solver, struct, doc, hash = {})
-  options = hash[:options]
-  config = hash[:config]
-
-  solver.to_solver(progress, "; --- Document: ---")
-  translate_struct_in_doc_to_SMTLIB(progress, solver, struct, doc, nil, hash)
-end
-
-def translate_struct_in_doc_to_SMTLIB(progress, solver, struct, doc, prefix, hash = {})
-  options = hash[:options]
-  config = hash[:config]
-
-  struct.each do |key, value|
-    next if key == :additional_smtlib || key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix
-    if prefix then
-      xpath = "#{prefix}/#{key.to_s}"
-    else
-      xpath = key.to_s
-    end
-    if value[:type] == :structure then
-      node = doc.at_xpath(xpath)
-      if node then
-        info = {
-          :assertion_type => :doc_structure_exists,
-          :xpath => xpath,
-        }
-        assertionID = "doc-#{xpath.gsub('/', '-')}"
-        solver.associateAssertionIDWith(assertionID, info)
-        solver.to_solver(progress, "(assert (! (= #{exists_var_for_structure(xpath)} true) :named #{assertionID}))")
-
-        translate_struct_in_doc_to_SMTLIB(progress, solver, value[:ref], doc, xpath, hash)
-      else
-        info = {
-          :assertion_type => :doc_structure_not_exists,
-          :xpath => xpath,
-        }
-        assertionID = "doc-#{xpath.gsub('/', '-')}"
-        solver.associateAssertionIDWith(assertionID, info)
-        solver.to_solver(progress, "(assert (! (= #{exists_var_for_structure(xpath)} false) :named #{assertionID}))")
-      end
-    elsif value[:type] == :field then
-      node = doc.at_xpath(xpath)
-      if node then
-        fvalue = node.text
-        info = {
-          :assertion_type => :doc_field_filled,
-          :xpath => xpath,
-          :value => fvalue
-        }
-        fvalue_str = value_of_type(fvalue, value[:datatype])
-        assertionID = "doc-#{xpath.gsub('/', '-')}"
-        solver.associateAssertionIDWith(assertionID, info)
-        solver.to_solver(progress, "(assert (! (and (= #{filled_var_for_field(xpath)} true) (= #{value_var_for_field(xpath)} #{fvalue_str})) :named #{assertionID}))")
-      else
-        info = {
-          :assertion_type => :doc_field_empty,
-          :xpath => xpath,
-        }
-        assertionID = "doc-#{xpath.gsub('/', '-')}"
-        solver.associateAssertionIDWith(assertionID, info)
-        solver.to_solver(progress, "(assert (! (= #{filled_var_for_field(xpath)} false) :named #{assertionID}))")
-      end
-    elsif value[:type] == :list then
-      node = doc.at_xpath(xpath)
-      if node then
-        fvalues = []
-        node.children.to_a.each do |c|
-          next unless c.is_a?(Nokogiri::XML::Element)          
-          fvalues << c.text
-        end
-        info = {
-          :assertion_type => :doc_list_filled,
-          :xpath => xpath,
-          :values => fvalues
-        }
-        fvalue_str = value_of_type(fvalues, value[:datatype])
-        assertionID = "doc-#{xpath.gsub('/', '-')}"
-        solver.associateAssertionIDWith(assertionID, info)
-        clauses = []
-        fvalues.each_with_index do |fvalue, index|
-          clauses << "(= #{value_var_for_list(xpath,index)} #{value_of_type(fvalue, value[:datatype])})"
-        end
-        solver.to_solver(progress, "(assert (! (and (= #{filled_var_for_field(xpath)} true) (= #{size_var_for_list(xpath)} #{fvalues.size}) #{clauses.join(' ')}) :named #{assertionID}))")
-      else
-        info = {
-          :assertion_type => :doc_field_empty,
-          :xpath => xpath,
-        }
-        assertionID = "doc-#{xpath.gsub('/', '-')}"
-        solver.associateAssertionIDWith(assertionID, info)
-        solver.to_solver(progress, "(assert (! (= #{filled_var_for_field(xpath)} false) :named #{assertionID}))")
-      end
-    else
-      raise RuntimeError.new("Model Error: encountered unknown type #{value[:type].inspect}")
-    end
-  end
-end
-
-def validate_doc(progress, struct, doc, options, config)
-  options[:solverLog] = "validation.smt2"
-
-  z3path = config[:z3path] || DEFAULT_Z3_PATH
-  z3 = Z3Solver.new(progress, z3path)
-  begin
-    result = z3.query(progress, options) do |solver|
-      set_standard_options(progress, solver)
-      translate_structure_to_SMTLIB(progress, solver, struct, :options => options, :config => config)
-      translate_doc_to_SMTLIB(progress, solver, struct, doc, :options => options, :config => config)
-      solver.to_solver(progress, "")
-    end
-    unless result == true then
-      raise FatalError.new("VALIDATION FAILED! SMTLIB log was written to #{options[:solverLog]}.\n\n#{result}")
-    end
-  ensure
-    z3.close()
-  end
-
-  return true
-end
-
-def generate(message, argv, message_config_filename = nil)
-  if ARGV[0] == "--help" || ARGV[0] == "-help" || ARGV[0] == "help" then
-    puts USAGE
-    exit(0)
-  end
-
-  options = { :negated_validation_rules => [], :date_now => (Date.today - Time.at(0).to_date).to_i, :timestamp_now => Time.now().to_i }
-  
-  opt = argv.shift
-  while opt do
-    if opt == "-negate" then
-      rule = argv.shift
-      options[:negated_validation_rules] << rule
-    elsif opt == "-list-keys" then
-      options[:list_keys] = true
-    elsif opt == "-list" then
-      options[:list] = argv.shift
-    elsif opt == "-list-validation-rules" then
-      options[:list_validation_rules] = true      
-    elsif opt == "-generate-docs-from" then
-      options[:combination_source] = argv.shift
-    elsif opt == "-skip" then
-      options[:skip] = Integer(argv.shift)
-    elsif opt == "-generate-docs-for-key" then
-      options[:docs_for_key] = argv.shift
-    elsif opt == "-count-docs-for-key" then
-      options[:count_docs_for_key] = argv.shift
-    elsif opt == "-continue" then
-      options[:continue] = true
-    elsif opt == "-max-num-docs" then
-      options[:max_num_docs] = Integer(argv.shift)
-      puts "flag -max-num-docs used. Will stop after generating #{options[:max_num_docs]} documents."
-    elsif opt == "-validate" then
-      if options.has_key?(:validate_files) then
-        options[:validate_files] << argv.shift
-      else
-        options[:validate_files] = [argv.shift]
-      end
-    else
-      puts "FATAL: unknown option #{opt.inspect}."
-      puts USAGE
-      raise FatalError.new("")
-    end
-    opt = argv.shift
-  end
-
-  unless options[:negated_validation_rules].empty? then
-    puts "negating rules: #{options[:negated_validation_rules].join(", ")}"
-
-    rules = collect_validation_rule_names(message)
-    unknown_negated_rules = options[:negated_validation_rules].filter { |rule_name| !rules.include?(rule_name) } 
-    unless unknown_negated_rules.empty? then  
-      throw RuntimeError.new("unknown negated rules #{unknown_negated_rules.join(", ")}")
-    end
-
-  end
-
-  if File.exist?(CONFIG_FILE) then
-    config = YAML.load_file(CONFIG_FILE)
-  else
-    config = DEFAULT_CONFIG
-    File.open(CONFIG_FILE, "w") do |f|
-      f.puts(config.to_yaml)
-    end
-    puts "No config file found. Default config written to #{CONFIG_FILE}"
-  end
-  if message_config_filename then
-    message_config = YAML.load_file(message_config_filename)
-    config = config.merge(message_config)
-  end
-
-  introduce_parent_links(message)
-
-  if options[:validate_files] then
-    puts "using #{options[:date_now]} as [Date.now]"
-    puts "using #{options[:timestamp_now]} as [Timestamp.now]"
-    options[:validate_files].each do |filename|
-      puts "validating file #{filename}"
-      doc = Nokogiri::XML(File.read(filename))
-      validate_doc_against_schema(message, doc)
-      progress = ProgressBar.new(1)
-      validate_doc(progress, message, doc, options, config)
-    end
-  elsif options[:count_docs_for_key] then
-    puts "there are #{calc_combinations(message, options[:count_docs_for_key])} combinations."
-  elsif options[:docs_for_key] then
-    puts "using #{options[:date_now]} as [Date.now]"
-    puts "using #{options[:timestamp_now]} as [Timestamp.now]"
-    validate_model(message)
-    CoreEnumerationAlgo.new().generate_docs_for_key(message, options[:docs_for_key], OUTPUT_DIR, config, options)
-  elsif options[:list_keys] then
-    list_keys(message)
-  elsif options[:list] then
-    list_fields_and_structures(message, options[:list])
-  elsif options[:list_validation_rules] then
-    list_validation_rules(message)
-  else
-    puts "using #{options[:date_now]} as [Date.now]"
-    puts "using #{options[:timestamp_now]} as [Timestamp.now]"
-    validate_model(message)
-    progress = ProgressBar.new(1)
-    generate_doc(progress, message, OUTPUT_FILE, config, options)
-  end
-end
+#!/usr/bin/ruby
+
+require 'date'
+require 'fileutils'
+require 'tmpdir'
+require 'yaml'
+require 'time'
+
+require 'nokogiri'
+
+require_relative 'solver-lib'
+require_relative 'progress'
+require_relative 'regexp-to-smtlib'
+
+
+CHUNK_SIZE = 10000
+DEFAULT_Z3_PATH = "z3"
+
+CONFIG_FILE = "config.yml"
+OUTPUT_FILE = "result.xml"
+CONTINUATION_FILE = "continuation.log"
+COMBINATION_LOG = "combinations.log"
+VARS_LOG = "vars.log"
+
+OUTPUT_DIR = "result"
+LISTS_DIR = "lists"
+QUERY_DIR = "queries"
+PARAMETERIZED_QUERY_DIR = "parameterized_queries"
+XML_TEMPLATE = ""
+
+DEFAULT_CONFIG = {
+  :z3path => "z3",
+  :generation_system_mode => "Production",
+  :generation_system => "Production"
+}
+
+USAGE = <<EOF
+usage:
+
+GENERATING A SINGLE MESSAGE:
+
+  ruby #{$PROGRAM_NAME} -list-validation-rules
+  - outputs a list of all validation rule names that have been specified in #{$PROGRAM_NAME}.
+
+  ruby #{$PROGRAM_NAME} <options>
+  - generates a message of the given type, which
+    - is valid XML
+    - adheres to the schema, and
+    - satisfies all validation rules specified for messages of this type.
+  where <options> is a combination of
+    -negate <validation_rule> - negates the named validation rule. This causes the resulting message to still satisfy all other validation rules, but to violate this one.
+  NOTE: the result can be found in #{OUTPUT_FILE}
+  
+GENERATING MESSAGES FOR ALL COMBINATIONS:
+
+  ruby #{$PROGRAM_NAME} -list-keys
+  - outputs a list of all keys that have been used in #{$PROGRAM_NAME} to mark fields or structures.
+
+  ruby #{$PROGRAM_NAME} -list <key>
+  - outputs a list of all fields and structures that have been marked with <key> in #{$PROGRAM_NAME}.
+
+  ruby #{$PROGRAM_NAME} -count-docs-for-key <key>
+  - calculates the number of combinations of all fields and structures marked with <key>
+
+  ruby #{$PROGRAM_NAME} -generate-docs-for-key <key> <options>
+  - generates all valid messages of the given type that can be derived from combinations of all the fields and structures marked with <key>.
+  where <options> is a combination of
+    -continue - continues a previously interrupted generation process. Must use the same <key> the the interrupted generation.ARGF
+    -max-num-docs <limit> - stops generation process after generating <limit> documents.
+  NOTE: enumerating large combination-spaces may take a long time.
+        - Use -count-docs-for-key to measure the size of the combination-space beforehand.
+        - Also: you can interrupt the generation process at any time using Ctrl+C. The intermediate result is stored in the directory #{OUTPUT_DIR} and the generation process can
+          be restarted later by adding -continue to the command line.
+  NOTE: do not expect the number of generated documents to equal the number of combinations as combinations may not have any valid message instances (for instance when they contradict the validation rules).
+  NOTE: the result can be found in the directory #{OUTPUT_DIR}
+
+VALIDATING THE MODEL:
+
+  ruby #{$PROGRAM_NAME} -validate <file.xml>
+  - reads in the given XML file and validates it against the model - that is, it checks whether it conforms to the specified XML schema and to the validation rules.
+EOF
+
+class FatalError < RuntimeError
+end
+
+class NormalProgramTermination < RuntimeError
+end
+
+def raw_field(xpath)
+  path_str = "base"
+  path_str = xpath.gsub("/", "-") if xpath
+  "field-#{path_str}"
+end
+
+def filled_var_for_field(xpath)
+  "#{raw_field(xpath)}-filled"
+end
+
+def value_var_for_field(xpath)
+  "#{raw_field(xpath)}-value"
+end
+
+def exists_var_for_structure(xpath, index = nil)
+  if index then
+    struct_str = "base"
+    struct_str = xpath.gsub("/", "-") if xpath
+    struct_str = "#{struct_str}-#{index}" if index
+    "struct-#{struct_str}-exists"
+  else
+    struct_str = "base"
+    struct_str = xpath.gsub("/", "-") if xpath
+    "struct-#{struct_str}-exists"
+  end
+end
+
+def size_var_for_list(xpath)
+  "#{raw_field(xpath)}-size"
+end
+
+def value_var_for_list(xpath, index)
+  "#{raw_field(xpath)}-index-#{index}-value"
+end
+
+def translate_datatype_to_SMTLIB(datatype)
+  case datatype
+  when :string
+    return "String"
+  when :int
+    return "Int"
+  when :bool
+    return "Bool"
+  when :date
+    return "Int"
+  when :timestamp
+    return "Int"
+  when :enum
+    return "String"
+  end
+end
+
+def translate_value_to_SMTLIB(value)
+  if value.is_a?(Integer) then
+    return value.to_s
+  elsif value.is_a?(String) then
+    return value.inspect
+  elsif value.is_a?(Date) then
+    return (value - (Time.at(0).to_date)).to_i  # days since 1970.01.01
+  elsif value.is_a?(Time) then
+    return value.to_i
+  else
+    raise RuntimeError.new("cannot translate value to SMTLIB: #{value.inspect}")
+  end
+end
+
+def min_constraint_for_expr(expr, lower_bound)
+  "(>= #{expr} #{lower_bound})"
+end
+
+def max_constraint_for_expr(expr, upper_bound)
+  "(<= #{expr} #{upper_bound})"
+end
+
+def regex_constraint_for_expr(expr, regex)
+  "(str.in.re #{expr} #{Regexp_to_SMTLIB.translate_regexp_to_SMTLIB(regex)})"
+end
+
+def maxLength_contraint_for_expr(expr, maxLength)
+  "(<= (str.len #{expr}) #{maxLength})"
+end
+
+def oneOf_contraint_for_expr(expr, values)
+  list = values.map do |val|
+    "(= #{expr} \"#{val}\")"
+  end
+  "(or #{list.join(" ")})"
+end
+
+def enum_constraint_for_expr(expr, values)
+  "(or #{values.map{|x| "(= #{expr} #{translate_value_to_SMTLIB(x)})"}.join(" ")})"
+end
+
+def translate_field_def_to_SMTLIB_constraints(progress, solver, fielddef, xpath, struct_var)
+  varname = raw_field(xpath)
+  if fielddef[:optional] == nil || fielddef[:optional] == false then
+    # field is required
+    assertionID = "required-#{varname}"
+    info = {
+      :assertion_type => :constraint,
+      :xpath => xpath,
+      :constraint_type => :required
+    }
+    solver.associateAssertionIDWith(assertionID, info)
+    solver.to_solver(progress, "(assert (! (=> #{struct_var} #{varname}-filled) :named #{assertionID}))")
+  end
+  datatype = fielddef[:datatype]
+  case datatype
+  when :int
+    if fielddef[:min] then
+        assertionID = "min-#{varname}"
+        info = {
+          :assertion_type => :constraint,
+          :xpath => xpath,
+          :datatype => :int,
+          :value => fielddef[:min],
+          :constraint_type => :min
+        }
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{min_constraint_for_expr("#{varname}-value", translate_value_to_SMTLIB(fielddef[:min]))}) :named #{assertionID}))") 
+    end
+    if fielddef[:max] then
+        assertionID = "max-#{varname}"
+        info = {
+          :assertion_type => :constraint,
+          :xpath => xpath,
+          :datatype => :int,
+          :value => fielddef[:max],
+          :constraint_type => :max
+        }
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{max_constraint_for_expr("#{varname}-value", translate_value_to_SMTLIB(fielddef[:max]))}) :named #{assertionID}))")
+    end
+  when :string
+    if fielddef[:regex] then
+        assertionID = "regex-#{varname}"
+        info = {
+          :assertion_type => :constraint,
+          :xpath => xpath,
+          :datatype => :string,
+          :pattern => fielddef[:regex],
+          :constraint_type => :regex
+        }
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{regex_constraint_for_expr("#{varname}-value", fielddef[:regex])}) :named #{assertionID}))") 
+    end 
+    # TODO: minLength?
+    if fielddef[:maxLength] then
+      assertionID = "maxLength-#{varname}"
+        info = {
+          :assertion_type => :constraint,
+          :xpath => xpath,
+          :datatype => :string,
+          :value => fielddef[:maxLength],
+          :constraint_type => :maxLength
+        }
+      solver.associateAssertionIDWith(assertionID, info)
+      solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{maxLength_contraint_for_expr("#{varname}-value", fielddef[:maxLength])}) :named #{assertionID}))") 
+    end
+    if fielddef[:from_list] then
+      assertionID = "oneOf-#{varname}"
+      list_filename = "#{LISTS_DIR}/#{fielddef[:from_list]}"
+      info = {
+        :assertion_type => :constraint,
+        :xpath => xpath,
+        :datatype => :string,
+        :filename => list_filename,
+        :constraint_type => :from_list
+      }
+      unless File.exist?(list_filename)
+        raise RuntimeError.new("could not find list file #{list_filename}.")
+      end
+      values = File.read(list_filename).split("\n").map{|v| v.chomp()}
+      solver.associateAssertionIDWith(assertionID, info)
+      solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{oneOf_contraint_for_expr("#{varname}-value", values)}) :named #{assertionID}))") 
+    end
+  when :enum
+    assertionID = "enum-#{varname}"
+    info = {
+      :assertion_type => :constraint,
+      :xpath => xpath,
+      :datatype => :enum,
+      :values => fielddef[:values],
+      :constraint_type => :values
+    }
+    solver.associateAssertionIDWith(assertionID, info)
+    solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{enum_constraint_for_expr("#{varname}-value", fielddef[:values])}) :named #{assertionID}))") 
+  end
+end
+
+def translate_list_def_to_SMTLIB_constraints(progress, solver, fielddef, varname)
+  datatype = fielddef[:datatype]
+  case datatype
+  when :int
+    if fielddef[:min] then
+        fielddef[:model_maxLength].times do |i|
+          solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{min_constraint_for_expr("#{varname}-index-#{i}-value", translate_value_to_SMTLIB(fielddef[:min]))}) :named list-min-#{varname}-list-index-#{i}))") 
+        end
+    end
+    if fielddef[:max] then
+        fielddef[:model_maxLength].times do |i|
+          solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{max_constraint_for_expr("#{varname}-index-#{i}-value", translate_value_to_SMTLIB(fielddef[:max]))}) :named list-max-#{varname}-list-index-#{i}))") 
+        end
+    end
+  when :string
+    if fielddef[:regex] then
+        fielddef[:model_maxLength].times do |i|
+          solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{regex_constraint_for_expr("#{varname}-index-#{i}-value", fielddef[:regex])}) :named list-regex-#{varname}-index-#{i}))") 
+        end
+    end 
+    # TODO: minLength?
+    if fielddef[:maxLength] then
+        fielddef[:model_maxLength].times do |i|
+          solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{maxLength_contraint_for_expr("#{varname}-index-#{i}-value", fielddef[:maxLength])}) :named list-maxLength-#{varname}-index-#{i}))") 
+        end
+    end 
+  when :enum
+    fielddef[:model_maxLength].times do |i|
+      solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{enum_constraint_for_expr("#{varname}-index-#{i}-value", fielddef[:values])}) :named list-enum-#{varname}-index-#{i}))") 
+    end
+  end
+end
+
+def default_value_for_fielddef(fielddef)
+  datatype = fielddef[:datatype]
+  case datatype
+  when :int
+    return "0"
+  when :string
+    return "".inspect
+  when :date
+    return "0"
+  when :timestamp
+    return "0"
+  when :enum
+    return "".inspect
+  else
+    raise RuntimeError.new("unknown datatype: #{datatype.inspect}")
+  end
+end
+
+def translate_structure_to_SMTLIB(progress, solver, struct, hash = {})
+  prefix = hash[:prefix]
+  config = hash[:config]
+  options = hash[:options]
+  translate_substructure_to_SMTLIB(progress, solver, struct, hash)
+  translate_validation_rules_to_SMTLIB(progress, solver, struct, prefix, config, options)
+  if options[:add_SMTLIB] then
+    solver.to_solver(progress, "; --- additional SMTLIB Code: ---")
+    solver.to_solver(progress, options[:add_SMTLIB])
+  end
+end
+
+def translate_substructure_to_SMTLIB(progress, solver, struct, hash = {})
+  prefix = hash[:prefix]
+  struct_varname = hash[:struct_varname]
+  config = hash[:config]
+  options = hash[:options]
+  solver.to_solver(progress, "; --- Structure: ---")
+  struct.each do |field, definition|
+    if field == :additional_smtlib then
+      solver.to_solver(progress, "; additional SMTLIB of structure #{prefix}")
+      solver.to_solver(progress, translate_validation_rule_to_SMTLIB(definition, struct, prefix, config, options, "<additional SMTLIB of structure #{prefix}>"))
+    elsif field == :validation_rules || field == :predicates || field == :parent_link || field == :struct_xpath_prefix
+      next 
+    else
+      solver.to_solver(progress, "; #{field.to_s}")
+      if definition[:type] == :field then
+        display_prefix = ""
+        display_prefix = "#{prefix}/" if prefix
+        xpath = field.to_s
+        xpath = "#{prefix}/#{xpath}" if prefix
+        solver.declare_const(progress, filled_var_for_field(xpath), "Bool", {:xpath => xpath})
+        solver.declare_const(progress, value_var_for_field(xpath), translate_datatype_to_SMTLIB(definition[:datatype]), {:xpath => xpath, :datatype => definition[:datatype]})
+        solver.to_solver(progress, "(assert (! (=> (not #{filled_var_for_field(xpath)}) (= #{value_var_for_field(xpath)} #{default_value_for_fielddef(definition)})) :named empty-field-has-no-value-for-#{raw_field(xpath)}))") 
+        if (struct_varname)
+          solver.to_solver(progress, "(assert (! (=> (not #{struct_varname}) (not #{filled_var_for_field(xpath)})) :named empty-struct-has-no-fields-for-#{raw_field(xpath)}))") 
+        end
+        translate_field_def_to_SMTLIB_constraints(progress, solver, definition, xpath, struct_varname)
+      elsif definition[:type] == :list then
+        display_prefix = ""
+        display_prefix = "#{prefix}/" if prefix
+        xpath = field.to_s
+        xpath = "#{prefix}/#{xpath}" if prefix
+        solver.declare_const(progress, filled_var_for_field(xpath), "Bool", {:xpath => xpath})
+        solver.declare_const(progress, size_var_for_list(xpath), "Int", {:xpath => xpath})
+        solver.to_solver(progress, "(assert (! (=> (not #{filled_var_for_field(xpath)}) (= #{size_var_for_list(xpath)} 0)) :named empty-list-field-has-zero-size-#{raw_field(xpath)}))")
+        solver.to_solver(progress, "(assert (! (=> (= #{size_var_for_list(xpath)} 0) (not #{filled_var_for_field(xpath)})) :named list-has-zero-size-implies-empty-list-#{raw_field(xpath)}))")
+        solver.to_solver(progress, "(assert (! (=> #{filled_var_for_field(xpath)} (> #{size_var_for_list(xpath)} 0)) :named filled-list-field-has-positive-size-#{raw_field(xpath)}))")
+        solver.to_solver(progress, "(assert (! (=> (> #{size_var_for_list(xpath)} 0) #{filled_var_for_field(xpath)}) :named list-has-positive-size-implies-filled-list-#{raw_field(xpath)}))")
+        definition[:model_maxLength].times do |i|
+          solver.declare_const(progress, value_var_for_list(xpath, i), translate_datatype_to_SMTLIB(definition[:datatype]), {:xpath => xpath, :xpath_element => definition[:xpath_element], :index => i, :datatype => definition[:datatype]})
+          solver.to_solver(progress, "(assert (! (=> (<= #{size_var_for_list(xpath)} #{i}) (= #{value_var_for_list(xpath, i)} #{default_value_for_fielddef(definition)})) :named empty-list-field-has-no-value-for-#{raw_field(xpath)}-index-#{i}))") 
+        end
+        if (struct_varname)
+          solver.to_solver(progress, "(assert (! (=> (not #{struct_varname}) (= #{size_var_for_list(xpath)} 0)) :named empty-struct-has-no-fields-for-#{raw_field(xpath)}))") 
+        end
+        translate_list_def_to_SMTLIB_constraints(progress, solver, definition, raw_field(xpath))
+      elsif definition[:type] == :objlist then
+        xpath = field.to_s
+        xpath = "#{prefix}/#{xpath}" if prefix
+        solver.declare_const(progress, filled_var_for_field(xpath), "Bool", {:xpath => xpath})
+        solver.declare_const(progress, size_var_for_list(xpath), "Int", {:xpath => xpath})
+        if (struct_varname)
+          solver.to_solver(progress, "(assert (! (=> (not #{struct_varname}) (not #{filled_var_for_field(xpath)})) :named empty-struct-has-no-objlist-#{raw_field(xpath)}))") 
+        end
+        if definition[:optional] == nil || definition[:optional] == false then
+          solver.to_solver(progress, "(assert (! (=> #{struct_varname} #{filled_var_for_field(xpath)}) :named required-objlist-#{raw_field(xpath)}))")
+        end
+        solver.to_solver(progress, "(assert (! (=> (not #{filled_var_for_field(xpath)}) (= #{size_var_for_list(xpath)} 0)) :named empty-objlist-has-zero-size-#{raw_field(xpath)}))")
+        solver.to_solver(progress, "(assert (! (=> (= #{size_var_for_list(xpath)} 0) (not #{filled_var_for_field(xpath)})) :named objlist-has-zero-size-implies-empty-objlist-#{raw_field(xpath)}))")
+        minSize = 0
+        solver.to_solver(progress, "(assert (! (=> #{filled_var_for_field(xpath)} (and (> #{size_var_for_list(xpath)} #{minSize}) (< #{size_var_for_list(xpath)} #{definition[:model_maxLength]}))) :named filled-objlist-has-positive-size-#{raw_field(xpath)}))")
+        solver.to_solver(progress, "(assert (! (=> (> #{size_var_for_list(xpath)} 0) #{filled_var_for_field(xpath)}) :named objlist-has-positive-size-implies-filled-objlist-#{raw_field(xpath)}))")
+        definition[:model_maxLength].times do |i|
+          ext_xpath = "#{xpath}/#{i}"
+          solver.declare_const(progress, exists_var_for_structure(xpath, i), "Bool", {:xpath => ext_xpath})
+          solver.to_solver(progress, "(assert (! (=> (<= #{size_var_for_list(xpath)} #{i}) (not #{exists_var_for_structure(xpath, i)})) :named objlist-struct-exist1-for-#{raw_field(xpath)}-index-#{i}))") 
+          solver.to_solver(progress, "(assert (! (=> (> #{size_var_for_list(xpath)} #{i}) #{exists_var_for_structure(xpath, i)}) :named objlist-struct-exist2-for-#{raw_field(xpath)}-index-#{i}))") 
+          translate_substructure_to_SMTLIB(progress, solver, definition[:ref], :prefix => ext_xpath, :struct_varname => exists_var_for_structure(xpath, i), :options => options, :config => config)
+        end
+      elsif definition[:type] == :structure then
+        xpath = field.to_s
+        xpath = "#{prefix}/#{xpath}" if prefix
+        solver.declare_const(progress, exists_var_for_structure(xpath), "Bool", {:xpath => xpath})
+        if (struct_varname)
+          solver.to_solver(progress, "(assert (! (=> (not #{struct_varname}) (not #{exists_var_for_structure(xpath)})) :named empty-struct-has-no-substructure-#{raw_field(xpath)}))") 
+        end
+        translate_substructure_to_SMTLIB(progress, solver, definition[:ref], :prefix => xpath, :struct_varname => exists_var_for_structure(xpath), :options => options, :config => config)
+      else
+        raise RuntimeError.new("Model Error: encountered unknown :type #{definition[:type].inspect} for element #{field.inspect}")
+      end
+    end
+  end
+end
+
+def resolve_field_path(path, relative_to_struct, prefix)
+  current = relative_to_struct
+  result = nil
+  path.split("/").each do |segment|
+    unless current then
+      raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect}: resolution worked up until #{result.inspect}, but then the last segment was a field or list.")
+    end
+    field_def = current[segment.to_sym]
+    unless field_def then
+      raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect}: structure #{prefix.inspect} does not have a member #{segment.to_sym.inspect}.")
+    end
+    if field_def[:type] == :structure then
+      unless field_def[:ref] then
+        raise RuntimeError.new("could not resolve field path: #{path.inspect}: substructure #{segment.inspect} of structure #{prefix.inspect} does not have a :ref.")
+      end
+      current = field_def[:ref]
+      if prefix then
+        prefix = "#{prefix}/#{segment}"
+      else
+        prefix = segment
+      end
+    elsif field_def[:type] == :field then
+      current = nil
+      result = segment
+      if prefix
+        result = "#{prefix}/#{result}"
+      end
+    elsif field_def[:type] == :list then
+      current = nil
+      result = segment
+      if prefix
+        result = "#{prefix}/#{result}"
+      end
+    end
+  end
+  return result
+end
+
+def resolve_field_def(path, relative_to_struct, prefix)
+  current = relative_to_struct
+  result = nil
+  path.split("/").each do |segment|
+    unless current then
+      raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect}: resolution worked up until #{result.inspect}, but then the last segment was a field or list.")
+    end
+    field_def = current[segment.to_sym]
+    unless field_def then
+      raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect}: structure #{prefix.inspect} does not have a member #{segment.to_sym.inspect}.")
+    end
+    if field_def[:type] == :structure then
+      unless field_def[:ref] then
+        raise RuntimeError.new("could not resolve field path: #{path.inspect}: substructure #{segment.inspect} of structure #{prefix.inspect} does not have a :ref.")
+      end
+      current = field_def[:ref]
+      if prefix then
+        prefix = "#{prefix}/#{segment}"
+      else
+        prefix = segment
+      end
+    elsif field_def[:type] == :field then
+      current = nil
+      result = field_def
+    elsif field_def[:type] == :list then
+      current = nil
+      result = field_def
+    end
+  end
+  return result
+end
+
+def resolve_struct_path(path, relative_to_struct, prefix)
+  current = relative_to_struct
+  result = nil
+  path.split("/").each do |segment|
+    unless current then
+      raise RuntimeError.new("could not resolve field path: #{path.inspect} relative to struct #{relative_to_struct.inspect}")
+    end
+    if segment == "." then
+      result = current[:struct_xpath_prefix]
+    elsif segment == ".." then
+      current = current[:parent_link]
+      if prefix then
+        prefix_segs = prefix.split("/")
+        prefix = prefix_segs[0,prefix_segs.size()-1].join("/")
+      else
+        prefix = current[:struct_xpath_prefix]
+      end
+    else
+      field_def = current[segment.to_sym]
+      unless field_def then
+        raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect} relative to struct #{relative_to_struct.inspect}")
+      end
+      if field_def[:type] == :structure then
+        current = field_def[:ref]
+        if prefix then
+          prefix = "#{prefix}/#{segment}"
+        else
+          prefix = segment
+        end
+        result = prefix
+      end
+    end
+  end
+  return result
+end
+
+def resolve_struct_def(path, relative_to_struct, prefix, context_desc = "")
+  current = relative_to_struct
+  result = nil
+  path.split("/").each do |segment|
+    unless current then
+      raise RuntimeError.new("could not resolve field path: #{path.inspect} relative to struct #{relative_to_struct.inspect} #{context_desc}")
+    end
+    if segment == "." then
+      result = current
+    elsif segment == ".." then
+      current = current[:parent_link]
+      if prefix then
+        prefix_segs = prefix.split("/")
+        prefix = prefix_segs[0,prefix_segs.size()-1].join("/")
+      else
+        prefix = current[:struct_xpath_prefix]
+      end
+    else
+      field_def = current[segment.to_sym]
+      unless field_def then
+        raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect} relative to struct #{relative_to_struct.inspect} #{context_desc}")
+      end
+      if field_def[:type] == :structure then
+        current = field_def[:ref]
+        if prefix then
+          prefix = "#{prefix}/#{segment}"
+        else
+          prefix = segment
+        end
+        result = current
+      end
+    end
+  end
+  return result
+end
+
+def translate_validation_rules_to_SMTLIB(progress, solver, struct, prefix, config, options)
+  solver.to_solver(progress, "; --- Validation Rules ---")
+  translate_validation_rules_for_structure_to_SMTLIB(progress, solver, struct, prefix, config, options)
+  struct.each_pair do |key, value|
+    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if value[:type] == :structure then
+      if prefix then
+        new_prefix = "#{prefix.to_s}/#{key}"
+      else
+        new_prefix = key.to_s
+      end
+      translate_validation_rules_to_SMTLIB(progress, solver, value[:ref], new_prefix, config, options)
+    end
+  end
+end
+
+def translate_validation_rule_to_SMTLIB(rule, context_struct, prefix, config, options, rule_name)
+  date_now = options[:date_now]
+  timestamp_now = options[:timestamp_now]
+  return rule.gsub(/FILLED\[[^\]]*\]/) do |m|
+                    unless m =~ /FILLED\[([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse FILLED[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    field = resolve_field_path($1, context_struct, prefix)
+                    filled_var_for_field(field)
+                  end
+                  .gsub(/VALUE\[[^\]]*\]/) do |m|
+                    unless m =~ /VALUE\[([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse VALUE[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    field = resolve_field_path($1, context_struct, prefix)
+                    value_var_for_field(field)
+                  end
+                  .gsub(/EXIST\[[^\]]*\]/) do |m|
+                    unless m =~ /EXIST\[([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse EXIST[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    s = resolve_struct_path($1, context_struct, prefix)
+                    exists_var_for_structure(s)
+                  end
+                  .gsub(/SIZE\[[^\]]*\]/) do |m|
+                    unless m =~ /SIZE\[([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse SIZE[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    l = resolve_field_path($1, context_struct, prefix)
+                    size_var_for_list(l)
+                  end
+                  .gsub(/LIST_CONTAINS\[[^\]]*,[^\]]*\]/) do |m|
+                    unless m =~ /LIST_CONTAINS\[([^\]]*),([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse LIST_CONTAINS[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    l = resolve_field_path($1, context_struct, prefix)
+                    l_def = resolve_field_def($1, context_struct, prefix)
+                    value = $2
+                    terms = []
+                    l_def[:model_maxLength].times do |i|
+                      terms << "(and (< #{i} #{size_var_for_list(l)}) (= #{value_var_for_list(l, i)} #{value}))"
+                    end
+                    "(or #{terms.join(" ")})"
+                  end
+                  .gsub(/LIST_VALUE_FIRST_ELEMENT\[[^\]]*\]/) do |m|
+                    unless m =~ /LIST_VALUE_FIRST_ELEMENT\[([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse LIST_VALUE_FIRST_ELEMENT[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    xpath = $1
+                    l = resolve_field_path(xpath, context_struct, prefix)
+                    value_var_for_list(l, 0)
+                  end
+                  .gsub("[Date.now]", date_now.to_s)
+                  .gsub("[Timestamp.now]", timestamp_now.to_s)
+                  .gsub(/PREDICATE\[[^\]]*,[^\]]*\]/) do |m|
+                    unless m =~ /PREDICATE\[([^\]]*),([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse PREDICATE[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    path = $1
+                    key = $2.strip().to_sym
+                    sdef = resolve_struct_def(path, context_struct, prefix, "in validation rule #{rule_name}")
+                    unless sdef
+                      raise RuntimeError.new("could not resolve struct path #{path.inspect} in validation rule #{rule_name}")
+                    end
+                    preds = sdef[:predicates]
+                    mdk = preds[key]
+                    if mdk then
+                      if mdk[:smtlib] then
+                        translate_validation_rule_to_SMTLIB(mdk[:smtlib], sdef, sdef[:struct_xpath_prefix], config, options, "#{rule_name}.predicate.#{key}[#{sdef[:struct_xpath_prefix]}]" )
+                      else
+                        raise RuntimeError.new("Predicate #{key} for structure #{sdef[:struct_xpath_prefix]} does not have an SMTLIB definition, but is referenced in validation rule #{rule_name}")
+                      end
+                    else
+                      raise RuntimeError.new("No predicate #{key.inspect} defined for structure #{sdef[:struct_xpath_prefix]}, but one referenced in validation rule #{rule_name}")
+                    end
+                  end
+                  .gsub(/CONFIG\[[^\]]*\]/) do |m|
+                    unless m =~ /CONFIG\[([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse LIST_CONTAINS[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    key = $1.to_sym
+                    if config.has_key?(key) then
+                      "\"#{config[key]}\"" 
+                    else
+                      raise RuntimeError.new("unknown CONFIG-key #{key.inspect} encountered in validation rule #{rule_name}" )
+                    end
+                  end
+                  .gsub(/FIELD_IN_QUERY\[[^\]]*\]/) do |m|
+                    unless m =~ /FIELD_IN_QUERY\[([^\]]*),([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse FIELD_IN_QUERY[..,..]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    xpath = $1
+                    query_filename = $2
+                    fpath = resolve_field_path(xpath, context_struct, prefix)
+                    fdef = resolve_field_def(xpath, context_struct, prefix)
+                    values = File.read("#{QUERY_DIR}/#{query_filename.strip()}").split("\n")
+                    if fdef[:datatype] == :string then
+                      terms = values.filter{|l| !l.start_with?("#")}.map do |val|
+                        "(= #{value_var_for_field(fpath)} \"#{val}\")"
+                      end
+                    elsif fdef[:datatype] == :int then
+                      terms = values.filter{|l| !l.start_with?("#")}.map do |val|
+                        "(= #{value_var_for_field(fpath)} #{Integer(val)})"
+                      end
+                    else
+                      raise RuntimeError.new("FIELD_IN_QUERY[] was applied to a field of datatype #{fdef[:datatype].inspect}. However, it can only be applied to fields of datatype string or int.")
+                    end
+                    "(or #{terms.join(" ")})"
+                  end
+                  .gsub(/FIELD_IN_SMTLIB_PARAMETERIZED_QUERY\[[^\]]*\]/) do |m|
+                    unless m =~ /FIELD_IN_SMTLIB_PARAMETERIZED_QUERY\[([^\]]*),([^\]]*),([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse FIELD_IN_SMTLIB_PARAMETERIZED_QUERY[..,..,..]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    field_xpath = $1
+                    param_smtlib = $2
+                    query_filename = $3
+                    fpath = resolve_field_path(field_xpath.strip(), context_struct, prefix)
+                    fdef = resolve_field_def(field_xpath.strip(), context_struct, prefix)
+                    query_result = YAML.load_file("#{PARAMETERIZED_QUERY_DIR}/#{query_filename.strip()}")
+                    terms = []
+                    if fdef[:datatype] == :string then
+                      terms << "(or #{query_result.keys.map { |key| "(= #{param_smtlib} \"#{key}\")" }.join(" ")})"
+                      query_result.each_pair do |key, values|
+                        terms << "(=> (= #{param_smtlib} \"#{key}\") (or #{values.map{ |v| "(= #{value_var_for_field(fpath)} \"#{v}\")" }.join(" ")}))"
+                      end
+                    else
+                      raise RuntimeError.new("FIELD_IN_SMTLIB_PARAMETERIZED_QUERY[] was applied to a field of datatype #{fdef[:datatype].inspect}. However, it can only be applied to fields of datatype string.")
+                    end
+                    "(and #{terms.join(" ")})"
+                  end
+                  .gsub(/CONFIG_IN_QUERY\[[^\]]*\]/) do |m|
+                    unless m =~ /CONFIG_IN_QUERY\[([^\]]*),([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse CONFIG_IN_QUERY[..,..]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    key = $1.to_sym
+                    unless config.has_key?(key) then
+                      raise RuntimeError.new("unknown CONFIG key #{key.inspect} encountered in validation rule #{rule_name}")
+                    end
+                    query_filename = $2
+                    values = File.read("#{QUERY_DIR}/#{query_filename.strip()}").split("\n")
+                    terms = values.filter{|l| !l.start_with?("#")}.map do |val|
+                      "(= \"#{config[key]}\" \"#{val}\")"
+                    end
+                    "(or #{terms.join(" ")})"
+                  end
+end
+
+def translate_validation_rules_for_structure_to_SMTLIB(progress, solver, struct, prefix, config, options)
+  return unless struct[:validation_rules]
+
+  solver.to_solver(progress, "; --- Validation Rules: ---")
+  negated_rules = options[:negated_validation_rules] || []
+  solver.to_solver(progress, "; negated rules: #{negated_rules.empty? ? "<none>" : negated_rules.join(", ")}")
+
+  rules = struct[:validation_rules]
+  rules.each do |rule_name, rule|
+    rule = rule[:smtlib]  # ignore the text representation
+    next unless rule # warning about this omission is the job of validate_model()
+    rule_SMTLIB = translate_validation_rule_to_SMTLIB(rule, struct, prefix, config, options, rule_name)
+    if negated_rules.include?(rule_name) then
+      rule_SMTLIB = "(not #{rule_SMTLIB})"
+    end
+    solver.to_solver(progress, "(assert (! #{rule_SMTLIB} :named validation_rule_#{rule_name}_instance_#{prefix.gsub("/", "_")}))")
+  end
+end
+
+def parse_field_value(fielddef, value)
+  value = value.gsub("\n", "")
+  datatype = fielddef[:datatype]
+  case datatype
+  when :int
+    return Integer(value)
+  when :string
+    return value
+  when :enum
+    return value
+  when :date
+    return Date.parse(value)
+  when :timestamp
+    return Time.new(value)
+  end
+end
+
+def solver_value_to_string(solver_value)
+  if solver_value =~ /\"(.*)\"/ then
+    return $1
+  else
+    return solver_value
+  end
+end
+
+def build_document_from_model(progress, struct, model)
+  doc = Nokogiri::XML(XML_TEMPLATE)
+  add_model_struct_to_doc(progress, struct, doc, model)
+  return doc
+end
+
+def add_model_struct_to_doc(progress, struct, doc, model, prefix = nil)
+  struct.each do |key, value|
+    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if prefix then
+      xpath = "#{prefix}/#{key}"
+    else
+      xpath = key.to_s
+    end
+    model_value = model[xpath]
+    if value[:type] == :field then
+      unless model_value then
+        raise RuntimeError.new("xpath '#{xpath}' not contained in model.")
+      end
+      if model_value[:filled] == true || model_value[:filled] == "true" then
+        doc.add_child("<#{key}>#{solver_value_to_string(model_value[:value])}</#{key}>")
+      end
+    elsif value[:type] == :list then
+      node = doc.add_child("<#{key}></#{key}>")
+      if node.is_a?(Nokogiri::XML::NodeSet) then
+        node = node.first 
+      end
+      xpath_element = value[:xpath_element]
+      if Integer(model_value[:size]) > 0 then
+        model_value[:value].slice(0, Integer(model_value[:size])).each do |mvalue|
+          node.add_child("<#{xpath_element}>#{solver_value_to_string(mvalue)}</#{xpath_element}>")
+        end
+      end
+    elsif value[:type] == :objlist then
+      node = doc.add_child("<#{key}></#{key}>")
+      if node.is_a?(Nokogiri::XML::NodeSet) then
+        node = node.first 
+      end
+      xpath_element = value[:xpath_element]
+      if Integer(model_value[:size]) > 0 then
+        model_value[:size].times do |index|
+          cnode = node.add_child("<#{xpath_element}></#{xpath_element}>")
+          if cnode.is_a?(Nokogiri::XML::NodeSet) then
+            cnode = cnode.first 
+          end
+          add_model_struct_to_doc(progress, value[:ref], cnode, model, "#{xpath}/#{index}")
+        end
+      end
+    elsif value[:type] == :structure then
+      unless value[:ref] then
+        raise RuntimeError.new("struct is broken: :structure #{key.inspect} does not have a :ref")
+      end
+      if model_value[:exists] == true || model_value[:exists] == "true" then
+        node = doc.add_child("<#{key}></#{key}>")
+        if node.is_a?(Nokogiri::XML::NodeSet) then
+          node = node.first 
+        end
+        add_model_struct_to_doc(progress, value[:ref], node, model, xpath)
+      end
+    else
+      raise RuntimeError.new("struct is broken: encountered unknown :type #{value[:type].inspect}")
+    end
+  end
+end
+
+def collect_validation_rule_names(struct)
+  result = []
+  struct.each do |key, value|
+    next if key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if key == :validation_rules
+      value.each do |key, value|
+        result << key
+      end
+    else
+      if value[:type] == :structure
+        result += collect_validation_rule_names(value[:ref])
+      end
+    end
+  end
+  return result
+end
+
+def set_standard_options(progress, solver)
+  solver.to_solver(progress, "(set-option :produce-unsat-cores true) ; enable generation of unsat cores")
+  solver.to_solver(progress, "(set-option :smt.core.minimize true) ; ensure that unsat cores are minimal")
+  solver.to_solver(progress, "")
+end
+
+def generate_doc(progress, message, output_filename, config, options)
+  raise RuntimeError.new("option :solverFactory is required") unless options[:solverFactory]
+  log = options[:log] || nil
+  log.puts "generating doc #{output_filename}" if log
+  options[:solverLog] = options[:solverLog] || "#{output_filename}.smt2"
+
+  solverFactory = options[:solverFactory]
+  model = solverFactory.query_model(options) do |solver|
+    set_standard_options(progress, solver)
+    log.puts "translating structure" if log
+    translate_structure_to_SMTLIB(progress, solver, message, :options => options, :config => config)
+    solver.to_solver(progress, "")
+  end
+
+  if model then
+    result_doc = build_document_from_model(progress, message, model)
+    dir = File.dirname(output_filename)
+    FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+    File.open(output_filename, "w") do |f|
+      f.puts result_doc.to_xml()
+    end
+    progress.print_line "DONE. Generated XML was written to #{output_filename}, SMTLIB log was written to #{options[:solverLog]}"
+    return true
+  else
+    progress.print_line "FAILED. No XML document was generated. For more details, please see the SMTLIB log in #{options[:solverLog]}"
+    return false
+  end
+end
+
+def z3ValueToRubyValue(value)
+  if value =~ /\"(.*)\"/ then # string
+    return $1
+  elsif value =~ /\d+/ then # int
+    return Integer(value)
+  elsif value == "true"
+    return true
+  elsif value == "false" then
+    return false 
+  else
+    raise RuntimeError.new("enountered unknown type of value in z3 model: #{value.inspect}")
+  end
+end
+
+def extract_key_field_values_from_model(key_fields, model)
+  result = {}
+  key_fields.each do |xpath, fieldDesc|
+    fieldInfo = model[xpath]
+    if fieldDesc[:type] == :field then
+      if fieldInfo[:filled] then
+        result[xpath] = z3ValueToRubyValue(fieldInfo[:value])
+      else
+        result[xpath] = nil
+      end
+    elsif fieldDesc[:type] == :struct || fieldDesc[:type] == :structure then
+      if fieldInfo.has_key?(:exists) then
+        result[xpath] = z3ValueToRubyValue(fieldInfo[:exists])
+      else
+        result[xpath] = false
+      end
+    else
+      raise RuntimeError.new("encountered unknown :type in solver model: #{fieldDesc[:type].inspect}")
+    end
+  end
+  return result
+end
+
+def all_values_for_field(field)
+  unless field[:type] == :field then
+    raise RuntimeError.new("something that is not a field definition was passed to all_values_for_field(): #{field.inspect}")
+  end
+  unless field[:datatype] == :enum then
+    raise RuntimeError.new("all_values_for_field() is only applicable to fields of datatype enum! not #{field[:datatype].inspect}.")
+  end
+  values = field[:values].dup
+  if field[:optional] == true then
+    values << nil
+  end
+  return values
+end
+
+def calc_combinations_for_key_fields(key_fields)
+  count = 1
+  key_fields.each do |field|
+    if field[1][:type] == :field then
+      if field[1][:datatype] == :enum then
+        num_values = field[1][:values].size
+        num_values += 1 if field[1][:optional]
+      elsif field[1][:datatype] == :int then
+        if field[1].has_key?(:min) and field[1].has_key?(:max) and field[1][:min] <= field[1][:max] then
+          if field[1][:optional] then
+            num_values = 2 + field[1][:max] - field[1][:min]
+          else
+            num_values = 1 + field[1][:max] - field[1][:min]
+          end
+        else
+          return "potentially infinite"
+        end
+      elsif field[1][:datatype] == :string then
+        return "potentially infinite"
+      end
+    elsif field[1][:type] == :structure then
+      num_values = 2
+    end
+    count *= num_values
+  end
+  return count
+end
+
+def calc_combinations(message, key)
+  key_fields = collect_key_fields(key, message)
+  if key_fields.empty?
+    raise FatalError.new("unknown key: #{key}")
+  end
+  return calc_combinations_for_key_fields(key_fields)
+end
+
+def read_docs_from_log_file(log_filename)
+  result = []
+  if File.exist?(log_filename) then
+    File.open(log_filename, "r") do |f|
+      f.each_line do |line|
+        doc = {}
+        if line =~ /(.*)\ <-\ \{(.*)\}/ then
+          filename = $1
+          combination = $2
+          combination.split(", ").each do |assignment|
+            if assignment =~ /(.*):\ (.*)/ then
+              key = $1
+              value = $2
+              if value =~ /\"(.*)\"/ then
+                value = $1
+              elsif value == "nil" then
+                value = nil
+              end
+              doc[key] = value
+            else
+              raise RuntimeError.new("could not parse logfile #{log_filename}: invalid assignment #{assignment.inspect}")
+            end
+          end
+        else
+          raise RuntimeError.new("could not parse logfile #{log_filename}: invalid line #{line.inspect}")
+        end
+        result << doc
+      end
+    end
+  end
+  return result
+end
+
+def prepare_output_doc_filename(number, output_dir)
+  chunk_number = number / 100000
+  dirname = "#{output_dir}/#{chunk_number}"
+  unless Dir.exist?(dirname) then
+    FileUtils.mkdir_p(dirname)
+  end
+  return "#{dirname}/doc#{number}.xml"
+end
+
+def values_for_key_field(field)
+  if field[:type] == :field then
+    if field[:datatype] == :enum then
+      if field[:optional] then
+        return field[:values] + [nil]
+      else
+        return field[:values]
+      end
+    elsif field[:datatype] == :int then
+      if field.has_key?(:min) and field.has_key?(:max) and field[:min] <= field[:max] then
+        return (field[:min]..field[:max]).to_a
+      else
+        raise RuntimeError.new("cannot determine values for unbounded :int field")
+      end
+    else
+      raise RuntimeError.new("cannot determine values for datatype #{field[:datatype].inspect}")
+    end
+  elsif field[:type] == :structure then
+    if field[:optional] then
+      return [true, false]
+    else
+      return [true]
+    end
+  else
+    raise RuntimeError.new("Encountered unknown :type in model: #{field[:type].inspect} - keys can only mark fields or structures.")
+  end
+end
+
+def collect_key_fields(fkey, struct, prefix = nil)
+  result = {}
+  struct.each do |key, value|
+    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if value[:type] == :structure
+      if value.has_key?(:keys) and value[:keys].include?(fkey) then
+        xpath = key.to_s
+        xpath = "#{prefix}/#{xpath}" if prefix
+        result[xpath] = value.clone().filter{|k,v| [:type, :optional, :datatype, :values].include?(k)}
+      end
+      if prefix then
+        new_prefix = "#{prefix}/#{key}" 
+      else
+        new_prefix = key
+      end
+      result = result.merge(collect_key_fields(fkey, value[:ref], new_prefix))
+    elsif value[:type] == :field and value.has_key?(:keys) and value[:keys].include?(fkey) then
+      xpath = key.to_s
+      xpath = "#{prefix}/#{xpath}" if prefix
+      result[xpath] = value.clone().filter{|k,v| [:type, :optional, :datatype, :values, :min, :max].include?(k)}
+    end
+  end
+  return result
+end
+
+def collect_key_fields_and_strctures(fkey, struct, prefix = nil)
+  fields = []
+  structures = []
+  struct.each do |key, value|
+    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if value[:type] == :structure
+      if value.has_key?(:keys) and value[:keys].include?(fkey) then
+        xpath = key.to_s
+        xpath = "#{prefix}/#{xpath}" if prefix
+        structures << xpath
+      end
+      if prefix then
+        new_prefix = "#{prefix}/#{key}" 
+      else
+        new_prefix = key
+      end
+      result_fields, result_structures = collect_key_fields_and_strctures(fkey, value[:ref], new_prefix)
+      fields += result_fields
+      structures += result_structures
+    elsif value[:type] == :field and value.has_key?(:keys) and value[:keys].include?(fkey) then
+      xpath = key.to_s
+      xpath = "#{prefix}/#{xpath}" if prefix
+      fields << xpath
+    end
+  end
+  return fields, structures
+end
+
+def collect_keys(struct)
+  result = []
+  struct.each do |key, value|
+    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if value[:type] == :structure
+      if value.has_key?(:keys) then
+        result += value[:keys]
+      end
+      if value[:ref] then
+        result += collect_keys(value[:ref])
+      else
+        raise RuntimeError.new("Model error: encountered structure without :ref: key=#{key.inspect} value=#{value.inspect}")
+      end
+    elsif value[:type] == :field and value.has_key?(:keys) then
+      result += value[:keys]
+    end
+  end
+  result.uniq
+end
+
+def collect_validation_rules(struct)
+  result = []
+  struct.each do |key, value|
+    next if key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if key == :validation_rules then
+      result += value.keys
+    else
+      if value[:type] == :structure
+        if value[:ref] then
+          result += collect_validation_rules(value[:ref])
+        else
+          raise RuntimeError.new("Model error: encountered structure without :ref: key=#{key.inspect} value=#{value.inspect}")
+        end
+      end
+    end
+  end
+  result.uniq
+end
+
+def list_keys(message)
+  keys = collect_keys(message)
+  puts "keys: #{keys.join(", ")}"
+end
+
+def list_fields_and_structures(message, fkey)
+  fields, structures = collect_key_fields_and_strctures(fkey, message)
+  puts "fields for key #{fkey}:"
+  fields.each { |f| puts "  #{f}" }
+  puts "structures for key #{fkey}:"
+  structures.each { |s| puts "  #{s}" }
+end
+
+def list_validation_rules(message)
+  rules = collect_validation_rules(message)
+  puts "Validation Rules:"
+  rules.each { |r| puts "  #{r}" }
+end
+  
+def value_of_type(value, type)
+  if type == :string or type == :enum then
+    "\"#{value}\""
+  elsif type == :int then
+    value.to_s
+  elsif type == :date then
+    (Date.parse(value) -  Time.at(0).to_date).to_i.to_s
+  elsif type == :timestamp then
+    (Time.new(value) - Time.at(0)).to_i.to_s
+  else
+    raise RuntimeError.new("encountered unknown :datatype #{type.inspect}")
+  end
+end
+
+class CoreEnumerationAlgo
+
+  def initialize()
+    @num_tries = 0
+    @num_docs = 0
+    @on_new_doc = nil
+    @on_unsat = nil
+  end
+
+
+  # fix values of all variables left from switched
+  def enumerate_fixing_clause(assignment, vars, switched)
+    clauses = []
+    switched.times do |index|
+      var_xpath = vars[index][0]
+      var_def = vars[index][1]
+      if var_def[:type] == :field then
+        if assignment[var_xpath] == nil then
+          clauses << "(assert (! (= #{filled_var_for_field(var_xpath)} false) :named fixing_clause_#{index}))"
+        else
+          clauses << "(assert (! (and (= #{filled_var_for_field(var_xpath)} true) (= #{value_var_for_field(var_xpath)} #{value_of_type(assignment[var_xpath], var_def[:datatype])})) :named fixing_clause_#{index}))"
+        end
+      elsif var_def[:type] == :structure then
+        clauses << "(assert (! (= #{exists_var_for_structure(var_xpath)} #{assignment[var_xpath].inspect}) :named fixing_clause_#{index}))"
+      end
+    end
+    "; fixing clauses:\n#{clauses.join("\n")}" 
+  end
+
+  def summarize_fixing_clause(assignment, vars, switched)
+    fixed = switched.times.to_a.map do |index|
+      var_xpath = vars[index][0]
+      "#{var_xpath}=#{assignment[var_xpath].inspect}"
+    end
+    "fixed(#{fixed.join(", ")})"
+  end
+  
+  # prevent the var from taking values that already have been enumerated exhaustively
+  def enumerate_blocking_clause(var_xpath, var_def, blocked)
+    clauses = []
+    num = 0
+    if var_def[:type] == :field then
+      blocked.each do |value|
+        if value == nil then
+          clauses << "(assert (! (not (= #{filled_var_for_field(var_xpath)} false)) :named blocking_clause_#{num}))"
+        else
+          clauses << "(assert (! (not (and (= #{filled_var_for_field(var_xpath)} true) (= #{value_var_for_field(var_xpath)} #{value_of_type(value, var_def[:datatype])}))) :named blocking_clause_#{num}))"
+        end
+        num += 1
+      end
+    elsif var_def[:type] == :structure then
+      blocked.each do |value|
+        clauses << "(assert (! (not (= #{exists_var_for_structure(var_xpath)} #{value.inspect})) :named blocking_clause_#{num}))"
+        num += 1
+      end
+    end
+    "; blocking clauses:\n#{clauses.join("\n")}" 
+  end
+
+  def summarize_blocking_clause(var_xpath, blocked)
+    "blocked(#{var_xpath})=[#{blocked.map{|v| v.inspect}.join(", ")}]"
+  end
+
+  def generate_sparse_doc(progress, message, key, config, options, code, output_doc_filename)
+    additional_options = {}
+    if code then
+        additional_options = { :add_SMTLIB => code }
+    end
+    return generate_doc_return_key_values(progress, message, key, output_doc_filename, config, options.merge(additional_options))
+  end
+
+  def log_filename_for_output_filename(output_filename)
+    "#{output_filename}.smt2"
+  end
+
+  def generate_doc_return_key_values(progress, message, key, output_filename, config, options)
+    raise RuntimeError.new("option :solverFactory is required.") unless options[:solverFactory]
+    log = options[:log] || nil
+    log.puts "generating doc #{output_filename}" if log
+    options[:solverLog] = log_filename_for_output_filename(output_filename)
+
+    key_fields = collect_key_fields(key, message)
+
+    solverFactory = options[:solverFactory]    
+    model = solverFactory.query_model(options) do |solver|
+      set_standard_options(progress, solver)
+      log.puts "translating structure" if log
+      translate_structure_to_SMTLIB(progress, solver, message, :options => options, :config => config)
+      solver.to_solver(progress, "")
+    end
+
+    if model then
+      result_doc = build_document_from_model(progress, message, model)
+      dir = File.dirname(output_filename)
+      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+      File.open(output_filename, "w") do |f|
+        f.puts result_doc.to_xml(:save_with => Nokogiri::XML::Node::SaveOptions::AS_XML)
+      end
+      progress.print_line "DONE. Generated XML was written to #{output_filename}, SMTLIB log was written to #{options[:solverLog]}"
+      return extract_key_field_values_from_model(key_fields, model)
+    else
+      progress.print_line "FAILED. No XML document was generated. For more details, please see the SMTLIB log in #{options[:solverLog]}"
+      return nil
+    end
+  end
+
+  def store_continuation_point(doc_number, state, assignment, new_assignment, filename, options, time)
+    File.open(filename, "w") do |f|
+      state_str = "state(#{state.map{|s| if s then "[#{s.map{|v| v.inspect}.join(",")}]" else "nil" end}.join("#")})"
+      assignment_str = "assignment(#{assignment.to_a.map{|k,v| "#{k}:#{v.inspect}"}.join(",")})"
+      new_assignment_str = "new_assignment(#{new_assignment.to_a.map{|k,v| "#{k}:#{v.inspect}"}.join(",")})"
+      f.puts("#{doc_number};#{@num_docs};#{state_str};#{assignment_str};#{new_assignment_str};#{time}")
+    end
+  end
+
+  def self.parse_state(state_str)
+    unless state_str =~ /state\((.*)\)/
+      return nil
+    end
+    split = $1.split("#")
+    split.map do |e|
+      if e == "nil" then
+        nil
+      elsif e =~ /\[(.*)\]/ then
+        $1.split(",").map do |i|
+          if i.strip == "nil" then
+            nil
+          elsif i.strip == "true" then
+            true
+          elsif i.strip == "false" then
+            false
+          elsif i.strip =~ /\"(.*)\"/ then
+            $1
+          end
+        end
+      end
+    end
+  end
+
+  def self.parse_assignment(str)
+    unless str =~ /assignment\((.*)\)/
+      return nil
+    end
+    split = $1.split(",")
+    result = {}
+    split.each do |e|
+      s2 = e.split(":")
+      key = s2[0]
+      value = s2[1]
+      if value == "nil" then
+        result[key] = nil
+      elsif value == "true" then
+        result[key] = true
+      elsif value == "false" then
+        result[key] = false
+      elsif value =~ /\"(.*)\"/
+        result[key] = $1
+      end
+    end
+    result
+  end
+
+  def self.parse_new_assignment(str)
+    unless str =~ /new_assignment\((.*)\)/
+      return nil
+    end
+    split = $1.split(",")
+    result = {}
+    split.each do |e|
+      s2 = e.split(":")
+      key = s2[0]
+      value = s2[1]
+      if value == "nil" then
+        result[key] = nil
+      elsif value == "true" then
+        result[key] = true
+      elsif value == "false" then
+        result[key] = false
+      elsif value =~ /\"(.*)\"/
+        result[key] = $1
+      end
+    end
+    result
+  end
+
+  def self.parse_continuation_point(line)
+    split = line.split(";")
+    doc_number = Integer(split[0])
+    num_docs = Integer(split[1])
+    state = CoreEnumerationAlgo.parse_state(split[2])
+    assignment = CoreEnumerationAlgo.parse_assignment(split[3])
+    new_assignment = CoreEnumerationAlgo.parse_assignment(split[4])
+    processessing_time = Integer(split[5])
+    return [doc_number, num_docs, state, assignment, new_assignment, processessing_time]
+  end
+
+  # TODO: logging
+  
+  def display_combination_space(assignment, vars, switched, blocked)
+    # enumerate_fixing_clause(assignment, vars, switched) + enumerate_blocking_clause(vars[switched][0], vars[switched][1], blocked)
+
+    # => all vars left of switched are fixed to their assigned values (when the value is nil, then they are empty)
+    # => the switched var is prevented to have any of the blocked values
+    res = switched.times.to_a.map do |index|
+      var_xpath = vars[index][0]
+      "#{var_xpath}=#{assignment[var_xpath].inspect}"
+    end + ["#{vars[switched][0]}\\{#{blocked.map{|v| v.inspect}.join(",")}}"]
+    "{#{res.join(", ")}}"
+  end
+  
+
+  # core algo
+  def enumerate(progress, switched, vars, assignment, message, key, output_dir, config, options, state = [nil] * vars.size(), start_time, prior_processing_time)
+    enumerate(progress, switched+1, vars, assignment, message, key, output_dir, config, options, state, start_time, prior_processing_time) if switched < vars.size()-1
+    is_finite_state = keys_are_finite_state(vars)
+    if state[switched] == nil then
+      blocked = [assignment[vars[switched][0]]] 
+      state[switched] = blocked
+    else
+      blocked = state[switched] + [assignment[vars[switched][0]]]
+      state[switched] = blocked
+    end
+    found = true
+    while found do
+      found = false
+      # - fix all vars left of switched to their values in assignment
+      # - block switched from obtaining the values in blocked
+      # progress.print_line "DEBUG: #{summarize_fixing_clause(assignment, vars, switched)}, #{summarize_blocking_clause(vars[switched][0], blocked)}"
+      clauses = enumerate_fixing_clause(assignment, vars, switched) + enumerate_blocking_clause(vars[switched][0], vars[switched][1], blocked)
+      output_doc_filename = prepare_output_doc_filename(@num_tries, output_dir)
+      new_assignment = generate_sparse_doc(progress, message, key, config, options, clauses, output_doc_filename)
+      generated_doc_number = @num_tries
+      time = prior_processing_time + (Time.now().to_i - options[:timestamp_now])
+      store_continuation_point(generated_doc_number, state, assignment, new_assignment, "#{output_dir}/#{CONTINUATION_FILE}", options, time)
+      progress.print_line "STORING continuation point"
+      progress.print_line "|time=#{time}"
+      progress.print_line "|doc_number=#{generated_doc_number}"
+      progress.print_line "|num_docs=#{@num_docs}"
+      progress.print_line "|state=#{state.inspect}"
+      progress.print_line "|assignment=#{assignment.inspect}"
+      progress.print_line "|new_assignment=#{new_assignment.inspect}"
+
+      if is_finite_state then
+        progress.report_progress(determine_progress(state, vars), start_time, prior_processing_time, @num_docs)
+      else
+        progress.report_progress(nil, start_time, prior_processing_time, @num_docs)
+      end
+      @num_tries += 1
+      if new_assignment then # sat
+        @num_docs += 1
+        @on_new_doc.call(generated_doc_number, output_doc_filename, new_assignment) if @on_new_doc
+        if switched < vars.size()-1 then
+          enumerate(progress, switched+1, vars, new_assignment, message, key, output_dir, config, options, state, start_time, prior_processing_time)
+        end
+        blocked << new_assignment[vars[switched][0]]
+        state[switched] = blocked
+        found = true
+      else # unsat
+        @on_unsat.call(generated_doc_number, log_filename_for_output_filename(output_doc_filename), display_combination_space(assignment, vars, switched, blocked), switched) if @on_unsat
+        found = false
+      end
+    end
+    state[switched] = nil
+  end
+  
+  # NOTE: can only be called when all key-fields are finite-state
+  def is_final_state(state, new_assignment, key_fields_list)
+    return state[1,state.size()] == [nil] * (state.size()-1) &&
+       state[0].to_set == values_for_key_field(key_fields_list[0][1]).to_set &&
+       new_assignment.empty?
+  end
+
+  # NOTE: can only be called when all key-fields are finite-state
+  def determine_progress(state, vars)
+    combinations_per_value = [nil] * vars.size()
+    comb = 1
+    vars.size().times.to_a.reverse_each do |i|
+      combinations_per_value[i] = comb
+      comb = values_for_key_field(vars[i][1]).size() * comb
+    end
+
+    blocked_combinations = 0
+    state.size().times do |i|
+      if state[i] then
+        blocked_combinations += state[i].size() * combinations_per_value[i]
+      end
+    end
+    blocked_combinations
+  end
+
+  def num_combinations(key_def)
+    if key_def[:type] == :field then
+      if key_def[:datatype] == :enum then
+        key_def[:values].size() + (key_def[:optional] ? 1 : 0)
+      elsif key_def[:datatype] == :int then
+        if key_def.has_key?(:min) and key_def.has_key?(:max) and key_def[:min] <= key_def[:max] then
+          1 + key_def[:max] - key_def[:min] + (key_def[:optional] ? 1 : 0)
+        else
+          nil
+        end
+      else
+        nil
+      end
+    elsif key_def[:type] == :structure then
+      2
+    end
+  end
+  
+  def keys_are_finite_state(list)
+    list.each do |field|
+      name = field[0]
+      if field[1][:type] == :field then
+        if field[1][:datatype] == :enum then
+          # enums are finite-state
+        elsif field[1][:datatype] == :int and field[1].has_key?(:min) and field[1].has_key?(:max) and field[1][:min] <= field[1][:max] then
+          # ints are finite-state when both bounds are given and :min <= :max
+        else
+          return false
+        end
+      end
+    end
+    return true
+  end
+
+  def generate_docs_for_key(progress, message, key, output_dir, config, options)
+
+    prior_processing_time = 0
+    start_time = options[:timestamp_now]
+    number_of_combinations = calc_combinations(message, key)
+    if number_of_combinations.is_a?(String) then
+      progress.set_max(100000000000000)
+    else
+      progress.set_max(number_of_combinations)
+    end
+    progress.report_progress(0.0, start_time, prior_processing_time, 0)
+    progress.print_line "generating sparse documents efficiently..."
+    progress.print_line "there is a theoretical maximum of #{number_of_combinations} combinations."
+    progress.print_line "Note, however, that we are generating only those documents that are valid, which are usually much less."
+
+    progress.print_line "preparing output dir #{output_dir}..."
+    if options[:continue] then
+      unless Dir.exist?(output_dir) then
+        raise FatalError.new("FATAL: directory #{output_dir} not found. There seems to be nothing to continue from...")
+      end
+      log_mode = "a"
+    else
+      unless Dir.exist?(output_dir) then
+        FileUtils.mkdir_p(output_dir)
+      end
+      log_mode = "w"
+    end
+    log_filename = "#{output_dir}/#{COMBINATION_LOG}"
+
+    key_fields = collect_key_fields(key, message)
+    key_fields_list = key_fields.each_pair.to_a
+    key_fields_list.sort! {|a,b| num_combinations(b[1]) <=> num_combinations(a[1]) }
+
+    if options[:continue] then
+      stored_key_fields_str = File.read("#{output_dir}/#{VARS_LOG}")
+      unless key_fields_list.to_yaml == stored_key_fields_str then
+        raise FatalError.new("vars are differing! please specify the same key in order to use -continue!")
+      end
+      progress.print_line "continuing enumerating documents for key #{key}!"
+    else
+      # store vars
+      File.open("#{output_dir}/#{VARS_LOG}", "w") do |f|
+        f.puts(key_fields_list.to_yaml)
+      end
+    end
+
+    cont_filename = "#{output_dir}/#{CONTINUATION_FILE}"
+    if options[:continue] then
+      unless File.exist?(cont_filename) then
+        raise FatalError.new("FATAL: file #{cont_filename} not found. There seems to be no continuation point to continue from...")
+      end
+      doc_number, num_docs, state, assignment, new_assignment, prior_processing_time = CoreEnumerationAlgo.parse_continuation_point(File.read(cont_filename).chomp)
+      if is_final_state(state, new_assignment, key_fields_list) then
+        progress.report_progress(number_of_combinations, start_time, prior_processing_time, num_docs)
+        raise FatalError.new("FATAL: The enumeration is finished. There is nothing left to do. Resuming the enumeration hence does not make sense.")
+      end
+      progress.print_line "RESUMING enumeration from the following continuation point:"
+      progress.print_line "|time=#{prior_processing_time}"
+      progress.print_line "|doc_number=#{doc_number}"
+      progress.print_line "|num_docs=#{num_docs}"
+      progress.print_line "|state=#{state.inspect}"
+      progress.print_line "|assignment=#{assignment.inspect}"
+      progress.print_line "|new_assignment=#{new_assignment.inspect}"
+      if keys_are_finite_state(key_fields_list) then
+        progress.report_progress(determine_progress(state, key_fields_list), start_time, prior_processing_time, num_docs)
+      else
+        progress.report_progress(nil, start_time, prior_processing_time, num_docs)
+      end
+
+      if new_assignment && !new_assignment.empty? then
+        assignment = new_assignment
+      end
+
+      # restore state at continuation point
+      @num_tries = doc_number+1 # we do not want to overwrite the last document written before the STOP
+      number_of_combinations += 1 # account for this in the safety mechanism
+      @num_docs = num_docs
+      File.open(log_filename, log_mode) do |log|
+
+        @on_new_doc = Proc.new { |num_doc, doc_filename, combination|
+          # combination logging
+          log.puts "#{doc_filename} <- {#{combination.map{|k,v| "#{k}=#{v.inspect}"}.join(", ")}}"
+
+          if options[:max_num_docs] && @num_docs >= options[:max_num_docs] then
+            raise NormalProgramTermination.new("-max-num-docs exceeded. Stopping!")
+          end
+
+          # safety mechanism
+          if @num_docs > number_of_combinations then
+            raise RuntimeError.new("This should never happen! Number of combinations exceeded. #{@num_docs} documents generated, but there are only #{number_of_combinations} possible combinations of the key vars.")
+          end
+        }
+        @on_unsat = Proc.new { |num_doc, log_filename, combination_str, switched|
+              log.puts "CONTRADICTION (see: #{log_filename}) <- #{combination_str}"
+              progress.print_line "doc#{num_doc} NOT generated"
+              progress.print_line "No more valid combinations of the free vars could be found. -> Switching (var #{switched})"
+        }
+
+        # core algo - resume from state
+        enumerate(progress, 0, key_fields_list, assignment, message, key, output_dir, config, options, state, start_time, prior_processing_time)
+      end
+    else
+      # init safety mechanism
+      @num_tries = 0
+      @num_docs = 0
+      # first try - a freely generated document
+      File.open(log_filename, log_mode) do |log|
+        output_doc_filename = prepare_output_doc_filename(@num_tries, output_dir)
+        @on_new_doc = Proc.new { |num_doc, doc_filename, combination|
+          # combination logging
+          log.puts "#{doc_filename} <- {#{combination.map{|k,v| "#{k}=#{v.inspect}"}.join(", ")}}"
+
+          if options[:max_num_docs] && @num_docs >= options[:max_num_docs] then
+            raise NormalProgramTermination.new("-max-num-docs exceeded. Stopping!")
+          end
+
+          # safety mechanism
+          unless number_of_combinations.is_a?(String) then
+            if @num_docs > number_of_combinations then
+              raise RuntimeError.new("This should never happen! Number of combinations exceeded. #{@num_docs} documents generated, but there are only #{number_of_combinations} possible combinations of the key vars.")
+            end
+          end
+        }
+        @on_unsat = Proc.new { |num_doc, log_filename, combination_str, switched|
+              log.puts "CONTRADICTION (see: #{log_filename}) <- #{combination_str}"
+              progress.print_line "doc#{num_doc} NOT generated"
+              progress.print_line "No more valid combinations of the free vars could be found. -> Switching (var #{switched})"
+        }
+        key_values = generate_doc_return_key_values(progress, message, key, output_doc_filename, config, options)
+        @num_tries += 1
+        if key_values then
+          @num_docs += 1
+          @on_new_doc.call(@num_tries, output_doc_filename, key_values)
+        else
+          # NOTE: this only occurs when the model is inconsistent and has no valid documents
+          progress.print_line("ERROR: Model inconsistent - no valid documents could be found.")
+          @on_unsat.call(@num_tries, log_filename_for_output_filename(output_doc_filename), "{}", 0)
+        end
+
+        # core algo - start enumeration
+        enumerate(progress, 0, key_fields_list, key_values, message, key, output_dir, config, options, start_time, prior_processing_time)
+      end
+    end
+    progress.print_line "No more valid documents could be found. -> Exiting"
+    progress.report_progress(number_of_combinations, start_time, prior_processing_time, @num_docs)
+  end
+end
+
+def introduce_parent_links(message, parent = nil, prefix = nil)
+  if parent != nil then
+    message[:parent_link] = parent
+  end
+  if prefix != nil then
+    message[:struct_xpath_prefix] = prefix
+  end
+  message.each_pair do |key, value|
+    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if value[:type] == :structure then
+      if prefix then
+        new_prefix = "#{prefix}/#{key}"
+      else
+        new_prefix = key
+      end
+      introduce_parent_links(value[:ref], message, new_prefix)
+    end
+  end
+end
+
+def validate_model(message, prefix=nil)
+
+  message.each_pair do |key, value|
+    if key == :validation_rules then
+      value.each_pair do |vr_key, vr_value|
+        unless vr_value[:text] && vr_value[:text] != "" then
+          puts "INFO: Model Problem: validation rule #{vr_key} does not have a textual representation."
+        end
+        unless vr_value[:smtlib] && vr_value[:smtlib] != "" then
+          puts "WARN: Model Problem: validation rule #{vr_key} does not have an SMTLIB representation. It will hence be omitted!"
+        end
+      end
+    elsif key == :predicates then
+      value.each_pair do |mkey, mvalue|
+        next unless mvalue
+        unless mvalue[:text] && mvalue[:text] != "" then
+            puts "INFO: Model Problem: structure #{prefix}'s predicate #{mkey} does not have a textual representation."
+        end
+        unless mvalue[:smtlib] && mvalue[:smtlib] != "" then
+            puts "WARN: Model Problem: structure #{prefix}'s predicate #{mkey} does not have an SMTLIB representation. It will be regarded as always true."
+        end
+      end
+    elsif key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+      next
+    else
+      if value[:type] == :structure then
+        unless value[:ref] then
+          raise RuntimeError.new("Model Problem: encountered structure without a :ref (key=#{key.inspect}, value=#{value.inspect})")
+        end
+        if prefix then
+          new_prefix = "#{prefix}/#{key}"
+        else
+          new_prefix = key
+        end
+        validate_model(value[:ref], new_prefix)
+      elsif value[:type] == :field then
+        unless value[:datatype] then
+          raise RuntimeError.new("Model Problem: encountered field without a :datatype (key=#{key.inspect}, value=#{value.inspect})")
+        end
+        if value[:datatype] == :enum then
+          unless value[:values] then
+            raise RuntimeError.new("Model Problem: encountered enum field without a :values (key=#{key.inspect}, value=#{value.inspect})")
+          end
+        end
+      elsif value[:type] == :list then
+        unless value[:datatype] then
+          raise RuntimeError.new("Model Problem: encountered list without a :datatype (key=#{key.inspect}, value=#{value.inspect})")
+        end
+        if value[:datatype] == :enum then
+          unless value[:values] then
+            raise RuntimeError.new("Model Problem: encountered enum list without a :values (key=#{key.inspect}, value=#{value.inspect})")
+          end
+        end
+      elsif value[:type] == :objlist then
+        unless value[:ref] then
+          raise RuntimeError.new("Model Problem: encountered objlist without a :ref (key=#{key.inspect}, value=#{value.inspect})")
+        end        
+        unless value[:model_maxLength] then
+          raise RuntimeError.new("Model Problem: encountered objlist without a :model_maxLength (key=#{key.inspect}, value=#{value.inspect})")
+        end
+        unless value[:model_maxLength].is_a?(Integer) then
+          raise RuntimeError.new("Model Problem: encountered objlist whose :model_maxLength property is not an Integer (key=#{key.inspect}, value=#{value.inspect})")
+        end
+        unless value[:xpath_element] then
+          raise RuntimeError.new("Model Problem: encountered objlist without a :xpath_element property (key=#{key.inspect}, value=#{value.inspect})")
+        end
+        unless value[:xpath_element].is_a?(String) then
+          raise RuntimeError.new("Model Problem: encountered objlist whose :xpath_element property is not a String (key=#{key.inspect}, value=#{value.inspect})")
+        end
+        if prefix then
+          new_prefix = "#{prefix}/#{key}"
+        else
+          new_prefix = key
+        end
+        validate_model(value[:ref], new_prefix)
+      else
+        raise RuntimeError.new("Model Problem: encountered unknown :type #{value[:type].inspect}")
+      end
+    end  
+  end
+end
+
+def validate_doc_against_schema(struct, doc)
+  root = doc.root
+  sdef = struct[root.name.to_sym]
+  unless sdef[:type] == :structure then
+    puts "ERROR: schema validation failed: root element should be a structure"
+  end  
+  # first, check that each doc element is defined for the struct
+  validate_doc_struct_against_schema(sdef[:ref], root, root.name)
+  # second, check that all required elements from the struct are actually present in the doc
+  validate_schema_against_doc_struct(sdef[:ref], root, root.name)
+end
+
+def validate_doc_struct_against_schema(struct, doc, prefix = nil)
+  doc.children.to_a.filter{|x| x.is_a?(Nokogiri::XML::Element) }.each do |c|
+    fdef = struct[c.name.to_sym]
+    if prefix then
+      xpath = "#{prefix}/#{c.name}"
+    else
+      xpath = c.name
+    end
+    if fdef
+      if fdef[:type] == :structure then
+        return unless validate_doc_struct_against_schema(fdef[:ref], c, xpath)
+      end
+    else
+      if prefix then
+        xpath = "#{prefix}/#{c.name}"
+      else
+        xpath = c.name
+      end
+      raise FatalError.new("ERROR: schema validation failed: #{c.name} of #{xpath} is present in document, but not defined in schema.")
+    end
+  end
+end
+
+def validate_schema_against_doc_struct(struct, doc, prefix = nil)
+  struct.each do |key, value|
+    next if key == :additional_smtlib || key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix
+      if prefix then
+        xpath = "#{prefix}/#{key}"
+      else
+        xpath = key
+      end
+    if value[:type] == :structure then
+      node = doc.at_xpath(key.to_s)
+      if node then
+        validate_schema_against_doc_struct(value[:ref], node, xpath)
+      else
+        if value[:optional] == false then
+          raise FatalError.new("ERROR: schema validation failed: non-optional structure #{xpath} not found in doc.")
+        end
+      end
+    elsif value[:type] == :field then
+      if value[:optional] == false then
+        unless doc.at_xpath(key.to_s) then
+          raise FatalError.new("ERROR: schema validation failed: non-optional field #{xpath} not found in doc.")
+        end
+      end
+    elsif value[:type] == :list then
+      if value[:optional] == false then
+        unless doc.at_xpath(key.to_s) then
+          raise FatalError.new("ERROR: schema validation failed: non-optional list #{xpath} not found in doc.")
+        end
+      end
+      list = doc.at_xpath(key.to_s)
+      if list then
+        list.children.to_a.each do |c|
+          next unless c.is_a?(Nokogiri::XML::Element)
+          unless c.name == value[:xpath_element] then
+            raise FatalError.new("ERROR: schema validation failed: encountered element #{c.name} in list #{xpath}, which should only contain #{value[:xpath_element]} elements.")
+          end
+        end
+      end
+    else
+      raise RuntimeError.new("Model error: encountered element #{xpath} of unknown type: #{value[:type]}.")
+    end
+  end  
+end
+
+def translate_doc_to_SMTLIB(progress, solver, struct, doc, hash = {})
+  options = hash[:options]
+  config = hash[:config]
+
+  solver.to_solver(progress, "; --- Document: ---")
+  translate_struct_in_doc_to_SMTLIB(progress, solver, struct, doc, nil, hash)
+end
+
+def translate_struct_in_doc_to_SMTLIB(progress, solver, struct, doc, prefix, hash = {})
+  options = hash[:options]
+  config = hash[:config]
+
+  struct.each do |key, value|
+    next if key == :additional_smtlib || key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix
+    if prefix then
+      xpath = "#{prefix}/#{key.to_s}"
+    else
+      xpath = key.to_s
+    end
+    if value[:type] == :structure then
+      node = doc.at_xpath(xpath)
+      if node then
+        info = {
+          :assertion_type => :doc_structure_exists,
+          :xpath => xpath,
+        }
+        assertionID = "doc-#{xpath.gsub('/', '-')}"
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (= #{exists_var_for_structure(xpath)} true) :named #{assertionID}))")
+
+        translate_struct_in_doc_to_SMTLIB(progress, solver, value[:ref], doc, xpath, hash)
+      else
+        info = {
+          :assertion_type => :doc_structure_not_exists,
+          :xpath => xpath,
+        }
+        assertionID = "doc-#{xpath.gsub('/', '-')}"
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (= #{exists_var_for_structure(xpath)} false) :named #{assertionID}))")
+      end
+    elsif value[:type] == :field then
+      node = doc.at_xpath(xpath)
+      if node then
+        fvalue = node.text
+        info = {
+          :assertion_type => :doc_field_filled,
+          :xpath => xpath,
+          :value => fvalue
+        }
+        fvalue_str = value_of_type(fvalue, value[:datatype])
+        assertionID = "doc-#{xpath.gsub('/', '-')}"
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (and (= #{filled_var_for_field(xpath)} true) (= #{value_var_for_field(xpath)} #{fvalue_str})) :named #{assertionID}))")
+      else
+        info = {
+          :assertion_type => :doc_field_empty,
+          :xpath => xpath,
+        }
+        assertionID = "doc-#{xpath.gsub('/', '-')}"
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (= #{filled_var_for_field(xpath)} false) :named #{assertionID}))")
+      end
+    elsif value[:type] == :list then
+      node = doc.at_xpath(xpath)
+      if node then
+        fvalues = []
+        node.children.to_a.each do |c|
+          next unless c.is_a?(Nokogiri::XML::Element)          
+          fvalues << c.text
+        end
+        info = {
+          :assertion_type => :doc_list_filled,
+          :xpath => xpath,
+          :values => fvalues
+        }
+        fvalue_str = value_of_type(fvalues, value[:datatype])
+        assertionID = "doc-#{xpath.gsub('/', '-')}"
+        solver.associateAssertionIDWith(assertionID, info)
+        clauses = []
+        fvalues.each_with_index do |fvalue, index|
+          clauses << "(= #{value_var_for_list(xpath,index)} #{value_of_type(fvalue, value[:datatype])})"
+        end
+        solver.to_solver(progress, "(assert (! (and (= #{filled_var_for_field(xpath)} true) (= #{size_var_for_list(xpath)} #{fvalues.size}) #{clauses.join(' ')}) :named #{assertionID}))")
+      else
+        info = {
+          :assertion_type => :doc_field_empty,
+          :xpath => xpath,
+        }
+        assertionID = "doc-#{xpath.gsub('/', '-')}"
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (= #{filled_var_for_field(xpath)} false) :named #{assertionID}))")
+      end
+    else
+      raise RuntimeError.new("Model Error: encountered unknown type #{value[:type].inspect}")
+    end
+  end
+end
+
+def validate_doc(progress, struct, doc, options, config)
+  raise RuntimeError.new("option :solverFactory is required") unless options[:solverFactory]
+  options[:solverLog] = "validation.smt2"
+
+  solverFactory = options[:solverFactory]
+  result = solverFactory.query(options) do |solver|
+    set_standard_options(progress, solver)
+    translate_structure_to_SMTLIB(progress, solver, struct, :options => options, :config => config)
+    translate_doc_to_SMTLIB(progress, solver, struct, doc, :options => options, :config => config)
+    solver.to_solver(progress, "")
+  end
+  unless result == true then
+    raise FatalError.new("VALIDATION FAILED! SMTLIB log was written to #{options[:solverLog]}.\n\n#{result}")
+  end
+
+  return true
+end
+
+def generate_unused_logfile_name()
+  number = 0
+  result = "logs/#{number}.log"
+  while File.exist?(result) do
+    number += 1
+    result = "logs/#{number}.log"
+  end
+  result
+end
+
+def generate(message, argv, message_config_filename = nil)
+  if ARGV[0] == "--help" || ARGV[0] == "-help" || ARGV[0] == "help" then
+    puts USAGE
+    exit(0)
+  end
+
+  options = { :negated_validation_rules => [], :date_now => (Date.today - Time.at(0).to_date).to_i, :timestamp_now => Time.now().to_i }
+  
+  opt = argv.shift
+  while opt do
+    if opt == "-negate" then
+      rule = argv.shift
+      options[:negated_validation_rules] << rule
+    elsif opt == "-list-keys" then
+      options[:list_keys] = true
+    elsif opt == "-list" then
+      options[:list] = argv.shift
+    elsif opt == "-list-validation-rules" then
+      options[:list_validation_rules] = true      
+    elsif opt == "-generate-docs-from" then
+      options[:combination_source] = argv.shift
+    elsif opt == "-skip" then
+      options[:skip] = Integer(argv.shift)
+    elsif opt == "-generate-docs-for-key" then
+      options[:docs_for_key] = argv.shift
+    elsif opt == "-count-docs-for-key" then
+      options[:count_docs_for_key] = argv.shift
+    elsif opt == "-continue" then
+      options[:continue] = true
+    elsif opt == "-max-num-docs" then
+      options[:max_num_docs] = Integer(argv.shift)
+      puts "flag -max-num-docs used. Will stop after generating #{options[:max_num_docs]} documents."
+    elsif opt == "-validate" then
+      if options.has_key?(:validate_files) then
+        options[:validate_files] << argv.shift
+      else
+        options[:validate_files] = [argv.shift]
+      end
+    else
+      puts "FATAL: unknown option #{opt.inspect}."
+      puts USAGE
+      raise FatalError.new("")
+    end
+    opt = argv.shift
+  end
+
+  ProgressBar.logging(1, generate_unused_logfile_name()) do |progress|
+    unless options[:negated_validation_rules].empty? then
+      progress.print_line "negating rules: #{options[:negated_validation_rules].join(", ")}"
+
+      rules = collect_validation_rule_names(message)
+      unknown_negated_rules = options[:negated_validation_rules].filter { |rule_name| !rules.include?(rule_name) } 
+      unless unknown_negated_rules.empty? then  
+        throw RuntimeError.new("unknown negated rules #{unknown_negated_rules.join(", ")}")
+      end
+
+    end
+
+    if File.exist?(CONFIG_FILE) then
+      config = YAML.load_file(CONFIG_FILE)
+    else
+      config = DEFAULT_CONFIG
+      File.open(CONFIG_FILE, "w") do |f|
+        f.puts(config.to_yaml)
+      end
+      progress.print_line "No config file found. Default config written to #{CONFIG_FILE}"
+    end
+    if message_config_filename then
+      message_config = YAML.load_file(message_config_filename)
+      config = config.merge(message_config)
+    end
+
+    introduce_parent_links(message)
+    z3path = config[:z3path] || DEFAULT_Z3_PATH
+    options[:solverFactory] = SolverFactory.new(progress, z3path)
+
+    if options[:validate_files] then
+      progress.print_line "using #{options[:date_now]} as [Date.now]"
+      progress.print_line "using #{options[:timestamp_now]} as [Timestamp.now]"
+      options[:validate_files].each do |filename|
+        progress.print_line "validating file #{filename}"
+        doc = Nokogiri::XML(File.read(filename))
+        validate_doc_against_schema(message, doc)
+        validate_doc(progress, message, doc, options, config)
+      end
+    elsif options[:count_docs_for_key] then
+      progress.print_line "there are #{calc_combinations(message, options[:count_docs_for_key])} combinations."
+    elsif options[:docs_for_key] then
+      progress.print_line "using #{options[:date_now]} as [Date.now]"
+      progress.print_line "using #{options[:timestamp_now]} as [Timestamp.now]"
+      validate_model(message)
+      CoreEnumerationAlgo.new().generate_docs_for_key(progress, message, options[:docs_for_key], OUTPUT_DIR, config, options)
+    elsif options[:list_keys] then
+      list_keys(message)
+    elsif options[:list] then
+      list_fields_and_structures(message, options[:list])
+    elsif options[:list_validation_rules] then
+      list_validation_rules(message)
+    else
+      progress.print_line "using #{options[:date_now]} as [Date.now]"
+      progress.print_line "using #{options[:timestamp_now]} as [Timestamp.now]"
+      validate_model(message)
+      generate_doc(progress, message, OUTPUT_FILE, config, options)
+    end
+  end
+end
+