mbt-gen 0.0.1

data/lib/mbt-gen.rb

@@ -0,0 +1,1774 @@
+#!/usr/bin/ruby
+
+require 'date'
+require 'fileutils'
+require 'tmpdir'
+require 'yaml'
+require 'time'
+
+require 'nokogiri'
+
+require_relative 'solver-lib'
+require_relative 'progress'
+require_relative 'regexp-to-smtlib'
+
+
+CHUNK_SIZE = 10000
+DEFAULT_Z3_PATH = "z3"
+
+CONFIG_FILE = "config.yml"
+OUTPUT_FILE = "result.xml"
+CONTINUATION_FILE = 'continuation.log'
+COMBINATION_LOG = 'combinations.log'
+VARS_LOG = 'vars.log'
+
+OUTPUT_DIR = "result"
+LISTS_DIR = "lists"
+QUERY_DIR = "queries"
+PARAMETERIZED_QUERY_DIR = "parameterized_queries"
+XML_TEMPLATE = ""
+
+DEFAULT_CONFIG = {
+  :z3path => "z3",
+  :generation_system_mode => "Production",
+  :generation_system => "Production"
+}
+
+USAGE = <<EOF
+usage:
+
+GENERATING A SINGLE MESSAGE:
+
+  ruby #{$PROGRAM_NAME} -list-validation-rules
+  - outputs a list of all validation rule names that have been specified in #{$PROGRAM_NAME}.
+
+  ruby #{$PROGRAM_NAME} <options>
+  - generates a message of the given type, which
+    - is valid XML
+    - adheres to the schema, and
+    - satisfies all validation rules specified for messages of this type.
+  where <options> is a combination of
+    -negate <validation_rule> - negates the named validation rule. This causes the resulting message to still satisfy all other validation rules, but to violate this one.
+  NOTE: the result can be found in #{OUTPUT_FILE}
+  
+GENERATING MESSAGES FOR ALL COMBINATIONS:
+
+  ruby #{$PROGRAM_NAME} -list-keys
+  - outputs a list of all keys that have been used in #{$PROGRAM_NAME} to mark fields or structures.
+
+  ruby #{$PROGRAM_NAME} -list <key>
+  - outputs a list of all fields and structures that have been marked with <key> in #{$PROGRAM_NAME}.
+
+  ruby #{$PROGRAM_NAME} -count-docs-for-key <key>
+  - calculates the number of combinations of all fields and structures marked with <key>
+
+  ruby #{$PROGRAM_NAME} -generate-docs-for-key <key> <options>
+  - generates all valid messages of the given type that can be derived from combinations of all the fields and structures marked with <key>.
+  where <options> is a combination of
+    -continue - continues a previously interrupted generation process. Must use the same <key> the the interrupted generation.ARGF
+    -max-num-docs <limit> - stops generation process after generating <limit> documents.
+  NOTE: enumerating large combination-spaces may take a long time.
+        - Use -count-docs-for-key to measure the size of the combination-space beforehand.
+        - Also: you can interrupt the generation process at any time using Ctrl+C. The intermediate result is stored in the directory #{OUTPUT_DIR} and the generation process can
+          be restarted later by adding -continue to the command line.
+  NOTE: do not expect the number of generated documents to equal the number of combinations as combinations may not have any valid message instances (for instance when they contradict the validation rules).
+  NOTE: the result can be found in the directory #{OUTPUT_DIR}
+EOF
+
+class FatalError < RuntimeError
+end
+
+class NormalProgramTermination < RuntimeError
+end
+
+def raw_field(xpath)
+  path_str = "base"
+  path_str = xpath.gsub("/", "-") if xpath
+  "field-#{path_str}"
+end
+
+def filled_var_for_field(xpath)
+  "#{raw_field(xpath)}-filled"
+end
+
+def value_var_for_field(xpath)
+  "#{raw_field(xpath)}-value"
+end
+
+def exists_var_for_structure(xpath)
+  struct_str = "base"
+  struct_str = xpath.gsub("/", "-") if xpath
+  "struct-#{struct_str}-exists"
+end
+
+def size_var_for_list(xpath)
+  "#{raw_field(xpath)}-size"
+end
+
+def value_var_for_list(xpath, index)
+  "#{raw_field(xpath)}-index-#{index}-value"
+end
+
+def translate_datatype_to_SMTLIB(datatype)
+  case datatype
+  when :string
+    return "String"
+  when :int
+    return "Int"
+  when :bool
+    return "Bool"
+  when :date
+    return "Int"
+  when :timestamp
+    return "Int"
+  when :enum
+    return "String"
+  end
+end
+
+def translate_value_to_SMTLIB(value)
+  if value.is_a?(Integer) then
+    return value.to_s
+  elsif value.is_a?(String) then
+    return value.inspect
+  elsif value.is_a?(Date) then
+    return (value - (Time.at(0).to_date)).to_i  # days since 1970.01.01
+  elsif value.is_a?(Time) then
+    return value.to_i
+  else
+    raise RuntimeError.new("cannot translate value to SMTLIB: #{value.inspect}")
+  end
+end
+
+def min_constraint_for_expr(expr, lower_bound)
+  "(>= #{expr} #{lower_bound})"
+end
+
+def max_constraint_for_expr(expr, upper_bound)
+  "(<= #{expr} #{upper_bound})"
+end
+
+def regex_constraint_for_expr(expr, regex)
+  "(str.in.re #{expr} #{Regexp_to_SMTLIB.translate_regexp_to_SMTLIB(regex)})"
+end
+
+def maxLength_contraint_for_expr(expr, maxLength)
+  "(<= (str.len #{expr}) #{maxLength})"
+end
+
+def oneOf_contraint_for_expr(expr, values)
+  list = values.map do |val|
+    "(= #{expr} \"#{val}\")"
+  end
+  "(or #{list.join(" ")})"
+end
+
+def enum_constraint_for_expr(expr, values)
+  "(or #{values.map{|x| "(= #{expr} #{translate_value_to_SMTLIB(x)})"}.join(" ")})"
+end
+
+def translate_field_def_to_SMTLIB_constraints(progress, solver, fielddef, varname)
+  if fielddef[:optional] == nil || fielddef[:optional] == false then
+    # field is required
+    solver.to_solver(progress, "(assert (! #{varname}-filled :named required-#{varname}))") 
+  end
+  datatype = fielddef[:datatype]
+  case datatype
+  when :int
+    if fielddef[:min] then
+        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{min_constraint_for_expr("#{varname}-value", translate_value_to_SMTLIB(fielddef[:min]))}) :named min-#{varname}))") 
+    end
+    if fielddef[:max] then
+        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{max_constraint_for_expr("#{varname}-value", translate_value_to_SMTLIB(fielddef[:max]))}) :named max-#{varname}))") 
+    end
+  when :string
+    if fielddef[:regex] then
+        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{regex_constraint_for_expr("#{varname}-value", fielddef[:regex])}) :named regex-#{varname}))") 
+    end 
+    # TODO: minLength?
+    if fielddef[:maxLength] then
+      solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{maxLength_contraint_for_expr("#{varname}-value", fielddef[:maxLength])}) :named maxLength-#{varname}))") 
+    end
+    if fielddef[:from_list] then
+      list_filename = "#{LISTS_DIR}/#{fielddef[:from_list]}"
+      unless File.exist?(list_filename)
+        raise RuntimeError.new("could not find list file #{list_filename}.")
+      end
+      values = File.read(list_filename).split("\n").map{|v| v.chomp()}
+      solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{oneOf_contraint_for_expr("#{varname}-value", values)}) :named oneOf-#{varname}))") 
+    end
+  when :enum
+    solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{enum_constraint_for_expr("#{varname}-value", fielddef[:values])}) :named enum-#{varname}))") 
+  end
+end
+
+def translate_list_def_to_SMTLIB_constraints(progress, solver, fielddef, varname)
+  datatype = fielddef[:datatype]
+  case datatype
+  when :int
+    if fielddef[:min] then
+        fielddef[:model_maxLength].times do |i|
+          solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{min_constraint_for_expr("#{varname}-index-#{i}-value", translate_value_to_SMTLIB(fielddef[:min]))}) :named list-min-#{varname}-list-index-#{i}))") 
+        end
+    end
+    if fielddef[:max] then
+        fielddef[:model_maxLength].times do |i|
+          solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{max_constraint_for_expr("#{varname}-index-#{i}-value", translate_value_to_SMTLIB(fielddef[:max]))}) :named list-max-#{varname}-list-index-#{i}))") 
+        end
+    end
+  when :string
+    if fielddef[:regex] then
+        fielddef[:model_maxLength].times do |i|
+          solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{regex_constraint_for_expr("#{varname}-index-#{i}-value", fielddef[:regex])}) :named list-regex-#{varname}-index-#{i}))") 
+        end
+    end 
+    # TODO: minLength?
+    if fielddef[:maxLength] then
+        fielddef[:model_maxLength].times do |i|
+          solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{maxLength_contraint_for_expr("#{varname}-index-#{i}-value", fielddef[:maxLength])}) :named list-maxLength-#{varname}-index-#{i}))") 
+        end
+    end 
+  when :enum
+    fielddef[:model_maxLength].times do |i|
+      solver.to_solver(progress, "(assert (! (=> (< #{i} #{varname}-size) #{enum_constraint_for_expr("#{varname}-index-#{i}-value", fielddef[:values])}) :named list-enum-#{varname}-index-#{i}))") 
+    end
+  end
+end
+
+def default_value_for_fielddef(fielddef)
+  datatype = fielddef[:datatype]
+  case datatype
+  when :int
+    return "0"
+  when :string
+    return "".inspect
+  when :date
+    return "0"
+  when :timestamp
+    return "0"
+  when :enum
+    return "".inspect
+  else
+    raise RuntimeError.new("unknown datatype: #{datatype.inspect}")
+  end
+end
+
+def translate_structure_to_SMTLIB(progress, solver, struct, hash = {})
+  prefix = hash[:prefix]
+  config = hash[:config]
+  options = hash[:options]
+  translate_substructure_to_SMTLIB(progress, solver, struct, hash)
+  translate_validation_rules_to_SMTLIB(progress, solver, struct, prefix, config, options)
+  if options[:add_SMTLIB] then
+    solver.to_solver(progress, "; --- additional SMTLIB Code: ---")
+    solver.to_solver(progress, options[:add_SMTLIB])
+  end
+end
+
+def translate_substructure_to_SMTLIB(progress, solver, struct, hash = {})
+  prefix = hash[:prefix]
+  struct_varname = hash[:struct_varname]
+  config = hash[:config]
+  options = hash[:options]
+  solver.to_solver(progress, "; --- Structure: ---")
+  struct.each do |field, definition|
+    if field == :additional_smtlib then
+      solver.to_solver(progress, "; additional SMTLIB of structure #{prefix}")
+      solver.to_solver(progress, translate_validation_rule_to_SMTLIB(definition, struct, prefix, config, options, "<additional SMTLIB of structure #{prefix}>"))
+    elsif field == :validation_rules || field == :predicates || field == :parent_link || field == :struct_xpath_prefix
+      next 
+    else
+      solver.to_solver(progress, "; #{field.to_s}")
+      if definition[:type] == :field then
+        display_prefix = ""
+        display_prefix = "#{prefix}/" if prefix
+        xpath = field.to_s
+        xpath = "#{prefix}/#{xpath}" if prefix
+        solver.declare_const(progress, filled_var_for_field(xpath), "Bool", {:xpath => xpath})
+        solver.declare_const(progress, value_var_for_field(xpath), translate_datatype_to_SMTLIB(definition[:datatype]), {:xpath => xpath, :datatype => definition[:datatype]})
+        solver.to_solver(progress, "(assert (! (=> (not #{filled_var_for_field(xpath)}) (= #{value_var_for_field(xpath)} #{default_value_for_fielddef(definition)})) :named empty-field-has-no-value-for-#{raw_field(xpath)}))") 
+        if (struct_varname)
+          solver.to_solver(progress, "(assert (! (=> (not #{struct_varname}) (not #{filled_var_for_field(xpath)})) :named empty-struct-has-no-fields-for-#{raw_field(xpath)}))") 
+        end
+        translate_field_def_to_SMTLIB_constraints(progress, solver, definition, raw_field(xpath))
+      elsif definition[:type] == :list then
+        display_prefix = ""
+        display_prefix = "#{prefix}/" if prefix
+        xpath = field.to_s
+        xpath = "#{prefix}/#{xpath}" if prefix
+        solver.declare_const(progress, size_var_for_list(xpath), "Int", {:xpath => xpath})
+        definition[:model_maxLength].times do |i|
+          solver.declare_const(progress, value_var_for_list(xpath, i), translate_datatype_to_SMTLIB(definition[:datatype]), {:xpath => xpath, :xpath_element => definition[:xpath_element], :index => i, :datatype => definition[:datatype]})
+          solver.to_solver(progress, "(assert (! (=> (<= #{size_var_for_list(xpath)} #{i}) (= #{value_var_for_list(xpath, i)} #{default_value_for_fielddef(definition)})) :named empty-list-field-has-no-value-for-#{raw_field(xpath)}-index-#{i}))") 
+        end
+        if (struct_varname)
+          solver.to_solver(progress, "(assert (! (=> (not #{struct_varname}) (= #{size_var_for_list(xpath)} 0)) :named empty-struct-has-no-fields-for-#{raw_field(xpath)}))") 
+        end
+        translate_list_def_to_SMTLIB_constraints(progress, solver, definition, raw_field(xpath))
+      elsif definition[:type] == :structure then
+        xpath = field.to_s
+        xpath = "#{prefix}/#{xpath}" if prefix
+        solver.declare_const(progress, exists_var_for_structure(xpath), "Bool", {:xpath => xpath})
+        if (struct_varname)
+          solver.to_solver(progress, "(assert (! (=> (not #{struct_varname}) (not #{exists_var_for_structure(xpath)})) :named empty-struct-has-no-substructure-#{raw_field(xpath)}))") 
+        end
+        translate_substructure_to_SMTLIB(progress, solver, definition[:ref], :prefix => xpath, :struct_varname => exists_var_for_structure(xpath), :options => options, :config => config)
+      else
+        next
+      end
+    end
+  end
+end
+
+def resolve_field_path(path, relative_to_struct, prefix)
+  current = relative_to_struct
+  result = nil
+  path.split("/").each do |segment|
+    unless current then
+      raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect}: resolution worked up until #{result.inspect}, but then the last segment was a field or list.")
+    end
+    field_def = current[segment.to_sym]
+    unless field_def then
+      raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect}: structure #{prefix.inspect} does not have a member #{segment.to_sym.inspect}.")
+    end
+    if field_def[:type] == :structure then
+      unless field_def[:ref] then
+        raise RuntimeError.new("could not resolve field path: #{path.inspect}: substructure #{segment.inspect} of structure #{prefix.inspect} does not have a :ref.")
+      end
+      current = field_def[:ref]
+      if prefix then
+        prefix = "#{prefix}/#{segment}"
+      else
+        prefix = segment
+      end
+    elsif field_def[:type] == :field then
+      current = nil
+      result = segment
+      if prefix
+        result = "#{prefix}/#{result}"
+      end
+    elsif field_def[:type] == :list then
+      current = nil
+      result = segment
+      if prefix
+        result = "#{prefix}/#{result}"
+      end
+    end
+  end
+  return result
+end
+
+def resolve_field_def(path, relative_to_struct, prefix)
+  current = relative_to_struct
+  result = nil
+  path.split("/").each do |segment|
+    unless current then
+      raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect}: resolution worked up until #{result.inspect}, but then the last segment was a field or list.")
+    end
+    field_def = current[segment.to_sym]
+    unless field_def then
+      raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect}: structure #{prefix.inspect} does not have a member #{segment.to_sym.inspect}.")
+    end
+    if field_def[:type] == :structure then
+      unless field_def[:ref] then
+        raise RuntimeError.new("could not resolve field path: #{path.inspect}: substructure #{segment.inspect} of structure #{prefix.inspect} does not have a :ref.")
+      end
+      current = field_def[:ref]
+      if prefix then
+        prefix = "#{prefix}/#{segment}"
+      else
+        prefix = segment
+      end
+    elsif field_def[:type] == :field then
+      current = nil
+      result = field_def
+    elsif field_def[:type] == :list then
+      current = nil
+      result = field_def
+    end
+  end
+  return result
+end
+
+def resolve_struct_path(path, relative_to_struct, prefix)
+  current = relative_to_struct
+  result = nil
+  path.split("/").each do |segment|
+    unless current then
+      raise RuntimeError.new("could not resolve field path: #{path.inspect} relative to struct #{relative_to_struct.inspect}")
+    end
+    if segment == "." then
+      result = current[:struct_xpath_prefix]
+    elsif segment == ".." then
+      current = current[:parent_link]
+      if prefix then
+        prefix_segs = prefix.split("/")
+        prefix = prefix_segs[0,prefix_segs.size()-1].join("/")
+      else
+        prefix = current[:struct_xpath_prefix]
+      end
+    else
+      field_def = current[segment.to_sym]
+      unless field_def then
+        raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect} relative to struct #{relative_to_struct.inspect}")
+      end
+      if field_def[:type] == :structure then
+        current = field_def[:ref]
+        if prefix then
+          prefix = "#{prefix}/#{segment}"
+        else
+          prefix = segment
+        end
+        result = prefix
+      end
+    end
+  end
+  return result
+end
+
+def resolve_struct_def(path, relative_to_struct, prefix, context_desc = "")
+  current = relative_to_struct
+  result = nil
+  path.split("/").each do |segment|
+    unless current then
+      raise RuntimeError.new("could not resolve field path: #{path.inspect} relative to struct #{relative_to_struct.inspect} #{context_desc}")
+    end
+    if segment == "." then
+      result = current
+    elsif segment == ".." then
+      current = current[:parent_link]
+      if prefix then
+        prefix_segs = prefix.split("/")
+        prefix = prefix_segs[0,prefix_segs.size()-1].join("/")
+      else
+        prefix = current[:struct_xpath_prefix]
+      end
+    else
+      field_def = current[segment.to_sym]
+      unless field_def then
+        raise RuntimeError.new("could not resolve segment #{segment.inspect} of field path: #{path.inspect} relative to struct #{relative_to_struct.inspect} #{context_desc}")
+      end
+      if field_def[:type] == :structure then
+        current = field_def[:ref]
+        if prefix then
+          prefix = "#{prefix}/#{segment}"
+        else
+          prefix = segment
+        end
+        result = current
+      end
+    end
+  end
+  return result
+end
+
+def translate_validation_rules_to_SMTLIB(progress, solver, struct, prefix, config, options)
+  solver.to_solver(progress, "; --- Validation Rules ---")
+  translate_validation_rules_for_structure_to_SMTLIB(progress, solver, struct, prefix, config, options)
+  struct.each_pair do |key, value|
+    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if value[:type] == :structure then
+      if prefix then
+        new_prefix = "#{prefix.to_s}/#{key}"
+      else
+        new_prefix = key.to_s
+      end
+      translate_validation_rules_to_SMTLIB(progress, solver, value[:ref], new_prefix, config, options)
+    end
+  end
+end
+
+def translate_validation_rule_to_SMTLIB(rule, context_struct, prefix, config, options, rule_name)
+  date_now = options[:date_now]
+  timestamp_now = options[:timestamp_now]
+  return rule.gsub(/FILLED\[[^\]]*\]/) do |m|
+                    unless m =~ /FILLED\[([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse FILLED[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    field = resolve_field_path($1, context_struct, prefix)
+                    filled_var_for_field(field)
+                  end
+                  .gsub(/VALUE\[[^\]]*\]/) do |m|
+                    unless m =~ /VALUE\[([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse VALUE[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    field = resolve_field_path($1, context_struct, prefix)
+                    value_var_for_field(field)
+                  end
+                  .gsub(/EXIST\[[^\]]*\]/) do |m|
+                    unless m =~ /EXIST\[([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse EXIST[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    s = resolve_struct_path($1, context_struct, prefix)
+                    exists_var_for_structure(s)
+                  end
+                  .gsub(/SIZE\[[^\]]*\]/) do |m|
+                    unless m =~ /SIZE\[([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse SIZE[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    l = resolve_field_path($1, context_struct, prefix)
+                    size_var_for_list(l)
+                  end
+                  .gsub(/LIST_CONTAINS\[[^\]]*,[^\]]*\]/) do |m|
+                    unless m =~ /LIST_CONTAINS\[([^\]]*),([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse LIST_CONTAINS[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    l = resolve_field_path($1, context_struct, prefix)
+                    l_def = resolve_field_def($1, context_struct, prefix)
+                    value = $2
+                    terms = []
+                    l_def[:model_maxLength].times do |i|
+                      terms << "(and (< #{i} #{size_var_for_list(l)}) (= #{value_var_for_list(l, i)} #{value}))"
+                    end
+                    "(or #{terms.join(" ")})"
+                  end
+                  .gsub(/LIST_VALUE_FIRST_ELEMENT\[[^\]]*\]/) do |m|
+                    unless m =~ /LIST_VALUE_FIRST_ELEMENT\[([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse LIST_VALUE_FIRST_ELEMENT[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    xpath = $1
+                    l = resolve_field_path(xpath, context_struct, prefix)
+                    value_var_for_list(l, 0)
+                  end
+                  .gsub("[Date.now]", date_now.to_s)
+                  .gsub("[Timestamp.now]", timestamp_now.to_s)
+                  .gsub(/PREDICATE\[[^\]]*,[^\]]*\]/) do |m|
+                    unless m =~ /PREDICATE\[([^\]]*),([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse PREDICATE[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    path = $1
+                    key = $2.strip().to_sym
+                    sdef = resolve_struct_def(path, context_struct, prefix, "in validation rule #{rule_name}")
+                    unless sdef
+                      raise RuntimeError.new("could not resolve struct path #{path.inspect} in validation rule #{rule_name}")
+                    end
+                    preds = sdef[:predicates]
+                    mdk = preds[key]
+                    if mdk then
+                      if mdk[:smtlib] then
+                        translate_validation_rule_to_SMTLIB(mdk[:smtlib], sdef, sdef[:struct_xpath_prefix], config, options, "#{rule_name}.predicate.#{key}[#{sdef[:struct_xpath_prefix]}]" )
+                      else
+                        raise RuntimeError.new("Predicate #{key} for structure #{sdef[:struct_xpath_prefix]} does not have an SMTLIB definition, but is referenced in validation rule #{rule_name}")
+                      end
+                    else
+                      raise RuntimeError.new("No predicate #{key.inspect} defined for structure #{sdef[:struct_xpath_prefix]}, but one referenced in validation rule #{rule_name}")
+                    end
+                  end
+                  .gsub(/CONFIG\[[^\]]*\]/) do |m|
+                    unless m =~ /CONFIG\[([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse LIST_CONTAINS[...]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    key = $1.to_sym
+                    if config.has_key?(key) then
+                      "\"#{config[key]}\"" 
+                    else
+                      raise RuntimeError.new("unknown CONFIG-key #{key.inspect} encountered in validation rule #{rule_name}" )
+                    end
+                  end
+                  .gsub(/FIELD_IN_QUERY\[[^\]]*\]/) do |m|
+                    unless m =~ /FIELD_IN_QUERY\[([^\]]*),([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse FIELD_IN_QUERY[..,..]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    xpath = $1
+                    query_filename = $2
+                    fpath = resolve_field_path(xpath, context_struct, prefix)
+                    fdef = resolve_field_def(xpath, context_struct, prefix)
+                    values = File.read("#{QUERY_DIR}/#{query_filename.strip()}").split("\n")
+                    if fdef[:datatype] == :string then
+                      terms = values.filter{|l| !l.start_with?("#")}.map do |val|
+                        "(= #{value_var_for_field(fpath)} \"#{val}\")"
+                      end
+                    elsif fdef[:datatype] == :int then
+                      terms = values.filter{|l| !l.start_with?("#")}.map do |val|
+                        "(= #{value_var_for_field(fpath)} #{Integer(val)})"
+                      end
+                    else
+                      raise RuntimeError.new("FIELD_IN_QUERY[] was applied to a field of datatype #{fdef[:datatype].inspect}. However, it can only be applied to fields of datatype string or int.")
+                    end
+                    "(or #{terms.join(" ")})"
+                  end
+                  .gsub(/FIELD_IN_SMTLIB_PARAMETERIZED_QUERY\[[^\]]*\]/) do |m|
+                    unless m =~ /FIELD_IN_SMTLIB_PARAMETERIZED_QUERY\[([^\]]*),([^\]]*),([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse FIELD_IN_SMTLIB_PARAMETERIZED_QUERY[..,..,..]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    field_xpath = $1
+                    param_smtlib = $2
+                    query_filename = $3
+                    fpath = resolve_field_path(field_xpath.strip(), context_struct, prefix)
+                    fdef = resolve_field_def(field_xpath.strip(), context_struct, prefix)
+                    query_result = YAML.load_file("#{PARAMETERIZED_QUERY_DIR}/#{query_filename.strip()}")
+                    terms = []
+                    if fdef[:datatype] == :string then
+                      terms << "(or #{query_result.keys.map { |key| "(= #{param_smtlib} \"#{key}\")" }.join(" ")})"
+                      query_result.each_pair do |key, values|
+                        terms << "(=> (= #{param_smtlib} \"#{key}\") (or #{values.map{ |v| "(= #{value_var_for_field(fpath)} \"#{v}\")" }.join(" ")}))"
+                      end
+                    else
+                      raise RuntimeError.new("FIELD_IN_SMTLIB_PARAMETERIZED_QUERY[] was applied to a field of datatype #{fdef[:datatype].inspect}. However, it can only be applied to fields of datatype string.")
+                    end
+                    "(and #{terms.join(" ")})"
+                  end
+                  .gsub(/CONFIG_IN_QUERY\[[^\]]*\]/) do |m|
+                    unless m =~ /CONFIG_IN_QUERY\[([^\]]*),([^\]]*)\]/ then
+                      raise RuntimeError.new("could not parse CONFIG_IN_QUERY[..,..]: #{m.inspect} in validation rule #{rule_name}")
+                    end
+                    key = $1.to_sym
+                    unless config.has_key?(key) then
+                      raise RuntimeError.new("unknown CONFIG key #{key.inspect} encountered in validation rule #{rule_name}")
+                    end
+                    query_filename = $2
+                    values = File.read("#{QUERY_DIR}/#{query_filename.strip()}").split("\n")
+                    terms = values.filter{|l| !l.start_with?("#")}.map do |val|
+                      "(= \"#{config[key]}\" \"#{val}\")"
+                    end
+                    "(or #{terms.join(" ")})"
+                  end
+end
+
+def translate_validation_rules_for_structure_to_SMTLIB(progress, solver, struct, prefix, config, options)
+  return unless struct[:validation_rules]
+
+  solver.to_solver(progress, "; --- Validation Rules: ---")
+  negated_rules = options[:negated_validation_rules] || []
+  solver.to_solver(progress, "; negated rules: #{negated_rules.empty? ? "<none>" : negated_rules.join(", ")}")
+
+  rules = struct[:validation_rules]
+  rules.each do |rule_name, rule|
+    rule = rule[:smtlib]  # ignore the text representation
+    next unless rule # warning about this omission is the job of validate_model()
+    rule_SMTLIB = translate_validation_rule_to_SMTLIB(rule, struct, prefix, config, options, rule_name)
+    if negated_rules.include?(rule_name) then
+      rule_SMTLIB = "(not #{rule_SMTLIB})"
+    end
+    solver.to_solver(progress, "(assert (! #{rule_SMTLIB} :named validation_rule_#{rule_name}_instance_#{prefix.gsub("/", "_")}))")
+  end
+end
+
+def parse_field_value(fielddef, value)
+  value = value.gsub("\n", "")
+  datatype = fielddef[:datatype]
+  case datatype
+  when :int
+    return Integer(value)
+  when :string
+    return value
+  when :enum
+    return value
+  when :date
+    return Date.parse(value)
+  when :timestamp
+    return Time.new(value)
+  end
+end
+
+def solver_value_to_string(solver_value)
+  if solver_value =~ /\"(.*)\"/ then
+    return $1
+  else
+    return solver_value
+  end
+end
+
+def build_document_from_model(progress, struct, model)
+  doc = Nokogiri::XML(XML_TEMPLATE)
+  add_model_struct_to_doc(progress, struct, doc, model)
+  return doc
+end
+
+def add_model_struct_to_doc(progress, struct, doc, model, prefix = nil)
+  struct.each do |key, value|
+    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if value[:type] == :field then
+      if prefix then
+        xpath = "#{prefix}/#{key}"
+      else
+        xpath = key.to_s
+      end
+      model_value = model[xpath]
+      unless model_value then
+        raise RuntimeError.new("xpath '#{xpath}' not contained in model.")
+      end
+      if model_value[:filled] == true || model_value[:filled] == "true" then
+        doc.add_child("<#{key}>#{solver_value_to_string(model_value[:value])}</#{key}>")
+      end
+    elsif value[:type] == :list then
+      node = doc.add_child("<#{key}></#{key}>")
+      if node.is_a?(Nokogiri::XML::NodeSet) then
+        node = node.first 
+      end
+      if prefix then
+        xpath = "#{prefix}/#{key}"
+      else
+        xpath = key.to_s
+      end
+      model_value = model[xpath]
+      xpath_element = value[:xpath_element]
+      if Integer(model_value[:size]) > 0 then
+        model_value[:value].slice(0, Integer(model_value[:size])).each do |mvalue|
+          node.add_child("<#{xpath_element}>#{solver_value_to_string(mvalue)}</#{xpath_element}>")
+        end
+      end
+    elsif value[:type] == :structure then
+      node = doc.add_child("<#{key}></#{key}>")
+      if node.is_a?(Nokogiri::XML::NodeSet) then
+        node = node.first 
+      end
+      unless value[:ref] then
+        raise RuntimeError.new("struct is broken: :structure #{key.inspect} does not have a :ref")
+      end
+      if prefix then
+        new_prefix = "#{prefix}/#{key}"
+      else
+        new_prefix = key
+      end
+      add_model_struct_to_doc(progress, value[:ref], node, model, new_prefix)
+    else
+      raise RuntimeError.new("struct is broken: encountered unknown :type #{value[:type].inspect}")
+    end
+  end
+end
+
+def collect_validation_rule_names(struct)
+  result = []
+  struct.each do |key, value|
+    next if key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if key == :validation_rules
+      value.each do |key, value|
+        result << key
+      end
+    else
+      if value[:type] == :structure
+        result += collect_validation_rule_names(value[:ref])
+      end
+    end
+  end
+  return result
+end
+
+def generate_doc(progress, message, output_filename, config, options)
+  log = options[:log] || nil
+  log.puts "generating doc #{output_filename}" if log
+  options[:solverLog] = options[:solverLog] || "#{output_filename}.smt2"
+
+  z3path = config[:z3path] || DEFAULT_Z3_PATH
+  z3 = Z3Solver.new(progress, z3path)
+  begin
+    model = z3.query_model(progress, options) do |solver|
+      solver.to_solver(progress, "(set-option :produce-unsat-cores true) ; enable generation of unsat cores")
+      solver.to_solver(progress, "")
+      log.puts "translating structure" if log
+      translate_structure_to_SMTLIB(progress, solver, message, :options => options, :config => config)
+      solver.to_solver(progress, "")
+    end
+  ensure
+    z3.close()
+  end
+
+  if model then
+    result_doc = build_document_from_model(progress, message, model)
+    dir = File.dirname(output_filename)
+    FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+    File.open(output_filename, "w") do |f|
+      f.puts result_doc.to_xml()
+    end
+    progress.print_line "DONE. Generated XML was written to #{output_filename}, SMTLIB log was written to #{options[:solverLog]}"
+    return true
+  else
+    progress.print_line "FAILED. No XML document was generated. For more details, please see the SMTLIB log in #{options[:solverLog]}"
+    return false
+  end
+end
+
+def z3ValueToRubyValue(value)
+  if value =~ /\"(.*)\"/ then # string
+    return $1
+  elsif value =~ /\d+/ then # int
+    return Integer(value)
+  elsif value == "true"
+    return true
+  elsif value == "false" then
+    return false 
+  else
+    raise RuntimeError.new("enountered unknown type of value in z3 model: #{value.inspect}")
+  end
+end
+
+def extract_key_field_values_from_model(key_fields, model)
+  result = {}
+  key_fields.each do |xpath, fieldDesc|
+    fieldInfo = model[xpath]
+    if fieldDesc[:type] == :field then
+      if fieldInfo[:filled] then
+        result[xpath] = z3ValueToRubyValue(fieldInfo[:value])
+      else
+        result[xpath] = nil
+      end
+    elsif fieldDesc[:type] == :struct || fieldDesc[:type] == :structure then
+      if fieldInfo.has_key?(:exists) then
+        result[xpath] = z3ValueToRubyValue(fieldInfo[:exists])
+      else
+        result[xpath] = false
+      end
+    else
+      raise RuntimeError.new("encountered unknown :type in solver model: #{fieldDesc[:type].inspect}")
+    end
+  end
+  return result
+end
+
+def all_values_for_field(field)
+  unless field[:type] == :field then
+    raise RuntimeError.new("something that is not a field definition was passed to all_values_for_field(): #{field.inspect}")
+  end
+  unless field[:datatype] == :enum then
+    raise RuntimeError.new("all_values_for_field() is only applicable to fields of datatype enum! not #{field[:datatype].inspect}.")
+  end
+  values = field[:values].dup
+  if field[:optional] == true then
+    values << nil
+  end
+  return values
+end
+
+def each_combination(fields, &block)
+  if fields.size < 1 then
+    return
+  end
+  if fields.size == 1 then
+    field = fields.first
+    values = field[1][:values].clone()
+    if field[1][:optional] == true then
+      values << nil
+    end
+    values.each do |value|
+      block.call({ field[0] => value })
+    end
+  else
+    first = fields.shift
+    values = first[1][:values].clone()
+    if first[1][:optional] == true then
+      values << nil
+    end
+    each_combination(fields) do |mapping|
+      values.each do |value|
+        block.call(mapping.merge({ first[0] => value }))
+      end
+    end
+  end
+end
+
+def calc_combinations_for_key_fields(key_fields)
+  count = 1
+  key_fields.each do |field|
+    if field[1][:type] == :field then
+      if field[1][:datatype] == :enum then
+        num_values = field[1][:values].size
+        num_values += 1 if field[1][:optional]
+      elsif field[1][:datatype] == :int then
+        if field[1].has_key?(:min) and field[1].has_key?(:max) and field[1][:min] <= field[1][:max] then
+          if field[1][:optional] then
+            num_values = 2 + field[1][:max] - field[1][:min]
+          else
+            num_values = 1 + field[1][:max] - field[1][:min]
+          end
+        else
+          return "potentially infinite"
+        end
+      elsif field[1][:datatype] == :string then
+        return "potentially infinite"
+      end
+    elsif field[1][:type] == :structure then
+      num_values = 2
+    end
+    count *= num_values
+  end
+  return count
+end
+
+def calc_combinations(message, key)
+  key_fields = collect_key_fields(key, message)
+  return calc_combinations_for_key_fields(key_fields)
+end
+
+def read_docs_from_log_file(log_filename)
+  result = []
+  if File.exist?(log_filename) then
+    File.open(log_filename, "r") do |f|
+      f.each_line do |line|
+        doc = {}
+        if line =~ /(.*)\ <-\ \{(.*)\}/ then
+          filename = $1
+          combination = $2
+          combination.split(", ").each do |assignment|
+            if assignment =~ /(.*):\ (.*)/ then
+              key = $1
+              value = $2
+              if value =~ /\"(.*)\"/ then
+                value = $1
+              elsif value == "nil" then
+                value = nil
+              end
+              doc[key] = value
+            else
+              raise RuntimeError.new("could not parse logfile #{log_filename}: invalid assignment #{assignment.inspect}")
+            end
+          end
+        else
+          raise RuntimeError.new("could not parse logfile #{log_filename}: invalid line #{line.inspect}")
+        end
+        result << doc
+      end
+    end
+  end
+  return result
+end
+
+def blocking_clause(docs, vars = docs.map{|e| e.keys}.flatten)
+  unless docs.empty?
+      codes = docs.map do |key_values|
+        value_codes = key_values.filter{|xpath,value| vars.include?(xpath) }.map do |xpath, value|
+          if value == nil then
+            "(= #{filled_var_for_field(xpath)} false)"
+          else
+            "(and (= #{filled_var_for_field(xpath)} true) (= #{value_var_for_field(xpath)} \"#{value}\"))"
+          end
+        end
+        if value_codes.empty? then
+          "true"
+        else
+          "(not (and #{value_codes.join(" ")}))"
+        end
+      end
+      return "" if codes.empty?
+      return "(assert (! (and #{codes.join(" ")}) :named blocking-clause))"
+  end
+  return nil
+end
+
+#def fixing_clause(key_values, fixed = key_value.keys)
+#  codes = []
+#  key_values.each do |xpath, value|
+#    if fixed.include?(xpath) then
+#      if value then
+#        codes << "(and (= #{filled_var_for_field(xpath)} true) (= #{value_var_for_field(xpath)} \"#{value}\"))"
+#      else
+#        codes << "(= #{filled_var_for_field(xpath)} false)"
+#      end
+#    end
+#  end
+#  if codes.empty? then
+#    return ""
+#  else
+#    return "(assert (! (and #{codes.join(" ")}) :named fixing-clause))"
+#  end
+#end
+
+def prepare_output_doc_filename(number, output_dir)
+  chunk_number = number / 100000
+  dirname = "#{output_dir}/#{chunk_number}"
+  unless Dir.exist?(dirname) then
+    FileUtils.mkdir_p(dirname)
+  end
+  return "#{dirname}/doc#{number}.xml"
+end
+
+def switch_blocking_clause(fixed, current_value, var_xpath, blocked_values)
+  clauses = []
+  blocked_values.each do |value|
+    if value != nil then
+      clauses << "(not (and (= #{filled_var_for_field(var_xpath)} true) (= #{value_var_for_field(var_xpath)} \"#{value}\")))"
+    else
+      clauses << "(not (= #{filled_var_for_field(var_xpath)} false))"
+    end
+  end
+  if fixed then
+    if current_value != nil then
+      result = "; switch fixing clause\n(assert (! (and (= #{filled_var_for_field(var_xpath)} true) (= #{value_var_for_field(var_xpath)} \"#{current_value}\")) :named switch_fixing_clause))"
+    else
+      result = "; switch fixing clause\n(assert (! (= #{filled_var_for_field(var_xpath)} false) :named switch_fixing_clause))"
+    end
+  else
+    result = ""
+  end
+  unless clauses.empty? then
+    result = result + "\n; switch blocking clause\n(assert (! (and #{clauses.join(" ")}) :named switch_blocking_clause))"
+  end
+  return result
+end
+
+def values_for_key_field(field)
+  if field[:type] == :field then
+    if field[:datatype] == :enum then
+      if field[:optional] then
+        return field[:values] + [nil]
+      else
+        return field[:values]
+      end
+    elsif field[:datatype] == :int then
+      if field.has_key?(:min) and field.has_key?(:max) and field[:min] <= field[:max] then
+        return (field[:min]..field[:max]).to_a
+      else
+        raise RuntimeError.new("cannot determine values for unbounded :int field")
+      end
+    else
+      raise RuntimeError.new("cannot determine values for datatype #{field[:datatype].inspect}")
+    end
+  elsif field[:type] == :structure then
+    if field[:optional] then
+      return [true, false]
+    else
+      return [true]
+    end
+  else
+    raise RuntimeError.new("Encountered unknown :type in model: #{field[:type].inspect} - keys can only mark fields or structures.")
+  end
+end
+
+def fixing_clause(xpath, switched_info)
+  if switched_info[:current_value] == nil then
+    return "(assert (! (= #{filled_var_for_field(xpath)} false) :named fixing-clause-#{raw_field(xpath)}))"
+  else
+    return "(assert (! (and (= #{filled_var_for_field(xpath)} true) (= #{value_var_for_field(xpath)} \"#{switched_info[:current_value]}\")) :named fixing-clause-#{raw_field(xpath)}))"
+  end
+end
+
+def collect_key_fields(fkey, struct, prefix = nil)
+  result = {}
+  struct.each do |key, value|
+    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if value[:type] == :structure
+      if value.has_key?(:keys) and value[:keys].include?(fkey) then
+        xpath = key.to_s
+        xpath = "#{prefix}/#{xpath}" if prefix
+        result[xpath] = value.clone().filter{|k,v| [:type, :optional, :datatype, :values].include?(k)}
+      end
+      if prefix then
+        new_prefix = "#{prefix}/#{key}" 
+      else
+        new_prefix = key
+      end
+      result = result.merge(collect_key_fields(fkey, value[:ref], new_prefix))
+    elsif value[:type] == :field and value.has_key?(:keys) and value[:keys].include?(fkey) then
+      xpath = key.to_s
+      xpath = "#{prefix}/#{xpath}" if prefix
+      result[xpath] = value.clone().filter{|k,v| [:type, :optional, :datatype, :values, :min, :max].include?(k)}
+    end
+  end
+  return result
+end
+
+def collect_key_fields_and_strctures(fkey, struct, prefix = nil)
+  fields = []
+  structures = []
+  struct.each do |key, value|
+    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if value[:type] == :structure
+      if value.has_key?(:keys) and value[:keys].include?(fkey) then
+        xpath = key.to_s
+        xpath = "#{prefix}/#{xpath}" if prefix
+        structures << xpath
+      end
+      if prefix then
+        new_prefix = "#{prefix}/#{key}" 
+      else
+        new_prefix = key
+      end
+      result_fields, result_structures = collect_key_fields_and_strctures(fkey, value[:ref], new_prefix)
+      fields += result_fields
+      structures += result_structures
+    elsif value[:type] == :field and value.has_key?(:keys) and value[:keys].include?(fkey) then
+      xpath = key.to_s
+      xpath = "#{prefix}/#{xpath}" if prefix
+      fields << xpath
+    end
+  end
+  return fields, structures
+end
+
+def collect_keys(struct)
+  result = []
+  struct.each do |key, value|
+    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if value[:type] == :structure
+      if value.has_key?(:keys) then
+        result += value[:keys]
+      end
+      if value[:ref] then
+        result += collect_keys(value[:ref])
+      else
+        raise RuntimeError.new("Model error: encountered structure without :ref: key=#{key.inspect} value=#{value.inspect}")
+      end
+    elsif value[:type] == :field and value.has_key?(:keys) then
+      result += value[:keys]
+    end
+  end
+  result.uniq
+end
+
+def collect_validation_rules(struct)
+  result = []
+  struct.each do |key, value|
+    next if key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if key == :validation_rules then
+      result += value.keys
+    else
+      if value[:type] == :structure
+        if value[:ref] then
+          result += collect_validation_rules(value[:ref])
+        else
+          raise RuntimeError.new("Model error: encountered structure without :ref: key=#{key.inspect} value=#{value.inspect}")
+        end
+      end
+    end
+  end
+  result.uniq
+end
+
+def list_keys(message)
+  keys = collect_keys(message)
+  puts "keys: #{keys.join(", ")}"
+end
+
+def list_fields_and_structures(message, fkey)
+  fields, structures = collect_key_fields_and_strctures(fkey, message)
+  puts "fields for key #{fkey}:"
+  fields.each { |f| puts "  #{f}" }
+  puts "structures for key #{fkey}:"
+  structures.each { |s| puts "  #{s}" }
+end
+
+def list_validation_rules(message)
+  rules = collect_validation_rules(message)
+  puts "Validation Rules:"
+  rules.each { |r| puts "  #{r}" }
+end
+
+class CoreEnumerationAlgo
+
+  def initialize()
+    @num_tries = 0
+    @num_docs = 0
+    @on_new_doc = nil
+    @on_unsat = nil
+  end
+
+
+  # fix values of all variables left from switched
+  def enumerate_fixing_clause(assignment, vars, switched)
+    clauses = []
+    switched.times do |index|
+      var_xpath = vars[index][0]
+      var_def = vars[index][1]
+      if var_def[:type] == :field then
+        if assignment[var_xpath] == nil then
+          clauses << "(assert (! (= #{filled_var_for_field(var_xpath)} false) :named fixing_clause_#{index}))"
+        else
+          clauses << "(assert (! (and (= #{filled_var_for_field(var_xpath)} true) (= #{value_var_for_field(var_xpath)} #{value_of_type(assignment[var_xpath], var_def[:datatype])})) :named fixing_clause_#{index}))"
+        end
+      elsif var_def[:type] == :structure then
+        clauses << "(assert (! (= #{exists_var_for_structure(var_xpath)} #{assignment[var_xpath].inspect}) :named fixing_clause_#{index}))"
+      end
+    end
+    "; fixing clauses:\n#{clauses.join("\n")}" 
+  end
+
+  def summarize_fixing_clause(assignment, vars, switched)
+    fixed = switched.times.to_a.map do |index|
+      var_xpath = vars[index][0]
+      "#{var_xpath}=#{assignment[var_xpath].inspect}"
+    end
+    "fixed(#{fixed.join(", ")})"
+  end
+  
+  def value_of_type(value, type)
+    if type == :string or type == :enum then
+      "\"#{value}\""
+    elsif type == :int then
+      value.to_s
+    end
+  end
+
+  # prevent the var from taking values that already have been enumerated exhaustively
+  def enumerate_blocking_clause(var_xpath, var_def, blocked)
+    clauses = []
+    num = 0
+    if var_def[:type] == :field then
+      blocked.each do |value|
+        if value == nil then
+          clauses << "(assert (! (not (= #{filled_var_for_field(var_xpath)} false)) :named blocking_clause_#{num}))"
+        else
+          clauses << "(assert (! (not (and (= #{filled_var_for_field(var_xpath)} true) (= #{value_var_for_field(var_xpath)} #{value_of_type(value, var_def[:datatype])}))) :named blocking_clause_#{num}))"
+        end
+        num += 1
+      end
+    elsif var_def[:type] == :structure then
+      blocked.each do |value|
+        clauses << "(assert (! (not (= #{exists_var_for_structure(var_xpath)} #{value.inspect})) :named blocking_clause_#{num}))"
+        num += 1
+      end
+    end
+    "; blocking clauses:\n#{clauses.join("\n")}" 
+  end
+
+  def summarize_blocking_clause(var_xpath, blocked)
+    "blocked(#{var_xpath})=[#{blocked.map{|v| v.inspect}.join(", ")}]"
+  end
+
+  def generate_sparse_doc(progress, message, key, config, options, code, output_doc_filename)
+    additional_options = {}
+    if code then
+        additional_options = { :add_SMTLIB => code }
+    end
+    return generate_doc_return_key_values(progress, message, key, output_doc_filename, config, options.merge(additional_options))
+  end
+
+  def log_filename_for_output_filename(output_filename)
+    "#{output_filename}.smt2"
+  end
+
+  def generate_doc_return_key_values(progress, message, key, output_filename, config, options)
+    log = options[:log] || nil
+    log.puts "generating doc #{output_filename}" if log
+    options[:solverLog] = log_filename_for_output_filename(output_filename)
+
+    key_fields = collect_key_fields(key, message)
+
+    z3path = config[:z3path] || DEFAULT_Z3_PATH
+    z3 = Z3Solver.new(progress, z3path)
+    begin
+      model = z3.query_model(progress, options) do |solver|
+        solver.to_solver(progress, "(set-option :produce-unsat-cores true) ; enable generation of unsat cores")
+        solver.to_solver(progress, "")
+        log.puts "translating structure" if log
+        translate_structure_to_SMTLIB(progress, solver, message, :options => options, :config => config)
+        solver.to_solver(progress, "")
+      end
+    ensure
+      z3.close()
+    end
+
+    if model then
+      result_doc = build_document_from_model(progress, message, model)
+      dir = File.dirname(output_filename)
+      FileUtils.mkdir_p(dir) unless Dir.exist?(dir)
+      File.open(output_filename, "w") do |f|
+        f.puts result_doc.to_xml(:save_with => Nokogiri::XML::Node::SaveOptions::AS_XML)
+      end
+      progress.print_line "DONE. Generated XML was written to #{output_filename}, SMTLIB log was written to #{options[:solverLog]}"
+      return extract_key_field_values_from_model(key_fields, model)
+    else
+      progress.print_line "FAILED. No XML document was generated. For more details, please see the SMTLIB log in #{options[:solverLog]}"
+      return nil
+    end
+  end
+
+  def store_continuation_point(doc_number, state, assignment, new_assignment, filename)
+    File.open(filename, "w") do |f|
+      f.puts("#{doc_number};#{@num_docs};state(#{state.map{|s| if s then "[#{s.map{|v| v.inspect}.join(",")}]" else "nil" end}.join("#")});assignment(#{assignment.to_a.map{|k,v| "#{k}:#{v.inspect}"}.join(",")});new_assignment(#{new_assignment.to_a.map{|k,v| "#{k}:#{v.inspect}"}.join(",")})")
+    end
+  end
+
+  def self.parse_state(state_str)
+    unless state_str =~ /state\((.*)\)/
+      return nil
+    end
+    split = $1.split("#")
+    split.map do |e|
+      if e == "nil" then
+        nil
+      elsif e =~ /\[(.*)\]/ then
+        $1.split(",").map do |i|
+          if i.strip == "nil" then
+            nil
+          elsif i.strip == "true" then
+            true
+          elsif i.strip == "false" then
+            false
+          elsif i.strip =~ /\"(.*)\"/ then
+            $1
+          end
+        end
+      end
+    end
+  end
+
+  def self.parse_assignment(str)
+    unless str =~ /assignment\((.*)\)/
+      return nil
+    end
+    split = $1.split(",")
+    result = {}
+    split.each do |e|
+      s2 = e.split(":")
+      key = s2[0]
+      value = s2[1]
+      if value == "nil" then
+        result[key] = nil
+      elsif value == "true" then
+        result[key] = true
+      elsif value == "false" then
+        result[key] = false
+      elsif value =~ /\"(.*)\"/
+        result[key] = $1
+      end
+    end
+    result
+  end
+
+  def self.parse_new_assignment(str)
+    unless str =~ /new_assignment\((.*)\)/
+      return nil
+    end
+    split = $1.split(",")
+    result = {}
+    split.each do |e|
+      s2 = e.split(":")
+      key = s2[0]
+      value = s2[1]
+      if value == "nil" then
+        result[key] = nil
+      elsif value == "true" then
+        result[key] = true
+      elsif value == "false" then
+        result[key] = false
+      elsif value =~ /\"(.*)\"/
+        result[key] = $1
+      end
+    end
+    result
+  end
+
+  def self.parse_continuation_point(line)
+    split = line.split(";")
+    doc_number = Integer(split[0])
+    num_docs = Integer(split[1])
+    state = CoreEnumerationAlgo.parse_state(split[2])
+    assignment = CoreEnumerationAlgo.parse_assignment(split[3])
+    new_assignment = CoreEnumerationAlgo.parse_assignment(split[4])
+    return [doc_number, num_docs, state, assignment, new_assignment]
+  end
+
+  # TODO: logging
+  
+  def display_combination_space(assignment, vars, switched, blocked)
+    # enumerate_fixing_clause(assignment, vars, switched) + enumerate_blocking_clause(vars[switched][0], vars[switched][1], blocked)
+
+    # => all vars left of switched are fixed to their assigned values (when the value is nil, then they are empty)
+    # => the switched var is prevented to have any of the blocked values
+    res = switched.times.to_a.map do |index|
+      var_xpath = vars[index][0]
+      "#{var_xpath}=#{assignment[var_xpath].inspect}"
+    end + ["#{vars[switched][0]}|#{blocked.map{|v| v.inspect}.join(",")}"]
+    "{#{res.join(", ")}}"
+  end
+
+  # core algo
+  def enumerate(progress, switched, vars, assignment, message, key, output_dir, config, options, state = [nil] * vars.size())
+    enumerate(progress, switched+1, vars, assignment, message, key, output_dir, config, options, state) if switched < vars.size()-1
+    is_finite_state = keys_are_finite_state(vars)
+    if state[switched] == nil then
+      blocked = [assignment[vars[switched][0]]]
+      state[switched] = blocked
+    else
+      blocked = state[switched] + [assignment[vars[switched][0]]]
+      state[switched] = blocked
+    end
+    found = true
+    while found do
+      found = false
+      # - fix all vars left of switched to their values in assignment
+      # - block switched from obtaining the values in blocked
+      # progress.print_line "DEBUG: #{summarize_fixing_clause(assignment, vars, switched)}, #{summarize_blocking_clause(vars[switched][0], blocked)}"
+      clauses = enumerate_fixing_clause(assignment, vars, switched) + enumerate_blocking_clause(vars[switched][0], vars[switched][1], blocked)
+      output_doc_filename = prepare_output_doc_filename(@num_tries, output_dir)
+      new_assignment = generate_sparse_doc(progress, message, key, config, options, clauses, output_doc_filename)
+      generated_doc_number = @num_tries
+      store_continuation_point(generated_doc_number, state, assignment, new_assignment, "#{output_dir}/#{CONTINUATION_FILE}")
+      if is_finite_state then
+        progress.report_progress(determine_progress(state, vars))
+      else
+        progress.report_progress(nil)
+      end
+      @num_tries += 1
+      if new_assignment then # sat
+        @num_docs += 1
+        @on_new_doc.call(generated_doc_number, output_doc_filename, new_assignment) if @on_new_doc
+        if switched < vars.size()-1 then
+          enumerate(progress, switched+1, vars, new_assignment, message, key, output_dir, config, options, state)
+        end
+        blocked << new_assignment[vars[switched][0]]
+        state[switched] = blocked
+        found = true
+      else # unsat
+        @on_unsat.call(generated_doc_number, log_filename_for_output_filename(output_doc_filename), display_combination_space(assignment, vars, switched, blocked)) if @on_unsat
+        found = false
+      end
+    end
+    state[switched] = nil
+  end
+  
+  # NOTE: can only be called when all key-fields are finite-state
+  def is_final_state(state, new_assignment, key_fields_list)
+    return state[1,state.size()] == [nil] * (state.size()-1) &&
+       state[0].to_set == values_for_key_field(key_fields_list[0][1]).to_set &&
+       new_assignment.empty?
+  end
+
+  # NOTE: can only be called when all key-fields are finite-state
+  def determine_progress(state, vars)
+    combinations_per_value = [nil] * vars.size()
+    comb = 1
+    vars.size().times.to_a.reverse_each do |i|
+      combinations_per_value[i] = comb
+      comb = values_for_key_field(vars[i][1]).size() * comb
+    end
+
+    blocked_combinations = 0
+    state.size().times do |i|
+      if state[i] then
+        blocked_combinations += state[i].size() * combinations_per_value[i]
+      end
+    end
+    blocked_combinations
+  end
+
+  def num_combinations(key_def)
+    if key_def[:type] == :field then
+      if key_def[:datatype] == :enum then
+        key_def[:values].size() + (key_def[:optional] ? 1 : 0)
+      elsif key_def[:datatype] == :int then
+        if key_def.has_key?(:min) and key_def.has_key?(:max) and key_def[:min] <= key_def[:max] then
+          1 + key_def[:max] - key_def[:min] + (key_def[:optional] ? 1 : 0)
+        else
+          nil
+        end
+      else
+        nil
+      end
+    elsif key_def[:type] == :structure then
+      2
+    end
+  end
+  
+  def keys_are_finite_state(list)
+    list.each do |field|
+      name = field[0]
+      if field[1][:type] == :field then
+        if field[1][:datatype] == :enum then
+          # enums are finite-state
+        elsif field[1][:datatype] == :int and field[1].has_key?(:min) and field[1].has_key?(:max) and field[1][:min] <= field[1][:max] then
+          # ints are finite-state when both bounds are given and :min <= :max
+        else
+          return false
+        end
+      end
+    end
+    return true
+  end
+
+  def generate_docs_for_key(message, key, output_dir, config, options)
+
+    number_of_combinations = calc_combinations(message, key)
+    if number_of_combinations.is_a?(String) then
+      progress = ProgressBar.new(100000000000000)
+    else
+      progress = ProgressBar.new(number_of_combinations)
+    end
+    begin
+    ensure
+    end
+    progress.report_progress(0.0)
+    progress.print_line "generating sparse documents efficiently..."
+    progress.print_line "there is a theoretical maximum of #{number_of_combinations} combinations."
+    progress.print_line "Note, however, that we are generating only those documents that are valid, which are usually much less."
+
+    progress.print_line "preparing output dir #{output_dir}..."
+    if options[:continue] then
+      unless Dir.exist?(output_dir) then
+        raise FatalError.new("FATAL: directory #{output_dir} not found. There seems to be nothing to continue from...")
+      end
+      log_mode = "a"
+    else
+      unless Dir.exist?(output_dir) then
+        FileUtils.mkdir_p(output_dir)
+      end
+      log_mode = "w"
+    end
+    log_filename = "#{output_dir}/#{COMBINATION_LOG}"
+
+    key_fields = collect_key_fields(key, message)
+    key_fields_list = key_fields.each_pair.to_a
+    key_fields_list.sort! {|a,b| num_combinations(b[1]) <=> num_combinations(a[1]) }
+
+    if options[:continue] then
+      stored_key_fields_str = File.read("#{output_dir}/#{VARS_LOG}")
+      unless key_fields_list.to_yaml == stored_key_fields_str then
+        raise FatalError.new("vars are differing! please specify the same key in order to use -continue!")
+      end
+      progress.print_line "continuing enumerating documents for key #{key}!"
+    else
+      # store vars
+      File.open("#{output_dir}/#{VARS_LOG}", "w") do |f|
+        f.puts(key_fields_list.to_yaml)
+      end
+    end
+
+    #progress.print_line "DEBUG: key_fields_list=#{key_fields_list.inspect}"
+
+    cont_filename = "#{output_dir}/#{CONTINUATION_FILE}"
+    if options[:continue] then
+      unless File.exist?(cont_filename) then
+        raise FatalError.new("FATAL: file #{cont_filename} not found. There seems to be no continuation point to continue from...")
+      end
+      doc_number, num_docs, state, assignment, new_assignment = CoreEnumerationAlgo.parse_continuation_point(File.read(cont_filename).chomp)
+      if is_final_state(state, new_assignment, key_fields_list) then
+        progress.report_progress(number_of_combinations)
+        raise FatalError.new("FATAL: The enumeration is finished. There is nothing left to do. Resuming the enumeration hence does not make sense.")
+      end
+      progress.print_line "RESUMING enumeration from the following continuation point:"
+      progress.print_line "|doc_number=#{doc_number}"
+      progress.print_line "|num_docs=#{num_docs}"
+      progress.print_line "|state=#{state.inspect}"
+      progress.print_line "|assignment=#{assignment.inspect}"
+      progress.print_line "|new_assignment=#{new_assignment.inspect}"
+      if keys_are_finite_state(key_fields_list) then
+        progress.report_progress(determine_progress(state, key_fields_list))
+      else
+        progress.report_progress(nil)
+      end
+
+      if new_assignment && !new_assignment.empty? then
+        assignment = new_assignment
+      end
+
+      # restore state at continuation point
+      @num_tries = doc_number+1 # we do not want to overwrite the last document written before the STOP
+      number_of_combinations += 1 # account for this in the safety mechanism
+      @num_docs = num_docs
+      doc_counter = 0
+      File.open(log_filename, log_mode) do |log|
+
+        @on_new_doc = Proc.new { |num_doc, doc_filename, combination|
+          # combination logging
+          log.puts "#{doc_filename} <- {#{combination.map{|k,v| "#{k}: #{v.inspect}"}.join(", ")}}"
+
+          doc_counter += 1
+          if options[:max_num_docs] && doc_counter >= options[:max_num_docs] then
+            raise NormalProgramTermination.new("-max-num-docs exceeded. Stopping!")
+          end
+
+          # safety mechanism
+          if @num_docs > number_of_combinations then
+            raise RuntimeError.new("This should never happen! Number of combinations exceeded. #{@num_docs} documents generated, but there are only #{number_of_combinations} possible combinations of the key vars.")
+          end
+        }
+        @on_unsat = Proc.new { |num_doc, log_filename, combination_str|
+              log.puts "CONTRADICTION (see: #{log_filename}) <- #{combination_str}"
+              progress.print_line "doc#{num_doc} NOT generated"
+              progress.print_line "No more valid combinations of the free vars could be found. -> Switching"
+        }
+
+        # core algo - resume from state
+        enumerate(progress, 0, key_fields_list, assignment, message, key, output_dir, config, options, state)
+      end
+    else
+      # init safety mechanism
+      @num_tries = 0
+      @num_docs = 0
+      doc_counter = 0
+      # first try - a freely generated document
+      File.open(log_filename, log_mode) do |log|
+        output_doc_filename = prepare_output_doc_filename(@num_tries, output_dir)
+        key_values = generate_doc_return_key_values(progress, message, key, output_doc_filename, config, options)
+        log.puts "#{output_doc_filename} <- {#{key_values.map{|k,v| "#{k}: #{v.inspect}"}.join(", ")}}"
+        doc_counter += 1
+        @num_tries += 1
+        @on_new_doc = Proc.new { |num_doc, doc_filename, combination|
+          # combination logging
+          log.puts "#{doc_filename} <- {#{combination.map{|k,v| "#{k}: #{v.inspect}"}.join(", ")}}"
+
+          doc_counter += 1
+          if options[:max_num_docs] && doc_counter >= options[:max_num_docs] then
+            raise NormalProgramTermination.new("-max-num-docs exceeded. Stopping!")
+          end
+
+          # safety mechanism
+          unless number_of_combinations.is_a?(String) then
+            if @num_docs > number_of_combinations then
+              raise RuntimeError.new("This should never happen! Number of combinations exceeded. #{@num_docs} documents generated, but there are only #{number_of_combinations} possible combinations of the key vars.")
+            end
+          end
+        }
+        @on_unsat = Proc.new { |num_doc, log_filename, combination_str|
+              log.puts "CONTRADICTION (see: #{log_filename}) <- #{combination_str}"
+              progress.print_line "doc#{num_doc} NOT generated"
+              progress.print_line "No more valid combinations of the free vars could be found. -> Switching"
+        }
+
+        # core algo - start enumeration
+        enumerate(progress, 0, key_fields_list, key_values, message, key, output_dir, config, options)
+      end
+      
+      progress.print_line "No more valid documents could be found. -> Exiting"
+      progress.report_progress(number_of_combinations)
+    end
+  end
+end
+
+def introduce_parent_links(message, parent = nil, prefix = nil)
+  if parent != nil then
+    message[:parent_link] = parent
+  end
+  if prefix != nil then
+    message[:struct_xpath_prefix] = prefix
+  end
+  message.each_pair do |key, value|
+    next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if value[:type] == :structure then
+      if prefix then
+        new_prefix = "#{prefix}/#{key}"
+      else
+        new_prefix = key
+      end
+      introduce_parent_links(value[:ref], message, new_prefix)
+    end
+  end
+end
+
+def validate_model(message, prefix=nil)
+
+  message.each_pair do |key, value|
+    if key == :validation_rules then
+      value.each_pair do |vr_key, vr_value|
+        unless vr_value[:text] && vr_value[:text] != "" then
+          puts "INFO: Model Problem: validation rule #{vr_key} does not have a textual representation."
+        end
+        unless vr_value[:smtlib] && vr_value[:smtlib] != "" then
+          puts "WARN: Model Problem: validation rule #{vr_key} does not have an SMTLIB representation. It will hence be omitted!"
+        end
+      end
+    elsif key == :predicates then
+      value.each_pair do |mkey, mvalue|
+        next unless mvalue
+        unless mvalue[:text] && mvalue[:text] != "" then
+            puts "INFO: Model Problem: structure #{prefix}'s predicate #{mkey} does not have a textual representation."
+        end
+        unless mvalue[:smtlib] && mvalue[:smtlib] != "" then
+            puts "WARN: Model Problem: structure #{prefix}'s predicate #{mkey} does not have an SMTLIB representation. It will be regarded as always true."
+        end
+      end
+    elsif key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+      next
+    else
+      if value[:type] == :structure then
+        unless value[:ref] then
+          raise RuntimeError.new("Model Problem: encountered structure without a :ref (key=#{key.inspect}, value=#{value.inspect})")
+        end
+        if prefix then
+          new_prefix = "#{prefix}/#{key}"
+        else
+          new_prefix = key
+        end
+        validate_model(value[:ref], new_prefix)
+      elsif value[:type] == :field then
+        unless value[:datatype] then
+          raise RuntimeError.new("Model Problem: encountered field without a :datatype (key=#{key.inspect}, value=#{value.inspect})")
+        end
+        if value[:datatype] == :enum then
+          unless value[:values] then
+            raise RuntimeError.new("Model Problem: encountered enum field without a :values (key=#{key.inspect}, value=#{value.inspect})")
+          end
+        end
+      end
+    end  
+  end
+end
+
+def generate(message, argv, message_config_filename = nil)
+  if ARGV[0] == "--help" || ARGV[0] == "-help" || ARGV[0] == "help" then
+    puts USAGE
+    exit(0)
+  end
+
+  options = { :negated_validation_rules => [], :date_now => (Date.today - Time.at(0).to_date).to_i, :timestamp_now => Time.now().to_i }
+  
+  opt = argv.shift
+  while opt do
+    if opt == "-negate" then
+      rule = argv.shift
+      options[:negated_validation_rules] << rule
+    elsif opt == "-list-keys" then
+      options[:list_keys] = true
+    elsif opt == "-list" then
+      options[:list] = argv.shift
+    elsif opt == "-list-validation-rules" then
+      options[:list_validation_rules] = true      
+    elsif opt == "-generate-docs-from" then
+      options[:combination_source] = argv.shift
+    elsif opt == "-skip" then
+      options[:skip] = Integer(argv.shift)
+    elsif opt == "-generate-docs-for-key" then
+      options[:docs_for_key] = argv.shift
+    elsif opt == "-count-docs-for-key" then
+      options[:count_docs_for_key] = argv.shift
+    elsif opt == "-continue" then
+      options[:continue] = true
+    elsif opt == "-max-num-docs" then
+      options[:max_num_docs] = Integer(argv.shift)
+      puts "flag -max-num-docs used. Will stop after generating #{options[:max_num_docs]} documents."
+    else
+      puts "FATAL: unknown option #{opt.inspect}."
+      puts USAGE
+      raise FatalError.new("")
+    end
+    opt = argv.shift
+  end
+
+  unless options[:negated_validation_rules].empty? then
+    puts "negating rules: #{options[:negated_validation_rules].join(", ")}"
+
+    rules = collect_validation_rule_names(message)
+    unknown_negated_rules = options[:negated_validation_rules].filter { |rule_name| !rules.include?(rule_name) } 
+    unless unknown_negated_rules.empty? then  
+      throw RuntimeError.new("unknown negated rules #{unknown_negated_rules.join(", ")}")
+    end
+
+  end
+
+  if File.exist?(CONFIG_FILE) then
+    config = YAML.load_file(CONFIG_FILE)
+  else
+    config = DEFAULT_CONFIG
+    File.open(CONFIG_FILE, "w") do |f|
+      f.puts(config.to_yaml)
+    end
+    puts "No config file found. Default config written to #{CONFIG_FILE}"
+  end
+  if message_config_filename then
+    message_config = YAML.load_file(message_config_filename)
+    config = config.merge(message_config)
+  end
+
+  introduce_parent_links(message)
+
+  if options[:count_docs_for_key] then
+    puts "there are #{calc_combinations(message, options[:count_docs_for_key])} combinations."
+  elsif options[:docs_for_key] then
+    puts "using #{options[:date_now]} as [Date.now]"
+    puts "using #{options[:timestamp_now]} as [Timestamp.now]"
+    validate_model(message)
+    CoreEnumerationAlgo.new().generate_docs_for_key(message, options[:docs_for_key], OUTPUT_DIR, config, options)
+  elsif options[:list_keys] then
+    list_keys(message)
+  elsif options[:list] then
+    list_fields_and_structures(message, options[:list])
+  elsif options[:list_validation_rules] then
+    list_validation_rules(message)
+  else
+    puts "using #{options[:date_now]} as [Date.now]"
+    puts "using #{options[:timestamp_now]} as [Timestamp.now]"
+    validate_model(message)
+    progress = ProgressBar.new(1)
+    generate_doc(progress, message, OUTPUT_FILE, config, options)
+  end
+end