mbt-gen 0.0.2 → 0.0.4

data/lib/mbt-gen.rb

@@ -73,6 +73,11 @@ GENERATING MESSAGES FOR ALL COMBINATIONS:
           be restarted later by adding -continue to the command line.
   NOTE: do not expect the number of generated documents to equal the number of combinations as combinations may not have any valid message instances (for instance when they contradict the validation rules).
   NOTE: the result can be found in the directory #{OUTPUT_DIR}
+
+VALIDATING THE MODEL:
+
+  ruby #{$PROGRAM_NAME} -validate <file.xml>
+  - reads in the given XML file and validates it against the model - that is, it checks whether it conforms to the specified XML schema and to the validation rules.
 EOF
 
 class FatalError < RuntimeError
@@ -167,38 +172,100 @@ def enum_constraint_for_expr(expr, values)
   "(or #{values.map{|x| "(= #{expr} #{translate_value_to_SMTLIB(x)})"}.join(" ")})"
 end
 
-def translate_field_def_to_SMTLIB_constraints(progress, solver, fielddef, varname)
+def translate_field_def_to_SMTLIB_constraints(progress, solver, fielddef, xpath)
+  varname = raw_field(xpath)
   if fielddef[:optional] == nil || fielddef[:optional] == false then
     # field is required
-    solver.to_solver(progress, "(assert (! #{varname}-filled :named required-#{varname}))") 
+    assertionID = "required-#{varname}"
+    info = {
+      :assertion_type => :constraint,
+      :xpath => xpath,
+      :constraint_type => :required
+    }
+    solver.associateAssertionIDWith(assertionID, info)
+    solver.to_solver(progress, "(assert (! #{varname}-filled :named #{assertionID}))")
   end
   datatype = fielddef[:datatype]
   case datatype
   when :int
     if fielddef[:min] then
-        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{min_constraint_for_expr("#{varname}-value", translate_value_to_SMTLIB(fielddef[:min]))}) :named min-#{varname}))") 
+        assertionID = "min-#{varname}"
+        info = {
+          :assertion_type => :constraint,
+          :xpath => xpath,
+          :datatype => :int,
+          :value => fielddef[:min],
+          :constraint_type => :min
+        }
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{min_constraint_for_expr("#{varname}-value", translate_value_to_SMTLIB(fielddef[:min]))}) :named #{assertionID}))") 
     end
     if fielddef[:max] then
-        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{max_constraint_for_expr("#{varname}-value", translate_value_to_SMTLIB(fielddef[:max]))}) :named max-#{varname}))") 
+        assertionID = "max-#{varname}"
+        info = {
+          :assertion_type => :constraint,
+          :xpath => xpath,
+          :datatype => :int,
+          :value => fielddef[:max],
+          :constraint_type => :max
+        }
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{max_constraint_for_expr("#{varname}-value", translate_value_to_SMTLIB(fielddef[:max]))}) :named #{assertionID}))")
     end
   when :string
     if fielddef[:regex] then
-        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{regex_constraint_for_expr("#{varname}-value", fielddef[:regex])}) :named regex-#{varname}))") 
+        assertionID = "regex-#{varname}"
+        info = {
+          :assertion_type => :constraint,
+          :xpath => xpath,
+          :datatype => :string,
+          :pattern => fielddef[:regex],
+          :constraint_type => :regex
+        }
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{regex_constraint_for_expr("#{varname}-value", fielddef[:regex])}) :named #{assertionID}))") 
     end 
     # TODO: minLength?
     if fielddef[:maxLength] then
-      solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{maxLength_contraint_for_expr("#{varname}-value", fielddef[:maxLength])}) :named maxLength-#{varname}))") 
+      assertionID = "maxLength-#{varname}"
+        info = {
+          :assertion_type => :constraint,
+          :xpath => xpath,
+          :datatype => :string,
+          :value => fielddef[:maxLength],
+          :constraint_type => :maxLength
+        }
+      solver.associateAssertionIDWith(assertionID, info)
+      solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{maxLength_contraint_for_expr("#{varname}-value", fielddef[:maxLength])}) :named #{assertionID}))") 
     end
     if fielddef[:from_list] then
+      assertionID = "oneOf-#{varname}"
       list_filename = "#{LISTS_DIR}/#{fielddef[:from_list]}"
+      info = {
+        :assertion_type => :constraint,
+        :xpath => xpath,
+        :datatype => :string,
+        :filename => list_filename,
+        :constraint_type => :from_list
+      }
       unless File.exist?(list_filename)
         raise RuntimeError.new("could not find list file #{list_filename}.")
       end
       values = File.read(list_filename).split("\n").map{|v| v.chomp()}
-      solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{oneOf_contraint_for_expr("#{varname}-value", values)}) :named oneOf-#{varname}))") 
+      solver.associateAssertionIDWith(assertionID, info)
+      solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{oneOf_contraint_for_expr("#{varname}-value", values)}) :named #{assertionID}))") 
     end
   when :enum
-    solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{enum_constraint_for_expr("#{varname}-value", fielddef[:values])}) :named enum-#{varname}))") 
+    assertionID = "enum-#{varname}"
+    info = {
+      :assertion_type => :constraint,
+      :xpath => xpath,
+      :datatype => :enum,
+      :values => fielddef[:values],
+      :constraint_type => :values
+    }
+    solver.associateAssertionIDWith(assertionID, info)
+    solver.to_solver(progress, "(assert (! (=> #{varname}-filled #{enum_constraint_for_expr("#{varname}-value", fielddef[:values])}) :named #{assertionID}))") 
   end
 end
 
@@ -290,13 +357,15 @@ def translate_substructure_to_SMTLIB(progress, solver, struct, hash = {})
         if (struct_varname)
           solver.to_solver(progress, "(assert (! (=> (not #{struct_varname}) (not #{filled_var_for_field(xpath)})) :named empty-struct-has-no-fields-for-#{raw_field(xpath)}))") 
         end
-        translate_field_def_to_SMTLIB_constraints(progress, solver, definition, raw_field(xpath))
+        translate_field_def_to_SMTLIB_constraints(progress, solver, definition, xpath)
       elsif definition[:type] == :list then
         display_prefix = ""
         display_prefix = "#{prefix}/" if prefix
         xpath = field.to_s
         xpath = "#{prefix}/#{xpath}" if prefix
+        solver.declare_const(progress, filled_var_for_field(xpath), "Bool", {:xpath => xpath})
         solver.declare_const(progress, size_var_for_list(xpath), "Int", {:xpath => xpath})
+        solver.to_solver(progress, "(assert (! (=> (not #{filled_var_for_field(xpath)}) (= #{size_var_for_list(xpath)} 0)) :named empty-list-field-has-zero-size-#{raw_field(xpath)}))")
         definition[:model_maxLength].times do |i|
           solver.declare_const(progress, value_var_for_list(xpath, i), translate_datatype_to_SMTLIB(definition[:datatype]), {:xpath => xpath, :xpath_element => definition[:xpath_element], :index => i, :datatype => definition[:datatype]})
           solver.to_solver(progress, "(assert (! (=> (<= #{size_var_for_list(xpath)} #{i}) (= #{value_var_for_list(xpath, i)} #{default_value_for_fielddef(definition)})) :named empty-list-field-has-no-value-for-#{raw_field(xpath)}-index-#{i}))") 
@@ -678,13 +747,13 @@ end
 def add_model_struct_to_doc(progress, struct, doc, model, prefix = nil)
   struct.each do |key, value|
     next if key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix || key == :additional_smtlib
+    if prefix then
+      xpath = "#{prefix}/#{key}"
+    else
+      xpath = key.to_s
+    end
+    model_value = model[xpath]
     if value[:type] == :field then
-      if prefix then
-        xpath = "#{prefix}/#{key}"
-      else
-        xpath = key.to_s
-      end
-      model_value = model[xpath]
       unless model_value then
         raise RuntimeError.new("xpath '#{xpath}' not contained in model.")
       end
@@ -696,12 +765,6 @@ def add_model_struct_to_doc(progress, struct, doc, model, prefix = nil)
       if node.is_a?(Nokogiri::XML::NodeSet) then
         node = node.first 
       end
-      if prefix then
-        xpath = "#{prefix}/#{key}"
-      else
-        xpath = key.to_s
-      end
-      model_value = model[xpath]
       xpath_element = value[:xpath_element]
       if Integer(model_value[:size]) > 0 then
         model_value[:value].slice(0, Integer(model_value[:size])).each do |mvalue|
@@ -709,19 +772,16 @@ def add_model_struct_to_doc(progress, struct, doc, model, prefix = nil)
         end
       end
     elsif value[:type] == :structure then
-      node = doc.add_child("<#{key}></#{key}>")
-      if node.is_a?(Nokogiri::XML::NodeSet) then
-        node = node.first 
-      end
       unless value[:ref] then
         raise RuntimeError.new("struct is broken: :structure #{key.inspect} does not have a :ref")
       end
-      if prefix then
-        new_prefix = "#{prefix}/#{key}"
-      else
-        new_prefix = key
+      if model_value[:exists] == true || model_value[:exists] == "true" then
+        node = doc.add_child("<#{key}></#{key}>")
+        if node.is_a?(Nokogiri::XML::NodeSet) then
+          node = node.first 
+        end
+        add_model_struct_to_doc(progress, value[:ref], node, model, xpath)
       end
-      add_model_struct_to_doc(progress, value[:ref], node, model, new_prefix)
     else
       raise RuntimeError.new("struct is broken: encountered unknown :type #{value[:type].inspect}")
     end
@@ -745,23 +805,24 @@ def collect_validation_rule_names(struct)
   return result
 end
 
+def set_standard_options(progress, solver)
+  solver.to_solver(progress, "(set-option :produce-unsat-cores true) ; enable generation of unsat cores")
+  solver.to_solver(progress, "(set-option :smt.core.minimize true) ; ensure that unsat cores are minimal")
+  solver.to_solver(progress, "")
+end
+
 def generate_doc(progress, message, output_filename, config, options)
+  raise RuntimeError.new("option :solverFactory is required") unless options[:solverFactory]
   log = options[:log] || nil
   log.puts "generating doc #{output_filename}" if log
   options[:solverLog] = options[:solverLog] || "#{output_filename}.smt2"
 
-  z3path = config[:z3path] || DEFAULT_Z3_PATH
-  z3 = Z3Solver.new(progress, z3path)
-  begin
-    model = z3.query_model(progress, options) do |solver|
-      solver.to_solver(progress, "(set-option :produce-unsat-cores true) ; enable generation of unsat cores")
-      solver.to_solver(progress, "")
-      log.puts "translating structure" if log
-      translate_structure_to_SMTLIB(progress, solver, message, :options => options, :config => config)
-      solver.to_solver(progress, "")
-    end
-  ensure
-    z3.close()
+  solverFactory = options[:solverFactory]
+  model = solverFactory.query_model(options) do |solver|
+    set_standard_options(progress, solver)
+    log.puts "translating structure" if log
+    translate_structure_to_SMTLIB(progress, solver, message, :options => options, :config => config)
+    solver.to_solver(progress, "")
   end
 
   if model then
@@ -860,6 +921,9 @@ end
 
 def calc_combinations(message, key)
   key_fields = collect_key_fields(key, message)
+  if key_fields.empty?
+    raise FatalError.new("unknown key: #{key}")
+  end
   return calc_combinations_for_key_fields(key_fields)
 end
 
@@ -1043,6 +1107,20 @@ def list_validation_rules(message)
   puts "Validation Rules:"
   rules.each { |r| puts "  #{r}" }
 end
+  
+def value_of_type(value, type)
+  if type == :string or type == :enum then
+    "\"#{value}\""
+  elsif type == :int then
+    value.to_s
+  elsif type == :date then
+    (Date.parse(value) -  Time.at(0).to_date).to_i.to_s
+  elsif type == :timestamp then
+    (Time.new(value) - Time.at(0)).to_i.to_s
+  else
+    raise RuntimeError.new("encountered unknown :datatype #{type.inspect}")
+  end
+end
 
 class CoreEnumerationAlgo
 
@@ -1081,14 +1159,6 @@ class CoreEnumerationAlgo
     "fixed(#{fixed.join(", ")})"
   end
   
-  def value_of_type(value, type)
-    if type == :string or type == :enum then
-      "\"#{value}\""
-    elsif type == :int then
-      value.to_s
-    end
-  end
-
   # prevent the var from taking values that already have been enumerated exhaustively
   def enumerate_blocking_clause(var_xpath, var_def, blocked)
     clauses = []
@@ -1128,24 +1198,19 @@ class CoreEnumerationAlgo
   end
 
   def generate_doc_return_key_values(progress, message, key, output_filename, config, options)
+    raise RuntimeError.new("option :solverFactory is required.") unless options[:solverFactory]
     log = options[:log] || nil
     log.puts "generating doc #{output_filename}" if log
     options[:solverLog] = log_filename_for_output_filename(output_filename)
 
     key_fields = collect_key_fields(key, message)
 
-    z3path = config[:z3path] || DEFAULT_Z3_PATH
-    z3 = Z3Solver.new(progress, z3path)
-    begin
-      model = z3.query_model(progress, options) do |solver|
-        solver.to_solver(progress, "(set-option :produce-unsat-cores true) ; enable generation of unsat cores")
-        solver.to_solver(progress, "")
-        log.puts "translating structure" if log
-        translate_structure_to_SMTLIB(progress, solver, message, :options => options, :config => config)
-        solver.to_solver(progress, "")
-      end
-    ensure
-      z3.close()
+    solverFactory = options[:solverFactory]    
+    model = solverFactory.query_model(options) do |solver|
+      set_standard_options(progress, solver)
+      log.puts "translating structure" if log
+      translate_structure_to_SMTLIB(progress, solver, message, :options => options, :config => config)
+      solver.to_solver(progress, "")
     end
 
     if model then
@@ -1163,9 +1228,12 @@ class CoreEnumerationAlgo
     end
   end
 
-  def store_continuation_point(doc_number, state, assignment, new_assignment, filename)
+  def store_continuation_point(doc_number, state, assignment, new_assignment, filename, options, time)
     File.open(filename, "w") do |f|
-      f.puts("#{doc_number};#{@num_docs};state(#{state.map{|s| if s then "[#{s.map{|v| v.inspect}.join(",")}]" else "nil" end}.join("#")});assignment(#{assignment.to_a.map{|k,v| "#{k}:#{v.inspect}"}.join(",")});new_assignment(#{new_assignment.to_a.map{|k,v| "#{k}:#{v.inspect}"}.join(",")})")
+      state_str = "state(#{state.map{|s| if s then "[#{s.map{|v| v.inspect}.join(",")}]" else "nil" end}.join("#")})"
+      assignment_str = "assignment(#{assignment.to_a.map{|k,v| "#{k}:#{v.inspect}"}.join(",")})"
+      new_assignment_str = "new_assignment(#{new_assignment.to_a.map{|k,v| "#{k}:#{v.inspect}"}.join(",")})"
+      f.puts("#{doc_number};#{@num_docs};#{state_str};#{assignment_str};#{new_assignment_str};#{time}")
     end
   end
 
@@ -1246,7 +1314,8 @@ class CoreEnumerationAlgo
     state = CoreEnumerationAlgo.parse_state(split[2])
     assignment = CoreEnumerationAlgo.parse_assignment(split[3])
     new_assignment = CoreEnumerationAlgo.parse_assignment(split[4])
-    return [doc_number, num_docs, state, assignment, new_assignment]
+    processessing_time = Integer(split[5])
+    return [doc_number, num_docs, state, assignment, new_assignment, processessing_time]
   end
 
   # TODO: logging
@@ -1259,16 +1328,17 @@ class CoreEnumerationAlgo
     res = switched.times.to_a.map do |index|
       var_xpath = vars[index][0]
       "#{var_xpath}=#{assignment[var_xpath].inspect}"
-    end + ["#{vars[switched][0]}|#{blocked.map{|v| v.inspect}.join(",")}"]
+    end + ["#{vars[switched][0]}\{#{blocked.map{|v| v.inspect}.join(",")}}"]
     "{#{res.join(", ")}}"
   end
+  
 
   # core algo
-  def enumerate(progress, switched, vars, assignment, message, key, output_dir, config, options, state = [nil] * vars.size())
-    enumerate(progress, switched+1, vars, assignment, message, key, output_dir, config, options, state) if switched < vars.size()-1
+  def enumerate(progress, switched, vars, assignment, message, key, output_dir, config, options, state = [nil] * vars.size(), start_time, prior_processing_time)
+    enumerate(progress, switched+1, vars, assignment, message, key, output_dir, config, options, state, start_time, prior_processing_time) if switched < vars.size()-1
     is_finite_state = keys_are_finite_state(vars)
     if state[switched] == nil then
-      blocked = [assignment[vars[switched][0]]]
+      blocked = [assignment[vars[switched][0]]] 
       state[switched] = blocked
     else
       blocked = state[switched] + [assignment[vars[switched][0]]]
@@ -1284,24 +1354,33 @@ class CoreEnumerationAlgo
       output_doc_filename = prepare_output_doc_filename(@num_tries, output_dir)
       new_assignment = generate_sparse_doc(progress, message, key, config, options, clauses, output_doc_filename)
       generated_doc_number = @num_tries
-      store_continuation_point(generated_doc_number, state, assignment, new_assignment, "#{output_dir}/#{CONTINUATION_FILE}")
+      time = prior_processing_time + (Time.now().to_i - options[:timestamp_now])
+      store_continuation_point(generated_doc_number, state, assignment, new_assignment, "#{output_dir}/#{CONTINUATION_FILE}", options, time)
+      progress.print_line "STORING continuation point"
+      progress.print_line "|time=#{time}"
+      progress.print_line "|doc_number=#{generated_doc_number}"
+      progress.print_line "|num_docs=#{@num_docs}"
+      progress.print_line "|state=#{state.inspect}"
+      progress.print_line "|assignment=#{assignment.inspect}"
+      progress.print_line "|new_assignment=#{new_assignment.inspect}"
+
       if is_finite_state then
-        progress.report_progress(determine_progress(state, vars))
+        progress.report_progress(determine_progress(state, vars), start_time, prior_processing_time, @num_docs)
       else
-        progress.report_progress(nil)
+        progress.report_progress(nil, start_time, prior_processing_time, @num_docs)
       end
       @num_tries += 1
       if new_assignment then # sat
         @num_docs += 1
         @on_new_doc.call(generated_doc_number, output_doc_filename, new_assignment) if @on_new_doc
         if switched < vars.size()-1 then
-          enumerate(progress, switched+1, vars, new_assignment, message, key, output_dir, config, options, state)
+          enumerate(progress, switched+1, vars, new_assignment, message, key, output_dir, config, options, state, start_time, prior_processing_time)
         end
         blocked << new_assignment[vars[switched][0]]
         state[switched] = blocked
         found = true
       else # unsat
-        @on_unsat.call(generated_doc_number, log_filename_for_output_filename(output_doc_filename), display_combination_space(assignment, vars, switched, blocked)) if @on_unsat
+        @on_unsat.call(generated_doc_number, log_filename_for_output_filename(output_doc_filename), display_combination_space(assignment, vars, switched, blocked), switched) if @on_unsat
         found = false
       end
     end
@@ -1367,18 +1446,17 @@ class CoreEnumerationAlgo
     return true
   end
 
-  def generate_docs_for_key(message, key, output_dir, config, options)
+  def generate_docs_for_key(progress, message, key, output_dir, config, options)
 
+    prior_processing_time = 0
+    start_time = options[:timestamp_now]
     number_of_combinations = calc_combinations(message, key)
     if number_of_combinations.is_a?(String) then
-      progress = ProgressBar.new(100000000000000)
+      progress.set_max(100000000000000)
     else
-      progress = ProgressBar.new(number_of_combinations)
-    end
-    begin
-    ensure
+      progress.set_max(number_of_combinations)
     end
-    progress.report_progress(0.0)
+    progress.report_progress(0.0, start_time, prior_processing_time, 0)
     progress.print_line "generating sparse documents efficiently..."
     progress.print_line "there is a theoretical maximum of #{number_of_combinations} combinations."
     progress.print_line "Note, however, that we are generating only those documents that are valid, which are usually much less."
@@ -1414,28 +1492,27 @@ class CoreEnumerationAlgo
       end
     end
 
-    #progress.print_line "DEBUG: key_fields_list=#{key_fields_list.inspect}"
-
     cont_filename = "#{output_dir}/#{CONTINUATION_FILE}"
     if options[:continue] then
       unless File.exist?(cont_filename) then
         raise FatalError.new("FATAL: file #{cont_filename} not found. There seems to be no continuation point to continue from...")
       end
-      doc_number, num_docs, state, assignment, new_assignment = CoreEnumerationAlgo.parse_continuation_point(File.read(cont_filename).chomp)
+      doc_number, num_docs, state, assignment, new_assignment, prior_processing_time = CoreEnumerationAlgo.parse_continuation_point(File.read(cont_filename).chomp)
       if is_final_state(state, new_assignment, key_fields_list) then
-        progress.report_progress(number_of_combinations)
+        progress.report_progress(number_of_combinations, start_time, prior_processing_time, num_docs)
         raise FatalError.new("FATAL: The enumeration is finished. There is nothing left to do. Resuming the enumeration hence does not make sense.")
       end
       progress.print_line "RESUMING enumeration from the following continuation point:"
+      progress.print_line "|time=#{prior_processing_time}"
       progress.print_line "|doc_number=#{doc_number}"
       progress.print_line "|num_docs=#{num_docs}"
       progress.print_line "|state=#{state.inspect}"
       progress.print_line "|assignment=#{assignment.inspect}"
       progress.print_line "|new_assignment=#{new_assignment.inspect}"
       if keys_are_finite_state(key_fields_list) then
-        progress.report_progress(determine_progress(state, key_fields_list))
+        progress.report_progress(determine_progress(state, key_fields_list), start_time, prior_processing_time, num_docs)
       else
-        progress.report_progress(nil)
+        progress.report_progress(nil, start_time, prior_processing_time, num_docs)
       end
 
       if new_assignment && !new_assignment.empty? then
@@ -1446,15 +1523,13 @@ class CoreEnumerationAlgo
       @num_tries = doc_number+1 # we do not want to overwrite the last document written before the STOP
       number_of_combinations += 1 # account for this in the safety mechanism
       @num_docs = num_docs
-      doc_counter = 0
       File.open(log_filename, log_mode) do |log|
 
         @on_new_doc = Proc.new { |num_doc, doc_filename, combination|
           # combination logging
-          log.puts "#{doc_filename} <- {#{combination.map{|k,v| "#{k}: #{v.inspect}"}.join(", ")}}"
+          log.puts "#{doc_filename} <- {#{combination.map{|k,v| "#{k}=#{v.inspect}"}.join(", ")}}"
 
-          doc_counter += 1
-          if options[:max_num_docs] && doc_counter >= options[:max_num_docs] then
+          if options[:max_num_docs] && @num_docs >= options[:max_num_docs] then
             raise NormalProgramTermination.new("-max-num-docs exceeded. Stopping!")
           end
 
@@ -1463,33 +1538,27 @@ class CoreEnumerationAlgo
             raise RuntimeError.new("This should never happen! Number of combinations exceeded. #{@num_docs} documents generated, but there are only #{number_of_combinations} possible combinations of the key vars.")
           end
         }
-        @on_unsat = Proc.new { |num_doc, log_filename, combination_str|
+        @on_unsat = Proc.new { |num_doc, log_filename, combination_str, switched|
               log.puts "CONTRADICTION (see: #{log_filename}) <- #{combination_str}"
               progress.print_line "doc#{num_doc} NOT generated"
-              progress.print_line "No more valid combinations of the free vars could be found. -> Switching"
+              progress.print_line "No more valid combinations of the free vars could be found. -> Switching (var #{switched})"
         }
 
         # core algo - resume from state
-        enumerate(progress, 0, key_fields_list, assignment, message, key, output_dir, config, options, state)
+        enumerate(progress, 0, key_fields_list, assignment, message, key, output_dir, config, options, state, start_time, prior_processing_time)
       end
     else
       # init safety mechanism
       @num_tries = 0
       @num_docs = 0
-      doc_counter = 0
       # first try - a freely generated document
       File.open(log_filename, log_mode) do |log|
         output_doc_filename = prepare_output_doc_filename(@num_tries, output_dir)
-        key_values = generate_doc_return_key_values(progress, message, key, output_doc_filename, config, options)
-        log.puts "#{output_doc_filename} <- {#{key_values.map{|k,v| "#{k}: #{v.inspect}"}.join(", ")}}"
-        doc_counter += 1
-        @num_tries += 1
         @on_new_doc = Proc.new { |num_doc, doc_filename, combination|
           # combination logging
-          log.puts "#{doc_filename} <- {#{combination.map{|k,v| "#{k}: #{v.inspect}"}.join(", ")}}"
+          log.puts "#{doc_filename} <- {#{combination.map{|k,v| "#{k}=#{v.inspect}"}.join(", ")}}"
 
-          doc_counter += 1
-          if options[:max_num_docs] && doc_counter >= options[:max_num_docs] then
+          if options[:max_num_docs] && @num_docs >= options[:max_num_docs] then
             raise NormalProgramTermination.new("-max-num-docs exceeded. Stopping!")
           end
 
@@ -1500,19 +1569,28 @@ class CoreEnumerationAlgo
             end
           end
         }
-        @on_unsat = Proc.new { |num_doc, log_filename, combination_str|
+        @on_unsat = Proc.new { |num_doc, log_filename, combination_str, switched|
               log.puts "CONTRADICTION (see: #{log_filename}) <- #{combination_str}"
               progress.print_line "doc#{num_doc} NOT generated"
-              progress.print_line "No more valid combinations of the free vars could be found. -> Switching"
+              progress.print_line "No more valid combinations of the free vars could be found. -> Switching (var #{switched})"
         }
+        key_values = generate_doc_return_key_values(progress, message, key, output_doc_filename, config, options)
+        @num_tries += 1
+        if key_values then
+          @num_docs += 1
+          @on_new_doc.call(@num_tries, output_doc_filename, key_values)
+        else
+          # NOTE: this only occurs when the model is inconsistent and has no valid documents
+          progress.print_line("ERROR: Model inconsistent - no valid documents could be found.")
+          @on_unsat.call(@num_tries, log_filename_for_output_filename(output_doc_filename), "{}", 0)
+        end
 
         # core algo - start enumeration
-        enumerate(progress, 0, key_fields_list, key_values, message, key, output_dir, config, options)
+        enumerate(progress, 0, key_fields_list, key_values, message, key, output_dir, config, options, start_time, prior_processing_time)
       end
-      
-      progress.print_line "No more valid documents could be found. -> Exiting"
-      progress.report_progress(number_of_combinations)
     end
+    progress.print_line "No more valid documents could be found. -> Exiting"
+    progress.report_progress(number_of_combinations, start_time, prior_processing_time, @num_docs)
   end
 end
 
@@ -1585,6 +1663,211 @@ def validate_model(message, prefix=nil)
   end
 end
 
+def validate_doc_against_schema(struct, doc)
+  root = doc.root
+  sdef = struct[root.name.to_sym]
+  unless sdef[:type] == :structure then
+    puts "ERROR: schema validation failed: root element should be a structure"
+  end  
+  # first, check that each doc element is defined for the struct
+  validate_doc_struct_against_schema(sdef[:ref], root, root.name)
+  # second, check that all required elements from the struct are actually present in the doc
+  validate_schema_against_doc_struct(sdef[:ref], root, root.name)
+end
+
+def validate_doc_struct_against_schema(struct, doc, prefix = nil)
+  doc.children.to_a.filter{|x| x.is_a?(Nokogiri::XML::Element) }.each do |c|
+    fdef = struct[c.name.to_sym]
+    if prefix then
+      xpath = "#{prefix}/#{c.name}"
+    else
+      xpath = c.name
+    end
+    if fdef
+      if fdef[:type] == :structure then
+        return unless validate_doc_struct_against_schema(fdef[:ref], c, xpath)
+      end
+    else
+      if prefix then
+        xpath = "#{prefix}/#{c.name}"
+      else
+        xpath = c.name
+      end
+      raise FatalError.new("ERROR: schema validation failed: #{c.name} of #{xpath} is present in document, but not defined in schema.")
+    end
+  end
+end
+
+def validate_schema_against_doc_struct(struct, doc, prefix = nil)
+  struct.each do |key, value|
+    next if key == :additional_smtlib || key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix
+      if prefix then
+        xpath = "#{prefix}/#{key}"
+      else
+        xpath = key
+      end
+    if value[:type] == :structure then
+      node = doc.at_xpath(key.to_s)
+      if node then
+        validate_schema_against_doc_struct(value[:ref], node, xpath)
+      else
+        if value[:optional] == false then
+          raise FatalError.new("ERROR: schema validation failed: non-optional structure #{xpath} not found in doc.")
+        end
+      end
+    elsif value[:type] == :field then
+      if value[:optional] == false then
+        unless doc.at_xpath(key.to_s) then
+          raise FatalError.new("ERROR: schema validation failed: non-optional field #{xpath} not found in doc.")
+        end
+      end
+    elsif value[:type] == :list then
+      if value[:optional] == false then
+        unless doc.at_xpath(key.to_s) then
+          raise FatalError.new("ERROR: schema validation failed: non-optional list #{xpath} not found in doc.")
+        end
+      end
+      list = doc.at_xpath(key.to_s)
+      if list then
+        list.children.to_a.each do |c|
+          next unless c.is_a?(Nokogiri::XML::Element)
+          unless c.name == value[:xpath_element] then
+            raise FatalError.new("ERROR: schema validation failed: encountered element #{c.name} in list #{xpath}, which should only contain #{value[:xpath_element]} elements.")
+          end
+        end
+      end
+    else
+      raise RuntimeError.new("Model error: encountered element #{xpath} of unknown type: #{value[:type]}.")
+    end
+  end  
+end
+
+def translate_doc_to_SMTLIB(progress, solver, struct, doc, hash = {})
+  options = hash[:options]
+  config = hash[:config]
+
+  solver.to_solver(progress, "; --- Document: ---")
+  translate_struct_in_doc_to_SMTLIB(progress, solver, struct, doc, nil, hash)
+end
+
+def translate_struct_in_doc_to_SMTLIB(progress, solver, struct, doc, prefix, hash = {})
+  options = hash[:options]
+  config = hash[:config]
+
+  struct.each do |key, value|
+    next if key == :additional_smtlib || key == :validation_rules || key == :predicates || key == :parent_link || key == :struct_xpath_prefix
+    if prefix then
+      xpath = "#{prefix}/#{key.to_s}"
+    else
+      xpath = key.to_s
+    end
+    if value[:type] == :structure then
+      node = doc.at_xpath(xpath)
+      if node then
+        info = {
+          :assertion_type => :doc_structure_exists,
+          :xpath => xpath,
+        }
+        assertionID = "doc-#{xpath.gsub('/', '-')}"
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (= #{exists_var_for_structure(xpath)} true) :named #{assertionID}))")
+
+        translate_struct_in_doc_to_SMTLIB(progress, solver, value[:ref], doc, xpath, hash)
+      else
+        info = {
+          :assertion_type => :doc_structure_not_exists,
+          :xpath => xpath,
+        }
+        assertionID = "doc-#{xpath.gsub('/', '-')}"
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (= #{exists_var_for_structure(xpath)} false) :named #{assertionID}))")
+      end
+    elsif value[:type] == :field then
+      node = doc.at_xpath(xpath)
+      if node then
+        fvalue = node.text
+        info = {
+          :assertion_type => :doc_field_filled,
+          :xpath => xpath,
+          :value => fvalue
+        }
+        fvalue_str = value_of_type(fvalue, value[:datatype])
+        assertionID = "doc-#{xpath.gsub('/', '-')}"
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (and (= #{filled_var_for_field(xpath)} true) (= #{value_var_for_field(xpath)} #{fvalue_str})) :named #{assertionID}))")
+      else
+        info = {
+          :assertion_type => :doc_field_empty,
+          :xpath => xpath,
+        }
+        assertionID = "doc-#{xpath.gsub('/', '-')}"
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (= #{filled_var_for_field(xpath)} false) :named #{assertionID}))")
+      end
+    elsif value[:type] == :list then
+      node = doc.at_xpath(xpath)
+      if node then
+        fvalues = []
+        node.children.to_a.each do |c|
+          next unless c.is_a?(Nokogiri::XML::Element)          
+          fvalues << c.text
+        end
+        info = {
+          :assertion_type => :doc_list_filled,
+          :xpath => xpath,
+          :values => fvalues
+        }
+        fvalue_str = value_of_type(fvalues, value[:datatype])
+        assertionID = "doc-#{xpath.gsub('/', '-')}"
+        solver.associateAssertionIDWith(assertionID, info)
+        clauses = []
+        fvalues.each_with_index do |fvalue, index|
+          clauses << "(= #{value_var_for_list(xpath,index)} #{value_of_type(fvalue, value[:datatype])})"
+        end
+        solver.to_solver(progress, "(assert (! (and (= #{filled_var_for_field(xpath)} true) (= #{size_var_for_list(xpath)} #{fvalues.size}) #{clauses.join(' ')}) :named #{assertionID}))")
+      else
+        info = {
+          :assertion_type => :doc_field_empty,
+          :xpath => xpath,
+        }
+        assertionID = "doc-#{xpath.gsub('/', '-')}"
+        solver.associateAssertionIDWith(assertionID, info)
+        solver.to_solver(progress, "(assert (! (= #{filled_var_for_field(xpath)} false) :named #{assertionID}))")
+      end
+    else
+      raise RuntimeError.new("Model Error: encountered unknown type #{value[:type].inspect}")
+    end
+  end
+end
+
+def validate_doc(progress, struct, doc, options, config)
+  raise RuntimeError.new("option :solverFactory is required") unless options[:solverFactory]
+  options[:solverLog] = "validation.smt2"
+
+  solverFactory = options[:solverFactory]
+  result = solverFactory.query(options) do |solver|
+    set_standard_options(progress, solver)
+    translate_structure_to_SMTLIB(progress, solver, struct, :options => options, :config => config)
+    translate_doc_to_SMTLIB(progress, solver, struct, doc, :options => options, :config => config)
+    solver.to_solver(progress, "")
+  end
+  unless result == true then
+    raise FatalError.new("VALIDATION FAILED! SMTLIB log was written to #{options[:solverLog]}.\n\n#{result}")
+  end
+
+  return true
+end
+
+def generate_unused_logfile_name()
+  number = 0
+  result = "logs/#{number}.log"
+  while File.exist?(result) do
+    number += 1
+    result = "logs/#{number}.log"
+  end
+  result
+end
+
 def generate(message, argv, message_config_filename = nil)
   if ARGV[0] == "--help" || ARGV[0] == "-help" || ARGV[0] == "help" then
     puts USAGE
@@ -1617,6 +1900,12 @@ def generate(message, argv, message_config_filename = nil)
     elsif opt == "-max-num-docs" then
       options[:max_num_docs] = Integer(argv.shift)
       puts "flag -max-num-docs used. Will stop after generating #{options[:max_num_docs]} documents."
+    elsif opt == "-validate" then
+      if options.has_key?(:validate_files) then
+        options[:validate_files] << argv.shift
+      else
+        options[:validate_files] = [argv.shift]
+      end
     else
       puts "FATAL: unknown option #{opt.inspect}."
       puts USAGE
@@ -1625,51 +1914,63 @@ def generate(message, argv, message_config_filename = nil)
     opt = argv.shift
   end
 
-  unless options[:negated_validation_rules].empty? then
-    puts "negating rules: #{options[:negated_validation_rules].join(", ")}"
+  ProgressBar.logging(1, generate_unused_logfile_name()) do |progress|
+    unless options[:negated_validation_rules].empty? then
+      progress.print_line "negating rules: #{options[:negated_validation_rules].join(", ")}"
+
+      rules = collect_validation_rule_names(message)
+      unknown_negated_rules = options[:negated_validation_rules].filter { |rule_name| !rules.include?(rule_name) } 
+      unless unknown_negated_rules.empty? then  
+        throw RuntimeError.new("unknown negated rules #{unknown_negated_rules.join(", ")}")
+      end
 
-    rules = collect_validation_rule_names(message)
-    unknown_negated_rules = options[:negated_validation_rules].filter { |rule_name| !rules.include?(rule_name) } 
-    unless unknown_negated_rules.empty? then  
-      throw RuntimeError.new("unknown negated rules #{unknown_negated_rules.join(", ")}")
     end
 
-  end
+    if File.exist?(CONFIG_FILE) then
+      config = YAML.load_file(CONFIG_FILE)
+    else
+      config = DEFAULT_CONFIG
+      File.open(CONFIG_FILE, "w") do |f|
+        f.puts(config.to_yaml)
+      end
+      progress.print_line "No config file found. Default config written to #{CONFIG_FILE}"
+    end
+    if message_config_filename then
+      message_config = YAML.load_file(message_config_filename)
+      config = config.merge(message_config)
+    end
 
-  if File.exist?(CONFIG_FILE) then
-    config = YAML.load_file(CONFIG_FILE)
-  else
-    config = DEFAULT_CONFIG
-    File.open(CONFIG_FILE, "w") do |f|
-      f.puts(config.to_yaml)
-    end
-    puts "No config file found. Default config written to #{CONFIG_FILE}"
-  end
-  if message_config_filename then
-    message_config = YAML.load_file(message_config_filename)
-    config = config.merge(message_config)
-  end
-
-  introduce_parent_links(message)
-
-  if options[:count_docs_for_key] then
-    puts "there are #{calc_combinations(message, options[:count_docs_for_key])} combinations."
-  elsif options[:docs_for_key] then
-    puts "using #{options[:date_now]} as [Date.now]"
-    puts "using #{options[:timestamp_now]} as [Timestamp.now]"
-    validate_model(message)
-    CoreEnumerationAlgo.new().generate_docs_for_key(message, options[:docs_for_key], OUTPUT_DIR, config, options)
-  elsif options[:list_keys] then
-    list_keys(message)
-  elsif options[:list] then
-    list_fields_and_structures(message, options[:list])
-  elsif options[:list_validation_rules] then
-    list_validation_rules(message)
-  else
-    puts "using #{options[:date_now]} as [Date.now]"
-    puts "using #{options[:timestamp_now]} as [Timestamp.now]"
-    validate_model(message)
-    progress = ProgressBar.new(1)
-    generate_doc(progress, message, OUTPUT_FILE, config, options)
+    introduce_parent_links(message)
+    z3path = config[:z3path] || DEFAULT_Z3_PATH
+    options[:solverFactory] = SolverFactory.new(progress, z3path)
+
+    if options[:validate_files] then
+      progress.print_line "using #{options[:date_now]} as [Date.now]"
+      progress.print_line "using #{options[:timestamp_now]} as [Timestamp.now]"
+      options[:validate_files].each do |filename|
+        progress.print_line "validating file #{filename}"
+        doc = Nokogiri::XML(File.read(filename))
+        validate_doc_against_schema(message, doc)
+        validate_doc(progress, message, doc, options, config)
+      end
+    elsif options[:count_docs_for_key] then
+      progress.print_line "there are #{calc_combinations(message, options[:count_docs_for_key])} combinations."
+    elsif options[:docs_for_key] then
+      progress.print_line "using #{options[:date_now]} as [Date.now]"
+      progress.print_line "using #{options[:timestamp_now]} as [Timestamp.now]"
+      validate_model(message)
+      CoreEnumerationAlgo.new().generate_docs_for_key(progress, message, options[:docs_for_key], OUTPUT_DIR, config, options)
+    elsif options[:list_keys] then
+      list_keys(message)
+    elsif options[:list] then
+      list_fields_and_structures(message, options[:list])
+    elsif options[:list_validation_rules] then
+      list_validation_rules(message)
+    else
+      progress.print_line "using #{options[:date_now]} as [Date.now]"
+      progress.print_line "using #{options[:timestamp_now]} as [Timestamp.now]"
+      validate_model(message)
+      generate_doc(progress, message, OUTPUT_FILE, config, options)
+    end
   end
 end