cloud-toaster 1.1.2

data/lib/toaster/state/syscall_tracer.rb

@@ -0,0 +1,317 @@
+
+#
+# Author: Waldemar Hummer (hummer@dsg.tuwien.ac.at)
+#
+
+require "toaster/state/transition_edge"
+require "toaster/markup/markup_util"
+require "toaster/state/ptrace_util"
+require "pty"
+
+include Toaster
+
+module Toaster
+
+  class SyscallTracer
+
+    FILE_DUMPSTATE = "/tmp/tracer.dumpstate"
+    FILE_STATEDUMP = "/tmp/tracer.state.dump"
+    FILE_TMPDIR = "/tmp/tracer.tmp.dir"
+    FILE_ACKS = "/tmp/tracer.acks"
+
+    def initialize()
+      # type 0 syscalls (see below for parsing)
+      @syscalls_0 = ["open", "write", "openat"]
+      # type 1 syscalls (see below for parsing)
+      @syscalls_1 = ["creat", "mkdir", "rmdir", "link", "unlink", "symlink", 
+        "chown", "lchown", "chmod"]
+      # type 2 syscalls (see below for parsing)
+      @syscalls_2 = ["rename", "unlinkat", "mkdirat", "fchownat", "mknodat"]
+      # type 3 syscalls (see below for parsing)
+      @syscalls_3 = ["fchmod", "fchown"]
+      # type 4 syscalls (see below for parsing)
+      @syscalls_4 = ["utimensat"]
+      @acked_syscalls = @syscalls_0.concat(@syscalls_1).concat(@syscalls_2).concat(@syscalls_3).concat(@syscalls_4)
+      @monitored_syscalls = @acked_syscalls.dup.concat(["open", "openat", "chdir"]) #, "close", "dup", "dup2", "dup3"
+      @grep_cmd = "ps aux | grep -v grep | grep -v bash | grep -v screen | grep -v SCREEN | grep strace | grep #{Process.pid}"
+      @pwd_map = {}
+      @num_acks_sent = 0
+      @num_acks_received = 0
+      @execution_prestate = {"files"=>{}}
+    end
+
+    def dump_execution_prestate
+      Util.write(FILE_DUMPSTATE, "1", true)
+
+      # HACK: this causes a syscall which initiates the dump further below..
+      Dir.mkdir(FILE_TMPDIR)
+      Dir.rmdir(FILE_TMPDIR)
+
+      state = File.read(FILE_STATEDUMP)
+      if state.strip == ""
+        puts "WARN: could not read pre-state dump from #{FILE_STATEDUMP}"
+        return {}
+      else
+        return MarkupUtil.parse_json(state)
+      end
+    end
+
+    def start()
+
+      #puts "INFO: writing values to #{FILE_ACKS}"
+      `echo ":#{@acked_syscalls.join(':')}:" > #{FILE_ACKS}`
+      @monitoring_active = true
+
+      `echo "" > #{FILE_DUMPSTATE}`
+      `echo "" > #{FILE_STATEDUMP}`
+
+      # start strace process
+      do_start_process = true
+      do_start_thread = false
+
+      if do_start_process
+
+        tracer_pid = start_strace()
+
+        if do_start_thread
+          __legacy_start_thread()
+        end
+
+      else
+        __legacy_ruby_ptrace()
+      end
+
+    end
+
+    def stop()
+      puts "DEBUG: writing empty string to #{FILE_ACKS}"
+      `echo "" > #{FILE_ACKS}`
+      @monitoring_active = false
+    end
+
+    def parse_line(line)
+      pwd_map = @pwd_map
+      if is_correct_strace_line(line)
+
+        syscall = extract_syscall(line)
+        syscall_pid = extract_pid(line)
+        if !pwd_map[syscall_pid]
+          # get initial pwd for the process which does the syscall
+          pwd_map[syscall_pid] = get_pwd_for_pid(syscall_pid)
+        end
+        pwd = pwd_map[syscall_pid]
+        if syscall == "chdir"
+          pwd = pwd_map[syscall_pid] = get_pwd_for_pid(syscall_pid)
+        elsif syscall == "write"
+          if line.match(/.*write\([0-9]+<(.*)>.*/)
+            mod_file = line.gsub(/.*write\([0-9]+<(.*)>.*/, '\1').strip
+            return report_file(mod_file, pwd)
+          elsif line.match(/.*write\([0-9]+,.*/)
+            fd = line.gsub(/.*write\(([0-9]+),.*/, '\1').strip
+            mod_file = PtraceUtil.get_filename_for_fd(syscall_pid, fd)
+            return report_file(mod_file, pwd)
+          end
+        elsif syscall == "open" || syscall == "openat"
+          if  line.match(/O_WRONLY/) ||
+              line.match(/O_RDWR/) ||
+              line.match(/O_APPEND/) ||
+              line.match(/O_CREAT/)
+            if syscall == "open"
+              mod_file = line.gsub(/.*open\("([^"]+)".*/, '\1').strip
+              return report_file(mod_file, pwd)
+            elsif syscall == "openat"
+              if line.match(/.*openat\(AT_FDCWD,\s*"([^"]+)".*/)
+                mod_file = line.gsub(/.*openat\(AT_FDCWD,\s*"([^"]+)".*/, '\1').strip
+                return report_file(mod_file, pwd)
+              elsif line.match(/.*openat\([0-9]+,\s*"([^"]+)".*/)
+                fd = line.gsub(/.*openat\(([0-9]+),\s*"[^"]+".*/, '\1').strip.to_i
+                mod_file = PtraceUtil.get_filename_for_fd(syscall_pid, fd)
+                return report_file(mod_file, pwd)
+              elsif line.match(/.*openat\("([^"]+)",\s*"[^"]+".*/)
+                mod_file = line.gsub(/.*openat\("([^"]+)",\s*"[^"]+".*/, '\1').strip
+                return report_file(mod_file, pwd)
+              end
+            end
+          end
+        else
+
+          if @syscalls_1.include?(syscall)
+            mod_file = line.gsub(/.*((#{@syscalls_1.join(')|(')}))\("([^"]+)".*/, "\\#{@syscalls_1.size + 2}").strip
+            return report_file(mod_file, pwd)
+          end
+          if @syscalls_2.include?(syscall)
+            mod_file = line.gsub(/.*((#{@syscalls_2.join(')|(')}))\([^,]+,\s*"([^"]+)".*/, "\\#{@syscalls_2.size + 2}").strip
+            return report_file(mod_file, pwd)
+          end
+          if @syscalls_3.include?(syscall)
+            fd = line.gsub(/.*((#{@syscalls_3.join(')|(')}))\(([^,]+),.*/, "\\#{@syscalls_3.size + 2}").strip.to_i
+            mod_file = PtraceUtil.get_filename_for_fd(syscall_pid, fd)
+            #puts "!! syscalls_3: #{mod_file}"
+            return report_file(mod_file, pwd)
+          end
+          if @syscalls_4.include?(syscall)
+            mod_file = nil
+            pattern1 = ".*((#{@syscalls_4.join(')|(')}))\\(([0-9]+),.*"
+            pattern2 = ".*((#{@syscalls_4.join(')|(')}))\\([^,]+,\\s*\"([^\"]+)\".*"
+            if line.match(/#{pattern1}/)
+              fd = line.gsub(/#{pattern1}/, "\\#{@syscalls_4.size + 2}").strip.to_i
+              mod_file = PtraceUtil.get_filename_for_fd(syscall_pid, fd)
+            elsif line.match(/#{pattern2}/)
+              mod_file = line.gsub(/#{pattern2}/, "\\#{@syscalls_4.size + 2}")
+            end
+            #puts "!! syscalls_3: #{mod_file}"
+            return report_file(mod_file, pwd)
+          end
+
+        end
+
+      end
+      return nil
+    end
+
+    def is_correct_strace_line(line)
+      line.match(/^(\[pid\s+)?[0-9]+\]?\s+[a-zA-Z0-2]+\(.*/)
+    end
+    def extract_syscall(line)
+      line.gsub(/^(\[pid\s+)?[0-9]+\]?\s+([a-zA-Z0-2]+)\(.*/, '\2').strip
+    end
+    def extract_pid(line)
+      line.gsub(/^(\[pid\s+)?([0-9]+)\]?\s+([a-zA-Z0-2]+)\(.*/, '\2').strip
+    end
+
+    def report_file(mod_file, pwd)
+      if !mod_file || mod_file.to_s.strip == ""
+        return
+      end
+      if  !mod_file.match(/pipe:.*/) && 
+          !mod_file.match(/socket:.*/) && 
+          !mod_file.match(/\/dev\/.*/) &&
+          !mod_file.match(/\/proc\/.*/) && 
+          !mod_file.match(/\/tmp\/chef-script.*/) && 
+          !mod_file.match(/\/var\/chef\/cache\/.*/) && 
+          !mod_file.match(/\/tmp\/tracer\.acks.*/)
+        if is_correct_strace_line(mod_file)
+          puts "WARN: Apparently could not parse syscall line from strace: #{mod_file}"
+          return
+        end
+        #puts "=====> #{pwd} - #{mod_file}"
+        if mod_file[0] != "/" && pwd != ""
+          mod_file = "#{pwd}/#{mod_file}"
+        end
+        # ignore /tmp files
+        if !mod_file.match(/^\/tmp\//)
+          return mod_file
+        end
+        return nil
+      end
+    end
+
+    private
+
+    def start_strace()
+
+      tracer_pid = get_tracer_pid()
+      if tracer_pid == ""
+        # Start monitoring process in background
+        exec_dir = File.join(File.dirname(File.expand_path(__FILE__)), 
+            "..", "..", "..", "bin", "strace-4.8_patched")
+        to_exec = nil
+        if (/x86_64/ =~ RUBY_PLATFORM) != nil
+          to_exec = "#{exec_dir}/strace-x86_64"
+        elsif (/i686/ =~ RUBY_PLATFORM) != nil
+          to_exec = "#{exec_dir}/strace-i686"
+        else
+          emsg = "Unsupported platform or CPU architecture (no strace binary available): #{RUBY_PLATFORM}"
+          puts "WARN: #{emsg}"
+          throw emsg
+        end
+        tracer_cmd = "#{to_exec} -p #{Process.pid} -f -e #{@monitored_syscalls.join(',')} -s 0"
+
+        if File.exist?(to_exec)
+          puts "INFO: Start monitoring process in background: #{tracer_cmd}"
+          fork_pid = Process.fork do
+            PTY.spawn( tracer_cmd ) do |stdout, stdin, pid|
+              begin
+                stdout.each { |line|
+                  #puts ": #{line}"
+  
+                  # write prestate dump to file if requested..
+                  if File.read(FILE_DUMPSTATE).strip == "1"
+                    puts "INFO: dumping state to file #{FILE_STATEDUMP}.."
+                    #puts "DEBUG: this syscall: #{line}"
+                    Util.write(FILE_STATEDUMP, MarkupUtil.to_json(@execution_prestate), true)
+                    @execution_prestate = {"files" => {}}
+                    Util.write(FILE_DUMPSTATE, "0", true)
+                  end
+  
+                  begin
+                    if line.match(/waiting for ack for syscall/)
+                      #puts "sending ACK for syscall: #{line}"
+                      stdin.puts("y")
+                      @num_acks_sent += 1
+                      #puts "num_acks_sent: #{@num_acks_sent}" if (@num_acks_sent % 100 == 0)
+                    elsif line.match(/got ack for syscall/)
+                      @num_acks_received += 1
+                      #puts "num_acks_received: #{@num_acks_received}" if @num_acks_received % 100 == 0
+                    else
+                      if @monitoring_active
+                        mod_file = parse_line(line)
+                        if mod_file
+                          #puts "=====> #{mod_file}"
+                          if !@execution_prestate["files"].include?(mod_file) # don't overwrite previous state info!
+                            Util.build_file_hash_for_ohai([mod_file], @execution_prestate["files"])
+                          end
+                        end
+                      end
+                    end
+  
+                  rescue => exc
+                    puts "WARN: Exception in strace processing: #{exc} - #{exc.backtrace.join("\n")}"
+                  end
+                }
+              rescue Errno::EIO
+                # swallow - this probably just means that the process has finished giving output"
+              end
+              puts "INFO: Strace process terminating..."
+            end
+          end
+
+        else
+
+          puts "WARN: Cannot find patched strace executable: #{tracer_cmd}"
+
+        end
+
+      end
+      tracer_pid = get_tracer_pid()
+      return tracer_pid
+    end
+
+    def kill_tracer_process()
+      pid = get_tracer_pid()
+      if pid && pid != ""
+        `kill #{pid}`
+      end
+    end
+
+    def get_tracer_pid()
+      tracer_pid = `#{@grep_cmd}`
+      #puts "grepping for strace - output: #{tracer_pid}"
+      tracer_pid = tracer_pid.gsub(/[^\s]+\s+([^\s]+)\s+.*/, '\1').strip
+      return tracer_pid
+    end
+
+    def get_pwd_for_pid(pid)
+      dir = `pwdx #{pid} 2> /dev/null`
+      dir = dir.gsub(/[0-9]*:\s+(.*)/, '\1').strip
+      if dir != ""
+        return dir
+      end
+      #puts "WARN: Unknown pwd for process #{pid}, fallback to '#{Dir.pwd}'"
+      # fallback to pwd
+      return Dir.pwd
+    end
+
+  end
+
+end

data/lib/toaster/state/system_state.rb

@@ -0,0 +1,349 @@
+
+#
+# Author: Waldemar Hummer (hummer@dsg.tuwien.ac.at)
+#
+
+require "toaster/markup/markup_util"
+require "toaster/util/util"
+require "ohai/application"
+
+include Toaster
+
+module Toaster
+  class SystemState
+
+    @@initialized = false
+    @@include_default_state_props = false
+    @@required_builtin_ohai_plugins = ["languages.rb", "ruby.rb", "kernel.rb", "os.rb"]
+    @@original_ohai_plugin_paths = []
+    @@ohai_dir = File.join(File.expand_path(File.dirname(__FILE__)), '..', 'ohai')
+    @@max_arglist_length = 5000 # maximum number of characters to pass as cmdline parameter to ohai
+
+    def self.get_system_state(state_change_config = {})
+      puts "INFO: Taking snapshot of system state..."
+      puts "DEBUG: State snapshot configuration: #{state_change_config}" 
+
+      if !@@initialized
+        @@original_ohai_plugin_paths.concat(Ohai::Config[:plugin_path])
+        @@original_ohai_plugin_paths.each do |path|
+          @@required_builtin_ohai_plugins.dup.each do |pl|
+            required_file = File.join(path, pl)
+            if File.exist?(required_file)
+              @@required_builtin_ohai_plugins.delete(pl)
+              @@required_builtin_ohai_plugins << required_file
+            end
+          end
+        end
+        @@initialized = true
+      end
+
+      Ohai::Config[:plugin_path] = []
+      if @@include_default_state_props
+        Ohai::Config[:plugin_path].concat(@@original_ohai_plugin_paths)
+      end
+
+      state_change_config.each do |name,config|
+        # register tailor-made Ohai extensions
+        path = File.expand_path(File.join(@@ohai_dir, name))
+        # puts "Registering ohai extensions within directory #{path}"
+        Ohai::Config[:plugin_path].push(path)
+      end
+
+      ENV["OHAI_PARAMS"] = state_change_config.to_json().to_s
+
+      # NOTE: We need to be careful here. OHAI_PARAMS is passed as
+      # command line argument to ohai, and if this hash becomes
+      # too big, we end up with an "Argument list too long" error.
+      # If the hash becomes too big, save it to a file and read it 
+      # from there afterwards!
+      if ENV["OHAI_PARAMS"].size > @@max_arglist_length
+        params_file = "/tmp/toaster.ohai_params.tmp"
+        Util.write(params_file, ENV["OHAI_PARAMS"], true)
+        ENV["OHAI_PARAMS"] = "{\"__read_from_file__\": \"#{params_file}\"}"
+      end
+
+      ohai = Ohai::System.new
+      @@required_builtin_ohai_plugins.each do |plugin_file|
+        begin
+          if !defined?(ohai.from_file(plugin_file))
+            # some ohai versions don't seem to mix-in "from_file"
+            Ohai::System.send(:include, Chef::Mixin::FromFile)
+          end
+          ohai.from_file(plugin_file)
+        rescue => ex
+          puts "WARN: Unable to include ohai plugin file '#{plugin_file}': #{ex}: #{ex.backtrace.join("\n")}"
+          throw ex
+        end
+      end
+
+      if Ohai::Config[:file]
+        ohai.from_file(Ohai::Config[:file])
+      else
+        ohai.all_plugins
+      end
+      json = JSON.parse(ohai.to_json)
+      filter_unimportant_properties(json)
+      #puts "DEBUG: System state json: #{json.inspect}"
+
+      return json
+    end
+
+    # Given two states, preprocess the state difference computation
+    # by removing those state properties which are usually very large
+    # and infeasible to process with the generic approach (structural
+    # diff of state property trees).
+    #
+    # * Returns: an array [s1,s2,diffs] with the (potentially) modified
+    #   states ("s1" and "s2") and part of the differences ("diffs")
+    #   between the original states.
+    def self.preprocess_state_diff(state1, state2)
+      diffs = []
+      keys1 = state1.keys.dup
+      keys2 = state2.keys.dup
+
+      keys1.each do |k|
+        file = File.join(@@ohai_dir, k, "_meta.rb")
+        if File.exist?(file)
+          require file
+          s1 = state1[k]
+          s2 = state2[k]
+          if !s1 || !s2
+            next
+          end
+          tmp_result = nil
+          begin
+            eval("tmp_result = diff__#{k}(s1, s2)")
+            if tmp_result && tmp_result.kind_of?(Array)
+              state1.delete(k)
+              state2.delete(k)
+              diffs.concat(tmp_result)
+            end
+          rescue => ex
+            puts "WARN: Unable to compute diff of state property '#{k}' using code in file #{file}:"
+            Util.print_backtrace(ex, 10)
+          end
+        end
+      end
+
+      return [state1, state2, diffs]
+    end
+
+    ## Reconstruct the final post-state that results from a
+    ## sequence of task executions and their individual
+    ## state changes
+    def self.reconstruct_state_from_execs_seq(execs_seq)
+      state = execs_seq[0].state_before
+      #puts "DEBUG: Eliminate map entries from state: #{state}"
+      MarkupUtil.eliminate_inserted_map_entries!(state)
+      return reconstruct_state_from_change_seq(
+        execs_seq.collect{ |ex| ex.state_changes },
+        state
+      )
+    end
+    ## Reconstruct the final post-state that results from a
+    ## sequence of state changes
+    def self.reconstruct_state_from_change_seq(state_change_seq, initial_state={})
+      state = initial_state
+      state_change_seq.each do |change_set|
+        change_set.each do |ch|
+          if ch.action == StateChange::ACTION_DELETE
+            #puts "==> delete property #{ch.property}"
+            MarkupUtil.delete_value_by_path(state, ch.property)
+          elsif ch.action == StateChange::ACTION_INSERT ||
+              ch.action == StateChange::ACTION_MODIFY
+            #puts "==> set property #{ch.property} = #{ch.value}"
+            MarkupUtil.set_value_by_path(state, ch.property, ch.value)
+          end
+        end
+      end
+      #puts "DEBUG: Reconstructed state from state change sequence: #{state.to_s}"
+      return state
+    end
+
+    # Compute the difference between two system state snapshots.
+    # Returns an array of StateChange objects.
+    def self.get_state_diff(s_before, s_after)
+      tmp = preprocess_state_diff(s_before, s_after)
+      s_before = tmp[0]
+      s_after = tmp[1]
+      prop_changes = tmp[2]
+
+      prop_changes.concat(MarkupUtil.hash_diff_as_prop_changes(s_before, s_after))
+      return prop_changes
+    end
+
+    # Given two states which are "too" big, reduce the size of both states
+    # by removing properties that are equal in both states and hence not
+    # relevant for the state change computation.
+    def self.reduce_state_size(state1, state2)
+      state1_copy = state1.dup
+      state2_copy = state2.dup
+
+      state1.keys.each do |k|
+        file = File.join(@@ohai_dir, k, "_meta.rb")
+        if File.exist?(file)
+          require file
+          s1 = state1[k]
+          s2 = state2[k]
+          tmp_result = nil
+          begin
+            eval("tmp_result = reduce__#{k}(s1, s2)")
+            if tmp_result && tmp_result.kind_of?(Array)
+              state1_copy[k] = tmp_result[0]
+              state2_copy[k] = tmp_result[1]
+            end
+          rescue => ex
+            puts "WARN: Unable to compute reduced hash of state property '#{k}' using code in file #{file}:"
+            Util.print_backtrace(ex, 10)
+          end
+        end
+      end
+      return [state1_copy, state2_copy]
+    end
+
+    def self.get_statechange_config_from_state(state)
+      cfg = {}
+      if state["files"]
+        cfg["files"] = {"paths" => []}
+        state["files"].each do |path,info|
+          cfg["files"]["paths"] << path
+        end
+      end
+      return cfg
+    end
+
+    def self.read_ignore_properties()
+      require "toaster/model/ignore_property"
+      result = Set.new
+      Dir["#{@@ohai_dir}/*"].each do |dir|
+        file = File.join(dir, "_meta.rb")
+        if File.exist?(file)
+          require file
+          tmp_result = nil
+          name = dir.sub(/.*\/([a-z0-9A-Z_\-]+)\/*/, '\1')
+          begin
+            eval("tmp_result = ignore_properties__#{name}()")
+            tmp_result = [tmp_result] if !tmp_result.kind_of?(Array)
+            tmp_result.each do |r|
+              result << IgnoreProperty.new(:key => r.to_s)
+            end
+          rescue => ex
+            puts "WARN: Unable to get ignore properties using code in file #{file}:"
+            Util.print_backtrace(ex, 10)
+          end
+        end
+      end
+      return result.to_a()
+    end
+
+    def self.remove_ignore_props!(props_hash, ignore_prop_names=nil, key_path=[], print_info=false)
+      if !ignore_prop_names
+        ignore_prop_names = read_ignore_properties()
+      end
+      #puts "TRACE: ignore_prop_names #{ignore_prop_names}"
+      ignore_prop_names.each do |key|
+        if props_hash.kind_of?(Array)
+          props_hash.dup.each do |k|
+
+            # check if we have an array of StateChange
+            if k.kind_of?(StateChange)
+              if k.property.eql?(key) ||
+              Util.starts_with?(k.property, "#{key}.") ||
+              k.property.match(key)
+                props_hash.delete(k)
+              end
+
+            else
+              # this is not a StateChange, but an
+              # array of values or hashes --> to be implemented
+              puts "WARN: SystemState.remove_ignore_props(..) not implemented for non-StateChange arrays!"
+            end
+          end
+
+        else
+          # assume this is an actual state properties hash
+          props_hash.keys.dup.each do |k|
+            new_path = key_path.dup
+            new_path << k
+            long_key = "'#{new_path.join("'.'")}'"
+            #puts "TRACE: long key #{long_key}"
+            if k == "#{key}" || Util.starts_with?(k, "#{key}.") || k.match("#{key}") ||
+            long_key == "#{key}" || Util.starts_with?(long_key, "#{key}.") || long_key.match("#{key}")
+              deleted = props_hash.delete(k)
+            elsif props_hash[k].kind_of?(Hash)
+              # --> recursion!
+              remove_ignore_props!(props_hash[k], ignore_prop_names, new_path)
+            end
+          end
+        end
+      end
+    end
+
+    def self.get_flat_attributes(current=nil, name_so_far="", list_so_far={})
+      if current.nil?
+        name_so_far = name_so_far[1..-1] if name_so_far[0] == "."
+        list_so_far[name_so_far] = nil
+        return list_so_far
+      end
+      if !current.kind_of?(Hash)
+        name_so_far = name_so_far[1..-1] if name_so_far[0] == "."
+        list_so_far[name_so_far] = current
+        return list_so_far
+      end
+      current.each do |name,value|
+        name = "#{name_so_far}.'#{name}'"
+        get_flat_attributes(value, name, list_so_far)
+      end
+      return list_so_far
+    end
+
+    private
+
+    def self.filter_unimportant_properties(json)
+      if @@include_default_state_props
+        remove_properties(json, ["network", "interfaces"])
+        remove_properties(json, ["counters", "network", "interfaces"])
+        remove_properties(json, ["etc", "passwd"])
+        remove_properties(json, ["etc", "group"])
+        remove_properties(json, ["cpu", "json--map--entry", "value", "flags"])
+        remove_properties(json, ["filesystem"])
+        remove_properties(json, ["uptime_seconds"])
+        remove_properties(json, ["uptime"])
+        remove_properties(json, ["idletime_seconds"])
+        remove_properties(json, ["ohai_time"])
+        remove_properties(json, ["idletime"])
+        remove_properties(json, ["kernel", "version"])
+        remove_properties(json, ["os_version"])
+        remove_properties(json, ["os"])
+        #remove_properties(json, ["memory", "anon-pages"])
+        #remove_properties(json, ["memory", "dirty"])
+        remove_properties(json, ["dmi"]) # for now, ignore the whole dmi section
+        remove_properties(json, ["memory"]) # for now, ignore the whole memory section
+        remove_properties(json, ["cpu"]) # for now, ignore the whole cpu section
+        remove_properties(json, ["block_device"]) # for now, ignore the whole block_device section
+        #remove_properties(json, ["block_device",/ram.*/])
+        remove_properties(json, ["keys"])
+        remove_properties(json, ["chef_packages"])
+        remove_properties(json, ["kernel","modules",/.*/,"size"])
+        remove_properties(json, ["kernel","modules",/.*/,"refcount"])
+      end
+
+      remove_properties(json, ["languages"])
+
+      remove_properties(json, ["nginx"]) # this is added automatically by the nginx Chef recipe
+
+      if json["kernel"] && json["kernel"]["modules"]
+        mod_names = ""
+        json["kernel"]["modules"].each do |key,value|
+          mod_names += key + " "
+        end
+        json["kernel"]["modules"] = mod_names.strip
+      end
+    end
+
+    def self.remove_properties(json, props)
+      MarkupUtil.remove_properties(json, props)
+    end
+
+  end
+end