puppet-parse 0.0.1 → 0.0.2

data/lib/vendor/puppet/rails/benchmark.rb

@@ -0,0 +1,63 @@
+require 'benchmark'
+require 'yaml'
+
+module Puppet::Rails::Benchmark
+  $benchmarks = {:accumulated => {}}
+
+  def time_debug?
+    Puppet::Rails::TIME_DEBUG
+  end
+
+  def railsmark(message)
+    result = nil
+    seconds = Benchmark.realtime { result = yield }
+    Puppet.debug(message + " in %0.2f seconds" % seconds)
+
+    $benchmarks[message] = seconds if time_debug?
+    result
+  end
+
+  def debug_benchmark(message)
+    return yield unless Puppet::Rails::TIME_DEBUG
+
+    railsmark(message) { yield }
+  end
+
+  # Collect partial benchmarks to be logged when they're
+  # all done.
+  #   These are always low-level debugging so we only
+  # print them if time_debug is enabled.
+  def accumulate_benchmark(message, label)
+    return yield unless time_debug?
+
+    $benchmarks[:accumulated][message] ||= Hash.new(0)
+    $benchmarks[:accumulated][message][label] += Benchmark.realtime { yield }
+  end
+
+  # Log the accumulated marks.
+  def log_accumulated_marks(message)
+    return unless time_debug?
+
+    return if $benchmarks[:accumulated].empty? or $benchmarks[:accumulated][message].nil? or $benchmarks[:accumulated][message].empty?
+
+    $benchmarks[:accumulated][message].each do |label, value|
+      Puppet.debug(message + ("(#{label})") + (" in %0.2f seconds" % value))
+    end
+  end
+
+  def write_benchmarks
+    return unless time_debug?
+
+    branch = %x{git branch}.split("\n").find { |l| l =~ /^\*/ }.sub("* ", '')
+
+    file = "/tmp/time_debugging.yaml"
+
+    if FileTest.exist?(file)
+      data = YAML.load_file(file)
+    else
+      data = {}
+    end
+    data[branch] = $benchmarks
+    Puppet::Util.replace_file(file, 0644) { |f| f.print YAML.dump(data) }
+  end
+end

data/lib/vendor/puppet/rails/database/001_add_created_at_to_all_tables.rb

@@ -0,0 +1,13 @@
+class AddCreatedAtToAllTables < ActiveRecord::Migration
+  def self.up
+    ActiveRecord::Base.connection.tables.each do |t|
+      add_column t.to_s, :created_at, :datetime unless ActiveRecord::Base.connection.columns(t).collect {|c| c.name}.include?("created_at")
+    end
+  end
+
+  def self.down
+    ActiveRecord::Base.connection.tables.each do |t|
+      remove_column t.to_s, :created_at unless ActiveRecord::Base.connection.columns(t).collect {|c| c.name}.include?("created_at")
+    end
+  end
+end

data/lib/vendor/puppet/rails/database/002_remove_duplicated_index_on_all_tables.rb

@@ -0,0 +1,13 @@
+class RemoveDuplicatedIndexOnAllTables < ActiveRecord::Migration
+  def self.up
+    ActiveRecord::Base.connection.tables.each do |t|
+      remove_index t.to_s, :id if ActiveRecord::Base.connection.indexes(t).collect {|c| c.columns}.include?("id")
+    end
+  end
+
+  def self.down
+    ActiveRecord::Base.connection.tables.each do |t|
+      add_index t.to_s, :id, :integer => true unless ActiveRecord::Base.connection.indexes(t).collect {|c| c.columns}.include?("id")
+    end
+  end
+end

data/lib/vendor/puppet/rails/database/003_add_environment_to_host.rb

@@ -0,0 +1,9 @@
+class AddEnvironmentToHost < ActiveRecord::Migration
+  def self.up
+    add_column :hosts, :environment, :string unless ActiveRecord::Base.connection.columns(:hosts).collect {|c| c.name}.include?("environment")
+  end
+
+  def self.down
+    remove_column :hosts, :environment if ActiveRecord::Base.connection.columns(:hosts).collect {|c| c.name}.include?("environment")
+  end
+end

data/lib/vendor/puppet/rails/database/004_add_inventory_service_tables.rb

@@ -0,0 +1,36 @@
+class AddInventoryServiceTables < ActiveRecord::Migration
+  def self.up
+    unless ActiveRecord::Base.connection.tables.include?("inventory_nodes")
+      create_table :inventory_nodes do |t|
+        t.column :name, :string, :null => false
+        t.column :timestamp, :datetime, :null => false
+        t.column :updated_at, :datetime
+        t.column :created_at, :datetime
+      end
+
+      add_index :inventory_nodes, :name, :unique => true
+    end
+
+    unless ActiveRecord::Base.connection.tables.include?("inventory_facts")
+      create_table :inventory_facts, :id => false do |t|
+        t.column :node_id, :integer, :null => false
+        t.column :name, :string, :null => false
+        t.column :value, :text, :null => false
+      end
+
+      add_index :inventory_facts, [:node_id, :name], :unique => true
+    end
+  end
+
+  def self.down
+    unless ActiveRecord::Base.connection.tables.include?("inventory_nodes")
+      remove_index :inventory_nodes, :name
+      drop_table :inventory_nodes
+    end
+
+    if ActiveRecord::Base.connection.tables.include?("inventory_facts")
+      remove_index :inventory_facts, [:node_id, :name]
+      drop_table :inventory_facts
+    end
+  end
+end

data/lib/vendor/puppet/rails/database/schema.rb

@@ -0,0 +1,131 @@
+class Puppet::Rails::Schema
+  def self.init
+    oldout = nil
+    Puppet::Util.benchmark(Puppet, :notice, "Initialized database") do
+      # We want to rewrite stdout, so we don't get migration messages.
+      oldout = $stdout
+      $stdout = File.open("/dev/null", "w")
+      ActiveRecord::Schema.define do
+        create_table :resources do |t|
+          t.column :title, :text, :null => false
+          t.column :restype,  :string, :null => false
+          t.column :host_id, :integer
+          t.column :source_file_id, :integer
+          t.column :exported, :boolean
+          t.column :line, :integer
+          t.column :updated_at, :datetime
+          t.column :created_at, :datetime
+        end
+        add_index :resources, :host_id, :integer => true
+        add_index :resources, :source_file_id, :integer => true
+
+        # Thanks, mysql!  MySQL requires a length on indexes in text fields.
+        # So, we provide them for mysql and handle everything else specially.
+        # Oracle doesn't index on CLOB fields, so we skip it
+        if ['mysql','mysql2'].include? Puppet[:dbadapter]
+          execute "CREATE INDEX typentitle ON resources (restype,title(50));"
+        elsif Puppet[:dbadapter] != "oracle_enhanced"
+          add_index :resources, [:title, :restype]
+        end
+
+        create_table :source_files do |t|
+          t.column :filename, :string
+          t.column :path, :string
+          t.column :updated_at, :datetime
+          t.column :created_at, :datetime
+        end
+        add_index :source_files, :filename
+
+        create_table :resource_tags do |t|
+          t.column :resource_id, :integer
+          t.column :puppet_tag_id, :integer
+          t.column :updated_at, :datetime
+          t.column :created_at, :datetime
+        end
+        add_index :resource_tags, :resource_id, :integer => true
+        add_index :resource_tags, :puppet_tag_id, :integer => true
+
+        create_table :puppet_tags do |t|
+          t.column :name, :string
+          t.column :updated_at, :datetime
+          t.column :created_at, :datetime
+        end
+
+        # Oracle automatically creates a primary key index
+        add_index :puppet_tags, :id, :integer => true if Puppet[:dbadapter] != "oracle_enhanced"
+
+        create_table :hosts do |t|
+          t.column :name, :string, :null => false
+          t.column :ip, :string
+          t.column :environment, :text
+          t.column :last_compile, :datetime
+          t.column :last_freshcheck, :datetime
+          t.column :last_report, :datetime
+          #Use updated_at to automatically add timestamp on save.
+          t.column :updated_at, :datetime
+          t.column :source_file_id, :integer
+          t.column :created_at, :datetime
+        end
+        add_index :hosts, :source_file_id, :integer => true
+        add_index :hosts, :name
+
+        create_table :fact_names do |t|
+          t.column :name, :string, :null => false
+          t.column :updated_at, :datetime
+          t.column :created_at, :datetime
+        end
+        add_index :fact_names, :name
+
+        create_table :fact_values do |t|
+          t.column :value, :text, :null => false
+          t.column :fact_name_id, :integer, :null => false
+          t.column :host_id, :integer, :null => false
+          t.column :updated_at, :datetime
+          t.column :created_at, :datetime
+        end
+        add_index :fact_values, :fact_name_id, :integer => true
+        add_index :fact_values, :host_id, :integer => true
+
+        create_table :param_values do |t|
+          t.column :value,  :text, :null => false
+          t.column :param_name_id, :integer, :null => false
+          t.column :line, :integer
+          t.column :resource_id, :integer
+          t.column :updated_at, :datetime
+          t.column :created_at, :datetime
+        end
+        add_index :param_values, :param_name_id, :integer => true
+        add_index :param_values, :resource_id, :integer => true
+
+        create_table :param_names do |t|
+          t.column :name, :string, :null => false
+          t.column :updated_at, :datetime
+          t.column :created_at, :datetime
+        end
+        add_index :param_names, :name
+
+        create_table :inventory_nodes do |t|
+          t.column :name, :string, :null => false
+          t.column :timestamp, :datetime, :null => false
+          t.column :updated_at, :datetime
+          t.column :created_at, :datetime
+        end
+
+        add_index :inventory_nodes, :name, :unique => true
+
+        create_table :inventory_facts, :id => false do |t|
+          t.column :node_id, :integer, :null => false
+          t.column :name, :string, :null => false
+          t.column :value, :text, :null => false
+        end
+
+        add_index :inventory_facts, [:node_id, :name], :unique => true
+      end
+    end
+  ensure
+    $stdout.close
+    $stdout = oldout if oldout
+    oldout = nil
+  end
+end
+

data/lib/vendor/puppet/rails/fact_name.rb

@@ -0,0 +1,6 @@
+require 'puppet/rails'
+require 'puppet/rails/fact_value'
+
+class Puppet::Rails::FactName < ActiveRecord::Base
+  has_many :fact_values, :dependent => :destroy
+end

data/lib/vendor/puppet/rails/fact_value.rb

@@ -0,0 +1,8 @@
+class Puppet::Rails::FactValue < ActiveRecord::Base
+  belongs_to :fact_name
+  belongs_to :host
+
+  def to_label
+    "#{self.fact_name.name}"
+  end
+end

data/lib/vendor/puppet/rails/host.rb

@@ -0,0 +1,258 @@
+require 'puppet/node/environment'
+require 'puppet/rails'
+require 'puppet/rails/resource'
+require 'puppet/rails/fact_name'
+require 'puppet/rails/source_file'
+require 'puppet/rails/benchmark'
+require 'puppet/util/rails/collection_merger'
+
+class Puppet::Rails::Host < ActiveRecord::Base
+  include Puppet::Rails::Benchmark
+  extend Puppet::Rails::Benchmark
+  include Puppet::Util
+  include Puppet::Util::CollectionMerger
+
+  has_many :fact_values, :dependent => :destroy, :class_name => "Puppet::Rails::FactValue"
+  has_many :fact_names, :through => :fact_values, :class_name => "Puppet::Rails::FactName"
+  belongs_to :source_file
+  has_many :resources, :dependent => :destroy, :class_name => "Puppet::Rails::Resource"
+
+  def self.from_puppet(node)
+    host = find_by_name(node.name) || new(:name => node.name)
+
+    {"ipaddress" => "ip", "environment" => "environment"}.each do |myparam, itsparam|
+      if value = node.send(myparam)
+        host.send(itsparam + "=", value)
+      end
+    end
+
+    host
+  end
+
+  # Override the setter for environment to force it to be a string, lest it
+  # be YAML encoded.  See #4487.
+  def environment=(value)
+    super value.to_s
+  end
+
+  # returns a hash of fact_names.name => [ fact_values ] for this host.
+  # Note that 'fact_values' is actually a list of the value instances, not
+  # just actual values.
+  def get_facts_hash
+    fact_values = self.fact_values.find(:all, :include => :fact_name)
+    return fact_values.inject({}) do | hash, value |
+      hash[value.fact_name.name] ||= []
+      hash[value.fact_name.name] << value
+      hash
+    end
+  end
+
+
+  # This is *very* similar to the merge_parameters method
+  # of Puppet::Rails::Resource.
+  def merge_facts(facts)
+    db_facts = {}
+
+    deletions = []
+    self.fact_values.find(:all, :include => :fact_name).each do |value|
+      deletions << value['id'] and next unless facts.include?(value['name'])
+      # Now store them for later testing.
+      db_facts[value['name']] ||= []
+      db_facts[value['name']] << value
+    end
+
+    # Now get rid of any parameters whose value list is different.
+    # This might be extra work in cases where an array has added or lost
+    # a single value, but in the most common case (a single value has changed)
+    # this makes sense.
+    db_facts.each do |name, value_hashes|
+      values = value_hashes.collect { |v| v['value'] }
+
+      unless values == facts[name]
+        value_hashes.each { |v| deletions << v['id'] }
+      end
+    end
+
+    # Perform our deletions.
+    Puppet::Rails::FactValue.delete(deletions) unless deletions.empty?
+
+    # Lastly, add any new parameters.
+    facts.each do |name, value|
+      next if db_facts.include?(name)
+      values = value.is_a?(Array) ? value : [value]
+
+      values.each do |v|
+        fact_values.build(:value => v, :fact_name => Puppet::Rails::FactName.find_or_create_by_name(name))
+      end
+    end
+  end
+
+  # Set our resources.
+  def merge_resources(list)
+    # keep only exported resources in thin_storeconfig mode
+    list = list.select { |r| r.exported? } if Puppet.settings[:thin_storeconfigs]
+
+    resources_by_id = nil
+    debug_benchmark("Searched for resources") {
+      resources_by_id = find_resources
+    }
+
+    debug_benchmark("Searched for resource params and tags") {
+      find_resources_parameters_tags(resources_by_id)
+    } if id
+
+    debug_benchmark("Performed resource comparison") {
+      compare_to_catalog(resources_by_id, list)
+    }
+  end
+
+  def find_resources
+    condition = { :exported => true } if Puppet.settings[:thin_storeconfigs]
+
+    resources.find(:all, :include => :source_file, :conditions => condition || {}).inject({}) do | hash, resource |
+      hash[resource.id] = resource
+      hash
+    end
+  end
+
+  def find_resources_parameters_tags(resources)
+    find_resources_parameters(resources)
+    find_resources_tags(resources)
+  end
+
+  def compare_to_catalog(existing, list)
+    compiled = list.inject({}) do |hash, resource|
+      hash[resource.ref] = resource
+      hash
+    end
+
+    resources = nil
+    debug_benchmark("Resource removal") {
+      resources = remove_unneeded_resources(compiled, existing)
+    }
+
+    # Now for all resources in the catalog but not in the db, we're pretty easy.
+    additions = nil
+    debug_benchmark("Resource merger") {
+      additions = perform_resource_merger(compiled, resources)
+    }
+
+    debug_benchmark("Resource addition") {
+      additions.each do |resource|
+        build_rails_resource_from_parser_resource(resource)
+      end
+
+      log_accumulated_marks "Added resources"
+    }
+  end
+
+  def add_new_resources(additions)
+    additions.each do |resource|
+      Puppet::Rails::Resource.from_parser_resource(self, resource)
+    end
+  end
+
+  # Turn a parser resource into a Rails resource.
+  def build_rails_resource_from_parser_resource(resource)
+    db_resource = nil
+    accumulate_benchmark("Added resources", :initialization) {
+      args = Puppet::Rails::Resource.rails_resource_initial_args(resource)
+
+      db_resource = self.resources.build(args)
+
+      # Our file= method does the name to id conversion.
+      db_resource.file = resource.file
+    }
+
+
+    accumulate_benchmark("Added resources", :parameters) {
+      resource.each do |param, value|
+        Puppet::Rails::ParamValue.from_parser_param(param, value).each do |value_hash|
+          db_resource.param_values.build(value_hash)
+        end
+      end
+    }
+
+    accumulate_benchmark("Added resources", :tags) {
+      resource.tags.each { |tag| db_resource.add_resource_tag(tag) }
+    }
+
+    db_resource.save
+
+    db_resource
+  end
+
+
+  def perform_resource_merger(compiled, resources)
+    return compiled.values if resources.empty?
+
+    # Now for all resources in the catalog but not in the db, we're pretty easy.
+    additions = []
+    compiled.each do |ref, resource|
+      if db_resource = resources[ref]
+        db_resource.merge_parser_resource(resource)
+      else
+        additions << resource
+      end
+    end
+    log_accumulated_marks "Resource merger"
+
+    additions
+  end
+
+  def remove_unneeded_resources(compiled, existing)
+    deletions = []
+    resources = {}
+    existing.each do |id, resource|
+      # it seems that it can happen (see bug #2010) some resources are duplicated in the
+      # database (ie logically corrupted database), in which case we remove the extraneous
+      # entries.
+      if resources.include?(resource.ref)
+        deletions << id
+        next
+      end
+
+      # If the resource is in the db but not in the catalog, mark it
+      # for removal.
+      unless compiled.include?(resource.ref)
+        deletions << id
+        next
+      end
+
+      resources[resource.ref] = resource
+    end
+    # We need to use 'destroy' here, not 'delete', so that all
+    # dependent objects get removed, too.
+    Puppet::Rails::Resource.destroy(deletions) unless deletions.empty?
+
+    resources
+  end
+
+  def find_resources_parameters(resources)
+    params = Puppet::Rails::ParamValue.find_all_params_from_host(self)
+
+    # assign each loaded parameters/tags to the resource it belongs to
+    params.each do |param|
+      resources[param['resource_id']].add_param_to_list(param) if resources.include?(param['resource_id'])
+    end
+  end
+
+  def find_resources_tags(resources)
+    tags = Puppet::Rails::ResourceTag.find_all_tags_from_host(self)
+
+    tags.each do |tag|
+      resources[tag['resource_id']].add_tag_to_list(tag) if resources.include?(tag['resource_id'])
+    end
+  end
+
+  def to_puppet
+    node = Puppet::Node.new(self.name)
+    {"ip" => "ipaddress", "environment" => "environment"}.each do |myparam, itsparam|
+      if value = send(myparam)
+        node.send(itsparam + "=", value)
+      end
+    end
+
+    node
+  end
+end

data/lib/vendor/puppet/rails/inventory_fact.rb

@@ -0,0 +1,5 @@
+require 'puppet/rails/inventory_node'
+
+class Puppet::Rails::InventoryFact < ::ActiveRecord::Base
+  belongs_to :node, :class_name => "Puppet::Rails::InventoryNode"
+end

data/lib/vendor/puppet/rails/inventory_node.rb

@@ -0,0 +1,43 @@
+require 'puppet/rails/inventory_fact'
+
+class Puppet::Rails::InventoryNode < ::ActiveRecord::Base
+  has_many :facts, :class_name => "Puppet::Rails::InventoryFact", :foreign_key => :node_id, :dependent => :delete_all
+
+  if Puppet::Util.activerecord_version < 3.0
+    # For backward compatibility, use the old named_scope with pre 3.0 activerecord
+    named_scope :has_fact_with_value, lambda { |name,value|
+      {
+        :conditions => ["inventory_facts.name = ? AND inventory_facts.value = ?", name, value.to_s],
+        :joins => :facts
+      }
+    }
+
+    named_scope :has_fact_without_value, lambda { |name,value|
+      {
+        :conditions => ["inventory_facts.name = ? AND inventory_facts.value != ?", name, value.to_s],
+        :joins => :facts
+      }
+    }
+  else
+    # Use scope for activerecord >= 3.0
+    scope :has_fact_with_value, lambda { |name,value|
+      {
+        :conditions => ["inventory_facts.name = ? AND inventory_facts.value = ?", name, value.to_s],
+        :joins => :facts
+      }
+    }
+
+    scope :has_fact_without_value, lambda { |name,value|
+      {
+        :conditions => ["inventory_facts.name = ? AND inventory_facts.value != ?", name, value.to_s],
+        :joins => :facts
+      }
+    }
+  end
+
+  def facts_to_hash
+    facts.inject({}) do |fact_hash,fact|
+      fact_hash.merge(fact.name => fact.value)
+    end
+  end
+end

data/lib/vendor/puppet/rails/param_name.rb

@@ -0,0 +1,25 @@
+require 'puppet/util/rails/collection_merger'
+require 'puppet/rails/param_value'
+require 'puppet/util/rails/cache_accumulator'
+
+class Puppet::Rails::ParamName < ActiveRecord::Base
+  include Puppet::Util::CollectionMerger
+  has_many :param_values, :dependent => :destroy
+
+  include Puppet::Util::CacheAccumulator
+  accumulates :name
+
+  def to_resourceparam(resource, source)
+    hash = {}
+    hash[:name] = self.name.to_sym
+    hash[:source] = source
+    hash[:value] = resource.param_values.find(:all, :conditions => [ "param_name_id = ?", self.id]).collect { |v| v.value }
+    if hash[:value].length == 1
+      hash[:value] = hash[:value].shift
+    elsif hash[:value].empty?
+      hash[:value] = nil
+    end
+    Puppet::Parser::Resource::Param.new hash
+  end
+end
+