machinery-tool 1.0.2

data/lib/system_description.rb

@@ -0,0 +1,228 @@
+# Copyright (c) 2013-2014 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+
+class SystemDescription < Machinery::Object
+  CURRENT_FORMAT_VERSION = 2
+  EXTRACTABLE_SCOPES = [
+    "changed_managed_files",
+    "config_files",
+    "unmanaged_files"
+  ]
+
+  attr_accessor :name
+  attr_accessor :store
+  attr_accessor :format_version
+
+  class << self
+    def from_json(name, json, store = nil)
+      begin
+        json_hash = JSON.parse(json)
+      rescue JSON::ParserError => e
+        lines = e.message.split("\n")
+        error_pos = json.split("\n").length - lines.length + 2
+        block_end = lines.index { |l| l =~ / [\}\]],?$/ }
+
+        # remove needless json error information
+        lines[0].gsub!(/^\d+: (.*)$/, "\\1")
+        json_error = lines[0..block_end].join("\n")
+
+        if error_pos == 1
+          json_error = "An opening bracket, a comma or quotation is missing " \
+            "in one of the global scope definitions or in the meta section. " \
+            "Unlike issues with the elements of the scopes, our JSON parser " \
+            "isn't able to locate issues like these."
+          error_pos = nil
+        end
+
+        error = "The JSON data of the system description '#{name}' " \
+          "couldn't be parsed. The following error occured"
+        error += " around line #{error_pos}" if error_pos
+        error += " in file '#{store.manifest_path(name)}'" if store
+        error += ":\n\n#{json_error}"
+
+        raise Machinery::Errors::SystemDescriptionError.new(error)
+      end
+
+      if compatible_json?(json_hash)
+        SystemDescriptionValidator.new(self).validate_json(json_hash)
+      end
+
+      begin
+        description = self.new(name, self.create_attrs(json_hash), store)
+      rescue NameError
+        raise Machinery::Errors::SystemDescriptionIncompatible.new(name)
+      end
+
+      json_format_version = json_hash["meta"]["format_version"] if json_hash["meta"]
+      description.format_version = json_format_version
+
+      description
+    end
+
+    def create_attrs(hash)
+      entries = hash.map do |key, value|
+        next if key == "meta"
+
+        class_name = "#{key.split("_").map(&:capitalize).join}Scope"
+        value_converted = Object.const_get(class_name).from_json(value)
+
+        # Set metadata
+        if hash["meta"] && hash["meta"][key]
+          value_converted.meta = Machinery::Object.from_json(hash["meta"][key])
+        end
+
+        [key, value_converted]
+      end.compact
+
+      Hash[entries]
+    end
+
+    private
+
+    def compatible_json?(json)
+      json.is_a?(Hash) &&
+        json["meta"].is_a?(Hash) &&
+        json["meta"]["format_version"] == CURRENT_FORMAT_VERSION
+    end
+  end
+
+  def initialize(name, hash = {}, store = nil)
+    @name = name
+    @store = store
+    @format_version = CURRENT_FORMAT_VERSION
+
+    super(hash)
+  end
+
+  def compatible?
+    !format_version.nil? &&
+      format_version == SystemDescription::CURRENT_FORMAT_VERSION
+  end
+
+  def validate_compatibility
+    if !compatible?
+      raise Machinery::Errors::SystemDescriptionIncompatible.new(self.name)
+    end
+  end
+
+  def to_hash
+    meta = {}
+    meta["format_version"] = self.format_version if self.format_version
+
+    attributes.each do |key, value|
+      meta[key] = self[key].meta.as_json if self[key].meta
+    end
+
+    hash = as_json
+    hash["meta"] = meta unless meta.empty?
+    hash
+  end
+
+  def to_json
+    JSON.pretty_generate(to_hash)
+  end
+
+  def scopes
+    scopes = attributes.map {
+      |key, value| key.to_s
+    }.sort
+    scopes
+  end
+
+  def assert_scopes(*scopes)
+    missing = scopes.reject do |e|
+      self.send(e) && !self.send(e).empty?
+    end
+
+    unless missing.empty?
+      raise Machinery::Errors::SystemDescriptionError.new(
+        "The system description misses the following section(s): #{missing.join(", ")}."
+      )
+    end
+  end
+
+  def short_os_version
+    assert_scopes("os")
+
+    case self.os.name
+      when /^SUSE Linux Enterprise Server/
+        "sles" + self.os.version[/\d+( SP\d+)*/].gsub(" ", "").downcase
+      when /^openSUSE/
+        self.os.version[/^\d+.\d+/]
+      else
+        "unknown"
+    end
+  end
+
+  def scope_extracted?(scope)
+    self[scope] && self[scope].is_extractable? && self[scope].extracted
+  end
+
+  def os_object
+    assert_scopes("os")
+
+    begin
+      Os.for(self.os.name)
+    rescue Machinery::Errors::UnknownOs => e
+      raise Machinery::Errors::SystemDescriptionError.new(e)
+    end
+  end
+
+  def missing_files(scope, file_list)
+    file_list.map! { |file| File.join(file_store(scope), file) }
+
+    file_list.select do |file|
+      !File.exists?(file)
+    end
+  end
+
+  def additional_files(scope, file_list)
+    file_list.map! { |file| File.join(file_store(scope), file) }
+    files = list_file_store_content(scope)
+
+    files - file_list
+  end
+
+  def validate_file_data
+    SystemDescriptionValidator.new(self).validate_file_data
+  end
+
+  # Filestore handling
+  def initialize_file_store(store_name)
+    @store.initialize_file_store(self.name, store_name)
+  end
+
+  def file_store(store_name)
+    @store.file_store(self.name, store_name)
+  end
+
+  def remove_file_store(store_name)
+    @store.remove_file_store(self.name, store_name)
+  end
+
+  def rename_file_store(old_name, new_name)
+    @store.rename_file_store(self.name, old_name, new_name)
+  end
+
+  def list_file_store_content(store_name)
+    @store.list_file_store_content(self.name, store_name)
+  end
+
+  def create_file_store_sub_dir(store_name, sub_dir)
+    @store.create_file_store_sub_dir(self.name, store_name, sub_dir)
+  end
+end

data/lib/system_description_store.rb

@@ -0,0 +1,167 @@
+# Copyright (c) 2013-2014 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+
+class SystemDescriptionStore
+  attr_reader :base_path
+
+  def default_path
+    Machinery::DEFAULT_CONFIG_DIR
+  end
+
+  def initialize(base_path = default_path)
+    @base_path = base_path
+    create_dir(@base_path)
+  end
+
+  def description_path(name)
+    File.join(@base_path, name)
+  end
+
+  def manifest_path(name)
+    File.join(description_path(name), "manifest.json")
+  end
+
+  def html_path(name)
+    File.join(description_path(name), "index.html")
+  end
+
+  def load_json(name)
+    validate_name(name)
+    file_name = manifest_path(name)
+    unless File.exists?(file_name)
+      raise Machinery::Errors::SystemDescriptionNotFound.new(
+        "A system description with the name #{name} was not found."
+      )
+    end
+    File.read(file_name)
+  end
+
+  def load(name)
+    json = load_json(name)
+    description = SystemDescription.from_json(name, json, self)
+    description.validate_compatibility
+    description.validate_file_data
+    description
+  end
+
+  def save(description)
+    validate_name(description.name)
+    create_dir(description_path(description.name))
+    path = manifest_path(description.name)
+    created = !File.exists?(path)
+    File.write(path, description.to_json)
+    File.chmod(0600,path) if created
+  end
+
+  def list
+    Dir["#{@base_path}/*"].
+      select { |item| File.exists?(manifest_path(File.basename(item)))}.
+      map { |item| File.basename(item) }
+  end
+
+  def remove(name)
+    unless name.empty?
+      validate_name(name)
+      FileUtils.rm_rf(description_path(name))
+    else
+      raise "The system description has no name specified and thus can't be deleted."
+    end
+  end
+
+  def copy(from, to)
+    validate_name(from)
+    validate_name(to)
+    if !list.include?(from)
+      raise Machinery::Errors::SystemDescriptionNotFound.new(
+        "System description \"#{from}\" does not exist."
+      )
+    end
+
+    if list.include?(to)
+      raise Machinery::Errors::SystemDescriptionError.new(
+        "A System description with the name \"#{to}\" does already exist."
+      )
+    end
+
+    FileUtils.cp_r(description_path(from), description_path(to))
+  end
+
+  def initialize_file_store(description_name, store_name)
+    dir = File.join(description_path(description_name), store_name)
+    create_dir(dir, new_dir_mode(description_name))
+  end
+
+  def file_store(description_name, store_name)
+    dir = File.join(description_path(description_name), store_name)
+    Dir.exists?(dir) ? dir : nil
+  end
+
+  def remove_file_store(description_name, store_name)
+    FileUtils.rm_rf(File.join(description_path(description_name), store_name))
+  end
+
+  def rename_file_store(description_name, store_old, store_new)
+    FileUtils.mv(
+      File.join(description_path(description_name), store_old),
+      File.join(description_path(description_name), store_new)
+    )
+  end
+
+  def create_file_store_sub_dir(description_name, store_name, sub_dir)
+    dir = File.join(description_path(description_name), store_name, sub_dir)
+    create_dir(dir, new_dir_mode(description_name))
+  end
+
+  def list_file_store_content(description_name, store_name)
+    dir = File.join(description_path(description_name), store_name)
+
+    files = Dir.glob(File.join(dir, "**/{*,.*}"))
+    # filter parent directories because they should not be listed separately
+    files.reject { |f| files.index { |e| e =~ /^#{f}\/.+/ } }
+  end
+
+  def new_dir_mode(name)
+    mode = 0700
+    if Dir.exists?(description_path(name))
+      mode = File.stat(description_path(name)).mode & 0777
+    end
+    mode
+  end
+
+  private
+
+  def create_dir(dir, mode = 0700)
+    unless Dir.exists?(dir)
+      FileUtils.mkdir_p(dir, :mode => mode)
+    end
+  end
+
+  def validate_name(name)
+    if ! /^[\w\.:-]*$/.match(name)
+      raise Machinery::Errors::SystemDescriptionError.new(
+        "System description name \"#{name}\" is invalid. Only \"a-zA-Z0-9_:.-\" are valid characters."
+      )
+    end
+
+    if name.start_with?(".")
+      raise Machinery::Errors::SystemDescriptionError.new(
+        "System description name \"#{name}\" is invalid. A dot is not allowed as first character."
+      )
+    end
+
+  end
+end

data/lib/system_description_validator.rb

@@ -0,0 +1,216 @@
+# Copyright (c) 2013-2014 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 3 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.   See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, contact SUSE LLC.
+#
+# To contact SUSE about this file by physical or electronic mail,
+# you may find current contact information at www.suse.com
+
+class SystemDescriptionValidator
+  class << self
+    def load_global_schema
+      JSON.parse(File.read(File.join(
+        Machinery::ROOT,
+        "schema",
+        "v#{SystemDescription::CURRENT_FORMAT_VERSION}",
+        "system-description-global.schema.json"
+      )))
+    end
+
+    def load_scope_schemas
+      schema_dir = File.join(
+        Machinery::ROOT,
+        "plugins",
+        "schema",
+        "v#{SystemDescription::CURRENT_FORMAT_VERSION}",
+      )
+
+      Hash[
+        Dir["#{schema_dir}/*.schema.json"].map do |file|
+          scope = file.match(/system-description-(.*)\.schema\.json$/)[1].tr("-", "_")
+          schema = JSON.parse(File.read(file))
+
+          [scope, schema]
+        end
+      ]
+    end
+  end
+
+  GLOBAL_SCHEMA = load_global_schema
+  SCOPE_SCHEMAS = load_scope_schemas
+
+  def initialize(description)
+    @description = description
+  end
+
+  def validate_json(json)
+    errors = JSON::Validator.fully_validate(GLOBAL_SCHEMA, json)
+
+    scopes = json.keys
+    scopes.delete("meta")
+
+    errors += scopes.flat_map do |scope|
+      schema = SCOPE_SCHEMAS[scope]
+
+      if schema
+        issue = JSON::Validator.fully_validate(schema, json[scope]).map do |error|
+          "In scope #{scope}: #{error}"
+        end
+        issue.map do |error|
+          cleanup_json_error_message(error, scope)
+        end
+      else
+        []
+      end
+    end
+
+    if !errors.empty?
+      raise Machinery::Errors::SystemDescriptionError.new(errors.join("\n"))
+    end
+  end
+
+  def missing_files_for_plain_scope(scope)
+    expected_files = @description[scope].files.reject { |file| file.changes.include?("deleted") }
+    missing_files = @description.missing_files(scope, expected_files.map(&:name))
+  end
+
+  def expected_files_for_tared_scope(scope)
+    has_files_tarball = @description[scope].files.any? { |f| f.type == "file" || f.type == "link" }
+    tree_tarballs = @description[scope].files.
+      select { |f| f.type == "dir" }.
+      map { |d| File.join("trees", d.name.sub(/\/$/, "") + ".tgz") }
+
+    expected_files = []
+    expected_files << "files.tgz" if has_files_tarball
+    expected_files += tree_tarballs
+  end
+
+  def missing_files_for_tared_scope(scope)
+    expected_files = expected_files_for_tared_scope(scope)
+    missing_files = @description.missing_files(scope, expected_files)
+  end
+
+  def missing_files_for_scope(scope)
+    if scope == "unmanaged_files"
+      missing_files_for_tared_scope(scope)
+    else
+      missing_files_for_plain_scope(scope)
+    end
+  end
+
+  def additional_files_for_tared_scope(scope)
+    expected_files = expected_files_for_tared_scope(scope)
+    additional_files = @description.additional_files(scope, expected_files)
+  end
+
+  def additional_files_for_plain_scope(scope)
+    expected_files = @description[scope].files.reject { |file| file.changes.include?("deleted") }
+    additional_files = @description.additional_files(scope, expected_files.map(&:name))
+  end
+
+  def additional_files_for_scope(scope)
+    if scope == "unmanaged_files"
+      additional_files_for_tared_scope(scope)
+    else
+      additional_files_for_plain_scope(scope)
+    end
+  end
+
+  def format_file_errors(scope, missing_files, additional_files)
+    errors = missing_files.map do |file|
+      "  * File '" + file + "' doesn't exist"
+    end
+    errors += additional_files.map do |file|
+      "  * File '" + file + "' doesn't have meta data"
+    end
+    "Scope '#{scope}':\n" + errors.join("\n")
+  end
+
+  def validate_file_data
+    errors = []
+
+    ["config_files", "changed_managed_files", "unmanaged_files"].each do |scope|
+      if @description.scope_extracted?(scope)
+        missing_files = missing_files_for_scope(scope)
+        additional_files = additional_files_for_scope(scope)
+
+        if !missing_files.empty? or !additional_files.empty?
+          errors.push(format_file_errors(scope, missing_files, additional_files))
+        end
+      end
+    end
+
+    if !errors.empty?
+      e = Machinery::Errors::SystemDescriptionValidationFailed.new(errors)
+      e.header = "Error validating description '#{@description.name}'"
+      raise e
+    end
+  end
+
+  def cleanup_json_error_message(message, scope)
+    message = cleanup_json_path(message, scope)
+    message = remove_json_error_uuid(message)
+    message
+  end
+
+  private
+
+  def cleanup_json_path(message, scope)
+    old_path = message[/The property '#\/(.*?)'/,1]
+
+    position = error_position_from_json_path(old_path)
+    details = extract_details_from_json_path(old_path, scope)
+
+    new_path = "The property"
+    new_path += " ##{position}" if position > -1
+    new_path += " (#{details})" if !details.empty?
+
+    message.gsub(/The property '#\/.*?'/, new_path)
+  end
+
+  def error_position_from_json_path(path)
+    elements = path.split("/")
+
+    position = -1
+    elements.each do |e|
+      if Machinery::is_int?(e)
+        number = e.to_i
+        position = number if number > position
+      end
+    end
+    position
+  end
+
+  def extract_details_from_json_path(path, scope)
+    elements = path.split("/")
+
+    elements.uniq!
+
+    # filter numbers since the position is calculated elswhere
+    elements.reject! { |e| Machinery::is_int?(e) }
+
+    # The json schema path often contains the word "type" in many messages
+    # but this information adds no value for the user since it is not related
+    # to our manifest.json
+    # So we filter it for all scopes except of repositories since this is the
+    # only scope which does have an attribute called "type"
+    if scope != "repositories"
+      elements.reject! { |e| e == "type" }
+    end
+
+    elements.join("/")
+  end
+
+  def remove_json_error_uuid(message)
+    message.gsub(/ in schema .*$/, ".")
+  end
+end