ki-repo 0.1.0 → 0.1.1

data/lib/data_access/repository_info.rb

@@ -0,0 +1,153 @@
+# encoding: UTF-8
+
+# Copyright 2012 Mikko Apo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Ki
+  # Combines version's information from all different repositories
+  # @see Component
+  # @see Repository::Version
+  # @see Repository::Component
+  # @see VersionIterator
+  # @see RepositoryFinder
+  # @see FileFinder
+  # @see VersionMetadataFile
+  # @see VersionStatusFile
+  class Version
+    attr_chain :component, :require
+    attr_chain :name, :require
+    attr_chain :version_id, :require
+    attr_chain :metadata, -> { find_metadata }
+    attr_chain :binaries, -> { find_binaries }
+    attr_chain :finder, -> { component.finder }
+    attr_chain :versions, :require
+    attr_chain :statuses, -> { collect_statuses }
+
+    # finds first Repository::Version directory for this version that contains binaries
+    def find_binaries
+      finder.all_repositories.each do |package_root|
+        binary_dir = package_root.go(version_id)
+        if binary_dir.exists?
+          return binary_dir
+        end
+      end
+      nil
+    end
+
+    # finds first Repository::Version directory that contains metadata
+    def find_metadata
+      versions.each do |v|
+        m = v.metadata
+        if m.exists?
+          return m
+        end
+      end
+      nil
+    end
+
+    # collects all statuses related to this version
+    def collect_statuses
+      ret = []
+      versions.each do |v|
+        s = v.statuses
+        if s.exists?
+          v.statuses.each do |status|
+            ret << [status["key"], status["value"]]
+          end
+        end
+      end
+      ret
+    end
+
+    # finds all versions referenced by this version
+    def version_iterator
+      VersionIterator.new.version(self)
+    end
+
+    # finds files from this version (recursive)
+    def find_files(*file_patterns)
+      FileFinder.new.version(self).files(file_patterns)
+    end
+
+    def exists?
+      metadata || binaries
+    end
+
+    # Initializes a Version and Repository::Version for files non-imported files
+    # * works for testing and showing
+    def self.create_version(file, binary_directory=nil)
+      dir = File.dirname(file)
+      if dir == "."
+        dir = Dir.pwd
+      end
+      version = Version.new
+      repo_ver = Repository::Version.new(dir)
+      repo_ver.metadata = VersionMetadataFile.new(File.basename(file)).parent(repo_ver)
+      version.versions=[repo_ver]
+      if binary_directory
+        version.binaries = DirectoryBase.new(binary_directory)
+      end
+      version
+    end
+  end
+
+  # Combine's component's information from all different repositories
+  # @see Repository::Component
+  # @see RepositoryFinder
+  class Component
+    attr_chain :component_id, :require
+    # Package collector contains
+    attr_chain :finder, :require
+    attr_chain :versions, -> { find_versions }
+    attr_chain :status_info, -> { find_status_info }
+    attr_chain :components, :require
+
+    # Returns version list from first component which has a version list
+    def find_versions
+      components.each do |c|
+        version_list_file = c.versions
+        if version_list_file.exists?
+          return version_list_file
+        end
+      end
+      nil
+    end
+
+    # Returns Version which references all existing version directories
+    # @see Version
+    def version_by_id(version_str)
+      version_id = File.join(component_id, version_str)
+      finder.versions.cache(version_id) do
+        info_versions = components.map do |c|
+          Repository::Version.new(version_str).version_id(version_id).parent(c)
+        end
+        existing_versions = info_versions.select do |v|
+          v.exists?
+        end
+        Version.new.component(self).version_id(version_id).name(version_str).versions(existing_versions)
+      end
+    end
+
+    def find_status_info
+      ret = {}
+      components.each do |c|
+        si = c.status_info
+        if si.exists?
+          ret.merge!(si.cached_data)
+        end
+      end
+      ret
+    end
+  end
+end

data/lib/data_access/version_helpers.rb

@@ -0,0 +1,242 @@
+# encoding: UTF-8
+
+# Copyright 2012 Mikko Apo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Ki
+  # Tests that a version is intact. Version can be in repository or as file.
+  # Checks that all files have correct hashes. If recursive is set to true, goes through all dependencies
+  # @see test_version
+  class VersionTester
+    attr_chain :ki_home, :require
+    attr_chain :finder, -> { ki_home.finder }
+    attr_chain :recursive, -> { true }
+    attr_chain :print, -> { false }
+    attr_chain :results, -> { Hash.new }
+
+    # Tests that a version is intact
+    # * test_version(version) expects a Version parameter
+    # @see VersionIterator
+    # @see RepositoryFinder
+    # @return [bool] returns true if there weren't any problems with the version
+    def test_version(root_version, &block)
+      all_ok = true
+      possible_hashes = KiCommand::KiExtensions.find!("/hashing")
+      # iterates through all versions
+      root_version.version_iterator.iterate_versions do |v|
+        binaries = v.binaries
+        metadata = v.metadata
+        metadata.cached_data
+        metadata.files.each do |file_hash|
+          file_path = file_hash["path"]
+          full_path = binaries.path(file_path)
+          issue = nil
+          if !File.exists?(full_path)
+            issue="missing"
+          elsif File.size(full_path) != file_hash["size"]
+            issue="wrong size"
+          elsif !verify_hash(file_hash, full_path, possible_hashes)
+            issue="wrong hash"
+          end
+          if issue
+            all_ok = false
+            (results[issue]||=[]) << [v, file_path]
+            if block
+              block.call(issue, v, file_path)
+            end
+            if print
+              puts "#{v.metadata.path}: '#{file_path}' #{issue} '#{v.binaries.path(file_path)}'"
+            end
+          end
+        end
+        if !recursive
+          break
+        end
+      end
+      all_ok
+    end
+
+    def verify_hash(file_hash, full_path, possible_hashes)
+      file_hashes = possible_hashes.service_names.select { |name| file_hash.include?(name) }
+      checked_hashes = VersionMetadataFile.calculate_hashes(full_path, file_hashes)
+      checked_hashes.each_pair do |id, result|
+        if file_hash[id] != result
+          return false
+        end
+      end
+      true
+    end
+  end
+
+  # Imports a version to KiHome
+  class VersionImporter
+    attr_chain :ki_home, :require
+    attr_chain :finder, -> { ki_home.finder}
+    attr_chain :tester, -> { VersionTester.new.recursive(false).print(true) }
+    attr_chain :move_files
+    attr_chain :create_new_version
+    attr_chain :specific_version_id
+
+    # Imports a version to KiHome
+    # * import(file, binary_directory) expects two String parameters defining version file location and directory base for binaries
+    def import(*args)
+      if args.size == 2
+        file, input = args
+        source = DirectoryBase.new(input)
+        metadata = VersionMetadataFile.new(file)
+      else
+        raise "Not supported: '#{args.inspect}'"
+      end
+      test_version(file, input)
+
+      import_from_metadata(metadata, source)
+    end
+
+    def import_from_metadata(metadata, source=nil)
+      if defined?(@specific_version_id) && defined?(@create_new_version)
+        raise "Can't define both specific_version_id '#{specific_version_id}' and create_new_version '#{create_new_version}'!"
+      end
+
+      if defined?(@specific_version_id)
+        version_id = @specific_version_id
+      elsif defined? @create_new_version
+        component_id = @create_new_version
+        version = finder.version(component_id)
+        if version
+          id = version.version_id.split("/").last
+          version_number = (Integer(id) + 1).to_s
+        else
+          version_number = "1"
+        end
+        version_id = File.join(component_id, version_number)
+      else
+        version_id = metadata.version_id
+      end
+
+      version_arr = version_id.split("/")
+      version_number = version_arr.delete_at(-1)
+      component_id = version_arr.join("/")
+
+      version = finder.version(version_id)
+      if version && version.exists?
+        raise "'#{version_id}' exists in repository already!"
+      end
+
+      # creates directories
+      components_dir = ki_home.repositories.add_item("local").mkdir.components
+      binary_dest = metadata_dir = components_dir.add_item(component_id).mkdir.versions.add_version(version_number).mkdir
+
+      metadata_dir.metadata.cached_data = metadata.cached_data
+      metadata_dir.metadata.version_id = version_id
+      metadata_dir.metadata.save
+      if defined? @move_files
+        FileUtils.rm(metadata.path)
+      end
+      source_dirs = []
+      metadata_dir.metadata.files.each do |file_info|
+        file_path = file_info["path"]
+        dir = File.dirname(file_path)
+        if dir != "."
+          source_dirs << dir
+          binary_dest.mkdir(dir)
+        end
+        to_repo(source.path(file_path), binary_dest.path(file_path))
+      end
+      delete_empty_source_dirs(source, source_dirs)
+    end
+
+
+    def delete_empty_source_dirs(source, source_dirs)
+      if defined? @move_files
+        expanded_source_dirs = {}
+        source_dirs.each do |d|
+          dir_entries(d).each do |expanded|
+            expanded_source_dirs[expanded] = true
+          end
+        end
+        expanded_source_dirs.keys.each do |dir|
+          checked_dir = source.path(dir)
+          if Dir.entries(checked_dir) == [".", ".."]
+            FileUtils.rmdir(checked_dir)
+          end
+        end
+      end
+    end
+
+    # splits dir path in to all components: foo/bar/baz, foo/bar, foo
+    def dir_entries(str)
+      arr = str.split("/")
+      ret = []
+      c = arr.size
+      while (c > 0)
+        ret << File.join(arr[0..c])
+        c-=1
+      end
+      ret
+    end
+
+    def to_repo(src, dest)
+      if defined? @move_files
+        FileUtils.mv(src, dest)
+      else
+        FileUtils.cp(src, dest)
+      end
+    end
+
+    def test_version(file, input)
+      all_ok = tester.ki_home(ki_home).test_version(Version.create_version(file, input))
+      if !all_ok
+        raise "Files are not ok!"
+      end
+    end
+  end
+
+  # Exports a version to directory
+  # * if test_dependencies set to true, tests the version before exporting
+  class VersionExporter
+    attr_chain :ki_home, :require
+    attr_chain :finder, -> { ki_home.finder }
+    attr_chain :test_dependencies
+    attr_chain :find_files, -> { FileFinder.new }
+    attr_chain :copy
+
+    # Exports a version to directory
+    def export(version, out)
+      ver = finder.version(version)
+      if test_dependencies
+        test_version(ver)
+      end
+      files = find_files.version(ver).file_map.sort
+      files.each do |file_path, full_path|
+        dir = File.dirname(file_path)
+        if dir != "."
+          FileUtils.mkdir_p File.join(out, dir)
+        end
+        if defined? @copy
+          FileUtils.cp(full_path, File.join(out, file_path))
+        else
+          FileUtils.ln_sf(full_path, File.join(out, file_path))
+        end
+      end
+    end
+
+    def test_version(version)
+      tester = VersionTester.new.ki_home(ki_home).finder(finder).recursive(true).print(true)
+      all_ok = tester.test_version(version)
+      if !all_ok
+        raise "Files are not ok!"
+      end
+    end
+  end
+end

data/lib/data_access/version_iterators.rb

@@ -0,0 +1,145 @@
+# encoding: UTF-8
+
+# Copyright 2012 Mikko Apo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Ki
+  #
+  #
+  class VersionIterator
+    attr_chain :version, :require
+    attr_chain :dependency
+    attr_chain :package_path
+    attr_chain :dependency_path
+    attr_chain :finder, -> { version.component.finder }
+    attr_chain :block
+    attr_chain :internals
+    attr_chain :exclude_dependencies, -> { [] }, :convert => lambda { |list| Array.wrap(list).map { |s| /#{s}/ } }
+
+    def iterate_versions(&block)
+      start_iteration do |version_iterator|
+        block.call(version_iterator.version)
+        version_iterator.iterate_dependencies
+      end
+    end
+
+    def start_iteration(&block)
+      @block = block
+      @internals = true
+      block.call(self)
+    end
+
+    def iterate_dependencies
+      version.metadata.dependencies.map do |dep|
+        if internals || !dep["internal"]
+          dep_v = VersionIterator.new
+          dep_v.version = finder.version(dep["version_id"])
+          dep_v.dependency = dep
+          dep_v.exclude_dependencies.concat(exclude_dependencies).concat(select_dep_rm(dep))
+          dep_v.block = block
+          if dep["path"] || package_path
+            dep_v.package_path = File.join([package_path, dep["path"]].compact)
+          end
+          if dependency_path || dep["name"]
+            dep_v.dependency_path = File.join([dependency_path, dep["name"]].compact)
+          end
+          if ok_to_iterate_dependency(dep_v)
+            [dep, dep_v.version, block.call(dep_v)]
+          end
+        end
+      end.compact
+    end
+
+    def select_dep_rm(dep_v)
+      ops = dep_v["operations"]
+      if ops
+        ops.map do |op|
+          if op.first == "dep-rm"
+            op[1..-1]
+          end
+        end.compact.flatten
+      else
+        []
+      end
+    end
+
+    def ok_to_iterate_dependency(dep_v)
+      !(exclude_dependencies.size > 0 && (exclude_dependencies.any_matches?(dep_v.version.version_id) || dep_v.dependency_path && exclude_dependencies.any_matches?(dep_v.dependency_path)))
+    end
+  end
+
+  class FileFinder < VersionIterator
+    attr_chain :files, -> { [] }, :convert => lambda { |list| Array.wrap(list).map { |s| FileRegexp.matcher(s) } }
+    attr_chain :exclude_files, -> { [] }, :convert => lambda { |list| Array.wrap(list).map { |s| FileRegexp.matcher(s) } }
+    attr_chain :tags, -> { [] }, :convert => lambda { |list| Array.wrap(list)}
+    attr_chain :exclude_tags, -> { [] }, :convert => lambda { |list| Array.wrap(list)}
+
+    def file_map
+      start_iteration do |ver_iterator|
+        ret = {}
+        ver_iterator.iterate_dependencies.each do |dependency, version, file_map|
+          file_operations(file_map, dependency)
+          ret.merge!(file_map)
+        end
+        ver = ver_iterator.version
+        binaries = ver.binaries
+        metadata = ver.metadata
+        # TODO: file operations should be applied to the files before the files are filtered
+        metadata.files.each do |file|
+          path = file["path"]
+          file_path = File.join([ver_iterator.package_path, path].compact)
+          if ok_to_add_file(file, file_path)
+            ret[file_path]=binaries.path(path)
+          end
+        end
+        file_operations(ret, metadata.cached_data)
+        ret
+      end
+    end
+
+    def file_list
+      file_map.values
+    end
+
+    # Modifies
+    def file_operations(file_map, dependency)
+      operations = dependency["operations"]
+      if operations
+        VersionFileOperations.new.edit_file_map(file_map, operations)
+      end
+    end
+
+    # File is added to the list if
+    # - files pattern list is empty (select all files) or file path matches any files pattern
+    # - it does not match any file exclude patterns
+    # - tags selection list is empty or file has any tags from tags selection list
+    # - no tags match tags from tags exclusion list
+    def ok_to_add_file(file, file_path)
+      file_tags = file["tags"] || []
+      (files.size == 0 || files.any_matches?(file_path)) &&
+          !exclude_files.any_matches?(file_path) &&
+          (tags.size == 0 || (cross_any_matches?(file_tags, tags)) &&
+              !cross_any_matches?(file_tags, exclude_tags))
+    end
+
+    def cross_any_matches?(arr, dest_arr)
+      arr.each do |i|
+        if dest_arr.any_matches?(i)
+          return true
+        end
+      end
+      false
+    end
+  end
+end

data/lib/data_access/version_operations.rb

@@ -0,0 +1,80 @@
+# encoding: UTF-8
+
+# Copyright 2012 Mikko Apo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Ki
+  class VersionFileOperations
+    def resolve_dest_file(file, dest, matching_pattern)
+      matcher = matching_pattern.match(file)
+      dest = dest.gsub(/\$\d/) do |str|
+        matcher[Integer(str[1..-1])]
+      end
+      if dest.end_with?("/")
+        dest = File.join(dest, File.basename(file))
+      end
+      if dest.start_with?("/")
+        dest = dest[1..-1]
+      end
+      dest
+    end
+
+    def copy_or_move(file_map, args, op)
+      delete = op == "mv"
+      dest = args.delete_at(-1)
+      patterns = args.map { |pattern| FileRegexp.matcher(pattern) }
+      matching_files = []
+      file_map.keys.each do |file|
+        matching_pattern = patterns.any_matches?(file)
+        if matching_pattern
+          matching_files << file
+          dest_file = resolve_dest_file(file, dest, matching_pattern)
+          file_map[dest_file]=file_map[file]
+          if delete
+            file_map.delete(file)
+          end
+        end
+      end
+    end
+
+    def delete(file_map, args, op)
+      patterns = args.map { |pattern| FileRegexp.matcher(pattern) }
+      file_map.keys.each do |file|
+        if patterns.any_matches?(file)
+          file_map.delete(file)
+        end
+      end
+    end
+
+    def edit_file_map(file_map, operations)
+      operations.each do |op, *args|
+        case op
+          when "cp"
+            copy_or_move(file_map, args, op)
+          when "mv"
+            copy_or_move(file_map, args, op)
+          when "rm"
+            delete(file_map, args, op)
+        end
+      end
+    end
+
+  end
+
+  class FileRegexp
+    def FileRegexp.matcher(s)
+      /^#{s.gsub(/\./, '\.').gsub('*', ".*")}$/
+    end
+  end
+end