ki-repo 0.1.0 → 0.1.1

data/lib/data_storage/dir_base.rb

@@ -0,0 +1,106 @@
+# encoding: UTF-8
+
+# Copyright 2012 Mikko Apo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Ki
+  module DirectoryBaseModule
+    attr_chain :parent, :require
+
+    def init_from_path(path)
+      @path = path
+    end
+
+    def name
+      File.basename(@path)
+    end
+
+    def go(*path)
+      if path.empty?
+        self
+      else
+        path = File.join(path).split(File::Separator)
+        child = child(path.delete_at(0)).parent(self)
+        if path.empty?
+          child
+        else
+          child.go(*path)
+        end
+      end
+    end
+
+    def exists?(*sub_path)
+      File.exists?(go(*sub_path).path)
+    end
+
+    def mkdir(*path)
+      dest = go(*path)
+      if !dest.exists?
+        FileUtils.mkdir_p(dest.path)
+      end
+      dest
+    end
+
+    def path(*sub_paths)
+      new_path = [@path, sub_paths].flatten
+      if defined? @parent
+        new_path.unshift(@parent.path)
+      end
+      File.join(new_path)
+    end
+
+    def root
+      if defined? @parent
+        @parent.root
+      else
+        self
+      end
+    end
+
+    def root?
+      !defined? @parent
+    end
+
+    def ki_path(*sub_paths)
+      if defined? @parent
+        paths = [@parent.ki_path, @path]
+      else
+        paths = ["/"]
+      end
+      File.join([paths, sub_paths].flatten)
+    end
+
+    def child(name)
+      DirectoryBase.new(name)
+    end
+  end
+
+  class DirectoryBase
+    include DirectoryBaseModule
+
+    def initialize(path)
+      init_from_path(path)
+    end
+
+    def self.find!(path, *locations)
+      locations.each do |loc|
+        dest = loc.go(path)
+        if dest.exists?
+          return dest
+        end
+      end
+      raise "Could not find '#{path}' from '#{locations.map { |l| l.path }.join("', '")}'"
+    end
+  end
+end

data/lib/data_storage/ki_home.rb

@@ -0,0 +1,44 @@
+# encoding: UTF-8
+
+# Copyright 2012 Mikko Apo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Ki
+  module RepositoryMethods
+    DirectoryWithChildrenInListFile.add_list_file(self, Repository::Repository)
+
+    # repositories are stored under repositories directory
+    class RepositoryListFile
+      undef :create_list_item
+      def create_list_item(item)
+        Repository::Repository.new("repositories/" + item).parent(parent).repository_id(item)
+      end
+    end
+
+    def finder
+      RepositoryFinder.new(self)
+    end
+
+    def version(*args, &block)
+      finder.version(*args, &block)
+    end
+  end
+
+  class KiHome < DirectoryBase
+    include RepositoryMethods
+    def self.ki_version
+      IO.read(File.expand_path(File.join(File.dirname(__FILE__),"../..", 'VERSION')))
+    end
+  end
+end

data/lib/data_storage/ki_json.rb

@@ -0,0 +1,153 @@
+# encoding: UTF-8
+
+# Copyright 2012 Mikko Apo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Ki
+  require 'json'
+
+  # Base implementation for json files.
+  # * DirectoryBase takes a path argument where file will exist
+  # * Classes inheriting this file should implement json_default() to define default data object
+  # * cached_data loads data from disk when first accessed and edit_data modifies it
+  # * helper methods should access data through cached_data
+  class KiJSONFile < DirectoryBase
+    attr_chain :cached_data, -> { load_data_from_file }
+
+    # Loads latest data from file path. Does not update cached_data
+    def load_data_from_file(default=json_default)
+      KiJSONFile.load_json(path, default)
+    end
+
+    # Loads data from file path, makes it editable and saves data
+    def edit_data(&block)
+      @cached_data = load_data_from_file
+      block.call(self)
+      File.safe_write(path, JSON.pretty_generate(@cached_data))
+      @cached_data
+    end
+
+    # Saves data to file path. Does not update cached_data
+    def save(data=cached_data)
+      File.safe_write(path, JSON.pretty_generate(data))
+    end
+
+    def size
+      cached_data.size
+    end
+
+    def KiJSONFile.load_json(path, default=nil)
+      if File.exists?(path)
+        JSON.parse(IO.read(path))
+      else
+        default
+      end
+    end
+
+    def reset_cached_data
+      remove_instance_variable(:@cached_data)
+    end
+  end
+
+  # Base implementation for Json list file
+  class KiJSONListFile < KiJSONFile
+    include Enumerable
+    attr_chain :json_default, -> { Array.new }
+
+    def create_list_item(obj)
+      obj
+    end
+
+    def add_item(obj)
+      edit_data do
+        @cached_data << obj
+      end
+      create_list_item(obj)
+    end
+
+    def each(&block)
+      cached_data.each do |obj|
+        block.call(create_list_item(obj))
+      end
+    end
+  end
+
+  # Base implementation Json hash file
+  #
+  # Inheriting classes should implement their values using CachedMapDataAccessor
+  #     class VersionMetadataFile < KiJSONHashFile {
+  #       attr_chain :source, -> { Hash.new }, :accessor => CachedData
+  class KiJSONHashFile < KiJSONFile
+    include Enumerable
+    attr_chain :json_default, -> { Hash.new }
+
+    class CachedMapDataAccessor
+      def get(object, name)
+        object.cached_data[name.to_s]
+      end
+
+      def set(object, name, value)
+        object.cached_data[name.to_s] = value
+      end
+
+      def defined?(object, name)
+        object.cached_data.include?(name.to_s)
+      end
+    end
+
+    CachedData = CachedMapDataAccessor.new
+  end
+
+  # Helper method for creating list files.
+  class DirectoryWithChildrenInListFile
+    # Helper method for creating list files. When called on a class it extends the class with:
+    # * class extending KiJSONListFile which stores list items: class VersionListFile
+    # * method to load the file: versions()
+    # * method to load a specific item from list file: version(version_id, versions_list=versions)
+    def self.add_list_file(obj, clazz, name=nil)
+      stripped_class_name = clazz.name.split("::").last
+      class_name = clazz.name
+      list_class_name = "#{stripped_class_name}ListFile"
+      create_id_name = stripped_class_name.gsub(/.[A-Z]/) { |s| "#{s[0]}_#{s[1]}" }.downcase
+      if name.nil?
+        name = create_id_name
+      end
+      pluralized_name = "#{name}s".gsub(/ys$/, "ies")
+      new_methods = <<METHODS
+  class #{list_class_name} < KiJSONListFile
+    def create_list_item(#{name}_id)
+      i = #{class_name}.new(#{name}_id)
+      i.parent(parent)
+      i.#{create_id_name}_id(#{name}_id)
+      i
+    end
+  end
+
+  def #{pluralized_name}
+      #{list_class_name}.new("ki-#{pluralized_name}.json").parent(self)
+  end
+
+  def #{name}(#{name}_id, #{pluralized_name}_list=#{pluralized_name})
+    #{pluralized_name}_list.each do |c|
+      if c.#{name}_id == #{name}_id
+        return c
+      end
+    end
+    raise "#{class_name} '\#{#{name}_id}' not found"
+  end
+METHODS
+      obj.class_eval(new_methods, __FILE__, (__LINE__ - new_methods.split("\n").size - 1))
+    end
+  end
+end

data/lib/data_storage/repository.rb

@@ -0,0 +1,91 @@
+# encoding: UTF-8
+
+# Copyright 2012 Mikko Apo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Ki
+  module Repository
+    # Contains information about a version for one repository
+    # * Files: ki-version.json, ki-statuses.json, ki-reverse-dependencies.json
+    # @see Component
+    # @see VersionMetadataFile
+    # @see VersionStatusFile
+    # @see KiJSONListFile
+    class Version < DirectoryBase
+      attr_chain :version_id, :require
+      attr_chain :metadata, -> {VersionMetadataFile.new("ki-version.json").json_default("version_id" => version_id).parent(self)}
+
+      def statuses
+        VersionStatusFile.new("ki-statuses.json").parent(self)
+      end
+
+      def reverse_dependencies
+        KiJSONListFile.new("ki-reverse-dependencies.json").parent(self)
+      end
+
+      def action_usage
+
+      end
+    end
+
+    # Contains information about a component for one repository
+    # * Files: ki-versions.json, status_info.json
+    # @see Version
+    # @see Repository
+    class Component < DirectoryBase
+      attr_chain :component_id, :require
+      DirectoryWithChildrenInListFile.add_list_file(self, Version)
+
+      # Chronological list of versions in this component
+      class VersionListFile
+        undef create_list_item
+        def create_list_item(item)
+          id = item["id"]
+          Version.new(id).version_id(File.join(parent.component_id, id)).parent(parent)
+        end
+
+        def add_version(id, time=Time.now)
+          obj = {"id" => id, "time" => time}
+          edit_data do
+            @cached_data.unshift obj
+          end
+          create_list_item(obj)
+        end
+      end
+
+      # Status information file. Hash that defines information about status fields
+      # * list defines order of statuses {"maturity": ["alpha","beta","gamma"]}
+      def status_info
+        KiJSONHashFile.new("status_info.json").parent(self)
+      end
+    end
+
+    # Repository root
+    # * Files: ki-components.json
+    # @see Component
+    class Repository < DirectoryBase
+      attr_chain :repository_id, :require
+      DirectoryWithChildrenInListFile.add_list_file(self, Component)
+
+      # finds version matching the last part of version string, for example: my/component/1 looks for version named 1
+      # @see Component#version
+      def version(str)
+        args = str.split("/")
+        args.delete_at(-1)
+        component(args.join("/")).version(str)
+      end
+    end
+  end
+end
+

data/lib/data_storage/version_metadata.rb

@@ -0,0 +1,141 @@
+# encoding: UTF-8
+
+# Copyright 2012 Mikko Apo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Ki
+# Metadata
+# * version_id String
+# * build_info Hash. keys: source, tag, commiter. values=Strings
+# * filesets Hash. keys: Sorted array of tags. values: List of File info Hash.
+#   * File info Hash. keys: path, size, executable, sha-1
+# * dependencies List. values: Dependency Hash. keys: dependency_id, path, internal, dependency_operations
+# * fileoperations
+  class VersionMetadataFile < KiJSONHashFile
+    attr_chain :version_id, :require, :accessor => CachedData
+    attr_chain :source, -> { Hash.new }, :accessor => CachedData
+    attr_chain :files, -> { Array.new }, :accessor => CachedData
+    attr_chain :operations, -> { Array.new }, :accessor => CachedData
+    attr_chain :dependencies, -> { Array.new }, :accessor => CachedData
+
+    def add_file_info(name, size, *args)
+      extra = (args.select { |arg| arg.kind_of?(Hash) }.size!(0..1).first or {})
+      tags = (args - [extra]).flatten.uniq
+      file_hash = {"path" => name, "size" => size}.merge(extra)
+      if tags.size > 0
+        file_hash["tags"]=tags
+      end
+      files << file_hash
+    end
+
+    # Comma separated list of dependency arguments
+    # * dependency parameters can be given in the hash
+    # TODO: version_id should be resolved through Version
+    def add_dependency(param_str, args={})
+      params = param_str.split(",")
+      version_id = params.delete_at(0)
+      dep_hash = {"version_id" => version_id}.merge(params.to_h("=")).merge(args)
+      if dep_hash["internal"]
+        dep_hash["internal"]=true
+      end
+      dep_hash.extend(DependencyMethods)
+      dependencies << dep_hash
+      dep_hash
+    end
+
+    def add_operation(args)
+      operations << args
+    end
+
+    def VersionMetadataFile.calculate_hashes(full_path, digester_ids)
+      digesters = {}
+      digester_ids.each do |h|
+        digesters[h] = KiCommand::KiExtensions.find!(File.join("/hashing", h)).digest
+      end
+      algos = digesters.values
+      File.open(full_path, "r") do |io|
+        while (!io.eof)
+          buf = io.readpartial(1024)
+          algos.each do |digester|
+            digester.update(buf)
+          end
+        end
+      end
+      digesters.each_pair do |h, digester|
+        digesters[h]=digester.hexdigest
+      end
+      digesters
+    end
+
+    # Processes all files from source that match patterns and for each file calculates hashes and stores tags based on default_parameters
+    def add_files(source, patterns, default_parameters={})
+      files_or_dirs = Array.wrap(patterns).map do |pattern|
+        Dir.glob(File.join(source, pattern))
+      end.flatten
+
+      files = files_or_dirs.map do |file_or_dir|
+        if File.directory?(file_or_dir)
+          Dir.glob(File.join(file_or_dir, "**/*"))
+        else
+          file_or_dir
+        end
+      end.flatten.sort
+
+      files.each do |file|
+        add_file(source, file, default_parameters)
+      end
+      self
+    end
+
+    def add_file(root, full_path, parameters)
+      stat = File.stat(full_path)
+      size = stat.size
+      extra = {}
+      if stat.executable?
+        extra["executable"]=true
+      end
+      if parameters["tags"]
+        tags = Array.wrap(parameters["tags"])
+        if tags && tags.size > 0
+          extra["tags"]= tags
+        end
+      end
+      if parameters["hashes"].nil?
+        parameters["hashes"]=["sha1"]
+      end
+      if parameters["hashes"].size > 0
+        extra.merge!(VersionMetadataFile.calculate_hashes(full_path, parameters["hashes"]))
+      end
+      add_file_info(full_path[root.size+1..-1], size, extra)
+    end
+  end
+
+  module DependencyMethods
+    attr_chain :operations, -> { Array.new }, :accessor => AttrChain::HashAccess
+
+    def add_operation(args)
+      operations << args
+    end
+  end
+
+  class VersionStatusFile < KiJSONListFile
+    def add_status(key, value, flags={})
+      add_item({"key" => key, "value" => value}.merge(flags))
+    end
+
+    def matching_statuses(key)
+      cached_data.select { |hash| hash["key"].match(key) }
+    end
+  end
+end

data/lib/ki_repo_all.rb

@@ -0,0 +1,42 @@
+# encoding: UTF-8
+
+# Copyright 2012 Mikko Apo
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'bundler/setup'
+
+require_relative 'util/attr_chain'
+require_relative 'util/exception_catcher'
+require_relative 'util/ruby_extensions'
+require_relative 'util/test'
+require_relative 'util/service_registry'
+require_relative 'util/hash'
+require_relative 'util/hash_cache'
+require_relative 'util/simple_optparse'
+
+require_relative 'data_storage/dir_base'
+require_relative 'data_storage/ki_json'
+require_relative 'data_storage/repository'
+require_relative 'data_storage/version_metadata'
+require_relative 'data_storage/ki_home'
+
+require_relative 'data_access/repository_info'
+require_relative 'data_access/repository_finder'
+require_relative 'data_access/version_helpers'
+require_relative 'data_access/version_operations'
+require_relative 'data_access/version_iterators'
+
+require_relative 'cmd/cmd'
+require_relative 'cmd/version_cmd'
+require_relative 'cmd/user_pref_cmd'