dtk-client 0.9.1 → 0.10.0

data/lib/client/operation/account.rb

@@ -0,0 +1,66 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::Client
+  class Operation
+    class Account < self
+      RoutePrefix = 'account'
+
+      # opts can have keys
+      #  :first_registration - Booelan (default: false)
+      #  :name - (default: 'dtk-client')
+      def self.add_key(path_to_key, opts = {})
+        match, matched_username = nil, nil
+
+        unless File.file?(path_to_key)
+          raise Error,"[ERROR] No ssh key file found at (#{path_to_key}). Path is wrong or it is necessary to generate the public rsa key (e.g., run `ssh-keygen -t rsa`)."
+        end
+        
+        response = add_user_access(path_to_key, opts)
+
+        if response.ok? and !response.data(:match)
+          repo_manager_fingerprint, repo_manager_dns = response.data_ret_and_remove!(:repo_manager_fingerprint, :repo_manager_dns)
+          SSHUtil.update_ssh_known_hosts(repo_manager_dns, repo_manager_fingerprint)
+          OsUtil.print_info("SSH key '#{response.data('new_username')}' added successfully!")
+        end
+        
+        response
+      end
+
+      private
+
+      # opts can have keys
+      #  :first_registration - Booelan (default: false)
+      #  :name - (default: 'dtk-client')
+      def self.add_user_access(path_to_key, opts = {})
+        first_registration = opts[:first_registration] || false
+        name = opts[:name] || 'dtk-client'
+
+        rsa_pub_key = SSHUtil.read_and_validate_pub_key(path_to_key)
+        
+        post_body  = { 
+          :rsa_pub_key        => rsa_pub_key.chomp,
+          :username?          => name && name.chomp,
+          :first_registration => first_registration,
+        }
+        rest_post "#{RoutePrefix}/add_user_direct_access", PostBody.new(post_body)
+      end
+    end
+  end
+end
+
+

data/lib/client/operation/client_module_dir/git_repo.rb

@@ -0,0 +1,286 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::Client
+  class Operation::ClientModuleDir
+    # Operations for managing module folders that are git repos
+    class GitRepo < self
+      def self.commit_and_push_to_service_repo(args)
+        wrap_operation(args) do |args| 
+          response_data_hash(:head_sha => Internal.commit_and_push_to_service_repo(args))
+        end
+      end
+
+      def self.clone_exists?(args)
+        wrap_operation(args) do |args|
+          type = args.required(:type)
+          local_dir_exists?(type, args.required("#{type}_name".to_sym))
+        end
+      end
+
+      def self.clone_service_repo(args)
+        wrap_operation(args) do |args|
+          response_data_hash(:target_repo_dir => Internal.clone_service_repo(args))
+        end
+      end
+
+      def self.clone_module_repo(args)
+        wrap_operation(args) do |args|
+          response_data_hash(:target_repo_dir => Internal.clone_module_repo(args))
+        end
+      end
+
+      def self.create_add_remote_and_push(args)
+        wrap_operation(args) do |args|
+          repo_dir      = args.required(:repo_dir)
+          repo_url      = args.required(:repo_url)
+          remote_branch = args.required(:remote_branch)
+          response_data_hash(:head_sha => Internal.create_add_remote_and_push(repo_dir, repo_url, remote_branch))
+        end
+      end
+
+      def self.create_add_remote_and_pull(args)
+        wrap_operation(args) do |args|
+          repo_dir      = args.required(:repo_dir)
+          repo_url      = args.required(:repo_url)
+          remote_branch = args.required(:remote_branch)
+          response_data_hash(:head_sha => Internal.create_add_remote_and_pull(repo_dir, repo_url, remote_branch))
+        end
+      end
+
+      def self.fetch_merge_and_push(args)
+        wrap_operation(args) do |args|
+          response_data_hash(:head_sha => Internal.fetch_merge_and_push(args))
+        end
+      end
+
+      def self.init_and_push_from_existing_repo(args)
+        wrap_operation(args) do |args|
+          repo_dir      = args.required(:repo_dir)
+          repo_url      = args.required(:repo_url)
+          remote_branch = args.required(:remote_branch)
+          response_data_hash(:head_sha => Internal.init_and_push_from_existing_repo(repo_dir, repo_url, remote_branch))
+        end
+      end
+
+      def self.pull_from_remote(args)
+        wrap_operation(args) do |args|
+          response_data_hash(:target_repo_dir => Internal.pull_from_remote(args))
+        end
+      end
+
+      def self.pull_from_service_repo(args)
+        wrap_operation(args) do |args| 
+          response_data_hash(:repo => Internal.pull_from_service_repo(args)) 
+        end
+      end
+
+      private
+
+      def self.response_data_hash(hash)
+        hash.inject({}) { |h, (k, v)| h.merge(k.to_s => v) }
+      end
+
+      # All Internal methods do not have wrap_operation and can only be accessed by a method that wraps it
+      class Internal < self
+        # Git Params for dtk server
+        module Dtk_Server
+          GIT_REMOTE   = 'dtk-server'
+        end
+        # Git params for dtkn
+        module Dtkn
+          GIT_REMOTE   = 'dtkn'
+          LOCAL_BRANCH = 'master'
+        end
+
+        # returns head_sha
+        def self.commit_and_push_to_service_repo(args)
+          branch           = args.required(:branch)
+          remote_branch    = args[:remote_branch] || branch
+          service_instance = args.required(:service_instance)
+          commit_message   = args[:commit_message]
+
+          repo_dir = ret_base_path(:service, service_instance)
+          repo = git_repo.new(repo_dir, :branch => branch)
+          repo.stage_and_commit
+          # TODO: want to switch over to using Dtk_Server::GIT_REMOTE rather than 'origin'
+          dtk_server_remote = 'origin'
+          repo.push(dtk_server_remote, remote_branch)
+          repo.head_commit_sha
+        end
+
+        def self.clone_service_repo(args)
+          repo_url         = args.required(:repo_url)
+          module_ref       = args.required(:module_ref)
+          branch           = args.required(:branch)
+          service_instance = args.required(:service_instance)
+          remove_existing  = args[:remove_existing]
+
+          target_repo_dir = create_service_dir(service_instance, :remove_existing => remove_existing)
+          begin
+            git_repo.clone(repo_url, target_repo_dir,  branch)
+          rescue => e
+            #cleanup by deleting directory
+            
+            FileUtils.rm_rf(target_repo_dir) if File.directory?(target_repo_dir)
+            # Log error details
+            Logger.instance.error_pp(e.message, e.backtrace)
+            
+            # User-friendly error
+            raise Error::Usage, "Clone to directory '#{target_repo_dir}' failed"
+          end
+          target_repo_dir
+        end
+
+        def self.clone_module_repo(args)
+          module_type     = args.required(:module_type)
+          repo_url        = args.required(:repo_url)
+          branch          = args.required(:branch)
+          module_name     = args.required(:module_name)
+          remove_existing = args[:remove_existing]
+          repo_dir        = args[:repo_dir]
+
+          target_repo_dir = create_module_dir(module_type, module_name, :remove_existing => remove_existing, :path => repo_dir)
+          begin
+            git_repo.clone(repo_url, target_repo_dir,  branch)
+          rescue => e
+            FileUtils.rm_rf(target_repo_dir) if File.directory?(target_repo_dir)
+            Logger.instance.error_pp(e.message, e.backtrace)
+
+            raise Error::Usage, "Clone to directory '#{target_repo_dir}' failed"
+          end
+
+          target_repo_dir
+        end
+        
+        def self.fetch_merge_and_push(args)
+          repo_dir      = args.required(:repo_dir)
+          repo_url      = args.required(:repo_url)
+          remote_branch = args.required(:branch)
+          
+          head_sha =
+            if git_repo.is_git_repo?(repo_dir)
+              init_and_push_from_existing_repo(repo_dir, repo_url, remote_branch)
+            else
+              create_repo_from_remote_and_push(repo_dir, repo_url, remote_branch)
+            end
+          
+          head_sha 
+        end
+        
+        def self.create_add_remote_and_push(repo_dir, repo_url, remote_branch)
+          repo = git_repo.new(repo_dir)
+          add_remote_and_push(repo, repo_url, remote_branch)
+          repo.head_commit_sha
+        end
+
+        def self.create_add_remote_and_pull(repo_dir, repo_url, remote_branch)
+          repo = create_repo_from_dtkn_remote(repo_dir, repo_url, remote_branch)
+          add_dtkn_remote_and_pull(repo, repo_url, remote_branch)
+          repo.head_commit_sha
+        end
+        
+        def self.init_and_push_from_existing_repo(repo_dir, repo_url, remote_branch)
+          repo = git_repo.new(repo_dir)
+          
+          if repo.is_there_remote?(Dtk_Server::GIT_REMOTE)
+            push_when_there_is_dtk_remote(repo, repo_dir, repo_url, remote_branch)
+          else
+            add_remote_and_push(repo, repo_url, remote_branch)
+          end
+          
+          repo.head_commit_sha
+        end
+        
+        def self.pull_from_remote(args)
+          repo_url       = args.required(:repo_url)
+          remote_branch  = args.required(:branch)
+          repo_dir       = args.required(:repo_dir)
+          
+          repo = git_repo.new(repo_dir, :branch => remote_branch)
+          repo.pull(repo.remotes.first, remote_branch)
+        end
+        
+        # returns the repo
+        def self.pull_from_service_repo(args)
+          repo_url         = args.required(:repo_url)
+          remote_branch    = args.required(:branch)
+          service_instance = args.required(:service_instance)
+          
+          repo_dir = ret_base_path(:service, service_instance)
+          repo = git_repo.new(repo_dir, :branch => remote_branch)
+          
+          repo.pull(repo.remotes.first, remote_branch)
+          repo
+        end
+
+        def self.push_when_there_is_dtk_remote(repo, repo_dir, repo_url, remote_branch)
+          # if there is only one remote and it is dtk-server; remove .git and initialize and push as new repo to dtk-server remote
+          # else if multiple remotes and dtk-server being one of them; remove dtk-server; add new dtk-server remote and push
+          if repo.remotes.size == 1
+            git_repo.unlink_local_clone?(repo_dir)
+            create_repo_from_remote_and_push(repo_dir, repo_url, remote_branch)
+          else
+            repo.remove_remote(Dtk_Server::GIT_REMOTE)
+            add_remote_and_push(repo, repo_url, remote_branch)
+          end
+        end
+
+        def self.create_repo_from_server_remote(repo_dir, repo_url, remote_branch)
+          repo = git_repo.new(repo_dir, :branch => Dtkn::LOCAL_BRANCH)
+          repo.checkout(Dtkn::LOCAL_BRANCH, :new_branch => true)
+          repo.add_remote(Dtk_Server::GIT_REMOTE, repo_url)
+          repo
+        end
+
+        def self.create_repo_from_dtkn_remote(repo_dir, repo_url, remote_branch)
+          repo = git_repo.new(repo_dir, :branch => Dtkn::LOCAL_BRANCH)
+          repo.checkout(Dtkn::LOCAL_BRANCH, :new_branch => true)
+          repo.stage_and_commit
+          repo.add_remote(Dtkn::GIT_REMOTE, repo_url)
+          repo
+        end
+
+        def self.create_repo_from_remote_and_push(repo_dir, repo_url, remote_branch)
+          repo = create_repo_from_server_remote(repo_dir, repo_url, remote_branch)
+          # repo = git_repo.new(repo_dir, :branch => Dtkn::LOCAL_BRANCH)
+          # repo.checkout(Dtkn::LOCAL_BRANCH, :new_branch => true)
+          # repo.add_remote(Dtk_Server::GIT_REMOTE, repo_url)
+          repo.stage_and_commit
+          repo.push(Dtk_Server::GIT_REMOTE, remote_branch, { :force => true })
+          repo.head_commit_sha
+        end
+        
+        def self.add_remote_and_push(repo, repo_url, remote_branch)
+          repo.add_remote(Dtk_Server::GIT_REMOTE, repo_url)
+          repo.stage_and_commit
+          repo.push(Dtk_Server::GIT_REMOTE, remote_branch, { :force => true })
+        end
+
+        def self.add_dtkn_remote_and_pull(repo, repo_url, remote_branch)
+          repo.add_remote(Dtkn::GIT_REMOTE, repo_url)
+          repo.pull(Dtkn::GIT_REMOTE, remote_branch)
+        end
+        
+        def self.git_repo
+          ::DTK::Client::GitRepo
+        end
+      end
+    end
+  end
+end
+

data/lib/client/operation/client_module_dir.rb

@@ -0,0 +1,138 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'fileutils'
+
+module DTK::Client
+  class Operation
+    # Operations for managing module folders
+    class ClientModuleDir < self
+      require_relative('client_module_dir/git_repo')
+      
+      NAMESPACE_SEPERATOR = ':'
+      # opts can have keys
+      #   :backup_if_exist - Boolean (default: false)
+      #   :remove_existing - Boolean (default: false)
+      def self.create_service_dir(service_instance, opts = {})
+        path = "#{base_path(:service)}/#{service_instance}"
+        if File.exists?(path)
+          if opts[:remove_existing]
+            FileUtils.rm_rf(path)
+          else
+            # TODO: put back in after referenced methods are ported over
+            # if local copy of module exists then move that module to backups location
+            # if opts[:backup_if_exist]
+            #  backup_dir = backup_dir(type, module_ref)
+            #  FileUtils.mv(target_repo_dir, backup_dir)
+            #  OsUtil.print_warning("Backup of existing module directory moved to '#{backup_dir}'")
+            # else
+            # raise Error::Usage, "Directory '#{path}' is not empty; it must be deleted or removed before retrying the command"
+            #end
+            raise Error::Usage, "Directory '#{path}' is not empty; it must be deleted or removed before retrying the command"
+          end
+        end
+        FileUtils.mkdir_p(path)
+        path
+      end
+
+      def self.create_module_dir(module_type, module_name, opts = {})
+        path = opts[:path]
+
+        unless path
+          base_module_path = base_path(module_type)
+
+          path_parts =
+            if (module_name.match(/(.*)#{NAMESPACE_SEPERATOR}(.*)/))
+              [base_module_path, "#{$1}", "#{$2}"]
+            else
+              [base_module_path, "#{module_name}"]
+            end
+
+          path = path_parts.compact.join('/')
+        end
+
+        if File.exists?(path)
+          if opts[:remove_existing]
+            FileUtils.rm_rf(path)
+          else
+            raise Error::Usage, "Directory '#{path}' is not empty; it must be deleted or removed before retrying the command" unless (Dir.entries(path) - %w{ . .. }).empty?
+            return path
+          end
+        end
+
+        FileUtils.mkdir_p(path)
+        path
+      end
+
+      def self.create_module_dir_from_path(path, opts = {})
+        if File.exists?(path)
+          if opts[:remove_existing]
+            FileUtils.rm_rf(path)
+          else
+            raise Error::Usage, "Directory '#{path}' is not empty; it must be deleted or removed before retrying the command" unless (Dir.entries(path) - %w{ . .. }).empty?
+            return path
+          end
+        end
+
+        FileUtils.mkdir_p(path)
+        path
+      end
+
+      def self.local_dir_exists?(type, name, opts = {})
+        File.exists?("#{base_path(type)}/#{name}")
+      end
+
+      def self.ret_base_path(type, name)
+        "#{base_path(type)}/#{name}"
+      end
+
+      def self.ret_path_with_current_dir(name)
+        "#{OsUtil.current_dir}/#{name.gsub(':','/')}"
+      end
+
+      def self.purge_service_instance_dir(dir_path)
+        FileUtils.rm_rf(dir_path)
+      end
+
+      def self.delete_directory_content(path)
+        FileUtils.rm_rf(Dir.glob("#{path}/*"))
+      end
+
+      def self.create_file_with_content(file_path, content)
+        File.open(file_path, 'w') { |f| f << content }
+      end
+    
+      private
+
+      def self.base_path(type)
+        path =
+          case type.to_sym
+          when :service_module then Config[:service_location]
+          when :component_module then Config[:module_location]
+          when :service then Config[:instance_location]
+          else raise Error, "Unexpected type (#{type}) when determining base path"
+          end
+        
+        # see if path is relative or not
+        final_path = path && path.start_with?('/') ? path : "#{OsUtil.dtk_local_folder}/#{path}"
+        # remove last slash if set in configuration by mistake
+        final_path.gsub(/\/$/,'')
+      end
+    end
+  end
+end
+

data/lib/client/operation/module/clone_module.rb

@@ -0,0 +1,52 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::Client
+  class Operation::Module
+    class CloneModule < self
+      def self.execute(args = Args.new)
+        wrap_operation(args) do |args|
+          module_ref       = args.required(:module_ref)
+          module_name      = args.required(:module_name)
+          target_directory = args[:target_directory]
+
+          unless module_info = module_exists?(module_ref, :type => :common_module)
+            raise Error::Usage, "DTK module '#{module_ref.print_form}' does not exist on server."
+          end
+
+          branch    = module_info.required(:branch, :name)
+          repo_url  = module_info.required(:repo, :url)
+          repo_name = module_info.required(:repo, :name)
+
+          clone_args = {
+            :module_type => :common_module,
+            :repo_url    => module_info.required(:repo, :url),
+            :branch      => module_info.required(:branch, :name),
+            :module_name => module_name,
+            :repo_dir    => target_directory || ClientModuleDir.ret_path_with_current_dir(module_name)
+            # :remove_existing  => remove_existing
+          }
+
+          ret = ClientModuleDir::GitRepo.clone_module_repo(clone_args)
+          OsUtil.print_info("DTK module '#{module_ref.print_form}' has been successfully cloned into '#{ret.required(:target_repo_dir)}'")
+        end
+      end
+    end
+  end
+end
+
+

data/lib/client/operation/module/install/common_module.rb

@@ -0,0 +1,58 @@
+#
+# Copyright (C) 2010-2016 dtk contributors
+#
+# This file is part of the dtk project.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module DTK::Client
+  class Operation::Module::Install
+    class CommonModule < self
+      BaseRoute = 'modules'
+
+      def self.install(module_ref, file_obj)
+        post_body = PostBody.new(
+          :module_name => module_ref.module_name,
+          :namespace   => module_ref.namespace,
+          :version?    => module_ref.version
+        )
+
+        response = rest_post("#{BaseRoute}/create_empty_module", post_body)
+
+        branch    = response.required(:branch, :name)
+        repo_url  = response.required(:repo, :url)
+        repo_name = response.required(:repo, :name)
+
+        # making repo dir to be directory that directly holds the base file object file_obj
+        repo_dir = file_obj.parent_dir
+
+        args = {
+          :repo_dir => repo_dir,
+          :repo_url => repo_url,
+          :branch   => branch
+        }
+
+        git_response = ClientModuleDir::GitRepo.fetch_merge_and_push(args)
+
+        post_body.merge!(
+          :repo_name  => repo_name,
+          :commit_sha => git_response.data(:head_sha)
+        )
+
+        rest_post("#{BaseRoute}/update_from_repo", post_body)
+      end
+    end
+  end
+end
+
+