dtk-client 0.10.0.1 → 0.10.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 27c7da3e73bf4e15f1588363817bf3d31cce7eaa
-  data.tar.gz: 9b67c4994a0f8e4e4a2646fa18658f48d3c8a4f6
+  metadata.gz: f28cd5b72fe1b22b5ed1c1acf5d7994c70e147d5
+  data.tar.gz: 3ed5a5af32baa1a5d22e5a493bd4907b7636eef2
 SHA512:
-  metadata.gz: ed1c406eec37ae5326c0ec53e022168b380f2cf0e95c91c49017726883a1a0058d1e86f14da84f707b375fec1a8a78fb9cc430c0e71bf43314e7ae321b286420
-  data.tar.gz: d00b0993f3c444c435f0f1ddd1d4c7f23aeda34761fd689694ea087140902edc607b3390f2079edc396379d4a7f30b72adc8617253a20d570ca50b8ecc3e5df5
+  metadata.gz: 609335a109d58bfc74ffa0b0b5580b6474fdadea1d2691d7270420de7f60bb1bd26363c7e55cd77e0f2dbd854bc4c9939d0587360f314b77b45814809f9fae1a
+  data.tar.gz: 3f3446bd3d10709e48c19c12886fde17da89e16bf0dcaa866a582a525fe81bb411352da2475100ca54abe75705b02f886805f03546b6a3a6ec846bdad6d7e8ec

data/.gitignore

@@ -2,3 +2,4 @@
 sftp-config*.json
 .remote-sync.json
 .tags
+Gemfile.lock

data/Gemfile

@@ -1,3 +1,2 @@
 source 'https://rubygems.org'
 gemspec
-gem 'dtk-dsl', :github => 'dtk/dtk-dsl'

data/dtk-client.gemspec

@@ -24,5 +24,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'git', '1.2.9'
   spec.add_dependency 'hirb', '0.7.3'
   spec.add_dependency 'mime-types', '~> 2.99.3'
+  spec.add_dependency 'dtk-dsl', '1.0.0'
   
 end

data/examples/simple/dtk.module.yaml

@@ -1,7 +1,7 @@
 ---
 dsl_version: 1.0.0
 module: rich/simple
-version: 0.0.2
+version: 0.0.1
 keywords: DTK-2554
 description: Tests for DTK-2554
 license: Apache 2.0

data/examples/spark/dtk.module.yaml

@@ -1,6 +1,5 @@
 ---
-dsl_version: 1.0.0
-module: rich/spark
+module: test/spark
 keywords: spark, hdfs, zeppelin
 description: Spark cluster
 license: Apache 2.0

data/lib/cli/command/module.rb

@@ -21,9 +21,7 @@ module DTK::Client
       module Module 
         include Command::Mixin
 
-        # TODO: removed push command until fixed by jira: DTK-2667
-        # ALL_SUBCOMMANDS = ['install', 'list', 'list-assemblies', 'push', 'uninstall', 'clone']
-         ALL_SUBCOMMANDS = ['install', 'list', 'list-assemblies', 'uninstall', 'clone']
+        ALL_SUBCOMMANDS = ['install', 'list', 'list-assemblies', 'push', 'uninstall', 'clone']
         command_def :desc => 'Subcommands for interacting with DTK modules'
         ALL_SUBCOMMANDS.each { |subcommand| require_relative("module/#{subcommand.gsub(/-/,'_')}") } 
       end

data/lib/cli/version.rb

@@ -1,7 +1,7 @@
 module DTK
   module Client
     module CLI
-      VERSION = '0.10.0.1'
+      VERSION = '0.10.0.2'
     end
   end
 end

data/lib/client/content_generator.rb

@@ -91,8 +91,8 @@ module DTK::Client
         content_hash     = convert_file_content_to_hash(assembly)
         name             = content_hash['name']
         assembly_content = content_hash['assembly']
-        workflows        = ret_workflows_hash(content_hash['workflow'])
-        
+        workflows        = ret_workflows_hash(content_hash)
+
         assembly_content.merge!('workflows' => workflows) if workflows
         assemblies.merge!(name => assembly_content)
       end
@@ -100,12 +100,15 @@ module DTK::Client
       assemblies.empty? ? nil : assemblies
     end
 
-    def ret_workflows_hash(workflow)
-      return if workflow.nil? || workflow.empty?
-      workflow_name = workflow.delete('assembly_action')
-      {
-        workflow_name => workflow
-      }
+    def ret_workflows_hash(content_hash)
+      if workflows = content_hash['workflow'] || content_hash['workflows']
+        # this is legacy workflow
+        if workflow_name = workflows.delete('assembly_action')
+          { workflow_name => workflows }
+        else
+          workflows
+        end
+      end
     end
 
     def ret_dependencies_hash

data/lib/client/operation/client_module_dir/git_repo.rb

@@ -83,11 +83,16 @@ module DTK::Client
         end
       end
       
-      def self.add_file(args)
+      def self.add_service_repo_file(args)
         wrap_operation(args) do |args| 
-          response_data_hash(:repo => Internal.add_file(args)) 
+          response_data_hash(:repo => Internal.add_service_repo_file(args)) 
+        end
+      end
+
+      def self.get_service_repo_file_content(args)
+        wrap_operation(args) do |args| 
+          response_data_hash(:content => Internal.get_service_repo_file_content(args))
         end
-        
       end
 
       def self.pull_from_service_repo(args)
@@ -283,33 +288,61 @@ module DTK::Client
           repo.pull(Dtkn::GIT_REMOTE, remote_branch)
         end
         
-        def self.add_file(args)
-          all_backup_file_paths = args.required(:backup_files).keys
-          final_path = args.required(:final_path)
-          if File.exists?(file_path = "#{final_path}/.gitignore")
-           has_backup = false
-           file = File.open(file_path, "r") do |f|
-             f.each_line do |line|
-               unless all_backup_file_paths.include?(line.strip)
-                 write = File.open(file_path, "a")
-                 write << line
-               end
-             end
-           end
-          else
-           File.open(".gitignore", "w") do |f|
-            all_backup_file_paths.each {|el| f.puts("#{el}\n")}
-           end
+        def self.add_service_repo_file(args)
+          branch           = args.required(:branch)
+          service_instance = args.required(:service_instance)
+          path             = args.required(:path)
+          content          = args.required(:content)
+          checkout_branch(service_instance, branch) do
+            File.open(qualified_path(service_instance, path), 'w') { |f| f.write(content) }
           end
-          args[:backup_files].each_pair do |file_path, content|
-            File.open("#{final_path}/#{file_path}", "w") {|f| f.write(content)}
-            File.open("#{final_path}/dtk.service.yaml", "w") {|f| f.write(content)}
+        end
+
+        # returns content if file exists
+        def self.get_service_repo_file_content(args)
+          branch           = args.required(:branch)
+          service_instance = args.required(:service_instance)
+          path             = args.required(:path)
+          checkout_branch(service_instance, branch) do
+            qualified_path = qualified_path(service_instance, path)
+            if File.exists?(qualified_path)
+              File.open(qualified_path, 'r').read
+            end
           end
         end
 
         def self.git_repo
           ::DTK::Client::GitRepo
         end
+
+        private
+
+        # relative_path is relative to top-leel repo directory
+        def self.qualified_path(service_instance, relative_path)
+          repo_dir = ret_base_path(:service, service_instance)
+          "#{repo_dir}/#{relative_path}"
+        end
+
+        CHECKOUT_LOCK = Mutex.new
+        def self.checkout_branch(service_instance, branch, &body)
+          ret = nil
+          CHECKOUT_LOCK.synchronize do
+            repo = git_repo.new(ret_base_path(:service, service_instance), :branch => branch)
+            current_branch = repo.current_branch.name
+            if current_branch == branch
+              ret = yield
+            else
+              begin
+                repo.checkout(branch) 
+                ret = yield
+              ensure
+                repo.checkout(current_branch)
+              end
+            end
+          end
+          ret
+        end
+
       end
     end
   end

data/lib/client/operation/module/install/external_module.rb

@@ -23,7 +23,7 @@ module DTK::Client
       def self.install_dependent_modules(module_refs, opts = {})
         module_refs.each do |module_ref|
           if module_exists?(module_ref, { :type => :component_module })
-            OsUtil.print("Using module '#{module_ref.namespace}:#{module_ref.module_name}'")
+            OsUtil.print("Using module '#{module_ref.namespace}:#{module_ref.module_name}'" + (module_ref.version.nil? ? "" : " version: #{module_ref.version} "))
             # If component module is imported, still check to see if it's dependencies are imported
             find_and_install_component_module_dependency(module_ref, opts.merge(:skip_if_no_remote => true))
           else

data/lib/client/operation/module/install_from_catalog.rb

@@ -44,6 +44,11 @@ module DTK::Client
         )
         module_info = rest_get "#{BaseRoute}/remote_module_info", query_string_hash
 
+        # unless version is explicitly provided, use latest version instead of master
+        unless @version
+          @version = module_info.required(:latest_version)
+        end
+
         git_repo_args = {
           :repo_dir      => target_repo_dir,
           :repo_url      => module_info.required(:remote_repo_url),

data/lib/client/operation/module/uninstall.rb

@@ -31,9 +31,9 @@ module DTK::Client
             :namespace   => module_ref.namespace,
             :version?    => module_ref.version
           )
-          ret = rest_post("#{BaseRoute}/delete", post_body)
+          rest_post("#{BaseRoute}/delete", post_body)
           OsUtil.print_info("DTK module '#{module_ref.print_form}' has been deleted successfully.")
-          ret 
+          nil
         end
       end
 

data/lib/client/operation/service/commit_and_push.rb

@@ -25,18 +25,24 @@ module DTK::Client
 
           response = rest_get("#{BaseRoute}/#{service_instance}/repo_info")
 
-          push_args = {
+          repo_info_args = Args.new(
             :service_instance => service_instance,
             :commit_message   => args[:commit_message] || default_commit_message(service_instance),
             :branch           => response.required(:branch, :name),
             :repo_url         => response.required(:repo, :url)
-          }
+          )
 
-          response = ClientModuleDir::GitRepo.commit_and_push_to_service_repo(push_args)
+          response = ClientModuleDir::GitRepo.commit_and_push_to_service_repo(repo_info_args)
           commit_sha = response.required(:head_sha)
 
           response = rest_post("#{BaseRoute}/#{service_instance}/update_from_repo", :commit_sha => commit_sha)
-          process_response(response, :service_instance => service_instance, :args => push_args)
+          print_msgs_of_type(:error_msgs, response)
+          print_msgs_of_type(:warning_msgs, response)
+          print_msgs_of_type(:info_msgs, response)
+
+          ClientModuleDir::GitRepo.pull_from_service_repo(repo_info_args) if response.data(:repo_updated)
+          process_backup_files(repo_info_args, response.data(:backup_files))
+          process_semantic_diffs(response.data(:semantic_diffs))
           nil
         end
       end
@@ -51,36 +57,37 @@ module DTK::Client
         raise Error, "Need to write"
       end
 
-      def self.process_response(response, opts = {})
-        print_msgs_of_type(:error_msgs, response)
-        print_msgs_of_type(:warning_msgs, response)
-        print_msgs_of_type(:info_msgs, response)
 
-        pull_repo_updates?(response, opts)
-        process_backup_files(response, opts)
-        process_semantic_diffs(response)
-      end
+      def self.process_backup_files(repo_info_args, backup_files)
+        return if (backup_files || {}).empty?
+        backup_files.each_pair do |path, content|
+          ClientModuleDir::GitRepo.add_service_repo_file(repo_info_args.merge(:path => path, :content => content))
+        end
+    
+        backup_file_paths = backup_files.keys
+        update_gitignore?(repo_info_args, backup_file_paths)
 
-      def self.pull_repo_updates?(response, opts = {})
-        ClientModuleDir::GitRepo.pull_from_service_repo(opts[:args]) if response.data(:repo_updated)
+        ClientModuleDir::GitRepo.commit_and_push_to_service_repo(repo_info_args)
       end
 
-      # TODO: DTK-2663: This is fine for now, but in 0.10.1 want to move logic that writes files to be in
-      # a new method we write in ClientModuleDir::GitRepo.add_file
-      def self.process_backup_files(response, opts = {})
-        backup_files = response.data(:backup_files) || {}
-        return if backup_files.empty?
-        service_instance_name = opts[:service_instance]
-        final_path = "#{OsUtil.dtk_local_folder}/#{service_instance_name}" 
-        
-        ClientModuleDir::GitRepo.add_file(:backup_files => backup_files, :final_path => final_path)
-
-        response = ClientModuleDir::GitRepo.commit_and_push_to_service_repo(opts[:args]) 
+      GITIGNORE_REL_PATH = '.gitignore'
+      def self.update_gitignore?(repo_info_args, backup_file_paths)
+        response = ClientModuleDir::GitRepo.get_service_repo_file_content(repo_info_args.merge(:path => GITIGNORE_REL_PATH))
+        gitignore_content = response.data(:content) || ''
+        gitignore_files = gitignore_content.split("\n")
+        to_add = ''
+        backup_file_paths.each do |backup_file_path|
+          to_add << "#{backup_file_path}\n" unless gitignore_files.include?(backup_file_path)
+        end
+        unless to_add.empty?
+          gitignore_content << "\n" unless gitignore_content.empty? or gitignore_content[-1] == "\n"
+          gitignore_content << to_add
+          ClientModuleDir::GitRepo.add_service_repo_file(repo_info_args.merge(:path => GITIGNORE_REL_PATH, :content => gitignore_content))
+        end
       end
-
-      def self.process_semantic_diffs(response)
-        semantic_diffs = response.data(:semantic_diffs) || {}
-        return if semantic_diffs.empty?
+      
+      def self.process_semantic_diffs(semantic_diffs)
+        return if (semantic_diffs || {}).empty?
         # TODO: DTK-2663; cleanup so pretty printed'
         OsUtil.print_info("\nDiffs that were pushed:")
         # TODO: get rid of use of STDOUT
@@ -91,7 +98,7 @@ module DTK::Client
       PRINT_FN = {
         :info_msgs    => lambda { |msg| OsUtil.print_info(msg) },
         :warning_msgs => lambda { |msg| OsUtil.print_warning(msg) },
-        :error_msgs => lambda { |msg| OsUtil.print_error(msg) }
+        :error_msgs   => lambda { |msg| OsUtil.print_error(msg) }
       }
       def self.print_msgs_of_type(msg_type, response)
         msgs = response.data(msg_type) || []

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dtk-client
 version: !ruby/object:Gem::Version
-  version: 0.10.0.1
+  version: 0.10.0.2
 platform: ruby
 authors:
 - Reactor8
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-20 00:00:00.000000000 Z
+date: 2016-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dtk-common-core
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.99.3
+- !ruby/object:Gem::Dependency
+  name: dtk-dsl
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 description: Command line tool to interact with a DTK Server and DTK Service Catalog.
 email: support@reactor8.com
 executables:
@@ -306,7 +320,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.1
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: DTK CLI client for DTK server interaction.