dtk-client 0.9.1 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 65f98d3a29d9f7cbd1368bdb0ef41a7d5f294fa6
-  data.tar.gz: 1edf46630c80023aac016dfa21414fb9a27098ba
+  metadata.gz: 63474c8dc3c49c71f705cc21ff398b64f72adac4
+  data.tar.gz: d54921f4093a817ecfc2d8f58766ab34e6b949af
 SHA512:
-  metadata.gz: 12ed89cec1938c97083425aa8528d9ab14d2ed478abcd4c9bd1bda3432e44ac9a1c26cbf16ec843f36cdb1a1ae62983c51c4e7d8f7aab42b42f7a86cf3d57301
-  data.tar.gz: 4b2664285e145724fb4eeebcb14390eb990ea0183b2dad8076801ea1baaab1623176a106351090cb1ea5f76d2b79b1939b3f7f2b5b318b4c868d81a79387ba04
+  metadata.gz: 50369f8f5f07e47c8edbb8c2eb257330dd4a0b02955fe507c8e958542ae3ccd85c65e33d49bb92679663f6efc69ff997cfcc052bed223dcda021fe713a0391ff
+  data.tar.gz: 5eeaa8383df1bc7e445c1a29ac1af5f6236cbe97929abc06706d16ab53e97ffc6b1d9d663563c45cfd8e279cd41f80151796f41dcc54a334fe43e79be0e246ef

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+sftp-config*.json
+.remote-sync.json
+.tags

data/Gemfile

@@ -1,5 +1,3 @@
 source 'https://rubygems.org'
-
-# dependcies are defined in gemspec file since to avoid duplication
-# gemspec :name => 'dtk-client'
 gemspec
+gem 'dtk-dsl', :github => 'dtk/dtk-dsl'

data/README.md

@@ -1,121 +1,2 @@
-DTK Client
-==============================
-
-
-#### Description
-
-
-DTK Client is a Ruby based CLI interface for communication with the DTK Server.
-It's main purpose is to provide an easy to use interface for importing modules, browsing modules repository and deploying assemblies and topologies.
-
----
-#### Sytstem requirements
-
-- Ruby 1.9.3
-- Unix OS
-
----
-#### Installation
-
-For DTK Client to work, following steps have to be done:
-
-- Git User set up
-- RSA Keys Generated
-
-To install DTK Client gem, execute:
-
-`gem install dtk-client`
-
----
-#### Initial setup
-
-There are two ways for DTK Client to run
-
-- via an interactive shell (<tt>dtk-shell</tt>)
-- by executing DTK Client commands (ie. <tt>dtk service list</tt>)
-
-If it is the first time that the DTK Client is being used (by either of the ways), following prompts will appear:
-
-```
-Please enter the DTK server address (example: instance.dtk.io)
-Server address:
-
-Please enter your DTK login details
-Username:
-Password:
-
-```
-After entering the correct data, following message will appear:
-
-`SSH key 'dtk-client' added successfully!`
-
-This means that the DTK Server has successfully registered DTK Client and the client is ready for use.
-
----
-#### DTK Client configuration
-
-All of the DTK Configuration, installed component and service modules as well as client logs are located in `~/dtk`.
-
-DTK Client configuration options, such as development option, verbose calls to DTK Server, log options and Client user credentials, can be configured in `~/dtk/client.conf`:
-
-```
-debug_task_frequency=5            # assembly - frequency between requests (seconds)
-auto_commit_changes=false         # autocommit for modules
-verbose_rest_calls=false          # logging of REST calls
-
-module_location=component_modules
-service_location=service_modules
-test_module_location=test_modules
-backups_location=backups
-
-
-server_port=80
-secure_connection_server_port=443
-secure_connection=true
-server_host=instance.dtk.io
-```
-
-
-User credentials are located in `~/dtk/.connection`
-
-Component and Service modules that are installed, or modules that are imported are located in `~/dtk/component_modules` and `~/dtk/service_modules/`
-
----
-
-## Advanced features
-#### Development mode
-
-When using DTK Client in Development mode, gems from `Gemfile_dev` must be installed and used when running DTK Client or DTK Client Shell:
-
-```
-bundle install --gemfile Gemfile_dev
-BUNDLE_GEMFILE=Gemfile_dev bundle exec ruby ./bin/dtk-shell
-```
-
-#### DTK Repoman
-
-DTK Repoman is a Git based repository for publishing and installing component modules and service modules. DTK Repoman has it's own users, known as catalog users. The inital setup should register the DTK Client to the Public Catalog users.
-
-Successfully registered DTK Client on DTK Repoman enables execution of commands such as:
-
-`dtk service-module list --remote` - lists service moduels on remote visible to the catalog user
-
-`dtk component-module install namespace/component-module-name` - install component module
-
-Switching from public user to a commercial user can be done with `dtk account set-catalog-credentials`. This will initate a prompt for catalog username and password
-
-## License
-
-dtk-client is copyright (C) 2010-2016 dtk contributors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this work except in compliance with the License.
-You may obtain a copy of the License in the [LICENSE](LICENSE) file, or at:
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+# dtk-client2
+DTK client

data/Rakefile

@@ -0,0 +1,44 @@
+require 'rake/clean'
+require 'rubygems'
+require 'rubygems/package_task'
+require 'rdoc/task'
+require 'cucumber'
+require 'cucumber/rake/task'
+Rake::RDocTask.new do |rd|
+  rd.main = "README.rdoc"
+  rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
+  rd.title = 'Your application title'
+end
+
+spec = eval(File.read('dtk.gemspec'))
+
+Gem::PackageTask.new(spec) do |pkg|
+end
+CUKE_RESULTS = 'results.html'
+CLEAN << CUKE_RESULTS
+desc 'Run features'
+Cucumber::Rake::Task.new(:features) do |t|
+  opts = "features --format html -o #{CUKE_RESULTS} --format progress -x"
+  opts += " --tags #{ENV['TAGS']}" if ENV['TAGS']
+  t.cucumber_opts =  opts
+  t.fork = false
+end
+
+desc 'Run features tagged as work-in-progress (@wip)'
+Cucumber::Rake::Task.new('features:wip') do |t|
+  tag_opts = ' --tags ~@pending'
+  tag_opts = ' --tags @wip'
+  t.cucumber_opts = "features --format html -o #{CUKE_RESULTS} --format pretty -x -s#{tag_opts}"
+  t.fork = false
+end
+
+task :cucumber => :features
+task 'cucumber:wip' => 'features:wip'
+task :wip => 'features:wip'
+require 'rake/testtask'
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/*_test.rb']
+end
+
+task :default => [:test,:features]

data/bin/dtk

@@ -1,92 +1,14 @@
 #!/usr/bin/env ruby
-#
-# Copyright (C) 2010-2016 dtk contributors
-#
-# This file is part of the dtk project.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# GLOBAL IDENTIFIER
-$shell_mode = false
-
-require File.expand_path('../lib/client', File.dirname(__FILE__))
-require File.expand_path('../lib/configurator', File.dirname(__FILE__))
-require File.expand_path('../lib/parser/adapters/thor', File.dirname(__FILE__))
-require File.expand_path('../lib/shell/context', File.dirname(__FILE__))
-require File.expand_path('../lib/shell/domain/context_entity', File.dirname(__FILE__))
-require File.expand_path('../lib/shell/domain/active_context', File.dirname(__FILE__))
-require File.expand_path('../lib/shell/domain/context_params', File.dirname(__FILE__))
-require File.expand_path('../lib/shell/domain/override_tasks', File.dirname(__FILE__))
-require File.expand_path('../lib/shell/domain/shadow_entity', File.dirname(__FILE__))
-require File.expand_path('../lib/commands/thor/account', File.dirname(__FILE__))
-require File.expand_path('../lib/shell/parse_monkey_patch', File.dirname(__FILE__))
-require File.expand_path('../lib/shell/help_monkey_patch', File.dirname(__FILE__))
-require File.expand_path('../lib/execute/cli_pure/cli_rerouter', File.dirname(__FILE__))
-
-paths = []
-paths << File.expand_path('../lib/commands/thor/*.rb', File.dirname(__FILE__))
-paths << File.expand_path('../lib/commands/common/thor/*.rb', File.dirname(__FILE__))
-
-paths.each do |path|
-  Dir[path].each do |thor_class_file|
-    require thor_class_file
+module DTK
+  require_relative '../lib/dtk_client'
+  module GlobalForDSL
+    Error  = ::DTK::Client::Error
+    ErrorUsage  = ::DTK::Client::Error::Usage
   end
-end
-
-
-
-require 'shellwords'
-require 'json'
-
-$: << "/usr/lib/ruby/1.8/" #TODO: put in to get around path problem in rvm 1.9.2 environment
+  #dtk_dsl must be loaded after GlobalForDSL set
+  require 'dtk_dsl'
+  # lib/dtk_cli must be loaded after lib/client and lib/dsl
+  require_relative '../lib/dtk_cli'
 
-config_exists = ::DTK::Client::Configurator.check_config_exists
-::DTK::Client::Configurator.check_git
-::DTK::Client::Configurator.create_missing_clone_dirs
-
-
-# check if .add_direct_access file exists, if not then add direct access and create .add_direct_access file
-resolve_direct_access(::DTK::Client::Configurator.check_direct_access, config_exists)
-entries = []
-
-if ARGV.size > 0
-  entries = ARGV
-  entries = DTK::Shell::Context.check_for_sym_link(entries)
-  entity_name = entries.shift
-end
-
-# special case for when no params are provided use help method
-if (entity_name == 'help' || entity_name.nil?)
-  entity_name = 'dtk'
-  args = ['help']
-else
-  args = entries
+  Client::CLI::Runner.run(ARGV)
 end
-
-context = DTK::Shell::Context.new(true)
-
-begin
-  if ::DTK::CLIRerouter.is_candidate?(entity_name, args)
-    response_obj = ::DTK::CLIRerouter.new(entity_name, args).run()
-    puts response_obj.to_json
-  else
-    # default execution
-    entity_name, method_name, context_params, thor_options = context.get_dtk_command_parameters(entity_name, args)
-    top_level_execute(entity_name, method_name, context_params, thor_options, false)
-  end
-rescue DTK::Client::DtkError => e
-  DtkLogger.instance.error(e.message, true)
-rescue Exception => e
-  DtkLogger.instance.error_pp(e.message, e.backtrace)
-end

data/dtk-client.gemspec

@@ -1,49 +1,26 @@
-# -*- encoding: utf-8 -*-
-require File.expand_path('../lib/dtk-client/version', __FILE__)
+require File.expand_path('../lib/cli/version', __FILE__)
 
-Gem::Specification.new do |gem|
-  gem.authors       = ["Rich PELAVIN"]
-  gem.email         = ["rich@reactor8.com"]
-  gem.description   = %q{The DTK Client is a command line tool to interact with your DTK Server and DTK Service Catalog instance(s).}
-  gem.summary       = %q{DTK CLI client for DTK server interaction.}
-  gem.homepage      = "https://github.com/rich-reactor8/dtk-client"
-  gem.licenses      = ["Apache-2.0"]
+Gem::Specification.new do |spec| 
+  spec.name        = 'dtk-client'
+  spec.version     = DTK::Client::CLI::VERSION
+  spec.author      = 'Reactor8'
+  spec.email       = 'support@reactor8.com'
+  spec.description = %q{Command line tool to interact with a DTK Server and DTK Service Catalog.}
+  spec.summary     = %q{DTK CLI client for DTK server interaction.}
+  spec.license     = 'Apache 2.0'
+  spec.platform    = Gem::Platform::RUBY
+  spec.required_ruby_version = Gem::Requirement.new('>= 1.9.3')
 
-  gem.files = %w(README.md Gemfile Gemfile_dev dtk-client.gemspec)
-  gem.files += Dir.glob("bin/**/*")
-  gem.files += Dir.glob("lib/**/*")
-  gem.files += Dir.glob("puppet/**/*")
-  gem.files += Dir.glob("spec/**/*")
-  gem.files += Dir.glob("views/**/*")
+  spec.require_paths = ['lib']
+  spec.files =  `git ls-files`.split("\n")
 
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.name          = "dtk-client"
-  gem.require_paths = ["lib"]
-  gem.version       = DtkClient::VERSION
+  spec.executables   = spec.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
 
-  gem.add_dependency 'mime-types','~> 1.25'
-  gem.add_dependency 'bundler','>= 1.2.4'
-  gem.add_dependency 'json_pure' ,'1.7.4'
-  gem.add_dependency 'diff-lcs','1.1.3'
-  gem.add_dependency 'hirb','~> 0.7.0'
-  gem.add_dependency 'thor','~> 0.15.4'
-  gem.add_dependency 'erubis','~> 2.7.0'
-  gem.add_dependency 'dtk-common-core','0.9.0'
-  gem.add_dependency 'git','1.2.9'
-  gem.add_dependency 'colorize','~> 0.5.8'
-  gem.add_dependency 'highline', '1.6.16'
-  gem.add_dependency 'awesome_print', '1.1.0'
-
-  # gem.add_dependency 'rb-readline', '0.5.0'
-  # gem.add_dependency 'activesupport','~> 3.2.12'
-  # gem.add_dependency 'i18n','0.6.1'
-  # gem.add_dependency 'puppet','~> 3.1.0'
-  # gem.add_dependency 'jenkins-client','~> 0.0.1'
-  # gem.add_dependency 'rspec','~> 2.12.0'
-  # gem.add_dependency 'awesome_print','~> 1.1.0'
-  # gem.add_dependency 'rdoc','= 3.12.1'l
-  # gem.add_development_dependency 'ruby-debug','>= 0.10.4'
-  # gem.add_development_dependency 'awesome_print','>= 1.1.0'
-  # gem.add_development_dependency 'rspec','~> 2.12.0'
+  spec.add_dependency 'dtk-common-core','0.10.0'
+  spec.add_dependency 'gli', '2.13.4'
+  spec.add_dependency 'highline', '1.7.8'
+  spec.add_dependency 'colorize', '0.7.7'
+  spec.add_dependency 'git', '1.2.9'
+  spec.add_dependency 'hirb', '0.7.3'
 end

data/examples/simple/dtk.module.yaml

@@ -0,0 +1,32 @@
+---
+dsl_version: 1.0.0
+module: rich/simple
+version: 0.0.2
+keywords: DTK-2554
+description: Tests for DTK-2554
+license: Apache 2.0
+
+dependencies:
+  aws/ec2:
+  dtk/host: 1.0.1
+assemblies:
+  simple:
+    description: Simple assembly for DTK-2554
+    attributes:
+      global_num: 5
+    nodes:
+      n1:
+        attributes:
+          image: amazon_hvm
+          size: micro
+        components:
+        - host::hostname:
+            attributes:
+              hostname: host1
+    workflows:
+      create:
+        subtasks:
+        - name: set hostname
+          components:
+          - host::hostname
+

data/examples/spark/dtk.module.yaml

@@ -0,0 +1,120 @@
+---
+dsl_version: 1.0.0
+module: rich/spark
+keywords: spark, hdfs, zeppelin
+description: Spark cluster
+license: Apache 2.0
+version: 0.1.2
+
+dependencies:
+- bigtop/bigtop_base
+- bigtop/bigtop_multiservice
+- bigtop-new/hadoop
+- bigtop-new/spark
+#- bigtop/zookeeper
+- bigtop/zeppelin
+assemblies:
+  cluster:
+    description: Spark cluster with Zeppelin on trusty
+    components:
+    - bigtop_multiservice
+    - hadoop::cluster:
+        component_links:
+          bigtop_multiservice: bigtop_multiservice
+    - spark::cluster:
+        component_links:
+          bigtop_multiservice: bigtop_multiservice
+    nodes:
+      master:
+        attributes:
+          image: trusty_hvm
+          size: medium
+          root_device_size: 30
+        components:
+        - bigtop_multiservice::hiera:
+            component_links:
+              bigtop_multiservice: bigtop_multiservice
+        - bigtop_base:
+            attributes:
+              with_maven: false
+
+        - hadoop::namenode
+        - hadoop::hdfs_directories
+
+        - spark::master:
+            attributes:
+              eventlog_enabled: true
+            component_links:
+              hadoop::hdfs_directories: master/hadoop::hdfs_directories
+        - spark::client
+
+        - zeppelin::server:
+            attributes:
+              install_mode: deb
+            component_links:
+              spark::master: master/spark::master
+      slaves:
+        attributes:
+          image: trusty_hvm
+          size: medium
+          cardinality: 2
+          type: group
+        components:
+        - bigtop_multiservice::hiera:
+            component_links:
+              bigtop_multiservice: bigtop_multiservice
+        - bigtop_base:
+            attributes:
+              with_maven: false
+
+        - hadoop::common_hdfs:
+            component_links:
+              hadoop::namenode: master/hadoop::namenode
+        - hadoop::datanode
+
+        - spark::common:
+            component_links:
+              spark::master: master/spark::master
+        - spark::worker
+    workflows:
+      create:
+        subtasks:
+        - name: bigtop_multiservice
+          components:
+          - bigtop_multiservice
+        - name: bigtop hiera
+          components:
+          - bigtop_multiservice::hiera
+        - name: bigtop_base
+          components:
+          - bigtop_base
+
+        - name: namenode
+          components:
+          - hadoop::namenode
+        - name: if needed leave safemode
+          actions:
+          - hadoop::namenode.leave_safemode
+        - name: namenode smoke test
+          actions:
+          - hadoop::namenode.smoke_test
+        - name: datanodes
+          ordered_components:
+          - hadoop::common_hdfs
+          - hadoop::datanode
+        - name: hdfs directories for spark
+          component:
+          - hadoop::hdfs_directories
+
+        - name: spark master and client
+          components:
+          - spark::master
+          - spark::client
+        - name: spark workers
+          ordered_components:
+          - spark::common
+          - spark::worker
+
+        - name: zeppelin server
+          components:
+          - zeppelin::server

data/examples/with_repo_content/deploy/puppet/manifests/hostname.pp

@@ -0,0 +1,54 @@
+class host::hostname(
+ $hostname = undef,
+ $append_node_group_index = true
+)
+{
+
+  if $hostname == undef {
+    $base_name = $::dtk_assembly_node_base_name
+  } else {
+    $base_name = $hostname
+  }
+
+  if "${append_node_group_index}" == 'true' and $::dtk_assembly_node_type == 'node_group_member' and $::dtk_assembly_node_index != '' {
+    $new_hostname = "${base_name}-${::dtk_assembly_node_index}"
+  } else {
+    $new_hostname = $base_name
+  }
+
+  file { '/etc/hostname':
+        ensure  => present,
+        content => "${new_hostname}\n"
+  }
+
+  exec { "hostname ${new_hostname}":
+    path => ['/bin']
+  }
+
+  host::hostname::set_hosts{ $new_hostname: }
+} 
+
+define host::hostname::set_hosts() {
+  $new_hostname = $name
+  $new_hostname_with_domain = "${$new_hostname}.${::domain}" 
+
+  if $::hostname != $new_hostname and $::hostname != $new_hostname_with_domain {
+    host { $::hostname:
+      ensure => 'absent'
+    }
+  }
+  if $::fqdn != $new_hostname and $::fqdn != $new_hostname_with_domain and $::hostname != $::fqdn {
+    host { $::fqdn:
+      ensure => 'absent'
+    }
+  }
+  # important for to make $new_hostname_with_domain and not $new_hostname primary
+  host { $new_hostname:
+    ensure => 'absent'
+  } -> 
+  host { $new_hostname_with_domain:
+    ensure       => 'present',
+    host_aliases => [$new_hostname],
+    ip           => $::ipaddress_eth0
+  }
+} 

data/examples/with_repo_content/dtk.module.yaml

@@ -0,0 +1,49 @@
+---
+dsl_version: 1.0.0
+module: rich/simple2
+keywords: DTK-2554
+description: Tests for DTK-2554
+license: Apache 2.0
+
+dependencies:
+- aws/ec2
+
+assemblies:
+  simple:
+    nodes:
+      n1:
+        attributes:
+          image: amazon_hvm
+          size: small
+        components:
+        - simple::hostname:
+            attributes:
+              hostname: host1
+    workflows:
+      create:
+        subtasks:
+        - name: set hostname
+          components:
+          - simple::hostname
+providers:
+  puppet:
+    content: 
+    - from: deploy/puppet
+      to: /etc/puppet/modules/rich
+      # For Puppet 'to:' can be omitted and default would be /etc/puppet/modules/<MODULENAME>
+
+components:
+  hostname:
+    attributes:
+      hostname:
+        type: string
+        required: true
+    actions:
+      create:
+        puppet_class: simple::hostname
+
+  dtk_hostname:
+    description: Use dtk node name as host name
+    actions:
+      create:
+        puppet_class: simple::hostname

data/features/dtk.feature

@@ -0,0 +1,8 @@
+Feature: My bootstrapped app kinda works
+  In order to get going on coding my awesome app
+  I want to have aruba and cucumber setup
+  So I don't have to do it myself
+
+  Scenario: App just runs
+    When I get help for "dtk"
+    Then the exit status should be 0

data/features/step_definitions/dtk_steps.rb

@@ -0,0 +1,6 @@
+When /^I get help for "([^"]*)"$/ do |app_name|
+  @app_name = app_name
+  step %(I run `#{app_name} help`)
+end
+
+# Add more step definitions here

data/features/support/env.rb

@@ -0,0 +1,15 @@
+require 'aruba/cucumber'
+
+ENV['PATH'] = "#{File.expand_path(File.dirname(__FILE__) + '/../../bin')}#{File::PATH_SEPARATOR}#{ENV['PATH']}"
+LIB_DIR = File.join(File.expand_path(File.dirname(__FILE__)),'..','..','lib')
+
+Before do
+  # Using "announce" causes massive warnings on 1.9.2
+  @puts = true
+  @original_rubylib = ENV['RUBYLIB']
+  ENV['RUBYLIB'] = LIB_DIR + File::PATH_SEPARATOR + ENV['RUBYLIB'].to_s
+end
+
+After do
+  ENV['RUBYLIB'] = @original_rubylib
+end