vmc_knife 0.0.8 → 0.0.9

data/README.md

@@ -11,4 +11,163 @@ Run the deployment file via a web-interface.
     prepare a vmc deployment file.
     vmc_knife vmc_deployment_file.json
 
+## Installation
+ssh into a Cloudfoundry VM
+load the cloudfoundry profile and install the gem:
 
+    source /home/ubuntu/.cloudfoundry_deployment_local
+    gem install vmc_knife
+
+Or to install from source:
+
+    git clone https://github.com/hmalphettes/vmc-knife.git
+    cd vmc-knife
+    gem build vmc_knife.gemspec
+    gem install vmc_knife
+
+## Example:
+Create a recipe with the mongodb app example and the sinatra example app.
+
+Make a new file example_recipe.json
+
+Enter:
+
+    {
+      "sub_domain": "vcap.me",
+      "target": "api.vcap.me",
+      "email": "vcap@vcap.me",
+      "password": "vcap",
+      "recipes": [
+        {
+          "name": "example_recipe",
+          "data_services": {
+            "mongo1": {
+              "name": "a_mongo",
+              "vendor": "mongodb"
+            }
+          },
+          "applications": {
+            "example_mongo": {
+              "name": "mongo_db_demo",
+              "uris": [
+                "mongodb-on-cf-demo.#{this['sub_domain']}"
+              ],
+              "staging": {
+                "stack": "ruby19",
+                "model": "rails3"
+              },
+              "resources": {
+                "memory": 256
+              },
+              "services": [
+                "#{this['recipes'][0]['data_services']['mongo1']['name']}"
+              ],
+              "env": [
+                "DERIVED_VALUE_EXAMPLE=http://#{this['recipes'][0]['applications']['example_mongo']['uris'][0]}"
+              ],
+              "repository": {
+                "url": "https://github.com/mccrory/cloud-foundry-mongodb-demo.git",
+                "branch":"master"
+              }
+            }
+          }
+        }
+      ]
+    }
+
+Navigate to the folder where the recipe is located.
+And use vmc_knife:
+
+    vmc_knife login
+    vmc_knife configure-apps
+    vmc_knife upload-apps
+    vmc_knife start-apps
+
+The console will look like this:
+
+    ubuntu@ubuntu:~/tmp$ vmc_knife configure-apps
+    Applications selected mongo_db_demo
+    Data-services selected a_mongo
+    {
+      "applications": {
+        "mongo_db_demo": {
+          "name": "Create mongo_db_demo",
+          "services": {
+            "add": [
+              "a_mongo"
+            ]
+          },
+          "env": {
+            "add": [
+              "DERIVED_VALUE_EXAMPLE=http://mongodb-on-cf-demo.vcap.me"
+            ]
+          },
+          "uris": {
+            "add": [
+              "mongodb-on-cf-demo.vcap.me"
+            ]
+          },
+          "memory": " => 256"
+        }
+      }
+    }
+    Creating mongo_db_demo with {:name=>"mongo_db_demo", "resources"=>{"memory"=>256}, "staging"=>{"model"=>"rails3", "stack"=>"ruby19"}, "uris"=>["mongodb-on-cf-demo.vcap.me"], "services"=>["a_mongo"], "env"=>["DERIVED_VALUE_EXAMPLE=http://mongodb-on-cf-demo.vcap.me"]}
+
+    ubuntu@ubuntu:~/tmp$ vmc_knife upload-apps
+    Applications selected mongo_db_demo
+    Data-services selected a_mongo
+    Dir.entries(/home/ubuntu/vmc_knife_downloads/mongo_db_demo).size 2
+    remote: Counting objects: 85, done.
+    remote: Compressing objects: 100% (65/65), done.
+    remote: Total 85 (delta 4), reused 84 (delta 4)
+    Unpacking objects: 100% (85/85), done.
+    fatal: Not a git repository (or any of the parent directories): .git
+    Uploading Application mongo_db_demo from /home/ubuntu/vmc_knife_downloads/mongo_db_demo:
+    Copying the files
+    Done copying the files
+      Checking for available resources: About to compute the fingerprints
+    Finished computing the fingerprints
+    Invoking check_resources with the fingerprints
+    OK
+      Processing resources: OK
+      Packing application: OK
+      Uploading (255K): client.upload_app about to start
+      Uploading (255K): OK   
+    Done client.upload_app
+    Push Status: OK
+
+
+
+### Updating an app:
+
+For example edit the memory parameter of the app. Then call:
+
+    vmc_knife configure-apps
+    vmc_knife restart-apps example_mongo
+
+Note that vmc_knife's start/stop/restart only sends the command to vcap's cloud_controller.
+It does not try to poll it to see if the command was successful.
+
+Delete data-services and apps in the recipe:
+
+    vmc_knife delete-all
+
+
+### Accessing the data-services:
+Assuming that vmc_knife is able to locate the cloud_controller.yml and mongo binary:
+
+    vmc_knife data-shell mongo1
+
+will drop the user to the mongo shell:
+
+    ubuntu@ubuntu:~/tmp$ vmc_knife data-shell mongo1
+    Applications selected mongo_db_demo
+    Data-services selected a_mongo
+    Executing /home/ubuntu/cloudfoundry/.deployments/intalio_devbox/deploy/mongodb/bin/mongo -u 6e409825-31f6-4a47-8eef-11f49e763a9a -p 153c52d6-26ab-467b-9c1d-bc1b071a5c86 192.168.0.138:25001/db
+    MongoDB shell version: 1.8.1
+    connecting to: 192.168.0.138:25001/db
+    >
+
+With postgresql export and import are supported.
+
+Todo: take advantage of the new vmc-tunnel.

data/lib/vmc_knife/cli_extensions.rb

@@ -168,6 +168,14 @@ class VMC::Cli::KnifeRunner < VMC::Cli::Runner
       else
         set_cmd(:knifeapps, :data_drop, @args.size) # too many
       end
+    when 'data-shrink'
+      usage('vmc_knife data-shrink [<data-service-name>] [<tables-collection-regexp>]')
+      @args.shift # consumes the argument.
+      if @args.size <= 3
+        set_cmd(:knifeapps, :data_shrink, @args.size)
+      else
+        set_cmd(:knifeapps, :data_shrink, @args.size) # too many
+      end
     when 'data-import'
       usage('vmc_knife data-import [<data-service-name>] [<tables-collection-regexp>]')
       @args.shift # consumes the argument.
@@ -185,7 +193,7 @@ class VMC::Cli::KnifeRunner < VMC::Cli::Runner
         set_cmd(:knifeapps, :data_export, @args.size) # too many
       end
     when 'help'
-      display "vmc_knife expand-manifest|login|start/stop/restart-apps|upload-apps|configure-all|configure-recipes|configure-apps|configure-services|delete-all|configure-vcap|configure-vcap-mdns|configure-vcap-etc-hosts|data-shell [<manifest_path>]"
+      display "vmc_knife expand-manifest|login|start/stop/restart-apps|upload-apps|configure-all|configure-recipes|configure-apps|configure-services|delete-all|configure-vcap|configure-vcap-mdns|configure-vcap-etc-hosts|data-shell|data-export/import/shrink/drop [<manifest_path>]"
     else
       super
     end

data/lib/vmc_knife/commands/knife_cmds.rb

@@ -5,8 +5,10 @@ module VMC::KNIFE::Cli
   #loads the manifest file.
   #when the path is not specified, look in the current directory.
   #when the path is a directory, look for the first json file it can find.
+  #if it still find nothing then use the default which is the value of the environment variable VMC_KNIFE_DEFAULT_RECIPE
   #the json file actually loaded is set as the attribute @manifest_path
   def load_manifest(manifest_file_path=nil)
+    was_nil = true if manifest_file_path.nil?
     manifest_file_path = Dir.pwd if manifest_file_path.nil?
     if File.directory? manifest_file_path
       #look for the first .json file if possible that is not an expanded.json
@@ -17,7 +19,12 @@ module VMC::KNIFE::Cli
         end
         return VMC::KNIFE::JSON_EXPANDER.expand_json @manifest_path
       end
-      raise "Unable to find a *.json file in #{manifest_file_path}"
+      if was_nil && !ENV['VMC_KNIFE_DEFAULT_RECIPE'].nil?
+        raise "Can't load the default recipe VMC_KNIFE_DEFAULT_RECIPE=#{ENV['VMC_KNIFE_DEFAULT_RECIPE']}" unless File.exists? ENV['VMC_KNIFE_DEFAULT_RECIPE']
+        load_manifest ENV['VMC_KNIFE_DEFAULT_RECIPE']
+      else
+        raise "Unable to find a *.json file in #{manifest_file_path}"
+      end
     else
       @manifest_path = manifest_file_path
       return VMC::KNIFE::JSON_EXPANDER.expand_json @manifest_path
@@ -171,10 +178,12 @@ module VMC::Cli::Command
     include VMC::KNIFE::Cli
 
     def configure_applications(app_names_regexp=nil,manifest_file_path=nil)
-      configure(nil,nil,app_names_regexp,manifest_file_path)
+      configure(nil,nil,app_names_regexp,manifest_file_path,
+                        {:apps_only=>true})
     end
     def configure_services(services_names_regexp=nil,manifest_file_path=nil)
-      configure(nil,nil,services_names_regexp,manifest_file_path)
+      configure(nil,nil,services_names_regexp,manifest_file_path,
+                        {:data_only=>true})
     end
     def configure_recipes(recipe_names_regexp=nil,manifest_file_path=nil)
       configure(recipe_names_regexp,nil,nil,manifest_file_path)
@@ -199,40 +208,51 @@ module VMC::Cli::Command
     end
     
     def upload_applications(app_names_regexp=nil,manifest_file_path=nil)
-      recipe_configuror(:upload,nil,app_names_regexp,nil,manifest_file_path)
+      recipe_configuror(:upload,nil,app_names_regexp,nil,manifest_file_path,
+                        {:apps_only=>true})
     end
     def start_applications(app_names_regexp=nil,manifest_file_path=nil)
-      recipe_configuror(:start,nil,app_names_regexp,nil,manifest_file_path)
+      recipe_configuror(:start,nil,app_names_regexp,nil,manifest_file_path,
+                        {:apps_only=>true})
     end
     def stop_applications(app_names_regexp=nil,manifest_file_path=nil)
-      recipe_configuror(:stop,nil,app_names_regexp,nil,manifest_file_path)
+      recipe_configuror(:stop,nil,app_names_regexp,nil,manifest_file_path,
+                        {:apps_only=>true})
     end
     def restart_applications(app_names_regexp=nil,manifest_file_path=nil)
-      recipe_configuror(:restart,nil,app_names_regexp,nil,manifest_file_path)
+      recipe_configuror(:restart,nil,app_names_regexp,nil,manifest_file_path,
+                        {:apps_only=>true})
     end
     def delete_all(app_names_regexp=nil,manifest_file_path=nil)
       recipe_configuror(:delete,nil,app_names_regexp,nil,manifest_file_path)
     end
     def data_shell(data_names_regexp=nil,manifest_file_path=nil)
-      recipe_configuror(:shell,nil,nil,data_names_regexp,manifest_file_path)
+      recipe_configuror(:shell,nil,nil,data_names_regexp,manifest_file_path,
+                        {:data_only=>true})
     end
     def data_credentials(data_names_regexp=nil,manifest_file_path=nil)
-      recipe_configuror(:credentials,nil,nil,data_names_regexp,manifest_file_path)
+      recipe_configuror(:credentials,nil,nil,data_names_regexp,manifest_file_path,
+                        {:data_only=>true})
     end
     def data_apply_privileges(data_names_regexp=nil,manifest_file_path=nil)
-      recipe_configuror(:apply_privileges,nil,nil,data_names_regexp,manifest_file_path)
+      recipe_configuror(:apply_privileges,nil,nil,data_names_regexp,manifest_file_path,
+                        {:data_only=>true})
     end
     def data_import(data_names_regexp=nil,app_name=nil,file_names=nil,manifest_file_path=nil)
       recipe_configuror(:import,nil,nil,data_names_regexp,manifest_file_path,
-                        {:file_names=>file_names, :app_name=>app_name})
+                        {:file_names=>file_names, :app_name=>app_name, :data_only=>true})
     end
     def data_export(data_names_regexp=nil,app_name=nil,file_names=nil,manifest_file_path=nil)
       recipe_configuror(:export,nil,nil,data_names_regexp,manifest_file_path,
-                        {:file_names=>file_names, :app_name=>app_name})
+                        {:file_names=>file_names, :app_name=>app_name, :data_only=>true})
     end
     def data_drop(data_names_regexp=nil,collection_or_table_names=nil,manifest_file_path=nil)
       recipe_configuror(:drop,nil,nil,data_names_regexp,manifest_file_path,
-                        {:collection_or_table_names=>collection_or_table_names})
+                        {:collection_or_table_names=>collection_or_table_names, :data_only=>true})
+    end
+    def data_shrink(data_names_regexp=nil,collection_or_table_names=nil,manifest_file_path=nil)
+      recipe_configuror(:shrink,nil,nil,data_names_regexp,manifest_file_path,
+                        {:collection_or_table_names=>collection_or_table_names, :data_only=>true})
     end
     
     def recipe_configuror(method_sym_name,recipes_regexp=nil,app_names_regexp=nil,service_names_regexp=nil,manifest_file_path=nil,opts=nil)

data/lib/vmc_knife/data_services.rb

@@ -2,6 +2,8 @@ require 'yaml'
 require "interact"
 require 'tempfile'
 require 'tmpdir'
+require 'pathname'
+require 'erb'
 
 module VMC
   module KNIFE
@@ -10,16 +12,21 @@ module VMC
     PSQL_RAW_RES_ARGS="-P format=unaligned -P footer=off -P tuples_only=on"
 
     # Reads the cloud_controller config file for the connection parameters to ccdb.
-    def self.get_ccdb_credentials(ccdb_yml_path="#{ENV['HOME']}/cloudfoundry/config/cloud_controller.yml", db_type='production')
+    def self.get_ccdb_credentials(ccdb_yml_path="#{ENV['CLOUD_FOUNDRY_CONFIG_PATH']}/cloud_controller.yml", db_type='production')
       cc_conf = File.open( ccdb_yml_path ) do |yf| YAML::load( yf ) end
       db = cc_conf['database_environment'][db_type]
       db
     end
     
-    def self.get_postgresql_node_credentials(postgresql_node_yml_path="#{ENV['HOME']}/cloudfoundry/config/postgresql_node.yml")
+    def self.get_postgresql_node_credentials(postgresql_node_yml_path="#{ENV['CLOUD_FOUNDRY_CONFIG_PATH']}/postgresql_node.yml")
       db = File.open( postgresql_node_yml_path ) do |yf| YAML::load( yf ) end
       db['postgresql']
     end
+    
+    def self.get_mongodb_node_config(mongodb_node_yml_path="#{ENV['CLOUD_FOUNDRY_CONFIG_PATH']}/mongodb_node.yml")
+      db = File.open( mongodb_node_yml_path ) do |yf| YAML::load( yf ) end
+      db
+    end
         
     def self.get_app_id(app_name)
        db=get_ccdb_credentials()
@@ -96,21 +103,31 @@ module VMC
     end
     
     # command_files or command.
-    def self.data_service_console(credentials_hash, commands_file="",as_admin=false)
+    def self.data_service_console(credentials_hash, commands_file="",as_admin=false,exec_name=nil)
       if credentials_hash['db'] #so far it has always been equal to 'db'
         # It is a mongo service
         #/home/ubuntu/cloudfoundry/.deployments/intalio_devbox/deploy/mongodb/bin/mongo 127.0.0.1:25003/db 
         #-u c417f26c-6f49-4dd5-a208-216107279c7a -p 8ab08355-6509-48d5-974f-27c853b842f5
         # Todo: compute the mongoshell path (?)
-        mongo_shell=find_mongo_exec()
-        cmd = "#{mongo_shell} -u #{credentials_hash['username']} -p #{credentials_hash['password']} #{credentials_hash['hostname']}:#{credentials_hash['port']}/#{credentials_hash['db']}"
+        exec_name ||= 'mongo'
+        mongo_shell=get_mongo_exec(exec_name)
+        if exec_name == 'mongo'
+          db_arg = "/#{credentials_hash['db']}"
+        elsif exec_name == 'mongodump'
+          db_arg = "" # dump all the databases including 'admin' which contains the users.
+        else
+          db_arg = "--db #{credentials_hash['db']}"
+        end
+        cmd = "#{mongo_shell} -u #{credentials_hash['username']} -p #{credentials_hash['password']} #{credentials_hash['hostname']}:#{credentials_hash['port']}#{db_arg}"
         puts "Executing #{cmd}"
         if commands_file
-          if File.exists? commands_file
-            # not supported yet.
-            commands_file = "--eval \"#{`cat commands_file`}"
-          else
-            commands_file = "--eval \"#{commands_file}\""
+          if mongo_shell == 'mongo'
+            if File.exists? commands_file
+              # not supported yet.
+              commands_file = "--eval \"#{`cat commands_file`}"
+            else
+              commands_file = "--eval \"#{commands_file}\""
+            end
           end
           `#{cmd} #{commands_file}`
         else
@@ -133,16 +150,24 @@ module VMC
       end
     end
     
-    def self.find_mongo_exec()
-      mongo=`which mongo`
-      return mongo unless mongo.nil? || mongo.empty?
-      mongo_files = Dir.glob("#{ENV['HOME']}/cloudfoundry/.deployments/*/deploy/mongodb/bin/mongo")
-      mongo_files.first unless mongo_files.empty?
+    def self.get_mongo_exec(exec_name=nil)
+      exec_name||='mongo'
+      mongo_bin_folder=File.dirname(get_mongodb_node_config()['mongod_path'])
+      File.join(mongo_bin_folder,exec_name)
+    end
+    
+    # Returns the path to the mongodb db files. /var/vcap/services/mongodb/
+    def self.get_mongodb_base_dir(mongodb_node_yml_path=nil)
+      mongodb_node_yml_path||="#{ENV['CLOUD_FOUNDRY_CONFIG_PATH']}/mongodb_node.yml"
+      db = File.open( mongodb_node_yml_path ) do |yf| YAML::load( yf ) end
+      base_dir = db['base_dir']
     end
     
     def self.as_regexp(arg)
       if arg != nil && arg.kind_of?(String) && !arg.strip.empty?
         Regexp.new(arg)
+      elsif arg.kind_of?(Regexp)
+        arg
       end
     end
     
@@ -183,6 +208,12 @@ module VMC
           data_service.drop(collection_or_table_names)
         end
       end
+      def shrink()
+        collection_or_table_names = @opts[:collection_or_table_names] if @opts
+        @data_services.each do |data_service|
+          data_service.shrink(collection_or_table_names)
+        end
+      end
       
     end
     
@@ -212,73 +243,103 @@ module VMC
           raise "Unable to locate the database file to import." if files.empty?
           file = files.first
         end
-        is_tmp = false
-        current_wd = Dir.pwd
-        begin
+        
+
+        tmp_download_filename="_download_.zip"
+        data_download_dir="#{ENV['HOME']}/vmc_knife_downloads/data_#{@wrapped['name']}"
+        current_wd=Dir.pwd
+        FileUtils.mkdir_p data_download_dir
+        Dir.chdir(data_download_dir) do
           if file =~ /^https?:\/\// || file =~ /^ftp:\/\//
-            url = file
-            is_tmp = true
-            if file =~ /[^\/]*$/
-              basename = $0
-            end
-            tempfile = Tempfile.new("import_db_#{basename}")
-            wget_args = @application_json['repository']['wget_args']
-            if wget_args.nil?
-              wget_args_str = ""
-            elsif wget_args.kind_of? Array
-              wget_args_str = wget_args.join(' ')
-            elsif wget_args.kind_of? String
-              wget_args_str = wget_args
+            basename = Pathname.new(URI.parse(file).path).basename.to_s
+          else
+            file=File.expand_path(file,current_wd)
+            basename = File.basename(file).to_s
+          end
+          if Dir.entries(Dir.pwd).size == 2
+            if file =~ /^https?:\/\// || file =~ /^ftp:\/\//
+              wget_args = @wrapped['director']['wget_args']
+              if wget_args.nil?
+                wget_args_str = ""
+              elsif wget_args.kind_of? Array
+                wget_args_str = wget_args.join(' ')
+              elsif wget_args.kind_of? String
+                wget_args_str = wget_args
+              end
+              `wget #{wget_args_str} --output-document=#{basename} #{file}`
+              if $? != 0
+                `rm #{data_download_dir}/#{basename}`
+                raise "Unable to successfully download #{file}"
+              end
+            else
+              `cp #{file} #{basename}`
             end
-            `wget #{wget_args_str} --output-document=#{tempfile.path} #{url}`
-            file = tempfile.path
           end
           #unzip if necessary (in progress)
-          
-          if /\.tgz$/ =~ file || /\.tar\.gz$/ =~ file
-            tmp_dir = Dir.mktmpdir
-            Dir.chdir(tmp_dir.path)
-            `tar zxvf #{file.path}`
-          elsif /\.tar$/ =~ file
-            tmp_dir = Dir.mktmpdir
-            Dir.chdir(tmp_dir.path)
-            `tar xvf #{file}`
-          elsif /\.zip$/ =~ file
-            tmp_dir = Dir.mktmpdir
-            Dir.chdir(tmp_dir.path)
-            `unzip #{file}`
+          is_unzipped=true
+          p "unzip #{basename}"
+          if /\.tgz$/ =~ basename || /\.tar\.gz$/ =~ basename
+            `tar zxvf #{basename}`
+          elsif /\.tar$/ =~ basename
+            `tar xvf #{basename}`
+          elsif /\.zip$/ =~ basename
+            `unzip #{basename}`
+          else
+            is_unzipped=false
           end
-          if tmp_dir
-            `rm #{file}`
+          
+          if is_unzipped
+            #`rm #{basename}`
             files = Dir.glob("*.sql") if is_postgresql
             files = Dir.glob("*.bson") if is_mongodb
             files ||= Dir.glob("*")
             raise "Can't find the db-dump file." if files.empty?
             file = files.first
+          else
+            file = basename
           end
-          
+            
           if is_postgresql
+            p "chmod o+w #{file}"
             `chmod o+w #{file}`
-            #TODO:
+            creds=credentials(app_name)
             if /\.sql$/ =~ file
               other_params="--file #{file} --quiet"
-              cmd = VMC::KNIFE.pg_connect_cmd(credentials(app_name), 'psql',as_admin=false, other_params)
+              cmd = VMC::KNIFE.pg_connect_cmd(creds, 'psql',as_admin=false, other_params)
               #`psql --dbname #{dbname} --file #{file} --clean --quiet --username #{rolename}`
             else
               other_params="--clean --no-acl --no-privileges --no-owner #{file}"
-              cmd = VMC::KNIFE.pg_connect_cmd(credentials(app_name), 'pg_restore',false, other_params)
+              cmd = VMC::KNIFE.pg_connect_cmd(creds, 'pg_restore',false, other_params)
               #`pg_restore --dbname=#{dbname} --username=#{username} --no-acl --no-privileges --no-owner #{file}`
             end
             puts cmd
             puts `#{cmd}`
             `chmod o-w #{file}`
+          elsif is_mongodb
+            
+            # see if we go through the filesystem to shrink or
+            # if we are only interested in the data itself.
+            base_dir=VMC::KNIFE.get_mongodb_base_dir()
+            instance_name=creds['name']
+            dbpath=File.join(base_dir, instance_name, 'data')            
+            mongod_lock=File.join(dbpath,'mongo.lock')
+            
+            if File.exists?(mongod_lock) && File.size(mongod_lock)>0
+              # the mongodb instance is currently working. connect to it and do the work.
+              # in that case import the 'db' alone. don't do the 'admin'
+              VMC::KNIFE.data_service_console(creds, File.dirname(file),false,'mongorestore')
+            else
+              # the mongodb instance is not currently working
+              # go directly on the filesystem
+              `rm -rf #{dbpath}`
+              `mkdir -p #{dbpath}`
+              #sudo mongorestore --dbpath /var/lib/mongodb
+              mongorestore_exec=VMC::KNIFE.get_mongo_exec('mongorestore')
+              `#{mongorestore_exec} --dbpath #{dbpath} #{File.dirname(File.dirname(file))}`
+            end
           else
-            raise "Unsupported type of data-service. Postgresql is the only supported service at the moment."
+            raise "Unsupported type of data-service. Postgresql and mongodb are the only supported services at the moment."
           end
-        ensure
-          file.unlink if is_tmp  # deletes the temp file
-          tmp_dir.unlink if tmp_dir  # deletes the temp directory
-          Dir.chdir(current_wd)
         end
       end
       
@@ -286,19 +347,55 @@ module VMC
         if is_postgresql
           if file.nil?
             extension = @wrapped['director']['file_extension'] if @wrapped['director']
-            extension = is_mongodb() ? "bson" : "sql"
+            extension ||= "sql"
             file = "#{name()}.#{extension}"
           end
           `touch #{file}`
           `chmod o+w #{file}`
           puts "Exports the database #{credentials(app_name)['name']} in #{file}"
-          
-          #sudo -u postgres env PGPASSWORD=$PGPASSWORD dbname=$DBNAME DUMPFILE=$DUMPFILE pg_dump --format=p --file=$DUMPFILE --no-owner --clean --blobs --no-acl --oid --no-tablespaces $DBNAME
+          #sudo -u postgres env PGPASSWORD=intalio DBNAME=intalio DUMPFILE=intalio_dump.sql pg_dump --format=p --file=$DUMPFILE --no-owner --clean --blobs --no-acl --oid --no-tablespaces $DBNAME
+          #sudo -u postgres env PGPASSWORD=$PGPASSWORD DUMPFILE=$DUMPFILE pg_dump --format=p --file=$DUMPFILE --no-owner --clean --blobs --no-acl --oid --no-tablespaces $DBNAME
 
           cmd = VMC::KNIFE.pg_connect_cmd(credentials(app_name), 'pg_dump', false, "--format=p --file=#{file} --no-owner --clean --oids --blobs --no-acl --no-privileges --no-tablespaces")
           puts cmd
           puts `#{cmd}`
           `chmod o-w #{file}`
+        elsif is_mongodb
+          if file.nil?
+            extension = @wrapped['director']['file_extension'] if @wrapped['director']
+            extension ||= "bson.tar.gz"
+            file = "#{name()}.#{extension}"
+          end
+          creds=credentials(app_name)
+          puts "Exports the database #{creds['name']} in #{file}"
+          #mongodump --host localhost:27017
+          mongodump_exec=VMC::KNIFE.get_mongo_exec('mongodump')
+          # see if we go through the filesystem or through the network:
+          base_dir=VMC::KNIFE.get_mongodb_base_dir()
+          instance_name=creds['name']
+          dbpath=File.join(base_dir, instance_name, 'data')            
+          mongod_lock=File.join(dbpath,'mongo.lock')
+          if File.exists?(mongod_lock) && File.size(mongod_lock)>0
+            cmd = "#{mongodump_exec} -u #{credentials_hash['username']} -p #{credentials_hash['password']} --host #{credentials_hash['hostname']}:#{credentials_hash['port']}"
+          else
+            cmd = "#{mongodump_exec} --dbpath #{dbpath}"
+          end
+          puts cmd
+          puts `#{cmd}`
+          
+          # this produces a dump folder in the working directory.
+          # let's zip it:
+          if /\.zip$/ =~ extension
+            # just zip
+            `zip -r #{file} dump/`
+          elsif /\.tar$/ =~ extension
+            # just tar
+            `tar cvf #{file} dump/`
+          else
+            # tar-gzip by default
+            `tar czvf #{file} dump/`
+          end
+          `rm -rf dump`
         end
       end
       
@@ -309,7 +406,11 @@ module VMC
       def is_mongodb()
         credentials()['db'] != nil
       end
-                                                                                                                                                      
+      
+      # Make sure that all users who can connect to the DB can also access
+      # the tables.
+      # This workarounds the privilege issue reported here:
+      # 
       def apply_privileges()
         if is_postgresql()
           cmd_acl="GRANT CREATE ON SCHEMA PUBLIC TO PUBLIC;\
@@ -320,6 +421,32 @@ module VMC
         end
       end
       
+      # shrink the size of the databses on the file system.
+      # Specifically act on the mongodb instances when they are stopped.
+      def shrink(collection_or_table_names=nil)
+        return unless is_mongodb
+        creds=credentials()
+        base_dir=VMC::KNIFE.get_mongodb_base_dir()
+        instance_name=creds['name']
+        dbpath=File.join(base_dir, instance_name, 'data')            
+        mongod_lock=File.join(dbpath,'mongo.lock')
+        raise "Can't shrink #{name}; the mongodb is currently running" if File.exists?(mongod_lock) && File.size(mongod_lock)>0
+        mongodump_exec=VMC::KNIFE.get_mongo_exec('mongodump')
+        raise "Can't find mongodump" unless File.exist? mongodump_exec
+        mongorestore_exec=VMC::KNIFE.get_mongo_exec('mongorestore')
+        raise "Can't find mongorestore" unless File.exist? mongorestore_exec
+        cmd = "#{mongodump_exec} --dbpath #{dbpath}"
+        puts "#{cmd}"
+        puts `#{cmd}`
+        
+        `rm -rf #{dbpath}`
+        `mkdir #{dbpath}`
+        cmd = "#{mongorestore_exec} --dbpath #{dbpath} dump/"
+        puts "#{cmd}"
+        puts `#{cmd}`
+        `rm -rf dump`
+      end
+      
       def drop(collection_or_table_names=nil)
         if is_postgresql
           sel_tables = "SELECT table_name FROM information_schema.tables WHERE table_schema='public'"
@@ -351,8 +478,35 @@ module VMC
           puts cmd
           puts `#{cmd}`
         elsif is_mongodb
+          # generate the command file from the erb template.
+          filter = ".*"
+          filterIsNegated = "false";
+          skipSystem = "true";
+          if collection_or_table_names
+            if collection_or_table_names.start_with?('!')
+              filterIsNegated = "true";
+              filter = collection_or_table_names[1..-1]
+            else
+              filter = collection_or_table_names
+            end
+          end
+          # this command is applied to each collection.
+          # the name of the variable is 'collection' as can bee seen in the erb file.
+          cmd="collection.drop();"
+          
+          file = Tempfile.new('dropcollections')
+          begin
+            File.open(file.path, 'w') do |f2|
+              template = ERB.new File.new("#{VMCKNIFE::ROOT_REL}/vmc_knife/mongo/mongo_cmd.js.erb").read, nil, "%"
+              f2.puts template.result(binding)
+            end
+            puts shell(file.path)
+          ensure
+            file.unlink   # deletes the temp file
+          end
+          
           #TODO: iterate over the collections and drop them according to the filter.
-          raise "TODO: Unsupported operation 'drop' for the data-service #{name()}"
+          #raise "TODO: Unsupported operation 'drop' for the data-service #{name()}"
         else
           puts "Unsupported operation 'drop' for the data-service #{name()}"
         end

data/lib/vmc_knife/json_expander.rb

@@ -10,7 +10,8 @@ module VMC
       def self.ip_auto(interface='eth0')
         res=`ifconfig | sed -n '/#{interface}/{n;p;}' | grep 'inet addr:' | grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $1}' | head -1`
         if interface == 'eth0' && (res.nil? || res.strip.empty?)
-          res = VcapUtilities.ip_auto "wlan0"
+          res = ip_auto "wlan0"
+          res = res[0] if res
           if res.strip.empty?
             #nevermind fetch the first IP you can find that is not 127.0.0.1
             res=`ifconfig | grep 'inet addr:' | grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $1}' | head -1`
@@ -20,10 +21,10 @@ module VMC
         unless res.empty?
           # gets the Mask
           line=`ifconfig | grep 'inet addr:#{res}' | awk '{ print $1}' | head -1`
-          puts "parsing ip and mask in line #{line}"
+#          puts "parsing ip and mask in line #{line}"
           mask=`ifconfig | grep 'inet addr:#{res}' | grep -v '127.0.0.1' | cut -d: -f4 | awk '{ print $1}' | head -1`
           mask.strip!
-          puts "got ip #{res} and mask #{mask}"
+#          puts "got ip #{res} and mask #{mask}"
           return [ res, mask ]
         end
       end
@@ -50,16 +51,16 @@ module VMC
       # Makes up to 10 passes evaluating ruby in the values that contain #{}.
       def self.expand_json(file_path)
         raise "The file #{file_path} does not exist" unless File.exists? file_path
-        data = JSON.parse File.open(file_path).read
+        data = File.open(file_path, "r") do |infile| JSON.parse infile.read end
         #puts "got data #{data.to_json}"
         passes = 0
-        while passes < 100
+        while passes < 150
           #puts "pass #{passes}"
           break unless expand_data(data,data)
           passes += 1
         end
-        puts data.to_json unless passes < 100
-        raise "More than 100 passes evaluating the ruby template in the json file" unless passes < 100
+        puts data.to_json unless passes < 150
+        raise "More than 100 passes evaluating the ruby template in the json file current state #{JSON.pretty_generate data}" unless passes < 100
         #puts "got data #{data.to_json}"
         
         data
@@ -90,22 +91,52 @@ module VMC
           end
         elsif current.kind_of? Array
           index = 0
+          current_evalled = Array.new
+          do_flatten=false
           current.each do | v |
             if v.kind_of? String
               if /\#{.+}/ =~ v
                 at_least_one_eval = true
                 begin
-                  evalled = eval_v(v,data,current)
-                  current[index] = evalled unless evalled.nil?
+                  evalled_ret = eval_v(v,data,current)
+                  if evalled_ret.kind_of? Array
+                    # we choose to support our use cases to flatten the arrays.
+                    # never mind the lists of list for now.
+                    do_flatten = true
+                    evalled_ret.each do |nitem|
+                      current_evalled[index] = nitem
+                      index+=1
+                    end
+                  elsif evalled_ret.nil? # skip the nil for the next pass
+                    current_evalled[index] = v
+                  else
+                    current_evalled[index] = evalled_ret
+                  end
                 rescue => e
-                  raise "Error thrown evaluating #{v}: #{e.inspect}"
+                  raise "Error thrown evaluating #{v}; current state #{JSON.pretty_generate data}: #{e.inspect}"
                 end
+              else
+                current_evalled[index] = v
               end
-            else
-              at_least_one_eval ||= expand_data(data,v)
+#            else
+#              at_least_one_eval ||= expand_data(data,v)
             end
             index+=1
           end
+          
+          # we don't support list of lists if there is a doubt when running the eval.
+          if at_least_one_eval
+            current_evalled.flatten! if do_flatten
+            current.clear
+            current.concat current_evalled
+          end
+          
+          current.each do | v |
+            unless v.kind_of? String
+              at_least_one_eval ||= expand_data(data,v)
+            end
+          end
+          
         end
         at_least_one_eval
       end
@@ -113,16 +144,26 @@ module VMC
       # internal eval a reference.
       # the reference is always wrapped in a json string.
       # however if it is purely a ruby script ("#{ruby here}" ) we unwrap it
-      # to avoid casting the result into a string.
-      def self.eval_v(v,data,current)
-        #puts "evalling #{v}"
+      def self.eval_v(v,data,current,recurse=0)
         if /^\#{([^}]*)}$/ =~ v
           val = $1
         else
           val = '"'+v+'"'
         end
-        evalled = eval(val,get_binding(data,current))
-        #puts "evaluating #{v} => #{evalled} class #{evalled.class.name}"
+          module_eval <<-"END"
+  def self.eval_block_from_template(this,current)
+    #{val}
+  end
+END
+        evalled = eval_block_from_template(data,current)
+        return nil if evalled.nil?
+        if evalled.kind_of?(String) && /\#{([^}]*)}/ =~ evalled
+          if recurse < 20
+            evalled = eval_v(evalled,data,current,recurse+1)
+          else
+            return nil
+          end
+        end
         evalled
       end
       

data/lib/vmc_knife/mongo/mongo_cmd.js.erb

@@ -0,0 +1,24 @@
+// Applies a command to all collections that name matches a particular filter
+var skipSystem = <%= skipSystem %>;
+var filter = /<%= filter %>/;
+var filterIsNegated = <%= filterIsNegated %>;
+var collectionNames = db.getCollectionNames();
+for (var i = 0; i < collectionNames.length; i++ ) {
+  var name = collectionNames[i];
+  if (!(/^system/).test(name) || !skipSystem) {
+    var collection = null;
+    if (filter.test(name)) {
+      if (!filterIsNegated) {
+        collection=db.getCollection(name);
+      }
+    } else {
+      if (filterIsNegated) {
+        collection=db.getCollection(name);
+      }
+    }
+    if (collection) {
+       print("Applying to " + collection.getName());
+       <%= cmd %>
+    }
+  }
+}

data/lib/vmc_knife/vmc_knife.rb

@@ -23,9 +23,6 @@ module VMC
       def sub_domain()
         @wrapped['sub_domain']
       end
-      def target()
-        @wrapped['target']
-      end
       def user()
         @wrapped['user']
       end
@@ -211,8 +208,10 @@ module VMC
           @applications = @applications + recipe.applications(application_sel)
           @data_services = @data_services + recipe.data_services(service_sel)
         end
-        app_names = @applications.collect {|app| app.name }
-        service_names = @data_services.collect {|service| service.name }
+        app_names = @applications.collect {|app| app.name } unless @opts && @opts[:data_only]
+        service_names = @data_services.collect {|service| service.name } unless @opts && @opts[:apps_only]
+        app_names ||= Array.new
+        service_names ||= Array.new
         if app_names.empty? && service_names.empty?
           puts "No applications and data-services were selected."
         else
@@ -431,10 +430,12 @@ module VMC
         return unless @application_json['repository']
         url = @application_json['repository']['url']
         Dir.chdir(ENV['HOME']) do
-          FileUtils.mkdir_p "vmc_knife_downloads/#{@application_json['name']}"
-          Dir.chdir("vmc_knife_downloads/#{@application_json['name']}") do
+          tmp_download_filename="_download_.zip"
+          app_download_dir="vmc_knife_downloads/#{@application_json['name']}"
+          `rm -rf #{app_download_dir}` if File.exist? "#{app_download_dir}/#{tmp_download_filename}"
+          FileUtils.mkdir_p app_download_dir
+          Dir.chdir(app_download_dir) do
             if Dir.entries(Dir.pwd).size == 2
-              puts "Dir.entries(#{Dir.pwd}).size #{Dir.entries(Dir.pwd).size}"
               #empty directory.
               if /\.git$/ =~ url
                 `git clone #{url} --depth 1`
@@ -442,28 +443,31 @@ module VMC
                 if branch
                   `git checkout #{branch}`
                 else
-                  branch=master
+                  branch='master'
                 end
                 `git pull origin #{branch}`
               else
-                wget_args = @application_json['repository']['wget_args']
-                if wget_args.nil?
-                  wget_args_str = ""
-                elsif wget_args.kind_of? Array
-                  wget_args_str = wget_args.join(' ')
-                elsif wget_args.kind_of? String
-                  wget_args_str = wget_args
-                end
-                `wget #{wget_args_str} --output-document=_download_.zip #{url}`
-                raise "Unable to download #{url}" unless $? == 0
-                if /\.tgz$/ =~ url || /\.tar\.gz$/ =~ url
-                  `tar zxvf _download_.zip`
-                elsif /\.tar$/ =~ url
-                  `tar xvf _download_.zip`
-                else
-                  `unzip _download_.zip`
+                begin
+                  wget_args = @application_json['repository']['wget_args']
+                  if wget_args.nil?
+                    wget_args_str = ""
+                  elsif wget_args.kind_of? Array
+                    wget_args_str = wget_args.join(' ')
+                  elsif wget_args.kind_of? String
+                    wget_args_str = wget_args
+                  end
+                  `wget #{wget_args_str} --output-document=#{tmp_download_filename} #{url}`
+                  raise "Unable to download #{url}" unless $? == 0
+                  if /\.tgz$/ =~ url || /\.tar\.gz$/ =~ url
+                    `tar zxvf #{tmp_download_filename}`
+                  elsif /\.tar$/ =~ url
+                    `tar xvf #{tmp_download_filename}`
+                  else
+                    `unzip #{tmp_download_filename}`
+                  end
+                ensure
+                  `rm #{tmp_download_filename}`
                 end
-                `rm _download_.zip`
               end
             end
             Dir.chdir(@application_json['repository']['sub_dir']) if @application_json['repository']['sub_dir']
@@ -531,23 +535,30 @@ module VMC
       end
       
       def update_name_pending()
-        if current[:name].nil?
+        curr_name=current[:name] || current['name']
+        if curr_name.nil?
           return "Create #{@application_json['name']}"
         end
-        if @application_json['name'] != @current[:name]
-          return "#{@current[:name]} => #{@application_json['name']}"
+        if @application_json['name'] != curr_name
+          return "#{curr_name} => #{@application_json['name']}"
         end
       end
       def update_memory_pending()
-        old_mem = current[:resources][:memory] unless current[:resources].nil?
+        old_mem = current[:resources][:memory] if current[:resources]
+        old_mem ||= current['resources']['memory'] if current['resources']
         new_mem = @application_json['resources']['memory'] unless @application_json['resources'].nil?
         if old_mem != new_mem
           return "#{old_mem} => #{new_mem}"
         end
       end
       def update_staging_pending()
-        old_model = current[:staging][:model] unless current[:staging].nil?
-        old_stack = current[:staging][:stack] unless current[:staging].nil?
+        if current[:staging] # sym style: real vmc.
+          old_model = current[:staging][:model]
+          old_stack = current[:staging][:stack]
+        elsif current['staging'] # string style: testing.
+          old_model = current['staging']['model']
+          old_stack = current['staging']['stack']
+        end
         new_model = @application_json['staging']['model'] unless @application_json['staging'].nil?
         new_stack = @application_json['staging']['stack'] unless @application_json['staging'].nil?
         if old_model != new_model
@@ -562,17 +573,17 @@ module VMC
         return { "stack" => stack_change, "model" => model_change }
       end
       def update_services_pending()
-        old_services = current[:services]
+        old_services = current[:services] || current['services']
         new_services = @application_json['services']
         diff_lists(old_services,new_services)
       end
       def update_env_pending()
-        old_services = current[:env]
+        old_services = current[:env] || current['env']
         new_services = @application_json['env']
         diff_lists(old_services,new_services)
       end
       def update_uris_pending()
-        old_services = current[:uris]
+        old_services = current[:uris] || current['uris']
         new_services = @application_json['uris']
         diff_lists(old_services,new_services)
       end
@@ -729,6 +740,7 @@ module VMC
     end
     
     # This is really a server-side feature.
+    # This is deprecated; use the dns_publisher vcap module instead.
     # Regenerates the urls to publish as aliases.
     # use vmc apps to read the uris of each app and also the manifest.
     class VCAPUpdateAvahiAliases
@@ -789,14 +801,18 @@ module VMC
       end
       def execute()
         return unless update_pending()
-        File.open(@config, "w") do |file|
-          all_uris().each do |uri|
-            file.puts uri + "\n"
+        # if the dns_publisher module is present let's not do this one.
+        # the dns_publisher does a better job at mdns
+        unless File.exist? "#{ENV['CLOUD_FOUNDRY_CONFIG_PATH']}/dns_publisher.yml"
+          File.open(@config, "w") do |file|
+            all_uris().each do |uri|
+              file.puts uri + "\n"
+            end
           end
+          #configured so that we don't need root privileges on /etc/avahi/aliases:
+          #the backticks don't work; system() works:
+          system('avahi-publish-aliases') if @do_exec
         end
-        #configured so that we don't need root privileges on /etc/avahi/aliases:
-        #the backticks don't work; system() works:
-        system('avahi-publish-aliases') if @do_exec
       end
       def update_pending()
         already = already_published_uris()

data/lib/vmc_knife.rb

@@ -1,23 +1,23 @@
-module VMCKNIFE; end
-
-ROOT_REL = File.expand_path(File.dirname(__FILE__))
-require "#{ROOT_REL}/restclient/restclient_add_timeout.rb"
+module VMCKNIFE
+  ROOT_REL = File.expand_path(File.dirname(__FILE__))
+end
+require "#{VMCKNIFE::ROOT_REL}/restclient/restclient_add_timeout.rb"
 
 
 module VMC
   module Cli
     module Command
-      autoload :Knife,         "#{ROOT_REL}/vmc_knife/commands/knife_cmds"
-      autoload :Knifeapps,         "#{ROOT_REL}/vmc_knife/commands/knife_cmds"
-      autoload :Knifemisc,         "#{ROOT_REL}/vmc_knife/commands/knife_cmds"
+      autoload :Knife,         "#{VMCKNIFE::ROOT_REL}/vmc_knife/commands/knife_cmds"
+      autoload :Knifeapps,         "#{VMCKNIFE::ROOT_REL}/vmc_knife/commands/knife_cmds"
+      autoload :Knifemisc,         "#{VMCKNIFE::ROOT_REL}/vmc_knife/commands/knife_cmds"
     end
   end
 end
 
 
-require "#{ROOT_REL}/vmc_knife/json_expander"
-require "#{ROOT_REL}/vmc_knife/vmc_helper"
-require "#{ROOT_REL}/vmc_knife/vmc_knife"
-require "#{ROOT_REL}/vmc_knife/data_services"
+require "#{VMCKNIFE::ROOT_REL}/vmc_knife/json_expander"
+require "#{VMCKNIFE::ROOT_REL}/vmc_knife/vmc_helper"
+require "#{VMCKNIFE::ROOT_REL}/vmc_knife/vmc_knife"
+require "#{VMCKNIFE::ROOT_REL}/vmc_knife/data_services"
 
-require "#{ROOT_REL}/vmc_knife/cli_extensions"
+require "#{VMCKNIFE::ROOT_REL}/vmc_knife/cli_extensions"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: vmc_knife
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 13
   prerelease: 
   segments: 
   - 0
   - 0
-  - 8
-  version: 0.0.8
+  - 9
+  version: 0.0.9
 platform: ruby
 authors: 
 - Intalio Pte
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-12-12 00:00:00 Z
+date: 2012-01-11 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: vmc
@@ -108,6 +108,7 @@ files:
 - lib/vmc_knife/commands/knife_cmds.rb
 - lib/vmc_knife/data_services.rb
 - lib/vmc_knife/json_expander.rb
+- lib/vmc_knife/mongo/mongo_cmd.js.erb
 - lib/vmc_knife/version.rb
 - lib/vmc_knife/vmc_helper.rb
 - lib/vmc_knife/vmc_knife.rb