safedb 0.5.1005 → 0.7.1001

data/lib/controller/api/terraform/README.md

@@ -3,8 +3,29 @@
 
 ### safe terraform | introduction
 
-This terraform use case exports the AWS IAM user access key, secret key and region key into (very safe) environment variables and then runs the specified terraform be it **init**, **plan**, **apply** or **destroy**.
+This terraform use case exports the AWS IAM user access key, secret key and region key into (very safe) environment variables and then runs the specified terraform be it **plan**, **apply** or **destroy**.
 
+The plan is to extend this command to directly cache terraform output variables.
+
+### Passing Input Variables
+
+The most powerful feature of **`safe terraform`** is the ability to pass safely stored input variables to terraform via environment variables. The safe exports data when the key
+
+- **either** begins with **`tfvar.`**
+- **or** begins with **`@tfvar.`** (for sensitive values)
+
+### safe input variables examples
+
+| **safe key**                  | **safe value**         | type      | exported env variable | usage                    |
+|:----------------------------- |:---------------------- |:--------- |:--------------------- |: ----------------------- |
+**tfvar.in_vpc_id**             | vpc-1234567890         | string    | TF_VAR_in_vpc_id      | var.in_vpc_id
+**tfvar.in_role_arn**           | arn:aws:iam::98764 ... | string    | TF_VAR_in_role_arn    | var.in_role_arn
+**@tfvar.in_db_password**       | secret-543+210=753     | string    | TF_VAR_in_db_password | var.in_db_password
+**tfvar.in_ingress**            | '[ "ssh", "http" ]'    | list      | TF_VAR_in_ingress     | var.in_ingress
+
+Mostly you pass string, number or boolean input variables to terraform. These examples also show how you can pass list and map variables to terraform.
+
+---
 
 ## safe terraform | credential creation
 
@@ -21,7 +42,7 @@ The first use case is importing the IAM user credentials into safe.
     $ safe put @secret.key 5678uvwx4321abcd9876 # Put IAM secret key in safe
     $ safe put region.key eu-west-1             # infrastructure in Dublin
 
-    safe logout
+    $ safe logout
 
 Take care to specify these 3 key names **@access.key**, **@secret.key**, **region.key** and note that safe's convention is to sensitively treat the value's of keys beginning with an **@** sign. **safe show** and other readers **mask out (redact)** these sensitive values.
 

data/lib/controller/api/terraform/terraform.rb

@@ -14,9 +14,9 @@ module SafeDb
   # ubiquitous safe open command.
   #
   #     safe open <<chapter>> <<verse>>
-  class Terraform < QueryVerse
+  class Terraform < EditVerse
 
-    attr_writer :command
+    attr_writer :command, :debug
 
     # This prefix is tagged onto environment variables which Terraform will read
     # and convert for consumption into module input variables.
@@ -26,15 +26,19 @@ module SafeDb
     # <tt>terraform apply</tt> command, it examines the lines at the opened
     # chapter and verse and any that start with this prefix will be substringed
     # to create an environment variable with the substringed name and key value.
-    ENV_VAR_PREFIX_A = "env-var."
+    ENV_VAR_PREFIX_A = "tfvar."
 
     # Secure var prefix for environment variable key (line). Before safe runs the
     # <tt>terraform apply</tt> command, it examines the lines at the opened
     # chapter and verse and any that start with this prefix will be substringed
     # to create an environment variable with the substringed name and key value.
-    ENV_VAR_PREFIX_B = "@env-var."
+    ENV_VAR_PREFIX_B = "@#{ENV_VAR_PREFIX_A}"
 
-    def query_verse()
+    TIMESTAMP_LINE_KEY = "tfvar.in_timestamp"
+    DESCRIBES_LINE_KEY = "tfvar.in_description"
+
+
+    def edit_verse()
 
       # ############## | ############################################################
       # @todo refactor | ############################################################
@@ -67,10 +71,25 @@ module SafeDb
       puts "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
       puts ""
 
+      command_name = @command ? @command : "apply"
+      is_apply = command_name.eql?( "apply" ) 
+      is_blast = command_name.eql?( "destroy" ) 
+
+      has_timestamp = @verse.has_key?( TIMESTAMP_LINE_KEY )
+      is_create_stamps = is_apply && !has_timestamp
+      is_remove_stamps = is_blast && has_timestamp
+
+      the_description = "was created on #{TimeStamp.readable()}."
+
+      @verse.store( TIMESTAMP_LINE_KEY, TimeStamp.yjjjhhmmsst() ) if is_create_stamps
+      @verse.store( DESCRIBES_LINE_KEY, the_description )       if is_create_stamps
+
       ENV[ "AWS_ACCESS_KEY_ID"     ] = @verse[ "@access.key" ]
       ENV[ "AWS_SECRET_ACCESS_KEY" ] = @verse[ "@secret.key" ]
       ENV[ "AWS_DEFAULT_REGION"    ] = @verse[ "region.key"  ]
 
+      ENV[ "TF_LOG" ] = "DEBUG" if @debug == true
+
       @verse.each do | key_str, value_object |
 
         is_env_var = key_str.start_with?( ENV_VAR_PREFIX_A ) || key_str.start_with?( ENV_VAR_PREFIX_B )
@@ -90,13 +109,25 @@ module SafeDb
       puts ""
 
       auto_approve = @command && @command.eql?( "plan" ) ? "" : "-auto-approve"
-      command_name = @command ? @command : "apply"
-      system "terraform #{command_name} #{auto_approve}"
+      exit_success = system "terraform #{command_name} #{auto_approve}"
 
       puts ""
       puts "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@"
       puts ""
 
+      return if ( exit_success.nil?() || !exit_success )
+
+      @verse.delete( TIMESTAMP_LINE_KEY ) if is_remove_stamps
+      @verse.delete( DESCRIBES_LINE_KEY ) if is_remove_stamps
+
+      return unless is_apply
+
+      puts "Successful terraform apply."
+      graph_filename = "network-#{@book.get_open_verse_name()}-#{TimeStamp.yyjjj_hhmm_sst()}.png"
+      system "terraform graph | dot -Tpng > #{graph_filename}"
+      puts "Resource graph #{graph_filename} created."
+      puts ""
+
     end
 
 

data/lib/controller/{admin → book}/commit.rb

@@ -37,10 +37,9 @@ module SafeDb
 
       end
 
-      StateMigrate.commit( @book )
+      EvolveState.commit( @book )
 
-      puts "The commit was on #{TimeStamp.readable()}\n"
-      puts "Commit from branch to master was successful.\n"
+      puts "Commit at #{TimeStamp.readable()} successful."
       puts ""
 
 

data/lib/controller/{admin → book}/refresh.rb

@@ -39,8 +39,8 @@ module SafeDb
       puts " == Book Mark := #{@book.get_open_chapter_name()}/#{@book.get_open_verse_name()}\n" if @book.is_opened?()
       puts ""
 
-      StateMigrate.refresh( @book )
-      StateMigrate.copy_commit_id_to_branch( @book )
+      EvolveState.refresh( @book )
+      EvolveState.copy_commit_id_to_branch( @book )
 
       puts "Refresh from master to branch was successful.\n"
       puts ""

data/lib/controller/db/pull.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/ruby
+	
+module SafeDb
+
+  # If the removable drive path is configured and exists and contains the master
+  # index file, the pull use case backs up both file and master crypts (if necessary)
+  # and then refreshes them with the state that exists in the remote mirrored git
+  # directory and the indices on the removable drive path.
+  class Pull < Controller
+
+    # If the removable drive path is configured and exists and contains the master
+    # index file, the pull use case backs up both file and master crypts (if necessary)
+    # and then refreshes them with the state that exists in the remote mirrored git
+    # directory and the indices on the removable drive path.
+    def execute()
+
+      puts ""
+
+      removable_drive_path = xxx # ~~~~ read this from the --to variable
+      removable_drive_file = File.join( removable_drive_path, Indices::MASTER_INDICES_FILE_NAME )
+      removable_drive_file_exists = File.exist?( removable_drive_file ) && File.file?( removable_drive_file )
+
+      puts "Removable Drive Location => #{removable_drive_path}"
+      puts "Removable Drive Filepath => #{removable_drive_file}"
+
+      if removable_drive_file_exists
+        drive_filename = TimeStamp.yyjjj_hhmm_sst() + "-" + Indices::MASTER_INDICES_FILE_NAME
+        drive_backup_filepath = File.join( removable_drive_path, drive_filename )
+        File.write( drive_backup_filepath, File.read( removable_drive_file ) )
+        puts "Backup of Clobbered File => #{drive_backup_filepath}"
+      end
+
+      clobbered_crypts_name = TimeStamp.yyjjj_hhmm_sst() + "-" + Indices::MASTER_CRYPTS_FOLDER_NAME
+      clobbered_crypts_path = File.join( Indices::SAFE_DATABASE_FOLDER, clobbered_crypts_name )
+
+      FileUtils.mkdir_p( clobbered_crypts_path )
+      FileUtils.copy_entry( Indices::MASTER_CRYPTS_FOLDER_PATH, clobbered_crypts_path )
+
+
+      puts "Backup of Clobbered Crypts => #{clobbered_crypts_path}"
+
+      is_git = File.exist?( Indices::MASTER_CRYPTS_GIT_PATH ) && File.directory?( Indices::MASTER_CRYPTS_GIT_PATH )
+
+
+
+=begin
+      require "octokit"
+############client = Octokit::Client.new(:login => 'defunkt', :password => 'c0d3b4ssssss!')
+
+client = Octokit::Client.new(:access_token => '')
+user = client.user
+puts "Company Name => #{user[:company]}"
+puts "User Name => #{user[:name]}"
+puts "User ID => #{user[:id]}"
+puts "Email => #{user[:email]}"
+puts "Login => #{user[:login]}"
+puts "Biography => #{user[:bio]}"
+=end
+
+      return
+
+
+    end
+
+
+  end
+
+
+end

data/lib/controller/db/push.rb

@@ -0,0 +1,352 @@
+#!/usr/bin/ruby
+	
+module SafeDb
+
+  # After backing up local assets the <b>push use case</b> creates a remoe github
+  # repository if necessary and initializes the master crypts as a git repository
+  # if necessary and then adds, commits and pushes the crypts up to the github
+  # remote for safe keeping.
+  #
+  # We also remember the commit reference and we add this to the master indices
+  # file before finally backing up, and then updating the master indices file on
+  # the locally accessible removable drive.
+  #
+  # == The First Push
+  #
+  # The first push on a machine
+  #
+  # - writes and secures the private key
+  # - creates an entry within ~/.ssh/config
+  # - does a git init and sets the git remote
+  #
+  # Subsequent pushes will always
+  #
+  # - add and commit to the local repository
+  # - push crypts to the remote repository
+  # - record the commit reference in the safe database tracker file
+  # - copy the database tracker file to the removable drive
+  #
+  class Push < Controller
+
+    # After backing up local assets the <b>push use case</b> creates a remoe github
+    # repository if necessary and initializes the master crypts as a git repository
+    # if necessary and then adds, commits and pushes the crypts up to the github
+    # remote for safe keeping.
+    def execute()
+
+      open_remote_backend_location()
+
+###########
+########### =========================================
+########### Instead of Private Keys Use Tokens  =====
+########### =========================================
+########### 
+########### git remote add origin https://<<GITHUB_TOKEN>>@github.com/<<REPO_USERNAME>>/<<REPO_NAME>>.git
+########### git remote set-url origin https://<<GITHUB_TOKEN>>@github.com/<<REPO_USERNAME>>/<<REPO_NAME>>.git
+########### git push origin master
+########### 
+########### Note ==== Query repository with git remote -v to see if an origin has been set
+###########      ==== If no origin set use the set-url variant otherwise use the add variant
+########### 
+########### ==========================================
+########### For Pulling (Cloning the Repository  =====
+########### ==========================================
+########### 
+########### git clone https://github.com/<<REPO_USERNAME>>/<<REPO_NAME>>.git [[safedb-master-crypts]]
+########### 
+########### 
+########### 
+
+      # @todo ------------------------------------------------------------ >>
+      # @todo REFACTOR the below into lib/utils/keys/keypair.rb
+      # @todo REFACTOR And create a utiliy class for bulk of file Writer functionality
+      # @todo Methods in keypair should NOT know about the Indices constants
+      # @todo Refactor name from [Indices] to [Constants]
+      # @todo ------------------------------------------------------------ >>
+      # @todo Method Names
+      # @todo ------------------------------------------------------------ >>
+      # @todo   (1) - Constants.write_private_key()
+      # @todo ------------------------------------------------------------ >>
+
+      private_key_path = File.join( Indices::SSH_DIRECTORY_PATH, @verse[ Indices::REMOTE_PRIVATE_KEY_KEYNAME ] )
+      private_key_exists = File.file?( private_key_path )
+      puts "private key found at #{private_key_path}" if private_key_exists
+
+      unless private_key_exists
+
+        puts "private key will be created at #{private_key_path}"
+        file_writer = Write.new()
+        file_writer.file_key = Indices::PRIVATE_KEY_DEFAULT_KEY_NAME
+        file_writer.to_dir = Indices::SSH_DIRECTORY_PATH
+        file_writer.flow()
+
+        FileUtils.chmod( 0600, private_key_path, :verbose => true )
+
+      end
+
+      git_username = @verse[ Indices::GIT_REPOSITORY_USER_KEYNAME ]
+      git_reponame = @verse[ Indices::GIT_REPOSITORY_NAME_KEYNAME ]
+
+      ssh_host_name = @verse[ Indices::REMOTE_MIRROR_SSH_HOST_KEYNAME ] 
+      ssh_config_exists = File.file?( Indices::SSH_CONFIG_FILE_PATH )
+      config_file_contents = File.read( Indices::SSH_CONFIG_FILE_PATH ) if ssh_config_exists
+      ssh_config_written = ssh_config_exists && config_file_contents.include?( ssh_host_name )
+      puts "ssh config for host #{ssh_host_name} has already been written" if ssh_config_written
+
+      unless ssh_config_written
+
+        puts "ssh config for host #{ssh_host_name} will be written"
+        config_backup_path = File.join( Indices::SSH_DIRECTORY_PATH, "safe.clobbered.ssh.config-#{TimeStamp.yyjjj_hhmm_sst()}" )
+        File.write( config_backup_path, config_file_contents ) if ssh_config_exists
+        puts "original ssh config at #{config_backup_path}" if ssh_config_exists
+
+        File.open( Indices::SSH_CONFIG_FILE_PATH, "a" ) do |line|
+          line.puts( "\n" )
+          line.puts( "Host #{ ssh_host_name }" )
+          line.puts( "HostName github.com" )
+          line.puts( "User #{ git_username }" )
+          line.puts( "IdentityFile #{ private_key_path }" )
+          line.puts( "StrictHostKeyChecking no" )
+        end
+
+        puts "ssh config has been successfully written"
+
+      end
+
+      puts ""
+
+      ssh_test_cmd_string = "ssh -i #{private_key_path} -vT git@github.com"
+      system( ssh_test_cmd_string )
+      ssh_cmd_exit_status = $?.exitstatus
+
+      unless ssh_cmd_exit_status == 1
+
+        puts ""
+        puts "The command exit status is #{ssh_test_exitstatus}"
+        puts ""
+        puts "### ##### : ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+        puts "### Error : SSH test result did not contain expected string."
+        puts "### Query : #{ ssh_test_cmd_string }"
+        puts "### ##### : ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+        puts ""
+
+        return
+
+      end
+
+      puts ""
+      puts "### ####### : ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+      puts "### Success : The SSH connection test was a roaring success."
+      puts "### Command : #{ ssh_test_cmd_string }"
+      puts "### ####### : ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
+      puts ""
+
+## ==========>>> git rev-parse HEAD
+
+# git init    
+# git clone `URLTORepository`
+# cd `into your cloned folder`
+# git checkout commithash
+
+      puts ""
+      return
+
+=begin
+ssh -i ~/.ssh/safedb.code.private.key.pem -vT git@safedb.code
+git clone https://github.com/devops4me/safedb.net safedb.net
+git remote set-url --push origin git@safedb.code:devops4me/safedb.net.git
+=end
+
+      unless ssh_config_file contains git_reponame
+
+        #write out the SSH private key
+        # @todo change the write method to change the file permissions
+
+# SAFE_PRIVATE_KEY_KEYNAME
+
+      # @todo - Write the chunk of text into .ssh/config file (name is git_reponame)
+      # @todo - the User is git_username
+      # @todo - the IdentityFile is Dir.home() joined to .ssh and User is git_username
+
+      user_host_name = "#{Etc.getlogin()}@#{Socket.gethostname()}"
+      @verse.store( Indices::REMOTE_LAST_PUSH_ON, TimeStamp.readable() )
+      @verse.store( Indices::REMOTE_LAST_PUSH_BY, user_host_name )
+
+      end # end the unless block
+
+
+# --  SAFE_REMOTE_SSH_HOST = "safe.remote"
+# --  SAFE_REMOTE_HOST_NAME = "github.com"
+
+  # @todo - link this to the Keys class to use the same string constant
+# --  SAFE_PRIVATE_KEY_KEYNAME = "private.key"
+
+
+
+
+      # Do a git init if no .git folder found
+      # do git local config (for name and email) if necessary
+      # do git set remote url add
+      # do git add
+      # do git commit
+      # do git push origin master
+
+#      @verse.store( Indices::REMOTE_LAST_PUSH_ID,  )
+   # @todo set git remote url (for push) in the @verse
+   # @todo set git clone url in the @verse
+   # @todo set git commit id in the @verse
+      
+#   @todo now set the git clone url and commit ID in the master index file
+
+      # Make sure git pull --from=/path/to/dir LOGS in and writes the /path/to/dir with KEY as the User@hostname
+
+      ## Now the git push --to=/path/to/this/dir => IF no path read from @verse
+      ## If no verse with user@host path the WRITE to present working directory
+
+
+
+=begin
+
+Setting up passwordless git interactions (cloning, pulling, pushing) is the same as setting up passwordless ssh login.
+
+To interact with Git without passwords you need to
+
+- setup a public private SSH keypair
+- install and lock down the private key
+- create a SSH IdentityFile called config in `$HOME/.ssh/config`
+- install the public key into BitBucket, GitLab, GitHub or a SSH accessible repo
+
+### Setup Passwordless SSH
+
+Passwordless SSH is a prerequisite to passwordless git interaction.
+
+### The SSH Identity File
+
+The Identity File is telling the SSH subsystem that when you see this particular hostname (IP Address) - you submit this private key because that host will for sure have the corresponding public key in its authorized keys cache.
+
+When using Github, Gitlab or BitBucket - you go to a screen and enter in the public key portion.
+
+```
+Host bitbucket.server
+StrictHostKeyChecking no
+HostName bitbucket.org
+User joebloggs276
+IdentityFile /home/joebloggs/.ssh/bitbucket-repo-private-key.pem
+```
+
+### The Passwordless SSH Setup Commands
+
+Our local user `joebloggs` has an account with `bitbucket.org` with username `joebloggs276` and has submitted the public key to it. He has created a private key at `/home/joebloggs/.ssh/bitbucket-repo-private-key.pem` (locked with a 400) and an identity file at `/home/joebloggs/.ssh/config`.
+
+``` bash
+ssh-keygen -t rsa                                              # enter /home/joebloggs/.ssh/bitbucket-repo-private-key.pem
+chmod 400 /home/joebloggs/.ssh/bitbucket-repo-private-key.pem  # restrict to user read-only permissions
+GIT_HOST_IP=bitbucket.org                                      # set the hostname as bitbucket.org
+ssh-keyscan $GIT_HOST_IP >> /home/joebloggs/.ssh/known_hosts   # prevents a authenticity of host cant be established prompt
+ssh -i /home/joebloggs/.ssh/bitbucket-repo-private-key.pem -vT "joebloggs276@$GIT_HOST_IP" # test that all will be okay
+git clone git@bitbucket.org:joeltd/bigdata.git mirror.bigdata  # this clone against bigdata account and repo is bigdata
+```
+
+BITBUCKET_USER=joebloggs276;
+# curl --user ${BITBUCKET_USER} https://api.bitbucket.org/2.0/repositories/joeltd
+curl --user ${BITBUCKET_USER} git@api.bitbucket.org/2.0/repositories/joeltd
+
+
+Note that the clone command uses the bitbucket account called joeltd and the repository is called big_data_scripts.
+
+The response to the SSH test against a bitbucket repository for user
+
+`ssh -i /home/joebloggs/.ssh/bitbucket-repo-private-key.pem -vT "joebloggs276@$GIT_HOST_IP"`
+
+## Setup Git in Existing Directory
+
+To hook up with a new repository from a directory with files you first
+
+- create the remote repository (use safe's github and gitlab tooling)
+- safe will have created a public / private keypair and installed it in the remote repo
+- locally their should be a private key (with 0600 permissions) and an entry in ~/.ssh/config
+- go to the git directory (without a .git folder)
+
+The commands to run
+
+git init
+git add -A
+git status
+git commit -am "First checkin of project."
+git remote add origin git@<<Host>>:<<userOrGroup>>/<<repo-name>>.git
+git remote -v
+git push --set-upstream origin master
+
+=end
+
+
+# @todo -- also see temp-git-code.rb class in this directory
+# @todo -- also see temp-git-code.rb class in this directory
+# @todo -- also see temp-git-code.rb class in this directory
+# @todo -- also see temp-git-code.rb class in this directory
+# @todo -- also see temp-git-code.rb class in this directory
+# @todo -- also see temp-git-code.rb class in this directory
+# @todo -- also see temp-git-code.rb class in this directory
+# @todo -- also see temp-git-code.rb class in this directory
+# @todo -- also see temp-git-code.rb class in this directory
+# @todo -- also see temp-git-code.rb class in this directory
+
+
+puts ""
+the_384_key = OpenSSL::PKey::EC.new('secp384r1')
+the_384_key.generate_key!
+
+puts "#############################"
+puts "the 384 key"
+puts "#############################"
+puts the_384_key.private_key.to_pem()
+puts "#############################"
+puts the_384_key.private_key.export()
+puts "#############################"
+puts the_384_key.public_key.export()
+puts "#############################"
+puts the_384_key.public_key.to_pem()
+puts "#############################"
+puts the_384_key.to_pem()
+puts "#############################"
+puts the_384_key.to_text()
+puts ""
+
+ec_private_key_encoded = Base64.urlsafe_encode64( the_384_key.to_pem() )
+
+puts "Private Key Encoded"
+puts "ec_private_key_encoded"
+puts ""
+return 
+
+return
+      puts ""
+
+      removable_drive_path = xxx # ~~~~ read this from the --to variable
+      removable_drive_file = File.join( removable_drive_path, Indices::MASTER_INDICES_FILE_NAME )
+      removable_drive_file_exists = File.exist?( removable_drive_file ) && File.file?( removable_drive_file )
+
+      puts "Removable Drive Location => #{removable_drive_path}"
+      puts "Removable Drive Filepath => #{removable_drive_file}"
+
+      if removable_drive_file_exists
+        drive_filename = TimeStamp.yyjjj_hhmm_sst() + "-" + Indices::MASTER_INDICES_FILE_NAME
+        drive_backup_filepath = File.join( removable_drive_path, drive_filename )
+        File.write( drive_backup_filepath, File.read( removable_drive_file ) )
+        puts "Backup of Clobbered File => #{drive_backup_filepath}"
+      end
+
+      is_git = File.exist?( Indices::MASTER_CRYPTS_GIT_PATH ) && File.directory?( Indices::MASTER_CRYPTS_GIT_PATH )
+
+
+
+      return
+
+
+    end
+
+
+  end
+
+
+end