safedb 0.5.1005 → 0.7.1001

data/lib/manual/remote.md

@@ -0,0 +1,62 @@
+
+#### create a remote backend store from which you can `safe push` and `safe pull` your encrypted database assets.
+
+# provision a github repository backend
+
+We want to provision (create) the safe's remote (github) backend so that we can access it from different machines. This is how we provision a Github remote backend do it.
+
+### safe remote --provision
+
+Before you use safe remote to provision a github git repository backend for your crypts ensure that
+
+- you have a github account (with username)
+- you have created a (40 character hexadecimal) github access token
+- you have inserted this data at a suitable chapter and verse
+- you run **`safe remote --provision`** from this opened book/chapter/verse
+
+Below is the sequence of commands leading up to **`safe remote --provision`**
+
+```
+safe init db.admin
+safe login db.admin
+safe open remote.backend github
+safe put @github.token 43210fedcba43210fedcba43210fedcba43210fedcba
+safe configure backend db.admin/remote.backend/github
+safe remote --provision
+```
+
+## focus | safe put @github.token
+
+**@todo** - *we need to refactor out this business of configuring the backend. This should be done **automagically** by the remote commaand expecting to be at the correct book/chapter/verse*
+
+safe knows how to talk to the Github Rest API as long as you provide a github access token. [Visit how to acquire a Github access token](https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line).
+
+## safe configure @github.token
+
+The **`safe configure`** command tell's safe which book, chapter and verse (in our case db.admin/remote.backend/github) that contains the backend repository access properties.
+
+## safe remote --provision
+
+A number of setup tasks are executed when you ask that the backend repository be created.
+
+- a repository is created in github
+- the git fetch (https) and git push (ssh) urls are fabricated
+- the fetch url is written into the master keys file
+- the push url is written to the configured chapter/verse location
+- a ssh public/private keypair (using EC25519) is created
+- the private and public keys are placed within the chapter/verse
+- the public (deploy) key is registered with the github repository
+
+Now you are ready to push and pull.
+
+Visit the [safe push](push-pull) and [safe pull](push-pull) documentation to discover howto use your safe database on as many machines as you need.
+
+
+## where is the safe database?
+
+A safe database is always encrypted at rest and consiss of just 3 simple parts
+
+- **backend** - a set of encrypted files kept either in **`~/.safedb.net/safedb-master-crypts`** or in a git repository
+- **frontend** - a single state tracking file called <tt>safedb-dataase-tracker.ini</tt> (usually) on a removable drive
+- one password - known by the database owner allowing them to login and access the information held within the database
+

data/lib/model/coordinates.rb

@@ -0,0 +1,59 @@
+#!/usr/bin/ruby
+
+module SafeDb
+
+  # Coordinates point to either a book, or a book/chapter or a book/chapter/verse
+  # location. The location pointed to may or may not exist and even if they do they
+  # may not be accessible if they exist within a book that (as of yet) we have not
+  # logged into.
+  class Coordinates
+
+
+    # Initialize coordinates to a location within a book and/or chapter and/or verse.
+    #
+    # @param coordinates_str [String]
+    #    this parameter should be a book/chapter/verse separated
+    #    by forward slashes.
+    def initialize( coordinates_str )
+
+      KeyError.not_new( coordinates_str, self )
+      @coords_list = @coords_list.split( "/" )
+      bcv_error_msg = "Invalid / separated book chapter and verse coordinates ~> #{@coords_list}"
+      raise ArgumentError.new( bcv_error_msg ) unless @coords_list.length() == 3
+      
+      @book_name = @coords_list[ 0 ].strip()
+      @chapter_name = @coords_list[ 1 ].strip()
+      @verse_name = @coords_list[ 2 ].strip()
+
+      log.info(x) { "Initializing a book chapter and verse coordinate within book [#{@book_name}]." }
+
+    end
+
+
+    # Do the (book, chapter or verse) that our said coordinates point
+    # to exist.
+    def exists?()
+    end
+
+
+    # Do our coordinates point to an existing verse in a chapter within
+    # the current logged in book.
+    def is_verse?()
+    end
+
+
+    # Do our coordinates point to an existing chapter within the currently
+    # logged in book.
+    def is_chapter?()
+    end
+
+
+    # Do our coordinates point to any currently logged in book.
+    def is_book?()
+    end
+
+
+  end
+
+
+end

data/lib/model/{safe_tree.rb → file_tree.rb}

@@ -22,9 +22,12 @@ module SafeDb
     end
 
 
+    # Get the path to the folder that holds the master crypts for the
+    # book ID specified in the parameter.
+    # @param book_id [String] the identifier of the book in question
+    # @return [File] path to the master crypts folder for the book
     def self.master_crypts_folder( book_id )
-      master_crypts_folder = File.join( Indices::SAFE_DATABASE_FOLDER, Indices::MASTER_CRYPTS_FOLDER_NAME )
-      return File.join( master_crypts_folder, "safedb.book.#{book_id}" )
+      return File.join( Indices::MASTER_CRYPTS_FOLDER_PATH, "safedb.book.#{book_id}" )
     end
 
 
@@ -34,14 +37,16 @@ module SafeDb
 
 
     def self.branch_crypts_folder( book_id, branch_id )
-      branch_crypts_folder = File.join( Indices::SAFE_DATABASE_FOLDER, Indices::BRANCH_CRYPTS_FOLDER_NAME  )
-      return File.join( branch_crypts_folder, "safedb-branch-#{book_id}-#{branch_id}" )
+      return File.join( Indices::BRANCH_CRYPTS_FOLDER_PATH, "safedb-branch-#{book_id}-#{branch_id}" )
     end
 
 
+    # Get the path to the branch indices file for the branch ID
+    # specified in the parameter.
+    # @param branch_id [String] the identifier of the branch in question
+    # @return [File] path to the branch indices file for the given branch
     def self.branch_indices_filepath( branch_id )
-      branch_indices_folder = File.join( Indices::SAFE_DATABASE_FOLDER, Indices::BRANCH_INDICES_FOLDER_NAME )
-      return File.join( branch_indices_folder, "safedb-indices-#{branch_id}.ini" )
+      return File.join( Indices::BRANCH_INDICES_FOLDER_PATH, "safedb-indices-#{branch_id}.ini" )
     end
 
 

data/lib/model/indices.rb

@@ -16,26 +16,119 @@ module SafeDb
     # The desired length of a safe book ergonomic identifier.
     SAFE_BOOK_ID_LENGTH = 12
 
+    # The file-system location of the safe database tree
+    SAFE_DATABASE_FOLDER = File.join( Dir.home, ".#{SAFE_URL_NAME}" )
+
     # The fully qualified domain name of the safedb home website
     SAFE_GEM_WEBSITE = "https://www.#{SAFE_URL_NAME}"
 
     # The safe database github clonable url for the ruby software
     SAFE_GITHUB_URL = "https://github.com/devops4me/#{SAFE_URL_NAME}"
 
-    # The name ofthe master crypts folder.
+    # The name of the master crypts folder.
     MASTER_CRYPTS_FOLDER_NAME = "safedb-master-crypts"
 
-    # The name ofthe branch indices folder.
-    BRANCH_INDICES_FOLDER_NAME = "safedb-branch-indices"
+    # The path to the master crypts folder.
+    MASTER_CRYPTS_FOLDER_PATH = File.join( SAFE_DATABASE_FOLDER, MASTER_CRYPTS_FOLDER_NAME )
+
+    # The path to the master crypts .git directory.
+    MASTER_CRYPTS_GIT_PATH = File.join( MASTER_CRYPTS_FOLDER_PATH, ".git" )
+
+    # The name of the branch indices folder.
+    BRANCH_INDICES_FOLDER_NAME = "safedb-branch-keys"
+
+    # The path to the branch indices folder.
+    BRANCH_INDICES_FOLDER_PATH = File.join( SAFE_DATABASE_FOLDER, BRANCH_INDICES_FOLDER_NAME )
 
-    # The name ofthe branch crypts folder.
+    # The name of the branch crypts folder.
     BRANCH_CRYPTS_FOLDER_NAME = "safedb-branch-crypts"
 
-    # The file-system location of the safe database tree
-    SAFE_DATABASE_FOLDER = File.join( Dir.home, ".#{SAFE_URL_NAME}" )
+    # The path to the branch crypts folder.
+    BRANCH_CRYPTS_FOLDER_PATH = File.join( SAFE_DATABASE_FOLDER, BRANCH_CRYPTS_FOLDER_NAME )
+
+    # The master indices file name
+    MASTER_INDICES_FILE_NAME = "safedb-master-keys.ini"
 
     # The path to the master indices file
-    MASTER_INDICES_FILEPATH = File.join( SAFE_DATABASE_FOLDER, "safedb-master-indices.ini" )
+    MASTER_INDICES_FILEPATH = File.join( SAFE_DATABASE_FOLDER, MASTER_INDICES_FILE_NAME )
+
+    # The path to the remote storage configuration INI file
+    MACHINE_CONFIG_FILEPATH = File.join( SAFE_DATABASE_FOLDER, "safedb-remote-storage.ini" )
+
+    # The name of the machine removable drive path location directive
+    MACHINE_REMOVABLE_DRIVE_PATH = "removable.drive"
+
+
+
+    # The keyname whose value denotes a local folder path to clone to
+    GIT_CLONE_BASE_PATH = "git.clone.base.path"
+
+
+
+
+    # The name of the keys section that holds remote mirror properties
+    REMOTE_MIRROR_SECTION_NAME = "remote.mirror"
+
+    # The name of the property that points to the book/chapter/verse (page)
+    REMOTE_MIRROR_PAGE_NAME = "remote.mirror.page"
+
+    # The ending of the private key filename for remote mirror push access
+    REMOTE_MIRROR_PRIVATE_KEY_POSTFIX = "private-key.pem"
+
+    # The key name that holds the remote mirror private key filename
+    REMOTE_PRIVATE_KEY_KEYNAME = "private.key.filename"
+
+    # The key name that holds the remote mirror ssh config host value
+    REMOTE_MIRROR_SSH_HOST_KEYNAME = "ssh.config.host"
+
+    # The path to the SSH directory
+    SSH_DIRECTORY_PATH = File.join( Dir.home(), ".ssh" )
+
+    # The path to the SSH config file
+    SSH_CONFIG_FILE_PATH = File.join( SSH_DIRECTORY_PATH, "config" )
+
+
+
+
+
+    # This access token allows us to talk to the Github API
+    GITHUB_ACCESS_TOKEN = "@github.access.token"
+
+    # Github repository keyname
+    GIT_REPOSITORY_NAME_KEYNAME = "repository.name"
+
+    # Github Username Keyname
+    GIT_REPOSITORY_USER_KEYNAME = "repository.user"
+
+    # Github Host Keyname
+    GIT_REPOSITORY_HOST_KEYNAME = "repository.host"
+
+
+
+
+    # Keyname for when the last backend push occured
+    REMOTE_LAST_PUSH_ON = "last.push.on"
+
+    # Keyname for the user and hostname that evoked the last push
+    REMOTE_LAST_PUSH_BY = "last.push.by"
+
+    # Keyname for the ID of the last push (Usually Git Commit Reference)
+    REMOTE_LAST_PUSH_ID = "last.push.id"
+
+    # Private Key Default Key Name
+    PRIVATE_KEY_DEFAULT_KEY_NAME = "private.key"
+
+    # Public Key Default Key Name
+    PUBLIC_KEY_DEFAULT_KEY_NAME = "public.key"
+
+    # The parameter key name to configure the backend coordinates
+    CONFIGURE_BACKEND_KEY_NAME = "backend"
+
+    # The name of the remote database git pull url key
+    REMOTE_DATABASE_GIT_PULL_URL = "git.pull.url"
+
+    # The name of the remote database git push url key
+    REMOTE_DATABASE_GIT_PUSH_URL = "git.push.url"
 
     # The desired length of a content identifier
     CONTENT_ID_LENGTH  = 14
@@ -90,7 +183,7 @@ module SafeDb
 
     # Character (randomly) repeated to mask credentials
     # Asterices, hyphens, plus and equal signs are common alternatives.
-    SECRET_MASK_STRING = "*" * rand( 7 .. 17 )
+    SECRET_MASK_STRING = "*" * rand( 18 .. 30 )
 
     # The birthday (initialization time) of this safe book.
     SAFE_BOOK_INITIALIZE_TIME = "book.init.time"
@@ -125,6 +218,18 @@ module SafeDb
     # Handle to the simple name of the ingested file in the submap verse
     INGESTED_FILE_BASE_NAME_KEY = "file.name"
 
+    # The permission setting (chmod) key name
+    FILE_CHMOD_PERMISSIONS_KEY = "file.access"
+
+    # The keypair name prefix for private keys.
+    PRIVATE_KEY_PREFIX = "private.key"
+
+    # The keypair name prefix for public keys.
+    PUBLIC_KEY_PREFIX = "public.key"
+
+    # Elliptic Curve SSL Key Type
+    ELLIPTIC_CURVE_KEY_TYPE = "secp384r1"
+
 
   end
 

data/lib/model/master.rb

@@ -0,0 +1,40 @@
+#!/usr/bin/ruby
+
+module SafeDb
+
+  # This master data structure controls the key publicly visible properties for
+  # the safe database be it in a local or remote location.
+  #
+  # This object mapper seals away details of the persistence engine involved.
+  # Who knows, it could be a local drive, an S3 bucket and even a database.
+  class Master
+
+    # Initialize an instance of this safe database's master properties.
+    def initialize()
+      @master = DataMap.new( Indices::MASTER_INDICES_FILEPATH )
+    end
+
+
+    # Get the coordinates (book, chapter and verse) of the verse that holds
+    # the remote backend properties. An exception will be thrown if no backend
+    # coordinates have been set.
+    # @return [String] the backend coordinates to set
+    def get_backend_coordinates()
+      @master.use( Indices::REMOTE_MIRROR_SECTION_NAME )
+      return @master.get( Indices::REMOTE_MIRROR_PAGE_NAME )
+    end
+
+
+    # Set the coordinates (book, chapter and verse) of the verse that holds
+    # the remote backend properties.
+    # @param backend_coordinates [String] the backend coordinates to set
+    def set_backend_coordinates( backend_coordinates )
+      @master.use( Indices::REMOTE_MIRROR_SECTION_NAME )
+      @master.set( Indices::REMOTE_MIRROR_PAGE_NAME, backend_coordinates )
+    end
+
+
+  end
+
+
+end

data/lib/model/{state.migrate.rb → state_evolve.rb}

@@ -10,7 +10,7 @@ module SafeDb
   # - <tt>commit</tt> - transfers state from branch to master
   # - <tt>refresh</tt> - transfers state from master to branch
   #
-  class StateMigrate
+  class EvolveState
 
     # The login process recycles the content encryption key by regenerating the human
     # key from the password text and salts and then accessing the old crypt key, generating
@@ -42,7 +42,7 @@ module SafeDb
     #    used for the asymmetric decryption of the content ciphertext which is in a
     #    file marked with the content identifier also within the book keys.
     #
-    # @param secret [String]
+    # @param human_secret [String]
     #    the secret text that can potentially be cryptographically weak (low entropy).
     #    This text is severely strengthened and morphed into a key using multiple key
     #    derivation functions like <b>PBKDF2, BCrypt</b> and <b>SCrypt</b>.
@@ -54,22 +54,30 @@ module SafeDb
     #    The key ring only stores the salts. This means the secret text based key can
     #    only be regenerated at the next login, which explains the inter-branch label.
     #
-    def self.login( book_keys, secret )
+    # @return [Boolean]
+    #    return false if failure decrypting with human password occurs.
+    #    True is returned if the login logic completes naturally.
+    def self.login( book_keys, human_secret )
 
       the_book_id = book_keys.section()
 
-      old_human_key = KdfApi.regenerate_from_salts( secret, book_keys )
+      old_human_key = KdfApi.regenerate_from_salts( human_secret, book_keys )
+      is_correct_password = old_human_key.can_decrypt_key( book_keys.get( Indices::CRYPT_CIPHER_TEXT ) )
+      return false unless is_correct_password
+
       the_crypt_key = old_human_key.do_decrypt_key( book_keys.get( Indices::CRYPT_CIPHER_TEXT ) )
       plain_content = Content.unlock_master( the_crypt_key, book_keys )
 
       first_login_since_boot = StateInspect.is_first_login?( book_keys )
       the_crypt_key = Key.from_random if first_login_since_boot
-      recycle_keys( the_crypt_key, the_book_id, secret, book_keys, plain_content )
+      recycle_keys( the_crypt_key, the_book_id, human_secret, book_keys, plain_content )
       set_bootup_id( book_keys ) if first_login_since_boot
 
       branch_id = Identifier.derive_branch_id( Branch.to_token() )
       clone_book_into_branch( the_book_id, branch_id, book_keys, the_crypt_key )
 
+      return true
+
     end
 
 

data/lib/model/{state.inspect.rb → state_query.rb}

@@ -2,7 +2,7 @@
 
 module SafeDb
 
-  # State queries are related to {StateMigrate} but they simple ask for information
+  # State queries are related to {EvolveState} but they simple ask for information
   # about the state without changing any state.
   #
   class StateInspect
@@ -42,6 +42,9 @@ module SafeDb
       branch_key_ciphertext = branch_keys.get( Indices::CRYPT_CIPHER_TEXT )
       branch_key = KeyDerivation.regenerate_shell_key( Branch.to_token() )
 
+      return branch_key.can_decrypt_key( branch_key_ciphertext )
+
+=begin
       begin
         branch_key.do_decrypt_key( branch_key_ciphertext )
         return true
@@ -50,6 +53,7 @@ module SafeDb
         log.warn(x) { "Login failure error message is #{e.message}" }
         return false
       end
+=end
 
     end
 

data/lib/plugin/github.rb

@@ -0,0 +1,53 @@
+#!/usr/bin/ruby
+
+module SafeDb
+
+  # This class knows how to talk to Github and callers can delegate common
+  # github functionality like creating repositories, listing repositories,
+  # downloading repositories and the like.
+  #
+  class Github
+
+    # Initialize an instance of this safe database's master properties.
+    def initialize()
+
+
+
+
+      #
+      #
+      #  Test the Github api using curl
+      #
+      #
+      #       curl -H "Authorization: token OAUTH-TOKEN" https://api.github.com
+      #
+      #
+
+
+      @master = DataMap.new( Indices::MASTER_INDICES_FILEPATH )
+    end
+
+
+    # Get the coordinates (book, chapter and verse) of the verse that holds
+    # the remote backend properties. An exception will be thrown if no backend
+    # coordinates have been set.
+    # @return [String] the backend coordinates to set
+    def get_backend_coordinates()
+      @master.use( Indices::REMOTE_MIRROR_SECTION_NAME )
+      return @master.get( Indices::REMOTE_MIRROR_PAGE_NAME )
+    end
+
+
+    # Set the coordinates (book, chapter and verse) of the verse that holds
+    # the remote backend properties.
+    # @param backend_coordinates [String] the backend coordinates to set
+    def set_backend_coordinates( backend_coordinates )
+      @master.use( Indices::REMOTE_MIRROR_SECTION_NAME )
+      @master.set( Indices::REMOTE_MIRROR_PAGE_NAME, backend_coordinates )
+    end
+
+
+  end
+
+
+end

data/lib/utils/keys/key.rb

@@ -478,6 +478,50 @@ module SafeDb
     end
 
 
+    # Return true if this key object can decrypt the parameter ciphertext that
+    # represents a key.
+    #
+    # The parameter cipher text must comprise of an 80 character key and a 16
+    # character random initialization vector (IV). When transformed to a bit string
+    # of ones and zeroes, the bit string length must be <tt>96 * 8 = 768</tt>.
+    #
+    # @param key_iv_ciphertext [String]
+    #    Provide the ciphertext produced by our sister key encryption method.
+    #    The ciphertext should hold 96 bytes which equates to 128 base64 characters.
+    #    The random initialization vector (iv) accounts for the first 16 bytes.
+    #    The actual crypt ciphertext then accounts for the final 80 bytes.
+    #
+    # @return [Boolean]
+    #    return true if the ciphertext is of the expected length and the
+    #    <tt>dry run of the decrypt operation</tt> completes without a
+    #    cipher error from {OpenSSL::Cipher::CipherError} being thrown.
+    #
+    def can_decrypt_key( key_iv_ciphertext )
+
+      bit_text = Key64.to_bits( key_iv_ciphertext )
+      ciphertext_size_msg = "Expected bit length of #{EXPECTED_CIPHER_BIT_LENGTH} not #{bit_text.length}."
+      is_expected_length = bit_text.length == EXPECTED_CIPHER_BIT_LENGTH
+      log.warn(x) { ciphertext_size_msg } unless is_expected_length
+      return false unless is_expected_length
+
+      the_cipher = OpenSSL::Cipher::AES256.new(:CBC)
+      the_cipher.decrypt()
+      rawbytes = [ bit_text ].pack("B*")
+      the_cipher.key = to_aes_key()
+      the_cipher.iv  = rawbytes[ 0 .. ( RANDOM_IV_BYTE_COUNT - 1 ) ]
+
+      begin
+        the_cipher.update( rawbytes[ RANDOM_IV_BYTE_COUNT .. -1 ] ) + the_cipher.final
+        return true
+      rescue OpenSSL::Cipher::CipherError => e
+        log.warn(x) { "Auth failure decrypting this ciphertext ~~ #{key_iv_ciphertext}" }
+        log.warn(x) { "Auth (maybe login failure) error message is ~~ #{e.message}" }
+        return false
+      end
+
+    end
+
+
     # Use the {OpenSSL::Cipher::AES256} block cipher in CBC mode and the binary
     # 256bit representation of this key to encrypt the parameter plaintext using
     # the parameter random initialization vector.

data/lib/utils/keys/keypair.rb

@@ -0,0 +1,52 @@
+#!/usr/bin/ruby
+# coding: utf-8
+
+module SafeDb
+
+  # This class creates and represents an Elliptic Curve cryptographic key.
+  # The generated key can then be comsumed via its various aspects like its
+  # ssh formatted public key and/or the pem formatted private key.
+  class Keypair
+
+    # Generate an elliptic curve cryptographic keypair. After the key is
+    # generated, both the public and private keys can be retrieved through
+    # the accessors.
+    #
+    def initialize
+
+      @ec_keypair = OpenSSL::PKey::EC.new( Indices::ELLIPTIC_CURVE_KEY_TYPE )
+      @ec_keypair.generate_key!
+
+      log.info(x) { "An elliptic curve keypair has just been generated." }
+
+    end
+
+
+    # Get the private key aspect of this elliptic curve cryptographic key
+    # in PEM format.
+    # @return [String] the PEM formatted private key
+    def private_key_pem()
+      return @ec_keypair.to_pem()
+    end
+
+
+    # Get the public key aspect of this elliptic curve cryptographic key
+    # in the long line SSH format. This format states the key type which
+    # will be **ecdsa-sha2-nistp384** followed by base64 encoded data.
+    #
+    # The returned one line public key will likely contain forward slashes
+    # and possibly equal signs at the end of the string.
+    #
+    # @return [String] the SSH formatted public key prefixed by the key type
+    def public_key_ssh()
+      require 'net/ssh'
+      key_type = @ec_keypair.ssh_type()
+      key_data = [ @ec_keypair.to_blob ].pack('m0')
+      return "#{key_type} #{key_data}"
+    end
+
+
+  end
+
+
+end

data/lib/utils/logs/logger.rb

@@ -88,7 +88,7 @@ module LogImpl
   def get_logger
 
     file_logger = Logger.new @@log_path
-    file_logger.level = Logger::INFO
+    file_logger.level = Logger::DEBUG
     original_formatter = Logger::Formatter.new
 
     file_logger.formatter = proc { |severity, datetime, progname, msg|

data/lib/utils/store/datastore.rb

@@ -5,7 +5,7 @@ module SafeDb
 
   require 'json'
 
-  # A Key/Value database knows how to manipulate a JSON backed data structure
+  # A Key/Value database knows how to manipulate a JSON backend data structure
   # (put, add etc) <b>after reading and then decrypting it</b> from a
   # file and <b>before encrypting and then writing it</b> to a file.
   #

data/lib/utils/store/github.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/ruby
+# coding: utf-8
+
+module SafeDb
+
+  # The Github class uses the REST API to talk to Github and create, query,
+  # change and delete assets within a specified hosted git repository.
+  class Github
+
+    # Initialize a Github repository given the parameter name.
+    # the parameter JSON string.
+    #
+    # @param repository_name [String] name of the Github repository
+    #
+    def initialize( repository_name )
+
+      data_db = DataStore.new()
+      data_db.merge!( JSON.parse( db_json_string ) )
+      return data_db
+
+    end
+
+
+  end
+
+
+end