kasefet 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ef83c9b70e27855bb629f577ac6fd37de8b5477d
-  data.tar.gz: b2b865ac6e19bcb17cc5143c5c3f0e0b7554f3e2
+  metadata.gz: a594687e14c4388b280060fb9684510f8f3151d0
+  data.tar.gz: 33368567de1b1ebda745b80552e58291eeb91bd9
 SHA512:
-  metadata.gz: ddd53fc18dd123c69c9b4113765b72c06d0240eede4efb976b10a7dec3b7886d2aa00ae2b9d1f7c6a90f9dfb2f032c853246bcb574255f8daff075edf3771182
-  data.tar.gz: 813b83d332a7bd0817d746978811ace68885ddfe9bf4d897f85a71871d1edf439b914a39425f563c77c8c6650129e232eb54226802e5c15e5d488d671afbf380
+  metadata.gz: fd9155c78f966ae7a061fbee9f4594d131f614a20360114d26771317b79c0883bd7b065815e19ff3323b036da649c8b0add3c743769e22c4eb397556b031c1de
+  data.tar.gz: 0662f1cadf77d4f873620b2f959e42227b0d2060fe58c036e4c3e19ca44f4b982204a9b360c631564ade8c6a259e13eda6de48b56cbe603cd770637ab0dd78e3

data/DESIGN.md

@@ -4,6 +4,8 @@ We reuse this concept in several places, so here's the generic concept. I need t
 
 ```
 root
+|--index
+|  |--index
 |--db
    |--c39d7d6b239c2e20a31672ed979fed2b45c88748d06ba3be6bff85767b5d3d
       |--20160224.184012.laptop
@@ -22,15 +24,40 @@ I leave it as an exercise to the sync program to correctly identify deleted valu
 
 When encrypting files in such a format, I like to keep them prefixed to the encrypted content and then the auth tag. Note that the first version only supports AES-256-GCM.
 
-# File format
+## File format
+
+The actual format of the value files is a binary file with the magic number "KSFT". The layout is simple:
+
+```
++-------------------------------------------------------------------------------------------+
+| Magic Number | Key Length (32 bit big endian unsigned integer) | Key    | Value until EOF |
+| KSFT         | 0x0006                                          | foobar | bazquux....     |
++-------------------------------------------------------------------------------------------+
+```
+
+## Index directory
+
+The index is a quick way to convert/iterate over all the keys in a KV store.
+
+There are two versions of the index file, which involve tradeoffs between change velocity and the likelihood of conflicts. The tradeoff is between storage size and expected velocity.
+
+## Low-velocity index
+
+This form is ideal for stores that don't change very often, as the index can be shared between all nodes, and generally doesn't need to be regenerated unless there's a conflict.
+
+The idea is that there is a single "index" file, which gets updated each time there's a new key added, and it's left to the sync program to leave a "conflicted copy" of the file, which indicates that the index should be rebuilt automatically.
+
+## High-velocity index
+
+This is the preferred form if you expect to have many concurrent changes. In this approach, each node maintains its own view of the index, and regenerates it pretty much every single chance it gets. The drawback is that changes are not automatically detected, and the extra storage, which can be non-trivial if there are a large number of keys.
+
+# Kasefet Password Wallets
 
 A kasefet wallet has the following layout:
 
 ```
 wallet
 |--key
-|--index
-|  |--index
 |--metadata (flat file kv directory)
 |--ksft (flat file kv directory)
 ```

data/PHILOSOPHY.md

@@ -1,6 +1,6 @@
 # The name
 
-Kasefet is the hebrew word for "safe", (the locked box meaning of safe). Seemed as good as any, and wasn't taken on rubygems
+Kasefet is the Hebrew word for "safe", (the locked box meaning of safe). Seemed as good as any, and wasn't taken on RubyGems
 
 # A bit of history
 

data/README.md

@@ -10,12 +10,12 @@ Kasefet is currently a work in progress. This is the todo list for version 1.0,
 
  - [x] Flat file wallet
  - [x] SSL encrypted wallet
- - [ ] support for multiple wallets
+ - [x] support for multiple wallets
  - [x] editor integration
- - [ ] clipboard integration (Mac)
- - [ ] autotype (Mac)
- - [ ] clipboard integration (Ubuntu)
- - [ ] autotype (Ubuntu)
+ - [x] clipboard integration (Mac Yosemite 10.10)
+ - [ ] autotype (Mac Yosemite 10.10)
+ - [x] clipboard integration (Ubuntu 14.04)
+ - [ ] autotype (Ubuntu 14.04)
 
 ## Installation
 

data/kasefet.gemspec

@@ -1,7 +1,9 @@
 # coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'kasefet/version'
+require "kasefet/version"
+
+require "rbconfig" # for platform data
 
 Gem::Specification.new do |spec|
   spec.name          = "kasefet"
@@ -19,8 +21,15 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency "thunder", "~> 0.7"
+  spec.add_runtime_dependency "clipboard", "~> 1.0"
+
+  case RbConfig::CONFIG['host_os']
+  when /windows/
+    spec.add_runtime_dependency "ffi"
+  end
+
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "thunder", "~> 0.7"
   spec.add_development_dependency "minitest"
 end

data/lib/kasefet/cli.rb

@@ -2,6 +2,7 @@ require "thunder"
 
 require "kasefet/config"
 require "kasefet/wallet"
+require "kasefet/multi_wallet"
 
 class Kasefet
   class CLI
@@ -14,17 +15,35 @@ class Kasefet
 
     include Thunder
 
-    def load_config(options)
+    desc "copy KEYNAME", "copy the contents of KEYNAME to the primary system clipboard"
+    def copy(keyname, **options)
+      load_config(options)
+      load_wallet(options)
+
+      require 'clipboard'
+
+      Clipboard.copy(@wallets.load(keyname))
+    end
+
+    def load_config(options = {})
       config_file = options[:config]
       config_file ||= GlobalConfigLocations.find { |file| File.exist?(File.expand_path(file)) }
       config_file ||= File.expand_path(GlobalConfigLocations.first)
       @config = Kasefet::Config.new(config_file)
+      @config.load
+      return @config
     end
 
-    def load_wallet
-      wallet_dir = @config["wallet"]
-      wallet_dir = @config["wallet"] = File.expand_path(DefaultWalletLocation) unless wallet_dir
-      @wallet = Kasefet::Wallet.new(directory: wallet_dir)
+    def load_wallet(options = {})
+      wallet_dirs = @config["wallet"]
+      wallet_dirs = @config["wallet"] = File.expand_path(DefaultWalletLocation) unless wallet_dirs
+      wallet_dirs = Array(wallet_dirs)
+      wallets = wallet_dirs.map do |wallet_dir|
+        [wallet_dir, Kasefet::Wallet.new(directory: wallet_dir)]
+      end.to_h
+
+      @wallets = Kasefet::MultiWallet.new(wallets)
+      return @wallets
     end
 
     def determine_editor
@@ -44,13 +63,13 @@ class Kasefet
     desc "edit KEYNAME", "open the contents of KEYNAME in an editor, and save the changes"
     def edit(keyname, *content, **options)
       load_config(options)
-      load_wallet
+      load_wallet(options)
 
       require 'tmpdir'
       require 'pathname'
       Dir.mktmpdir do |tmpdir|
         tmpdir = Pathname.new(tmpdir)
-        content = @wallet.load(keyname)
+        content = @wallets.load(keyname)
         File.binwrite(tmpdir + keyname, content)
 
         # invoke the editor
@@ -63,7 +82,7 @@ class Kasefet
           puts "`#{editor}` exited with error status: #{$?}. Not saving contents"
         else
           new_content = File.binread(tmpdir + keyname)
-          @wallet.store(keyname, new_content)
+          @wallets.store(keyname, new_content)
         end
       end
     end
@@ -71,11 +90,11 @@ class Kasefet
     desc "add KEYNAME CONTENTS...", "store the given CONTENTS in KEYNAME"
     def add(keyname, *content, **options)
       load_config(options)
-      load_wallet
+      load_wallet(options)
 
       content = content.join(" ")
 
-      @wallet.store(keyname, content)
+      @wallets.store(keyname, content)
 
       return content
     end
@@ -83,9 +102,9 @@ class Kasefet
     desc "show KEYNAME", "print the contents of KEYNAME to stdout"
     def show(keyname, **options)
       load_config(options)
-      load_wallet
+      load_wallet(options)
 
-      content = @wallet.load(keyname)
+      content = @wallets.load(keyname)
 
       puts content
       return content

data/lib/kasefet/config.rb

@@ -3,8 +3,6 @@ class Kasefet
     def initialize(file)
       @file = file
       @settings = {}
-      return unless File.exists?(file)
-      load
     end
 
     attr_accessor :file
@@ -18,17 +16,30 @@ class Kasefet
       last_segment[key.split(".")[-1]] = value
     end
 
+    def each_keys()
+      return @settings.keys.each unless block_given?
+
+      @settings.keys.each do |key|
+        yield key
+      end
+    end
+
     def load
-      case File.extname(@file)
+      return unless File.exists?(@file)
+      parse(File.read(@file))
+    end
+
+    def parse(contents, file_path = @file)
+      case File.extname(file_path)
       when ".yaml", ".yml"
         require 'yaml'
-        @settings = YAML.load(File.read(file))
+        @settings = YAML.load(contents)
       when ".json"
         require 'json'
-        @settings = JSON.parse(File.read(file))
+        @settings = JSON.parse(contents)
       else
         # try the key=value format
-        content = File.read(file)
+        content = contents
         content.split("\n").each do |line|
           key, value = line.split("=")
           value = value.to_i if value =~ /\d+/
@@ -43,15 +54,19 @@ class Kasefet
     end
 
     def save
-      case File.extname(@file)
+      File.write(@file, format())
+    end
+
+    def format(file_path = @file)
+      case File.extname(file_path)
       when ".json"
         require 'json'
-        File.write(@file, @settings.to_json)
+        return @settings.to_json
       when ".yaml", ".yml"
         require 'yaml'
-        File.write(@file, @settings.to_yaml)
+        return @settings.to_yaml
       else
-        to_write = flatten_hash(@settings).map do |key, value|
+        return flatten_hash(@settings).map do |key, value|
           if value.is_a? Array
             value.map do |array_value|
               "#{key}=#{array_value}"
@@ -60,7 +75,6 @@ class Kasefet
             "#{key}=#{value}"
           end
         end.join("\n")
-        File.write(@file, to_write)
       end
     end
 

data/lib/kasefet/encrypted_flat_kv.rb

@@ -8,13 +8,19 @@ class Kasefet
     CipherIVLength = 12
     CipherAuthTagLength = 16
 
-    def initialize(cipher_key:, **options)
+    def initialize(cipher_key:, key_salt: nil, **options)
       super(**options)
       @cipher = OpenSSL::Cipher.new("aes-256-gcm")
       @cipher_key = cipher_key
+      @key_salt = key_salt
     end
 
-    attr_accessor :cipher_key
+    attr_accessor :cipher_key, :key_salt
+
+    def key_to_digest(key)
+      key = "#{@key_salt}/#{key}/#{@key_salt}" if @key_salt
+      return super(key)
+    end
 
     def reencrypt_all_values!(new_key)
       old_key = @cipher_key
@@ -51,14 +57,14 @@ class Kasefet
       return value
     end
 
-    def [](key)
-      encrypted_value = super(key)
-      return nil if encrypted_value.nil?
-      return decrypt_value(encrypted_value, @cipher_key)
+    def read_file(file_path)
+      encrypted_contents = super(file_path)
+      return nil if encrypted_contents.nil?
+      return decrypt_value(encrypted_contents, @cipher_key)
     end
 
-    def []=(key, value)
-      super(key, encrypt_value(value, @cipher_key))
+    def write_file(path, contents)
+      return super(path, encrypt_value(contents, @cipher_key))
     end
   end
 end

data/lib/kasefet/flat_kv.rb

@@ -8,6 +8,8 @@ class Kasefet
   #
   # Flat-file based key-value storage engine that is designed to be compatible with naive sync programs
   class FlatKV
+    MagicNumber = "KSFT"
+
     # @option [String] :root The root directory of the FlatKV store
     # @option [String] :device_name The device name to use to identify the writer
     # @option [String] :extension The default extension to use for the value files
@@ -26,15 +28,44 @@ class Kasefet
 
     def [](key)
       value_file = file_for_key(key)
-      return nil unless value_file
-      return File.binread(value_file)
+      contents = read_file(value_file)
+      _, value = read_value(contents)
+      return value
+    end
+
+    def read_file(file_path)
+      return nil unless file_path
+      return File.binread(file_path)
+    end
+
+    def read_value(file_contents)
+      return nil, nil unless file_contents
+
+      raise "FlatKV value file must be at least 8 bytes long" unless file_contents.bytesize >= 8
+      magic_number, key_size, file_contents = file_contents.unpack("A4NA*")
+      raise "FlatKV value file must be a KSFT file" unless magic_number == MagicNumber
+      raise "FlatKV value file has been corrupted. Key length in header is longer than file" unless file_contents.bytesize >= key_size
+      key_name, file_contents = file_contents.unpack("A#{key_size}A*")
+
+      return key_name, file_contents
     end
 
     def []=(key, value)
       key_dir = dir_for_key(key)
       FileUtils.mkdir_p(key_dir)
       value_file_name = Time.now.strftime("%Y%m%d.%H%M%S%6N.") + @device_name + @extension
-      File.binwrite(key_dir + value_file_name, value)
+
+      value = format_value(key, value)
+
+      write_file(key_dir + value_file_name, value)
+    end
+
+    def format_value(key, value)
+      [MagicNumber, key.bytesize, key, value].pack("A4NA*A*")
+    end
+
+    def write_file(path, contents)
+      File.binwrite(path, contents)
     end
 
     def file_for_key(key)
@@ -44,8 +75,12 @@ class Kasefet
       return files.last
     end
 
+    def key_to_digest(key)
+      OpenSSL::Digest::SHA256.hexdigest(key)
+    end
+
     def dir_for_key(key)
-      digest = OpenSSL::Digest::SHA256.hexdigest(key)
+      digest = key_to_digest(key)
       return @root + digest[0..1] + digest[2..-1]
     end
   end

data/lib/kasefet/indexed_flat_kv.rb

@@ -0,0 +1,76 @@
+require 'fileutils'
+
+require 'kasefet/flat_kv'
+require 'kasefet/config'
+
+class Kasefet
+  class IndexedFlatKV
+    def initialize(flat_kv:, index_dir: "index", index_ext: "")
+      @flat_kv = flat_kv
+      @root = flat_kv.root
+      @index_dir = @root + "#{index_dir}"
+      @index_file = @index_dir + "index#{index_ext}"
+      @index = Kasefet::Config.new(@index_file)
+    end
+
+    attr_accessor :index, :index_file
+
+    def [](key)
+      return @flat_kv[key]
+    end
+
+    def []=(key, value)
+      mark_key(key, @flat_kv.key_to_digest(key))
+      return @flat_kv[key] = value
+    end
+
+    def load_index
+      index_contents = @flat_kv.read_file(@index_file)
+      @index.parse(index_contents)
+    end
+
+    def conflicted?
+      Dir[@index_dir + "*"].size != 1
+    end
+
+    def rebuild_index
+      Dir.foreach(@root) do |prefix|
+        next unless prefix =~ /\h\h/
+        Dir.foreach(@root + prefix) do |key_dir|
+          next unless key_dir =~ /\h{62}/ # SHA256 digest is 64 bytes, first two are the prefix
+          Dir.foreach(@root + prefix + key_dir) do |value_file|
+            next if [".", ".."].include?(value_file)
+            value_path = @root + prefix + key_dir + value_file
+            contents = @flat_kv.read_file(value_path)
+            key_name, value = @flat_kv.read_value(contents)
+            mark_key(key_name, prefix + key_dir)
+          end
+        end
+      end
+    end
+
+    def mark_key(key, digest)
+      @index["dig:" + digest] = key
+      @index["key:" + key] = digest
+    end
+
+    def save_index
+      index_contents = @index.format
+      FileUtils.mkdir_p(@index_dir)
+      @flat_kv.write_file(@index.file, index_contents)
+    end
+
+    def has_key?(key)
+      return !! @index["key:" + key]
+    end
+
+    def each_keys
+      return enum_for(__method__) unless block_given?
+
+      @index.each_keys do |key|
+        next unless key.start_with?("key:")
+        yield key[4..-1]
+      end
+    end
+  end
+end

data/lib/kasefet/multi_wallet.rb

@@ -0,0 +1,24 @@
+require "kasefet/wallet"
+
+class Kasefet
+  class MultiWallet
+    def initialize(wallets)
+      @wallets = wallets
+    end
+
+    attr_accessor :wallets
+
+    def load(keyname)
+      @wallets.each do |name, wallet|
+        contents = wallet.load(keyname)
+        return contents unless contents.nil?
+      end
+      return nil
+    end
+
+    def store(keyname, content, name = nil)
+      name ||= @wallets.keys.first
+      @wallets[name].store(keyname, content)
+    end
+  end
+end

data/lib/kasefet/version.rb

@@ -1,3 +1,3 @@
 class Kasefet
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/kasefet/wallet.rb

@@ -27,22 +27,21 @@ class Kasefet
       @wallet_version = @wallet_version.to_i
       raise "Unknown Kasefet Wallet version: #{@wallet_version}" unless @wallet_version <= VERSION
 
-      @name_salt = @metadata["kasefet.name_salt"]
-      @credentials = Kasefet::EncryptedFlatKV.new(root: @root + CREDENTIALS_DIR, cipher_key: @master_key.key)
+      @credentials = Kasefet::EncryptedFlatKV.new(
+        root: @root + CREDENTIALS_DIR,
+        cipher_key: @master_key.key,
+        key_salt: @metadata["kasefet.name_salt"],
+      )
     end
 
     attr_accessor :root
 
-    def salted_keyname(name)
-      return "#{@name_salt}/#{name}/#{@name_salt}"
-    end
-
     def load(name)
-      return @credentials[salted_keyname(name)]
+      return @credentials[name]
     end
 
     def store(name, creds)
-      @credentials[salted_keyname(name)] = creds
+      @credentials[name] = creds
     end
   end
 end

metadata

@@ -1,57 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: kasefet
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Steven Karas
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-03-07 00:00:00.000000000 Z
+date: 2016-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: thunder
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
-  type: :development
+        version: '0.7'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0.7'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: clipboard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.10'
 - !ruby/object:Gem::Dependency
-  name: thunder
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.7'
+        version: '10.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -91,7 +105,9 @@ files:
 - lib/kasefet/config.rb
 - lib/kasefet/encrypted_flat_kv.rb
 - lib/kasefet/flat_kv.rb
+- lib/kasefet/indexed_flat_kv.rb
 - lib/kasefet/master_key.rb
+- lib/kasefet/multi_wallet.rb
 - lib/kasefet/version.rb
 - lib/kasefet/wallet.rb
 homepage: https://github.com/stevenkaras/kasefet