e3db 1.0.0 → 2.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8130f8d97af89a3b0dadd56a137f215950df2ff6
-  data.tar.gz: 1d8c7ea1c68e443417fe8426ae742e3546f3e378
+  metadata.gz: 4654bc1065622f2d3f1ad6fd263688780063f4c5
+  data.tar.gz: 0be6154d88d0cd709a7fca0e67013cc05ad7d1fd
 SHA512:
-  metadata.gz: 40e554e311c5c25c8ea9eeffb2a33c0a27073b9d852ae23895b2577f0d279e909c6401eccc60b0725c0e6a395f558b6059a80801b3a4c65566f4f14795c6ede4
-  data.tar.gz: ae4c58bacbeba5924d47a181b2d9e9f179bbdda47bc5987edc3a4baaa8e5b47dc19be87b206042d8832747bf813a9e2468c6414c3063715937a71629e2f591b6
+  metadata.gz: 1401984cc27238a0a168b7e71265bbdfe4b44d225b7e4edfede2519e4cd384a699f9fbd984ec61932ee7d0cc0347704bedcd08002a16081eb96d703b254324c3
+  data.tar.gz: 30ff287c9585276086c4fb9a7a936e3e20b9656dc5f410016b70c3833af4e29ade36207ad49e6c95db172d780b7c2abffaad261b53e3427a912c4c76750a6dfc

data/.travis.yml

@@ -1,5 +1,19 @@
 sudo: false
 language: ruby
 rvm:
-  - 2.0.0
+  - 2.3.4
+  - 2.4.1
+
+cache:
+  bundler: true
+  directories:
+  - $HOME/libsodium
+
 before_install: gem install bundler -v 1.12.3
+
+install:
+  - ./travis-install-libsodium.sh
+  - ./travis-install-configfile.sh
+  - export PKG_CONFIG_PATH=$HOME/libsodium/lib/pkgconfig:$PKG_CONFIG_PATH
+  - export LD_LIBRARY_PATH=$HOME/libsodium/lib:$LD_LIBRARY_PATH
+  - bundle install

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown

data/Gemfile

@@ -1,4 +1,4 @@
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in e3db.gemspec
-gemspec
+gemspec

data/README.md

@@ -1,3 +1,4 @@
+[![Gem Version][gem-image]][gem-url] [![Build Status][travis-image]][travis-url] [![Coverage Status][coveralls-image]][coveralls-url]
 
 # Introduction
 
@@ -83,19 +84,18 @@ client = E3DB::Client.new(config)
 
 ## Writing a record
 
-To write new records to the database, first create a blank record
-of the correct type using `E3DB::Client#new_record`. Then fill in
-the fields of the record's `data` hash. Finally, write the record
-to the database with `E3DB::Client#write`, which returns the
-unique ID of the newly created record.
+To write new records to the database, call the `E3DB::Client#write`
+method with a string describing the type of data to be written,
+along with a hash containing the fields of the record.  `E3DB::Client#write`
+returns the newly created record.
 
 ```ruby
-record = client.new_record('contact')
-record.data[:first_name] = 'Jon'
-record.data[:last_name] = 'Snow'
-record.data[:phone] = '555-555-1212'
-record_id = client.write(record)
-printf("Wrote record %s\n", record_id)
+record = client.write('contact', {
+  :first_name => 'Jon',
+  :last_name => 'Snow',
+  :phone => '555-555-1212'
+})
+printf("Wrote record %s\n", record.meta.record_id)
 ```
 
 ## Querying Records
@@ -114,6 +114,29 @@ client.query(type: 'contact') do |record|
 end
 ```
 
+In this example, the `E3DB::Client#query` method takes a block that will
+execute for each record that matches the query. Records will be streamed
+efficiently from the server in batches, allowing processing of large data
+sets without consuming excessive memory.
+
+In some cases, it is more convenient to load all results into memory
+for processing. To achieve this, instead of passing a block to
+`E3DB::Client#query`, you can call `Enumerable` methods on the query result,
+including `Enumerable#to_a` to convert the results to an array.
+
+For example:
+
+```ruby
+results = client.query(type: 'contact').to_a
+printf("There were %d results.\n", results.length)
+results.each do |record|
+  puts record
+end
+```
+
+## More examples
+See the [simple example code](examples/simple.rb) for runnable detailed examples.
+
 ## Development
 
 Before running tests, register an `integration-test` profile using
@@ -133,6 +156,12 @@ then run `bundle exec rake release`, which will create a git tag for the
 version, push git commits and tags, and push the `.gem` file to
 [rubygems.org](https://rubygems.org).
 
+## Documentation
+
+General E3DB documentation is [on our web site](https://tozny.com/documentation/e3db/).
+
+Comprehensive documentation for the SDK can be found online [via RubyDoc.info](http://www.rubydoc.info/gems/e3db/1.0.0).
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/tozny/e3db-ruby.
@@ -140,3 +169,10 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/tozny/
 ## License
 
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+
+[gem-image]: https://badge.fury.io/rb/e3db.svg
+[gem-url]: https://rubygems.org/gems/e3db
+[travis-image]: https://travis-ci.org/tozny/e3db-ruby.svg?branch=master
+[travis-url]: https://travis-ci.org/tozny/e3db-ruby
+[coveralls-image]: https://coveralls.io/repos/github/tozny/e3db-ruby/badge.svg?branch=master
+[coveralls-url]: https://coveralls.io/github/tozny/e3db-ruby

data/e3db.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency 'simplecov', '~> 0.14.1'
+  spec.add_development_dependency 'coveralls', '~> 0.8.0'
 
   spec.add_dependency 'dry-struct', '~> 0.2.1'
   spec.add_dependency 'lru_redux', '~> 1.1'

data/examples/simple.rb

@@ -0,0 +1,148 @@
+# This program provides a few simple examples of reading, writing, and
+# querying e3db records. For more detailed information, please see the
+# documentation home page: https://tozny.com/documentation/e3db/
+#
+# Author::    Isaac Potoczny-Jones  (mailto:ijones@tozny.com)
+# Copyright:: Copyright (c) 2017 Tozny, LLC
+# License::   Public Domain
+
+# ---------------------------------------------------------
+# Initialization
+# ---------------------------------------------------------
+
+require 'e3db'
+
+# Configuration files live in ~/.tozny and you can have several
+# different "profiles" like *dev* and *production*.
+config = E3DB::Config.default
+
+# Now create a client using that configuration.
+client = E3DB::Client.new(config)
+
+# ---------------------------------------------------------
+# Writing a record
+# ---------------------------------------------------------
+
+# Create a record by first creating a local version as a map:
+data = {
+    :name => 'Jon Snow',
+    :what_he_knows => 'Nothing'
+}
+
+# Now encrypt the *value* part of the record, write it to the server and
+# the server returns the newly created record:
+record = client.write('test-contact', data)
+record_id = record.meta.record_id
+puts("Wrote:    " + record_id)
+
+# ---------------------------------------------------------
+# Simple reading and queries
+# ---------------------------------------------------------
+
+# Use the new record's unique ID to read the same record again from E3DB:
+newRecord = client.read(record.meta.record_id)
+puts 'Record:   ' + newRecord.data[:name] + ' ' + record.data[:what_he_knows]
+
+# Query for all records of type 'test-contact' and print out
+# a little bit of data and metadata.
+client.query(type: 'test-contact').each do |record|
+    puts 'Data:     ' + record.data[:name] + ' ' + record.data[:what_he_knows]
+    puts 'Metadata: ' + record.meta.record_id + ' ' + record.meta.type
+end
+
+# ---------------------------------------------------------
+# Simple sharing by record type
+# ---------------------------------------------------------
+
+# Share all of the records of type 'test-contact' with Isaac's client ID:
+isaac_client_id = 'db1744b9-3fb6-4458-a291-0bc677dba08b'
+client.share('test-contact', isaac_client_id)
+
+# Share all of the records of type 'test-contact' with Isaac's email address.
+# This only works if the client has opted into discovery of their client_id.
+client.share('test-contact', 'ijones+feedback@tozny.com')
+
+# ---------------------------------------------------------
+# More complex queries
+# ---------------------------------------------------------
+
+# Create some new records of the same type (note that they are also shared
+# automatically since they are a type that we have shared above. We
+# will also add some "plain" fields that are not secret but can be used
+# for efficient querying:
+
+bran_data = { :name => 'Bran', :what_he_knows => 'Crow' }
+bran_plain = { :house => 'Stark', :ageRange => 'child' }
+client.write('test-contact', bran_data, bran_plain)
+
+hodor_data = { :name => 'Hodor', :what_he_knows => 'Hodor' }
+hodor_plain = { :house => 'Stark', :ageRange => 'adult' }
+client.write('test-contact', hodor_data, hodor_plain)
+
+doran_data = { :name => 'Doran', :what_he_knows => 'Oberyn' }
+doran_plain = { :house => 'Martell', :ageRange => 'adult' }
+client.write('test-contact', doran_data, doran_plain)
+
+# Create a query that finds everyone from house Stark, but not others:
+queryWesteros = Hash.new
+queryWesteros = {:eq => {:name => 'house', :value => 'Stark'} }
+
+# Execute that query:
+client.query(plain: queryWesteros).each do |record|
+    puts record.data[:name]
+end
+
+# Now create a  more complex query with only the adults from house Stark:
+queryWesteros = {:and => [
+                       {:eq => {:name => 'house', :value => 'Stark'} },
+                       {:eq => {:name => 'ageRange', :value => 'adult'} }
+                       ]}
+
+# Execute that query:
+client.query(plain: queryWesteros) do |record|
+    puts record.data[:name]
+end
+
+# ---------------------------------------------------------
+# Learning about other clients
+# ---------------------------------------------------------
+isaac_client_info = client.client_info('ijones+feedback@tozny.com')
+puts isaac_client_info.inspect
+
+# Fetch the public key:
+isaac_pub_key = client.client_key(isaac_client_id)
+puts isaac_pub_key.inspect
+
+# ---------------------------------------------------------
+# More reading and inspection of records
+# ---------------------------------------------------------
+
+# read_raw gets a record without decrypting its data
+rawRecord = client.read_raw(record_id)
+newRecord = client.read(record_id)
+
+# So let's compare them:
+
+puts (rawRecord.meta == newRecord.meta).to_s # true
+puts (rawRecord.data == newRecord.data).to_s # false
+
+puts newRecord.data[:name] + ' encrypts to ' + rawRecord.data[:name]
+
+# Records contain a few other fields that are fun to look at, and this gives
+# you a good sense for what's encrypted and what's not:
+puts rawRecord.inspect
+
+# ---------------------------------------------------------
+# Clean up - Comment these out if you want to experiment
+# ---------------------------------------------------------
+
+# Revoke the sharing created by the client.share
+client.revoke('test-contact', 'ijones+feedback@tozny.com')
+
+# Delete the record we created above
+client.delete(record_id)
+
+# Delete all of the records of type test-contact from previous runs:
+client.query(type: 'test-contact') do |record|
+    client.delete(record.meta.record_id)
+end

data/lib/e3db/client.rb

@@ -35,6 +35,23 @@ module E3DB
 
   private_constant :TokenHelper
 
+  # Exception thrown by {Client#update} when a concurrent modification
+  # is detected. Upon catching this exception, a client should re-fetch
+  # the affected record and retry the update operation.
+  class ConflictError < StandardError
+    def initialize(record)
+      super('Conflict updating record: ' + record.meta.record_id)
+      @record = record
+    end
+
+    # Return the record from the failing update attempt.
+    #
+    # @return [Record] the affected record
+    def record
+      @record
+    end
+  end
+
   # A client's public key information.
   #
   # @!attribute curve25519
@@ -72,6 +89,8 @@ module E3DB
   #   @return [Time, nil] when this record was created, or nil if unavailable
   # @!attribute last_modified
   #   @return [Time, nil] when this record was last modified, or nil if unavailable
+  # @!attribute version
+  #   @return [String] opaque version identifier updated by server on changes
   class Meta < Dry::Struct
     attribute :record_id, Types::Strict::String.optional
     attribute :writer_id, Types::Strict::String
@@ -80,6 +99,7 @@ module E3DB
     attribute :plain, Types::Strict::Hash.default { Hash.new }
     attribute :created, Types::Json::DateTime.optional
     attribute :last_modified, Types::Json::DateTime.optional
+    attribute :version, Types::Strict::String.optional
   end
 
   # A E3DB record containing data and metadata. Records are
@@ -88,9 +108,8 @@ module E3DB
   # to the server for storage, and decrypted in the client after
   # they are read.
   #
-  # The {Client#new_record} method should be used to create a
-  # new record that can be written to the database with
-  # {Client#write}.
+  # New records are written to the database by calling the
+  # {Client#write} method.
   #
   # To read a record by their unique ID, use {Client#read}, or to
   # query a set of records based on their attributes, use {Client#query}.
@@ -102,6 +121,49 @@ module E3DB
   class Record < Dry::Struct
     attribute :meta, Meta
     attribute :data, Types::Strict::Hash.default { Hash.new }
+
+    # Allow updating metadata, used on destructive update.
+    def meta=(meta)
+      @meta = meta
+    end
+  end
+
+  # Information about records shared with this client.
+  #
+  # The {Client#incoming_sharing} method returns a list of
+  # {IncomingSharingPolicy} instances, each of which describes
+  # a rule allowing this client to read records of a specific
+  # type, written by another client.
+  #
+  # @!attribute writer_id
+  #   @return [String] unique ID of the writer that shared with this client
+  # @!attribute writer_name
+  #   @return [String] display name of the writer, if available
+  # @!attribute record_type
+  #   @return [String] type of record shared with this client
+  class IncomingSharingPolicy < Dry::Struct
+    attribute :writer_id, Types::Strict::String
+    attribute :writer_name, Types::Strict::String.optional
+    attribute :record_type, Types::Strict::String
+  end
+
+  # Information about records shared with another client.
+  #
+  # The {Client#outgoing_sharing} method returns a list of
+  # {OutgoingSharingPolicy} instances, each of which describes
+  # a rule allowing other E3DB clients to read records of a
+  # specific type.
+  #
+  # @!attribute reader_id
+  #   @return [String] unique ID of the authorized reader
+  # @!attribute reader_name
+  #   @return [String] display name of reader, if available
+  # @!attribute record_type
+  #   @return [String] type of record shared with reader
+  class OutgoingSharingPolicy < Dry::Struct
+    attribute :reader_id, Types::Strict::String
+    attribute :reader_name, Types::Strict::String.optional
+    attribute :record_type, Types::Strict::String
   end
 
   # A connection to the E3DB service used to perform database operations.
@@ -146,10 +208,17 @@ module E3DB
 
     # Query the server for information about an E3DB client.
     #
-    # @param client_id [String] client ID to look up
+    # @param client_id [String] client ID or e-mail address to look up
     # @return [ClientInfo] information about this client
     def client_info(client_id)
-      resp = @conn.get(get_url('v1', 'storage', 'clients', client_id))
+      if client_id.include? "@"
+        base_url = get_url('v1', 'storage', 'clients', 'find')
+        url = base_url + sprintf('?email=%s', CGI.escape(client_id))
+        resp = @conn.post(url)
+      else
+        resp = @conn.get(get_url('v1', 'storage', 'clients', client_id))
+      end
+
       ClientInfo.new(JSON.parse(resp.body, symbolize_names: true))
     end
 
@@ -184,30 +253,48 @@ module E3DB
       decrypt_record(read_raw(record_id))
     end
 
-    # Create a new, empty record that can be written to E3DB
-    # by calling {Client#write}.
+    # Write a new record to the E3DB storage service.
     #
-    # @param type [String] free-form content type of this record
-    # @return [Record] an empty record of `type`
-    def new_record(type)
+    # @param type [String] free-form content type name of this record
+    # @param data [Hash<String, String>] record data to be stored encrypted
+    # @param plain [Hash<String, String>] record data to be stored unencrypted for querying
+    # @return [Record] the newly created record object
+    def write(type, data, plain=Hash.new)
+      url = get_url('v1', 'storage', 'records')
       id = @config.client_id
       meta = Meta.new(record_id: nil, writer_id: id, user_id: id,
-                      type: type, plain: Hash.new, created: nil,
-                      last_modified: nil)
-      Record.new(meta: meta, data: Hash.new)
+                      type: type, plain: plain, created: nil,
+                      last_modified: nil, version: nil)
+      record = Record.new(meta: meta, data: data)
+      resp = @conn.post(url, encrypt_record(record).to_hash)
+      decrypt_record(Record.new(JSON.parse(resp.body, symbolize_names: true)))
     end
 
-    # Write a new record to the E3DB storage service.
+    # Update an existing record in the E3DB storage service.
     #
-    # Create new records with {Client#new_record}.
+    # If the record has been modified by another client since it was
+    # read, this method raises {ConflictError}, which should be caught
+    # by the caller so that the record can be re-fetched and the update retried.
     #
-    # @param record [Record] record to write
-    # @return [String] the unique ID of the written record
-    def write(record)
-      url = get_url('v1', 'storage', 'records')
-      resp = @conn.post(url, encrypt_record(record).to_hash)
+    # The metadata of the input record will be updated in-place to reflect
+    # the new version number and modification time returned by the server.
+    #
+    # @param record [Record] the record to update
+    def update(record)
+      record_id = record.meta.record_id
+      version = record.meta.version
+      url = get_url('v1', 'storage', 'records', 'safe', record_id, version)
+      begin
+        resp = @conn.put(url, encrypt_record(record).to_hash)
+      rescue Faraday::ClientError => e
+        if e.response[:status] == 409
+          raise E3DB::ConflictError, record
+        else
+          raise e   # re-raise on other failures
+        end
+      end
       json = JSON.parse(resp.body, symbolize_names: true)
-      json[:meta][:record_id]
+      record.meta = Meta.new(json[:meta])
     end
 
     # Delete a record from the E3DB storage service.
@@ -218,14 +305,15 @@ module E3DB
     end
 
     class Query < Dry::Struct
-      attribute :count,         Types::Int
-      attribute :include_data,  Types::Bool.optional
-      attribute :writer_ids,    Types::Coercible::Array.member(Types::String).optional
-      attribute :user_ids,      Types::Coercible::Array.member(Types::String).optional
-      attribute :record_ids,    Types::Coercible::Array.member(Types::String).optional
-      attribute :content_types, Types::Coercible::Array.member(Types::String).optional
-      attribute :plain,         Types::Hash.optional
-      attribute :after_index,   Types::Int.optional
+      attribute :count,               Types::Int
+      attribute :include_data,        Types::Bool.optional
+      attribute :writer_ids,          Types::Coercible::Array.member(Types::String).optional
+      attribute :user_ids,            Types::Coercible::Array.member(Types::String).optional
+      attribute :record_ids,          Types::Coercible::Array.member(Types::String).optional
+      attribute :content_types,       Types::Coercible::Array.member(Types::String).optional
+      attribute :plain,               Types::Hash.optional
+      attribute :after_index,         Types::Int.optional
+      attribute :include_all_writers, Types::Bool.optional
 
       def after_index=(index)
         @after_index = index
@@ -241,49 +329,117 @@ module E3DB
     DEFAULT_QUERY_COUNT = 100
     private_constant :DEFAULT_QUERY_COUNT
 
+    # A set of records returned by {Client#query}. This implements the
+    # `Enumerable` interface which can be used to loop over the records
+    # in the result set (using eg: `Enumerable#each`).
+    #
+    # Every traversal of the result set will execute a query to the server,
+    # so if multiple in-memory traversals are needed, use `Enumerable#to_a` to
+    # fetch all records into an array first.
+    class Result
+      include Enumerable
+
+      def initialize(client, query, raw)
+        @client = client
+        @query = query
+        @raw = raw
+      end
+
+      # Invoke a block for each record matching a query.
+      def each
+        # Every invocation of 'each' gets its own copy of the query since
+        # it will be modified as we loop through the result pages. This
+        # allows multiple traversals of the same result set to start from
+        # the beginning each time.
+        q = Query.new(@query.to_hash)
+        loop do
+          json = @client.instance_eval { query1(q) }
+          results = json[:results]
+          results.each do |r|
+            record = Record.new(meta: r[:meta], data: r[:record_data] || Hash.new)
+            if q.include_data && !@raw
+              access_key = r[:access_key]
+              if access_key
+                record = @client.instance_eval {
+                  ak = decrypt_eak(access_key)
+                  decrypt_record_with_key(record, ak)
+                }
+              else
+                record = @client.instance_eval { decrypt_record(record) }
+              end
+            end
+            yield record
+          end
+
+          if results.length < q.count
+            break
+          end
+
+          q.after_index = json[:last_index]
+        end
+      end
+    end
+
     # Query E3DB records according to a set of selection criteria.
     #
-    # Each record (optionally including data) is yielded to the block
-    # argument.
+    # The default behavior is to return all records written by the
+    # current authenticated client.
+    #
+    # To restrict the results to a particular type, pass a type or
+    # list of types as the `type` argument.
+    #
+    # To restrict the results to a set of clients, pass a single or
+    # list of client IDs as the `writer` argument. To list records
+    # written by any client that has shared with the current client,
+    # pass the special token `:any` as the `writer` argument.
     #
-    # @param writer [String,Array<String>] select records written by these client IDs
+    # If a block is supplied, each record matching the query parameters
+    # is fetched from the server and yielded to the block.
+    #
+    # If no block is supplied, a {Result} is returned that will
+    # iterate over the records matching the query parameters. This
+    # iterator is lazy and will query the server each time it is used,
+    # so calling `Enumerable#to_a` to convert to an array is recommended
+    # if multiple traversals are necessary.
+    #
+    # @param writer [String,Array<String>,:all] select records written by these client IDs or :all for all writers
     # @param record [String,Array<String>] select records with these record IDs
     # @param type [String,Array<string>] select records with these types
     # @param plain [Hash] plaintext query expression to select
     # @param data [Boolean] include data in records
     # @param raw [Boolean] when true don't decrypt record data
-    def query(data: true, raw: false, writer: nil, record: nil, type: nil, plain: nil)
+    # @param page_size [Integer] number of records to fetch per request
+    # @return [Result] a result set object enumerating matched records
+    def query(data: true, raw: false, writer: nil, record: nil, type: nil, plain: nil, page_size: DEFAULT_QUERY_COUNT)
+      all_writers = false
+      if writer == :all
+        all_writers = true
+        writer = []
+      end
+
       q = Query.new(after_index: 0, include_data: data, writer_ids: writer,
                     record_ids: record, content_types: type, plain: plain,
-                    user_ids: nil, count: DEFAULT_QUERY_COUNT)
-      url = get_url('v1', 'storage', 'search')
-      loop do
-        resp = @conn.post(url, q.as_json)
-        json = JSON.parse(resp.body, symbolize_names: true)
-        results = json[:results]
-        results.each do |r|
-          record = Record.new(meta: r[:meta], data: r[:record_data] || Hash.new)
-          if data && !raw
-            record = decrypt_record(record)
-          end
-          yield record
-        end
-
-        if results.length < q.count
-          break
+                    user_ids: nil, count: page_size,
+                    include_all_writers: all_writers)
+      result = Result.new(self, q, raw)
+      if block_given?
+        result.each do |rec|
+          yield rec
         end
-
-        q.after_index = json[:last_index]
+      else
+        result
       end
     end
 
     # Grant another E3DB client access to records of a particular type.
     #
     # @param type [String] type of records to share
-    # @param reader_id [String] client ID of reader to grant access to
+    # @param reader_id [String] client ID or e-mail address of reader to grant access to
     def share(type, reader_id)
       if reader_id == @config.client_id
         return
+      elsif reader_id.include? "@"
+        reader_id = client_info(reader_id).client_id
       end
 
       id = @config.client_id
@@ -301,6 +457,8 @@ module E3DB
     def revoke(type, reader_id)
       if reader_id == @config.client_id
         return
+      elsif reader_id.include? "@"
+        reader_id = client_info(reader_id).client_id
       end
 
       id = @config.client_id
@@ -308,9 +466,31 @@ module E3DB
       @conn.put(url, JSON.generate({:deny => [{:read => {}}]}))
     end
 
+    def outgoing_sharing
+      url = get_url('v1', 'storage', 'policy', 'outgoing')
+      resp = @conn.get(url)
+      json = JSON.parse(resp.body, symbolize_names: true)
+      return json.map {|x| OutgoingSharingPolicy.new(x)}
+    end
+
+    def incoming_sharing
+      url = get_url('v1', 'storage', 'policy', 'incoming')
+      resp = @conn.get(url)
+      json = JSON.parse(resp.body, symbolize_names: true)
+      return json.map {|x| IncomingSharingPolicy.new(x)}
+    end
+
     private
+
+    # Fetch a single page of query results. Used internally by {Client#query}.
+    def query1(query)
+      url = get_url('v1', 'storage', 'search')
+      resp = @conn.post(url, query.as_json)
+      return JSON.parse(resp.body, symbolize_names: true)
+    end
+
     def get_url(*paths)
-      sprintf('%s/%s', @config.api_url.chomp('/'), paths.map { |x| URI.escape x }.join('/'))
+      sprintf('%s/%s', @config.api_url.chomp('/'), paths.map { |x| CGI.escape x }.join('/'))
     end
   end
 end

data/lib/e3db/config.rb

@@ -7,14 +7,14 @@
 
 
 module E3DB
-  DEFAULT_API_URL = 'https://dev.e3db.com/'
+  DEFAULT_API_URL = 'https://api.e3db.com/'
 
   # Configuration and credentials for E3DB.
   #
   # Typically a configuration is loaded from a JSON file generated
   # during registration via the E3DB administration console
   # or command-line tool. To load a configuration from a JSON file,
-  # use {Config.load}.
+  # use {E3DB::Config.load}.
   #
   # @!attribute version
   #   @return [Int] the version number of the configuration format (currently 1)
@@ -47,7 +47,7 @@ module E3DB
     attribute :logging, Types::Bool
 
     # Load configuration from a JSON file created during registration
-    # or with {Config.save}.
+    # or with {E3DB::Config.save}.
     #
     # The configuration file should contain a single JSON object
     # with the following structure:

data/lib/e3db/crypto.rb

@@ -9,50 +9,16 @@
 module E3DB
   class Client
     private
-    def get_access_key(writer_id, user_id, reader_id, type)
-      ak_cache_key = [writer_id, user_id, type]
-      if @ak_cache.key? ak_cache_key
-        return @ak_cache[ak_cache_key]
-      end
-
-      url = get_url('v1', 'storage', 'access_keys', writer_id, user_id, reader_id, type)
-      resp = @conn.get(url)
-      json = JSON.parse(resp.body, symbolize_names: true)
-
-      k = json[:authorizer_public_key][:curve25519]
-      authorizer_pubkey = Crypto.decode_public_key(k)
-
-      fields     = json[:eak].split('.', 2)
-      ciphertext = Crypto.base64decode(fields[0])
-      nonce      = Crypto.base64decode(fields[1])
-      box        = RbNaCl::Box.new(authorizer_pubkey, @private_key)
-
-      ak = box.decrypt(nonce, ciphertext)
-      @ak_cache[ak_cache_key] = ak
-      ak
-    end
-
-    def put_access_key(writer_id, user_id, reader_id, type, ak)
-      ak_cache_key = [writer_id, user_id, type]
-      @ak_cache[ak_cache_key] = ak
-
-      reader_key = client_key(reader_id)
-      nonce = RbNaCl::Random.random_bytes(RbNaCl::Box.nonce_bytes)
-      eak   = RbNaCl::Box.new(reader_key, @private_key).encrypt(nonce, ak)
-
-      encoded_eak = sprintf('%s.%s', Crypto.base64encode(eak), Crypto.base64encode(nonce))
-
-      url = get_url('v1', 'storage', 'access_keys', writer_id, user_id, reader_id, type)
-      @conn.put(url, { :eak => encoded_eak })
-    end
-
-    def decrypt_record(encrypted_record)
-      record = Record.new(meta: encrypted_record.meta.clone, data: Hash.new)
-
+    def decrypt_record(record)
       writer_id = record.meta.writer_id
       user_id = record.meta.user_id
       type = record.meta.type
       ak = get_access_key(writer_id, user_id, @config.client_id, type)
+      decrypt_record_with_key(record, ak)
+    end
+
+    def decrypt_record_with_key(encrypted_record, ak)
+      record = Record.new(meta: encrypted_record.meta.clone, data: Hash.new)
 
       encrypted_record.data.each do |k, v|
         fields = v.split('.', 4)
@@ -99,6 +65,48 @@ module E3DB
 
       record
     end
+
+    def decrypt_eak(json)
+      k = json[:authorizer_public_key][:curve25519]
+      authorizer_pubkey = Crypto.decode_public_key(k)
+
+      fields     = json[:eak].split('.', 2)
+      ciphertext = Crypto.base64decode(fields[0])
+      nonce      = Crypto.base64decode(fields[1])
+      box        = RbNaCl::Box.new(authorizer_pubkey, @private_key)
+
+      box.decrypt(nonce, ciphertext)
+    end
+
+    def get_access_key(writer_id, user_id, reader_id, type)
+      ak_cache_key = [writer_id, user_id, type]
+      if @ak_cache.key? ak_cache_key
+        return @ak_cache[ak_cache_key]
+      end
+
+      url = get_url('v1', 'storage', 'access_keys', writer_id, user_id, reader_id, type)
+      resp = @conn.get(url)
+      json = JSON.parse(resp.body, symbolize_names: true)
+
+      ak = decrypt_eak(json)
+      @ak_cache[ak_cache_key] = ak
+      ak
+    end
+
+    def put_access_key(writer_id, user_id, reader_id, type, ak)
+      ak_cache_key = [writer_id, user_id, type]
+      @ak_cache[ak_cache_key] = ak
+
+      reader_key = client_key(reader_id)
+      nonce = RbNaCl::Random.random_bytes(RbNaCl::Box.nonce_bytes)
+      eak   = RbNaCl::Box.new(reader_key, @private_key).encrypt(nonce, ak)
+
+      encoded_eak = sprintf('%s.%s', Crypto.base64encode(eak), Crypto.base64encode(nonce))
+
+      url = get_url('v1', 'storage', 'access_keys', writer_id, user_id, reader_id, type)
+      @conn.put(url, { :eak => encoded_eak })
+    end
+
   end
 
   class Crypto

data/lib/e3db/version.rb

@@ -1,3 +1,3 @@
 module E3DB
-  VERSION = "1.0.0"
+  VERSION = "2.0.0.rc1"
 end

data/travis-install-configfile.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# The purpose of this file is to install a default
+# e3db profile configuration so tests can execute
+# against a live server.
+
+set -e
+
+# Check if the config is already set
+if [ ! -d "$HOME/.tozny/integration-test" ]; then
+    mkdir -p "$HOME/.tozny/integration-test"
+fi
+
+cat > "$HOME/.tozny/integration-test/e3db.json" <<EOT
+{
+    "version":1,
+    "api_url":"${API_URL}",
+    "api_key_id":"${API_KEY_ID}",
+    "api_secret":"${API_SECRET}",
+    "client_id":"${CLIENT_ID}",
+    "client_email":"${CLIENT_EMAIL}",
+    "public_key":"${PUBLIC_KEY}",
+    "private_key":"${PRIVATE_KEY}"
+}
+EOT

data/travis-install-libsodium.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+# The purpose of this file is to install libsodium in
+# the Travis CI environment. Outside this environment,
+# you would probably not want to install it like this.
+
+set -e
+
+# check if libsodium is already installed
+if [ ! -d "$HOME/libsodium/lib" ]; then
+  wget https://github.com/jedisct1/libsodium/releases/download/1.0.12/libsodium-1.0.12.tar.gz
+  tar xvfz libsodium-1.0.12.tar.gz
+  cd libsodium-1.0.12
+  ./configure --prefix=$HOME/libsodium
+  make
+  make install
+else
+  echo 'Using cached directory.'
+fi

metadata

@@ -1,165 +1,179 @@
 --- !ruby/object:Gem::Specification
 name: e3db
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 2.0.0.rc1
 platform: ruby
 authors:
 - Tozny, LLC
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-04 00:00:00.000000000 Z
+date: 2017-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.12'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.14.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.14.1
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: dry-struct
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: lru_redux
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
   name: rbnacl
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 4.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 4.0.2
 - !ruby/object:Gem::Dependency
   name: net-http-persistent
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.9.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.9.4
 - !ruby/object:Gem::Dependency
   name: faraday_middleware
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.11.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.11.0.1
 - !ruby/object:Gem::Dependency
   name: oauth2
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.3.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.3.1
 description: 
@@ -169,9 +183,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
-- .travis.yml
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- ".yardopts"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -179,12 +194,15 @@ files:
 - bin/console
 - bin/setup
 - e3db.gemspec
+- examples/simple.rb
 - lib/e3db.rb
 - lib/e3db/client.rb
 - lib/e3db/config.rb
 - lib/e3db/crypto.rb
 - lib/e3db/types.rb
 - lib/e3db/version.rb
+- travis-install-configfile.sh
+- travis-install-libsodium.sh
 homepage: https://tozny.com/e3db
 licenses:
 - MIT
@@ -195,17 +213,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14.1
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: e3db client SDK