selfsdk 0.0.217 → 0.0.219

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b46d28c22f1f2bc5580d68f247bec53aac495765ea0199e59e21ecefd6f29e9
-  data.tar.gz: b0b84f89bdda28f94c2a81e56c71c420c480fe3b69e9425ab393e85e92a0e199
+  metadata.gz: 390f0ac2c3808caf3c347697209a0cea518844c1fc716412c76377715dff18b7
+  data.tar.gz: 8507869d086a970157451a51bafc1e3585e3f053c34c75ffcbf8752677f02c9c
 SHA512:
-  metadata.gz: 77173c08e398e75416d6711da44edc8a12062fae4dcf51e7b302dd9fab7fb8654af3904bd504aad7a980dd269fe6bd831453d005ec6352f7d60c0d08b754bb71
-  data.tar.gz: c9c443f7570a352bcfd5c65250e7b6b7ca5b3c9bc0dd1017170186df0bf706aa227e74924a24427dfd350ff0f967e126fd55da1517e7e8a6c1d6541cd982a6b1
+  metadata.gz: 3f212c0f88eeb44cfca786cb3a04e08748b63e7dba9dc56c356da598963c7f0917d37b70595925eaf31e15c35d1d7dff818caa08a789533cb6cc2a0303391141
+  data.tar.gz: adb386b6f42bea923763019ddad12310ff7719ff13c6191f6ed06d49a19f1f8aa111c62f1a5a9eff62e06fe13f1f551d1ddfc26830cd8e66b1a61f3cde2692df

data/lib/client.rb

@@ -99,7 +99,7 @@ module SelfSDK
           next if body[:error_id] == 'token_expired'
 
           break
-        rescue StandardError => e
+        rescue StandardError
           # retry if the server is down
         end
         sleep 2

data/lib/crypto.rb

@@ -1,37 +1,43 @@
 # Copyright 2020 Self Group Ltd. All Rights Reserved.
 
 require 'self_crypto'
+require_relative 'storage'
 
 module SelfSDK
   class Crypto
-    def initialize(client, device, storage_folder, storage_key)
+    def initialize(client, device, storage, storage_key)
       @client = client
       @device = device
       @storage_key = storage_key
-      @storage_folder = "#{storage_folder}/#{@client.jwt.key_id}"
       @lock_strategy = true
       @mode = "r+"
+      @storage = storage
+      @keys = {}
+      @mutex = Mutex.new
+
+      @storage.tx do
+        ::SelfSDK.logger.debug('Checking if account exists...')
+        if @storage.account_exists?
+          # 1a) if alice's account file exists load the pickle from the file
+          @account = SelfCrypto::Account.from_pickle(@storage.account_olm, @storage_key)
+        else
+          # 1b-i) if create a new account for alice if one doesn't exist already
+          @account = SelfCrypto::Account.from_seed(@client.jwt.key)
 
-      if File.exist?(account_path)
-        # 1a) if alice's account file exists load the pickle from the file
-        @account = SelfCrypto::Account.from_pickle(File.read(account_path), @storage_key)
-      else
-        # 1b-i) if create a new account for alice if one doesn't exist already
-        @account = SelfCrypto::Account.from_seed(@client.jwt.key)
-
-        # 1b-ii) generate some keys for alice and publish them
-        @account.gen_otk(100)
+          # 1b-ii) generate some keys for alice and publish them
+          @account.gen_otk(100)
 
-        # 1b-iii) convert those keys to json
-        keys = @account.otk['curve25519'].map{|k,v| {id: k, key: v}}.to_json
+          # 1b-iii) convert those keys to json
+          @keys = @account.otk['curve25519'].map{|k,v| {id: k, key: v}}
+          keys = @keys.to_json
 
-        # 1b-iv) post those keys to POST /v1/identities/<selfid>/devices/1/pre_keys/
-        res = @client.post("/v1/apps/#{@client.jwt.id}/devices/#{@device}/pre_keys", keys)
-        raise 'unable to push prekeys, please try in a few minutes' if res.code != 200
+          # 1b-iv) post those keys to POST /v1/identities/<selfid>/devices/1/pre_keys/
+          res = @client.post("/v1/apps/#{@client.jwt.id}/devices/#{@device}/pre_keys", keys)
+          raise 'unable to push prekeys, please try in a few minutes' if res.code != 200
 
-        # 1b-v) store the account to a file
-        FileUtils.mkdir_p(@storage_folder)
-        File.write(account_path, @account.to_pickle(storage_key))
+          # 1b-v) store the account to a file
+          @storage.account_create(@account.to_pickle(@storage_key))
+        end
       end
     end
 
@@ -43,119 +49,79 @@ module SelfSDK
       gs = SelfCrypto::GroupSession.new("#{@client.jwt.id}:#{@device}")
 
       sessions = {}
-      locks = {}
-      ::SelfSDK.logger.debug('- [crypto] managing sessions with all recipients')
-
-      recipients.each do |r|
-        f = nil
-        next if r[:id] == @client.jwt.id && r[:device_id] == @device
-
-        session_file_name = session_path(r[:id], r[:device_id])
-        session_with_bob = nil
-
-        begin
-          if File.exist?(session_file_name)
-            # Lock the session file
-            locks[session_file_name] = File.open(session_file_name, @mode)
-            locks[session_file_name].flock(File::LOCK_EX)
-          end
-          session_with_bob = get_outbound_session_with_bob(locks[session_file_name], r[:id], r[:device_id])
+      ct = ""
+      tx do
+        gs = SelfCrypto::GroupSession.new("#{@client.jwt.id}:#{@device}")
+        recipients.each do |r|
+          sid = @storage.sid(r[:id], r[:device_id])
+          next if sid == @storage.app_id
+
+          session_with_bob = get_outbound_session_with_bob(@storage.session_get_olm(sid), r[:id], r[:device_id])
+          ::SelfSDK.logger.debug("- [crypto]   adding group participant #{r[:id]}:#{r[:device_id]}")
+          gs.add_participant("#{r[:id]}:#{r[:device_id]}", session_with_bob)
+          sessions[sid] = session_with_bob
         rescue => e
           ::SelfSDK.logger.warn("- [crypto]   there is a problem adding group participant #{r[:id]}:#{r[:device_id]}, skipping...")
           ::SelfSDK.logger.warn("- [crypto] #{e}")
           next
         end
 
-        ::SelfSDK.logger.debug("- [crypto]   adding group participant #{r[:id]}:#{r[:device_id]}")
-        gs.add_participant("#{r[:id]}:#{r[:device_id]}", session_with_bob)
-        sessions[session_file_name] = session_with_bob
-      end
-
-      # 5) encrypt a message
-      ::SelfSDK.logger.debug("- [crypto] group encrypting message")
-      ct = gs.encrypt(message).to_s
+        # 5) encrypt a message
+        ::SelfSDK.logger.debug("- [crypto] group encrypting message")
+        ct = gs.encrypt(message).to_s
 
-      # 6) store the session to a file
-      ::SelfSDK.logger.debug("- [crypto] storing sessions")
-      sessions.each do |session_file_name, session_with_bob|
-        pickle = session_with_bob.to_pickle(@storage_key)
-        if locks[session_file_name]
-          locks[session_file_name].rewind
-          locks[session_file_name].write(pickle)
-          locks[session_file_name].truncate(locks[session_file_name].pos)
-        else
-          File.write(session_file_name, pickle)
+        # 6) store the session to a file
+        ::SelfSDK.logger.debug("- [crypto] storing sessions")
+        sessions.each do |sid, session_with_bob|
+          pickle = session_with_bob.to_pickle(@storage_key)
+          @storage.session_update(sid, pickle)
         end
       end
-
       ct
-    ensure
-      locks.each do |session_file_name, lock|
-        # Unlock the file
-        if lock
-          lock.flock(File::LOCK_UN)
-        end
-      end
     end
 
-    def decrypt(message, sender, sender_device)
-      f = nil
-      ::SelfSDK.logger.debug("- [crypto] decrypting a message")
-      session_file_name = session_path(sender, sender_device)
+    def decrypt(message, sender, sender_device, offset)
+      ::SelfSDK.logger.debug("- [crypto] decrypting a message #{message}")
+      sid = @storage.sid(sender, sender_device)
 
-      if File.exist?(session_file_name)
-        # Lock the session file
-        f = File.open(session_file_name, @mode)
-        f.flock(File::LOCK_EX)
-      end
-
-      ::SelfSDK.logger.debug("- [crypto] loading sessions")
-      session_with_bob = get_inbound_session_with_bob(f, message)
+      pt = ""
+      tx do
+        ::SelfSDK.logger.debug("- [crypto] loading sessions for #{sid}")
+        ::SelfSDK.logger.debug("- [crypto] #{@account.one_time_keys["curve25519"]}")
+        session_with_bob = get_inbound_session_with_bob(@storage.session_get_olm(sid), message)
 
-      # 8) create a group session and set the identity of the account you're using
-      ::SelfSDK.logger.debug("- [crypto] create a group session and set the identity of the account #{@client.jwt.id}:#{@device}")
-      gs = SelfCrypto::GroupSession.new("#{@client.jwt.id}:#{@device}")
+        # 8) create a group session and set the identity of the account you're using
+        ::SelfSDK.logger.debug("- [crypto] create a group session and set the identity of the account #{@client.jwt.id}:#{@device}")
+        gs = SelfCrypto::GroupSession.new("#{@client.jwt.id}:#{@device}")
 
-      # 9) add all recipients and their sessions
-      ::SelfSDK.logger.debug("- [crypto] add all recipients and their sessions #{@sender}:#{@sender_device}")
-      gs.add_participant("#{sender}:#{sender_device}", session_with_bob)
+        # 9) add all recipients and their sessions
+        ::SelfSDK.logger.debug("- [crypto] add all recipients and their sessions #{sender}:#{sender_device}")
+        gs.add_participant("#{sender}:#{sender_device}", session_with_bob)
 
-      # 10) decrypt the message ciphertext
-      ::SelfSDK.logger.debug("- [crypto] decrypt the message ciphertext")
-      pt = gs.decrypt("#{sender}:#{sender_device}", message).to_s
+        # 10) decrypt the message ciphertext
+        ::SelfSDK.logger.debug("- [crypto] decrypt the message ciphertext")
+        pt = gs.decrypt("#{sender}:#{sender_device}", message).to_s
 
-      # 11) store the session to a file
-      ::SelfSDK.logger.debug("- [crypto] store the session to a file")
+        # 11) store the session to a file
+        ::SelfSDK.logger.debug("- [crypto] store the session to a file")
 
-      pickle = session_with_bob.to_pickle(@storage_key)
-      if !f.nil?
-        f.rewind
-        f.write(pickle)
-        f.truncate(f.pos)
-      else
-        File.write(session_file_name, pickle)
+        pickle = session_with_bob.to_pickle(@storage_key)
+        @storage.session_update(sid, pickle)
+        @storage.account_set_offset(offset)
       end
-
       pt
-    ensure
-      # Unlock the session file
-      f&.flock(File::LOCK_UN)
-      f&.close
+    rescue => e
+      ::SelfSDK.logger.debug("- [crypto] ERROR DECRYPTING: original message:  #{message}")
+      ::SelfSDK.logger.debug("- [crypto] ERROR DECRYPTING: exception message: #{e.message}")
+      ::SelfSDK.logger.debug("- [crypto] ERROR DECRYPTING: exception backtrace: #{e.backtrace}")
+      @storage.account_set_offset(offset)
+      pt
     end
 
     private
 
-    def account_path
-      "#{@storage_folder}/account.pickle"
-    end
-
-    def session_path(selfid, device)
-      "#{@storage_folder}/#{selfid}:#{device}-session.pickle"
-    end
-
-    def get_outbound_session_with_bob(f, recipient, recipient_device)
-      if !f.nil?
-        pickle = f.read
+    def get_outbound_session_with_bob(pickle, recipient, recipient_device)
+      if !pickle.nil?
         # 2a) if bob's session file exists load the pickle from the file
         session_with_bob = SelfCrypto::Session.from_pickle(pickle, @storage_key)
       else
@@ -184,18 +150,24 @@ module SelfSDK
       session_with_bob
     end
 
-    def get_inbound_session_with_bob(f, message)
-      if !f.nil?
-        pickle = f.read
+    def get_inbound_session_with_bob(pickle, message)
+      if !pickle.nil?
         # 7a) if carol's session file exists load the pickle from the file
         session_with_bob = SelfCrypto::Session.from_pickle(pickle, @storage_key)
       end
 
+      ::SelfSDK.logger.debug("- [crypto] pickle nil? #{pickle.nil?}")
+      ::SelfSDK.logger.debug("- [crypto] getting one time message for #{@client.jwt.id}:#{@device}")
+
       # 7b-i) if you have not previously sent or received a message to/from bob,
       #       you should extract the initial message from the group message intended
       #       for your account id.
       m = SelfCrypto::GroupMessage.new(message.to_s).get_message("#{@client.jwt.id}:#{@device}")
 
+      ::SelfSDK.logger.debug("- [crypto] one time message #{m}")
+      ::SelfSDK.logger.debug("- [crypto] session is nil? #{session_with_bob.nil?}")
+      ::SelfSDK.logger.debug("- [crypto] is prekey message? #{m.instance_of?(SelfCrypto::PreKeyMessage)}")
+
       # if there is no session, create one
       # if there is an existing session and we are sent a one time key message, check
       # if it belongs to this current session and create a new inbound session if it doesn't
@@ -204,6 +176,7 @@ module SelfSDK
         session_with_bob = @account.inbound_session(m)
 
         # 7b-iii) remove the session's prekey from the account
+        ::SelfSDK.logger.debug "- [crypto] removing one time keys for bob #{session_with_bob}"
         @account.remove_one_time_keys(session_with_bob)
 
         current_one_time_keys = @account.otk['curve25519']
@@ -221,12 +194,20 @@ module SelfSDK
           res = @client.post("/v1/apps/#{@client.jwt.id}/devices/#{@device}/pre_keys", keys.to_json)
           raise 'unable to push prekeys, please try in a few minutes' if res.code != 200
         end
-
-        File.write(account_path, @account.to_pickle(@storage_key))
+        ::SelfSDK.logger.debug "- [crypto] updating account with removed keys"
+        @storage.account_update(@account.to_pickle(@storage_key))
       end
 
       session_with_bob
     end
 
+    def tx
+      @mutex.lock
+      @storage.tx do
+        yield
+      end
+    ensure
+      @mutex.unlock
+    end
   end
 end

data/lib/messages/message.rb

@@ -31,7 +31,7 @@ module SelfSDK
              else
                 envelope = input
                 issuer = input.sender.split(":")
-                messaging.encryption_client.decrypt(input.ciphertext, issuer.first, issuer.last)
+                messaging.encryption_client.decrypt(input.ciphertext, issuer.first, issuer.last, input.offset)
              end
 
       jwt = JSON.parse(body, symbolize_names: true)

data/lib/messaging.rb

@@ -13,6 +13,7 @@ require_relative 'messages/message'
 module SelfSDK
   class WebsocketClient
     ON_DEMAND_CLOSE_CODE=3999
+    CONNECTION_SUPERCEDED=1011
 
     attr_accessor :ws
 
@@ -42,7 +43,7 @@ module SelfSDK
       @ws.on :close do |event|
         SelfSDK.logger.debug "connection closed detected : #{event.code} #{event.reason}"
 
-        if event.code != ON_DEMAND_CLOSE_CODE
+        if not [ON_DEMAND_CLOSE_CODE, CONNECTION_SUPERCEDED].include? event.code
           raise StandardError('websocket connection closed') if !@auto_reconnect
 
           if !@reconnection_delay.nil?
@@ -59,6 +60,7 @@ module SelfSDK
 
     # Sends a closing message to the websocket client.
     def close
+      SelfSDK.logger.debug "connection closed by the client"
       @ws.close(ON_DEMAND_CLOSE_CODE, "connection closed by the client")
     end
 
@@ -80,7 +82,6 @@ module SelfSDK
   class MessagingClient
     DEFAULT_DEVICE="1"
     DEFAULT_AUTO_RECONNECT=true
-    DEFAULT_STORAGE_DIR="./.self_storage"
 
     PRIORITIES = { 
       'chat.invite':                 SelfSDK::Messages::PRIORITY_VISIBLE,
@@ -104,12 +105,12 @@ module SelfSDK
     #
     # @param url [string] self-messaging url
     # @params client [Object] SelfSDK::Client object
-    # @option opts [string] :storage_dir  the folder where encryption sessions and settings will be stored
+    # @option opts [string] :storage  the library used to persist session data.
     # @params storage_key [String] seed to encrypt messaging
     # @params storage_folder [String] folder to perist messaging encryption
     # @option opts [Bool] :auto_reconnect Automatically reconnects to websocket if connection is lost (defaults to true).
     # @option opts [String] :device_id The device id to be used by the app defaults to "1".
-    def initialize(url, client, storage_key, options = {})
+    def initialize(url, client, storage_key, storage, options = {})
       @mon = Monitor.new
       @messages = {}
       @acks = {}
@@ -121,18 +122,13 @@ module SelfSDK
       @timeout = 120 # seconds
       @auth_id = SecureRandom.uuid
       @device_id = options.fetch(:device_id, DEFAULT_DEVICE)
-      @raw_storage_dir = options.fetch(:storage_dir, DEFAULT_STORAGE_DIR)
-      @storage_dir = "#{@raw_storage_dir}/apps/#{@jwt.id}/devices/#{@device_id}"
-      FileUtils.mkdir_p @storage_dir unless File.exist? @storage_dir
-      @offset_file = "#{@storage_dir}/#{@jwt.id}:#{@device_id}.offset"
-      @offset = read_offset
       @source = SelfSDK::Sources.new("#{__dir__}/sources.json")
+      @storage = storage
 
       unless options.include? :no_crypto
-        crypto_path = "#{@storage_dir}/keys"
-        FileUtils.mkdir_p crypto_path unless File.exist? crypto_path
-        @encryption_client = Crypto.new(@client, @device_id, crypto_path, storage_key)
+        @encryption_client = Crypto.new(@client, @device_id, @storage, storage_key)
       end
+      @offset = @storage.account_offset
 
       @ws = if options.include? :ws
               options[:ws]
@@ -352,7 +348,7 @@ module SelfSDK
 
       SelfSDK.logger.debug " - notifying by type"
       SelfSDK.logger.debug " - #{message.typ}"
-      SelfSDK.logger.debug " - #{message}"
+      SelfSDK.logger.debug " - #{message.payload}"
       SelfSDK.logger.debug " - #{@type_observer.keys.join(',')}"
 
       # Return if there is no observer setup for this kind of message
@@ -371,6 +367,7 @@ module SelfSDK
 
     def subscribe(type, &block)
       type = @source.message_type(type) if type.is_a? Symbol
+      SelfSDK.logger.debug "Subscribing to messages by type: #{type}"
       @type_observer[type] = { block: block }
     end
 
@@ -431,7 +428,7 @@ module SelfSDK
     end
 
     def process_incomming_message(input)
-      message = parse_and_write_offset(input)
+      message = parse(input)
 
       if @messages.include? message.id
         message.validate! @messages[message.id][:original_message]
@@ -450,21 +447,17 @@ module SelfSDK
       nil
     end
 
-    def parse_and_write_offset(input)
+    def parse(input)
       msg = SelfSDK::Messages.parse(input, self)
-      write_offset(input.offset)
-      # Avoid catching any other decryption errors.
       msg
-    rescue SelfSDK::Messages::UnmappedMessage => e
-      # this is an ummapped message, let's ignore it but write the offset.
-      write_offset(input.offset)
+    rescue SelfSDK::Messages::UnmappedMessage
       nil
     end
 
     # Authenticates current client on the websocket server.
     def authenticate
       @auth_id = SecureRandom.uuid if @auth_id.nil?
-      @offset = read_offset
+      @offset = @storage.account_offset
 
       SelfSDK.logger.debug "authenticating with offset (#{@offset})"
 
@@ -500,23 +493,6 @@ module SelfSDK
       end
     end
 
-    def read_offset
-      return 0 unless File.exist? @offset_file
-
-      File.open(@offset_file, 'rb') do |f|
-        return f.read.to_i
-      end
-    end
-
-    def write_offset(offset)
-      File.open(@offset_file, 'wb') do |f|
-        f.flock(File::LOCK_EX)
-        f.write(offset.to_s.rjust(19, "0"))
-      end
-      SelfSDK.logger.debug "offset written #{offset}"
-      @offset = offset
-    end
-
     def select_priority(mtype)
       PRIORITIES[mtype] || SelfSDK::Messages::PRIORITY_VISIBLE
     end

data/lib/selfsdk.rb

@@ -49,6 +49,7 @@ module SelfSDK
     # @option opts [String] :messaging_url The messaging self provider url.
     # @option opts [Bool] :auto_reconnect Automatically reconnects to websocket if connection is lost (defaults to true).
     # @option opts [Symbol] :env The environment to be used, defaults to ":production".
+    # @option opts [String] :storage  the library used to persist session data.
     def initialize(app_id, app_key, storage_key, storage_dir, opts = {})
       app_key = cleanup_key(app_key)
 
@@ -60,9 +61,12 @@ module SelfSDK
       messaging_url = messaging_url(opts)
       @started = false
       unless messaging_url.nil?
+        device_id = opts.fetch(:device_id, MessagingClient::DEFAULT_DEVICE)
+        storage = opts.fetch(:storage, SelfSDK::Storage.new(@client.jwt.id, device_id, storage_dir, storage_key))
         @messaging_client = MessagingClient.new(messaging_url,
                                                 @client,
                                                 storage_key,
+                                                storage,
                                                 storage_dir: storage_dir,
                                                 auto_reconnect: opts.fetch(:auto_reconnect, MessagingClient::DEFAULT_AUTO_RECONNECT),
                                                 device_id: opts.fetch(:device_id, MessagingClient::DEFAULT_DEVICE))

data/lib/storage.rb

@@ -0,0 +1,236 @@
+# Copyright 2020 Self Group Ltd. All Rights Reserved.
+require 'sqlite3'
+
+module SelfSDK
+  class Storage
+    attr_accessor :app_id
+
+    def initialize(app_id, app_device, storage_folder, _key_id)
+      @app_id = sid(app_id, app_device)
+
+      # Create the storage folder if it does not exist
+      create_directory_skel("#{storage_folder}/identities/#{app_id}/devices/#{app_device}")
+
+      # Create the database
+      @db = SQLite3::Database.new(File.join("#{storage_folder}/identities/#{app_id}/devices/#{app_device}/", 'self.db'))
+      set_pragmas
+      create_accounts_table
+      create_sessions_table
+      m = StorageMigrator.new(@db, "#{storage_folder}/apps", @app_id)
+      m.migrate
+    end
+
+    def tx
+      @db.transaction
+      yield
+      @db.commit
+    rescue SQLite3::Exception => e
+      @db.rollback
+    rescue => e
+      @db.rollback
+      raise e
+    end
+
+    def account_exists?
+      row = @db.execute("SELECT olm_account FROM accounts WHERE as_identifier = ?", [@app_id.encode("UTF-8")]).first
+      !row.nil?
+    end
+
+    def account_create(olm)
+      @db.execute("INSERT INTO accounts (as_identifier, offset, olm_account) VALUES (?, ?, ?);", [ @app_id.encode("UTF-8"), 0, olm.encode("UTF-8") ])
+    rescue
+    end
+
+    def account_update(olm)
+      @db.execute("UPDATE accounts SET olm_account = ? WHERE as_identifier = ?", [ olm.encode("UTF-8"), @app_id.encode("UTF-8") ])
+    end
+
+    def account_olm
+      row = @db.execute("SELECT olm_account FROM accounts WHERE as_identifier = ?;", [@app_id.encode("UTF-8")]).first
+      return nil unless row
+
+      row.first
+    end
+
+    def account_offset
+      row = @db.execute("SELECT offset FROM accounts WHERE as_identifier = ?;", @app_id.encode("UTF-8")).first
+      return nil unless row
+
+      row.first
+    end
+
+    def account_set_offset(offset)
+      @db.execute("UPDATE accounts SET offset = ? WHERE as_identifier = ?;", [ offset, @app_id.encode("UTF-8") ])
+    end
+
+    def session_create(sid, olm)
+      @db.execute("INSERT INTO sessions (as_identifier, with_identifier, olm_session) VALUES (?, ?, ?);", [ @app_id.encode("UTF-8"), sid.encode("UTF-8"), olm.encode("UTF-8") ])
+    end
+
+    def session_update(sid, olm)
+      row = @db.execute("SELECT olm_session FROM sessions WHERE as_identifier = ? AND with_identifier = ?", [@app_id.encode("UTF-8"), sid.encode("UTF-8")]).first
+      if row.nil?
+        session_create(sid, olm)
+      else
+        @db.execute("UPDATE sessions SET olm_session = ? WHERE as_identifier = ? AND with_identifier = ?;", [ olm.encode("UTF-8"), @app_id.encode("UTF-8"), sid.encode("UTF-8") ])
+      end
+    end
+
+    def session_get_olm(sid)
+      row = @db.execute("SELECT olm_session FROM sessions WHERE as_identifier = ? AND with_identifier = ?", [@app_id.encode("UTF-8"), sid.encode("UTF-8")]).first
+      return nil if row.nil?
+
+      row.first
+    end
+
+    def sid(selfid, device)
+      "#{selfid}:#{device}"
+    end
+
+    private
+
+    # Create a folder if it does not exist
+    def create_directory_skel(storage_folder)
+      FileUtils.mkdir_p storage_folder unless File.exist? storage_folder
+    rescue Errno::ENOENT
+      raise ERR_INVALID_DIRECTORY
+    rescue
+    end
+
+    def set_pragmas
+      pragma_statement = <<~SQL
+        PRAGMA synchronous = NORMAL;
+        PRAGMA journal_mode = WAL;
+        PRAGMA temp_store = MEMORY;
+      SQL
+
+      @db.execute_batch(pragma_statement)
+    rescue SQLite3::Exception => e
+      puts "Exception occurred: #{e}"
+    end
+
+    def create_accounts_table
+      accounts_table_statement = <<~SQL
+        CREATE TABLE IF NOT EXISTS accounts (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          as_identifier TEXT NOT NULL,
+          offset INTEGER NOT NULL,
+          olm_account BLOB NOT NULL
+        );
+        CREATE UNIQUE INDEX IF NOT EXISTS idx_accounts_as_identifier
+        ON accounts (as_identifier);
+      SQL
+
+      @db.execute_batch(accounts_table_statement)
+    rescue SQLite3::Exception => e
+      puts "Exception occurred: #{e}"
+    end
+
+    def create_sessions_table
+      # TODO we could deduplicate as_identifier and with_identifier here
+      # by creating a record for each on a new identifier table,
+      # but this is only temporary
+      session_table_statement = <<~SQL
+        CREATE TABLE IF NOT EXISTS sessions (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          as_identifier TEXT NOT NULL,
+          with_identifier TEXT NOT NULL,
+          olm_session BLOB NOT NULL
+        );
+        CREATE UNIQUE INDEX IF NOT EXISTS idx_sessions_with_identifier
+        ON sessions (as_identifier, with_identifier);
+      SQL
+
+      @db.execute_batch(session_table_statement)
+    rescue SQLite3::Exception => e
+      puts "Exception occurred: #{e}"
+    end
+  end
+
+  class StorageMigrator
+    def initialize(db, storage_folder, app_id)
+      @db = db
+      # Old versions of the sdk using that same storage folder shouldn't be affected in any way
+
+      @base_path = "#{storage_folder}/#{app_id.split(':').first}"
+      @app_id = app_id
+    end
+
+    def migrate
+      return unless File.exist?(@base_path)
+
+      # Parse the account information.
+      accounts = parse_accounts
+
+      persist_accounts(accounts)
+
+      # Depreciate the base path.
+      File.rename("#{@base_path}", "#{@base_path}-depreciated")
+    end
+
+    private
+
+    def parse_accounts
+      accounts = {}
+
+      Dir.glob(File.join(@base_path, "**/*")).each do |path|
+        if File.directory?(path)
+          next
+        end
+
+        case File.extname(path)
+        when ".offset"
+          file_name = File.basename(path, ".offset")
+          offset = File.read(path)[0, 19].to_i
+
+          accounts[file_name] = {} unless accounts.key? file_name
+          accounts[file_name][:offset] = offset
+        when ".pickle"
+          file_name = File.basename(path, ".pickle")
+          content = File.read(path)
+
+          accounts[@app_id] = {} unless accounts.key? @app_id
+          if file_name == "account"
+            accounts[@app_id][:account] = content
+          else
+            if accounts.key? @app_id
+              accounts[@app_id][:sessions] = [] unless accounts[@app_id].key? :sessions
+              accounts[@app_id][:sessions] << {
+                with: file_name.sub("-session", ""),
+                session: content
+              }
+              next
+            end
+            accounts[@app_id][:account] = content
+          end
+        end
+      end
+
+      accounts
+    end
+
+    def persist_accounts(accounts)
+      @db.transaction
+      accounts.each do |inbox_id, account|
+        @db.execute(
+          "INSERT INTO accounts (as_identifier, offset, olm_account) VALUES ($1, $2, $3)",
+          [inbox_id, account[:offset], account[:account]]
+        )
+
+        account[:sessions].each do |session|
+          @db.execute(
+            "INSERT INTO sessions (as_identifier, with_identifier, olm_session) VALUES ($1, $2, $3)",
+            [inbox_id, session[:with], session[:session]]
+          )
+        end
+      end
+
+      @db.commit
+    rescue SQLite3::Exception => e
+      puts "Exception occurred"
+      puts e
+      puts e.backtrace
+      @db.rollback
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: selfsdk
 version: !ruby/object:Gem::Version
-  version: 0.0.217
+  version: 0.0.219
 platform: ruby
 authors:
 - Self Group Ltd.
@@ -178,6 +178,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -395,6 +409,7 @@ files:
 - lib/signature_graph.rb
 - lib/source_definition.rb
 - lib/sources.rb
+- lib/storage.rb
 homepage: https://www.joinself.com/
 licenses:
 - MIT