standard-file 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7507a20e41218ab29adb02614a3d56c8093b920e
-  data.tar.gz: d03b8b5fe61838f446b550bc289a4ba34226903a
+  metadata.gz: 4713fd9c76de2ac82f16a1fe84b27a53a25d4c3a
+  data.tar.gz: c85729b0055a6b4df9038f4804a4442905ed4dc5
 SHA512:
-  metadata.gz: 7f4b70e3fee538bf9709b2328f2832bed09d081042e779cf6762dc56453022a136797353c723315c910723f29d28ac4d417015c5845e31ed3d0b0d4fa03004ed
-  data.tar.gz: c3a9908c6eae5dc5ea533482a6e25daa3a5d52ce3e635e2a4bcc5b6e9e19e0510028e34bc7e97fdbead59b3b044f278d307ec529632e186c8c8325d697429ba8
+  metadata.gz: 6d42f737a016dba90c00cc661170b5b93f21a830664e65a3a2568f778bfd120e2ee344bbb8fa63ebbb9b7eb1a880a320f0685cf042090337c2c13f85535a7da1
+  data.tar.gz: 94fe38ec72e42df859cbae76cb50ac1affd49b131e8fa5e5dc366a4f5134c4003a2e3611663b235d1ae58553c40ee8357d4fb724c315a31863b5c424b806c927

data/lib/standard_file.rb

@@ -1,6 +1,10 @@
 require "standard_file/engine"
-require_relative 'standard_file/sync_manager'
-require_relative 'standard_file/user_manager'
+require_relative 'standard_file/abstract/sync_manager'
+require_relative 'standard_file/abstract/user_manager'
+require_relative 'standard_file/2016_12_15/sync_manager'
+require_relative 'standard_file/2019_05_20/sync_manager'
+require_relative 'standard_file/2016_12_15/user_manager'
+require_relative 'standard_file/2019_05_20/user_manager'
 require_relative 'standard_file/jwt_helper'
 
 module StandardFile

data/lib/standard_file/2016_12_15/sync_manager.rb

@@ -0,0 +1,143 @@
+module StandardFile
+  module V20161215
+    class SyncManager < StandardFile::AbstractSyncManager
+
+      def sync(item_hashes, options, request)
+        in_sync_token = options[:sync_token]
+        in_cursor_token = options[:cursor_token]
+        limit = options[:limit]
+        content_type = options[:content_type] # optional, only return items of these type if present
+
+        retrieved_items, cursor_token = _sync_get(in_sync_token, in_cursor_token, limit, content_type).to_a
+        last_updated = DateTime.now
+        saved_items, unsaved_items = _sync_save(item_hashes, request)
+        if saved_items.length > 0
+          last_updated = saved_items.sort_by{|m| m.updated_at}.last.updated_at
+        end
+
+        check_for_conflicts(saved_items, retrieved_items, unsaved_items)
+
+        # add 1 microsecond to avoid returning same object in subsequent sync
+        last_updated = (last_updated.to_time + 1/100000.0).to_datetime.utc
+
+        sync_token = sync_token_from_datetime(last_updated)
+        return {
+          :retrieved_items => retrieved_items,
+          :saved_items => saved_items,
+          :unsaved => unsaved_items,
+          :sync_token => sync_token,
+          :cursor_token => cursor_token
+        }
+      end
+
+      def check_for_conflicts(saved_items, retrieved_items, unsaved_items)
+        # conflicts occur when you are trying to save an item for which there is a pending change already
+        min_conflict_interval = 20
+
+        if Rails.env.development?
+          min_conflict_interval = 1
+        end
+
+        saved_ids = saved_items.map{|x| x.uuid }
+        retrieved_ids = retrieved_items.map{|x| x.uuid }
+        conflicts = saved_ids & retrieved_ids # & is the intersection
+        # saved items take precedence, retrieved items are duplicated with a new uuid
+        conflicts.each do |conflicted_uuid|
+          # if changes are greater than min_conflict_interval seconds apart,
+          # push the retrieved item in the unsaved array so that the client can duplicate it
+          saved = saved_items.find{|i| i.uuid == conflicted_uuid}
+          conflicted = retrieved_items.find{|i| i.uuid == conflicted_uuid}
+          if (saved.updated_at - conflicted.updated_at).abs > min_conflict_interval
+            # puts "\n\n\n Creating conflicted copy of #{saved.uuid}\n\n\n"
+
+            unsaved_items.push({
+              :item => conflicted,
+              :error => {:tag => "sync_conflict"}
+            })
+
+          end
+
+          # We remove the item from retrieved items whether or not it satisfies the min_conflict_interval
+          # This is because the 'saved' value takes precedence, since that's the current value in the database.
+          # So by removing it from retrieved, we are forcing the client to ignore this change.
+          retrieved_items.delete(conflicted)
+        end
+      end
+
+      private
+
+      def _sync_save(item_hashes, request)
+        if !item_hashes
+          return [], []
+        end
+        saved_items = []
+        unsaved = []
+
+        item_hashes.each do |item_hash|
+          begin
+            item = @user.items.find_or_create_by(:uuid => item_hash[:uuid])
+          rescue => error
+            unsaved.push({
+              :item => item_hash,
+              :error => {:message => error.message, :tag => "uuid_conflict"}
+              })
+            next
+          end
+
+          item.last_user_agent = request.user_agent
+          item.update(item_hash.permit(*permitted_params))
+          # we want to force update the updated_at field, even if no changes were made
+          # item.touch
+
+
+          if item.deleted == true
+            set_deleted(item)
+            item.save
+          end
+
+          saved_items.push(item)
+        end
+
+        return saved_items, unsaved
+      end
+
+      def _sync_get(sync_token, input_cursor_token, limit, content_type)
+        cursor_token = nil
+        if limit == nil
+          limit = 100000
+        end
+
+        # if both are present, cursor_token takes precendence as that would eventually return all results
+        # the distinction between getting results for a cursor and a sync token is that cursor results use a
+        # >= comparison, while a sync token uses a > comparison. The reason for this is that cursor tokens are
+        # typically used for initial syncs or imports, where a bunch of notes could have the exact same updated_at
+        # by using >=, we don't miss those results on a subsequent call with a cursor token
+        if input_cursor_token
+          date = datetime_from_sync_token(input_cursor_token)
+          items = @user.items.order(:updated_at).where("updated_at >= ?", date)
+        elsif sync_token
+          date = datetime_from_sync_token(sync_token)
+          items = @user.items.order(:updated_at).where("updated_at > ?", date)
+        else
+          # if no cursor token and no sync token, this is an initial sync. No need to return deleted items.
+          items = @user.items.order(:updated_at).where(:deleted => false)
+        end
+
+        if content_type
+          items = items.where(:content_type => content_type)
+        end
+
+        items = items.sort_by{|m| m.updated_at}
+
+        if items.count > limit
+          items = items.slice(0, limit)
+          date = items.last.updated_at
+          cursor_token = sync_token_from_datetime(date)
+        end
+
+        return items, cursor_token
+      end
+
+    end
+  end
+end

data/lib/standard_file/2016_12_15/user_manager.rb

@@ -0,0 +1,7 @@
+module StandardFile
+  module V20161215
+    class UserManager < StandardFile::AbstractUserManager
+
+    end
+  end
+end

data/lib/standard_file/2019_05_20/sync_manager.rb

@@ -0,0 +1,161 @@
+module StandardFile
+  module V20190520
+    class SyncManager < StandardFile::AbstractSyncManager
+
+      def sync(item_hashes, options, request)
+        in_sync_token = options[:sync_token]
+        in_cursor_token = options[:cursor_token]
+        limit = options[:limit]
+        content_type = options[:content_type] # optional, only return items of these type if present
+
+        retrieved_items, cursor_token = _sync_get(in_sync_token, in_cursor_token, limit, content_type).to_a
+        last_updated = DateTime.now
+        saved_items, conflicts = _sync_save(item_hashes, request, retrieved_items)
+
+        if saved_items.length > 0
+          last_updated = saved_items.sort_by{|m| m.updated_at}.last.updated_at
+        end
+
+        # add 1 microsecond to avoid returning same object in subsequent sync
+        last_updated = (last_updated.to_time + 1/100000.0).to_datetime.utc
+        sync_token = sync_token_from_datetime(last_updated)
+
+        return {
+          :retrieved_items => retrieved_items,
+          :saved_items => saved_items,
+          :conflicts => conflicts,
+          :sync_token => sync_token,
+          :cursor_token => cursor_token
+        }
+      end
+      
+
+      # Ignore differences that are at most this many seconds apart
+      # Anything over this threshold will be conflicted.
+      MIN_CONFLICT_INTERVAL = 1.0
+
+      def _sync_save(item_hashes, request, retrieved_items)
+        if !item_hashes
+          return [], []
+        end
+
+        saved_items = []
+        conflicts = []
+
+        item_hashes.each do |item_hash|
+          is_new_record = false
+          begin
+            item = @user.items.find_or_create_by(:uuid => item_hash[:uuid]) do |created_item|
+              # this block is executed if this is a new record.
+              is_new_record = true
+            end
+          rescue => error
+            conflicts.push({
+              :unsaved_item => item_hash,
+              :type => "uuid_conflict"
+            })
+            next
+          end
+
+          # SFJS did not send updated_at prior to 0.3.59.
+          # updated_at value from client will not be saved, as it is not a permitted_param.
+          if item_hash['updated_at']
+            incoming_updated_at = DateTime.parse(item_hash['updated_at'])
+          else
+            # Default to epoch
+            incoming_updated_at = Time.at(0).to_datetime
+          end
+
+          if !is_new_record
+            # We want to check if this updated_at value is equal to the item's current updated_at value.
+            # If they differ, it means the client is attempting to save an item which hasn't been updated.
+            # In this case, if the incoming_item.updated_at < server_item.updated_at, always conflict.
+            # We don't want old items overriding newer ones.
+            # incoming_item.updated_at > server_item.updated_at would seem to be impossible, as only servers are responsible for setting updated_at.
+            # But assuming a rogue client has gotten away with it,
+            # we should also conflict in this case if the difference between the dates is greater than MIN_CONFLICT_INTERVAL seconds.
+
+            save_incoming = true
+
+            our_updated_at = item.updated_at
+            difference = incoming_updated_at.to_f - our_updated_at.to_f
+
+            if difference < 0
+              # incoming is less than ours. This implies stale data. Don't save if greater than interval
+              save_incoming = difference.abs < MIN_CONFLICT_INTERVAL
+            elsif difference > 0
+              # incoming is greater than ours. Should never be the case. If so though, don't save.
+              save_incoming = difference.abs < MIN_CONFLICT_INTERVAL
+            else
+              # incoming is equal to ours (which is desired, healthy behavior), continue with saving.
+              save_incoming = true
+            end
+
+            if !save_incoming
+              # Dont save incoming and send it back. At this point the server item is likely to be included
+              # in retrieved_items in a subsequent sync, so when that value comes into the client,
+              server_value = item.as_json({})
+              conflicts.push({
+                :server_item => server_value, # as_json to get values as-is, befor modifying below,
+                :type => "sync_conflict"
+              })
+
+              retrieved_items.delete(item)
+              next
+            end
+          end
+
+          item.last_user_agent = request.user_agent
+          item.update(item_hash.permit(*permitted_params))
+
+          if item.deleted == true
+            set_deleted(item)
+            item.save
+          end
+
+          saved_items.push(item)
+        end
+
+        return saved_items, conflicts
+      end
+
+      def _sync_get(sync_token, input_cursor_token, limit, content_type)
+        cursor_token = nil
+        if limit == nil
+          limit = 100000
+        end
+
+        # if both are present, cursor_token takes precendence as that would eventually return all results
+        # the distinction between getting results for a cursor and a sync token is that cursor results use a
+        # >= comparison, while a sync token uses a > comparison. The reason for this is that cursor tokens are
+        # typically used for initial syncs or imports, where a bunch of notes could have the exact same updated_at
+        # by using >=, we don't miss those results on a subsequent call with a cursor token
+        if input_cursor_token
+          date = datetime_from_sync_token(input_cursor_token)
+          items = @user.items.order(:updated_at).where("updated_at >= ?", date)
+        elsif sync_token
+          date = datetime_from_sync_token(sync_token)
+          items = @user.items.order(:updated_at).where("updated_at > ?", date)
+        else
+          # if no cursor token and no sync token, this is an initial sync. No need to return deleted items.
+          items = @user.items.order(:updated_at).where(:deleted => false)
+        end
+
+        if content_type
+          items = items.where(:content_type => content_type)
+        end
+
+        items = items.sort_by{|m| m.updated_at}
+
+        if items.count > limit
+          items = items.slice(0, limit)
+          date = items.last.updated_at
+          cursor_token = sync_token_from_datetime(date)
+        end
+
+        return items, cursor_token
+      end
+
+    end
+  end
+end

data/lib/standard_file/2019_05_20/user_manager.rb

@@ -0,0 +1,7 @@
+module StandardFile
+  module V20190520
+    class UserManager < StandardFile::AbstractUserManager
+
+    end
+  end
+end

data/lib/standard_file/abstract/sync_manager.rb

@@ -0,0 +1,61 @@
+module StandardFile
+  class AbstractSyncManager
+
+    attr_accessor :sync_fields
+
+    def initialize(user)
+      @user = user
+      raise "User must be set" unless @user
+    end
+
+    def set_sync_fields(val)
+      @sync_fields = val
+    end
+
+    def sync_fields
+      return @sync_fields || [:content, :enc_item_key, :content_type, :auth_hash, :deleted, :created_at]
+    end
+
+    def destroy_items(uuids)
+      items = @user.items.where(uuid: uuids)
+      items.destroy_all
+    end
+
+    private
+
+    def sync_token_from_datetime(datetime)
+      version = 2
+      Base64.encode64("#{version}:" + "#{datetime.to_f}")
+    end
+
+    def datetime_from_sync_token(sync_token)
+      decoded = Base64.decode64(sync_token)
+      parts = decoded.rpartition(":")
+      timestamp_string = parts.last
+      version = parts.first
+      if version == "1"
+        date = DateTime.strptime(timestamp_string,'%s')
+      elsif version == "2"
+        date = Time.at(timestamp_string.to_f).to_datetime.utc
+      end
+
+      return date
+    end
+
+    def set_deleted(item)
+      item.deleted = true
+      item.content = nil if item.has_attribute?(:content)
+      item.enc_item_key = nil if item.has_attribute?(:enc_item_key)
+      item.auth_hash = nil if item.has_attribute?(:auth_hash)
+    end
+
+    def item_params
+      params.permit(*permitted_params)
+    end
+
+    def permitted_params
+      sync_fields
+    end
+
+  end
+end

data/lib/standard_file/{user_manager.rb → abstract/user_manager.rb}

@@ -1,5 +1,5 @@
 module StandardFile
-  class UserManager
+  class AbstractUserManager
 
     def initialize(user_class)
       @user_class = user_class

data/lib/standard_file/version.rb

@@ -1,3 +1,3 @@
 module StandardFile
-  VERSION = '0.3.1'
+  VERSION = '0.3.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: standard-file
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Standard File
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-10 00:00:00.000000000 Z
+date: 2019-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -64,10 +64,14 @@ files:
 - README.md
 - Rakefile
 - lib/standard_file.rb
+- lib/standard_file/2016_12_15/sync_manager.rb
+- lib/standard_file/2016_12_15/user_manager.rb
+- lib/standard_file/2019_05_20/sync_manager.rb
+- lib/standard_file/2019_05_20/user_manager.rb
+- lib/standard_file/abstract/sync_manager.rb
+- lib/standard_file/abstract/user_manager.rb
 - lib/standard_file/engine.rb
 - lib/standard_file/jwt_helper.rb
-- lib/standard_file/sync_manager.rb
-- lib/standard_file/user_manager.rb
 - lib/standard_file/version.rb
 - lib/tasks/standard_file_tasks.rake
 homepage: https://standardnotes.org

data/lib/standard_file/sync_manager.rb

@@ -1,197 +0,0 @@
-module StandardFile
-  class SyncManager
-
-    attr_accessor :sync_fields
-
-    def initialize(user)
-      @user = user
-      raise "User must be set" unless @user
-    end
-
-    def set_sync_fields(val)
-      @sync_fields = val
-    end
-
-    def sync_fields
-      return @sync_fields || [:content, :enc_item_key, :content_type, :auth_hash, :deleted, :created_at]
-    end
-
-    def sync(item_hashes, options, request)
-
-      in_sync_token = options[:sync_token]
-      in_cursor_token = options[:cursor_token]
-      limit = options[:limit]
-      content_type = options[:content_type] # optional, only return items of these type if present
-
-      retrieved_items, cursor_token = _sync_get(in_sync_token, in_cursor_token, limit, content_type).to_a
-      last_updated = DateTime.now
-      saved_items, unsaved_items = _sync_save(item_hashes, request)
-      if saved_items.length > 0
-        last_updated = saved_items.sort_by{|m| m.updated_at}.last.updated_at
-      end
-
-      check_for_conflicts(saved_items, retrieved_items, unsaved_items)
-
-      # add 1 microsecond to avoid returning same object in subsequent sync
-      last_updated = (last_updated.to_time + 1/100000.0).to_datetime.utc
-
-      sync_token = sync_token_from_datetime(last_updated)
-      return {
-        :retrieved_items => retrieved_items,
-        :saved_items => saved_items,
-        :unsaved => unsaved_items,
-        :sync_token => sync_token,
-        :cursor_token => cursor_token
-      }
-    end
-
-    def check_for_conflicts(saved_items, retrieved_items, unsaved_items)
-      # conflicts occur when you are trying to save an item for which there is a pending change already
-      min_conflict_interval = 20
-
-      if Rails.env.development?
-        min_conflict_interval = 1
-      end
-
-      saved_ids = saved_items.map{|x| x.uuid }
-      retrieved_ids = retrieved_items.map{|x| x.uuid }
-      conflicts = saved_ids & retrieved_ids # & is the intersection
-      # saved items take precedence, retrieved items are duplicated with a new uuid
-      conflicts.each do |conflicted_uuid|
-        # if changes are greater than min_conflict_interval seconds apart,
-        # push the retrieved item in the unsaved array so that the client can duplicate it
-        saved = saved_items.find{|i| i.uuid == conflicted_uuid}
-        conflicted = retrieved_items.find{|i| i.uuid == conflicted_uuid}
-        if (saved.updated_at - conflicted.updated_at).abs > min_conflict_interval
-          # puts "\n\n\n Creating conflicted copy of #{saved.uuid}\n\n\n"
-
-          unsaved_items.push({
-            :item => conflicted,
-            :error => {:tag => "sync_conflict"}
-          })
-
-        end
-
-        # We remove the item from retrieved items whether or not it satisfies the min_conflict_interval
-        # This is because the 'saved' value takes precedence, since that's the current value in the database.
-        # So by removing it from retrieved, we are forcing the client to ignore this change.
-        retrieved_items.delete(conflicted)
-      end
-    end
-
-    def destroy_items(uuids)
-      items = @user.items.where(uuid: uuids)
-      items.destroy_all
-    end
-
-
-    private
-
-    def sync_token_from_datetime(datetime)
-      version = 2
-      Base64.encode64("#{version}:" + "#{datetime.to_f}")
-    end
-
-    def datetime_from_sync_token(sync_token)
-      decoded = Base64.decode64(sync_token)
-      parts = decoded.rpartition(":")
-      timestamp_string = parts.last
-      version = parts.first
-      if version == "1"
-        date = DateTime.strptime(timestamp_string,'%s')
-      elsif version == "2"
-        date = Time.at(timestamp_string.to_f).to_datetime.utc
-      end
-
-      return date
-    end
-
-    def _sync_save(item_hashes, request)
-      if !item_hashes
-        return [], []
-      end
-      saved_items = []
-      unsaved = []
-
-      item_hashes.each do |item_hash|
-        begin
-          item = @user.items.find_or_create_by(:uuid => item_hash[:uuid])
-        rescue => error
-          unsaved.push({
-            :item => item_hash,
-            :error => {:message => error.message, :tag => "uuid_conflict"}
-            })
-          next
-        end
-
-        item.last_user_agent = request.user_agent
-        item.update(item_hash.permit(*permitted_params))
-        # we want to force update the updated_at field, even if no changes were made
-        # item.touch
-
-
-        if item.deleted == true
-          set_deleted(item)
-          item.save
-        end
-
-        saved_items.push(item)
-      end
-
-      return saved_items, unsaved
-    end
-
-    def _sync_get(sync_token, input_cursor_token, limit, content_type)
-      cursor_token = nil
-      if limit == nil
-        limit = 100000
-      end
-
-      # if both are present, cursor_token takes precendence as that would eventually return all results
-      # the distinction between getting results for a cursor and a sync token is that cursor results use a
-      # >= comparison, while a sync token uses a > comparison. The reason for this is that cursor tokens are
-      # typically used for initial syncs or imports, where a bunch of notes could have the exact same updated_at
-      # by using >=, we don't miss those results on a subsequent call with a cursor token
-      if input_cursor_token
-        date = datetime_from_sync_token(input_cursor_token)
-        items = @user.items.order(:updated_at).where("updated_at >= ?", date)
-      elsif sync_token
-        date = datetime_from_sync_token(sync_token)
-        items = @user.items.order(:updated_at).where("updated_at > ?", date)
-      else
-        # if no cursor token and no sync token, this is an initial sync. No need to return deleted items.
-        items = @user.items.order(:updated_at).where(:deleted => false)
-      end
-
-      if content_type
-        items = items.where(:content_type => content_type)
-      end
-
-      items = items.sort_by{|m| m.updated_at}
-
-      if items.count > limit
-        items = items.slice(0, limit)
-        date = items.last.updated_at
-        cursor_token = sync_token_from_datetime(date)
-      end
-
-      return items, cursor_token
-    end
-
-    def set_deleted(item)
-      item.deleted = true
-      item.content = nil if item.has_attribute?(:content)
-      item.enc_item_key = nil if item.has_attribute?(:enc_item_key)
-      item.auth_hash = nil if item.has_attribute?(:auth_hash)
-    end
-
-    def item_params
-      params.permit(*permitted_params)
-    end
-
-    def permitted_params
-      sync_fields
-    end
-
-  end
-end