nexo 0.1.5 → 0.1.7

data/app/lib/nexo/api_client/google_calendar_service.rb

@@ -1,106 +1,221 @@
 module Nexo
   # Wrapper around +Google::Apis::CalendarV3+
   #
-  # @raise [Google::Apis::ClientError] possible messages:
+  # TODO: handle ServerError to be retried
+  # @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried
+  # @raise [Google::Apis::AuthorizationError] Authorization is required
+  # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification
+  #   possible messages:
   #   - duplicate: The requested identifier already exists.
   #   - notFound: Not Found (calendar not exists or was deleted)
   #   - forbidden: Forbidden (event to update was deleted)
+  #   - conditionNotMet: Precondition Failed (etag / if-match header verification failed)
+  #   - invalid: Invalid sequence value. The specified sequence number is below
+  #              the current sequence number of the resource. Re-fetch the resource and
+  #              use its sequence number on the following request.
   #
   # TODO! when event to update was deleted, create a new one and warn
   class GoogleCalendarService < CalendarService
     # Create an event in a Google Calendar
     #
-    # @param [Folder] folder
+    # @param [Element] folder
     #
-    # @todo Debería recibir un {Element}?
-    def insert(folder, calendar_event)
-      validate_folder_state!(folder)
+    def insert(element)
+      validate_folder_state!(element.folder)
 
-      event = build_event(calendar_event)
-      response = client.insert_event(folder.external_identifier, event)
-      ApiResponse.new(payload: response.to_json, status: :ok, etag: response.etag, id: response.id)
+      event = build_event(element)
+      response = client.insert_event(element.folder.external_identifier, event)
+      ApiResponse.new(payload: response.to_h, status: :ok, etag: response.etag, id: response.id)
     end
 
     # Update an event in a Google Calendar
     def update(element)
       validate_folder_state!(element.folder)
+      # sidebranch
+      # TODO!: validate uuid presence
+
+      event = build_event(element)
+
+      response = client.update_event(element.folder.external_identifier, element.uuid, event, options: ifmatch_options(element))
 
-      event = build_event(element.synchronizable)
-      response = client.update_event(element.folder.external_identifier, element.uuid, event)
-      ApiResponse.new(payload: response.to_json, status: :ok, etag: response.etag)
+      ApiResponse.new(payload: response.to_h, status: :ok, etag: response.etag)
+    rescue Google::Apis::ClientError => e
+      if e.message.match? /conditionNotMet/
+        raise Errors::ConflictingRemoteElementChange, e
+      else
+        raise
+      end
     end
 
     # Delete an event in a Google Calendar
     def remove(element)
       validate_folder_state!(element.folder)
+      # sidebranch
+      # TODO!: validate uuid presence
 
-      # TODO: try with cancelled
-      client.delete_event(element.folder.external_identifier, element.uuid)
+      # TODO: try with cancelled / maybe its the same
+      client.delete_event(element.folder.external_identifier, element.uuid, options: ifmatch_options(element))
       ApiResponse.new(payload: nil, status: :ok, etag: nil)
+    rescue Google::Apis::ClientError => e
+      if e.message.match? /conditionNotMet/
+        raise Errors::ConflictingRemoteElementChange, e
+      else
+        raise
+      end
+    end
+
+    def get_event(element)
+      validate_folder_state!(element.folder)
+      # sidebranch
+      # TODO!: validate uuid presence
+
+      # would be nice to send If-None-Match header, but Google API doesn't seem
+      # to accept it
+      response = client.get_event(element.folder.external_identifier, element.uuid)
+
+      ApiResponse.new(payload: response.to_h, status: :ok, etag: response.etag, id: response.id)
+    rescue Google::Apis::ClientError => e
+      if e.message.match? "notFound"
+        Nexo.logger.warn("Event not found for #{element.to_gid}")
+        nil
+      else
+        raise
+      end
+    end
+
+    def get_calendar(folder)
+      validate_folder_state!(folder)
+
+      response = client.get_calendar(folder.external_identifier)
+      ApiResponse.new(payload: response.to_h, status: :ok, etag: response.etag, id: response.id)
     end
 
     # Create a Google calendar
     def insert_calendar(folder)
-      unless folder.integration.token?
-        raise Errors::Error, "folder has no token"
-      end
+      validate_folder_state!(folder, verify_external_identifier_presence: false)
 
       cal = build_calendar(folder)
       response = client.insert_calendar(cal)
-      ApiResponse.new(payload: response.to_json, status: :ok, etag: response.etag, id: response.id)
+      ApiResponse.new(payload: response.to_h, status: :ok, etag: response.etag, id: response.id)
     end
 
-    # Create a Google calendar
+    # Update a Google calendar
     def update_calendar(folder)
-      unless folder.integration.token?
-        raise Errors::Error, "folder has no token"
-      end
+      validate_folder_state!(folder)
 
       cal = build_calendar(folder)
       response = client.update_calendar(folder.external_identifier, cal)
-      ApiResponse.new(payload: response.to_json, status: :ok, etag: response.etag, id: response.id)
+      ApiResponse.new(payload: response.to_h, status: :ok, etag: response.etag, id: response.id)
     end
 
     def remove_calendar(folder)
+      validate_folder_state!(folder)
+
       client.delete_calendar(folder.external_identifier)
       ApiResponse.new(status: :ok)
     end
 
-    # @!visibility private
-    # :nocov: non-production
-    def clear_calendars
-      Folder.all.each do |folder|
-        cid = folder.external_identifier
-        events = client.list_events(cid).items
-        events.each do |event|
-          client.delete_event(cid, event.id)
-        end
-      end
+    def fields_from_payload(payload)
+      event = validate_version!(payload)
+
+      {
+        date_from: parse_date(event.start),
+        date_to: parse_date(event.end),
+        time_from: parse_time(event.start),
+        time_to: parse_time(event.end),
+        summary: event.summary,
+        description: event.description,
+
+        # Posible status values:
+        # - confirmed
+        # - tentative
+        # - cancelled (deleted)
+        status: event.status
+      }
     end
-    # :nocov:
 
     private
 
-    def validate_folder_state!(folder)
+    def validate_version!(payload)
+      event_data = ActiveSupport::HashWithIndifferentAccess.new(payload)
+      event = Google::Apis::CalendarV3::Event.new(**event_data)
+
+      validate_datetime!(event.start)
+      validate_datetime!(event.end)
+
+      event
+    end
+
+    def parse_date(datetime)
+      if datetime["date"].present?
+        Date.parse(datetime["date"])
+      else
+        Date.parse(datetime["date_time"])
+      end
+    end
+
+    def parse_time(datetime)
+      if datetime["date"].present?
+        nil
+      else
+        Time.parse(datetime["date_time"])
+      end
+    end
+
+    def validate_datetime!(datetime)
+      unless datetime.is_a?(Hash) && (datetime["date"].present? || datetime["date_time"].present?)
+        # :nocov: borderline
+        raise "invalid datetime"
+        # :nocov:
+      end
+    end
+
+    def validate_folder_state!(folder, verify_external_identifier_presence: true)
       unless folder.integration.token?
         raise Errors::Error, "folder has no token"
       end
 
-      if folder.external_identifier.blank?
+      if verify_external_identifier_presence && folder.external_identifier.blank?
         raise Errors::InvalidFolderState, folder
       end
     end
 
-    def build_event(calendar_event)
+    def build_event(element)
+      calendar_event = element.synchronizable
       estart = build_event_date_time(calendar_event.datetime_from)
       eend = build_event_date_time(calendar_event.datetime_to)
 
-      Google::Apis::CalendarV3::Event.new(
+      # TODO: sequence is managed by both google and nexo
+      # google accepts the sent sequence only if its valid (>= current_sequence)
+      # when we receive a remote change with updated sequence, we should
+      # update the secuence accordingly. and if we receive a remote change without a
+      # secuence increment, then we could:
+      #   1. not update the local secuence. but maybe thats not viable
+      #   2. increment the local secuence and update the remote
+      #   3. increment the local secuence and not update the remote
+      #   4. <current> ignore the sequence from google
+
+      from = element.last_remote_version
+
+      base_event =
+        if from.present?
+          event_data = ActiveSupport::HashWithIndifferentAccess.new(from.payload)
+          Google::Apis::CalendarV3::Event.new(**event_data)
+        else
+          Google::Apis::CalendarV3::Event.new
+        end
+
+      base_event.update!(
         start: estart,
         end: eend,
         summary: calendar_event.summary,
-        description: calendar_event.description
+        description: calendar_event.description,
+        transparency: calendar_event.transparency,
+        status: calendar_event.nce_status,
+        # sequence: calendar_event.sequence
       )
+
+      base_event
     end
 
     def build_calendar(folder)
@@ -128,5 +243,14 @@ module Nexo
         end
       # :nocov:
     end
+
+    def ifmatch_options(element)
+      ifmatch = element.etag
+      Nexo.logger.debug { "ifmatch: #{ifmatch}" }
+
+      raise Errors::Error, "an etag is required to perform the request" if ifmatch.blank?
+
+      Google::Apis::RequestOptions.new(header: { "If-Match" => ifmatch })
+    end
   end
 end

data/app/lib/nexo/api_client/google_calendar_sync_service.rb

@@ -0,0 +1,88 @@
+module Nexo
+  class GoogleCalendarSyncService < GoogleCalendarService
+    def full_sync!(folder)
+      Nexo.logger.info("performing full sync")
+      sync!(folder)
+    end
+
+    def incremental_sync!(folder)
+      raise Nexo::Errors::Error, "no sync token" if folder.google_next_sync_token.blank?
+
+      Nexo.logger.info("performing incremental sync")
+
+      sync!(folder, sync_token: folder.google_next_sync_token)
+    rescue Google::Apis::ClientError => e
+      if e.message.match /fullSyncRequired/
+        Nexo.logger.warn "Full sync required: #{e}"
+        full_sync!(folder)
+      else
+        raise
+      end
+    end
+
+    def full_or_incremental_sync!(folder)
+      if folder.google_next_sync_token.present?
+        incremental_sync!(folder)
+      else
+        full_sync!(folder)
+      end
+    end
+
+    private
+
+    # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification
+    #   TODO: handle 410 (Gone) to discard the sync_token and schedule a full_sync
+    #    (fullSyncRequired: Sync token is no longer valid, a full sync is required.
+    def sync!(folder, page_token: nil, sync_token: nil)
+      page_token = nil
+      loop do
+        if page_token.present?
+          Nexo.logger.debug("Calling list_events with page_token")
+        elsif sync_token.present?
+          Nexo.logger.debug("Calling list_events with sync_token")
+          # raise Google::Apis::ClientError, "fullSyncRequired: Sync token is no longer valid..."
+        else
+          Nexo.logger.debug("Calling list_events without sync_token")
+        end
+
+        events = client.list_events(folder.external_identifier, page_token:, sync_token:)
+        Nexo.logger.debug("retrieved a page with #{events.items.length} items")
+        sync_token = nil
+
+        events.items.each do |event|
+          response = ApiResponse.new(payload: event.to_h, etag: event.etag, id: event.id)
+
+          element = Element.kept.where(uuid: response.id).first
+
+          if element.present?
+            Nexo.logger.debug("Element found for event")
+
+            FetchRemoteResourceJob.new.handle_response(element, response)
+          elsif event.status == "cancelled"
+            Nexo.logger.debug("Skipping cancelled event")
+          else
+            element = ElementService.new.create_element_for_remote_resource!(folder, response)
+
+            # TODO!: rename handle_response
+            FetchRemoteResourceJob.new.handle_response(element, response)
+          end
+        end
+
+        page_token = events.next_page_token
+        if page_token
+          Nexo.logger.debug("Page token present. Fetching next page.")
+        else
+          sync_token = events.next_sync_token
+          if sync_token.present?
+            Nexo.logger.debug("Sync token present. Saving it to Folder")
+            folder.update!(google_next_sync_token: sync_token)
+          else
+            Nexo.logger.error("Sync token should have been present!")
+          end
+
+          break
+        end
+      end
+    end
+  end
+end

data/app/lib/nexo/element_service.rb

@@ -0,0 +1,249 @@
+module Nexo
+  class ElementService
+    attr_accessor :element, :element_version
+
+    def initialize(element_version: nil, element: nil)
+      @element_version = element_version
+      @element = element || element_version&.element
+    end
+
+    def create_element_version!(attributes)
+      element.with_lock do
+        _create_element_version!(attributes)
+      end
+    end
+
+    def update_element_version!(attributes)
+      element.with_lock do
+        element_version.update!(attributes)
+        _update_ne_status!
+      end
+    end
+
+    def update_element!(attributes)
+      element.update!(attributes)
+    end
+
+    def discard!
+      element.update!(discarded_at: Time.current)
+      _update_ne_status!
+    end
+
+    # The reason of `flagged_for_removal` is that there is a time gap between
+    # the the user action of removing the element and the actual API call that
+    # deletes the remote element. At some point is necesary to know if the
+    # element is being deleted, like when fetching a remote version previous to
+    # the actual delete API call.
+    def flag_for_removal!(removal_reason)
+      Nexo.logger.debug("Flagging an element for removal")
+
+      element.update!(flagged_for_removal: true, removal_reason:)
+      _update_ne_status!
+    end
+
+    # @raise ActiveRecord::RecordNotUnique
+    def update_synchronizable!
+      Nexo.logger.debug("update_synchronizable!")
+
+      element.with_lock do
+        service = ServiceBuilder.instance.build_protocol_service(element_version.element.folder)
+        fields = service.fields_from_payload(element_version.payload)
+
+        # and set the Synchronizable fields according to the Folder#nexo_protocol
+        synchronizable = element_version.element.synchronizable
+        if element.flagged_for_removal?
+          Nexo.logger.info("Element flagged for removal")
+          ElementService.new(element_version:).update_element_version!(
+            nev_status: :ignored_by_deletion
+          )
+        elsif synchronizable.present?
+          synchronizable.update_from_fields!(fields)
+
+          # synchronizable could have been destroyed
+          if synchronizable.persisted?
+            # si esto se ejecuta en paralelo con SynchronizableChangedJob? (para otro
+            # element del mismo synchronizable) puede haber race conditions
+            synchronizable.increment_sequence!
+            synchronizable.reload
+
+            ElementService.new(element_version:).update_element_version!(
+              sequence: synchronizable.sequence,
+              nev_status: :synced
+            )
+
+            SynchronizableChangedJob.perform_later(synchronizable, excluded_folders: [ element.folder.id ])
+          else
+            Nexo.logger.debug("Synchronizable destroyed. Removing other elements")
+            ElementService.new(element_version:).update_element_version!(
+              sequence: nil,
+              nev_status: :synced
+            )
+
+            FolderService.new.destroy_elements(
+              synchronizable, :synchronizable_destroyed, exclude_elements: [ element.id ])
+          end
+        else
+          Nexo.logger.info("Synchronizable not found")
+          policies = PolicyService.instance.policies_for(element.folder)
+          importer_rule = policies.select { |p| p.import_payload?(element_version.payload) }.first
+          if importer_rule.present?
+            Nexo.logger.debug("Found an importer rule")
+            synchronizable = importer_rule.create_synchronizable_from_payload!(element_version.payload)
+            ElementService.new(element:).update_element!(synchronizable:)
+            ElementService.new(element_version:).update_element_version!(
+              nev_status: :synced,
+              sequence: synchronizable.sequence
+            )
+          else
+            Nexo.logger.info("No importer rule found for event. Skipping")
+          end
+        end
+      end
+    end
+
+    def resolve_conflict!
+      unless element.conflicted?
+        raise "element not conflicted"
+      end
+
+      # both lock on Element and start a transaction
+      element.with_lock do
+        external_change = element.element_versions.where(origin: :external, nev_status: :pending_sync).order(:etag).last
+        local_change = element.element_versions.where(origin: :internal, nev_status: :pending_sync).order(:sequence).last
+        last_synced = element.element_versions.where(nev_status: :synced).order(:sequence).last
+
+        if local_change.sequence < last_synced.sequence
+          raise "there a newer synced sequence"
+        end
+
+        if external_change.etag < last_synced.etag
+          raise "there a newer synced etag"
+        end
+
+        Nexo.logger.debug { "resolving conflict" }
+        remote_update = Time.zone.parse(external_change.payload["updated"])
+        local_update = element.synchronizable.updated_at
+        Nexo.logger.debug { "Remote updated at: #{remote_update}. Local updated at #{local_update}" }
+        if remote_update > local_update
+          Nexo.logger.debug { "Remote wins, ignoring local change" }
+          _update_status_on_conflict_with_winner!(external_change)
+          ImportRemoteElementVersion.new.perform(external_change)
+        else
+          Nexo.logger.debug { "Local wins, discarding remote changes" }
+          _update_status_on_conflict_with_winner!(local_change)
+          UpdateRemoteResourceJob.perform_later(local_change)
+        end
+      end
+    end
+
+    def update_ne_status!
+      element.with_lock do
+        _update_ne_status!
+      end
+    end
+
+    def create_internal_version_if_none!
+      # NOTE: though synchronizable it's not locked and could change in between
+      # the block, this shouldn't run concurrently because its called from
+      # SynchronizableChangedJob that its limited to one perform at a time per
+      # synchronizable
+      element.with_lock do
+        if element.element_versions.where(sequence: element.synchronizable.sequence).any?
+          Nexo.logger.debug { "There is a version for current sequence, nothing to do" }
+          return
+        end
+
+        _create_internal_version!
+      end
+    end
+
+    def create_element_for_remote_resource!(folder, response)
+      Element.create!(
+        folder:,
+        uuid: response.id,
+        ne_status: :pending_external_sync
+      )
+    end
+
+    def create_element_for!(folder, synchronizable)
+      Element.transaction do
+        @element = Element.create!(
+          synchronizable:,
+          folder:,
+          ne_status: :pending_local_sync
+        )
+        Nexo.logger.debug { "Element created" }
+
+        _create_internal_version!
+      end
+    end
+
+    private
+
+    def _update_status_on_conflict_with_winner!(win_version)
+      element.element_versions.where(nev_status: :pending_sync)
+        .where.not(id: win_version.id)
+        .update_all(nev_status: :ignored_in_conflict)
+
+      _update_ne_status!
+    end
+
+    def _create_internal_version!
+      nev_status =
+        element.folder.sync_internal_changes? ? :pending_sync : :ignored_by_sync_direction
+
+      _create_element_version!(
+        origin: :internal,
+        sequence: element.synchronizable.sequence,
+        nev_status:
+      ).tap do |element_version|
+        Nexo.logger.debug("ElementVersion created")
+
+        if element.pending_local_sync?
+          Nexo.logger.debug("Enqueuing UpdateRemoteResourceJob")
+
+          UpdateRemoteResourceJob.perform_later(element_version)
+        elsif element.conflicted?
+          Nexo.logger.info("Element conflicted, so not enqueuing UpdateRemoteResourceJob")
+        elsif element.synced?
+          Nexo.logger.info("Element's ne_status is: #{element.ne_status}. No need to push any changes.")
+        else
+          # :nocov: borderline
+          Nexo.logger.info("Element's ne_status is: #{element.ne_status}. That's weird.")
+          # :nocov:
+        end
+      end
+    end
+
+    def _create_element_version!(attributes)
+      ElementVersion.create!(attributes.merge(element:)).tap do
+        _update_ne_status!
+      end
+    end
+
+    def _update_ne_status!
+      external_change =
+        element.folder.sync_external_changes? &&
+          element.element_versions.where(origin: :external, nev_status: :pending_sync).any?
+
+      local_change =
+        element.folder.sync_internal_changes? &&
+          element.element_versions.where(origin: :internal, nev_status: :pending_sync).any?
+
+      element.ne_status =
+        if external_change && local_change
+          :conflicted
+        elsif external_change
+          :pending_external_sync
+        elsif local_change
+          :pending_local_sync
+        elsif element.flagged_for_removal? && element.discarded_at.nil?
+          :pending_remote_delete
+        else
+          :synced
+        end
+
+      element.save!
+    end
+  end
+end

data/app/lib/nexo/errors.rb

@@ -2,13 +2,16 @@ module Nexo
   class Errors
     class Error < StandardError; end
 
-    class ElementConflicted < Error; end
-    class ExternalUnsyncedChange < Error; end
+    class SynchronizableInvalid < Error; end
+    class ConflictingRemoteElementChange < Error; end
+    class UpdateRemoteVersionFailed < Error; end
+
+    # From here on, classes are subject to review
+    # A lot of them are never rescued explicitly
     class ElementAlreadySynced < Error; end
     class MoreThanOneElementInFolderForSynchronizable < Error; end
     class InvalidFolderState < Error; end
     class FolderDiscarded < Error; end
-    class SynchronizableDiscarded < Error; end
 
     # on ControllerHelper
     class InvalidParamsError < Error; end
@@ -19,8 +22,6 @@ module Nexo
     # on EventReceiver
     class InvalidSynchronizableState < Error; end
 
-    # on SyncElementJob
-    class SyncElementJobError < Errors::Error; end
     class SynchronizableConflicted < Error; end
     class ElementDiscarded < Error; end
     class SynchronizableNotFound < Error; end

data/app/lib/nexo/event_receiver.rb

@@ -21,6 +21,8 @@ module Nexo
     end
 
     def folder_changed(folder)
+      return unless folder.sync_internal_changes?
+
       if folder.discarded?
         raise "folder discarded"
       end
@@ -28,6 +30,8 @@ module Nexo
     end
 
     def folder_discarded(folder)
+      return unless folder.sync_internal_changes?
+
       FolderDestroyJob.perform_later(folder)
     end