nexo 0.1.4 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e9cc07511456093aa96d53df983acf4bfc770ba3de05df5bacfb683f80840ca
-  data.tar.gz: 5b1918e753e917ac5e0331aa5d49fe03a5d0fcb87d64fc36bab85af638b79246
+  metadata.gz: aca3e3e57c524de464408274ab6309fbea580931420c9da736556d7997ed6bd6
+  data.tar.gz: 770c4033d4dc280ad32471800e59a6d6d5d5af7a150897d52e5fa69de043b393
 SHA512:
-  metadata.gz: 973db6311a60eb0534b3ca072976190bb3b5be953bc3b8f6cbb6e39207ba127b54711dd5ec0adbcc3f2c5cc4c4272302ca558ad0572c687f6864402e7bec4bde
-  data.tar.gz: 79ba89ed8a6dbbd17bdf8c6c055df802fb9ec4cca686075d2f5e2653c776d1b38599371158bd58fb3779aef6dffd8718bd9ea8b5c42e2403a30d917a5aa72505
+  metadata.gz: 299fe496fcaa3041699cf37fa39e6d897005a4d7a5343c912694ff72cec18a91d19d317c91898c6cd1d55aeba606967298f1ce73ea40008cc14efa4acd62d807
+  data.tar.gz: 63057165522974406b55ec157a46e1c664d17b81e341e7dd04a551e93266e94bc38a0501297d3151f4a0636ad7d8c01e1cd2f739a7bc37cc340f37f53b435d8b

data/README.md

@@ -26,6 +26,17 @@ Configure good_job
     max_threads:
     queues:
 
+## Unhandled cases / TODO's
+
+- Restore locally discarded Events. It creates a new Google Calendar event?
+
+- Recurring google events
+
+- IMPORTANTE: loggear las exceptions de los jobs
+
+- Llevar mucho control de los jobs y las exceptions que puedan surgir como
+  ActiveRecord::PreparedStatementCacheExpired
+
 ## Contributing
 Contribution directions go here.
 

data/app/controllers/nexo/element_versions_controller.rb

@@ -0,0 +1,28 @@
+# :nocov: TODO
+module Nexo
+  class ElementVersionsController < NexoController
+    def show
+      @element_version = ElementVersion.find(params[:id])
+    end
+
+    def sync
+      @element_version = ElementVersion.find(params[:id])
+
+      case params[:operation]
+      when "import"
+        ImportRemoteElementVersion.new.perform(@element_version)
+        notice = "Imported"
+      when "update_remote"
+        UpdateRemoteResourceJob.perform_later(@element_version)
+        notice = "enqueued UpdateRemoteResourceJob"
+      else
+        raise "unkown action"
+      end
+
+      redirect_to @element_version, notice:
+    rescue StandardError => e
+      redirect_to @element_version, alert: e.message
+    end
+  end
+end
+# :nocov:

data/app/controllers/nexo/elements_controller.rb

@@ -0,0 +1,64 @@
+# :nocov: TODO
+module Nexo
+  class ElementsController < NexoController
+    before_action except: :index do
+      @element = Element.find(params[:id])
+    end
+
+    def index
+      @elements =
+        Element.includes(:synchronizable).order(id: :desc)
+               .page(params[:page]).per(params[:page_size] || 10)
+
+      if params[:not_synced]
+        @elements = @elements.where.not(ne_status: :synced)
+      end
+      if params[:without_synchronizable]
+        @elements = @elements.where(synchronizable_id: nil)
+      end
+      I18n.locale = :en
+    end
+
+    def show
+    end
+
+    def update_status
+      ElementService.new(element: @element).update_ne_status!
+
+      redirect_to @element, notice: "Updated status"
+    end
+
+    def modify_local
+      unless Rails.env.local?
+        redirect_to @element, alert: "Available only on local env"
+        return
+      end
+
+      event = @element.synchronizable
+      event.date_from = event.date_from + 1.day
+      event.date_to = event.date_to + 1.day
+      event.save!
+
+      EventReceiver.new.synchronizable_updated(event)
+
+      redirect_to @element, notice: "Modified"
+    rescue StandardError => e
+      redirect_to @element, alert: e.message
+    end
+
+    def fetch_remote
+      FetchRemoteResourceJob.perform_later(@element)
+
+      redirect_to @element, notice: "Enqueued FetchRemoteResourceJob"
+    end
+
+    def resolve_conflict
+      ElementService.new(element: @element).resolve_conflict!
+
+      redirect_to @element, notice: "Conflict solved"
+    rescue StandardError => e
+      redirect_to @element, alert: e.message
+    end
+  end
+end
+# :nocov:

data/app/controllers/nexo/folders_controller.rb

@@ -0,0 +1,41 @@
+# :nocov: TODO
+module Nexo
+  class FoldersController < NexoController
+    before_action except: :index do
+      @folder = Folder.find(params[:id])
+    end
+
+    def index
+      page = params[:page].to_i || 0
+      page_size = 100
+      @folders = Folder.offset(page * page_size).limit(page_size).order(id: :desc)
+    end
+
+    def show
+    end
+
+    def check_status
+      FolderCheckStatusJob.perform_later(@folder)
+
+      redirect_to @folder, notice: "Checking status"
+    end
+
+    def sync
+      FolderDownloadJob.perform_later(@folder, "full_or_incremental_sync")
+
+      redirect_to @folder, notice: "Enqueued sync"
+    end
+
+    def full_sync
+      FolderDownloadJob.perform_later(@folder, "full_sync")
+
+      redirect_to @folder, notice: "Enqueued full sync"
+    end
+
+    def incremental_sync
+      FolderDownloadJob.perform_later(@folder, "incremental_sync")
+
+      redirect_to @folder, notice: "Enqueued incremental sync"
+    end
+  end
+end

data/app/controllers/nexo/nexo_controller.rb

@@ -0,0 +1,5 @@
+module Nexo
+  class NexoController < ApplicationController
+    layout "nexo"
+  end
+end

data/app/jobs/nexo/api_clients.rb

@@ -3,29 +3,31 @@ module Nexo
     extend ActiveSupport::Concern
 
     included do
-      include GoodJob::ActiveJobExtensions::Concurrency
+      queue_as :nexo_api_clients
 
-      queue_as :api_clients
+      # :nocov: tricky
+      if defined? GoodJob
+        include GoodJob::ActiveJobExtensions::Concurrency
 
-      # TODO!: configure good job queues and thread pools
-      # https://github.com/bensheldon/good_job?tab=readme-ov-file#optimize-queues-threads-and-processes
-      # TODO: make this configurable, so other job backends are allowed
-      good_job_control_concurrency_with(
-        perform_limit: 1,
+        # https://github.com/bensheldon/good_job?tab=readme-ov-file#optimize-queues-threads-and-processes
+        good_job_control_concurrency_with(
+          perform_limit: 1,
 
-        perform_throttle: (Nexo.api_jobs_throttle || [ 100, 5.minute ]),
+          perform_throttle: (Nexo.api_jobs_throttle || [ 100, 5.minute ]),
 
-        key: -> { "#{queue_name}" }
-      )
+          key: -> { "#{queue_name}" }
+        )
 
-      retry_on(
-        GoodJob::ActiveJobExtensions::Concurrency::ConcurrencyExceededError,
-        attempts: Float::INFINITY,
-        wait: :polynomially_longer,
-        jitter: 0.99,
-        # wait: ->(executions) { ((executions**4) + (Kernel.rand * (executions**4) * jitter)) + 2 }
-        # wait: ->(executions) { ((executions**3) + (Kernel.rand * (executions**3) * 0.5)) + 2 }
-      )
+        retry_on(
+          GoodJob::ActiveJobExtensions::Concurrency::ConcurrencyExceededError,
+          attempts: Float::INFINITY,
+          wait: :polynomially_longer,
+          jitter: 0.99,
+          # wait: ->(executions) { ((executions**4) + (Kernel.rand * (executions**4) * jitter)) + 2 }
+          # wait: ->(executions) { ((executions**3) + (Kernel.rand * (executions**3) * 0.5)) + 2 }
+        )
+      end
+      # :nocov:
     end
   end
 end

data/app/jobs/nexo/base_job.rb

@@ -1,8 +1,10 @@
 module Nexo
   class BaseJob < ActiveJob::Base
-    def self.limits_concurrency(*)
-      # TODO: implementar
-      Rails.logger.error "ERROR: Implementar limits_concurrency"
+    # Like ActiveJob does with Rails.logger, we do with Nexo.logger
+    # push job's name and id tags
+    around_perform do |job, block|
+      tags = job.class.name, job.job_id
+      Nexo.plain_logger.tagged(*tags, &block)
     end
   end
 end

data/app/jobs/nexo/delete_remote_resource_job.rb

@@ -1,10 +1,20 @@
 module Nexo
+  # - Removing/discarding Element's
   class DeleteRemoteResourceJob < BaseJob
     include ApiClients
 
     def perform(element)
+      if element.discarded?
+        raise "element already discarded"
+      end
+
+      unless element.flagged_for_removal?
+        raise "element not flagged for removal"
+      end
+
       ServiceBuilder.instance.build_protocol_service(element.folder).remove(element)
-      # TODO!: mark as deleted, or something
+
+      ElementService.new(element:).discard!
     end
   end
 end

data/app/jobs/nexo/fetch_remote_resource_job.rb

@@ -0,0 +1,61 @@
+module Nexo
+  # Always must be executed asynchronously to ensure the concurrency limit applies
+  #
+  # Responsabilities:
+  #   - Creating a ElementVersion on external incoming change
+  #   - Updating a Synchronizable for that external change
+  #
+  # @raise ActiveRecord::RecordNotUnique
+  class FetchRemoteResourceJob < BaseJob
+    include ApiClients
+
+    attr_reader :element
+
+    def perform(element)
+      @element = element
+
+      remote_service = ServiceBuilder.instance.build_protocol_service(element.folder)
+
+      response = remote_service.get_event(element)
+      # TODO!: handle calendar change
+
+      handle_response(element, response)
+    end
+
+    def handle_response(element, response)
+      # TODO!: refactor y que no tenga que repetir esta asignación
+      #        extraer a algún Service?
+      @element = element
+
+      if response.present? && element.element_versions.where(etag: response.etag).empty?
+        Nexo.logger.debug { "Fetched new element version from remote server" }
+        Nexo.logger.debug { response.payload }
+
+        element_version = save_element_version(response)
+
+        if element.folder.sync_external_changes?
+          ImportRemoteElementVersion.new.perform(element_version)
+        else
+          Nexo.logger.info("Element version ignored_by_sync_direction")
+        end
+      else
+        Nexo.logger.debug { "No new version fetched from remote server" }
+      end
+    end
+
+    private
+
+    def save_element_version(service_response)
+      nev_status =
+        element.folder.sync_external_changes? ? :pending_sync : :ignored_by_sync_direction
+
+      ElementService.new(element:).create_element_version!(
+        origin: :external,
+        etag: service_response.etag,
+        payload: service_response.payload,
+        sequence: nil,
+        nev_status:
+      )
+    end
+  end
+end

data/app/jobs/nexo/folder_check_status_job.rb

@@ -0,0 +1,10 @@
+module Nexo
+  class FolderCheckStatusJob < BaseJob
+    def perform(folder)
+      calendar = GoogleCalendarService.new(folder.integration).get_calendar(folder)
+      folder.update(nf_status: :ok)
+    rescue StandardError => e
+      folder.update(nf_status: :not_found)
+    end
+  end
+end

data/app/jobs/nexo/folder_destroy_job.rb

@@ -1,11 +1,12 @@
 module Nexo
   class FolderDestroyJob < BaseJob
+    # TODO!: limit concurrency
     def perform(folder)
       if folder.external_identifier.present?
         protocol_service = ServiceBuilder.instance.build_protocol_service(folder)
         response = protocol_service.remove_calendar(folder)
       else
-        Rails.logger.info("Folder doesn't have external_identifier: #{folder.to_gid}")
+        Nexo.logger.info("Folder doesn't have external_identifier: #{folder.to_gid}")
       end
     end
   end

data/app/jobs/nexo/folder_download_job.rb

@@ -0,0 +1,15 @@
+module Nexo
+  class FolderDownloadJob < BaseJob
+    def perform(folder, type)
+      if type == "full_sync"
+        GoogleCalendarSyncService.new(folder.integration).full_sync!(folder)
+      elsif type == "incremental_sync"
+        GoogleCalendarSyncService.new(folder.integration).incremental_sync!(folder)
+      elsif type == "full_or_incremental_sync"
+        GoogleCalendarSyncService.new(folder.integration).full_or_incremental_sync!(folder)
+      else
+        raise "unknown sync type"
+      end
+    end
+  end
+end

data/app/jobs/nexo/folder_sync_job.rb

@@ -1,5 +1,6 @@
 module Nexo
   class FolderSyncJob < BaseJob
+    # TODO!: limit concurrency
     def perform(folder)
       protocol_service = ServiceBuilder.instance.build_protocol_service(folder)
       if folder.external_identifier.blank?
@@ -10,18 +11,23 @@ module Nexo
       end
 
       policies = PolicyService.instance.policies_for(folder)
+      Nexo.logger.debug { "Found #{policies.length} policies" }
       # flat_map should be equivalent to:
       #   policies.map(&:synchronizable_queries).flatten(1)
       queries = policies.flat_map(&:synchronizable_queries)
+      Nexo.logger.debug { "Found #{queries.length} queries" }
 
       GoodJob::Bulk.enqueue do
         queries.each do |query|
           # TODO: avoid calling more than once per synchronizable
           query.find_each do |synchronizable|
+            Nexo.logger.debug { "Processing synchronizable: #{synchronizable}" }
+
             folder_service.find_element_and_sync(folder, synchronizable)
           end
         end
       end
+      Nexo.logger.debug { "Finished processing queries" }
     end
 
     private

data/app/jobs/nexo/synchronizable_changed_job.rb

@@ -1,15 +1,30 @@
 module Nexo
   class SynchronizableChangedJob < BaseJob
-    limits_concurrency key: ->(synchronizable) { synchronizable.to_gid }
+    # :nocov: tricky
+    if defined? GoodJob
+      include GoodJob::ActiveJobExtensions::Concurrency
+
+      good_job_control_concurrency_with(
+        perform_limit: 1,
+        key: -> { arguments.first.to_gid.to_s }
+      )
+    end
+    # :nocov:
 
     # TODO: check
     # https://github.com/rails/solid_queue?tab=readme-ov-file#jobs-and-transactional-integrity
     #
     # TODO: handle exceptions
 
-    def perform(synchronizable)
+    # @raise [ActiveRecord::PreparedStatementCacheExpired]
+    def perform(synchronizable, excluded_folders: [])
       # Maybe restrict this query to a more specific scope
       scope = Folder.kept
+      if excluded_folders.any?
+        scope = scope.where.not(id: excluded_folders)
+      end
+      Nexo.logger.debug("Processing #{scope.count} folders")
+
       # TODO: test
       GoodJob::Bulk.enqueue do
         scope.each do |folder|

data/app/jobs/nexo/update_remote_resource_job.rb

@@ -2,34 +2,74 @@ module Nexo
   class UpdateRemoteResourceJob < BaseJob
     include ApiClients
 
-    attr_reader :element
+    # https://github.com/rails/solid_queue?tab=readme-ov-file#jobs-and-transactional-integrity
+    self.enqueue_after_transaction_commit = true
 
-    def perform(element)
-      @element = element
+    attr_reader :element, :element_version
+
+    # @raise Google::Apis::ClientError
+    # @raise [ActiveRecord::RecordNotUnique] on ElementVersion update
+    # @raise [Errno::ENETUNREACH]
+    #   - (Network is unreachable - Network is unreachable - connect(2) for "www.googleapis.com"
+    # @raise [Signet::AuthorizationError]
+    #   - Unexpected error: #<Faraday::ConnectionFailed
+    #       wrapped=#<Socket::ResolutionError: Failed to open TCP connection to
+    #       oauth2.googleapis.com:443 (getaddrinfo: Name or service not known)>>"
+    def perform(element_version)
+      @element_version = element_version
+      @element = element_version.element
 
       validate_element_state!
 
       remote_service = ServiceBuilder.instance.build_protocol_service(element.folder)
 
       response =
-        if element.element_versions.any?
+        if element.uuid.present?
           remote_service.update(element)
         else
-          remote_service.insert(element.folder, element.synchronizable).tap do |response|
-            element.update(uuid: response.id)
+          remote_service.insert(element).tap do |response|
+            ElementService.new(element:).update_element!(uuid: response.id)
           end
         end
 
-      save_element_version(response)
+      ElementService.new(element_version:).update_element_version!(
+        nev_status: :synced,
+        etag: response.etag,
+        payload: response.payload
+      )
+    rescue Errors::ConflictingRemoteElementChange => e
+      Nexo.logger.warn <<~STR
+        ConflictingRemoteElementChange for #{element.to_gid}. \
+        Enqueuing FetchRemoteResourceJob, which should lead to conflicted element
+      STR
+      FetchRemoteResourceJob.perform_later(element)
     end
 
     private
 
     def validate_element_state!
+      if element.element_versions.where(nev_status: :synced)
+                .where("sequence > ?", element_version.sequence).any?
+
+        ElementService.new(element_version:).update_element_version!(nev_status: :superseded)
+
+        raise Errors::Error, "version superseded"
+      end
+
+      unless element.folder.sync_internal_changes?
+        raise Errors::Error, "sync direction excludes internal changes"
+      end
+
       if element.synchronizable.blank?
         raise Errors::SynchronizableNotFound
       end
 
+      element.synchronizable.validate_synchronizable!
+
+      if element.discarded?
+        raise Errors::ElementDiscarded
+      end
+
       if element.synchronizable.respond_to?(:discarded?) && element.synchronizable.discarded?
         raise Errors::SynchronizableDiscarded
       end
@@ -39,30 +79,24 @@ module Nexo
       end
 
       if element.synchronizable.conflicted?
-        raise Errors::ElementConflicted
+        raise Errors::UpdateRemoteVersionFailed, "synchronizable conflicted"
       end
 
-      if element.external_unsynced_change?
-        raise Errors::ExternalUnsyncedChange
+      unless element.pending_local_sync?
+        raise Errors::Error, "invalid ne_status: #{element.ne_status}"
       end
 
-      current_sequence = element.synchronizable.sequence
-      last_synced_sequence = element.last_synced_sequence
+      if !element_version.internal?
+        raise Errors::Error, "invalid ElementVersion: must be internal"
+      end
 
-      unless current_sequence > last_synced_sequence
-        raise Errors::ElementAlreadySynced
+      if element_version.etag.present?
+        raise Errors::Error, "invalid ElementVersion: etag must be blank"
       end
-    end
 
-    # @todo sequence should be fetched before to avoid being outdated
-    def save_element_version(service_response)
-      ElementVersion.create!(
-        element:,
-        origin: :internal,
-        etag: service_response.etag,
-        payload: service_response.payload,
-        sequence: element.synchronizable.sequence,
-      )
+      unless element_version.pending_sync?
+        raise Errors::Error, "invalid ElementVersion: must be pending_sync"
+      end
     end
   end
 end

data/app/lib/nexo/active_record_google_token_store.rb

@@ -17,7 +17,7 @@ module Nexo
     def store(integration, token)
       ActiveRecord::Base.transaction do
         # Maybe these should be destroyed
-        integration.tokens.active.update_all(tpt_status: :expired)
+        integration.tokens.active.update_all(nt_status: :expired)
 
         Token.create!(integration:, secret: token)
       end
@@ -28,7 +28,7 @@ module Nexo
       token = find_by_id(id)
 
       if token.present?
-        token.update!(tpt_status: :revoked)
+        token.update!(nt_status: :revoked)
       else
         # TODO: pg_warn("Couldn't find token for revocation: #{id}")
       end
@@ -37,7 +37,7 @@ module Nexo
     private
 
     def find_by_id(id)
-      Token.where(environment: Rails.env, integration: id, tpt_status: :active).last
+      Token.where(environment: Rails.env, integration: id, nt_status: :active).last
     end
   end
 end

data/app/lib/nexo/api_client/google_auth_service.rb

@@ -53,21 +53,26 @@ module Nexo
     # Guarda el Token
     # Si el client tiene más permisos que los que el user solicitó
     def get_credentials(request = nil)
+      # :nocov: tricky
       if request.present? && request.session["code_verifier"].present?
-        # :nocov: tricky
         authorizer.code_verifier = request.session["code_verifier"]
-        # :nocov:
       end
-      authorizer.get_credentials @integration, request
+      # :nocov:
+
+      authorizer.get_credentials(@integration, request).tap do |credentials|
+        if credentials.nil? && request.present? && !request.session["code_verifier"].present?
+          Nexo.logger.warn("Request has no code_verifier")
+        end
+      end
     rescue Signet::AuthorizationError
       # TODO: log
     end
 
-    def get_authorization_url(request)
+    def get_authorization_url(request, login_hint: nil)
       request.session["code_verifier"] ||= Google::Auth::WebUserAuthorizer.generate_code_verifier
       authorizer.code_verifier = request.session["code_verifier"]
-      authorizer.get_authorization_url(request:)
-      # authorizer.get_authorization_url(request:, login_hint: "bla@gmail.com")
+      # authorizer.get_authorization_url(request:)
+      authorizer.get_authorization_url(request:, login_hint:)
     end
 
     private