tasker-engine 1.0.0

data/app/graphql/tasker/queries/tasks_by_annotation.rb

@@ -0,0 +1,31 @@
+# typed: false
+# frozen_string_literal: true
+
+module Tasker
+  module Queries
+    class TasksByAnnotation < BaseQuery
+      include Helpers
+      type [Tasker::GraphQLTypes::TaskType], null: true
+
+      description 'Find and sort tasks by status'
+      argument :annotation_type, String, required: true
+      argument :annotation_key, String, required: true
+      argument :annotation_value, String, required: true
+      argument :limit, Integer, default_value: 20, prepare: ->(limit, _ctx) { [limit, 100].min }, required: false
+      argument :offset, Integer, default_value: 0, required: false
+      argument :sort_by, String, default_value: :requested_at, required: false
+      argument :sort_order, String, default_value: :desc, required: false
+
+      def resolve(limit:, offset:, sort_by:, sort_order:, annotation_type:, annotation_key:, annotation_value:)
+        sorts = page_sort_params(model: Tasker::Task, limit: limit, offset: offset, sort_by: sort_by,
+                                 sort_order: sort_order)
+        Tasker::Task
+          .with_all_associated
+          .by_annotation(annotation_type, annotation_key, annotation_value)
+          .limit(sorts[:limit])
+          .offset(sorts[:offset])
+          .order(sorts[:order])
+      end
+    end
+  end
+end

data/app/graphql/tasker/queries/tasks_by_status.rb

@@ -0,0 +1,30 @@
+# typed: false
+# frozen_string_literal: true
+
+module Tasker
+  module Queries
+    class TasksByStatus < BaseQuery
+      include Helpers
+      type [Tasker::GraphQLTypes::TaskType], null: true
+
+      description 'Find and sort tasks by status'
+      argument :status, String, required: true
+      argument :limit, Integer, default_value: 20, prepare: ->(limit, _ctx) { [limit, 100].min }, required: false
+      argument :offset, Integer, default_value: 0, required: false
+      argument :sort_by, String, default_value: :requested_at, required: false
+      argument :sort_order, String, default_value: :desc, required: false
+
+      def resolve(limit:, offset:, sort_by:, sort_order:, status:)
+        sorts = page_sort_params(model: Tasker::Task, limit: limit, offset: offset, sort_by: sort_by,
+                                 sort_order: sort_order)
+
+        # Use the ActiveRecord scope to query tasks by current state
+        Tasker::Task.with_all_associated
+                    .by_current_state(status)
+                    .limit(sorts[:limit])
+                    .offset(sorts[:offset])
+                    .order(sorts[:order])
+      end
+    end
+  end
+end

data/app/graphql/tasker/tasker_rails_schema.rb

@@ -0,0 +1,52 @@
+# typed: false
+# frozen_string_literal: true
+
+require_relative 'graph_ql_types'
+
+module Tasker
+  class TaskerRailsSchema < GraphQL::Schema
+    mutation(Tasker::GraphQLTypes::MutationType)
+    query(Tasker::GraphQLTypes::QueryType)
+
+    # Custom exception for GraphQL type resolution errors
+    class TypeResolutionError < GraphQL::ExecutionError; end
+    class UnknownInterfaceError < TypeResolutionError; end
+    class InvalidObjectTypeError < TypeResolutionError; end
+
+    # Union and Interface Resolution
+    def self.resolve_type(abstract_type, obj, _ctx)
+      case abstract_type.graphql_name
+      when 'TaskInterface'
+        # TaskInterface is only implemented by TaskType
+        unless obj.is_a?(Tasker::Task) || (obj.is_a?(Hash) && obj.key?(:named_task_id))
+          raise InvalidObjectTypeError, "Unable to resolve TaskInterface for object: #{obj.class}"
+        end
+
+        GraphQLTypes::TaskType
+
+      else
+        raise UnknownInterfaceError, "Unknown abstract type: #{abstract_type.graphql_name}"
+      end
+    end
+
+    # Relay-style Object Identification:
+
+    # Return a string UUID for `object`
+    def self.id_from_object(object, type_definition, query_ctx)
+      # Here's a simple implementation which:
+      # - joins the type name & object.id
+      # - encodes it with base64:
+      # GraphQL::Schema::UniqueWithinType.encode(type_definition.name, object.id)
+    end
+
+    # Given a string UUID, find the object
+    def self.object_from_id(id, query_ctx)
+      # For example, to decode the UUIDs generated above:
+      # type_name, item_id = GraphQL::Schema::UniqueWithinType.decode(id)
+      #
+      # Then, based on `type_name` and `id`
+      # find an object in your application
+      # ...
+    end
+  end
+end

data/app/jobs/tasker/application_job.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# typed: ignore
+
+module Tasker
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/jobs/tasker/metrics_export_job.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+
+module Tasker
+  # ActiveJob for exporting metrics with distributed coordination
+  #
+  # **Phase 4.2.2.3.3**: Thin wrapper around MetricsExportService that handles
+  # ActiveJob concerns (queueing, retries, timeouts) while delegating the actual
+  # export business logic to the service.
+  #
+  # Key Features:
+  # - Rails ActiveJob integration for any backend (Sidekiq, SQS, etc.)
+  # - Distributed coordination through ExportCoordinator
+  # - Retry handling with exponential backoff
+  # - Timeout protection
+  # - Comprehensive logging
+  #
+  # @example Scheduled by ExportCoordinator
+  #   MetricsExportJob.set(wait_until: 5.minutes.from_now).perform_later(
+  #     format: :prometheus,
+  #     coordinator_instance: "web-1-12345",
+  #     timing: { export_time: Time.current + 5.minutes }
+  #   )
+  #
+  class MetricsExportJob < ApplicationJob
+    include Tasker::Concerns::StructuredLogging
+
+    # Configure job retry behavior with exponential backoff
+    # This replaces the sleep-based retry in ExportCoordinator
+    retry_on StandardError,
+             wait: :exponentially_longer,
+             attempts: 3,
+             queue: :metrics_export_retry
+
+    # Use separate queue for metrics export to avoid blocking other jobs
+    queue_as :metrics_export
+
+    # Job timeout to prevent hanging exports
+    around_perform :with_timeout
+
+    # @param format [Symbol] Export format (:prometheus, :json, :csv)
+    # @param coordinator_instance [String] ExportCoordinator instance identifier
+    # @param scheduled_by [String] Who scheduled this job
+    # @param timing [Hash] Export timing information
+    # @param include_instances [Boolean] Include per-instance metrics breakdown
+    def perform(format:, coordinator_instance:, scheduled_by: nil, timing: nil, include_instances: false)
+      @format = format.to_sym
+      @coordinator_instance = coordinator_instance
+      @scheduled_by = scheduled_by
+      @timing = timing
+      @include_instances = include_instances
+      @job_start_time = Time.current
+
+      log_job_started
+
+      # Get metrics data from coordinator
+      coordinator = Tasker::Telemetry::ExportCoordinator.instance
+      export_result = coordinator.execute_coordinated_export(
+        format: @format,
+        include_instances: @include_instances
+      )
+
+      if export_result[:success]
+        # Delegate actual export to service
+        service_result = export_with_service(export_result[:result])
+        log_job_completed(service_result)
+      else
+        # Export coordination failed
+        handle_failed_coordination(export_result)
+      end
+    rescue StandardError => e
+      log_job_error(e)
+
+      # Extend cache TTL before retry to prevent data loss
+      extend_cache_ttl_for_retry
+
+      raise # Re-raise to trigger ActiveJob retry logic
+    end
+
+    private
+
+    # Export metrics using the MetricsExportService
+    #
+    # @param metrics_data [Hash] Metrics data from coordinator
+    # @return [Hash] Export service result
+    def export_with_service(metrics_data)
+      service = Tasker::Telemetry::MetricsExportService.new
+
+      export_context = {
+        job_id: job_id,
+        coordinator_instance: @coordinator_instance,
+        scheduled_by: @scheduled_by,
+        timing: @timing
+      }
+
+      service.export_metrics(
+        format: @format,
+        metrics_data: metrics_data,
+        context: export_context
+      )
+    end
+
+    # Handle failed export coordination
+    #
+    # @param export_result [Hash] Export result with error information
+    def handle_failed_coordination(export_result)
+      log_coordination_failure(export_result)
+
+      # If this is a lock timeout, it's not necessarily an error
+      # Another container is handling the export
+      unless export_result[:error]&.include?('lock timeout')
+        raise StandardError, "Export coordination failed: #{export_result[:error]}"
+      end
+
+      log_concurrent_export_detected
+
+      # Actual coordination failure - raise error to trigger retry
+    end
+
+    # Timeout wrapper for job execution
+    #
+    # @param job [ActiveJob] The job being executed
+    # @yield Block to execute with timeout
+    def with_timeout(_job, &)
+      timeout_duration = job_timeout_duration
+
+      Timeout.timeout(timeout_duration, &)
+    rescue Timeout::Error
+      log_job_timeout(timeout_duration)
+      raise StandardError, "Export job timed out after #{timeout_duration} seconds"
+    end
+
+    # Get job timeout duration from configuration
+    #
+    # @return [Integer] Timeout duration in seconds
+    def job_timeout_duration
+      prometheus_config = begin
+        Tasker.configuration.telemetry.prometheus
+      rescue StandardError
+        {}
+      end
+      prometheus_config[:job_timeout] || 5.minutes
+    end
+
+    # Extend cache TTL to prevent data loss during job retry
+    #
+    # Uses the job's executions count to calculate appropriate extension
+    def extend_cache_ttl_for_retry
+      return unless defined?(executions) && executions > 1
+
+      # Calculate extension based on retry attempt and expected delay
+      retry_delay = calculate_job_retry_delay(executions)
+      safety_margin = 1.minute
+      extension_duration = retry_delay + safety_margin
+
+      coordinator = Tasker::Telemetry::ExportCoordinator.instance
+      result = coordinator.extend_cache_ttl(extension_duration)
+
+      log_ttl_extension_for_retry(extension_duration, result)
+    rescue StandardError => e
+      log_ttl_extension_error(e)
+      # Don't fail the job if TTL extension fails
+    end
+
+    # Calculate expected retry delay for current execution attempt
+    #
+    # @param execution_count [Integer] Current execution attempt (1-based)
+    # @return [Duration] Expected delay until next retry
+    def calculate_job_retry_delay(execution_count)
+      # ActiveJob exponentially_longer uses: attempt ** 4 + 2 seconds
+      # But we use a more conservative estimate based on our config
+      base_delay = 30.seconds
+      (base_delay * (2**(execution_count - 2))).clamp(30.seconds, 10.minutes)
+    end
+
+    # Logging methods for job execution events
+
+    def log_job_started
+      log_structured(:info, 'Metrics export job started',
+                     job_id: job_id,
+                     format: @format,
+                     coordinator_instance: @coordinator_instance,
+                     scheduled_by: @scheduled_by,
+                     timing: @timing,
+                     include_instances: @include_instances)
+    end
+
+    def log_job_completed(service_result)
+      duration = Time.current - @job_start_time
+
+      log_structured(:info, 'Metrics export job completed',
+                     job_id: job_id,
+                     format: @format,
+                     success: service_result[:success],
+                     duration: duration,
+                     service_duration: service_result[:duration],
+                     exported_at: service_result[:exported_at])
+    end
+
+    def log_job_error(error)
+      # Get first 5 backtrace lines and pad to exactly 5 elements with nil
+      backtrace_lines = error.backtrace&.first(5) || []
+      backtrace = backtrace_lines + Array.new(5 - backtrace_lines.size, nil)
+
+      log_structured(:error, 'Metrics export job error',
+                     job_id: job_id,
+                     format: @format,
+                     error: error.message,
+                     error_class: error.class.name,
+                     backtrace: backtrace)
+    end
+
+    def log_job_timeout(timeout_duration)
+      log_structured(:error, 'Metrics export job timeout',
+                     job_id: job_id,
+                     format: @format,
+                     timeout_duration: timeout_duration)
+    end
+
+    def log_coordination_failure(export_result)
+      log_structured(:warn, 'Export coordination failed',
+                     job_id: job_id,
+                     format: @format,
+                     error: export_result[:error],
+                     attempts: export_result[:attempts])
+    end
+
+    def log_concurrent_export_detected
+      log_structured(:info, 'Concurrent export detected - skipping',
+                     job_id: job_id,
+                     format: @format,
+                     reason: 'another_container_exporting')
+    end
+
+    def log_ttl_extension_for_retry(extension_duration, result)
+      log_structured(:info, 'Cache TTL extended for job retry',
+                     job_id: job_id,
+                     format: @format,
+                     execution_attempt: defined?(executions) ? executions : 1,
+                     extension_duration: extension_duration,
+                     ttl_extension_success: result[:success],
+                     metrics_extended: result[:metrics_extended])
+    end
+
+    def log_ttl_extension_error(error)
+      log_structured(:warn, 'Failed to extend cache TTL for retry',
+                     job_id: job_id,
+                     format: @format,
+                     error: error.message,
+                     note: 'Job will continue but metrics may expire during retry delay')
+    end
+  end
+end

data/app/jobs/tasker/task_runner_job.rb

@@ -0,0 +1,224 @@
+# typed: false
+# frozen_string_literal: true
+
+require 'tasker/logging/correlation_id_generator'
+require_relative '../../../lib/tasker/errors'
+
+module Tasker
+  class TaskRunnerJob < Tasker::ApplicationJob
+    include Tasker::Concerns::StructuredLogging
+    include Tasker::Concerns::EventPublisher
+
+    queue_as :default
+
+    # Retry retryable errors with exponential backoff
+    retry_on Tasker::RetryableError, wait: :exponentially_longer, attempts: 5
+
+    # Don't retry permanent errors
+    discard_on Tasker::PermanentError
+
+    # Retry other errors with shorter attempts
+    retry_on StandardError, wait: 3.seconds, attempts: 3
+
+    def handler_factory
+      @handler_factory ||= Tasker::HandlerFactory.instance
+    end
+
+    def perform(task_id, correlation_id: nil)
+      job_start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      task = nil
+
+      # Establish correlation ID for the entire job execution
+      job_correlation_id = correlation_id || generate_correlation_id
+
+      with_correlation_id(job_correlation_id) do
+        log_structured(:info, 'TaskRunnerJob execution started',
+                       task_id: task_id,
+                       job_id: job_id,
+                       correlation_id: job_correlation_id,
+                       queue_name: queue_name,
+                       attempt: executions + 1)
+
+        # Find and validate task
+        task = find_and_validate_task(task_id)
+        return unless task
+
+        # Publish task enqueue event for observability
+        publish_task_enqueue(task,
+                             job_id: job_id,
+                             correlation_id: job_correlation_id,
+                             queue_name: queue_name,
+                             attempt: executions + 1)
+
+        # Execute task with monitoring
+        execute_task_with_monitoring(task, job_correlation_id, job_start_time)
+
+        job_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - job_start_time
+
+        # Safely get final status
+        final_status = task&.reload&.status || 'unknown'
+
+        log_performance_event('task_runner_job_execution', job_duration,
+                              task_id: task_id,
+                              job_id: job_id,
+                              final_status: final_status,
+                              correlation_id: job_correlation_id)
+
+        log_structured(:info, 'TaskRunnerJob execution completed',
+                       task_id: task_id,
+                       job_id: job_id,
+                       final_task_status: task&.status,
+                       duration_ms: (job_duration * 1000).round(2),
+                       correlation_id: job_correlation_id)
+      end
+    rescue StandardError => e
+      job_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - job_start_time
+
+      log_exception(e, context: {
+                      task_id: task_id,
+                      job_id: job_id,
+                      correlation_id: job_correlation_id,
+                      operation: 'task_runner_job_execution',
+                      duration: job_duration,
+                      attempt: executions + 1
+                    })
+
+      Rails.logger.error("TaskRunnerJob: Error processing task #{task_id}: #{e.message}")
+      raise
+    end
+
+    private
+
+    # Find and validate the task for execution
+    #
+    # @param task_id [String] The task ID to find
+    # @return [Tasker::Task, nil] The task if found and valid, nil otherwise
+    def find_and_validate_task(task_id)
+      task = Tasker::Task.where(task_id: task_id).first
+
+      unless task
+        log_structured(:error, 'TaskRunnerJob: Task not found',
+                       task_id: task_id,
+                       job_id: job_id,
+                       error: 'Task not found in database')
+        return nil
+      end
+
+      log_structured(:debug, 'TaskRunnerJob: Task found and validated',
+                     task_id: task_id,
+                     task_name: task.name,
+                     task_status: task.status,
+                     job_id: job_id)
+
+      task
+    end
+
+    # Execute the task with comprehensive monitoring
+    #
+    # @param task [Tasker::Task] The task to execute
+    # @param job_correlation_id [String] The correlation ID for this job
+    # @param job_start_time [Float] The job start time for duration calculation
+    # @return [void]
+    def execute_task_with_monitoring(task, job_correlation_id, _job_start_time)
+      Rails.logger.info "TaskRunnerJob: Starting execution for task #{task.task_id}"
+
+      log_task_event(task, :job_execution_started,
+                     job_id: job_id,
+                     correlation_id: job_correlation_id,
+                     queue_name: queue_name,
+                     attempt: executions + 1)
+
+      # Get the appropriate task handler with timing
+      handler_lookup_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      handler = handler_factory.get(task.name, namespace_name: task.namespace_name, version: task.version)
+      handler_lookup_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - handler_lookup_start
+
+      log_performance_event('handler_factory_lookup', handler_lookup_duration,
+                            task_id: task.task_id,
+                            task_name: task.name,
+                            namespace_name: task.namespace_name,
+                            version: task.version,
+                            handler_class: handler.class.name)
+
+      log_structured(:debug, 'TaskRunnerJob: Handler retrieved',
+                     task_id: task.task_id,
+                     handler_class: handler.class.name,
+                     lookup_duration_ms: (handler_lookup_duration * 1000).round(2))
+
+      # Execute the task handler with timing
+      task_execution_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      handler.handle(task)
+      task_execution_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - task_execution_start
+
+      log_performance_event('task_handler_execution', task_execution_duration,
+                            task_id: task.task_id,
+                            task_name: task.name,
+                            handler_class: handler.class.name)
+
+      # Validate task completion
+      validate_task_completion(task, job_correlation_id, task_execution_duration)
+
+      Rails.logger.info "TaskRunnerJob: Completed execution for task #{task.task_id}"
+    end
+
+    # Validate that the task completed successfully
+    #
+    # @param task [Tasker::Task] The task to validate
+    # @param job_correlation_id [String] The correlation ID for this job
+    # @param task_execution_duration [Float] How long the task execution took
+    # @return [void]
+    # @raise [RuntimeError] If task did not complete successfully
+    def validate_task_completion(task, job_correlation_id, task_execution_duration)
+      task.reload
+
+      log_task_event(task, :job_execution_completed,
+                     job_id: job_id,
+                     final_status: task.status,
+                     correlation_id: job_correlation_id,
+                     execution_duration_ms: (task_execution_duration * 1000).round(2))
+
+      if task.status == Tasker::Constants::TaskStatuses::COMPLETE
+        log_structured(:info, 'TaskRunnerJob: Task completed successfully',
+                       task_id: task.task_id,
+                       final_status: task.status,
+                       execution_duration_ms: (task_execution_duration * 1000).round(2))
+      else
+        error_message = "Task processing failed for task #{task.task_id} - final status: #{task.status}"
+
+        log_task_event(task, :job_execution_failed,
+                       job_id: job_id,
+                       final_status: task.status,
+                       error: error_message,
+                       correlation_id: job_correlation_id)
+
+        Rails.logger.error("TaskRunnerJob: #{error_message}")
+
+        # Raise appropriate error type based on task status
+        case task.status
+        when Tasker::Constants::TaskStatuses::FAILED
+          raise Tasker::PermanentError.new(
+            error_message,
+            error_code: 'TASK_EXECUTION_FAILED',
+            context: { task_id: task.task_id, final_status: task.status }
+          )
+        else
+          raise Tasker::RetryableError.new(
+            error_message,
+            context: { task_id: task.task_id, final_status: task.status }
+          )
+        end
+      end
+    end
+
+    # Generate a correlation ID for job execution
+    #
+    # @return [String] A unique correlation ID
+    def generate_correlation_id
+      if defined?(Tasker::Logging::CorrelationIdGenerator)
+        Tasker::Logging::CorrelationIdGenerator.generate
+      else
+        "job_#{SecureRandom.hex(8)}"
+      end
+    end
+  end
+end

data/app/models/tasker/annotation_type.rb

@@ -0,0 +1,26 @@
+# typed: false
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: annotation_types
+#
+#  description        :string(255)
+#  name               :string(64)       not null
+#  created_at         :datetime         not null
+#  updated_at         :datetime         not null
+#  annotation_type_id :integer          not null, primary key
+#
+# Indexes
+#
+#  annotation_types_name_index   (name)
+#  annotation_types_name_unique  (name) UNIQUE
+#
+
+module Tasker
+  class AnnotationType < ApplicationRecord
+    self.primary_key = :annotation_type_id
+    has_many :task_annotations, dependent: :destroy
+    validates :name, presence: true, uniqueness: true
+  end
+end