natswork-server 0.0.1

data/lib/natswork/protocol.rb

@@ -0,0 +1,454 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'time'
+
+module NatsWork
+  module Protocol
+    VERSION = '1.0.0'
+
+    # Message types
+    module MessageType
+      JOB_REQUEST = 'job_request'
+      JOB_RESPONSE = 'job_response'
+      JOB_ERROR = 'job_error'
+      WORKER_HEARTBEAT = 'worker_heartbeat'
+      WORKER_REGISTER = 'worker_register'
+      WORKER_UNREGISTER = 'worker_unregister'
+      QUEUE_STATUS = 'queue_status'
+      PROTOCOL_VERSION = 'protocol_version'
+    end
+
+    # Standard error codes
+    module ErrorCode
+      UNKNOWN = 'UNKNOWN'
+      JOB_NOT_FOUND = 'JOB_NOT_FOUND'
+      WORKER_NOT_FOUND = 'WORKER_NOT_FOUND'
+      QUEUE_NOT_FOUND = 'QUEUE_NOT_FOUND'
+      TIMEOUT = 'TIMEOUT'
+      INVALID_ARGUMENTS = 'INVALID_ARGUMENTS'
+      SERIALIZATION_ERROR = 'SERIALIZATION_ERROR'
+      EXECUTION_ERROR = 'EXECUTION_ERROR'
+      RETRY_LIMIT_EXCEEDED = 'RETRY_LIMIT_EXCEEDED'
+      UNSUPPORTED_VERSION = 'UNSUPPORTED_VERSION'
+    end
+
+    # Message schemas
+    class Schema
+      # Job request message schema
+      def self.job_request
+        {
+          type: MessageType::JOB_REQUEST,
+          version: VERSION,
+          job_id: 'string (UUID)',
+          job_class: 'string',
+          queue: 'string',
+          arguments: 'array|hash',
+          metadata: {
+            created_at: 'ISO8601 timestamp',
+            enqueued_at: 'ISO8601 timestamp',
+            retry_count: 'integer',
+            max_retries: 'integer',
+            timeout: 'integer (seconds)',
+            language: 'string (optional)',
+            language_version: 'string (optional)',
+            worker_constraints: 'hash (optional)',
+            priority: 'integer (optional)',
+            idempotency_key: 'string (optional)',
+            correlation_id: 'string (optional)',
+            parent_job_id: 'string (optional)'
+          }
+        }
+      end
+
+      # Job response message schema
+      def self.job_response
+        {
+          type: MessageType::JOB_RESPONSE,
+          version: VERSION,
+          job_id: 'string (UUID)',
+          status: 'success|failure|partial',
+          result: 'any',
+          metadata: {
+            started_at: 'ISO8601 timestamp',
+            completed_at: 'ISO8601 timestamp',
+            duration_ms: 'float',
+            worker_id: 'string',
+            worker_language: 'string',
+            worker_version: 'string',
+            retry_count: 'integer'
+          }
+        }
+      end
+
+      # Job error message schema
+      def self.job_error
+        {
+          type: MessageType::JOB_ERROR,
+          version: VERSION,
+          job_id: 'string (UUID)',
+          error_code: 'string (ErrorCode)',
+          error_message: 'string',
+          error_class: 'string',
+          backtrace: 'array of strings (optional)',
+          metadata: {
+            occurred_at: 'ISO8601 timestamp',
+            worker_id: 'string',
+            worker_language: 'string',
+            retry_count: 'integer',
+            retryable: 'boolean',
+            retry_at: 'ISO8601 timestamp (optional)'
+          }
+        }
+      end
+
+      # Worker heartbeat schema
+      def self.worker_heartbeat
+        {
+          type: MessageType::WORKER_HEARTBEAT,
+          version: VERSION,
+          worker_id: 'string',
+          timestamp: 'ISO8601 timestamp',
+          status: 'running|paused|stopping',
+          stats: {
+            jobs_processed: 'integer',
+            jobs_failed: 'integer',
+            active_jobs: 'integer',
+            queues: 'array of strings',
+            concurrency: 'integer',
+            memory_usage: 'integer (bytes)',
+            cpu_usage: 'float (percentage)',
+            uptime_seconds: 'integer'
+          },
+          capabilities: {
+            language: 'string',
+            language_version: 'string',
+            protocol_version: 'string',
+            supported_job_types: 'array (optional)',
+            max_job_size: 'integer (bytes)',
+            features: 'array of strings'
+          }
+        }
+      end
+    end
+
+    # Message builder
+    class MessageBuilder
+      def self.build_job_request(job_class, arguments, options = {})
+        {
+          type: MessageType::JOB_REQUEST,
+          version: VERSION,
+          job_id: options[:job_id] || generate_uuid,
+          job_class: job_class,
+          queue: options[:queue] || 'default',
+          arguments: arguments,
+          metadata: {
+            created_at: Time.now.utc.iso8601,
+            enqueued_at: options[:enqueued_at] || Time.now.utc.iso8601,
+            retry_count: options[:retry_count] || 0,
+            max_retries: options[:max_retries] || 3,
+            timeout: options[:timeout] || 30,
+            language: options[:language] || 'ruby',
+            language_version: options[:language_version] || RUBY_VERSION,
+            worker_constraints: options[:worker_constraints],
+            priority: options[:priority] || 0,
+            idempotency_key: options[:idempotency_key],
+            correlation_id: options[:correlation_id],
+            parent_job_id: options[:parent_job_id]
+          }.compact
+        }
+      end
+
+      def self.build_job_response(job_id, result, options = {})
+        {
+          type: MessageType::JOB_RESPONSE,
+          version: VERSION,
+          job_id: job_id,
+          status: options[:status] || 'success',
+          result: result,
+          metadata: {
+            started_at: options[:started_at],
+            completed_at: Time.now.utc.iso8601,
+            duration_ms: options[:duration_ms],
+            worker_id: options[:worker_id],
+            worker_language: 'ruby',
+            worker_version: VERSION,
+            retry_count: options[:retry_count] || 0
+          }.compact
+        }
+      end
+
+      def self.build_job_error(job_id, error, options = {})
+        {
+          type: MessageType::JOB_ERROR,
+          version: VERSION,
+          job_id: job_id,
+          error_code: options[:error_code] || ErrorCode::EXECUTION_ERROR,
+          error_message: error.message,
+          error_class: error.class.name,
+          backtrace: error.backtrace&.first(20),
+          metadata: {
+            occurred_at: Time.now.utc.iso8601,
+            worker_id: options[:worker_id],
+            worker_language: 'ruby',
+            retry_count: options[:retry_count] || 0,
+            retryable: options[:retryable] != false,
+            retry_at: options[:retry_at]
+          }.compact
+        }
+      end
+
+      def self.build_worker_heartbeat(worker_id, stats, capabilities)
+        {
+          type: MessageType::WORKER_HEARTBEAT,
+          version: VERSION,
+          worker_id: worker_id,
+          timestamp: Time.now.utc.iso8601,
+          status: stats[:status] || 'running',
+          stats: {
+            jobs_processed: stats[:jobs_processed] || 0,
+            jobs_failed: stats[:jobs_failed] || 0,
+            active_jobs: stats[:active_jobs] || 0,
+            queues: stats[:queues] || [],
+            concurrency: stats[:concurrency] || 1,
+            memory_usage: stats[:memory_usage],
+            cpu_usage: stats[:cpu_usage],
+            uptime_seconds: stats[:uptime_seconds]
+          }.compact,
+          capabilities: {
+            language: 'ruby',
+            language_version: RUBY_VERSION,
+            protocol_version: VERSION,
+            supported_job_types: capabilities[:supported_job_types],
+            max_job_size: capabilities[:max_job_size] || 10_485_760, # 10MB default
+            features: capabilities[:features] || []
+          }.compact
+        }
+      end
+
+      def self.generate_uuid
+        require 'securerandom'
+        SecureRandom.uuid
+      end
+    end
+
+    # Message validator
+    class Validator
+      def self.valid_job_request?(message)
+        return false unless message.is_a?(Hash)
+
+        required = %w[type version job_id job_class queue arguments metadata]
+        required.all? { |key| message.key?(key) } &&
+          message['type'] == MessageType::JOB_REQUEST &&
+          message['version'] =~ /^\d+\.\d+\.\d+$/ &&
+          message['metadata'].is_a?(Hash) &&
+          message['metadata']['created_at'].is_a?(String)
+      rescue StandardError
+        false
+      end
+
+      def self.valid_job_response?(message)
+        return false unless message.is_a?(Hash)
+
+        required = %w[type version job_id status result metadata]
+        required.all? { |key| message.key?(key) } &&
+          message['type'] == MessageType::JOB_RESPONSE &&
+          %w[success failure partial].include?(message['status'])
+      rescue StandardError
+        false
+      end
+
+      def self.valid_job_error?(message)
+        return false unless message.is_a?(Hash)
+
+        required = %w[type version job_id error_code error_message error_class metadata]
+        required.all? { |key| message.key?(key) } &&
+          message['type'] == MessageType::JOB_ERROR
+      rescue StandardError
+        false
+      end
+
+      def self.validate_message(message)
+        case message['type']
+        when MessageType::JOB_REQUEST
+          valid_job_request?(message)
+        when MessageType::JOB_RESPONSE
+          valid_job_response?(message)
+        when MessageType::JOB_ERROR
+          valid_job_error?(message)
+        else
+          false
+        end
+      end
+    end
+
+    # Type mapper for cross-language support
+    class TypeMapper
+      # Ruby to Python type mapping
+      RUBY_TO_PYTHON = {
+        'NilClass' => 'None',
+        'TrueClass' => 'bool',
+        'FalseClass' => 'bool',
+        'Integer' => 'int',
+        'Float' => 'float',
+        'String' => 'str',
+        'Symbol' => 'str',
+        'Array' => 'list',
+        'Hash' => 'dict',
+        'Time' => 'datetime',
+        'Date' => 'date',
+        'DateTime' => 'datetime'
+      }.freeze
+
+      # Python to Ruby type mapping
+      PYTHON_TO_RUBY = {
+        'None' => 'nil',
+        'bool' => 'Boolean',
+        'int' => 'Integer',
+        'float' => 'Float',
+        'str' => 'String',
+        'list' => 'Array',
+        'tuple' => 'Array',
+        'dict' => 'Hash',
+        'datetime' => 'Time',
+        'date' => 'Date',
+        'bytes' => 'String'
+      }.freeze
+
+      def self.serialize_for_language(value, target_language)
+        case target_language.downcase
+        when 'python'
+          serialize_for_python(value)
+        when 'javascript', 'node', 'nodejs'
+          serialize_for_javascript(value)
+        when 'go', 'golang'
+          serialize_for_go(value)
+        else
+          value
+        end
+      end
+
+      def self.serialize_for_python(value)
+        case value
+        when Time, DateTime
+          value.utc.iso8601
+        when Date
+          value.iso8601
+        when Symbol
+          value.to_s
+        when nil
+          nil
+        else
+          value
+        end
+      end
+
+      def self.serialize_for_javascript(value)
+        case value
+        when Time, DateTime
+          value.utc.iso8601
+        when Date
+          value.iso8601
+        when Symbol
+          value.to_s
+        when BigDecimal
+          value.to_f
+        else
+          value
+        end
+      end
+
+      def self.serialize_for_go(value)
+        case value
+        when Time, DateTime
+          value.utc.iso8601
+        when Date
+          value.iso8601
+        when Symbol
+          value.to_s
+        else
+          value
+        end
+      end
+
+      def self.deserialize_from_language(value, source_language)
+        case source_language.downcase
+        when 'python'
+          deserialize_from_python(value)
+        else
+          value
+        end
+      end
+
+      def self.deserialize_from_python(value)
+        if value.is_a?(String) && value =~ /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/
+          Time.parse(value)
+        else
+          value
+        end
+      end
+    end
+
+    # Language router for directing jobs to appropriate workers
+    class LanguageRouter
+      def self.route_job(job_message, available_workers)
+        constraints = job_message.dig('metadata', 'worker_constraints') || {}
+        language = constraints['language']
+
+        if language
+          # Filter workers by language
+          workers = available_workers.select do |worker|
+            worker[:language] == language
+          end
+
+          return workers.first if workers.any?
+        end
+
+        # Fallback to any available worker
+        available_workers.first
+      end
+
+      def self.queue_for_language(base_queue, language = nil)
+        return base_queue unless language
+
+        "#{base_queue}.#{language.downcase}"
+      end
+
+      def self.parse_queue_language(queue_name)
+        parts = queue_name.split('.')
+        return [queue_name, nil] if parts.length == 1
+
+        language = parts.last
+        base_queue = parts[0..-2].join('.')
+
+        [base_queue, language]
+      end
+    end
+
+    # Protocol version negotiation
+    class VersionNegotiator
+      SUPPORTED_VERSIONS = ['1.0.0'].freeze
+
+      def self.negotiate(client_version, server_versions = SUPPORTED_VERSIONS)
+        return VERSION if client_version.nil?
+
+        client_major = parse_major_version(client_version)
+
+        # Find highest compatible version
+        compatible = server_versions.select do |version|
+          parse_major_version(version) == client_major
+        end
+
+        compatible.max || nil
+      end
+
+      def self.compatible?(version1, version2)
+        parse_major_version(version1) == parse_major_version(version2)
+      end
+
+      def self.parse_major_version(version)
+        version.to_s.split('.').first.to_i
+      end
+    end
+  end
+end

data/lib/natswork/queue_manager.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+require 'concurrent'
+
+module NatsWork
+  class QueueManager
+    attr_reader :subscriptions, :priorities, :concurrency_limits
+
+    def initialize(connection)
+      @connection = connection
+      @subscriptions = Concurrent::Hash.new
+      @callbacks = Concurrent::Hash.new
+      @priorities = Concurrent::Hash.new
+      @concurrency_limits = Concurrent::Hash.new
+      @paused_queues = Concurrent::Array.new
+      @active_counts = Concurrent::Hash.new { 0 }
+      @mutex = Mutex.new
+    end
+
+    def subscribe(queue, &block)
+      subject = queue_subject(queue)
+
+      sid = @connection.subscribe(subject) do |msg|
+        process_message(queue, msg, &block) unless paused?(queue)
+      end
+
+      @subscriptions[queue] = sid
+      @callbacks[queue] = block
+      sid
+    end
+
+    def unsubscribe(queue)
+      @mutex.synchronize do
+        sid = @subscriptions.delete(queue)
+        @connection.unsubscribe(sid) if sid
+        @callbacks.delete(queue)
+      end
+    end
+
+    def unsubscribe_all
+      @mutex.synchronize do
+        @subscriptions.each_value do |sid|
+          @connection.unsubscribe(sid)
+        end
+        @subscriptions.clear
+        @callbacks.clear
+      end
+    end
+
+    def add_queue(queue, &block)
+      subscribe(queue, &block)
+    end
+
+    def remove_queue(queue)
+      unsubscribe(queue)
+    end
+
+    def pause(queue)
+      @paused_queues << queue unless @paused_queues.include?(queue)
+    end
+
+    def pause_all
+      @subscriptions.each_key { |queue| pause(queue) }
+    end
+
+    def resume(queue)
+      @paused_queues.delete(queue)
+    end
+
+    def resume_all
+      @paused_queues.clear
+    end
+
+    def paused?(queue)
+      @paused_queues.include?(queue)
+    end
+
+    def paused_queues
+      @paused_queues.to_a
+    end
+
+    def set_priority(queue, priority)
+      @priorities[queue] = priority
+    end
+
+    def set_concurrency(queue, limit)
+      @concurrency_limits[queue] = limit
+    end
+
+    def drain(queue, timeout: 30)
+      deadline = Time.now + timeout
+
+      # Process any pending messages
+      process_pending_messages(queue)
+
+      # Wait for timeout
+      sleep 0.1 while Time.now < deadline
+    end
+
+    def drain_all(timeout: 30)
+      threads = @subscriptions.keys.map do |queue|
+        Thread.new { drain(queue, timeout: timeout) }
+      end
+      threads.each(&:join)
+    end
+
+    def stats
+      {
+        queues: @subscriptions.keys,
+        priorities: @priorities.to_h,
+        paused: paused_queues,
+        total_subscriptions: @subscriptions.size,
+        concurrency_limits: @concurrency_limits.to_h,
+        active_counts: @active_counts.to_h
+      }
+    end
+
+    private
+
+    def queue_subject(queue)
+      "natswork.queue.#{queue}"
+    end
+
+    def process_message(queue, msg, &block)
+      # Check concurrency limit
+      return if limited?(queue) && at_concurrency_limit?(queue)
+
+      begin
+        increment_active(queue)
+        block&.call(msg)
+      ensure
+        decrement_active(queue)
+      end
+    end
+
+    def process_pending_messages(queue)
+      # In real implementation, this would process any buffered messages
+      # For now, it's a placeholder for draining logic
+    end
+
+    def limited?(queue)
+      @concurrency_limits.key?(queue)
+    end
+
+    def at_concurrency_limit?(queue)
+      limit = @concurrency_limits[queue]
+      return false unless limit
+
+      @active_counts[queue] >= limit
+    end
+
+    def increment_active(queue)
+      @mutex.synchronize do
+        @active_counts[queue] = @active_counts[queue] + 1
+      end
+    end
+
+    def decrement_active(queue)
+      @mutex.synchronize do
+        @active_counts[queue] = [@active_counts[queue] - 1, 0].max
+      end
+    end
+  end
+end