ignis-collective 0.0.1

data/lib/nvruby/collective/transport/base.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Ignis
+  module Collective
+    module Transport
+      # Abstract base class for all transport implementations
+      # Each transport handles GPU-to-GPU or GPU-to-network data movement
+      class Base
+        # Transport type identifier
+        # @return [Symbol] Transport type
+        def self.transport_type
+          raise NotImplementedError, "Subclass must define transport_type"
+        end
+
+        # @return [Integer] Source device ID
+        attr_reader :src_device
+
+        # @return [Integer] Destination device ID
+        attr_reader :dst_device
+
+        # @param src_device [Integer] Source GPU device ID
+        # @param dst_device [Integer] Destination GPU device ID
+        def initialize(src_device:, dst_device:)
+          @src_device = src_device
+          @dst_device = dst_device
+          @initialized = false
+        end
+
+        # Initialize the transport (called once per communicator)
+        # @return [void]
+        def initialize!
+          raise NotImplementedError
+        end
+
+        # Check if transport is initialized and ready
+        # @return [Boolean] True if ready for use
+        def ready?
+          @initialized
+        end
+
+        # Send data asynchronously
+        # @param buffer [FFI::Pointer] Device pointer to send
+        # @param size [Integer] Bytes to send
+        # @param stream [CUDA::Stream, FFI::Pointer] CUDA stream for async execution
+        # @return [void]
+        def send_async(buffer, size, stream)
+          raise NotImplementedError
+        end
+
+        # Receive data asynchronously
+        # @param buffer [FFI::Pointer] Device pointer to receive into
+        # @param size [Integer] Bytes to receive
+        # @param stream [CUDA::Stream, FFI::Pointer] CUDA stream for async execution
+        # @return [void]
+        def recv_async(buffer, size, stream)
+          raise NotImplementedError
+        end
+
+        # Synchronous send (waits for completion)
+        # @param buffer [FFI::Pointer] Device pointer to send
+        # @param size [Integer] Bytes to send
+        # @return [void]
+        def send_sync(buffer, size)
+          null_stream = FFI::Pointer::NULL
+          send_async(buffer, size, null_stream)
+          synchronize!
+        end
+
+        # Synchronous receive (waits for completion)
+        # @param buffer [FFI::Pointer] Device pointer to receive into
+        # @param size [Integer] Bytes to receive
+        # @return [void]
+        def recv_sync(buffer, size)
+          null_stream = FFI::Pointer::NULL
+          recv_async(buffer, size, null_stream)
+          synchronize!
+        end
+
+        # Wait for all pending operations to complete
+        # @return [void]
+        def synchronize!
+          CUDA::RuntimeAPI.ensure_loaded!
+          status = CUDA::RuntimeAPI.cudaDeviceSynchronize
+          CUDA::RuntimeAPI.check_status!(status, "Transport synchronize")
+        end
+
+        # Estimated bandwidth in GB/s
+        # @return [Float] Bandwidth estimate
+        def estimated_bandwidth
+          raise NotImplementedError
+        end
+
+        # Estimated latency in microseconds
+        # @return [Float] Latency estimate
+        def estimated_latency
+          raise NotImplementedError
+        end
+
+        # Check if this transport is available for the given GPU pair
+        # @param src [Integer] Source GPU
+        # @param dst [Integer] Destination GPU
+        # @return [Boolean] True if available
+        def self.available?(src, dst)
+          raise NotImplementedError
+        end
+
+        # Clean up resources
+        # @return [void]
+        def destroy!
+          @initialized = false
+        end
+
+        # @return [String] Human-readable description
+        def to_s
+          "#{self.class.transport_type}[#{@src_device}→#{@dst_device}]"
+        end
+
+        protected
+
+        # Get stream pointer for FFI calls
+        # @param stream [CUDA::Stream, FFI::Pointer, nil] Stream
+        # @return [FFI::Pointer] Stream pointer
+        def stream_ptr(stream)
+          case stream
+          when FFI::Pointer
+            stream
+          when CUDA::Stream
+            stream.ptr
+          when NilClass
+            FFI::Pointer::NULL
+          else
+            raise ArgumentError, "Invalid stream type: #{stream.class}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nvruby/collective/transport/host_staged_transport.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Ignis
+  module Collective
+    module Transport
+      # Host-Staged Transport - Fallback when P2P is not available
+      #
+      # Uses host (pinned) memory as an intermediate staging buffer.
+      # Data path: GPU_A -> Host -> GPU_B
+      #
+      # Slower than P2P but always works regardless of topology.
+      # Bandwidth limited by PCIe x2 (upload + download).
+      class HostStagedTransport < Base
+        # @return [Symbol] Transport type identifier
+        def self.transport_type
+          :host_staged
+        end
+
+        # @return [Float] Estimated bandwidth (GB/s)
+        def estimated_bandwidth
+          12.0  # PCIe 4.0 x16 / 2 (round trip)
+        end
+
+        # @return [Float] Estimated latency (microseconds)
+        def estimated_latency
+          25.0  # Higher due to double copy
+        end
+
+        # Initialize the transport
+        # @return [void]
+        def initialize!
+          return if @initialized
+
+          CUDA::RuntimeAPI.ensure_loaded!
+          @staging_buffers = {}  # size -> pinned host buffer
+          @initialized = true
+        end
+
+        # Copy data via host staging
+        # @param dst_buffer [FFI::Pointer] Destination GPU buffer
+        # @param src_buffer [FFI::Pointer] Source GPU buffer
+        # @param size [Integer] Size in bytes
+        # @param stream [FFI::Pointer] CUDA stream (for async ops)
+        # @return [void]
+        def copy_async(dst_buffer, src_buffer, size, stream)
+          ensure_initialized!
+
+          # Get or allocate staging buffer
+          staging = get_staging_buffer(size)
+
+          # Step 1: GPU_src -> Host (async)
+          CUDA::RuntimeAPI.cudaSetDevice(@src_device)
+          status = CUDA::RuntimeAPI.cudaMemcpyAsync(
+            staging,
+            src_buffer,
+            size,
+            CUDA::RuntimeAPI::MEMCPY_DEVICE_TO_HOST,
+            stream
+          )
+          CUDA::RuntimeAPI.check_status!(status, "Host-staged D2H copy")
+
+          # Synchronize to ensure data is in host memory
+          sync_stream(stream)
+
+          # Step 2: Host -> GPU_dst (async)
+          CUDA::RuntimeAPI.cudaSetDevice(@dst_device)
+          status = CUDA::RuntimeAPI.cudaMemcpyAsync(
+            dst_buffer,
+            staging,
+            size,
+            CUDA::RuntimeAPI::MEMCPY_HOST_TO_DEVICE,
+            stream
+          )
+          CUDA::RuntimeAPI.check_status!(status, "Host-staged H2D copy")
+        end
+
+        # Synchronous copy
+        # @param dst_buffer [FFI::Pointer] Destination
+        # @param src_buffer [FFI::Pointer] Source
+        # @param size [Integer] Size in bytes
+        # @return [void]
+        def copy_sync(dst_buffer, src_buffer, size)
+          ensure_initialized!
+
+          staging = get_staging_buffer(size)
+
+          # GPU_src -> Host
+          CUDA::RuntimeAPI.cudaSetDevice(@src_device)
+          status = CUDA::RuntimeAPI.cudaMemcpy(
+            staging,
+            src_buffer,
+            size,
+            CUDA::RuntimeAPI::MEMCPY_DEVICE_TO_HOST
+          )
+          CUDA::RuntimeAPI.check_status!(status, "Host-staged D2H sync")
+
+          # Host -> GPU_dst
+          CUDA::RuntimeAPI.cudaSetDevice(@dst_device)
+          status = CUDA::RuntimeAPI.cudaMemcpy(
+            dst_buffer,
+            staging,
+            size,
+            CUDA::RuntimeAPI::MEMCPY_HOST_TO_DEVICE
+          )
+          CUDA::RuntimeAPI.check_status!(status, "Host-staged H2D sync")
+        end
+
+        # Async send (GPU to host staging)
+        # @param buffer [FFI::Pointer] Source GPU buffer
+        # @param size [Integer] Size in bytes
+        # @param stream [FFI::Pointer] CUDA stream
+        # @return [FFI::Pointer] Staging buffer with data
+        def send_async(buffer, size, stream)
+          ensure_initialized!
+
+          staging = get_staging_buffer(size)
+
+          CUDA::RuntimeAPI.cudaSetDevice(@src_device)
+          status = CUDA::RuntimeAPI.cudaMemcpyAsync(
+            staging,
+            buffer,
+            size,
+            CUDA::RuntimeAPI::MEMCPY_DEVICE_TO_HOST,
+            stream
+          )
+          CUDA::RuntimeAPI.check_status!(status, "Host-staged send")
+
+          staging
+        end
+
+        # Async receive (host staging to GPU)
+        # @param buffer [FFI::Pointer] Destination GPU buffer
+        # @param staging [FFI::Pointer] Host staging buffer with data
+        # @param size [Integer] Size in bytes
+        # @param stream [FFI::Pointer] CUDA stream
+        # @return [void]
+        def recv_async(buffer, staging, size, stream)
+          ensure_initialized!
+
+          CUDA::RuntimeAPI.cudaSetDevice(@dst_device)
+          status = CUDA::RuntimeAPI.cudaMemcpyAsync(
+            buffer,
+            staging,
+            size,
+            CUDA::RuntimeAPI::MEMCPY_HOST_TO_DEVICE,
+            stream
+          )
+          CUDA::RuntimeAPI.check_status!(status, "Host-staged recv")
+        end
+
+        # Check if host-staged is available (always true)
+        # @return [Boolean] Always true
+        def self.available?
+          true
+        end
+
+        # Clean up staging buffers
+        # @return [void]
+        def destroy!
+          @staging_buffers.each_value do |buf|
+            CUDA::RuntimeAPI.cudaFreeHost(buf) rescue nil
+          end
+          @staging_buffers.clear
+          @initialized = false
+        end
+
+        private
+
+        # Get or allocate a pinned host staging buffer
+        # @param size [Integer] Required size
+        # @return [FFI::Pointer] Pinned host buffer
+        def get_staging_buffer(size)
+          # Round up to nearest power of 2 for reuse efficiency
+          rounded_size = next_power_of_2(size)
+
+          return @staging_buffers[rounded_size] if @staging_buffers[rounded_size]
+
+          # Allocate pinned host memory
+          ptr_ptr = FFI::MemoryPointer.new(:pointer)
+          status = CUDA::RuntimeAPI.cudaHostAlloc(
+            ptr_ptr,
+            rounded_size,
+            1  # cudaHostAllocDefault
+          )
+          CUDA::RuntimeAPI.check_status!(status, "Alloc pinned staging buffer")
+
+          @staging_buffers[rounded_size] = ptr_ptr.read_pointer
+        end
+
+        # Round up to next power of 2
+        def next_power_of_2(n)
+          return 1 if n <= 0
+
+          n -= 1
+          n |= n >> 1
+          n |= n >> 2
+          n |= n >> 4
+          n |= n >> 8
+          n |= n >> 16
+          n |= n >> 32
+          n + 1
+        end
+
+        # Synchronize stream
+        def sync_stream(stream)
+          if stream.null?
+            CUDA::RuntimeAPI.cudaDeviceSynchronize
+          else
+            CUDA::RuntimeAPI.cudaStreamSynchronize(stream)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/nvruby/collective/transport/ipc_transport.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../p2p_bindings"
+
+module Ignis
+  module Collective
+    module Transport
+      # CUDA IPC transport for inter-process GPU memory sharing
+      # Uses cudaIpcGetMemHandle/cudaIpcOpenMemHandle for zero-copy cross-process access
+      # Supports both legacy IPC and cuMem VMM API
+      class IPCTransport < Base
+        # @return [Symbol] Transport type identifier
+        def self.transport_type
+          :cuda_ipc
+        end
+
+        # Estimated bandwidth (host memory throughput for handle exchange)
+        BANDWIDTH_GBS = 25.0
+
+        # Estimated latency (IPC overhead)
+        LATENCY_US = 10.0
+
+        # Handle exchange methods
+        EXCHANGE_METHODS = [:named_pipe, :shared_memory, :socket].freeze
+
+        # @return [P2PBindings::CudaIpcMemHandle, nil] Exported handle
+        attr_reader :exported_handle
+
+        # @return [Symbol] Handle exchange method
+        attr_reader :exchange_method
+
+        # @param src_device [Integer] Source GPU
+        # @param dst_device [Integer] Destination GPU
+        # @param exchange_method [Symbol] Method for handle exchange
+        def initialize(src_device:, dst_device:, exchange_method: :named_pipe)
+          super(src_device: src_device, dst_device: dst_device)
+          @exchange_method = exchange_method
+          @exported_handles = {}  # device_ptr -> handle
+          @imported_handles = {}  # handle -> mapped_ptr
+        end
+
+        # Initialize IPC transport
+        # @return [void]
+        def initialize!
+          return if @initialized
+
+          P2PBindings.ensure_loaded!
+          CUDA::RuntimeAPI.ensure_loaded!
+
+          @initialized = true
+        end
+
+        # Export GPU memory for sharing with another process
+        # @param device_ptr [FFI::Pointer] GPU memory pointer to export
+        # @return [String] Binary handle (64 bytes) for transfer to other process
+        def export_handle(device_ptr)
+          ensure_initialized!
+
+          # Set source device context
+          status = CUDA::RuntimeAPI.cudaSetDevice(@src_device)
+          CUDA::RuntimeAPI.check_status!(status, "Set device for IPC export")
+
+          handle = P2PBindings::CudaIpcMemHandle.new
+          status = P2PBindings.cudaIpcGetMemHandle(handle, device_ptr)
+          P2PBindings.check_status!(status, "Get IPC memory handle")
+
+          # Cache the handle for cleanup
+          handle_bytes = handle[:reserved].to_a.pack("C*")
+          @exported_handles[device_ptr.address] = handle_bytes
+
+          handle_bytes
+        end
+
+        # Import GPU memory handle from another process
+        # @param handle_bytes [String] 64-byte binary handle
+        # @param flags [Integer] IPC memory flags
+        # @return [FFI::Pointer] Mapped device pointer usable in this process
+        def import_handle(handle_bytes, flags: P2PBindings::IPC_MEM_LAZY_ENABLE_PEER_ACCESS)
+          ensure_initialized!
+
+          # Set destination device context
+          status = CUDA::RuntimeAPI.cudaSetDevice(@dst_device)
+          CUDA::RuntimeAPI.check_status!(status, "Set device for IPC import")
+
+          # Reconstruct handle from bytes
+          handle = P2PBindings::CudaIpcMemHandle.new
+          handle_bytes.each_byte.with_index do |byte, i|
+            handle[:reserved][i] = byte
+          end
+
+          # Open the handle
+          mapped_ptr_ptr = FFI::MemoryPointer.new(:pointer)
+          status = P2PBindings.cudaIpcOpenMemHandle(mapped_ptr_ptr, handle, flags)
+          P2PBindings.check_status!(status, "Open IPC memory handle")
+
+          mapped_ptr = mapped_ptr_ptr.read_pointer
+          @imported_handles[handle_bytes] = mapped_ptr
+
+          mapped_ptr
+        end
+
+        # Close an imported handle
+        # @param mapped_ptr [FFI::Pointer] Previously imported pointer
+        # @return [void]
+        def close_imported_handle(mapped_ptr)
+          ensure_initialized!
+
+          status = P2PBindings.cudaIpcCloseMemHandle(mapped_ptr)
+          P2PBindings.check_status!(status, "Close IPC memory handle")
+
+          @imported_handles.delete_if { |_, ptr| ptr == mapped_ptr }
+        end
+
+        # Send data using IPC (export phase)
+        # In IPC, send means making buffer available to receiver
+        # @param buffer [FFI::Pointer] Buffer to share
+        # @param size [Integer] Buffer size (for validation)
+        # @param stream [Object] CUDA stream (unused for export, sync only)
+        # @return [String] Handle bytes to transmit to receiver
+        def send_async(buffer, size, stream)
+          export_handle(buffer)
+        end
+
+        # Receive data using IPC (import phase)
+        # @param buffer [FFI::Pointer] NOT USED - returns new mapped pointer
+        # @param size [Integer] Expected size
+        # @param stream [Object] CUDA stream
+        # @return [FFI::Pointer] Mapped device pointer
+        def recv_async(buffer, size, stream)
+          # IPC recv requires handle from sender - actual implementation
+          # would receive handle via exchange_method and then import
+          raise TransportError, "IPC recv requires handle - use recv_with_handle"
+        end
+
+        # Receive using provided handle
+        # @param handle_bytes [String] Handle from sender
+        # @return [FFI::Pointer] Mapped device pointer
+        def recv_with_handle(handle_bytes)
+          import_handle(handle_bytes)
+        end
+
+        # @return [Float] Bandwidth in GB/s
+        def estimated_bandwidth
+          BANDWIDTH_GBS
+        end
+
+        # @return [Float] Latency in microseconds
+        def estimated_latency
+          LATENCY_US
+        end
+
+        # IPC is always available on Windows for same-node communication
+        # @param src [Integer] Source GPU
+        # @param dst [Integer] Destination GPU
+        # @return [Boolean] True (always available)
+        def self.available?(src, dst)
+          true
+        end
+
+        # Clean up all handles
+        # @return [void]
+        def destroy!
+          @imported_handles.each_value do |ptr|
+            P2PBindings.cudaIpcCloseMemHandle(ptr)
+          rescue StandardError
+            # Ignore cleanup errors
+          end
+          @imported_handles.clear
+          @exported_handles.clear
+          super
+        end
+
+        # @return [String] Transport description
+        def to_s
+          "IPC[#{@src_device}→#{@dst_device}, #{@exchange_method}]"
+        end
+
+        private
+
+        def ensure_initialized!
+          raise TransportError, "IPC Transport not initialized" unless @initialized
+        end
+      end
+    end
+  end
+end

data/lib/nvruby/collective/transport/p2p_transport.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../p2p_bindings"
+
+module Ignis
+  module Collective
+    module Transport
+      # PCIe/NVLink peer-to-peer transport for direct GPU-to-GPU transfers
+      # Uses cudaMemcpyPeerAsync for high-bandwidth same-process communication
+      class P2PTransport < Base
+        # @return [Symbol] Transport type identifier
+        def self.transport_type
+          :pcie_p2p
+        end
+
+        # Estimated bandwidth based on interconnect type
+        BANDWIDTH_ESTIMATES = {
+          nvlink: 900.0,    # GB/s - NVLink 4.0
+          pcie_p2p: 32.0,   # GB/s - PCIe Gen4 x16
+        }.freeze
+
+        # Estimated latency
+        LATENCY_ESTIMATES = {
+          nvlink: 1.0,      # microseconds
+          pcie_p2p: 5.0,    # microseconds
+        }.freeze
+
+        # @return [Symbol] Actual interconnect type (:nvlink or :pcie_p2p)
+        attr_reader :interconnect_type
+
+        # @param src_device [Integer] Source GPU
+        # @param dst_device [Integer] Destination GPU
+        # @param interconnect_type [Symbol] Detected interconnect type
+        def initialize(src_device:, dst_device:, interconnect_type: :pcie_p2p)
+          super(src_device: src_device, dst_device: dst_device)
+          @interconnect_type = interconnect_type
+          @peer_access_enabled = false
+        end
+
+        # Initialize P2P transport by enabling peer access
+        # @return [void]
+        def initialize!
+          return if @initialized
+
+          P2PBindings.ensure_loaded!
+          CUDA::RuntimeAPI.ensure_loaded!
+
+          # Set source device context
+          status = CUDA::RuntimeAPI.cudaSetDevice(@src_device)
+          CUDA::RuntimeAPI.check_status!(status, "Set device #{@src_device}")
+
+          # Enable peer access to destination
+          status = P2PBindings.cudaDeviceEnablePeerAccess(@dst_device, 0)
+
+          # Status 0 = success, 704 = already enabled (cudaErrorPeerAccessAlreadyEnabled)
+          unless status.zero? || status == 704
+            P2PBindings.check_status!(status, "Enable peer access #{@src_device}→#{@dst_device}")
+          end
+
+          @peer_access_enabled = true
+          @initialized = true
+        end
+
+        # Send data to destination GPU asynchronously
+        # @param buffer [FFI::Pointer] Source buffer on src_device
+        # @param size [Integer] Bytes to send
+        # @param stream [CUDA::Stream, FFI::Pointer] CUDA stream
+        # @return [void]
+        def send_async(buffer, size, stream)
+          ensure_initialized!
+
+          # For P2P, we use the receive buffer on dst, so this is a no-op
+          # The actual transfer happens in recv_async with combined send buffer
+          # This is for the half-duplex ring pattern
+          nil
+        end
+
+        # Receive data from source GPU asynchronously
+        # @param dst_buffer [FFI::Pointer] Destination buffer on dst_device
+        # @param src_buffer [FFI::Pointer] Source buffer on src_device
+        # @param size [Integer] Bytes to receive
+        # @param stream [CUDA::Stream, FFI::Pointer] CUDA stream
+        # @return [void]
+        def copy_async(dst_buffer, src_buffer, size, stream)
+          ensure_initialized!
+
+          status = P2PBindings.cudaMemcpyPeerAsync(
+            dst_buffer,
+            @dst_device,
+            src_buffer,
+            @src_device,
+            size,
+            stream_ptr(stream)
+          )
+          P2PBindings.check_status!(status, "P2P copy #{@src_device}→#{@dst_device}")
+        end
+
+        # Alias for copy_async with reversed semantics
+        # @param buffer [FFI::Pointer] Destination buffer
+        # @param size [Integer] Bytes to receive
+        # @param stream [CUDA::Stream, FFI::Pointer] CUDA stream
+        # @return [void]
+        def recv_async(buffer, size, stream)
+          # Note: For full implementation, source buffer would come from send side
+          # In ring allreduce, we manage buffers differently
+          nil
+        end
+
+        # @return [Float] Bandwidth in GB/s
+        def estimated_bandwidth
+          BANDWIDTH_ESTIMATES[@interconnect_type] || 12.0
+        end
+
+        # @return [Float] Latency in microseconds
+        def estimated_latency
+          LATENCY_ESTIMATES[@interconnect_type] || 20.0
+        end
+
+        # Check if P2P is available between two GPUs
+        # @param src [Integer] Source GPU
+        # @param dst [Integer] Destination GPU
+        # @return [Boolean] True if P2P available
+        def self.available?(src, dst)
+          P2PBindings.ensure_loaded!
+
+          can_access_ptr = FFI::MemoryPointer.new(:int)
+          status = P2PBindings.cudaDeviceCanAccessPeer(can_access_ptr, src, dst)
+          status.zero? && can_access_ptr.read_int == 1
+        end
+
+        # Clean up by disabling peer access
+        # @return [void]
+        def destroy!
+          if @peer_access_enabled
+            CUDA::RuntimeAPI.cudaSetDevice(@src_device)
+            P2PBindings.cudaDeviceDisablePeerAccess(@dst_device)
+            @peer_access_enabled = false
+          end
+          super
+        end
+
+        # @return [String] Transport description
+        def to_s
+          bw = estimated_bandwidth
+          "P2P[#{@src_device}→#{@dst_device}, #{@interconnect_type}, #{bw} GB/s]"
+        end
+
+        private
+
+        def ensure_initialized!
+          raise TransportError, "Transport not initialized" unless @initialized
+        end
+      end
+    end
+  end
+end