io-stream 0.6.1 → 0.8.0

data/lib/io/stream/readable.rb

@@ -0,0 +1,358 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2023-2024, by Samuel Williams.
+
+require_relative "string_buffer"
+
+module IO::Stream
+	# The default block size for IO buffers. Defaults to 256KB (optimized for modern SSDs and networks).
+	BLOCK_SIZE = ENV.fetch("IO_STREAM_BLOCK_SIZE", 1024*256).to_i
+
+	# The minimum read size for efficient I/O operations. Defaults to the same as BLOCK_SIZE.
+	MINIMUM_READ_SIZE = ENV.fetch("IO_STREAM_MINIMUM_READ_SIZE", BLOCK_SIZE).to_i
+
+	# The maximum read size for a single read operation. This limit exists because:
+	# 1. System calls like read() cannot handle requests larger than SSIZE_MAX
+	# 2. Very large reads can cause memory pressure and poor interactive performance  
+	# 3. Most socket buffers and pipe capacities are much smaller anyway
+	# On 64-bit systems SSIZE_MAX is ~8.8 million MB, on 32-bit it's ~2GB.
+	# Our default of 16MB provides a good balance of throughput and responsiveness, and is page aligned.
+	# It is also a multiple of the minimum read size, so that we can read in chunks without exceeding the maximum.
+	MAXIMUM_READ_SIZE = ENV.fetch("IO_STREAM_MAXIMUM_READ_SIZE", MINIMUM_READ_SIZE * 64).to_i
+	
+	# A module providing readable stream functionality.
+	#
+	# You must implement the `sysread` method to read data from the underlying IO.
+	module Readable
+		# Initialize readable stream functionality.
+		# @parameter minimum_read_size [Integer] The minimum size for read operations.
+		# @parameter maximum_read_size [Integer] The maximum size for read operations.
+		# @parameter block_size [Integer] Legacy parameter, use minimum_read_size instead.
+		def initialize(minimum_read_size: MINIMUM_READ_SIZE, maximum_read_size: MAXIMUM_READ_SIZE, block_size: nil, **, &block)
+			@done = false
+			@read_buffer = StringBuffer.new
+			# Used as destination buffer for underlying reads.
+			@input_buffer = StringBuffer.new
+			
+			# Support legacy block_size parameter for backwards compatibility
+			@minimum_read_size = block_size || minimum_read_size
+			@maximum_read_size = maximum_read_size
+			
+			super(**, &block) if defined?(super)
+		end
+
+		attr_accessor :minimum_read_size
+
+		# Legacy accessor for backwards compatibility
+		# @returns [Integer] The minimum read size.
+		def block_size
+			@minimum_read_size
+		end
+
+		# Legacy setter for backwards compatibility
+		# @parameter value [Integer] The minimum read size.
+		def block_size=(value)
+			@minimum_read_size = value
+		end
+
+		# Read data from the stream.
+		# @parameter size [Integer | Nil] The number of bytes to read. If nil, read until end of stream.
+		# @returns [String] The data read from the stream.
+		def read(size = nil)
+			return String.new(encoding: Encoding::BINARY) if size == 0
+			
+			if size
+				until @done or @read_buffer.bytesize >= size
+					# Compute the amount of data we need to read from the underlying stream:
+					read_size = size - @read_buffer.bytesize
+					
+					# Don't read less than @minimum_read_size to avoid lots of small reads:
+					fill_read_buffer(read_size > @minimum_read_size ? read_size : @minimum_read_size)
+				end
+			else
+				until @done
+					fill_read_buffer
+				end
+			end
+			
+			return consume_read_buffer(size)
+		end
+		
+		# Read at most `size` bytes from the stream. Will avoid reading from the underlying stream if possible.
+		def read_partial(size = nil)
+			return String.new(encoding: Encoding::BINARY) if size == 0
+		
+			if !@done and @read_buffer.empty?
+				fill_read_buffer
+			end
+			
+			return consume_read_buffer(size)
+		end
+		
+		# Read exactly the specified number of bytes.
+		# @parameter size [Integer] The number of bytes to read.
+		# @parameter exception [Class] The exception to raise if not enough data is available.
+		# @returns [String] The data read from the stream.
+		def read_exactly(size, exception: EOFError)
+			if buffer = read(size)
+				if buffer.bytesize != size
+					raise exception, "Could not read enough data!"
+				end
+				
+				return buffer
+			end
+			
+			raise exception, "Encountered done while reading data!"
+		end
+		
+		# This is a compatibility shim for existing code that uses `readpartial`.
+		def readpartial(size = nil)
+			read_partial(size) or raise EOFError, "Encountered done while reading data!"
+		end
+		
+		# Find the index of a pattern in the read buffer, reading more data if needed.
+		# @parameter pattern [String] The pattern to search for.
+		# @parameter offset [Integer] The offset to start searching from.
+		# @parameter limit [Integer | Nil] The maximum number of bytes to read while searching.
+		# @returns [Integer | Nil] The index of the pattern, or nil if not found.
+		private def index_of(pattern, offset, limit, discard = false)
+			# We don't want to split on the pattern, so we subtract the size of the pattern.
+			split_offset = pattern.bytesize - 1
+			
+			until index = @read_buffer.index(pattern, offset)
+				offset = @read_buffer.bytesize - split_offset
+				
+				offset = 0 if offset < 0
+				
+				if limit and offset >= limit
+					return nil
+				end
+				
+				unless fill_read_buffer
+					return nil
+				end
+				
+				if discard
+					# If we are discarding, we should consume the read buffer up to the offset:
+					consume_read_buffer(offset)
+					offset = 0
+				end
+			end
+			
+			return index
+		end
+		
+		# Efficiently read data from the stream until encountering pattern.
+		# @parameter pattern [String] The pattern to match.
+		# @parameter offset [Integer] The offset to start searching from.
+		# @parameter limit [Integer] The maximum number of bytes to read, including the pattern (even if chomped).
+		# @parameter chomp [Boolean] Whether to remove the pattern from the returned data.
+		# @returns [String | Nil] The contents of the stream up until the pattern, or nil if the pattern was not found. 
+		def read_until(pattern, offset = 0, limit: nil, chomp: true)
+			if index = index_of(pattern, offset, limit)
+				return nil if limit and index >= limit
+				
+				@read_buffer.freeze
+				matched = @read_buffer.byteslice(0, index+(chomp ? 0 : pattern.bytesize))
+				@read_buffer = @read_buffer.byteslice(index+pattern.bytesize, @read_buffer.bytesize)
+				
+				return matched
+			end
+		end
+		
+		# Efficiently discard data from the stream until encountering pattern.
+		# @parameter pattern [String] The pattern to match.
+		# @parameter offset [Integer] The offset to start searching from.
+		# @parameter limit [Integer] The maximum number of bytes to read, including the pattern.
+		# @returns [String | Nil] The contents of the stream up until the pattern, or nil if the pattern was not found.
+		def discard_until(pattern, offset = 0, limit: nil)
+			if index = index_of(pattern, offset, limit, true)
+				@read_buffer.freeze
+				
+				if limit and index >= limit
+					@read_buffer = @read_buffer.byteslice(limit, @read_buffer.bytesize)
+					
+					return nil
+				end
+				
+				matched = @read_buffer.byteslice(0, index+pattern.bytesize)
+				@read_buffer = @read_buffer.byteslice(index+pattern.bytesize, @read_buffer.bytesize)
+				
+				return matched
+			end
+		end
+		
+		# Peek at data in the buffer without consuming it.
+		# @parameter size [Integer | Nil] The number of bytes to peek at. If nil, peek at all available data.
+		# @returns [String] The data in the buffer without consuming it.
+		def peek(size = nil)
+			if size
+				until @done or @read_buffer.bytesize >= size
+					# Compute the amount of data we need to read from the underlying stream:
+					read_size = size - @read_buffer.bytesize
+					
+					# Don't read less than @minimum_read_size to avoid lots of small reads:
+					fill_read_buffer(read_size > @minimum_read_size ? read_size : @minimum_read_size)
+				end
+				return @read_buffer[..([size, @read_buffer.size].min - 1)]
+			end
+			until (block_given? && yield(@read_buffer)) or @done
+				fill_read_buffer
+			end
+			return @read_buffer
+		end
+		
+		# Read a line from the stream, similar to IO#gets.
+		# @parameter separator [String] The line separator to search for.
+		# @parameter limit [Integer | Nil] The maximum number of bytes to read.
+		# @parameter chomp [Boolean] Whether to remove the separator from the returned line.
+		# @returns [String | Nil] The line read from the stream, or nil if at end of stream.
+		def gets(separator = $/, limit = nil, chomp: false)
+			# Compatibility with IO#gets:
+			if separator.is_a?(Integer)
+				limit = separator
+				separator = $/
+			end
+			
+			# We don't want to split in the middle of the separator, so we subtract the size of the separator from the start of the search:
+			split_offset = separator.bytesize - 1
+			
+			offset = 0
+			
+			until index = @read_buffer.index(separator, offset)
+				offset = @read_buffer.bytesize - split_offset
+				offset = 0 if offset < 0
+				
+				# If a limit was given, and the offset is beyond the limit, we should return up to the limit:
+				if limit and offset >= limit
+					# As we didn't find the separator, there is nothing to chomp either.
+					return consume_read_buffer(limit)
+				end
+				
+				# If we can't read any more data, we should return what we have:
+				return consume_read_buffer unless fill_read_buffer
+			end
+			
+			# If the index of the separator was beyond the limit:
+			if limit and index >= limit
+				# Return up to the limit:
+				return consume_read_buffer(limit)
+			end
+			
+			# Freeze the read buffer, as this enables us to use byteslice without generating a hidden copy:
+			@read_buffer.freeze
+			
+			line = @read_buffer.byteslice(0, index+(chomp ? 0 : separator.bytesize))
+			@read_buffer = @read_buffer.byteslice(index+separator.bytesize, @read_buffer.bytesize)
+			
+			return line
+		end
+		
+		# Determins if the stream has consumed all available data. May block if the stream is not readable.
+		# See {readable?} for a non-blocking alternative.
+		#
+		# @returns [Boolean] If the stream is at file which means there is no more data to be read.
+		def done?
+			if !@read_buffer.empty?
+				return false
+			elsif @done
+				return true
+			else
+				return !self.fill_read_buffer
+			end
+		end
+		
+		alias eof? done?
+		
+		# Mark the stream as done and raise `EOFError`.
+		def done!
+			@read_buffer.clear
+			@done = true
+			
+			raise EOFError
+		end
+		
+		alias eof! done!
+		
+		# Whether there is a chance that a read operation will succeed or not.
+		# @returns [Boolean] If the stream is readable, i.e. a `read` operation has a chance of success.
+		def readable?
+			# If we are at the end of the file, we can't read any more data:
+			if @done
+				return false
+			end
+			
+			# If the read buffer is not empty, we can read more data:
+			if !@read_buffer.empty?
+				return true
+			end
+			
+			# If the underlying stream is readable, we can read more data:
+			return !closed?
+		end
+		
+		# Close the read end of the stream.
+		def close_read
+		end
+		
+		private
+		
+		# Fills the buffer from the underlying stream.
+		def fill_read_buffer(size = @minimum_read_size)
+			# Limit the read size to avoid exceeding SSIZE_MAX and to manage memory usage.
+			# Very large reads can also hurt interactive performance by blocking for too long.
+			if size > @maximum_read_size
+				size = @maximum_read_size
+			end
+			
+			# This effectively ties the input and output stream together.
+			flush
+			
+			if @read_buffer.empty?
+				if sysread(size, @read_buffer)
+					# Console.info(self, name: "read") {@read_buffer.inspect}
+					return true
+				end
+			else
+				if chunk = sysread(size, @input_buffer)
+					@read_buffer << chunk
+					# Console.info(self, name: "read") {@read_buffer.inspect}
+					
+					return true
+				end
+			end
+			
+			# else for both cases above:
+			@done = true
+			return false
+		end
+		
+		# Consumes at most `size` bytes from the buffer.
+		# @parameter size [Integer | Nil] The amount of data to consume. If nil, consume entire buffer.
+		def consume_read_buffer(size = nil)
+			# If we are at done, and the read buffer is empty, we can't consume anything.
+			return nil if @done && @read_buffer.empty?
+			
+			result = nil
+			
+			if size.nil? or size >= @read_buffer.bytesize
+				# Consume the entire read buffer:
+				result = @read_buffer
+				@read_buffer = StringBuffer.new
+			else
+				# This approach uses more memory.
+				# result = @read_buffer.slice!(0, size)
+				
+				# We know that we are not going to reuse the original buffer.
+				# But byteslice will generate a hidden copy. So let's freeze it first:
+				@read_buffer.freeze
+				
+				result = @read_buffer.byteslice(0, size)
+				@read_buffer = @read_buffer.byteslice(size, @read_buffer.bytesize)
+			end
+			
+			return result
+		end
+	end
+end

data/lib/io/stream/shim/buffered.rb

@@ -5,10 +5,14 @@
 
 unless IO.method_defined?(:buffered?, false)
 	class IO
+		# Check if the IO is buffered.
+		# @returns [Boolean] True if the IO is buffered (not synchronized).
 		def buffered?
 			return !self.sync
 		end
 		
+		# Set the buffered state of the IO.
+		# @parameter value [Boolean] True to enable buffering, false to disable.
 		def buffered=(value)
 			self.sync = !value
 		end
@@ -18,13 +22,18 @@ end
 require "socket"
 
 unless BasicSocket.method_defined?(:buffered?, false)
+	# Socket extensions for buffering support.
 	class BasicSocket
+		# Check if this socket uses TCP protocol.
+		# @returns [Boolean] True if the socket is TCP over IPv4 or IPv6.
 		def ip_protocol_tcp?
 			local_address = self.local_address
 			
 			return (local_address.afamily == ::Socket::AF_INET || local_address.afamily == ::Socket::AF_INET6) && local_address.socktype == ::Socket::SOCK_STREAM
 		end
 		
+		# Check if the socket is buffered.
+		# @returns [Boolean] True if the socket is buffered.
 		def buffered?
 			return false unless super
 			
@@ -35,6 +44,8 @@ unless BasicSocket.method_defined?(:buffered?, false)
 			end
 		end
 		
+		# Set the buffered state of the socket.
+		# @parameter value [Boolean] True to enable buffering, false to disable.
 		def buffered=(value)
 			super
 			
@@ -53,11 +64,16 @@ end
 require "stringio"
 
 unless StringIO.method_defined?(:buffered?, false)
+	# StringIO extensions for buffering support.
 	class StringIO
+		# Check if the StringIO is buffered.
+		# @returns [Boolean] True if the StringIO is buffered (not synchronized).
 		def buffered?
 			return !self.sync
 		end
 		
+		# Set the buffered state of the StringIO.
+		# @parameter value [Boolean] True to enable buffering, false to disable.
 		def buffered=(value)
 			self.sync = !value
 		end

data/lib/io/stream/shim/readable.rb

@@ -5,6 +5,8 @@
 
 class IO
 	unless method_defined?(:readable?, false)
+		# Check if the IO is readable.
+		# @returns [Boolean] True if the IO is readable (not closed).
 		def readable?
 			# Do not call `eof?` here as it is not concurrency-safe and it can block.
 			!closed?
@@ -16,6 +18,8 @@ require "socket"
 
 class BasicSocket
 	unless method_defined?(:readable?, false)
+		# Check if the socket is readable.
+		# @returns [Boolean] True if the socket is readable.
 		def readable?
 			# If we can wait for the socket to become readable, we know that the socket may still be open.
 			result = self.recv_nonblock(1, ::Socket::MSG_PEEK, exception: false)
@@ -36,6 +40,8 @@ require "stringio"
 
 class StringIO
 	unless method_defined?(:readable?, false)
+		# Check if the StringIO is readable.
+		# @returns [Boolean] True if the StringIO is readable (not at EOF).
 		def readable?
 			!eof?
 		end
@@ -46,6 +52,8 @@ require "openssl"
 
 class OpenSSL::SSL::SSLSocket
 	unless method_defined?(:readable?, false)
+		# Check if the SSL socket is readable.
+		# @returns [Boolean] True if the SSL socket is readable.
 		def readable?
 			to_io.readable?
 		end

data/lib/io/stream/string_buffer.rb

@@ -4,15 +4,20 @@
 # Copyright, 2023-2024, by Samuel Williams.
 
 module IO::Stream
+	# A specialized string buffer for binary data with automatic encoding handling.
 	class StringBuffer < String
 		BINARY = Encoding::BINARY
 		
+		# Initialize a new string buffer with binary encoding.
 		def initialize
 			super
 			
 			force_encoding(BINARY)
 		end
 		
+		# Append a string to the buffer, converting to binary encoding if necessary.
+		# @parameter string [String] The string to append.
+		# @returns [StringBuffer] Self for method chaining.
 		def << string
 			if string.encoding == BINARY
 				super(string)

data/lib/io/stream/version.rb

@@ -4,5 +4,5 @@
 # Copyright, 2023-2024, by Samuel Williams.
 
 module IO::Stream
-	VERSION = "0.6.1"
+	VERSION = "0.8.0"
 end

data/lib/io/stream/writable.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2023-2024, by Samuel Williams.
+
+require_relative "readable"
+
+module IO::Stream
+	# The minimum write size before flushing. Defaults to 64KB.
+	MINIMUM_WRITE_SIZE = ENV.fetch("IO_STREAM_MINIMUM_WRITE_SIZE", BLOCK_SIZE).to_i
+	
+	# A module providing writable stream functionality.
+	#
+	# You must implement the `syswrite` method to write data to the underlying IO.
+	module Writable
+		# Initialize writable stream functionality.
+		# @parameter minimum_write_size [Integer] The minimum buffer size before flushing.
+		def initialize(minimum_write_size: MINIMUM_WRITE_SIZE, **, &block)
+			@writing = ::Thread::Mutex.new
+			@write_buffer = StringBuffer.new
+			@minimum_write_size = minimum_write_size
+			
+			super(**, &block) if defined?(super)
+		end
+		
+		attr_accessor :minimum_write_size
+		
+		# Flushes buffered data to the stream.
+		def flush
+			return if @write_buffer.empty?
+			
+			@writing.synchronize do
+				self.drain(@write_buffer)
+			end
+		end
+		
+		# Writes `string` to the buffer. When the buffer is full or #sync is true the
+		# buffer is flushed to the underlying `io`.
+		# @parameter string [String] the string to write to the buffer.
+		# @returns [Integer] the number of bytes appended to the buffer.
+		def write(string, flush: false)
+			@writing.synchronize do
+				@write_buffer << string
+				
+				flush |= (@write_buffer.bytesize >= @minimum_write_size)
+				
+				if flush
+					self.drain(@write_buffer)
+				end
+			end
+			
+			return string.bytesize
+		end
+		
+		# Appends `string` to the buffer and returns self for method chaining.
+		# @parameter string [String] the string to write to the stream.
+		def <<(string)
+			write(string)
+			
+			return self
+		end
+		
+		# Write arguments to the stream followed by a separator and flush immediately.
+		# @parameter arguments [Array] The arguments to write to the stream.
+		# @parameter separator [String] The separator to append after each argument.
+		def puts(*arguments, separator: $/)
+			return if arguments.empty?
+			
+			@writing.synchronize do
+				arguments.each do |argument|
+					@write_buffer << argument << separator
+				end
+				
+				self.drain(@write_buffer)
+			end
+		end
+		
+		# Close the write end of the stream by flushing any remaining data.
+		def close_write
+			flush
+		end
+		
+		private def drain(buffer)
+			begin
+				syswrite(buffer)
+			ensure
+				# If the write operation fails, we still need to clear this buffer, and the data is essentially lost.
+				buffer.clear
+			end
+		end
+	end
+end

data/lib/io/stream.rb

@@ -6,10 +6,15 @@
 require_relative "stream/version"
 require_relative "stream/buffered"
 
+# @namespace
 class IO
+	# @namespace
 	module Stream
 	end
 	
+	# Convert any IO-like object into a buffered stream.
+	# @parameter io [IO] The IO object to wrap.
+	# @returns [IO::Stream::Buffered] A buffered stream wrapper.
 	def self.Stream(io)
 		if io.is_a?(Stream::Buffered)
 			io

data/license.md

@@ -1,6 +1,6 @@
 # MIT License
 
-Copyright, 2023-2024, by Samuel Williams.  
+Copyright, 2023-2025, by Samuel Williams.  
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/readme.md

@@ -8,6 +8,64 @@ Provide a buffered stream implementation for Ruby, independent of the underlying
 
 Please see the [project documentation](https://socketry.github.io/io-stream) for more details.
 
+## Releases
+
+Please see the [project releases](https://socketry.github.io/io-streamreleases/index) for all releases.
+
+### v0.8.0
+
+  - On Ruby v3.3+, use `IO#write` directly instead of `IO#write_nonblock`, for better performance.
+  - Introduce support for `Readable#discard_until` method to discard data until a specific pattern is found.
+
+### v0.7.0
+
+  - Split stream functionality into separate `Readable` and `Writable` modules for better modularity and composition.
+  - Remove unused timeout shim functionality.
+  - 100% documentation coverage.
+
+### v0.6.1
+
+  - Fix compatibility with Ruby v3.3.0 - v3.3.6 where broken `@io.close` could hang.
+
+### v0.6.0
+
+  - Improve compatibility of `gets` implementation to better match Ruby's IO\#gets behavior.
+
+### v0.5.0
+
+  - Add support for `read_until(limit:)` parameter to limit the amount of data read.
+  - Minor documentation improvements.
+
+### v0.4.3
+
+  - Add comprehensive tests for `buffered?` method on `SSLSocket`.
+  - Ensure TLS connections have correct buffering behavior.
+  - Improve test suite organization and readability.
+
+### v0.4.2
+
+  - Add external test suite for better integration testing.
+  - Update dependencies and improve code style with RuboCop.
+
+### v0.4.1
+
+  - Add compatibility fix for `SSLSocket` raising `EBADF` errors.
+  - Fix `IO#close` hang issue in certain scenarios.
+  - Add `#to_io` method to `IO::Stream::Buffered` for better compatibility.
+  - Modernize gem structure and dependencies.
+
+### v0.4.0
+
+  - Add convenient `IO.Stream()` constructor method for creating buffered streams.
+
+### v0.3.0
+
+  - Add support for timeouts with compatibility shims for various IO types.
+
+## See Also
+
+  - [async-io](https://github.com/socketry/async-io) — Where this implementation originally came from.
+
 ## Contributing
 
 We welcome contributions to this project.
@@ -25,7 +83,3 @@ In order to protect users of this project, we require all contributors to comply
 ### Community Guidelines
 
 This project is best served by a collaborative and respectful environment. Treat each other professionally, respect differing viewpoints, and engage constructively. Harassment, discrimination, or harmful behavior is not tolerated. Communicate clearly, listen actively, and support one another. If any issues arise, please inform the project maintainers.
-
-## See Also
-
-  - [async-io](https://github.com/socketry/async-io) — Where this implementation originally came from.