rubcask 0.1.0

data/lib/rubcask/hinted_file.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "tempfile"
+require "fileutils"
+
+module Rubcask
+  # HintedFile represents DataFile with the associated hint file
+  # it delegated all read/write responsibility to the @data_file
+  class HintedFile
+    extend Forwardable
+
+    ID_REGEX = /(\d+)\.data$/
+    HINT_EXTENSION_REGEX = /\.data$/
+
+    def_delegators :@data_file, :seek, :[], :close, :flush, :each, :pos, :write_pos
+
+    # @return [String] path of the file
+    attr_reader :path
+
+    # @return [Integer] id of the file
+    attr_reader :id
+
+    # @return [String] Path of the hint file associated with the data file
+    attr_reader :hint_path
+
+    # @param [String] file_path Path of the data_file
+    # @param [Boolean] os_sync Should O_SYNC flag be used on the data file?
+    # @param [Boolean] read_only Should the data file be opened read-only?
+    # @param [Boolean] ruby_sync Should ruby I/O buffers by bupassed?
+    def initialize(file_path, os_sync: false, read_only: false, ruby_sync: false)
+      @id = file_path.scan(ID_REGEX)[0][0].to_i
+      @hint_path = file_path.sub(HINT_EXTENSION_REGEX, ".hint")
+      @path = file_path
+      @read_only = read_only
+
+      io = nil
+      size = nil
+      flags = (os_sync && ruby_sync) ? File::SYNC : 0
+      if File.exist?(file_path)
+        size = File.size(file_path)
+        @dirty = false
+        io = File.open(file_path, "#{read_only ? "r" : "a+"}b", flags: flags)
+      else # If file does not exist we ignore read_only as it does not make sense
+        size = 0
+        @dirty = true
+        io = File.open(file_path, "a+b", flags: flags)
+      end
+      @data_file = DataFile.new(io, size)
+
+      if ruby_sync
+        @data_file.sync = true
+      end
+    end
+
+    # yields every KeydirEntry in the file
+    # @yield [keydir_entry]
+    # @yieldparam [KeydirEntry] keydirEntry
+    # @return [Enumerator] if no block given
+    def each_keydir_entry(&block)
+      return to_enum(__method__) unless block
+      if has_hint_file?
+        return each_hint_file_keydir_entry(&block)
+      end
+      each_data_file_keydir_entry(&block)
+    end
+
+    # Appends the entry to the end of the file
+    # @param [DataEntry] entry entry to append
+    # @return [KeydirEntry]
+    def append(entry)
+      if !dirty?
+        FileUtils.rm_f(hint_path)
+        @dirty = true
+      end
+      write_entry = @data_file.append(entry)
+      KeydirEntry.new(id, write_entry.value_size, write_entry.value_pos, entry.expire_timestamp)
+    end
+
+    # Creates a new hint file
+    def save_hint_file
+      tempfile = Tempfile.new("hint")
+      current_pos = 0
+      map = {}
+      data_file.each do |entry|
+        new_pos = data_file.pos
+        new_entry = HintEntry.new(entry.expire_timestamp, entry.key, current_pos, new_pos - current_pos)
+        current_pos = new_pos
+        map[entry.key] = new_entry
+      end
+
+      begin
+        hint_file = HintFile.new(tempfile)
+        map.each_value do |entry|
+          hint_file.append(entry)
+        end
+        hint_file.close
+        FileUtils.mv(tempfile.path, hint_path)
+        @dirty = false
+      ensure
+        tempfile.close(true)
+      end
+    end
+
+    # @return true if hint path exists
+    def has_hint_file?
+      File.exist?(hint_path)
+    end
+
+    # @return true if there were any appends to the data file
+    def dirty?
+      @dirty
+    end
+
+    private
+
+    attr_reader :data_file
+
+    def each_data_file_keydir_entry
+      current_pos = 0
+      @data_file.each do |entry|
+        new_pos = @data_file.pos
+        value_size = new_pos - current_pos
+        value_pos = current_pos
+        current_pos = new_pos
+        yield [
+          entry.key,
+          KeydirEntry.new(
+            id, value_size, value_pos, entry.expire_timestamp
+          )
+        ]
+      end
+    end
+
+    def each_hint_file_keydir_entry
+      File.open(hint_path, "rb") do |file|
+        HintFile.new(file).each do |entry|
+          yield [
+            entry.key,
+            KeydirEntry.new(
+              id, entry.value_size, entry.value_pos, entry.expire_timestamp
+            )
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/rubcask/keydir_entry.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative "expirable_entry"
+
+module Rubcask
+  KeydirEntry = Struct.new(:file_id, :value_size, :value_pos, :expire_timestamp) do
+    include ExpirableEntry
+  end
+end

data/lib/rubcask/merge_directory.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require_relative "hinted_file"
+
+module Rubcask
+  # A temporary directory that is used during the merge operation.
+  # You probably should not use this class outside of this context.
+  # @see Rubcask::Directory
+  class MergeDirectory
+    def initialize(dir, max_id_ref:, config: Config.new)
+      @dir = dir
+      @config = config
+      @max_id = max_id_ref
+
+      @data_files = []
+
+      create_new_file!
+    end
+
+    def append(entry)
+      value_pos = active.write_pos
+      active.append(entry)
+      value_size = active.write_pos
+      @active_hints[entry.key] = HintEntry.new(entry.expire_timestamp, entry.key, value_pos, value_size)
+
+      if active.write_pos >= config.max_file_size
+        prepare_old_file!
+        create_new_file!
+      end
+    end
+
+    def close
+      if active.write_pos == 0
+        File.delete(active.path)
+      else
+        prepare_old_file!
+      end
+    end
+
+    private
+
+    attr_reader :config
+
+    def prepare_old_file!
+      active.close
+      save_active_hint_file!
+    end
+
+    def save_active_hint_file!
+      File.open(active.hint_path, "ab") do |io|
+        hint_file = HintFile.new(io)
+        @active_hints.each_value do |entry|
+          hint_file.append(entry)
+        end
+      end
+    end
+
+    def active
+      @data_files.last
+    end
+
+    def create_new_file!
+      @active_hints = {}
+
+      id = @max_id.increment
+      file = HintedFile.new(
+        File.join(@dir, "#{id}.data"),
+        os_sync: false,
+        read_only: false,
+        ruby_sync: config.io_strategy != :ruby
+      )
+      @data_files << file
+    end
+  end
+end

data/lib/rubcask/protocol.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Rubcask
+  # Rubcask protocol is inspired by Redis, but is even simpler implementation-wise
+  # The format of response is
+  #   "#{message.byte_size}"\r\n#{message}
+  #   eg: "2\r\nOK"
+  # The format of request it is
+  #   "#{message.byte_size}\r\n#{method}\r\n#{first_argument.byte_size}\r\n#{first_argument}
+  #   eg: "13\r\nget\r\n5\r\nlorem"
+  module Protocol
+    # Success
+    OK = "ok"
+
+    # Repesents no data
+    NIL = "nil"
+
+    # Error message
+    ERROR = "error"
+
+    PING = "ping"
+    PONG = "pong"
+
+    SEPARATOR = "\r\n"
+
+    module_function
+
+    # Returns the provided message with the header of the start
+    # @param [String] message Message to encode
+    # @return [String]
+    def encode_message(message)
+      buffer = (+"").b
+      buffer << message.bytesize.to_s
+      buffer << SEPARATOR
+      buffer << message
+      buffer
+    end
+
+    # @param [String] method Name of the method
+    # @param [Array<String>] args method arguments
+    # @return [String]
+    def create_call_message(method, *args)
+      buffer = (+"").b
+      buffer << method
+      buffer << SEPARATOR
+      args.each do |arg|
+        buffer << encode_message(arg)
+      end
+
+      encode_message(buffer)
+    end
+
+    class << self
+      private
+
+      # @!macro [attach] generate_cached_message
+      #   @method $1_message
+      #   @note This method is autogenerated
+      #   @return [String] Encoded "$1" messege.
+      def generate_cached_message(method)
+        value = encode_message(const_get(method.upcase)).freeze
+        define_method "#{method}_message" do
+          value
+        end
+      end
+    end
+
+    generate_cached_message "ok"
+    generate_cached_message "nil"
+    generate_cached_message "error"
+    generate_cached_message "ping"
+    generate_cached_message "pong"
+  end
+end

data/lib/rubcask/server/abstract_server.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require_relative "../bytes"
+require_relative "../protocol"
+require_relative "config"
+
+module Rubcask
+  module Server
+    class AbstractServer
+      BLOCK_SIZE = Rubcask::Bytes::KILOBYTE * 64
+      MAX_READ_SIZE = BLOCK_SIZE * 128
+
+      include Protocol
+
+      attr_reader :dir
+
+      private
+
+      def client_loop(conn)
+        loop do
+          length = conn.gets(Protocol::SEPARATOR)
+
+          break unless length
+          length = length.to_i
+
+          command_body = read_command_body(conn, length)
+
+          break unless command_body
+          break if command_body.bytesize != length
+
+          reader = StringIO.new(command_body)
+
+          command = reader.gets(SEPARATOR)
+          command&.chomp!(SEPARATOR)
+
+          args = parse_args(reader)
+
+          conn.write(execute_command!(command, args))
+        end
+      end
+
+      def execute_command!(command, args)
+        begin
+          if command == "ping"
+            return pong_message
+          end
+
+          if command == "get"
+            return error_message if args.size != 1
+            val = @dir[args[0]]
+            return val ? encode_message(val) : nil_message
+          end
+
+          if command == "set"
+            return error_message if args.size != 2
+
+            @dir[args[0]] = args[1]
+
+            return ok_message
+          end
+
+          if command == "setex"
+            return error_message if args.size != 3
+            ttl = args[2].to_i
+            return error_message if ttl.negative?
+            @dir.set_with_ttl(args[0], args[1], ttl)
+            return ok_message
+          end
+
+          if command == "del"
+            return error_message if args.size != 1
+
+            return @dir.delete(args[0]) ? ok_message : nil_message
+          end
+        rescue => e
+          logger.warn("Error " + e.to_s)
+        end
+
+        error_message
+      end
+
+      def parse_word(reader)
+        length = reader.gets(SEPARATOR).to_i
+        return nil if length.zero?
+        reader.read(length)
+      end
+
+      def read_command_body(conn, length)
+        command_body = (+"").b
+        size = 0
+
+        while size < length
+          val = conn.read([MAX_READ_SIZE, length - size].min)
+          return nil if val.nil?
+          size += val.bytesize
+          command_body << val
+        end
+
+        command_body
+      end
+
+      def parse_args(reader)
+        args = []
+
+        while (word = parse_word(reader))
+          args << word
+        end
+
+        args
+      end
+    end
+  end
+end

data/lib/rubcask/server/async.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require "async/io"
+require "async/io/trap"
+require "async/io/stream"
+
+require_relative "abstract_server"
+
+module Rubcask
+  module Server
+    # Async-based server supporting Rubcask protocol
+    # It requires "async-io" gem.
+    class Async < AbstractServer
+      def initialize(dir, config: Server::Config.new)
+        @dir = dir
+        @config = config
+        @hostname = config.hostname
+        @port = config.port
+        @logger = Logger.new($stdout)
+        @endpoint = ::Async::IO::Endpoint.tcp(@hostname, @port)
+      end
+
+      # Shuts down the server
+      # @note You might want to use it inside signal trap
+      def shutdown
+        return unless @task
+        Sync do
+          @shutdown_condition.signal
+          @task.wait
+        end
+      end
+
+      # Starts the server
+      # @param [::Async::Condition, nil] on_start_condition The condition will be signalled after a successful bind
+      def start(on_start_condition = nil)
+        Async do
+          @shutdown_condition = ::Async::Condition.new
+
+          _, @task = @endpoint.bind do |server, task|
+            if @config.keepalive
+              server.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
+            end
+
+            define_close_routine(server, task)
+
+            Console.logger.info(server) { "Accepting connections on #{server.local_address.inspect}" }
+
+            server.listen(Socket::SOMAXCONN)
+            on_start_condition&.signal
+
+            server.accept_each do |conn|
+              conn.binmode
+              client_loop(::Async::IO::Stream.new(conn))
+            end
+          end
+        end
+      end
+
+      private
+
+      def define_close_routine(server, task)
+        task.async do |subtask|
+          @shutdown_condition.wait
+
+          Console.logger.info(server) { "Shutting down connections on #{server.local_address.inspect}" }
+
+          server.close
+
+          task.stop
+        end
+      end
+
+      def read_command_body(conn, length)
+        conn.read(length) # Async does the looping for us
+      end
+    end
+  end
+end

data/lib/rubcask/server/client.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require "socket"
+
+require_relative "../protocol"
+require_relative "pipeline"
+
+module Rubcask
+  module Server
+    class Client
+      # @!macro [new] raises_invalid_response
+      #   @raise [InvalidResponseError] If the response is invalid
+
+      class InvalidResponseError < Error; end
+
+      include Protocol
+
+      # yields a new client to the block
+      # closes the client after the block is terminated
+      # @param host [String] hostname of the server
+      # @param port [String] port of the server
+      # @yieldparam [Client] the running client
+      def self.with_client(host, port)
+        client = new(host, port)
+        begin
+          yield client
+        ensure
+          client.close
+        end
+      end
+
+      # @param host [String] hostname of the server
+      # @param port [String] port of the server
+      def initialize(host, port)
+        @socket = TCPSocket.new(host, port)
+      end
+
+      # Get value associated with the key
+      # @param [String] key
+      # @return [String] Binary string representing the value
+      # @return [Protocol::NIL] If no data associated with the key
+      # @macro raises_invalid_response
+      def get(key)
+        call_method("get", key)
+      end
+
+      # Set value associated with the key
+      # @param [String] key
+      # @param [String] value
+      # @return [Protocol::OK] If set succeeded
+      # @return [Protocol::ERROR] If failed to set the value
+      # @macro raises_invalid_response
+      def set(key, value)
+        call_method("set", key, value)
+      end
+
+      # Remove value associated with the key
+      # @param [String] key
+      # @return [Protocol::OK] If delete succeeded
+      # @return [Protocol::NIL] Otherwise
+      # @macro raises_invalid_response
+      def del(key)
+        call_method("del", key)
+      end
+
+      # Ping the server
+      # Use this method to check if server is running and responding
+      # @return [Protocol::PONG]
+      # @macro raises_invalid_response
+      def ping
+        call_method("ping")
+      end
+
+      # Ping the server
+      # Use this method to check if server is running and responding
+      # @param [String] key
+      # @param [String] value
+      # @param [Integer, String] ttl
+      # @return [String] Binary string representing the value
+      # @return [Protocol::NIL] If no data associated with the key
+      # @macro raises_invalid_response
+      def setex(key, value, ttl)
+        call_method("setex", key, value, ttl.to_s)
+      end
+
+      # Run the block in the pipeline
+      # @note pipeline execution IS NOT atomic
+      # @note instance_eval is used so you can call methods directly instead of using block argument
+      # @yield_param [Pipeline] pipeline
+      # @return [Array<String>] List of responses to the executed methods
+      # @macro raises_invalid_response
+      def pipelined(&block)
+        pipeline = Pipeline.new
+        pipeline.instance_eval(&block)
+        call(pipeline.out)
+        pipeline.count.times.map { get_response }
+      end
+
+      # Close the client
+      def close
+        @socket.close
+      end
+
+      private
+
+      def call_method(method, *args)
+        call(create_call_message(method, *args))
+        get_response
+      end
+
+      def call(message)
+        @socket.write(message)
+      end
+
+      def get_response
+        length = @socket.gets(Protocol::SEPARATOR)
+
+        if length.nil?
+          raise InvalidResponseError, "no response"
+        end
+        length = length.to_i
+
+        response = @socket.read(length)
+        if response.bytesize < length
+          raise InvalidResponseError, "response too short"
+        end
+        response
+      end
+    end
+  end
+end

data/lib/rubcask/server/config.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Rubcask
+  module Server
+    # @!attribute hostname
+    #   @return [String] Hostname of the server
+    # @!attribute port
+    #   @return [Integer] Port of the server
+    # @!attribute timeout
+    #   Timeut of the server
+    #
+    #   If the client does not send any messages for provided number of seconds the connection with it s closed
+    #   @return [Integer]
+    # @!attribute keepalive
+    #    @return [boolean] Flag whether to set TCP's keepalive
+    Config = Struct.new(:hostname, :port, :timeout, :keepalive) do
+      def initialize
+        self.hostname = "localhost"
+        self.timeout = nil
+        self.keepalive = true
+        self.port = 8080
+
+        yield(self) if block_given?
+      end
+
+      def self.configure(&block)
+        new(&block).freeze
+      end
+    end
+  end
+end

data/lib/rubcask/server/pipeline.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Rubcask
+  module Server
+    # @!macro [new] see_client
+    #   @see Client#$0
+
+    # Pipeline represents a sequence of commands.
+    # @note Pipeline execution IS NOT atomic.
+    # @see Client
+    class Pipeline
+      include Protocol
+
+      attr_reader :out, :count
+
+      def initialize
+        @out = (+"").b
+        @count = 0
+      end
+
+      # @macro see_client
+      def get(key)
+        @out << create_call_message("get", key)
+      end
+
+      # @macro see_client
+      def set(key, value)
+        @out << create_call_message("set", key, value)
+      end
+
+      # @macro see_client
+      def del(key)
+        @out << create_call_message("del", key)
+      end
+
+      # @macro see_client
+      def ping
+        @out << create_call_message("ping")
+      end
+
+      private
+
+      def create_call_message(method, *args)
+        @count += 1
+        super
+      end
+    end
+  end
+end