rubcask 0.1.0

data/lib/rubcask/server/runner/config.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Rubcask
+  module Server
+    class Runner
+      # @!attribute merge_interval
+      #   How frequent in seconds should merge operation by run.
+      #
+      #   Default: 3600
+      #   @return [Integer, null]
+      # @!attribute server_type
+      #   Which type of server should be run.
+      #
+      #   Use threaded if you are not on MRI. If you are on mri and can install `async-io` use :async.
+      #
+      #   Default: :threaded
+      #   @return [:threaded, :async]
+      # @!attribute directory_path
+      #   Path of the directory in which the data is stored.
+      #
+      #   Default: no default value, user has to set it manually.
+      #   @return [String]
+      # Server runner config
+      Config = Struct.new(:merge_interval, :server_type, :directory_path) do
+        # Overide fields with the block
+        # @yieldparam [self] config
+        def initialize
+          self.server_type = :threaded
+          self.merge_interval = 3_600
+          self.directory_path = nil
+
+          yield(self) if block_given?
+        end
+
+        # Calls new and freezes the config
+        # @see .initialize
+        def self.configure(&block)
+          new(&block).freeze
+        end
+      end
+    end
+  end
+end

data/lib/rubcask/server/runner.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "concurrent"
+require_relative "./config"
+require_relative "./runner/config"
+require_relative "../config"
+
+module Rubcask
+  module Server
+    # ServerRunner runs a server alongside merge worker
+    # It supports graceful shutdown with Ctrl-c
+    class Runner
+      def initialize(
+        server_config: Rubcask::Server::Config.new,
+        dir_config: Rubcask::Config::DEFAULT_SERVER_CONFIG,
+        runner_config: Rubcask::Server::Runner::Config.new
+      )
+        @dir = Rubcask::Directory.new(
+          runner_config.directory_path,
+          config: dir_config
+        )
+        @server = new_server(runner_config.server_type, server_config)
+        @merge_worker = if runner_config.merge_interval && runner_config.merge_interval > 0
+          Concurrent::TimerTask.new(
+            execution_interval: runner_config.merge_interval
+          ) do
+            merge_dir
+          end
+        end
+      end
+
+      # Starts the runner.
+      # @note It blocks the current thread
+      def start
+        install_trap!
+        @merge_worker.execute
+        @server.start
+      end
+
+      # Stops the server
+      def close
+        close_server
+        mutex_close
+      end
+
+      private
+
+      def close_server
+        puts "Shutting down server!"
+        begin
+          @server.shutdown
+        rescue
+        end
+      end
+
+      def mutex_close
+        if @merge_worker
+          puts "Stoping merge worker"
+          @merge_worker.shutdown
+          if @merge_worker.wait_for_termination(60)
+            puts "Closed merge worker"
+          else
+            puts "Failed to close worker"
+          end
+        end
+
+        puts "Closing Dir!"
+        begin
+          @dir.close
+        rescue
+        end
+        puts "Closed dir"
+      end
+
+      def install_trap!
+        Signal.trap("INT") do
+          puts ""
+          # Close server in the same thread
+          close_server
+
+          # Other things might needs mutex so a new thread is needed
+          Thread.new do
+            mutex_close
+          end.join
+        end
+      end
+
+      def new_server(type, config)
+        if type == :threaded
+          require_relative "threaded"
+          Rubcask::Server::Threaded.new(@dir, config: config)
+        elsif type == :async
+          require_relative "async"
+          Rubcask::Server::Async.new(@dir, config: config)
+        else
+          raise ArgumentError, "Unknown server typ #{type}"
+        end
+      end
+
+      def merge_dir
+        @server.dir.merge
+      rescue => e
+        puts e
+      end
+    end
+  end
+end

data/lib/rubcask/server/threaded.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require "logger"
+require "socket"
+require "io/wait"
+require "stringio"
+
+require_relative "../bytes"
+require_relative "../protocol"
+require_relative "config"
+require_relative "abstract_server"
+
+module Rubcask
+  module Server
+    # Thread-based server supporting Rubcask protocol
+    # If you are running on CRuby you should consider using Server::Async as it is generally more performant
+    class Threaded < AbstractServer
+      include Protocol
+
+      def initialize(dir, config: Server::Config.new)
+        @dir = dir
+        @config = config
+        @hostname = config.hostname
+        @port = config.port
+        @logger = Logger.new($stdout)
+        @logger.level = Logger::INFO
+        @threads = ThreadGroup.new
+        @connected = false
+        @status = :stopped
+        @listeners = []
+      end
+
+      # Creates sockets
+      # @return [self]
+      def connect
+        return if @connected
+        @connected = true
+        @listeners = Socket.tcp_server_sockets(@hostname, @port)
+        if @config.keepalive
+          @listeners.each do |s|
+            s.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
+          end
+        end
+        @listeners.each do |s|
+          address = s.connect_address
+          logger.info "Listening on #{address.ip_address}:#{address.ip_port}"
+        end
+        self
+      end
+
+      # Starts the server
+      # @note It blocks the current thread
+      def start
+        connect
+
+        setup_shutdown_pipe
+
+        @status = :running
+
+        Thread.handle_interrupt(Exception => :never) do
+          Thread.handle_interrupt(Exception => :immediate) do
+            accept_loop
+          end
+        ensure
+          cleanup_shutdown_pipe
+          @status = :shutdown
+          cleanup_listeners
+          @threads.list.each(&:kill)
+          @status = :stopped
+          @connected = false
+          logger.info "Closed server"
+        end
+      end
+
+      # Shuts down the server
+      # @note You probably want to use it in a signal trap
+      def shutdown
+        if @status == :running
+          @status = :shutdown
+        end
+        @shutdown_pipe[1].write_nonblock("\0")
+        @shutdown_pipe[1].close
+      end
+
+      # Prepares an IO pipe that is used in shutdown process
+      # Call if you need to shutdown the server from a different thread
+      # @return [self]
+      def setup_shutdown_pipe
+        @shutdown_pipe ||= IO.pipe
+        self
+      end
+
+      private
+
+      attr_reader :logger
+
+      def cleanup_listeners
+        @listeners.each do |listener|
+          listener.shutdown
+        rescue Errno::ENOTCONN
+          listener.close
+        else
+          listener.close
+        end
+        @listeners.clear
+      end
+
+      def cleanup_shutdown_pipe
+        pipe = @shutdown_pipe
+        pipe&.each(&:close)
+        @shutdown_pipe = nil
+      end
+
+      def accept_loop
+        shutdown_read = @shutdown_pipe[0]
+        while @status == :running
+          begin
+            fds = IO.select([shutdown_read, *@listeners])[0]
+            if fds.include?(shutdown_read)
+              consume_pipe(shutdown_read)
+              break
+            end
+            fds.each do |listener|
+              client = accept_client(listener)
+              next unless client
+              @threads.add(
+                Thread.start(client) { |conn| client_block(conn) }
+              )
+            end
+          rescue Errno::EBADF, Errno::ENOTSOCK, IOError
+            # Possible if socket was manually shut down
+          end
+        end
+      end
+
+      def accept_client(listener)
+        sock = listener.accept_nonblock(exception: false)
+        return nil if sock == :wait_readable
+        sock[0]
+      rescue
+        nil
+      end
+
+      def consume_pipe(pipe)
+        buf = +""
+        while String === pipe.read_nonblock([pipe.nread, 8].max, buf, exception: false)
+        end
+      end
+
+      def client_block(conn)
+        conn.binmode
+        with_interrupt_handle(conn) do |io|
+          client_loop(io)
+        end
+      end
+
+      def with_interrupt_handle(conn)
+        Thread.handle_interrupt(Exception => :never) do
+          Thread.handle_interrupt(Exception => :immediate) do
+            yield conn
+          end
+        ensure
+          begin
+            conn.close
+          rescue # It sometimes failes on jruby
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rubcask/task/clean_directory.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Rubcask
+  module Task
+    # Removes all files marked as executable in the directory
+    class CleanDirectory
+      def initialize(directory)
+        @directory = directory
+      end
+
+      def call
+        Dir.glob(["*.data", "*.hint"].map! { |ext| File.join(@directory, ext) }).each do |file|
+          next unless File.executable?(file)
+          File.delete(file)
+        end
+      end
+    end
+  end
+end

data/lib/rubcask/tombstone.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Rubcask
+  # Tombstone represents deleted value
+
+  # The prev_file_id is
+  # stored to support merge of subset of directory files, that is currently not implemented
+  module Tombstone
+    extend self
+
+    PREFIX = "TOMBSTONE".b
+    PREFIX_SIZE = PREFIX.bytesize
+    FILE_ID_FORMAT = "N"
+    FULL_BYTE_SIZE = PREFIX_SIZE + 4
+
+    # @param [String] value value to check
+    # @return true if value is a tombstone
+    # @return false otherwise
+    def is_tombstone?(value)
+      value.bytesize <= FULL_BYTE_SIZE && value.start_with?(PREFIX)
+    end
+
+    # Creates a new tombstone value
+    # @param [Integer] current_file_id Id of the active file
+    # @param [Integer] prev_file_id Id of the file where the record is currently located
+    # @return [String]
+    def new_tombstone(current_file_id, prev_file_id)
+      return PREFIX if prev_file_id == current_file_id
+      PREFIX.b << [prev_file_id].pack(FILE_ID_FORMAT)
+    end
+
+    # Gets file id from tombstone value
+    # @param [String] value Tombstone value
+    # @return [Integer, nil]
+    def tombstone_file_id(value)
+      return nil if value.bytesize < FULL_BYTE_SIZE
+      value.byteslice(PREFIX_SIZE, 4).unpack1(FILE_ID_FORMAT)
+    end
+  end
+end

data/lib/rubcask/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Rubcask
+  VERSION = "0.1.0"
+end

data/lib/rubcask/worker/direct_worker.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module Rubcask
+  module Worker
+    # Worker implementation that executes the job in the current thread
+    class DirectWorker
+      def initialize
+        @logger = Logger.new($stdout)
+      end
+
+      def push(task)
+        task.call
+      rescue => e
+        @logger.warn("Error while executing task #{e}")
+      end
+
+      def close
+      end
+    end
+  end
+end

data/lib/rubcask/worker/factory.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "direct_worker"
+require_relative "ractor_worker"
+require_relative "thread_worker"
+
+module Rubcask
+  module Worker
+    module Factory
+      extend self
+
+      # Returns a new worker of provided type
+      # @param [:direct, :thread, :reactor] type Type of worker to create
+      # @return [Worker]
+      # @raise [ArgumentError] if unknown type
+      def new_worker(type)
+        case type
+        when :direct
+          DirectWorker.new
+        when :thread
+          ThreadWorker.new
+        when :ractor
+          RactorWorker.new
+        else
+          raise ArgumentError, "#{type} is not a known worker type"
+        end
+      end
+
+      # Class for documentation purposes
+      class Worker
+        # @param [#call] arg job to execute
+        # @return [void]
+        def push(arg)
+        end
+
+        # @return [void]
+        def close
+        end
+      end
+    end
+  end
+end

data/lib/rubcask/worker/ractor_worker.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "logger"
+require "forwardable"
+
+module Rubcask
+  module Worker
+    # Worker implementation that delegates work to a dedicated ractor
+    class RactorWorker
+      extend Forwardable
+
+      def_delegator :@ractor, :send, :push
+
+      def initialize
+        @ractor = new_ractor
+        @logger = Logger.new($stdout)
+      end
+
+      def close
+        push(nil)
+        @ractor.take
+      end
+
+      private
+
+      def new_ractor
+        Ractor.new(@logger) do |logger|
+          while (value = Ractor.receive)
+            begin
+              value.call
+            rescue => e
+              logger.warn("Error while executing task " + e)
+            end
+            Ractor.yield(nil)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rubcask/worker/thread_worker.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Rubcask
+  module Worker
+    # Worker implementation that delegates work to a dedicated thread
+    class ThreadWorker
+      extend Forwardable
+
+      def_delegator :@queue, :push
+
+      def initialize
+        @queue = Queue.new
+        @logger = Logger.new($stdout)
+        @thread = new_thread
+      end
+
+      def close
+        @queue.close
+        @thread.join
+        nil
+      end
+
+      private
+
+      def new_thread
+        Thread.new(@queue) do |queue|
+          while (el = queue.pop)
+            begin
+              el.call
+            rescue => e
+              @logger.warn(e)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rubcask.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative "rubcask/version"
+require_relative "rubcask/directory"
+require_relative "rubcask/bytes"
+
+module Rubcask
+  class Error < StandardError; end
+
+  class LoadError < Error; end
+
+  class ChecksumError < LoadError; end
+
+  class MergeAlreadyInProgressError < Error; end
+
+  class ConfigValidationError < Error; end
+
+  NO_EXPIRE_TIMESTAMP = 0
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: rubcask
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Marcin Henryk Bartkowiak
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-01-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+description: Bitcask-like Key/Value storege library with a TCP server included
+email:
+- mhbartkowiak@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".standard.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- benchmark/benchmark_io.rb
+- benchmark/benchmark_server.rb
+- benchmark/benchmark_server_pipeline.rb
+- benchmark/benchmark_worker.rb
+- benchmark/op_times.rb
+- benchmark/profile.rb
+- benchmark/server_benchmark_helper.rb
+- example/server_runner.rb
+- lib/rubcask.rb
+- lib/rubcask/bytes.rb
+- lib/rubcask/concurrency/fake_atomic_fixnum.rb
+- lib/rubcask/concurrency/fake_lock.rb
+- lib/rubcask/concurrency/fake_monitor_mixin.rb
+- lib/rubcask/config.rb
+- lib/rubcask/data_entry.rb
+- lib/rubcask/data_file.rb
+- lib/rubcask/directory.rb
+- lib/rubcask/expirable_entry.rb
+- lib/rubcask/hint_entry.rb
+- lib/rubcask/hint_file.rb
+- lib/rubcask/hinted_file.rb
+- lib/rubcask/keydir_entry.rb
+- lib/rubcask/merge_directory.rb
+- lib/rubcask/protocol.rb
+- lib/rubcask/server/abstract_server.rb
+- lib/rubcask/server/async.rb
+- lib/rubcask/server/client.rb
+- lib/rubcask/server/config.rb
+- lib/rubcask/server/pipeline.rb
+- lib/rubcask/server/runner.rb
+- lib/rubcask/server/runner/config.rb
+- lib/rubcask/server/threaded.rb
+- lib/rubcask/task/clean_directory.rb
+- lib/rubcask/tombstone.rb
+- lib/rubcask/version.rb
+- lib/rubcask/worker/direct_worker.rb
+- lib/rubcask/worker/factory.rb
+- lib/rubcask/worker/ractor_worker.rb
+- lib/rubcask/worker/thread_worker.rb
+homepage: https://github.com/mhib/rubcask
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.1
+signing_key:
+specification_version: 4
+summary: Key/Value storage library
+test_files: []