rubcask 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0568bcb47da50a78d9679c88b735cb5b2c259efb00297bb87f749463520cb371'
-  data.tar.gz: 210aaa3e1dd5eb5b0dd26a105f3151717661ff92f24ff54ded3077306b07b838
+  metadata.gz: e54816bf08930641478dc6a0438debb94e82f774b2fcaa857fd0f6786f7adc6b
+  data.tar.gz: be69c187580c84f89572a445296728f5b987a48252c6470cf43853700aa6b1a0
 SHA512:
-  metadata.gz: f52adc24589f583eb1e777c5fe42c1c672e9b2c39e6cd2da5ca75f9be61d7c208a90d146a43adee28978c6217e3557e3b2c89a4290051f10fc2bf92290ab3d6c
-  data.tar.gz: 626f339669b988a5f844b90285e92985122a08d6144d22bd1a311411ca298876656ef375b13e4230da1c49edb00120191d53d046358fef107e13b34ab1634812
+  metadata.gz: 026aaaffc7a9ae96447a21cdd9690fe9bde5b0af59be11e3a04cafea0cce7630916e789baa9fd9f68b6c2082275ca5c096e95441efe35b12df70101a37ae69b8
+  data.tar.gz: 870854934247751b5b0597f09c73f5ae256ea02540b5a93b383071c6c0324f515cb0c56e77eed4d586f6d0ed937fadb0fcb2aa3fc17c0a9b5d189d142637f304

data/.standard.yml

@@ -1,3 +1,3 @@
-ruby_version: 3.2.0
+ruby_version: 2.7.0
 ignore:
   - 'benchmark/**/*'

data/Gemfile

@@ -7,9 +7,9 @@ gemspec
 
 gem "rake", "~> 13.0"
 
-gem "minitest", "~> 5.0"
+gem "minitest", "~> 5.25"
 
-gem "standard", "~> 1.20"
+gem "standard", "~> 1.33"
 
 gem "benchmark-ips", "~> 2.10"
 

data/Gemfile.lock

@@ -3,6 +3,7 @@ PATH
   specs:
     rubcask (0.2.0)
       concurrent-ruby (~> 1.1)
+      stringio (~> 3.1)
 
 GEM
   remote: https://rubygems.org/
@@ -11,50 +12,65 @@ GEM
     benchmark-ips (2.10.0)
     concurrent-ruby (1.2.0)
     docile (1.4.0)
-    json (2.6.3)
-    json (2.6.3-java)
+    json (2.7.1)
+    json (2.7.1-java)
     kalibera (0.1.2)
       memoist (~> 0.16)
       rbzip2 (~> 0.3)
-    language_server-protocol (3.17.0.2)
+    language_server-protocol (3.17.0.3)
+    lint_roller (1.1.0)
     memoist (0.16.2)
-    minitest (5.16.3)
-    parallel (1.22.1)
-    parser (3.1.3.0)
+    minitest (5.25.4)
+    parallel (1.24.0)
+    parser (3.2.2.4)
       ast (~> 2.4.1)
+      racc
+    racc (1.7.3)
+    racc (1.7.3-java)
     rainbow (3.1.1)
     rake (13.0.6)
     rbzip2 (0.3.0)
-    regexp_parser (2.6.1)
-    rexml (3.2.5)
-    rubocop (1.40.0)
+    regexp_parser (2.8.3)
+    rexml (3.2.6)
+    rubocop (1.59.0)
       json (~> 2.3)
+      language_server-protocol (>= 3.17.0)
       parallel (~> 1.10)
-      parser (>= 3.1.2.1)
+      parser (>= 3.2.2.4)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.23.0, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.24.1)
-      parser (>= 3.1.1.0)
-    rubocop-performance (1.15.1)
-      rubocop (>= 1.7.0, < 2.0)
-      rubocop-ast (>= 0.4.0)
-    ruby-progressbar (1.11.0)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.30.0)
+      parser (>= 3.2.1.0)
+    rubocop-performance (1.20.1)
+      rubocop (>= 1.48.1, < 2.0)
+      rubocop-ast (>= 1.30.0, < 2.0)
+    ruby-progressbar (1.13.0)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
-    standard (1.20.0)
+    standard (1.33.0)
       language_server-protocol (~> 3.17.0.2)
-      rubocop (= 1.40.0)
-      rubocop-performance (= 1.15.1)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.59.0)
+      standard-custom (~> 1.0.0)
+      standard-performance (~> 1.3)
+    standard-custom (1.0.2)
+      lint_roller (~> 1.0)
+      rubocop (~> 1.50)
+    standard-performance (1.3.0)
+      lint_roller (~> 1.1)
+      rubocop-performance (~> 1.20.1)
+    stringio (3.1.0)
+    stringio (3.1.0-java)
     timecop (0.9.6)
-    unicode-display_width (2.3.0)
+    unicode-display_width (2.5.0)
 
 PLATFORMS
   ruby
@@ -63,11 +79,11 @@ PLATFORMS
 DEPENDENCIES
   benchmark-ips (~> 2.10)
   kalibera (~> 0.1.2)
-  minitest (~> 5.0)
+  minitest (~> 5.25)
   rake (~> 13.0)
   rubcask!
   simplecov (~> 0.22.0)
-  standard (~> 1.20)
+  standard (~> 1.33)
   timecop (~> 0.9.6)
 
 BUNDLED WITH

data/lib/rubcask/data_entry.rb

@@ -3,7 +3,7 @@
 require_relative "expirable_entry"
 
 module Rubcask
-  DataEntry = Struct.new(:expire_timestamp, :key, :value) do
+  DataEntry = Struct.new(:expire_timestamp, :key, :value, :deleted?) do
     include ExpirableEntry
   end
 end

data/lib/rubcask/data_file.rb

@@ -12,8 +12,14 @@ module Rubcask
 
     attr_reader :write_pos
 
-    HEADER_FORMAT = "NQ>nN"
+    HEADER_SIZE = 4 + 8 + 2 + 4
+
     HEADER_WITHOUT_CRC_FORMAT = "Q>nN"
+    HEADER_FORMAT = "N#{HEADER_WITHOUT_CRC_FORMAT}"
+
+    EXPIRE_MASK = ~(1 << 63)
+    DELETED_MASK = (1 << 63)
+    MAX_EXPIRE_VALUE = DELETED_MASK - 1
 
     # @param [File] file File with the data
     # @param [Integer] file_size Current size of `file` in bytes
@@ -92,14 +98,15 @@ module Rubcask
 
       key_size = entry.key.bytesize
       value_size = entry.value.bytesize
-
+      timestamp_with_deleted = entry.expire_timestamp
+      timestamp_with_deleted |= DELETED_MASK if entry.deleted?
       crc = Zlib.crc32([
-        entry.expire_timestamp,
+        timestamp_with_deleted,
         key_size,
         value_size
       ].pack(HEADER_WITHOUT_CRC_FORMAT) + entry.key + entry.value)
       @write_pos += @file.write(
-        [crc, entry.expire_timestamp, key_size, value_size].pack(HEADER_FORMAT),
+        [crc, timestamp_with_deleted, key_size, value_size].pack(HEADER_FORMAT),
         entry.key,
         entry.value
       )
@@ -110,16 +117,18 @@ module Rubcask
     private
 
     def read_from_io(io)
-      header = io.read(18)
+      header = io.read(HEADER_SIZE)
 
       return nil unless header
 
-      crc, expire_timestamp, key_size, value_size = header.unpack(HEADER_FORMAT)
+      crc, expire_timestamp_with_deleted, key_size, value_size = header.unpack(HEADER_FORMAT)
       key = io.read(key_size)
       value = io.read(value_size)
+      expire_timestamp = (expire_timestamp_with_deleted & EXPIRE_MASK)
+      deleted = (expire_timestamp_with_deleted & DELETED_MASK) != 0
 
       raise ChecksumError, "Checksums do not match" if crc != Zlib.crc32(header[4..] + key + value)
-      DataEntry.new(expire_timestamp, key, value)
+      DataEntry.new(expire_timestamp, key, value, deleted)
     end
   end
 end

data/lib/rubcask/directory.rb

@@ -61,6 +61,7 @@ module Rubcask
     def initialize(dir, config: Config.new)
       @dir = dir
       @config = check_config(config)
+      @active = nil
 
       max_id = 0
       files = dir_data_files
@@ -107,7 +108,6 @@ module Rubcask
     # @param [String] value
     # @param [Integer] ttl Time to live
     # @return [String] the value provided by the user
-    # @return [String] the value provided by the user
     # @raise [ArgumentError] if ttl is negative
     def set_with_ttl(key, value, ttl)
       raise ArgumentError, "Negative ttl" if ttl.negative?
@@ -122,8 +122,6 @@ module Rubcask
     # @return [nil] If no value associated with the key
     def [](key)
       key = normalize_key(key)
-      entry = nil
-      data_file = nil
       @lock.with_read_lock do
         entry = @keydir[key]
         return nil unless entry
@@ -135,9 +133,9 @@ module Rubcask
         data_file = @files[entry.file_id]
 
         # We are using pread so there's no need to synchronize the read
-        value = data_file.pread(entry.value_pos, entry.value_size).value
-        return nil if Tombstone.is_tombstone?(value)
-        return value
+        entry = data_file.pread(entry.value_pos, entry.value_size)
+        return nil if entry.deleted?
+        return entry.value
       end
     end
 
@@ -199,9 +197,9 @@ module Rubcask
       @lock.with_read_lock do
         @keydir.each do |key, entry|
           file = @files[entry.file_id]
-          value = file[entry.value_pos, entry.value_size].value
-          next if Tombstone.is_tombstone?(value)
-          yield [key, value]
+          entry = file[entry.value_pos, entry.value_size]
+          next if entry.deleted?
+          yield [key, entry.value]
         end
       end
     end
@@ -271,10 +269,11 @@ module Rubcask
     attr_reader :config, :active, :worker, :logger
 
     def put(key, value, expire_timestamp)
+      expire_timestamp = expire_timestamp.clamp(NO_EXPIRE_TIMESTAMP, DataFile::MAX_EXPIRE_VALUE)
       key = normalize_key(key)
       @lock.with_write_lock do
         @keydir[key] = active.append(
-          DataEntry.new(expire_timestamp, key, value)
+          DataEntry.new(expire_timestamp, key, value, false)
         )
         if active.write_pos >= @config.max_file_size
           create_new_file!
@@ -286,7 +285,7 @@ module Rubcask
     # @note This method assumes write lock and normalized key
     def do_delete(key, prev_file_id)
       active.append(
-        DataEntry.new(NO_EXPIRE_TIMESTAMP, key, Tombstone.new_tombstone(active.id, prev_file_id))
+        DataEntry.new(NO_EXPIRE_TIMESTAMP, key, Tombstone.new_tombstone(active.id, prev_file_id), true)
       )
       @keydir.delete(key)
       if active.write_pos >= @config.max_file_size
@@ -329,9 +328,10 @@ module Rubcask
         end
       end
 
+      merging_paths.each { |_id, path| FileUtils.chmod("+x", path) }
+
       @lock.with_write_lock do
         close_not_active
-        merging_paths.each { |_id, path| FileUtils.chmod("+x", path) }
         reload!
       end
       clear_files
@@ -345,8 +345,7 @@ module Rubcask
           start_pos = pos
           pos = file.pos
 
-          next if entry.expired?
-          next if Tombstone.is_tombstone?(entry.value)
+          next if entry.expired? || entry.deleted?
 
           @lock.acquire_read_lock
           begin

data/lib/rubcask/marshaled_directory.rb

@@ -0,0 +1,58 @@
+require "forwardable"
+require_relative "directory"
+
+module Rubcask
+  class MarshaledDirectory
+    def initialize(directory)
+      @directory = directory
+    end
+
+    # Set value associated with given key.
+    # @param [Object] key
+    # @param [Object] value
+    # @return [Object] the value provided by the user
+    def []=(key, value)
+      @directory[Marshal.dump(key)] = Marshal.dump(value)
+      value # rubocop:disable Lint/Void
+    end
+
+    # Set value associated with given key with given ttl
+    # @param [Object] key
+    # @param [Object] value
+    # @param [Integer] ttl Time to live
+    # @return [Object] the value provided by the user
+    # @raise [ArgumentError] if ttl is negative
+    def set_with_ttl(key, value, ttl)
+      @directory.set_with_ttl(
+        Marshal.dump(key),
+        Marshal.dump(value),
+        ttl
+      )
+      value
+    end
+
+    # Gets value associated with the key
+    # @param [Object] key
+    # @return [Object] value associatiod with the key
+    # @return [nil] If no value associated with the key
+    def [](key)
+      value = @directory[Marshal.dump(key)]
+      if value.nil?
+        value
+      else
+        Marshal.load(value)
+      end
+    end
+
+    # Remove entry associated with the key.
+    # @param [Object] key
+    # @return false if the existing value does not exist
+    # @return true if the delete was succesfull
+    def delete(key)
+      @directory.delete(Marshal.dump(key))
+    end
+
+    extend Forwardable
+    def_delegators :@directory, *(Directory.public_instance_methods(false) - MarshaledDirectory.public_instance_methods(false))
+  end
+end

data/lib/rubcask/protocol.rb

@@ -59,7 +59,7 @@ module Rubcask
       #   @return [String] Encoded "$1" messege.
       def generate_cached_message(method)
         value = encode_message(const_get(method.upcase)).freeze
-        define_method "#{method}_message" do
+        define_method :"#{method}_message" do
           value
         end
       end

data/lib/rubcask/server/abstract_server.rb

@@ -18,25 +18,29 @@ module Rubcask
 
       private
 
-      def client_loop(conn)
-        loop do
-          length = conn.gets(Protocol::SEPARATOR)
+      def read_command_args(conn)
+        length = conn.gets(SEPARATOR, chomp: true)
 
-          break unless length
-          length = length.to_i
+        return nil unless length
+        length = length.to_i
 
-          command_body = read_command_body(conn, length)
+        command_body = read_command_body(conn, length)
 
-          break unless command_body
-          break if command_body.bytesize != length
+        return nil unless command_body
+        return nil if command_body.bytesize != length
 
-          reader = StringIO.new(command_body)
+        reader = StringIO.new(command_body)
 
-          command = reader.gets(SEPARATOR)
-          command&.chomp!(SEPARATOR)
+        command = reader.gets(SEPARATOR, chomp: true)
 
-          args = parse_args(reader)
+        args = parse_args(reader)
+        [command, args]
+      end
 
+      def client_loop(conn)
+        while running?
+          command_args = read_command_args(conn)
+          break unless command_args
           conn.write(execute_command!(command, args))
         end
       end
@@ -82,23 +86,13 @@ module Rubcask
       end
 
       def parse_word(reader)
-        length = reader.gets(SEPARATOR).to_i
+        length = reader.gets(SEPARATOR, chomp: true).to_i
         return nil if length.zero?
         reader.read(length)
       end
 
       def read_command_body(conn, length)
-        command_body = (+"").b
-        size = 0
-
-        while size < length
-          val = conn.read([MAX_READ_SIZE, length - size].min)
-          return nil if val.nil?
-          size += val.bytesize
-          command_body << val
-        end
-
-        command_body
+        conn.read(length)
       end
 
       def parse_args(reader)

data/lib/rubcask/server/async.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require "async/io"
-require "async/io/trap"
+require "async/queue"
 require "async/io/stream"
 
 require_relative "abstract_server"
@@ -18,6 +18,7 @@ module Rubcask
         @port = config.port
         @logger = Logger.new($stdout)
         @endpoint = ::Async::IO::Endpoint.tcp(@hostname, @port)
+        @running = false
       end
 
       # Shuts down the server
@@ -25,6 +26,7 @@ module Rubcask
       def shutdown
         return unless @task
         Sync do
+          @running = false
           @shutdown_condition.signal
           @task.wait
         end
@@ -33,7 +35,7 @@ module Rubcask
       # Starts the server
       # @param [::Async::Condition, nil] on_start_condition The condition will be signalled after a successful bind
       def start(on_start_condition = nil)
-        Async do
+        Sync do
           @shutdown_condition = ::Async::Condition.new
 
           _, @task = @endpoint.bind do |server, task|
@@ -47,10 +49,13 @@ module Rubcask
 
             server.listen(Socket::SOMAXCONN)
             on_start_condition&.signal
+            @running = true
 
             server.accept_each do |conn|
               conn.binmode
               client_loop(::Async::IO::Stream.new(conn))
+            ensure
+              ::Async::Task.current.stop
             end
           end
         end
@@ -58,6 +63,27 @@ module Rubcask
 
       private
 
+      def running?
+        @running
+      end
+
+      def client_loop(conn)
+        q = ::Async::Queue.new
+        Async do
+          @shutdown_condition.wait
+          q << nil
+        end
+        while running?
+          Async { q << read_command_args(conn) }
+          command_args = q.dequeue
+          return unless command_args
+
+          conn.write(execute_command!(*command_args))
+        end
+      rescue => error
+        Console::Event::Failure.for(error).emit(server, "Error while handling connection")
+      end
+
       def define_close_routine(server, task)
         task.async do |subtask|
           @shutdown_condition.wait
@@ -66,12 +92,18 @@ module Rubcask
 
           server.close
 
+          subtask.with_timeout(30) do
+            task.children.each { |t| t.wait unless t == subtask }
+          rescue ::Async::TimeoutError
+            Console.logger.warn("Could not terminate child connections...")
+          end
+
           task.stop
         end
       end
 
       def read_command_body(conn, length)
-        conn.read(length) # Async does the looping for us
+        conn.read(length)
       end
     end
   end

data/lib/rubcask/server/client.rb

@@ -113,7 +113,7 @@ module Rubcask
       end
 
       def get_response
-        length = @socket.gets(Protocol::SEPARATOR)
+        length = @socket.gets(SEPARATOR, chomp: true)
 
         if length.nil?
           raise InvalidResponseError, "no response"

data/lib/rubcask/server/runner.rb

@@ -2,8 +2,8 @@
 
 require "concurrent/timer_task"
 
-require_relative "./config"
-require_relative "./runner/config"
+require_relative "config"
+require_relative "runner/config"
 require_relative "../config"
 
 module Rubcask

data/lib/rubcask/server/threaded.rb

@@ -65,6 +65,10 @@ module Rubcask
           @status = :shutdown
           cleanup_listeners
           @threads.list.each(&:kill)
+          @threads.list.each do |t|
+            t.join
+          rescue
+          end
           @status = :stopped
           @connected = false
           logger.info "Closed server"
@@ -77,8 +81,12 @@ module Rubcask
         if @status == :running
           @status = :shutdown
         end
-        @shutdown_pipe[1].write_nonblock("\0")
-        @shutdown_pipe[1].close
+        begin
+          @shutdown_pipe[1].write_nonblock("\0")
+          @shutdown_pipe[1].close
+        rescue
+          # We might have race with cleanup shutdown pipe
+        end
       end
 
       # Prepares an IO pipe that is used in shutdown process
@@ -93,6 +101,10 @@ module Rubcask
 
       attr_reader :logger
 
+      def running?
+        @status == :running
+      end
+
       def cleanup_listeners
         @listeners.each do |listener|
           listener.shutdown
@@ -146,6 +158,16 @@ module Rubcask
         end
       end
 
+      def client_loop(conn)
+        while running?
+          command_args = read_command_args(conn)
+          return nil unless command_args
+          Thread.handle_interrupt(Exception => :never) do
+            conn.write(execute_command!(*command_args))
+          end
+        end
+      end
+
       def client_block(conn)
         conn.binmode
         with_interrupt_handle(conn) do |io|

data/lib/rubcask/tombstone.rb

@@ -8,25 +8,16 @@ module Rubcask
   module Tombstone
     extend self
 
-    PREFIX = "TOMBSTONE".b
-    PREFIX_SIZE = PREFIX.bytesize
-    FILE_ID_FORMAT = "N"
-    FULL_BYTE_SIZE = PREFIX_SIZE + 4
-
-    # @param [String] value value to check
-    # @return true if value is a tombstone
-    # @return false otherwise
-    def is_tombstone?(value)
-      value.bytesize <= FULL_BYTE_SIZE && value.start_with?(PREFIX)
-    end
+    FILE_ID_FORMAT = "Q>"
+    BYTE_SIZE = 8
 
     # Creates a new tombstone value
     # @param [Integer] current_file_id Id of the active file
     # @param [Integer] prev_file_id Id of the file where the record is currently located
     # @return [String]
     def new_tombstone(current_file_id, prev_file_id)
-      return PREFIX if prev_file_id == current_file_id
-      PREFIX.b << [prev_file_id].pack(FILE_ID_FORMAT)
+      return "" if prev_file_id == current_file_id
+      [prev_file_id].pack(FILE_ID_FORMAT)
     end
 
     # Gets file id from tombstone value
@@ -34,7 +25,7 @@ module Rubcask
     # @return [Integer, nil]
     def tombstone_file_id(value)
       return nil if value.bytesize < FULL_BYTE_SIZE
-      value.byteslice(PREFIX_SIZE, 4).unpack1(FILE_ID_FORMAT)
+      value.unpack1(FILE_ID_FORMAT)
     end
   end
 end

data/lib/rubcask/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Rubcask
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/rubcask.rb

@@ -2,6 +2,7 @@
 
 require_relative "rubcask/version"
 require_relative "rubcask/directory"
+require_relative "rubcask/marshaled_directory"
 require_relative "rubcask/bytes"
 
 module Rubcask

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: rubcask
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Marcin Henryk Bartkowiak
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-02-20 00:00:00.000000000 Z
+date: 2025-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -24,6 +23,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: stringio
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
 description: Bitcask-like Key/Value storege library with a TCP server included
 email:
 - mhbartkowiak@gmail.com
@@ -59,6 +72,7 @@ files:
 - lib/rubcask/hint_file.rb
 - lib/rubcask/hinted_file.rb
 - lib/rubcask/keydir_entry.rb
+- lib/rubcask/marshaled_directory.rb
 - lib/rubcask/merge_directory.rb
 - lib/rubcask/protocol.rb
 - lib/rubcask/server/abstract_server.rb
@@ -80,7 +94,6 @@ homepage: https://github.com/mhib/rubcask
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -95,8 +108,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
-signing_key:
+rubygems_version: 3.6.2
 specification_version: 4
 summary: Key/Value storage library
 test_files: []