rubcask 0.1.0

data/lib/rubcask/concurrency/fake_monitor_mixin.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Rubcask
+  module Concurrency
+    # Fake of MonitorMixin module that implements
+    # a subset of methods
+    # It does not do any synchronization
+    module FakeMonitorMixin
+      def mon_synchronize
+        yield
+      end
+      alias_method :synchronize, :mon_synchronize
+
+      def mon_enter
+      end
+
+      def mon_exit
+      end
+    end
+  end
+end

data/lib/rubcask/config.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative "bytes"
+module Rubcask
+  # @!attribute max_file_size
+  #   Maximum single file size.
+  #
+  #   New file is created after the current file is larger than this field
+  #
+  #   Default: Bytes::GIGABYTE * 2
+  #   @return [Integer]
+  # @!attribute io_strategy
+  #   Guarantees; listed fastest first
+  #
+  #   :ruby is safe as long as you exit gracefully
+  #
+  #   :os is safe as long as no os or hardware failures occures
+  #
+  #   :os_sync is always safe
+  #
+  #   Default :ruby
+  #   @return [:ruby, :os, :os_sync]
+  # @!attribute threadsafe
+  #   Set to true if you want to use Rubcask with many threads concurrently
+  #
+  #   Default: true
+  #   @return [boolean]
+  # @!attribute worker
+  #   Type of worker used for async jobs
+  #
+  #   Currently it is only used for deleting files after merge
+  #
+  #   Default: :direct
+  #   @return [:direct, :ractor, :thread]
+  # Server runner config
+  Config = Struct.new(:max_file_size, :io_strategy, :threadsafe, :worker) do
+    # @yieldparam [self] config
+    def initialize
+      self.max_file_size = Bytes::GIGABYTE * 2
+      self.io_strategy = :ruby
+      self.threadsafe = true
+      self.worker = :direct
+
+      yield(self) if block_given?
+    end
+
+    def self.configure(&block)
+      new(&block).freeze
+    end
+  end
+
+  class Config
+    DEFAULT_SERVER_CONFIG = configure { |c| c.io_strategy = :os }
+  end
+end

data/lib/rubcask/data_entry.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative "expirable_entry"
+
+module Rubcask
+  DataEntry = Struct.new(:expire_timestamp, :key, :value) do
+    include ExpirableEntry
+  end
+end

data/lib/rubcask/data_file.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "stringio"
+require "zlib"
+
+module Rubcask
+  # DataFile is a file where the key and values are actually stored
+  class DataFile
+    extend Forwardable
+    def_delegators :@file, :seek, :pos, :close, :sync=, :flush
+
+    attr_reader :write_pos
+
+    HEADER_FORMAT = "NQ>nN"
+    HEADER_WITHOUT_CRC_FORMAT = "Q>nN"
+
+    # @param [File] file File with the data
+    # @param [Integer] file_size Current size of `file` in bytes
+    def initialize(file, file_size)
+      @file = file
+      @write_pos = file_size
+    end
+
+    # Fetch entry at given offset.
+    # Optional size parameter is size of the record. With it we make one less I/O
+    # @param [Integer] offset File offset in bytes
+    # @param [Integer, nil] size Record size in bytes
+    def [](offset, size = nil)
+      seek(offset)
+      read(size)
+    end
+
+    # yields each record in the file
+    # @return [Enumerator] if no block given
+    # @yieldparam [DataEntry]
+    def each
+      return to_enum(__method__) unless block_given?
+
+      seek(0)
+
+      loop do
+        val = read
+        break unless val
+        yield val
+      end
+    end
+
+    # Read entry at the current file position
+    # @return [DataEntry]
+    # @return [nil] if at the end of file
+    # @raise [ChecksumError] if the entry has an incorrect checksum
+    def read(size = nil)
+      io = size ? StringIO.new(@file.read(size)) : @file
+      header = io.read(18)
+
+      return nil unless header
+
+      crc, expire_timestamp, key_size, value_size = header.unpack(HEADER_FORMAT)
+      key = io.read(key_size)
+      value = io.read(value_size)
+
+      raise ChecksumError, "Checksums do not match" if crc != Zlib.crc32(header[4..] + key + value)
+      DataEntry.new(expire_timestamp, key, value)
+    end
+
+    AppendResult = Struct.new(:value_pos, :value_size)
+    # Append a record at the end of the file
+    # @param [DataEntry] entry Entry to write to the file
+    # @return [AppendResult] struct containing position and size of the record
+    def append(entry)
+      current_pos = @write_pos
+
+      key_size = entry.key.bytesize
+      value_size = entry.value.bytesize
+
+      crc = Zlib.crc32([
+        entry.expire_timestamp,
+        key_size,
+        value_size
+      ].pack(HEADER_WITHOUT_CRC_FORMAT) + entry.key + entry.value)
+      @write_pos += @file.write(
+        [crc, entry.expire_timestamp, key_size, value_size].pack(HEADER_FORMAT),
+        entry.key,
+        entry.value
+      )
+      @file.flush
+      AppendResult.new(current_pos, @write_pos - current_pos)
+    end
+  end
+end

data/lib/rubcask/directory.rb

@@ -0,0 +1,437 @@
+# frozen_string_literal: true
+
+require "concurrent"
+
+require "forwardable"
+require "logger"
+require "monitor"
+require "tmpdir"
+
+require_relative "concurrency/fake_lock"
+require_relative "concurrency/fake_atomic_fixnum"
+require_relative "concurrency/fake_monitor_mixin"
+
+require_relative "task/clean_directory"
+
+require_relative "worker/factory"
+
+require_relative "config"
+require_relative "data_entry"
+require_relative "data_file"
+require_relative "hint_entry"
+require_relative "hint_file"
+require_relative "hinted_file"
+require_relative "keydir_entry"
+require_relative "merge_directory"
+require_relative "tombstone"
+
+module Rubcask
+  class Directory
+    extend Forwardable
+
+    # yields directory to the block and closes it after the block is terminated
+    # @see #initialize
+    # @yieldparam [Directory] directory
+    # @return [void]
+    def self.with_directory(dir, config: Config.new)
+      directory = new(dir, config: config)
+      begin
+        yield directory
+      ensure
+        directory.close
+      end
+    end
+
+    # @!macro [new] key_is_bytearray
+    #   @note key is always treated as byte array, encoding is ignored
+
+    # @!macro [new] deleted_keys
+    #   @note It might include deleted keys
+
+    # @!macro [new] lock_block_for_iteration
+    #   @note This method blocks writes for the entire iteration
+
+    # @!macro [new] key_any_order
+    #   @note Keys might be in any order
+
+    # @param [String] dir Path to the directory where data is stored
+    # @param [Config] config Config of the directory
+    def initialize(dir, config: Config.new)
+      @dir = dir
+      @config = check_config(config)
+
+      max_id = 0
+      files = dir_data_files
+      @files = files.each_with_object({}) do |file, hash|
+        next if File.executable?(file)
+        if file.equal?(files.last) && File.size(file) < config.max_file_size
+          hinted_file = open_write_file(file)
+          @active = hinted_file
+        else
+          hinted_file = open_read_only_file(file)
+        end
+
+        id = hinted_file.id
+
+        hash[id] = hinted_file
+        max_id = id # dir_data_files returns an already sorted collection
+      end
+      @max_id = (config.threadsafe ? Concurrent::AtomicFixnum : Concurrency::FakeAtomicFixnum).new(max_id)
+      @lock = config.threadsafe ? Concurrent::ReentrantReadWriteLock.new : Concurrency::FakeLock.new
+      @worker = Worker::Factory.new_worker(@config.worker)
+
+      @logger = Logger.new($stdin)
+      @logger.level = Logger::INFO
+
+      @merge_mutex = Thread::Mutex.new
+
+      load_keydir!
+      create_new_file! unless @active
+    end
+
+    # Set value associated with given key.
+    # @macro key_is_bytearray
+    # @param [String] key
+    # @param [String] value
+    # @return [String] the value provided by the user
+    def []=(key, value)
+      put(key, value, NO_EXPIRE_TIMESTAMP)
+      value # rubocop:disable Lint/Void
+    end
+
+    # Set value associated with given key with given ttl
+    # @macro key_is_bytearray
+    # @param [String] key
+    # @param [String] value
+    # @param [Integer] ttl Time to live
+    # @return [String] the value provided by the user
+    # @return [String] the value provided by the user
+    # @raise [ArgumentError] if ttl is negative
+    def set_with_ttl(key, value, ttl)
+      raise ArgumentError, "Negative ttl" if ttl.negative?
+      put(key, value, Time.now.to_i + ttl)
+      value # rubocop:disable Lint/Void
+    end
+
+    # Gets value associated with the key
+    # @macro key_is_bytearray
+    # @param [String] key
+    # @return [String] value associatiod with the key
+    # @return [nil] If no value associated with the key
+    def [](key)
+      key = normalize_key(key)
+      entry = nil
+      data_file = nil
+      @lock.with_read_lock do
+        entry = @keydir[key]
+        return nil unless entry
+
+        if entry.expired?
+          return nil
+        end
+
+        data_file = @files[entry.file_id]
+        data_file.synchronize do
+          value = data_file[entry.value_pos, entry.value_size].value
+          return nil if Tombstone.is_tombstone?(value)
+          return value
+        end
+      end
+    end
+
+    # Remove entry associated with the key.
+    # @param [String] key
+    # @macro key_is_bytearray
+    # @return false if the existing value does not exist
+    # @return true if the delete was succesfull
+    def delete(key)
+      key = normalize_key(key)
+      @lock.with_write_lock do
+        prev_val = @keydir[key]
+        if prev_val.nil?
+          return false
+        end
+        if prev_val.expired?
+          @keydir.delete(key)
+          return false
+        end
+        do_delete(key, prev_val.file_id)
+        true
+      end
+    end
+
+    # Starts the merge operation.
+    # @raise [MergeAlreadyInProgress] if another merge operation is in progress
+    def merge
+      unless @merge_mutex.try_lock
+        raise MergeAlreadyInProgressError, "Merge is already in progress"
+      end
+      begin
+        non_synced_merge
+      rescue => ex
+        logger.error("Error while merging #{ex}")
+      ensure
+        @merge_mutex.unlock
+      end
+    end
+
+    # Closes all the files and the worker
+    def close
+      @lock.with_write_lock do
+        @files.each_value(&:close)
+        if active.write_pos == 0
+          File.delete(active.path)
+        end
+      end
+      worker.close
+    end
+
+    # @yieldparam [String] key
+    # @yieldparam [String] value
+    # @macro lock_block_for_iteration
+    # @macro key_any_order
+    # @return Enumerator if block not given
+    def each
+      return to_enum(__method__) unless block_given?
+
+      @lock.with_read_lock do
+        @keydir.each do |key, entry|
+          file = @files[entry.file_id]
+          file.mon_enter
+          begin
+            value = file[entry.value_pos, entry.value_size].value
+            next if Tombstone.is_tombstone?(value)
+            yield [key, value]
+          ensure
+            file.mon_exit
+          end
+        end
+      end
+    end
+
+    # @yieldparam [String] key
+    # @macro deleted_keys
+    # @macro key_any_order
+    # @macro lock_block_for_iteration
+    # @return Enumerator if block not given
+    def each_key(&block)
+      return to_enum(__method__) unless block
+
+      @lock.with_read_lock do
+        @keydir.each_key(&block)
+      end
+    end
+
+    # Generate hint files for data files that do not have hint files
+    def generate_missing_hint_files!
+      @lock.with_read_lock do
+        @files.each_value do |data_file|
+          next if data_file.has_hint_file? && !data_file.dirty?
+          data_file.synchronize do
+            data_file.save_hint_file
+          end
+        end
+      end
+    end
+
+    # Generate hint files for all the data files
+    def regenerate_hint_files!
+      @lock.with_read_lock do
+        @files.each_value do |data_file|
+          data_file.synchronize do
+            data_file.save_hint_file
+          end
+        end
+      end
+    end
+
+    # Removes files that are not needed after the merge
+    def clear_files
+      worker.push(Rubcask::Task::CleanDirectory.new(@dir))
+    end
+
+    # Returns number of keys in the store
+    # @note It might count some deleted keys
+    # @return [Integer]
+    def key_count
+      @lock.with_read_lock do
+        @keydir.size
+      end
+    end
+
+    # Returns array of keys in store
+    # @macro deleted_keys
+    # @macro key_any_order
+    # @return [Array<String>]
+    def keys
+      @lock.with_read_lock do
+        @keydir.keys
+      end
+    end
+
+    private
+
+    attr_reader :config, :active, :worker, :logger
+
+    def put(key, value, expire_timestamp)
+      key = normalize_key(key)
+      @lock.with_write_lock do
+        @keydir[key] = active.append(
+          DataEntry.new(expire_timestamp, key, value)
+        )
+        if active.write_pos >= @config.max_file_size
+          create_new_file!
+        end
+      end
+      value # rubocop:disable Lint/Void
+    end
+
+    # @note This method assumes write lock and normalized key
+    def do_delete(key, prev_file_id)
+      active.append(
+        DataEntry.new(NO_EXPIRE_TIMESTAMP, key, Tombstone.new_tombstone(active.id, prev_file_id))
+      )
+      @keydir.delete(key)
+      if active.write_pos >= @config.max_file_size
+        create_new_file!
+      end
+    end
+
+    # This methods does not provide synchronization and should be run with write lock
+    def close_not_active
+      @files.each_value do |file|
+        next if file == active
+        file.close
+      end
+    end
+
+    def synchronize_hinted_file!(file)
+      file.extend(
+        @config.threadsafe ? MonitorMixin : Concurrency::FakeMonitorMixin
+      )
+    end
+
+    def non_synced_merge
+      merging_paths = nil
+      @lock.with_write_lock do
+        merging_paths = @files.sort_by(&:first).map! { |k, v| [k, v.path] }
+        create_new_file!
+      end
+
+      Dir.mktmpdir do |tmpdir|
+        out = MergeDirectory.new(tmpdir, config: @config, max_id_ref: @max_id)
+
+        merging_paths.each do |id, path|
+          merge_single_file(out, id, path)
+        end
+
+        out.close
+
+        Dir.each_child(tmpdir) do |child|
+          FileUtils.mv(File.join(tmpdir, child), @dir)
+        end
+      end
+
+      @lock.with_write_lock do
+        close_not_active
+        merging_paths.each { |_id, path| FileUtils.chmod("+x", path) }
+        reload!
+      end
+      clear_files
+    end
+
+    def merge_single_file(out, id, path)
+      File.open(path, "rb") do |io|
+        pos = 0
+        file = DataFile.new(io, 0)
+        file.each do |entry|
+          start_pos = pos
+          pos = file.pos
+
+          next if entry.expired?
+          next if Tombstone.is_tombstone?(entry.value)
+
+          @lock.acquire_read_lock
+          begin
+            keydir_entry = @keydir[entry.key]
+            next unless keydir_entry
+
+            # Ignore records overwritten in a new file
+            next if keydir_entry.file_id > id
+            # Ignore records overwritten in the data file
+            next if keydir_entry.file_id == id && keydir_entry.value_pos > start_pos
+          ensure
+            @lock.release_read_lock
+          end
+          out.append(entry)
+        end
+      end
+    end
+
+    def reload!
+      @files = dir_data_files.each_with_object({}) do |file, hash|
+        next if File.executable?(file) || file == @active.path
+        hinted_file = open_read_only_file(file)
+
+        id = hinted_file.id
+
+        hash[id] = hinted_file
+      end
+      @files[@active.id] = @active
+      load_keydir!
+    end
+
+    def load_keydir!
+      @keydir = {}
+
+      # Note that file iteration is oldest to newest
+      @files.each_value do |file|
+        file.each_keydir_entry do |key, entry|
+          if entry.expired?
+            @keydir.delete(key)
+          else
+            @keydir[key] = entry
+          end
+        end
+      end
+    end
+
+    def open_write_file(file)
+      HintedFile.new(
+        file,
+        os_sync: config.io_strategy == :os_sync,
+        read_only: false,
+        ruby_sync: config.io_strategy != :ruby
+      ).tap { |f| synchronize_hinted_file!(f) }
+    end
+
+    def open_read_only_file(file)
+      HintedFile.new(
+        file,
+        read_only: true
+      ).tap { |f| synchronize_hinted_file!(f) }
+    end
+
+    def create_new_file!
+      id = @max_id.increment
+      file = open_write_file(File.join(@dir, "#{id}.data"))
+      @active = file
+      @files[id] = file
+    end
+
+    def dir_data_files
+      Dir.glob(File.join(@dir, "*.data")).sort_by! { |el| File.basename(el).to_i }
+    end
+
+    def check_config(config)
+      config
+    end
+
+    def normalize_key(key)
+      key = key.to_s
+      return key if key.encoding.equal?(Encoding::ASCII_8BIT)
+
+      key.b
+    end
+  end
+end

data/lib/rubcask/expirable_entry.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Rubcask
+  module ExpirableEntry
+    def expired?
+      expire_timestamp != Rubcask::NO_EXPIRE_TIMESTAMP && expire_timestamp < Time.now.to_i
+    end
+  end
+end

data/lib/rubcask/hint_entry.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative "expirable_entry"
+
+module Rubcask
+  HintEntry = Struct.new(:expire_timestamp, :key, :value_pos, :value_size) do
+    include ExpirableEntry
+  end
+end

data/lib/rubcask/hint_file.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Rubcask
+  # HintFile stores only keys, and information on where the value of the key is located
+  class HintFile
+    extend Forwardable
+    def_delegators :@file, :seek, :close, :flush
+
+    HEADER_FORMAT = "Q>nNQ>"
+
+    # @param [File] file An already opened file
+    def initialize(file)
+      @file = file
+    end
+
+    # Yields each hint entry from the file
+    # @yieldparam [HintEntry] hint_entry
+    # @return [Enumerator] if no block given
+    def each
+      return to_enum(__method__) unless block_given?
+
+      seek(0)
+
+      loop do
+        val = read
+        break unless val
+        yield val
+      end
+    end
+
+    # Reads hint entry at the current offset
+    # @return [HintEntry]
+    # @return [nil] If at the end of file
+    # @raise LoadError if unable to read from the file
+    def read
+      header = @file.read(22)
+
+      return nil unless header
+
+      expire_timestamp, key_size, value_size, value_pos = header.unpack(HEADER_FORMAT)
+      key = @file.read(key_size)
+
+      HintEntry.new(expire_timestamp, key, value_pos, value_size)
+    end
+
+    # Appends an entry to the file
+    # @param [HintEntry] entry
+    # @return [Integer] Number of bytes written
+    def append(entry)
+      @file.write(
+        [entry.expire_timestamp, entry.key.bytesize, entry.value_size, entry.value_pos].pack(HEADER_FORMAT),
+        entry.key
+      )
+    end
+  end
+end