rubcask 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c7d54e3884c99955ecbab6624997f84c8469d1f60ad73a34631e206efffebbeb
+  data.tar.gz: d2c62d450a8ebd9abfc5e12ac26c2747365afbe98ff72fe8117fc8659c02b1d0
+SHA512:
+  metadata.gz: 6be6603b4238edd7a93548e6440c4dcee3263afb254b40999b774e7b8e60dc39d021457bd88a4f3cc0017a2812685c48d574aacbc1f35a0ae0eda35c46fad468
+  data.tar.gz: 57e451dd32e7e555e6692bbaf441370a90df623cffef4aa52169db0869ad1b99650372e20abed61ca0eefe3f1247ca4e8ae39f06d3a142d2b932713387c787dc

data/.standard.yml

@@ -0,0 +1,3 @@
+ruby_version: 3.2.0
+ignore:
+  - 'benchmark/**/*'

data/Gemfile

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in rubcask.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "minitest", "~> 5.0"
+
+gem "standard", "~> 1.20"
+
+gem "benchmark-ips", "~> 2.10"
+
+gem "kalibera", "~> 0.1.2"
+
+gem "timecop", "~> 0.9.6"
+
+gem "simplecov", "~> 0.22.0"

data/Gemfile.lock

@@ -0,0 +1,74 @@
+PATH
+  remote: .
+  specs:
+    rubcask (0.1.0)
+      concurrent-ruby (~> 1.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+    benchmark-ips (2.10.0)
+    concurrent-ruby (1.1.10)
+    docile (1.4.0)
+    json (2.6.3)
+    json (2.6.3-java)
+    kalibera (0.1.2)
+      memoist (~> 0.16)
+      rbzip2 (~> 0.3)
+    language_server-protocol (3.17.0.2)
+    memoist (0.16.2)
+    minitest (5.16.3)
+    parallel (1.22.1)
+    parser (3.1.3.0)
+      ast (~> 2.4.1)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    rbzip2 (0.3.0)
+    regexp_parser (2.6.1)
+    rexml (3.2.5)
+    rubocop (1.40.0)
+      json (~> 2.3)
+      parallel (~> 1.10)
+      parser (>= 3.1.2.1)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.23.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.24.1)
+      parser (>= 3.1.1.0)
+    rubocop-performance (1.15.1)
+      rubocop (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    ruby-progressbar (1.11.0)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.4)
+    standard (1.20.0)
+      language_server-protocol (~> 3.17.0.2)
+      rubocop (= 1.40.0)
+      rubocop-performance (= 1.15.1)
+    timecop (0.9.6)
+    unicode-display_width (2.3.0)
+
+PLATFORMS
+  ruby
+  universal-java-17
+
+DEPENDENCIES
+  benchmark-ips (~> 2.10)
+  kalibera (~> 0.1.2)
+  minitest (~> 5.0)
+  rake (~> 13.0)
+  rubcask!
+  simplecov (~> 0.22.0)
+  standard (~> 1.20)
+  timecop (~> 0.9.6)
+
+BUNDLED WITH
+   2.3.6

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 Marcin Henryk Bartkowiak
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,111 @@
+# Rubcask
+Rubcask is a Bitcask-like log-structured Key/Value storage library.
+
+It ships with a TCP server and client implementing a custom protocol.
+
+It has design very similar to bitcask including merge operation, moving to a next file after reaching a configurable limit; timestamp however is used for expiration only.
+
+## Documentation
+https://rubydoc.info/github/mhib/rubcask/master
+
+## Disclaimer
+This library is not production-ready.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rubcask'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install rubcask
+
+## Usage
+Rubcask's main methods are very similar to ruby's hash.
+
+```ruby
+dir = Rubcask::Directory.new("path_to_directory")
+dir["key"] = "value"
+dir["key"] # => "value"
+dir.delete("key") # => true
+dir.close
+```
+
+You can also set value with a ttl.
+
+```ruby
+dir = Rubcask::Directory.new("path_to_directory")
+dir.set_with_ttl("key", "value", 10)
+dir["key"] # => "value"
+sleep(11)
+dir["key"] # => nil
+dir.close
+```
+
+Rubcask does not store encoding information and stores keys as bytes, so using utf-8 strings is the same as using ASCII strings.
+The same goes with values.
+
+```ruby
+dir = Rubcask::Directory.new("path_to_directory")
+dir["jeż"] = "3"
+dir["jeż".b] # => 3
+dir.close
+```
+
+Rubcask can be used both as a library and as a server.
+
+See `examples/server_runner.rb` for example configuration with a TCP server and a merge worker.
+
+See `lib/rubcask/server/client.rb` for server client
+
+## Thread safety
+By default `Rubcask::Directory` is thread safe.
+
+Consider installing `concurrent-ruby-ext` for some performance gains.
+
+If you do not want to pay performance penalty for synchronization, it is possible to disable thread synchronization in config.
+
+```ruby
+config = Rubcask::Config.configure { |c| c.threadsafe = false }
+dir = Rubcask::Directory.new("path_to_directory", config: config)
+```
+
+It is only safe to do that when using Rubcask as a library; server implementations assumes that Directory is run with `threadsafe = true`.
+
+## Server implementations
+Projects is shipped with threaded server that does not require any dependencies, and with async server that requires `async-io`.
+
+They implement the same custom protocol.
+
+Generally async server is faster especially for pipelines as it buffers reads.
+
+Note that async server might not work on JRuby.
+
+## Supported Ruby implementations
+Tested against supported versions of CRuby and JRuby. TruffleRuby currently does not work due to some `IO` incompatibilities.
+
+## Todo
+* A script in `bin/` for running server runner easily.
+* (maybe) Nice drb support
+* (maybe) Using trie instead of key-dir with ordered iteration support
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/mhib/rubcask.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/test_*.rb"]
+end
+
+require "standard/rake"
+
+task default: %i[test standard]

data/benchmark/benchmark_io.rb

@@ -0,0 +1,49 @@
+require "benchmark/ips"
+require "securerandom"
+
+require_relative "../lib/rubcask"
+
+@values = Array.new(10_000) { SecureRandom.hex(128) }
+
+Dir.mktmpdir do |path|
+  # This does not reflect raw performance well as there is dir creation overhead but works good enough for comparison
+  Benchmark.ips do |x|
+    x.warmup = 15
+    x.time = 30
+
+    x.stats = :bootstrap
+    x.confidence = 95
+
+    x.report("put_get_os") do
+      Dir.mktmpdir do |path|
+        dir = Rubcask::Directory.new(path, config: Rubcask::Config.configure { |x| x.io_strategy = :os })
+
+        (0...10_000).each do |idx|
+          dir[idx.to_s] = @values[idx]
+        end
+      end
+    end
+
+    x.report("put_get_ruby") do
+      Dir.mktmpdir do |path|
+        dir = Rubcask::Directory.new(path, config: Rubcask::Config.new)
+
+        (0...10_000).each do |idx|
+          dir[idx.to_s] = @values[idx]
+        end
+      end
+    end
+
+    x.report("put_get_os_sync") do
+      Dir.mktmpdir do |path|
+        dir = Rubcask::Directory.new(path, config: Rubcask::Config.configure { |x| x.io_strategy = :os_sync })
+
+        (0...10_000).each do |idx|
+          dir[idx.to_s] = @values[idx]
+        end
+      end
+    end
+
+    x.compare!
+  end
+end

data/benchmark/benchmark_server.rb

@@ -0,0 +1,10 @@
+require_relative "server_benchmark_helper"
+include ServerBenchmarkHelper
+
+run_benchmark_with_threaded_server("Threaded") do |client|
+  client.get("key")
+end
+
+run_benchmark_with_async_server("Async") do |client|
+  client.get("key")
+end

data/benchmark/benchmark_server_pipeline.rb

@@ -0,0 +1,24 @@
+require_relative "./server_benchmark_helper"
+
+include ServerBenchmarkHelper
+
+pipeline_block = proc do |client|
+  client.pipelined do |pipe|
+    pipe.get("key")
+    pipe.get("unknown_key")
+    pipe.ping
+    pipe.get("key")
+  end
+end
+
+without_pipeline_block = proc do |client|
+  client.get("key")
+  client.get("unknown_key")
+  client.ping
+  client.get("key")
+end
+
+run_benchmark_with_async_server("Asyc pipelined", &pipeline_block)
+run_benchmark_with_async_server("Async without pipeline", &without_pipeline_block)
+run_benchmark_with_threaded_server("Threaded pipelined", &pipeline_block)
+run_benchmark_with_threaded_server("Threaded without pipeline", &without_pipeline_block)

data/benchmark/benchmark_worker.rb

@@ -0,0 +1,46 @@
+require "benchmark/ips"
+
+require "fileutils"
+require "securerandom"
+require "tmpdir"
+
+require_relative "../lib/rubcask/worker/factory"
+require_relative "../lib/rubcask/task/clean_directory"
+
+class SimulateIOTask
+  def call
+    10_000.times { IO.read("/dev/null") }
+    sleep(0.01)
+  end
+end
+
+@benchmark_procedure = lambda do |type, times|
+  times.times do
+    worker = Rubcask::Worker::Factory.new_worker(type)
+    10.times { worker.push(SimulateIOTask.new) }
+    100_000.times { IO.read("/dev/null") } # Simulate some IO
+    worker.close
+  end
+end
+
+Benchmark.ips do |x|
+  x.warmup = 15
+  x.time = 30
+
+  x.stats = :bootstrap
+  x.confidence = 95
+
+  x.report("thread") do |times|
+    @benchmark_procedure[:thread, times]
+  end
+
+  x.report("ractor") do |times|
+    @benchmark_procedure[:ractor, times]
+  end
+
+  x.report("direct") do |times|
+    @benchmark_procedure[:direct, times]
+  end
+
+  x.compare!
+end

data/benchmark/op_times.rb

@@ -0,0 +1,32 @@
+require "benchmark/ips"
+require "securerandom"
+
+require_relative "../lib/rubcask"
+
+Benchmark.ips do |x|
+  x.time = 30
+
+  x.report("1 milion writes") do
+    Dir.mktmpdir do |path|
+      dir = Rubcask::Directory.new(path, config: Rubcask::Config.new)
+
+      1_000_000.times do |idx|
+        dir[idx.to_s] = SecureRandom.hex(128)
+      end
+    end
+  end
+
+  x.report("1 milion writes gets") do
+    Dir.mktmpdir do |path|
+      dir = Rubcask::Directory.new(path, config: Rubcask::Config.new)
+
+      1_000_000.times do |idx|
+        dir[idx.to_s] = SecureRandom.hex(128)
+      end
+
+      1_000_000.times do |idx|
+        dir[idx.to_s]
+      end
+    end
+  end
+end

data/benchmark/profile.rb

@@ -0,0 +1,15 @@
+require "tempfile"
+require "securerandom"
+require "stackprof"
+
+require_relative "../lib/rubcask"
+
+StackProf.run(out: "tmp/stackprof-cpu-myapp.dump", raw: true) do
+  Dir.mktmpdir do |path|
+    dir = Rubcask::Directory.new(path, config: Rubcask::Config.new)
+
+    1_000_000.times do |idx|
+      dir[idx.to_s] = SecureRandom.hex(128)
+    end
+  end
+end

data/benchmark/server_benchmark_helper.rb

@@ -0,0 +1,138 @@
+require "concurrent"
+require "descriptive_statistics/safe"
+
+require "tempfile"
+
+require_relative "../lib/rubcask"
+require_relative "../lib/rubcask/server/client"
+
+require_relative "../lib/rubcask/server/threaded"
+require_relative "../lib/rubcask/server/async"
+
+module ServerBenchmarkHelper
+  NUMBER_OF_THREADS = 128
+  VALUE = ("8  bytes" * Rubcask::Bytes::KILOBYTE).freeze
+
+  def run_benchmark(seconds, threads, hostname, port)
+    res = Concurrent::Array.new
+
+    threads.times.map do
+      Thread.new do
+        start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        client = Rubcask::Server::Client.new(hostname, port)
+        array = []
+        while (task_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)) - start < seconds
+          task_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          yield(client)
+          array << Process.clock_gettime(Process::CLOCK_MONOTONIC) - task_start
+        end
+        res.concat(array)
+        client.close
+      end
+    end.map(&:join)
+
+    res.extend(DescriptiveStatistics)
+    {
+      "count" => res.size,
+      "median" => res.median,
+      "mean" => res.mean,
+      "standard deviation" => res.standard_deviation,
+      "95 percentile" => res.percentile(95),
+      "99 percentile" => res.percentile(99),
+      "99.99 percentile" => res.percentile(99.99),
+      "max" => res.max
+    }
+  end
+
+  def run_benchmark_with_async_server(label, &block)
+    Dir.mktmpdir do |path|
+      dir = Rubcask::Directory.new(path, config: Rubcask::Config::DEFAULT_SERVER_CONFIG)
+      dir["key"] = VALUE
+
+      config = Rubcask::Server::Config.new { |c| c.port = get_free_port }
+      server = Rubcask::Server::Async.new(dir, config: config)
+      from_thread_q = Thread::Queue.new
+      to_thread_q = Thread::Queue.new
+      server_thread = Thread.new(server) do |s|
+        Sync do
+          condition = Async::Condition.new
+          Async do
+            condition.wait
+            from_thread_q << nil
+          end
+
+          s.start(condition)
+
+          to_thread_q.pop
+          server.shutdown
+        end
+      end
+      from_thread_q.pop
+      begin
+        run_benchmark(10, 10, config.hostname, config.port, &block) # warmup
+
+        rd, wr = IO.pipe
+
+        Process.fork do
+          val = run_benchmark(10, NUMBER_OF_THREADS, config.hostname, config.port, &block)
+          rd.close
+          wr.write(val.to_s)
+          wr.close
+        end
+
+        wr.close
+        puts "#{label}: #{rd.read}"
+        rd.close
+      ensure
+        to_thread_q << nil
+        server_thread.join
+        dir.close
+      end
+    end
+  end
+
+  def run_benchmark_with_threaded_server(label, &block)
+    Dir.mktmpdir do |path|
+      dir = Rubcask::Directory.new(path, config: Rubcask::Config::DEFAULT_SERVER_CONFIG)
+      dir["key"] = VALUE
+
+      config = Rubcask::Server::Config.new { |c| c.port = get_free_port }
+      server = Rubcask::Server::Threaded.new(dir, config: config)
+      server.setup_shutdown_pipe
+      begin
+        server.connect
+      rescue Errno::EADDRINUSE
+        retry
+      end
+      server_thread = Thread.new(server, &:start)
+      begin
+        run_benchmark(10, 10, config.hostname, config.port, &block) # warmup
+        rd, wr = IO.pipe
+
+        Process.fork do
+          rd.close
+          val = run_benchmark(10, NUMBER_OF_THREADS, config.hostname, config.port, &block)
+          wr.write(val.to_s)
+          wr.close
+        end
+
+        wr.close
+        puts "#{label}: #{rd.read}"
+        rd.close
+      ensure
+        server.shutdown
+        server_thread.join
+        dir.close
+      end
+    end
+  end
+
+  def get_free_port
+    server = TCPServer.new("127.0.0.1", 0)
+    begin
+      server.addr[1]
+    ensure
+      server.close
+    end
+  end
+end

data/example/server_runner.rb

@@ -0,0 +1,15 @@
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+
+require "tempfile"
+
+require "rubcask"
+require "rubcask/server/runner"
+
+Dir.mktmpdir do |tmpdir|
+  runner_config = Rubcask::Server::Runner::Config.configure do |c|
+    c.directory_path = tmpdir
+  end
+
+  runner = Rubcask::Server::Runner.new(runner_config: runner_config)
+  runner.start
+end

data/lib/rubcask/bytes.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Rubcask
+  # Containssize of units of data in bytes
+  module Bytes
+    BYTE = 1
+    KILOBYTE = 1 << 10
+    MEGABYTE = 1 << 20
+    GIGABYTE = 1 << 30
+  end
+end

data/lib/rubcask/concurrency/fake_atomic_fixnum.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Rubcask
+  module Concurrency
+    # A fake class that implements interface of Concurrent::AtomicFixnum
+    # without actually doing any synchronization
+    class FakeAtomicFixnum
+      attr_accessor :value
+      def initialize(initial = 0)
+        @value = initial
+      end
+
+      def compare_and_set(expected, update)
+        @value = update if @value == expected
+        @value
+      end
+
+      def decrement(delta = 1)
+        @value -= delta
+        @value
+      end
+
+      def increment(delta = 1)
+        @value += delta
+        @value
+      end
+
+      def update
+        @value = yield(@value)
+        @value
+      end
+    end
+  end
+end

data/lib/rubcask/concurrency/fake_lock.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Rubcask
+  module Concurrency
+    # Fake of Concurrent::ReentrantReadWriteLock
+    # It does not do any synchronization
+    class FakeLock
+      def acquire_read_lock
+        true
+      end
+
+      def acquire_write_lock
+        true
+      end
+
+      def release_read_lock
+        false
+      end
+
+      def release_write_lock
+        false
+      end
+
+      def has_waiters?
+        false
+      end
+
+      def with_write_lock
+        yield
+      end
+
+      def with_read_lock
+        yield
+      end
+
+      def write_locked?
+        false
+      end
+    end
+  end
+end