garcun 0.0.2

data/lib/garcon/stash/format.rb

@@ -0,0 +1,114 @@
+# encoding: UTF-8
+#
+# Author: Stefano Harding <riddopic@gmail.com>
+#
+# Copyright (C) 2014-2015 Stefano Harding
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Garcon
+  module Stash
+    # Database format serializer and deserializer. You can create your own
+    # implementation of this class and define your own database format!
+    #
+    # @api public
+    class Format
+      # Read database header from input stream
+      #
+      # @param [#read] input
+      #   the input stream
+      #
+      # @return void
+      #
+      def read_header(input)
+        raise 'Not a stash' if input.read(MAGIC.bytesize) != MAGIC
+        ver = input.read(2).unpack('n').first
+        raise "Expected stash version #{VERSION}, got #{ver}" if ver != VERSION
+      end
+
+      # Return database header as string
+      #
+      # @return [String] database file header
+      #
+      def header
+        MAGIC + [VERSION].pack('n')
+      end
+
+      # Serialize record and return string.
+      #
+      # @param [Array] rec
+      #   an array with [key, value] or [key] if the record is deleted
+      #
+      # @return [String] serialized record
+      #
+      def dump(rec)
+        data = if rec.size == 1
+          [rec[0].bytesize, DELETE].pack('NN') << rec[0]
+        else
+          [rec[0].bytesize, rec[1].bytesize].pack('NN') << rec[0] << rec[1]
+        end
+        data << crc32(data)
+      end
+
+      # Deserialize records from buffer, and yield them.
+      #
+      # @param [String] buf
+      #   the buffer to read from
+      #
+      # @yield [Array] block
+      #   deserialized record [key, value] or [key] if the record is deleted
+      #
+      # @return [Fixnum] number of records
+      #
+      def parse(buf)
+        n, count = 0, 0
+        while n < buf.size
+          key_size, value_size = buf[n, 8].unpack('NN')
+          data_size = key_size + 8
+          data_size += value_size if value_size != DELETE
+          data = buf[n, data_size]
+          n += data_size
+          unless buf[n, 4] == crc32(data)
+            raise 'CRC mismatch: your stash might be corrupted!'
+          end
+          n += 4
+          yield(value_size == DELETE ? [data[8, key_size]] : [data[8, key_size], data[8 + key_size, value_size]])
+          count += 1
+        end
+        count
+      end
+
+      protected #      A T T E N Z I O N E   A R E A   P R O T E T T A
+
+      # Magic string of the file header.
+      MAGIC = 'ABRACADABRA'
+
+      # Database file format version (it is a hash after all).
+      VERSION = 420
+
+      # Special value size used for deleted records
+      DELETE = (1 << 32) - 1
+
+      # Compute crc32 of string
+      #
+      # @param [String] s a string
+      #
+      # @return [Fixnum]
+      #
+      def crc32(s)
+        [Zlib.crc32(s, 0)].pack('N')
+      end
+    end
+  end
+end

data/lib/garcon/stash/journal.rb

@@ -0,0 +1,226 @@
+# encoding: UTF-8
+#
+# Author: Stefano Harding <riddopic@gmail.com>
+#
+# Copyright (C) 2014-2015 Stefano Harding
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Garcon
+  module Stash
+    # Stash::Io handles background io, compaction and is the arbiter
+    # of multiprocess safety.
+    #
+    # @api private
+    class Journal < Queue
+      attr_reader :size, :file
+
+      def initialize(file, format, serializer, &block)
+        super()
+        @file, @format, @serializer, @emit = file, format, serializer, block
+        open
+        @worker = Thread.new(&method(:worker))
+        @worker.priority = -1
+        load
+      end
+
+      # Is the journal closed?
+      #
+      def closed?
+        @fd.closed?
+      end
+
+      # Clear the queue and close the file handler
+      #
+      def close
+        self << nil
+        @worker.join
+        @fd.close
+        super
+      end
+
+      # Load new journal entries
+      #
+      def load
+        flush
+        replay
+      end
+
+      # Lock the logfile across thread and process boundaries
+      #
+      def lock
+        flush
+        with_flock(File::LOCK_EX) do
+          replay
+          result = yield
+          flush
+          result
+        end
+      end
+
+      # Clear the database log and yield
+      #
+      def clear
+        flush
+        with_tmpfile do |path, file|
+          file.write(@format.header)
+          file.close
+          with_flock(File::LOCK_EX) do
+            File.rename(path, @file)
+          end
+        end
+        open
+      end
+
+      # Compact the logfile to represent the in-memory state
+      #
+      def compact
+        load
+        with_tmpfile do |path, file|
+          # Compactified database has the same size -> return
+          return self if @pos == file.write(dump(yield, @format.header))
+          with_flock(File::LOCK_EX) do
+            if @pos != nil
+              file.write(read)
+              file.close
+              File.rename(path, @file)
+            end
+          end
+        end
+        open
+        replay
+      end
+
+      # Return byte size of journal
+      #
+      def bytesize
+        @fd.stat.size
+      end
+
+      private #        P R O P R I E T À   P R I V A T A   Vietato L'accesso
+
+      # Emit records as we parse them
+      def replay
+        buf = read
+        @size += @format.parse(buf, &@emit)
+      end
+
+      # Open or reopen file
+      #
+      def open
+        @fd.close if @fd
+        @fd = File.open(@file, 'ab+')
+        @fd.advise(:sequential) if @fd.respond_to? :advise
+        stat = @fd.stat
+        @inode = stat.ino
+        write(@format.header) if stat.size == 0
+        @pos = nil
+      end
+
+      # Read new file content
+      #
+      def read
+        with_flock(File::LOCK_SH) do
+          unless @pos
+            @fd.pos = 0
+            @format.read_header(@fd)
+            @size = 0
+            @emit.call(nil)
+          else
+            @fd.pos = @pos
+          end
+          buf = @fd.read
+          @pos = @fd.pos
+          buf
+        end
+      end
+
+      # Return database dump as string
+      #
+      def dump(records, dump = '')
+        records.each do |record|
+          record[1] = @serializer.dump(record.last)
+          dump << @format.dump(record)
+        end
+        dump
+      end
+
+      # Worker thread
+      #
+      def worker
+        while (record = first)
+          tries = 0
+          begin
+            if Hash === record
+              write(dump(record))
+              @size += record.size
+            else
+              record[1] = @serializer.dump(record.last) if record.size > 1
+              write(@format.dump(record))
+              @size += 1
+            end
+          rescue Exception => e
+            tries += 1
+            warn "Stash worker, try #{tries}: #{e.message}"
+            tries <= 3 ? retry : raise
+          ensure
+            pop
+          end
+        end
+      rescue Exception => e
+        warn "Stash worker terminated: #{e.message}"
+      end
+
+      # Write data to output stream and advance @pos
+      #
+      def write(dump)
+        with_flock(File::LOCK_EX) do
+          @fd.write(dump)
+          @fd.flush
+        end
+        @pos = @fd.pos if @pos && @fd.pos == @pos + dump.bytesize
+      end
+
+      # Block with file lock
+      #
+      def with_flock(mode)
+        return yield if @locked
+        begin
+          loop do
+            Thread.pass until @fd.flock(mode)
+            stat = @fd.stat
+            break if stat.nlink > 0 && stat.ino == @inode
+            open
+          end
+          @locked = true
+          yield
+        ensure
+          @fd.flock(File::LOCK_UN)
+          @locked = false
+        end
+      end
+
+      # Open temporary file and pass it to the block
+      #
+      def with_tmpfile
+        path = [@file, $$.to_s(36), Thread.current.object_id.to_s(36)].join
+        file = File.open(path, 'wb')
+        yield(path, file)
+      ensure
+        file.close unless file.closed?
+        File.unlink(path) if File.exists?(path)
+      end
+    end
+  end
+end

data/lib/garcon/stash/queue.rb

@@ -0,0 +1,83 @@
+# encoding: UTF-8
+#
+# Author: Stefano Harding <riddopic@gmail.com>
+#
+# Copyright (C) 2014-2015 Stefano Harding
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Garcon
+  module Stash
+    class Queue
+      def initialize
+        @queue, @full, @empty = [], [], []
+        @stop = false
+        @heartbeat = Thread.new(&method(:heartbeat))
+        @heartbeat.priority = -9
+      end
+
+      def <<(x)
+        @queue << x
+        thread = @full.first
+        thread.wakeup if thread
+      end
+
+      def pop
+        @queue.shift
+        if @queue.empty?
+          thread = @empty.first
+          thread.wakeup if thread
+        end
+      end
+
+      def first
+        while @queue.empty?
+          begin
+            @full << Thread.current
+            Thread.stop while @queue.empty?
+          ensure
+            @full.delete(Thread.current)
+          end
+        end
+        @queue.first
+      end
+
+      def flush
+        until @queue.empty?
+          begin
+            @empty << Thread.current
+            Thread.stop until @queue.empty?
+          ensure
+            @empty.delete(Thread.current)
+          end
+        end
+      end
+
+      def close
+        @stop = true
+        @heartbeat.join
+      end
+
+      private #        P R O P R I E T À   P R I V A T A   Vietato L'accesso
+
+      def heartbeat
+        until @stop
+          @empty.each(&:wakeup)
+          @full.each(&:wakeup)
+          sleep 0.1
+        end
+      end
+    end
+  end
+end

data/lib/garcon/stash/serializer.rb

@@ -0,0 +1,86 @@
+# encoding: UTF-8
+#
+# Author: Stefano Harding <riddopic@gmail.com>
+#
+# Copyright (C) 2014-2015 Stefano Harding
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Garcon
+  module Stash
+    module Serializer
+      # Serializer which only encodes key in binary
+      #
+      # @api public
+      class None
+        # (see Stash::Serializer::Default#key_for)
+        if ''.respond_to? :force_encoding
+          def key_for(key)
+            if key.encoding != Encoding::BINARY
+              key = key.dup if key.frozen?
+              key.force_encoding(Encoding::BINARY)
+            end
+            key
+          end
+        else
+          def key_for(key)
+            key
+          end
+        end
+
+        # (see Stash::Serializer::Default#dump)
+        def dump(value)
+          value
+        end
+
+        # (see Stash::Serializer::Default#load)
+        def load(value)
+          value
+        end
+      end
+
+      # Default serializer which converts keys to strings and marshalls values.
+      #
+      # @api public
+      class Default < None
+        # Transform the key to a string.
+        #
+        # @param [Object] key
+        # @return [String]
+        #   The key transformed to string.
+        def key_for(key)
+          super(key.to_s)
+        end
+
+        # Serialize a value.
+        #
+        # @param [Object] value
+        # @return [String]
+        #   The value transformed to string.
+        def dump(value)
+          Marshal.dump(value)
+        end
+
+        # Parse a value.
+        #
+        # @param [String] value
+        # @return [Object]
+        #   The deserialized value.
+        def load(value)
+          Marshal.load(value)
+        end
+      end
+    end
+  end
+end