large_object_store 1.3.4 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 7330eb4955ee45e4ff4fcad4580a6fcd2f55ad48
-  data.tar.gz: e61edd740a283e28ec308600a271a095d199a44b
+SHA256:
+  metadata.gz: e72d3b11da68461fa95e282ad1f8727c0c397f87602a01557754a438c111f88d
+  data.tar.gz: a79ab553fd00bb63b89c584dda277c4faba58ce82d4095a7a73a16407d4995ca
 SHA512:
-  metadata.gz: 5a9e62798f1fcc24a298e5d36cf8896c99aed6d3070a0d62dfec72a2c2279949acf003030ed5bbdcd9bd442b30eef9fb56cb9d44b98821f1365024383345403c
-  data.tar.gz: dfc7717490b507a35ad4020aca46d3dd07dc1d2d773b1b6245ea4da1d06d61de78e4f85420cff6fa47e3d8f112f6cb70b4e57211b6be94fbceec099e2c66cf29
+  metadata.gz: b2eab6d7bf39128dda4096fc96dc583128e08d0b08acbce8ea879611be48c03024aacc2f0cb55d27827988ee024c1d291495f8c85bdeb12ba55becd34a634407
+  data.tar.gz: b991e5e5337333eeb39daade26edad8957819f8de768f2632df09f77873d9f351f5913d7dac19d5595115d3624369c88a8ddc9fd90a6967028db522221cadd96

data/Readme.md

@@ -26,6 +26,19 @@ store.write("a" * 1000, compress: true, compress_limit: 100) # compress when gre
 store.write("a" * 1000, raw: true)                           # store as string to avoid marshaling overhead
 ```
 
+zstd
+====
+
+[zstd compression](https://engineering.fb.com/2016/08/31/core-data/smaller-and-faster-data-compression-with-zstandard/), a modern improvement over the venerable zlib compression algorithm, is supported by passing the `zstd` flag when writing items:
+
+```
+store.write("a" * 10_000_000, compress: true, zstd: true)
+```
+
+For backwards compatibility and to enable safe roll-out of the change in working systems, the `zstd` flag defaults to `false`.
+
+zstd decompression is used when the zstd magic number is detected at the beginning of compressed data, so `zstd: true` does not need to be passed when reading/fetching items.
+
 Author
 ======
 [Ana Martinez](https://github.com/anamartinez)<br/>
@@ -33,4 +46,4 @@ acemacu@gmail.com<br/>
 [Michael Grosser](https://github.com/grosser)<br/>
 michael@grosser.it<br/>
 License: MIT<br/>
-[![Build Status](https://travis-ci.org/anamartinez/large_object_store.png)](https://travis-ci.org/anamartinez/large_object_store)
+[![CI](https://github.com/anamartinez/large_object_store/actions/workflows/actions.yml/badge.svg)](https://github.com/anamartinez/large_object_store/actions/workflows/actions.yml)

data/lib/large_object_store/version.rb

@@ -1,3 +1,3 @@
 module LargeObjectStore
-  VERSION = "1.3.4"
+  VERSION = "1.7.0"
 end

data/lib/large_object_store.rb

@@ -1,11 +1,12 @@
 require "large_object_store/version"
 require "zlib"
+require "zstd-ruby"
 require "securerandom"
 
 module LargeObjectStore
   UUID_BYTES = 16
   UUID_SIZE = UUID_BYTES * 2
-  CACHE_VERSION = 3
+  CACHE_VERSION = 4
   MAX_OBJECT_SIZE = 1024**2
   ITEM_HEADER_SIZE = 100
   DEFAULT_COMPRESS_LIMIT = 16*1024
@@ -13,35 +14,40 @@ module LargeObjectStore
   COMPRESSED = 1
   RAW = 2
   FLAG_RADIX = 32 # we can store 32 different states
+  ZSTD_MAGIC = "\x28\xB5\x2F\xFD".force_encoding('ASCII-8BIT')
+  ZSTD_COMPRESS_LEVEL = 3 # Default level recommended by zstd authors
 
-  def self.wrap(store)
-    RailsWrapper.new(store)
+  def self.wrap(*args)
+    RailsWrapper.new(*args)
+  end
+  class << self
+    ruby2_keywords :wrap if respond_to?(:ruby2_keywords, true)
   end
 
   class RailsWrapper
     attr_reader :store
 
-    def initialize(store)
+    def initialize(store, serializer: Marshal, max_slice_size: MAX_OBJECT_SIZE)
       @store = store
+      @serializer = serializer
+      @max_slice_size = [max_slice_size, MAX_OBJECT_SIZE].min
+      @namespace = (store.respond_to?(:options) && store.options[:namespace]) || ""
     end
 
-    def write(key, value, options = {})
+    def write(key, value, **options)
       options = options.dup
       value = serialize(value, options)
 
-      # calculate slice size; note that key length is a factor because
-      # the key is stored on the same slab page as the value
-      slice_size = MAX_OBJECT_SIZE - ITEM_HEADER_SIZE - UUID_SIZE - key.bytesize
-
+      slice_size = safe_slice_size(key)
       # store number of pages
       pages = (value.size / slice_size.to_f).ceil
 
       if pages == 1
-        !!@store.write(key(key, 0), value, options)
+        !!@store.write(key(key, 0), value, **options)
       else
         # store meta
         uuid = SecureRandom.hex(UUID_BYTES)
-        return false unless @store.write(key(key, 0), [pages, uuid], options) # invalidates the old cache
+        return false unless @store.write(key(key, 0), [pages, uuid], **options) # invalidates the old cache
 
         # store object
         page = 1
@@ -49,7 +55,7 @@ module LargeObjectStore
           slice = value.slice!(0, slice_size)
           break if slice.size == 0
 
-          return false unless @store.write(key(key, page), slice.prepend(uuid), options.merge(raw: true))
+          return false unless @store.write(key(key, page), slice.prepend(uuid), raw: true, **options)
           page += 1
         end
         true
@@ -83,11 +89,11 @@ module LargeObjectStore
       deserialize(data)
     end
 
-    def fetch(key, options={})
+    def fetch(key, **options)
       value = read(key)
       return value unless value.nil?
       value = yield
-      write(key, value, options)
+      write(key, value, **options)
       value
     end
 
@@ -101,6 +107,19 @@ module LargeObjectStore
 
     private
 
+    # calculate slice size; note that key length is a factor because
+    # the key is stored on the same slab page as the value
+    def safe_slice_size(key)
+      namespace_length = @namespace.empty? ? 0 : @namespace.size + 1
+      overhead = ITEM_HEADER_SIZE + UUID_SIZE + key.bytesize + namespace_length
+      slice_size = @max_slice_size - overhead
+      if slice_size <= 0
+        MAX_OBJECT_SIZE - overhead
+      else
+        slice_size
+      end
+    end
+
     # convert a object to a string
     # modifies options
     def serialize(value, options)
@@ -110,31 +129,56 @@ module LargeObjectStore
         flag |= RAW
         value = value.to_s
       else
-        value = Marshal.dump(value)
+        value = @serializer.dump(value)
       end
 
       if compress?(value, options)
         flag |= COMPRESSED
-        value = Zlib::Deflate.deflate(value)
+        value = compress(value, options)
       end
 
       value.prepend(flag.to_s(FLAG_RADIX))
     end
 
+    def compress(value, options)
+      if options[:zstd]
+        Zstd.compress(value, ZSTD_COMPRESS_LEVEL)
+      else
+        Zlib::Deflate.deflate(value)
+      end
+    end
+
+    def decompress(data)
+      if data.start_with?(ZSTD_MAGIC)
+        Zstd.decompress(data)
+      else
+        Zlib::Inflate.inflate(data)
+      end
+    end
+
     # opposite operations and order of serialize
     def deserialize(raw_data)
       data = raw_data.dup
       flag = data.slice!(0, 1).to_i(FLAG_RADIX)
-      data = Zlib::Inflate.inflate(data) if flag & COMPRESSED == COMPRESSED
-      data = Marshal.load(data) if flag & RAW != RAW
+      data = decompress(data) if flag & COMPRESSED == COMPRESSED
+      data = @serializer.load(data) if flag & RAW != RAW
       data
     end
 
     # Don't pass compression on to Rails, we're doing it ourselves.
     def compress?(value, options)
-      return unless options.delete(:compress)
-      compress_limit = options.delete(:compress_limit) || DEFAULT_COMPRESS_LIMIT
-      value.bytesize > compress_limit
+      return unless options[:compress]
+
+      compress_limit = options[:compress_limit] || DEFAULT_COMPRESS_LIMIT
+      should_compress = value.bytesize > compress_limit
+
+      if should_compress
+        # Pass compress: false to Rails in case the default is true
+        options[:compress] = false
+        options.delete(:compress_limit)
+      end
+
+      should_compress
     end
 
     def key(key, i)

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: large_object_store
 version: !ruby/object:Gem::Version
-  version: 1.3.4
+  version: 1.7.0
 platform: ruby
 authors:
 - Ana Martinez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-27 00:00:00.000000000 Z
-dependencies: []
+date: 2024-01-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: zstd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.5
 description: 
 email: acemacu@gmail.com
 executables: []
@@ -31,15 +45,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.0'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 3.5.3
 signing_key: 
 specification_version: 4
 summary: Store large objects in memcache or others