large_object_store 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 92be121f082879185822f734dd5fe7935ce04aacb05e900ce4dbfe89b23d712c
-  data.tar.gz: 14854eba125bd1837eb444ec25d65da9ae99228455f600bb4c9cb38902d3ac8a
+  metadata.gz: 9e41ad488600c979693e5e2c58958c8fd9eb3f3d51ae53c908415b920f1180a9
+  data.tar.gz: 22063ad5444b0f50a39c48a08e52e537bba6b21f31b8ee7751d331dfb3574412
 SHA512:
-  metadata.gz: 489ecb7a501deda4f45feb995585999717a595ef6e4539641a6866215d23cf89eb3f745aac02d3b95a4581356405acb21f40f79b010954a3a037cd053172fe5d
-  data.tar.gz: 6c8ad1eead221be4324a5d6557926b8702469797448d68e62a19ded103ad52a0a1dc146599d2f69b1dd6ab5b237659e275d07f9a1542b9aa7a735dc5bb95acf3
+  metadata.gz: 6e76ee6a7a7679cdf1b797aa65e3d59656dafeade0240db262de7312d640fdc15e3de1af92323aaed10a8e2d26c762265afc1ae9bb05978b39220224096b0276
+  data.tar.gz: '05937831063dbe4d29766cb4237b82b11fd950a3c879d6fe9461cbf557493ed235b0e181447cca254a9043166145c278b3c1171ad7f9c9f81ed92e70b5864b1a'

data/Readme.md

@@ -26,6 +26,19 @@ store.write("a" * 1000, compress: true, compress_limit: 100) # compress when gre
 store.write("a" * 1000, raw: true)                           # store as string to avoid marshaling overhead
 ```
 
+zstd
+====
+
+[zstd compression](https://engineering.fb.com/2016/08/31/core-data/smaller-and-faster-data-compression-with-zstandard/), a modern improvement over the venerable zlib compression algorithm, is supported by passing the `zstd` flag when writing items:
+
+```
+store.write("a" * 10_000_000, compress: true, zstd: true)
+```
+
+For backwards compatibility and to enable safe roll-out of the change in working systems, the `zstd` flag defaults to `false`.
+
+zstd decompression is used when the zstd magic number is detected at the beginning of compressed data, so `zstd: true` does not need to be passed when reading/fetching items.
+
 Author
 ======
 [Ana Martinez](https://github.com/anamartinez)<br/>

data/lib/large_object_store/version.rb

@@ -1,3 +1,3 @@
 module LargeObjectStore
-  VERSION = "1.5.0"
+  VERSION = "1.6.0"
 end

data/lib/large_object_store.rb

@@ -1,11 +1,12 @@
 require "large_object_store/version"
 require "zlib"
+require "zstd-ruby"
 require "securerandom"
 
 module LargeObjectStore
   UUID_BYTES = 16
   UUID_SIZE = UUID_BYTES * 2
-  CACHE_VERSION = 3
+  CACHE_VERSION = 4
   MAX_OBJECT_SIZE = 1024**2
   ITEM_HEADER_SIZE = 100
   DEFAULT_COMPRESS_LIMIT = 16*1024
@@ -13,6 +14,8 @@ module LargeObjectStore
   COMPRESSED = 1
   RAW = 2
   FLAG_RADIX = 32 # we can store 32 different states
+  ZSTD_MAGIC = "\x28\xB5\x2F\xFD".force_encoding('ASCII-8BIT')
+  ZSTD_COMPRESS_LEVEL = 3 # Default level recommended by zstd authors
 
   def self.wrap(*args)
     RailsWrapper.new(*args)
@@ -31,7 +34,7 @@ module LargeObjectStore
 
     def write(key, value, **options)
       options = options.dup
-      value = serialize(value, **options)
+      value = serialize(value, options)
 
       # calculate slice size; note that key length is a factor because
       # the key is stored on the same slab page as the value
@@ -107,7 +110,7 @@ module LargeObjectStore
 
     # convert a object to a string
     # modifies options
-    def serialize(value, **options)
+    def serialize(value, options)
       flag = NORMAL
 
       if options.delete(:raw)
@@ -117,28 +120,53 @@ module LargeObjectStore
         value = @serializer.dump(value)
       end
 
-      if compress?(value, **options)
+      if compress?(value, options)
         flag |= COMPRESSED
-        value = Zlib::Deflate.deflate(value)
+        value = compress(value, options)
       end
 
       value.prepend(flag.to_s(FLAG_RADIX))
     end
 
+    def compress(value, options)
+      if options[:zstd]
+        Zstd.compress(value, ZSTD_COMPRESS_LEVEL)
+      else
+        Zlib::Deflate.deflate(value)
+      end
+    end
+
+    def decompress(data)
+      if data.start_with?(ZSTD_MAGIC)
+        Zstd.decompress(data)
+      else
+        Zlib::Inflate.inflate(data)
+      end
+    end
+
     # opposite operations and order of serialize
     def deserialize(raw_data)
       data = raw_data.dup
       flag = data.slice!(0, 1).to_i(FLAG_RADIX)
-      data = Zlib::Inflate.inflate(data) if flag & COMPRESSED == COMPRESSED
+      data = decompress(data) if flag & COMPRESSED == COMPRESSED
       data = @serializer.load(data) if flag & RAW != RAW
       data
     end
 
     # Don't pass compression on to Rails, we're doing it ourselves.
-    def compress?(value, **options)
-      return unless options.delete(:compress)
-      compress_limit = options.delete(:compress_limit) || DEFAULT_COMPRESS_LIMIT
-      value.bytesize > compress_limit
+    def compress?(value, options)
+      return unless options[:compress]
+
+      compress_limit = options[:compress_limit] || DEFAULT_COMPRESS_LIMIT
+      should_compress = value.bytesize > compress_limit
+
+      if should_compress
+        # Pass compress: false to Rails in case the default is true
+        options[:compress] = false
+        options.delete(:compress_limit)
+      end
+
+      should_compress
     end
 
     def key(key, i)

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: large_object_store
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - Ana Martinez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-03-30 00:00:00.000000000 Z
-dependencies: []
+date: 2023-06-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: zstd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.5
 description: 
 email: acemacu@gmail.com
 executables: []
@@ -31,7 +45,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="