rzstd 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f499740d25842f109c142aaf1ff24648a317e10f3ac2600157fe73151bad7fc7
-  data.tar.gz: 21a645b344ae469809a95a1500d87cf72326e72c5d4d1005cc0cf7b15c07b05d
+  metadata.gz: 4d839dd8c43cda81d339bbcb624508952c77468320439d429699d75b3f441537
+  data.tar.gz: e57e1a1f7d8f782e1243fccca978833e51fe829b8dbdf4d7d652f6b64106ae14
 SHA512:
-  metadata.gz: 070d5a73c25d55de6b04b2b373dd6bfd925381984346bf7683a57027dd77fadcd4c60941c2a408550116292f7181cefbd6ae62b618e529433519bbf4abb39569
-  data.tar.gz: 807f3d87206810f19d528630ea1e1054067e3c0e79ae6eebff97605715dbc8abdcf30f94898f3d21196be04cb7819d7e54382de0a76a2dc65cdea6521bf41a7a
+  metadata.gz: 21c2663f86669e9d6bbfccb6620786aebdc8e7e793cf4e96e3c6ce863324c1065f85a1d2f4ab68be77231d09536e490f2467bba0b70635cf41f6fcbc114fdc25
+  data.tar.gz: 54b0e0f46779782a089706618fbad3a0af26ee72205c9e7c835a90a25c184034a62db669e6a2ca0caa0b308e6a77fda1d4a5f81b399e8678a8b37a89c0c401b8

data/ext/rzstd/src/lib.rs

@@ -257,9 +257,11 @@ fn dict_initialize(ruby: &Ruby, rb_dict: RString, id: u32, level: i32) -> Result
         })?;
     // Enable the content checksum so decoding with the wrong dictionary
     // (or wrong bytes generally) fails fast at the trailing XXH64 check
-    // instead of silently returning garbage. Raw-content dictionaries
-    // always carry a frame `dictID` of 0 by spec, so the checksum is the
-    // only on-wire signal we have to detect dict mismatch.
+    // instead of silently returning garbage. For ZDICT-format dicts
+    // the frame header additionally carries the dict's own dict_id
+    // (via `ZSTD_c_dictIDFlag`, enabled by default), which lets peers
+    // dispatch by id; raw-content dicts carry a frame `dictID` of 0
+    // and rely solely on the checksum to detect mismatch.
     cctx.set_parameter(CParameter::ContentSizeFlag(true))
         .map_err(|code| {
             Error::new(

data/lib/rzstd/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RZstd
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

data/lib/rzstd.rb

@@ -8,6 +8,7 @@ require_relative "rzstd/version"
 module RZstd
   DEFAULT_LEVEL = 3
 
+
   # Wrap the native `_native_compress(bytes, level)` so callers get a
   # `level:` kwarg with a sensible default. Defined as a real method
   # (not a Proc-bound singleton method) so it remains Ractor-shareable.
@@ -15,6 +16,7 @@ module RZstd
     _native_compress(bytes, Integer(level))
   end
 
+
   # Bounded single-shot decompression. When `max_output_size:` is given,
   # the Rust extension reads the frame's Frame_Content_Size header, raises
   # MissingContentSizeError if absent, and raises OutputSizeLimitError if
@@ -25,20 +27,38 @@ module RZstd
     _native_decompress(bytes, Integer(max_output_size || 0))
   end
 
+
   class Dictionary
-    def decompress(bytes, max_output_size: nil)
-      _native_decompress(bytes, Integer(max_output_size || 0))
-    end
-    # Public constructor. Derives the Zstd `Dict_ID` from the dictionary
-    # bytes (sha256 truncated to the first 4 bytes, little-endian) and
-    # forwards to the Rust extension. The id is for out-of-band peer
-    # negotiation (e.g. via the `dict:sha256:<hex>` profile string in
-    # the application protocol) — raw-content zstd dictionaries always
-    # write a frame `dictID` of 0, so the on-wire frame doesn't carry
-    # this id. Wrong-dict decoding is caught by the content checksum
-    # the encoder enables.
+    ZDICT_MAGIC = "\x37\xA4\x30\xEC".b.freeze
+
+    # Public Dict_ID range per the Zstandard spec. IDs `0..32_767` are
+    # reserved for a future registrar, and `>= 2**31` is reserved. Only
+    # `32_768..(2**31 - 1)` is available for private/auto-generated dicts.
+    USER_DICT_ID_MIN  = 32_768
+    USER_DICT_ID_MAX  = (2**31) - 1
+    USER_DICT_ID_SIZE = USER_DICT_ID_MAX - USER_DICT_ID_MIN + 1
+
+    # Public constructor. Resolves the Zstd `Dict_ID`:
+    #
+    # - If `bytes` begins with the ZDICT magic (`0x EC30A437` LE), the
+    #   id is read from bytes `[4..7]` of the dictionary header. This is
+    #   the same id zstd writes into every compressed frame header via
+    #   `ZSTD_c_dictIDFlag` (enabled by default), so on-wire frames and
+    #   `Dictionary#id` agree.
+    # - Otherwise the dict is raw content: zstd writes a frame `dictID`
+    #   of 0, and this wrapper falls back to `sha256(bytes)[0..3]` LE
+    #   mapped into the public range `32_768..(2**31 - 1)`, purely as
+    #   an out-of-band identifier for the Ruby side. Wrong-dict decoding
+    #   of raw dicts is caught by the content checksum the encoder
+    #   enables.
     def self.new(bytes, level: DEFAULT_LEVEL)
-      id = Digest::SHA256.digest(bytes).byteslice(0, 4).unpack1("V")
+      id = if bytes.byteslice(0, 4) == ZDICT_MAGIC
+        bytes.byteslice(4, 4).unpack1("V")
+      else
+        raw = Digest::SHA256.digest(bytes).byteslice(0, 4).unpack1("V")
+        USER_DICT_ID_MIN + (raw % USER_DICT_ID_SIZE)
+      end
+
       _native_new(bytes, id, Integer(level))
     end
 
@@ -59,5 +79,11 @@ module RZstd
       samples.each { |s| buffer << s.b }
       _native_train(buffer, sizes, Integer(capacity))
     end
+
+
+    def decompress(bytes, max_output_size: nil)
+      _native_decompress(bytes, Integer(max_output_size || 0))
+    end
+
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rzstd
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Patrik Wenger