herringbone 0.4.0 → 0.6.0

data/lib/herringbone/bloom_filter.rb

@@ -300,8 +300,6 @@ module Herringbone
       "#<#{self.class.name} #{num_bytes} bytes#{" for #{@column.dotted_path}" if @column}>"
     end
 
-    private
-
     # Header for this filter: split block algorithm, XXH64 hash, uncompressed
     #
     # @return [Format::BloomFilterHeader] the header
@@ -314,57 +312,4 @@ module Herringbone
       )
     end
   end
-
-  class Reader
-    # Internal (used by reads with where:): the bloom filter of a column chunk. +column+ is a
-    # dotted path ("a.b"), an Array path or a Schema::Column. Returns a BloomFilter, or nil when
-    # the chunk has none (or one of an unknown kind).
-    #
-    # @param row_group_index [Integer] index of the row group
-    # @param column [String, Array<String, Symbol>, Symbol, Schema::Column] the leaf column
-    # @return [BloomFilter, nil] the filter, or nil when there is none or it is unsupported
-    # @raise [IndexError] when there is no such row group
-    # @raise [ArgumentError] when there is no such column
-    # @raise [FormatError] when the filter is truncated or its header cannot be decoded
-    def bloom_filter(row_group_index, column)
-      col = bloom_filter_column(column)
-      rg = row_groups.fetch(row_group_index) { raise IndexError, "No row group #{row_group_index}" }
-      meta = rg.columns.fetch(col.index).meta_data
-      offset = meta&.bloom_filter_offset
-      return nil unless offset
-      length = meta.bloom_filter_length
-      @io.seek(offset)
-      if length
-        buf = @io.read(length)
-        raise FormatError, "Truncated bloom filter" if buf.nil? || buf.bytesize < length
-        return BloomFilter.decode(buf, column: col)
-      end
-      # Without a length (older writers), read the header first, then the bitset it announces
-      head = @io.read(256) || "".b
-      reader = Thrift::Reader.new(head)
-      header = reader.read_struct(Format::BloomFilterHeader)
-      return nil unless BloomFilter.supported_header?(header)
-      @io.seek(offset + reader.pos)
-      bitset = @io.read(header.num_bytes)
-      raise FormatError, "Truncated bloom filter" if bitset.nil? || bitset.bytesize != header.num_bytes
-      BloomFilter.new(bitset: bitset, column: col)
-    rescue Thrift::Error => e
-      raise FormatError, "Corrupt bloom filter header for #{col.dotted_path}: #{e.message}"
-    end
-
-    private
-
-    # Resolves the +column+ argument of #bloom_filter to a leaf column
-    #
-    # @param column [String, Array<String, Symbol>, Symbol, Schema::Column] dotted path, path
-    #   Array or column
-    # @return [Schema::Column] the column
-    # @raise [ArgumentError] when the schema has no such column
-    def bloom_filter_column(column)
-      return column if column.is_a?(Schema::Column)
-      col = schema.column(column.is_a?(Array) ? column.map(&:to_s) : column.to_s)
-      raise ArgumentError, "No column #{column.inspect}" unless col
-      col
-    end
-  end
 end

data/lib/herringbone/byte_values.rb

@@ -45,6 +45,15 @@ module Herringbone
     # @return [Boolean] true while still dictionary-encoding (not yet switched to raw bytes)
     def dictionary? = !@indices.nil?
 
+    # Short summary for the console, without the values
+    #
+    # @return [String] value count, the dictionary size (or "raw" after switching to raw bytes),
+    #   the width for FIXED_LEN_BYTE_ARRAY and the estimated memory
+    def inspect
+      mode = dictionary? ? "dictionary=#{@dictionary.size}" : "raw"
+      "#<#{self.class.name} size=#{size} #{mode}#{" width=#{@width}" if @width} memory_bytes=#{memory_bytes}>"
+    end
+
     # Appends a value, switching to raw bytes when the dictionary limits are exceeded.
     # @param value [String] bytes of the value; for FIXED_LEN_BYTE_ARRAY it must be +width+ bytes
     #   long (not checked here)

data/lib/herringbone/compression.rb

@@ -83,6 +83,28 @@ module Herringbone
     # Codec name => codec id, the inverse of NAMES
     CODECS_BY_NAME = NAMES.invert.freeze
 
+    # Codec id => compression levels it accepts, for the writer's compression_level: option
+    LEVELS = Ractor.make_shareable({
+      Format::Codec::ZSTD => (-(1 << 17))..22,
+      Format::Codec::GZIP => 0..9,
+      Format::Codec::BROTLI => 0..11
+    })
+
+    # Checks that +codec+ takes a compression level and that +level+ is in its range
+    #
+    # @param codec [Integer] Format::Codec id
+    # @param level [Integer, nil] compression level, nil for the codec's default
+    # @return [Integer, nil] the level
+    # @raise [ArgumentError] when the codec has no levels or +level+ is out of its range
+    def check_level!(codec, level)
+      return if level.nil?
+      name = NAMES.fetch(codec, codec).inspect
+      range = LEVELS.fetch(codec) { raise ArgumentError, "#{name} compression has no levels" }
+      level = Integer(level)
+      raise ArgumentError, "#{name} compression level must be in #{range}, got #{level}" unless range.cover?(level)
+      level
+    end
+
     # Codec id for a codec name; Integers are taken to be ids already and returned unchecked
     #
     # @param name [Integer, Symbol, String] codec id, or a name from NAMES (case-insensitive)
@@ -129,18 +151,19 @@ module Herringbone
     #
     # @param codec [Integer] Format::Codec id
     # @param data [String] bytes to compress
+    # @param level [Integer, nil] compression level checked with check_level!, nil for the default
     # @return [String] compressed bytes (binary)
     # @raise [UnsupportedError] for a codec herringbone does not implement
     # @raise [MissingCodecError] when the codec's gem is not loaded
-    def compress(codec, data)
+    def compress(codec, data, level = nil)
       case codec
       when Format::Codec::UNCOMPRESSED then data
       when Format::Codec::SNAPPY then Codecs::Snappy.compress(data)
-      when Format::Codec::GZIP then Zlib.gzip(data)
+      when Format::Codec::GZIP then level ? Zlib.gzip(data, level: level) : Zlib.gzip(data)
       when Format::Codec::LZ4_RAW then Codecs::LZ4.compress_block(data)
       when Format::Codec::LZ4 then Codecs::LZ4.compress_hadoop(data)
-      when Format::Codec::ZSTD then library(codec).compress(data)
-      when Format::Codec::BROTLI then library(codec).deflate(data)
+      when Format::Codec::ZSTD then level ? library(codec).compress(data, level: level) : library(codec).compress(data)
+      when Format::Codec::BROTLI then level ? library(codec).deflate(data, quality: level) : library(codec).deflate(data)
       else
         raise UnsupportedError, "Unsupported compression codec #{Format::Codec::NAMES[codec] || codec}"
       end.b