purplelight 0.1.13 → 0.1.16

data/lib/purplelight/snapshot.rb

@@ -19,7 +19,7 @@ module Purplelight
       format: :jsonl,
       compression: :zstd,
       batch_size: 2_000,
-      partitions: [Etc.respond_to?(:nprocessors) ? [Etc.nprocessors * 2, 4].max : 4, 32].min,
+      partitions: (Etc.nprocessors * 2).clamp(4, 32),
       queue_size_bytes: 256 * 1024 * 1024,
       rotate_bytes: 256 * 1024 * 1024,
       read_concern: { level: :majority },
@@ -68,6 +68,10 @@ module Purplelight
       @parquet_max_rows = parquet_max_rows
 
       @running = true
+      if @on_progress
+        @progress_mutex = Mutex.new
+        @progress_cv = ConditionVariable.new
+      end
       @telemetry_enabled = telemetry ? telemetry.enabled? : (ENV['PL_TELEMETRY'] == '1')
       @telemetry = telemetry || (
         @telemetry_enabled ? Telemetry.new(enabled: true) : Telemetry::NULL
@@ -105,43 +109,55 @@ module Purplelight
                             read_concern: (@read_concern.is_a?(Hash) ? @read_concern : { level: @read_concern }),
                             no_cursor_timeout: @no_cursor_timeout,
                             writer_threads: @writer_threads,
-                            compression_level: @compression_level || (ENV['PL_ZSTD_LEVEL']&.to_i if @compression.to_s == 'zstd') || ENV['PL_ZSTD_LEVEL']&.to_i,
+                            compression_level: @compression_level || ENV['PL_ZSTD_LEVEL']&.to_i,
                             write_chunk_bytes: @write_chunk_bytes || ENV['PL_WRITE_CHUNK_BYTES']&.to_i,
                             parquet_row_group: @parquet_row_group || ENV['PL_PARQUET_ROW_GROUP']&.to_i,
                             parquet_max_rows: @parquet_max_rows,
                             sharding: @sharding,
                             resume_overwrite_incompatible: @resume && @resume[:overwrite_incompatible] ? true : false,
                             telemetry: @telemetry_enabled
-                          })
-      manifest.ensure_partitions!(@partitions)
+                          }, partition_count: @partitions)
 
-      # Plan partitions
+      # Reuse the original ranges on resume. Replanning against a growing collection
+      # can move partition boundaries and replay rows under the old checkpoints.
       t_plan = @telemetry.start(:partition_plan_time)
-      partition_filters = Partitioner.object_id_partitions(collection: @collection, query: @query,
-                                                           partitions: @partitions, telemetry: @telemetry)
+      partition_filters = manifest.partition_filters
+      unless partition_filters
+        partition_filters = Partitioner.object_id_partitions(collection: @collection, query: @query,
+                                                             partitions: @partitions, telemetry: @telemetry)
+        manifest.configure_partition_filters!(partition_filters)
+      end
       @telemetry.finish(:partition_plan_time, t_plan)
 
       # Reader queue
       queue = ByteQueue.new(max_bytes: @queue_size_bytes)
 
       # Writer
-      writer = case @format
-               when :jsonl
-                 WriterJSONL.new(directory: dir, prefix: prefix, compression: @compression,
-                                 rotate_bytes: @rotate_bytes, logger: @logger, manifest: manifest)
-               when :csv
-                 single_file = @sharding && @sharding[:mode].to_s == 'single_file'
-                 WriterCSV.new(directory: dir, prefix: prefix, compression: @compression, rotate_bytes: @rotate_bytes,
-                               logger: @logger, manifest: manifest, single_file: single_file)
-               when :parquet
-                 single_file = @sharding && @sharding[:mode].to_s == 'single_file'
-                 row_group = @parquet_row_group || ENV['PL_PARQUET_ROW_GROUP']&.to_i || WriterParquet::DEFAULT_ROW_GROUP_SIZE
-                 WriterParquet.new(directory: dir, prefix: prefix, compression: @compression, logger: @logger,
-                                   manifest: manifest, single_file: single_file, row_group_size: row_group,
-                                   rotate_rows: @parquet_max_rows)
-               else
-                 raise ArgumentError, "format not implemented: #{@format}"
-               end
+      writer_count = @format == :jsonl ? [@writer_threads.to_i, 1].max : 1
+      part_sequence = WriterJSONL::PartSequence.new(manifest.parts.length) if @format == :jsonl
+      writers = case @format
+                when :jsonl
+                  Array.new(writer_count) do
+                    WriterJSONL.new(directory: dir, prefix: prefix, compression: @compression,
+                                    rotate_bytes: @rotate_bytes, logger: @logger, manifest: manifest,
+                                    compression_level: @compression_level, write_chunk_bytes: @write_chunk_bytes,
+                                    part_sequence:)
+                  end
+                when :csv
+                  single_file = @sharding && @sharding[:mode].to_s == 'single_file'
+                  [WriterCSV.new(directory: dir, prefix: prefix, compression: @compression,
+                                 rotate_bytes: @rotate_bytes, logger: @logger, manifest: manifest,
+                                 single_file:, compression_level: @compression_level)]
+                when :parquet
+                  single_file = @sharding && @sharding[:mode].to_s == 'single_file'
+                  row_group = @parquet_row_group || ENV['PL_PARQUET_ROW_GROUP']&.to_i ||
+                              WriterParquet::DEFAULT_ROW_GROUP_SIZE
+                  [WriterParquet.new(directory: dir, prefix: prefix, compression: @compression, logger: @logger,
+                                     manifest:, single_file:, row_group_size: row_group,
+                                     rotate_rows: @parquet_max_rows)]
+                else
+                  raise ArgumentError, "format not implemented: #{@format}"
+                end
 
       # Start reader threads
       readers = partition_filters.each_with_index.map do |pf, idx|
@@ -153,37 +169,49 @@ module Purplelight
         end
       end
 
-      # Writer loop
-      writer_telemetry = @telemetry_enabled ? Telemetry.new(enabled: true) : Telemetry::NULL
-      writer_thread = Thread.new do
-        Thread.current[:pl_telemetry] = writer_telemetry if @telemetry_enabled
-        loop do
-          batch = queue.pop
-          break if batch.nil?
+      # Writer loops
+      writer_telemetries = []
+      writer_threads = writers.map do |writer|
+        writer_telemetry = @telemetry_enabled ? Telemetry.new(enabled: true) : Telemetry::NULL
+        writer_telemetries << writer_telemetry
+        Thread.new(writer, writer_telemetry) do |worker_writer, worker_telemetry|
+          Thread.current[:pl_telemetry] = worker_telemetry if @telemetry_enabled
+          loop do
+            batch = queue.pop
+            break if batch.nil?
 
-          writer.write_many(batch)
+            worker_writer.write_many(batch)
+          end
+        ensure
+          worker_writer.close
         end
-      ensure
-        writer.close
       end
 
-      progress_thread = Thread.new do
-        Time.now
-        loop do
-          sleep 2
-          break unless @running
+      progress_thread = if @on_progress
+                          Thread.new do
+                            loop do
+                              @progress_mutex.synchronize { @progress_cv.wait(@progress_mutex, 2) }
+                              break unless @running
 
-          @on_progress&.call({ queue_bytes: queue.size_bytes })
-        end
-      end
+                              @on_progress.call({ queue_bytes: queue.size_bytes })
+                            end
+                          end
+                        end
 
       # Join readers
       readers.each(&:join)
       queue.close
-      writer_thread.join
-      @telemetry.merge!(writer_telemetry) if @telemetry_enabled
-      @running = false
-      progress_thread.join
+      writer_threads.each(&:join)
+      writer_telemetries.each { |writer_telemetry| @telemetry.merge!(writer_telemetry) } if @telemetry_enabled
+      if progress_thread
+        @progress_mutex.synchronize do
+          @running = false
+          @progress_cv.broadcast
+        end
+        progress_thread.join
+      else
+        @running = false
+      end
       if @telemetry_enabled
         total = @telemetry.timers.values.sum
         breakdown = @telemetry.timers
@@ -222,7 +250,7 @@ module Purplelight
       hint = @hint || filter_spec[:hint] || { _id: 1 }
 
       # Resume from checkpoint if present
-      checkpoint = manifest.partitions[idx] && manifest.partitions[idx]['last_id_exclusive']
+      checkpoint = manifest.partition_checkpoint(idx)
       if checkpoint
         filter = filter.dup
         filter['_id'] = (filter['_id'] || {}).merge({ '$gt' => checkpoint })
@@ -248,11 +276,12 @@ module Purplelight
       string_batch = +''
       buffer = []
       buffer_bytes = 0
+      buffer_rows = 0
       json_state = if encode_lines
                      JSON::Ext::Generator::State.new(ascii_only: false, max_nesting: false,
                                                      buffer_initial_length: 4_096)
                    end
-      size_state = encode_lines ? nil : JSON::Ext::Generator::State.new(ascii_only: false, max_nesting: false)
+      bson_size_buffer = BSON::ByteBuffer.new unless encode_lines
       last_id = checkpoint
       begin
         cursor.each do |doc|
@@ -266,37 +295,43 @@ module Purplelight
             string_batch << "\n"
             bytes = json.bytesize + 1
           else
-            # For CSV/Parquet keep raw docs to allow schema/row building
-            json = size_state.generate(doc)
-            bytes = json.bytesize + 1
+            previous_length = bson_size_buffer.length
+            doc.to_bson(bson_size_buffer)
+            bytes = bson_size_buffer.length - previous_length
             telemetry.finish(:serialize_time, t_ser)
             buffer << doc
           end
           buffer_bytes += bytes
-          # For JSONL, we count rows via newline accumulation; for others, use array length
-          ready = encode_lines ? (buffer_bytes >= 1_000_000 || (string_batch.length >= 1_000_000)) : (buffer.length >= batch_size || buffer_bytes >= 1_000_000)
+          buffer_rows += 1
+          # Flush on the configured row count or before a batch exceeds 1 MB.
+          ready = buffer_rows >= batch_size || buffer_bytes >= 1_000_000
           next unless ready
 
           t_q = telemetry.start(:queue_wait_time)
           if encode_lines
-            queue.push(string_batch, bytes: buffer_bytes)
+            batch = WriterJSONL::EncodedBatch.new(data: string_batch, rows: buffer_rows, bytes: buffer_bytes)
+            queue.push(batch, bytes: buffer_bytes)
             string_batch = +''
           else
             queue.push(buffer, bytes: buffer_bytes)
             buffer = []
+            bson_size_buffer = BSON::ByteBuffer.new
           end
           telemetry.finish(:queue_wait_time, t_q)
           manifest.update_partition_checkpoint!(idx, last_id)
           buffer_bytes = 0
+          buffer_rows = 0
         end
         if encode_lines
           unless string_batch.empty?
             t_q = telemetry.start(:queue_wait_time)
-            queue.push(string_batch, bytes: buffer_bytes)
+            batch = WriterJSONL::EncodedBatch.new(data: string_batch, rows: buffer_rows, bytes: buffer_bytes)
+            queue.push(batch, bytes: buffer_bytes)
             telemetry.finish(:queue_wait_time, t_q)
             manifest.update_partition_checkpoint!(idx, last_id)
             string_batch = +''
             buffer_bytes = 0
+            buffer_rows = 0
           end
         elsif !buffer.empty?
           t_q = telemetry.start(:queue_wait_time)
@@ -305,6 +340,7 @@ module Purplelight
           manifest.update_partition_checkpoint!(idx, last_id)
           buffer = []
           buffer_bytes = 0
+          buffer_rows = 0
         end
         manifest.mark_partition_complete!(idx)
       end

data/lib/purplelight/telemetry.rb

@@ -7,7 +7,6 @@ module Purplelight
       @enabled = enabled
       @counters = Hash.new(0)
       @timers = Hash.new(0.0)
-      @mutex = Mutex.new
     end
 
     def enabled?

data/lib/purplelight/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Purplelight
-  VERSION = '0.1.13'
+  VERSION = '0.1.16'
 end

data/lib/purplelight/writer_csv.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require 'csv'
 require 'json'
 require 'zlib'
 require 'fileutils'
 
+# simplecov:disable
 begin
   require 'zstd-ruby'
 rescue LoadError
@@ -14,14 +14,17 @@ rescue LoadError
     # no zstd backend; gzip fallback used later
   end
 end
+# simplecov:enable
 
 module Purplelight
   # WriterCSV writes documents to CSV files with optional compression.
   class WriterCSV
     DEFAULT_ROTATE_BYTES = 256 * 1024 * 1024
+    DEFAULT_ZSTD_LEVEL = 9
+    DEFAULT_GZIP_LEVEL = 1
 
     def initialize(directory:, prefix:, compression: :zstd, rotate_bytes: DEFAULT_ROTATE_BYTES, logger: nil,
-                   manifest: nil, single_file: false, columns: nil, headers: true)
+                   manifest: nil, single_file: false, columns: nil, headers: true, compression_level: nil)
       @directory = directory
       @prefix = prefix
       @compression = compression
@@ -29,7 +32,7 @@ module Purplelight
       @logger = logger
       @manifest = manifest
       env_level = ENV['PL_ZSTD_LEVEL']&.to_i
-      @compression_level = (env_level&.positive? ? env_level : nil)
+      @compression_level = compression_level || (env_level&.positive? ? env_level : nil)
       @single_file = single_file
 
       @columns = columns&.map(&:to_s)
@@ -37,10 +40,7 @@ module Purplelight
 
       @part_index = nil
       @io = nil
-      @csv = nil
-      @bytes_written = 0
-      @rows_written = 0
-      @file_seq = 0
+      @file_seq = manifest ? manifest.parts.length : 0
       @closed = false
 
       @effective_compression = determine_effective_compression(@compression)
@@ -52,22 +52,24 @@ module Purplelight
     def write_many(array_of_docs)
       ensure_open!
 
-      # infer columns if needed from docs
+      output = +''
+      # Infer columns if needed from docs.
       if @columns.nil?
         sample_docs = array_of_docs.is_a?(Array) ? array_of_docs : []
-        sample_docs = sample_docs.reject { |d| d.is_a?(String) }
+        sample_docs = sample_docs.grep_v(String)
         @columns = infer_columns(sample_docs)
-        @csv << @columns if @headers
+        append_csv_row(output, @columns) if @headers
       end
 
+      rows = 0
       array_of_docs.each do |doc|
         next if doc.is_a?(String)
 
-        row = @columns.map { |k| extract_value(doc, k) }
-        @csv << row
-        @rows_written += 1
+        append_csv_document(output, doc)
+        rows += 1
       end
-      @manifest&.add_progress_to_part!(index: @part_index, rows_delta: array_of_docs.size, bytes_delta: 0)
+      @io.write(output) unless output.empty?
+      @manifest&.add_progress_to_part!(index: @part_index, rows_delta: rows, bytes_delta: 0)
 
       rotate_if_needed
     end
@@ -76,7 +78,7 @@ module Purplelight
       return if @single_file
       return if @rotate_bytes.nil?
 
-      raw_bytes = @io.respond_to?(:pos) ? @io.pos : @bytes_written
+      raw_bytes = @io.respond_to?(:pos) ? @io.pos : @io.bytes_written
       return if raw_bytes < @rotate_bytes
 
       rotate!
@@ -85,7 +87,6 @@ module Purplelight
     def close
       return if @closed
 
-      @csv&.flush
       if @io
         t = Thread.current[:pl_telemetry]&.start(:rotate_time)
         finalize_current_part!
@@ -100,26 +101,20 @@ module Purplelight
     # Minimal wrapper to count bytes written for rotate logic when
     # underlying compressed writer doesn't expose position (e.g., zstd-ruby).
     class CountingIO
-      def initialize(io, on_write:)
+      attr_reader :bytes_written
+
+      def initialize(io)
         @io = io
-        @on_write = on_write
+        @bytes_written = 0
       end
 
       def write(data)
         bytes_written = @io.write(data)
-        @on_write.call(bytes_written) if bytes_written && @on_write
+        @bytes_written += bytes_written
         bytes_written
       end
 
-      # CSV calls '<<' on the underlying IO in some code paths
-      def <<(data)
-        write(data)
-      end
-
-      # CSV#flush may forward flush to underlying IO; make it a no-op if unavailable
-      def flush
-        @io.flush if @io.respond_to?(:flush)
-      end
+      alias << write
 
       def method_missing(method_name, *, &)
         @io.send(method_name, *, &)
@@ -135,31 +130,33 @@ module Purplelight
 
       FileUtils.mkdir_p(@directory)
       path = next_part_path
-      @part_index = @manifest&.open_part!(path) if @manifest
+      @part_index = @manifest.open_part!(path) if @manifest
+      # The compression stream owns and closes this file handle.
+      # rubocop:disable Style/FileOpen
       raw = File.open(path, 'wb')
+      # rubocop:enable Style/FileOpen
       compressed = build_compressed_io(raw)
-      @io = CountingIO.new(compressed, on_write: ->(n) { @bytes_written += n })
-      @csv = CSV.new(@io)
-      @bytes_written = 0
-      @rows_written = 0
+      @io = CountingIO.new(compressed)
+      return unless @headers && @columns
+
+      header = +''
+      append_csv_row(header, @columns)
+      @io.write(header)
     end
 
     def build_compressed_io(raw)
       case @effective_compression.to_s
       when 'zstd'
+        level = @compression_level || DEFAULT_ZSTD_LEVEL
         if Object.const_defined?(:Zstd) && defined?(::Zstd::StreamWriter)
-          level = @compression_level || 10
-          return ::Zstd::StreamWriter.new(raw, level: level)
-        elsif defined?(ZSTDS)
-          level = @compression_level || 10
-          return ZSTDS::Stream::Writer.new(raw, compression_level: level)
+          ::Zstd::StreamWriter.new(raw, level: level)
+        else
+          ZSTDS::Stream::Writer.new(raw, compression_level: level)
         end
 
-        @logger&.warn('zstd gem not loaded; using gzip')
-        Zlib::GzipWriter.new(raw)
-
       when 'gzip'
-        Zlib::GzipWriter.new(raw)
+        level = @compression_level || DEFAULT_GZIP_LEVEL
+        Zlib::GzipWriter.new(raw, level)
       when 'none'
         raw
       else
@@ -175,7 +172,6 @@ module Purplelight
       @io.close
       Thread.current[:pl_telemetry]&.finish(:rotate_time, t)
       @io = nil
-      @csv = nil
       ensure_open!
     end
 
@@ -216,6 +212,39 @@ module Purplelight
       :gzip
     end
 
+    def append_csv_document(output, document)
+      index = 0
+      while index < @columns.length
+        output << ',' unless index.zero?
+        append_csv_value(output, extract_value(document, @columns[index]))
+        index += 1
+      end
+      output << "\n"
+    end
+
+    def append_csv_row(output, values)
+      values.each_with_index do |value, index|
+        output << ',' unless index.zero?
+        append_csv_value(output, value)
+      end
+      output << "\n"
+    end
+
+    def append_csv_value(output, value)
+      return if value.nil?
+
+      string = value.to_s
+      contains_quote = string.include?('"')
+      unless string.empty? || contains_quote || string.include?(',') || string.include?("\n") || string.include?("\r")
+        output << string
+        return
+      end
+
+      output << '"'
+      output << (contains_quote ? string.gsub('"', '""') : string)
+      output << '"'
+    end
+
     def infer_columns(docs)
       keys = {}
       docs.each do |d|
@@ -223,12 +252,14 @@ module Purplelight
       end
       # Put _id first if present, then other keys sorted
       cols = []
-      cols << '_id' if docs.first.key?('_id') || docs.first.key?(:_id)
+      first = docs.first
+      cols << '_id' if first && (first.key?('_id') || first.key?(:_id))
       cols + keys.keys.sort
     end
 
     def extract_value(doc, key)
-      val = doc[key] || doc[key.to_sym]
+      val = doc[key]
+      val = doc[key.to_sym] if val.nil? && !doc.key?(key)
       case val
       when Hash, Array
         JSON.generate(val)

data/lib/purplelight/writer_jsonl.rb

@@ -4,6 +4,7 @@ require 'json'
 require 'zlib'
 require 'fileutils'
 
+# simplecov:disable
 begin
   require 'zstd-ruby'
 rescue LoadError
@@ -13,14 +14,34 @@ rescue LoadError
     # no zstd backend; gzip fallback
   end
 end
+# simplecov:enable
 
 module Purplelight
   # WriterJSONL writes newline-delimited JSON with optional compression.
   class WriterJSONL
     DEFAULT_ROTATE_BYTES = 256 * 1024 * 1024
+    DEFAULT_ZSTD_LEVEL = 3
+    DEFAULT_GZIP_LEVEL = 1
+    EncodedBatch = Data.define(:data, :rows, :bytes)
+
+    # Allocates globally unique output part numbers across writer threads.
+    class PartSequence
+      def initialize(next_value = 0)
+        @next_value = next_value
+        @mutex = Mutex.new
+      end
+
+      def next
+        @mutex.synchronize do
+          value = @next_value
+          @next_value += 1
+          value
+        end
+      end
+    end
 
     def initialize(directory:, prefix:, compression: :zstd, rotate_bytes: DEFAULT_ROTATE_BYTES, logger: nil,
-                   manifest: nil, compression_level: nil)
+                   manifest: nil, compression_level: nil, write_chunk_bytes: nil, part_sequence: nil)
       @directory = directory
       @prefix = prefix
       @compression = compression
@@ -29,18 +50,20 @@ module Purplelight
       @manifest = manifest
       env_level = ENV['PL_ZSTD_LEVEL']&.to_i
       @compression_level = compression_level || (env_level&.positive? ? env_level : nil)
+      @write_chunk_bytes = write_chunk_bytes
+      @part_sequence = part_sequence
 
       @part_index = nil
       @io = nil
       @bytes_written = 0
-      @rows_written = 0
-      @file_seq = 0
+      @file_seq = manifest ? manifest.parts.length : 0
       @closed = false
+      @thread_telemetry = false
 
       @effective_compression = determine_effective_compression(@compression)
       @json_state = JSON::Ext::Generator::State.new(ascii_only: false, max_nesting: false)
       if @logger
-        level_disp = @compression_level || (ENV['PL_ZSTD_LEVEL']&.to_i if @effective_compression.to_s == 'zstd')
+        level_disp = @compression_level
         @logger.info("WriterJSONL using compression='#{@effective_compression}' level='#{level_disp || 'default'}'")
       end
       return unless @effective_compression.to_s != @compression.to_s
@@ -51,12 +74,16 @@ module Purplelight
     def write_many(batch)
       ensure_open!
 
-      chunk_threshold = ENV['PL_WRITE_CHUNK_BYTES']&.to_i || (8 * 1024 * 1024)
+      chunk_threshold = @write_chunk_bytes || ENV['PL_WRITE_CHUNK_BYTES']&.to_i || (8 * 1024 * 1024)
       total_bytes = 0
       rows = 0
 
-      if batch.is_a?(String)
-        # Fast-path: writer received a preassembled buffer string
+      if batch.is_a?(EncodedBatch)
+        write_buffer(batch.data)
+        rows = batch.rows
+        total_bytes = batch.bytes
+      elsif batch.is_a?(String)
+        # Fast path for callers that don't provide row metadata.
         buffer = batch
         rows = buffer.count("\n")
         write_buffer(buffer)
@@ -104,7 +131,6 @@ module Purplelight
         end
       end
 
-      @rows_written += rows
       @manifest&.add_progress_to_part!(index: @part_index, rows_delta: rows, bytes_delta: total_bytes)
     end
 
@@ -132,30 +158,27 @@ module Purplelight
 
       FileUtils.mkdir_p(@directory)
       path = next_part_path
-      @part_index = @manifest&.open_part!(path) if @manifest
+      @part_index = @manifest.open_part!(path) if @manifest
+      # The compression stream owns and closes this file handle.
+      # rubocop:disable Style/FileOpen
       raw = File.open(path, 'wb')
+      # rubocop:enable Style/FileOpen
       @io = build_compressed_io(raw)
       @bytes_written = 0
-      @rows_written = 0
     end
 
     def build_compressed_io(raw)
       case @effective_compression.to_s
       when 'zstd'
         # Prefer zstd-ruby if available, else ruby-zstds
+        level = @compression_level || DEFAULT_ZSTD_LEVEL
         if Object.const_defined?(:Zstd) && defined?(::Zstd::StreamWriter)
-          level = @compression_level || 3
-          return ::Zstd::StreamWriter.new(raw, level: level)
-        elsif defined?(ZSTDS)
-          level = @compression_level || 3
-          return ZSTDS::Stream::Writer.new(raw, compression_level: level)
+          ::Zstd::StreamWriter.new(raw, level: level)
+        else
+          ZSTDS::Stream::Writer.new(raw, compression_level: level)
         end
-
-        @logger&.warn('zstd gems not loaded; falling back to gzip')
-        level = @compression_level || Zlib::DEFAULT_COMPRESSION
-        Zlib::GzipWriter.new(raw, level)
       when 'gzip'
-        level = @compression_level || 1
+        level = @compression_level || DEFAULT_GZIP_LEVEL
         Zlib::GzipWriter.new(raw, level)
       when 'none'
         raw
@@ -165,9 +188,18 @@ module Purplelight
     end
 
     def write_buffer(buffer)
-      t = Thread.current[:pl_telemetry]&.start(:write_time)
-      @io.write(buffer)
-      Thread.current[:pl_telemetry]&.finish(:write_time, t)
+      telemetry = @thread_telemetry
+      if telemetry.equal?(false)
+        telemetry = Thread.current[:pl_telemetry]
+        @thread_telemetry = telemetry
+      end
+      if telemetry
+        ticket = telemetry.start(:write_time)
+        @io.write(buffer)
+        telemetry.finish(:write_time, ticket)
+      else
+        @io.write(buffer)
+      end
       @bytes_written += buffer.bytesize
       rotate_if_needed
     end
@@ -192,7 +224,8 @@ module Purplelight
 
     def next_part_path
       ext = 'jsonl'
-      filename = format('%<prefix>s-part-%<seq>06d.%<ext>s', prefix: @prefix, seq: @file_seq, ext: ext)
+      sequence = @part_sequence ? @part_sequence.next : @file_seq
+      filename = format('%<prefix>s-part-%<seq>06d.%<ext>s', prefix: @prefix, seq: sequence, ext: ext)
       filename += '.zst' if @effective_compression.to_s == 'zstd'
       filename += '.gz' if @effective_compression.to_s == 'gzip'
       File.join(@directory, filename)