purplelight 0.1.4 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 707f5c45aade0603b8a3c500e91015b9850c2c92deb91492f06cb3f8bbee76d5
-  data.tar.gz: dd6217a55fa5bcd3f008699bd8710e8e0ac72f9bb0b8bf5dd71d2f2eccbc5d84
+  metadata.gz: 941ef04ede346a29c0afe19a02d69bd9a25d08ce241e21c6c47960498f4a42c6
+  data.tar.gz: c34b089d2842082f5d4be60c96d58e208e079f92712d397726a964a7361ae895
 SHA512:
-  metadata.gz: 23da05fd59362787069ae1df5168d85c210d2c7cfc0fd254c064cae26212ee177b84dff8c72ba825a612b3aef3072188e54fad267c60331105dc026e96b42d50
-  data.tar.gz: 8b8a728f0002f60d55c31270ee2751aaddab422777aa2a51d6a8b8d3a25a93fe2be27d218316b6fd077103d2901f7561b4983a27a27fde895b80fdc6ee67089f
+  metadata.gz: 8d666cb565283e6410fb0412d8ed369db18b0eaa0e80c573d677295910319b809396e99fca6627214d15942d531c9ac378c7d5a850842a7124d2ff4c9b03d7d8
+  data.tar.gz: 9188cd0b55e0d3e54fb2d209b85eeeacfb1c0d8daa73b1ace0fbb3bf4a2af865d6dd5c125f1e487458f096077a70cc93e7c0191c1222f6edd58701487f5479da

data/README.md

@@ -9,7 +9,7 @@ Purplelight is published on RubyGems: [purplelight on RubyGems](https://rubygems
 Add to your Gemfile:
 
 ```ruby
-gem 'purplelight', '~> 0.1.4'
+gem 'purplelight', '~> 0.1.6'
 ```
 
 Or install directly:

data/bin/purplelight

@@ -40,6 +40,7 @@ parser = OptionParser.new do |opts|
     options[:sharding][:prefix] = v
   end
   opts.on('-q', '--query JSON', 'Filter query as JSON') { |v| options[:query] = JSON.parse(v) }
+  opts.on('--projection JSON', 'Projection as JSON (e.g., {"_id":1,"field":1})') { |v| options[:projection] = JSON.parse(v) }
   opts.on('--read-preference MODE',
           'Read preference mode: primary|primary_preferred|secondary|secondary_preferred|nearest') do |v|
     options[:read_preference] = v.to_sym
@@ -106,6 +107,7 @@ ok = Purplelight.snapshot(
   partitions: options[:partitions],
   batch_size: options[:batch_size],
   query: options[:query],
+  projection: options[:projection],
   sharding: options[:sharding],
   read_preference: effective_read || options[:read_preference],
   resume: { enabled: true },

data/lib/purplelight/partitioner.rb

@@ -11,7 +11,86 @@ module Purplelight
   class Partitioner
     # Builds contiguous _id range filters for N partitions.
     # For ObjectId _id, we sample quantiles to split into near-equal document counts.
-    def self.object_id_partitions(collection:, query:, partitions:)
+    def self.object_id_partitions(collection:, query:, partitions:, mode: nil, telemetry: nil)
+      # Choose planning mode: :timestamp (fast), :cursor (legacy)
+      chosen_mode = (mode || ENV['PL_PARTITIONER_MODE'] || :timestamp).to_sym
+      telemetry ||= (defined?(Telemetry) ? Telemetry::NULL : nil)
+
+      return cursor_sampling_partitions(collection: collection, query: query, partitions: partitions) if chosen_mode == :cursor
+
+      timestamp_partitions(collection: collection, query: query, partitions: partitions, telemetry: telemetry)
+    end
+
+    def self.simple_ranges(collection:, query:, partitions:)
+      # Split by _id quantiles using min/max endpoints
+      min_id = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(1).first&.dig('_id')
+      max_id = collection.find(query || {}).projection(_id: 1).sort(_id: -1).limit(1).first&.dig('_id')
+      return [{ filter: query || {}, sort: { _id: 1 } }] if min_id.nil? || max_id.nil?
+
+      # Create numeric-ish interpolation by sampling
+      ids = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(partitions - 1).to_a.map { |d| d['_id'] }
+      boundaries = [min_id] + ids + [max_id]
+      ranges = []
+      boundaries.each_cons(2) do |a, b|
+        ranges << build_range(a, b)
+      end
+
+      ranges.map do |r|
+        filter = query ? query.dup : {}
+        filter['_id'] = r
+        { filter: filter, sort: { _id: 1 }, hint: { _id: 1 } }
+      end
+    end
+
+    # Faster planning using ObjectId timestamps: O(partitions) indexed lookups
+    def self.timestamp_partitions(collection:, query:, partitions:, telemetry: nil)
+      t_minmax = telemetry&.start(:plan_minmax_time)
+      min_id = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(1).first&.dig('_id')
+      max_id = collection.find(query || {}).projection(_id: 1).sort(_id: -1).limit(1).first&.dig('_id')
+      telemetry&.finish(:plan_minmax_time, t_minmax)
+
+      return [{ filter: query || {}, sort: { _id: 1 } }] if min_id.nil? || max_id.nil?
+
+      min_ts = min_id.respond_to?(:generation_time) ? min_id.generation_time.to_i : nil
+      max_ts = max_id.respond_to?(:generation_time) ? max_id.generation_time.to_i : nil
+
+      # Fallback to cursor sampling if _id isn't an ObjectId
+      return cursor_sampling_partitions(collection: collection, query: query, partitions: partitions) if min_ts.nil? || max_ts.nil? || max_ts <= min_ts
+
+      step = [(max_ts - min_ts) / partitions, 1].max
+      inner_boundaries = []
+      t_boundaries = telemetry&.start(:plan_boundary_queries_time)
+      1.upto(partitions - 1) do |i|
+        target_ts = min_ts + (step * i)
+        candidate = BSON::ObjectId.from_time(Time.at(target_ts))
+        f = query ? query.dup : {}
+        f['_id'] = { '$gt' => candidate }
+        b = collection.find(f).projection(_id: 1).sort(_id: 1).hint(_id: 1).limit(1).first&.dig('_id')
+        inner_boundaries << b if b
+      end
+      telemetry&.finish(:plan_boundary_queries_time, t_boundaries)
+
+      # Build ranges: first range has nil lower bound to include min_id,
+      # middle ranges are (prev, current], and last is (last, +inf)
+      ranges = []
+      t_ranges = telemetry&.start(:plan_ranges_build_time)
+      prev = nil
+      inner_boundaries.each do |b|
+        ranges << build_range(prev, b)
+        prev = b
+      end
+      ranges << build_range(prev, nil)
+      telemetry&.finish(:plan_ranges_build_time, t_ranges)
+
+      ranges.map do |r|
+        filter = query ? query.dup : {}
+        filter['_id'] = r
+        { filter: filter, sort: { _id: 1 }, hint: { _id: 1 } }
+      end
+    end
+
+    # Legacy cursor sampling planner
+    def self.cursor_sampling_partitions(collection:, query:, partitions:)
       # Ensure sort order for sampling
       base_query = collection.find(query || {}, {}.merge(sort: { _id: 1 }))
 
@@ -49,27 +128,6 @@ module Purplelight
       end
     end
 
-    def self.simple_ranges(collection:, query:, partitions:)
-      # Split by _id quantiles using min/max endpoints
-      min_id = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(1).first&.dig('_id')
-      max_id = collection.find(query || {}).projection(_id: 1).sort(_id: -1).limit(1).first&.dig('_id')
-      return [{ filter: query || {}, sort: { _id: 1 } }] if min_id.nil? || max_id.nil?
-
-      # Create numeric-ish interpolation by sampling
-      ids = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(partitions - 1).to_a.map { |d| d['_id'] }
-      boundaries = [min_id] + ids + [max_id]
-      ranges = []
-      boundaries.each_cons(2) do |a, b|
-        ranges << build_range(a, b)
-      end
-
-      ranges.map do |r|
-        filter = query ? query.dup : {}
-        filter['_id'] = r
-        { filter: filter, sort: { _id: 1 }, hint: { _id: 1 } }
-      end
-    end
-
     def self.build_range(from_id, to_id)
       if from_id && to_id
         { '$gt' => from_id, '$lte' => to_id }

data/lib/purplelight/snapshot.rb

@@ -10,6 +10,7 @@ require_relative 'writer_csv'
 require_relative 'writer_parquet'
 require_relative 'manifest'
 require_relative 'errors'
+require_relative 'telemetry'
 
 module Purplelight
   # Snapshot orchestrates partition planning, parallel reads, and writing.
@@ -37,7 +38,7 @@ module Purplelight
                    resume: { enabled: true, state_path: nil, overwrite_incompatible: false },
                    sharding: { mode: :by_size, part_bytes: DEFAULTS[:rotate_bytes], prefix: nil },
                    logger: nil, on_progress: nil, read_concern: DEFAULTS[:read_concern], read_preference: DEFAULTS[:read_preference],
-                   no_cursor_timeout: DEFAULTS[:no_cursor_timeout])
+                   no_cursor_timeout: DEFAULTS[:no_cursor_timeout], telemetry: nil)
       @client = client
       @collection = client[collection]
       @output = output
@@ -60,6 +61,10 @@ module Purplelight
       @no_cursor_timeout = no_cursor_timeout
 
       @running = true
+      @telemetry_enabled = telemetry ? telemetry.enabled? : (ENV['PL_TELEMETRY'] == '1')
+      @telemetry = telemetry || (
+        @telemetry_enabled ? Telemetry.new(enabled: true) : Telemetry::NULL
+      )
     end
 
     # rubocop:disable Naming/PredicateMethod
@@ -90,8 +95,10 @@ module Purplelight
       manifest.ensure_partitions!(@partitions)
 
       # Plan partitions
+      t_plan = @telemetry.start(:partition_plan_time)
       partition_filters = Partitioner.object_id_partitions(collection: @collection, query: @query,
-                                                           partitions: @partitions)
+                                                           partitions: @partitions, telemetry: @telemetry)
+      @telemetry.finish(:partition_plan_time, t_plan)
 
       # Reader queue
       queue = ByteQueue.new(max_bytes: @queue_size_bytes)
@@ -116,12 +123,17 @@ module Purplelight
       # Start reader threads
       readers = partition_filters.each_with_index.map do |pf, idx|
         Thread.new do
-          read_partition(idx: idx, filter_spec: pf, queue: queue, batch_size: @batch_size, manifest: manifest)
+          local_telemetry = @telemetry_enabled ? Telemetry.new(enabled: true) : Telemetry::NULL
+          read_partition(idx: idx, filter_spec: pf, queue: queue, batch_size: @batch_size, manifest: manifest, telemetry: local_telemetry)
+          # Merge per-thread telemetry
+          @telemetry.merge!(local_telemetry) if @telemetry_enabled
         end
       end
 
       # Writer loop
+      writer_telemetry = @telemetry_enabled ? Telemetry.new(enabled: true) : Telemetry::NULL
       writer_thread = Thread.new do
+        Thread.current[:pl_telemetry] = writer_telemetry if @telemetry_enabled
         loop do
           batch = queue.pop
           break if batch.nil?
@@ -146,8 +158,22 @@ module Purplelight
       readers.each(&:join)
       queue.close
       writer_thread.join
+      @telemetry.merge!(writer_telemetry) if @telemetry_enabled
       @running = false
       progress_thread.join
+      if @telemetry_enabled
+        total = @telemetry.timers.values.sum
+        breakdown = @telemetry.timers
+                              .sort_by { |_k, v| -v }
+                              .map { |k, v| [k, v, total.zero? ? 0 : ((v / total) * 100.0)] }
+        if @logger
+          @logger.info('Telemetry (seconds and % of timed work):')
+          breakdown.each { |k, v, pct| @logger.info("  #{k}: #{v.round(3)}s (#{pct.round(1)}%)") }
+        else
+          puts 'Telemetry (seconds and % of timed work):'
+          breakdown.each { |k, v, pct| puts "  #{k}: #{v.round(3)}s (#{pct.round(1)}%)" }
+        end
+      end
       true
     end
     # rubocop:enable Naming/PredicateMethod
@@ -167,7 +193,7 @@ module Purplelight
       [dir, prefix]
     end
 
-    def read_partition(idx:, filter_spec:, queue:, batch_size:, manifest:)
+    def read_partition(idx:, filter_spec:, queue:, batch_size:, manifest:, telemetry: Telemetry::NULL)
       filter = filter_spec[:filter]
       sort = filter_spec[:sort] || { _id: 1 }
       hint = @hint || filter_spec[:hint] || { _id: 1 }
@@ -195,6 +221,8 @@ module Purplelight
       cursor = @collection.find(filter, opts)
 
       encode_lines = (@format == :jsonl)
+      # When JSONL, build one big string per batch to offload join cost from writer.
+      string_batch = +''
       buffer = []
       buffer_bytes = 0
       last_id = checkpoint
@@ -202,25 +230,48 @@ module Purplelight
         cursor.each do |doc|
           last_id = doc['_id']
           doc = @mapper.call(doc) if @mapper
+          t_ser = telemetry.start(:serialize_time)
           if encode_lines
-            line = "#{JSON.generate(doc)}\n"
+            line = "#{JSON.fast_generate(doc)}\n"
+            telemetry.finish(:serialize_time, t_ser)
             bytes = line.bytesize
-            buffer << line
+            string_batch << line
           else
             # For CSV/Parquet keep raw docs to allow schema/row building
-            bytes = (JSON.generate(doc).bytesize + 1)
+            bytes = (JSON.fast_generate(doc).bytesize + 1)
+            telemetry.finish(:serialize_time, t_ser)
             buffer << doc
           end
           buffer_bytes += bytes
-          next unless buffer.length >= batch_size || buffer_bytes >= 1_000_000
+          # For JSONL, we count rows via newline accumulation; for others, use array length
+          ready = encode_lines ? (buffer_bytes >= 1_000_000 || (string_batch.length >= 1_000_000)) : (buffer.length >= batch_size || buffer_bytes >= 1_000_000)
+          next unless ready
 
-          queue.push(buffer, bytes: buffer_bytes)
+          t_q = telemetry.start(:queue_wait_time)
+          if encode_lines
+            queue.push(string_batch, bytes: buffer_bytes)
+            string_batch = +''
+          else
+            queue.push(buffer, bytes: buffer_bytes)
+            buffer = []
+          end
+          telemetry.finish(:queue_wait_time, t_q)
           manifest.update_partition_checkpoint!(idx, last_id)
-          buffer = []
           buffer_bytes = 0
         end
-        unless buffer.empty?
+        if encode_lines
+          unless string_batch.empty?
+            t_q = telemetry.start(:queue_wait_time)
+            queue.push(string_batch, bytes: buffer_bytes)
+            telemetry.finish(:queue_wait_time, t_q)
+            manifest.update_partition_checkpoint!(idx, last_id)
+            string_batch = +''
+            buffer_bytes = 0
+          end
+        elsif !buffer.empty?
+          t_q = telemetry.start(:queue_wait_time)
           queue.push(buffer, bytes: buffer_bytes)
+          telemetry.finish(:queue_wait_time, t_q)
           manifest.update_partition_checkpoint!(idx, last_id)
           buffer = []
           buffer_bytes = 0

data/lib/purplelight/telemetry.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Purplelight
+  # Lightweight, low-overhead timing and counters with a ticket API.
+  class Telemetry
+    def initialize(enabled: true)
+      @enabled = enabled
+      @counters = Hash.new(0)
+      @timers = Hash.new(0.0)
+      @mutex = Mutex.new
+    end
+
+    def enabled?
+      @enabled
+    end
+
+    # Start a timer. Returns a ticket (Float) when enabled, or nil when disabled.
+    def start(_key)
+      return nil unless @enabled
+
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    # Finish a timer using a ticket from start. No-ops if ticket is nil.
+    def finish(key, ticket)
+      return unless @enabled && ticket
+
+      dt = Process.clock_gettime(Process::CLOCK_MONOTONIC) - ticket
+      @timers[key] += dt
+    end
+
+    def add(key, count = 1)
+      return unless @enabled
+
+      @counters[key] += count
+    end
+
+    def merge!(other)
+      return self unless @enabled
+
+      other.counters.each { |k, v| @counters[k] += v }
+      other.timers.each { |k, v| @timers[k] += v }
+      self
+    end
+
+    attr_reader :counters, :timers
+
+    # A disabled singleton for zero overhead checks if needed.
+    NULL = new(enabled: false)
+  end
+end

data/lib/purplelight/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Purplelight
-  VERSION = '0.1.4'
+  VERSION = '0.1.6'
 end

data/lib/purplelight/writer_csv.rb

@@ -11,6 +11,12 @@ rescue LoadError
   # zstd not available; fallback handled later via gzip
 end
 
+begin
+  require 'zstd-ruby'
+rescue LoadError
+  # alternative zstd gem not available
+end
+
 module Purplelight
   # WriterCSV writes documents to CSV files with optional compression.
   class WriterCSV
@@ -24,6 +30,8 @@ module Purplelight
       @rotate_bytes = rotate_bytes
       @logger = logger
       @manifest = manifest
+      env_level = ENV['PL_ZSTD_LEVEL']&.to_i
+      @compression_level = (env_level && env_level > 0 ? env_level : nil)
       @single_file = single_file
 
       @columns = columns&.map(&:to_s)
@@ -81,14 +89,49 @@ module Purplelight
 
       @csv&.flush
       if @io
+        t = Thread.current[:pl_telemetry]&.start(:rotate_time)
         finalize_current_part!
         @io.close
+        Thread.current[:pl_telemetry]&.finish(:rotate_time, t)
       end
       @closed = true
     end
 
     private
 
+    # Minimal wrapper to count bytes written for rotate logic when
+    # underlying compressed writer doesn't expose position (e.g., zstd-ruby).
+    class CountingIO
+      def initialize(io, on_write:)
+        @io = io
+        @on_write = on_write
+      end
+
+      def write(data)
+        bytes_written = @io.write(data)
+        @on_write.call(bytes_written) if bytes_written && @on_write
+        bytes_written
+      end
+
+      # CSV calls '<<' on the underlying IO in some code paths
+      def <<(data)
+        write(data)
+      end
+
+      # CSV#flush may forward flush to underlying IO; make it a no-op if unavailable
+      def flush
+        @io.flush if @io.respond_to?(:flush)
+      end
+
+      def method_missing(method_name, *args, &block)
+        @io.send(method_name, *args, &block)
+      end
+
+      def respond_to_missing?(method_name, include_private = false)
+        @io.respond_to?(method_name, include_private)
+      end
+    end
+
     def ensure_open!
       return if @io
 
@@ -96,7 +139,8 @@ module Purplelight
       path = next_part_path
       @part_index = @manifest&.open_part!(path) if @manifest
       raw = File.open(path, 'wb')
-      @io = build_compressed_io(raw)
+      compressed = build_compressed_io(raw)
+      @io = CountingIO.new(compressed, on_write: ->(n) { @bytes_written += n })
       @csv = CSV.new(@io)
       @bytes_written = 0
       @rows_written = 0
@@ -105,7 +149,13 @@ module Purplelight
     def build_compressed_io(raw)
       case @effective_compression.to_s
       when 'zstd'
-        return ZSTDS::Writer.open(raw, level: 10) if defined?(ZSTDS)
+        if Object.const_defined?(:Zstd) && defined?(::Zstd::StreamWriter)
+          level = @compression_level || 10
+          return ::Zstd::StreamWriter.new(raw, level: level)
+        elsif defined?(ZSTDS)
+          level = @compression_level || 10
+          return ZSTDS::Stream::Writer.new(raw, compression_level: level)
+        end
 
         @logger&.warn('zstd gem not loaded; using gzip')
         Zlib::GzipWriter.new(raw)
@@ -122,8 +172,10 @@ module Purplelight
     def rotate!
       return unless @io
 
+      t = Thread.current[:pl_telemetry]&.start(:rotate_time)
       finalize_current_part!
       @io.close
+      Thread.current[:pl_telemetry]&.finish(:rotate_time, t)
       @io = nil
       @csv = nil
       ensure_open!
@@ -150,7 +202,7 @@ module Purplelight
     def determine_effective_compression(requested)
       case requested.to_s
       when 'zstd'
-        (defined?(ZSTDS) ? :zstd : :gzip)
+        ((defined?(ZSTDS) || (Object.const_defined?(:Zstd) && defined?(::Zstd::StreamWriter))) ? :zstd : :gzip)
       when 'none'
         :none
       else

data/lib/purplelight/writer_jsonl.rb

@@ -10,6 +10,12 @@ rescue LoadError
   # zstd not available; will fallback to gzip
 end
 
+begin
+  require 'zstd-ruby'
+rescue LoadError
+  # alternative zstd gem not available
+end
+
 module Purplelight
   # WriterJSONL writes newline-delimited JSON with optional compression.
   class WriterJSONL
@@ -23,7 +29,8 @@ module Purplelight
       @rotate_bytes = rotate_bytes
       @logger = logger
       @manifest = manifest
-      @compression_level = compression_level
+      env_level = ENV['PL_ZSTD_LEVEL']&.to_i
+      @compression_level = compression_level || (env_level && env_level > 0 ? env_level : nil)
 
       @part_index = nil
       @io = nil
@@ -33,23 +40,71 @@ module Purplelight
       @closed = false
 
       @effective_compression = determine_effective_compression(@compression)
-      return unless @effective_compression.to_s != @compression.to_s
-
-      @logger&.warn("requested compression '#{@compression}' not available; using '#{@effective_compression}'")
+      if @logger
+        level_disp = @compression_level || (ENV['PL_ZSTD_LEVEL']&.to_i if @effective_compression.to_s == 'zstd')
+        @logger.info("WriterJSONL using compression='#{@effective_compression}' level='#{level_disp || 'default'}'")
+      end
+      if @effective_compression.to_s != @compression.to_s
+        @logger&.warn("requested compression '#{@compression}' not available; using '#{@effective_compression}'")
+      end
     end
 
-    def write_many(array_of_docs)
+    def write_many(batch)
       ensure_open!
-      # If upstream already produced newline-terminated strings, join fast.
-      buffer = if array_of_docs.first.is_a?(String)
-                 array_of_docs.join
-               else
-                 array_of_docs.map { |doc| "#{JSON.generate(doc)}\n" }.join
-               end
-      rows = array_of_docs.size
-      write_buffer(buffer)
+
+      chunk_threshold = (ENV['PL_WRITE_CHUNK_BYTES']&.to_i || (8 * 1024 * 1024))
+      total_bytes = 0
+      rows = 0
+
+      if batch.is_a?(String)
+        # Fast-path: writer received a preassembled buffer string
+        buffer = batch
+        rows = buffer.count("\n")
+        write_buffer(buffer)
+        total_bytes = buffer.bytesize
+      elsif batch.first.is_a?(String)
+        # Join and write in chunks to avoid large intermediate allocations
+        chunk = +''
+        chunk_bytes = 0
+        batch.each do |line|
+          chunk << line
+          rows += 1
+          chunk_bytes += line.bytesize
+          next unless chunk_bytes >= chunk_threshold
+
+          write_buffer(chunk)
+          total_bytes += chunk.bytesize
+          chunk = +''
+          chunk_bytes = 0
+        end
+        unless chunk.empty?
+          write_buffer(chunk)
+          total_bytes += chunk.bytesize
+        end
+      else
+        # Fallback: encode docs here (JSON.fast_generate preferred) and write in chunks
+        chunk = +''
+        chunk_bytes = 0
+        batch.each do |doc|
+          line = "#{JSON.fast_generate(doc)}\n"
+          rows += 1
+          chunk << line
+          chunk_bytes += line.bytesize
+          next unless chunk_bytes >= chunk_threshold
+
+          write_buffer(chunk)
+          total_bytes += chunk.bytesize
+          chunk = +''
+          chunk_bytes = 0
+        end
+        unless chunk.empty?
+          write_buffer(chunk)
+          total_bytes += chunk.bytesize
+        end
+      end
+
       @rows_written += rows
-      @manifest&.add_progress_to_part!(index: @part_index, rows_delta: rows, bytes_delta: buffer.bytesize)
+      @manifest&.add_progress_to_part!(index: @part_index, rows_delta: rows, bytes_delta: total_bytes)
     end
 
     def rotate_if_needed
@@ -86,15 +141,18 @@ module Purplelight
     def build_compressed_io(raw)
       case @effective_compression.to_s
       when 'zstd'
-        if defined?(ZSTDS)
-          # ZSTDS::Writer supports IO-like interface
+        # Prefer zstd-ruby if available, else ruby-zstds
+        if Object.const_defined?(:Zstd) && defined?(::Zstd::StreamWriter)
+          level = @compression_level || 3
+          return ::Zstd::StreamWriter.new(raw, level: level)
+        elsif defined?(ZSTDS)
           level = @compression_level || 3
-          ZSTDS::Writer.open(raw, level: level)
-        else
-          @logger&.warn('zstd gem not loaded; this should have been handled earlier')
-          level = @compression_level || Zlib::DEFAULT_COMPRESSION
-          Zlib::GzipWriter.new(raw, level)
+          return ZSTDS::Stream::Writer.new(raw, compression_level: level)
         end
+
+        @logger&.warn('zstd gems not loaded; falling back to gzip')
+        level = @compression_level || Zlib::DEFAULT_COMPRESSION
+        Zlib::GzipWriter.new(raw, level)
       when 'gzip'
         level = @compression_level || 1
         Zlib::GzipWriter.new(raw, level)
@@ -106,7 +164,9 @@ module Purplelight
     end
 
     def write_buffer(buffer)
+      t = Thread.current[:pl_telemetry]&.start(:write_time)
       @io.write(buffer)
+      Thread.current[:pl_telemetry]&.finish(:write_time, t)
       @bytes_written += buffer.bytesize
       rotate_if_needed
     end
@@ -114,8 +174,10 @@ module Purplelight
     def rotate!
       return unless @io
 
+      t = Thread.current[:pl_telemetry]&.start(:rotate_time)
       finalize_current_part!
       @io.close
+      Thread.current[:pl_telemetry]&.finish(:rotate_time, t)
       @io = nil
       ensure_open!
     end
@@ -138,7 +200,7 @@ module Purplelight
     def determine_effective_compression(requested)
       case requested.to_s
       when 'zstd'
-        (defined?(ZSTDS) ? :zstd : :gzip)
+        ((defined?(ZSTDS) || (Object.const_defined?(:Zstd) && defined?(::Zstd::StreamWriter))) ? :zstd : :gzip)
       when 'none'
         :none
       else

data/lib/purplelight/writer_parquet.rb

@@ -44,8 +44,13 @@ module Purplelight
 
       ensure_open!
       unless @buffer_docs.empty?
+        t_tbl = Thread.current[:pl_telemetry]&.start(:parquet_table_build_time)
         table = build_table(@buffer_docs)
+        Thread.current[:pl_telemetry]&.finish(:parquet_table_build_time, t_tbl)
+
+        t_w = Thread.current[:pl_telemetry]&.start(:parquet_write_time)
         write_table(table, @writer_path, append: false)
+        Thread.current[:pl_telemetry]&.finish(:parquet_write_time, t_w)
       end
       finalize_current_part!
       @closed = true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: purplelight
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.6
 platform: ruby
 authors:
 - Alexander Nicholson
@@ -13,44 +13,62 @@ dependencies:
   name: csv
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.3.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.3.5
 - !ruby/object:Gem::Dependency
   name: logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: 1.7.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: 1.7.0
 - !ruby/object:Gem::Dependency
   name: mongo
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.21'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.19'
+        version: 2.21.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.21'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.19'
+        version: 2.21.3
 description: High-throughput, resumable snapshots of MongoDB collections with partitioning,
   multi-threaded readers, and size-based sharded outputs.
 email:
@@ -70,15 +88,16 @@ files:
 - lib/purplelight/partitioner.rb
 - lib/purplelight/queue.rb
 - lib/purplelight/snapshot.rb
+- lib/purplelight/telemetry.rb
 - lib/purplelight/version.rb
 - lib/purplelight/writer_csv.rb
 - lib/purplelight/writer_jsonl.rb
 - lib/purplelight/writer_parquet.rb
+homepage: https://github.com/alexandernicholson/purplelight
 licenses:
 - MIT
 metadata:
   rubygems_mfa_required: 'true'
-  homepage_uri: https://github.com/alexandernicholson/purplelight
   source_code_uri: https://github.com/alexandernicholson/purplelight
   changelog_uri: https://github.com/alexandernicholson/purplelight/releases
 rdoc_options: []