purplelight 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 707f5c45aade0603b8a3c500e91015b9850c2c92deb91492f06cb3f8bbee76d5
-  data.tar.gz: dd6217a55fa5bcd3f008699bd8710e8e0ac72f9bb0b8bf5dd71d2f2eccbc5d84
+  metadata.gz: a650fdd2113129b151396a1a90a83a6f1ede97eb5c34c60e028eb7639d5cc4fd
+  data.tar.gz: 1ab1bc421ddaf1c457639ae2ac5968245c6141b8504897ab762a49bb69c51a69
 SHA512:
-  metadata.gz: 23da05fd59362787069ae1df5168d85c210d2c7cfc0fd254c064cae26212ee177b84dff8c72ba825a612b3aef3072188e54fad267c60331105dc026e96b42d50
-  data.tar.gz: 8b8a728f0002f60d55c31270ee2751aaddab422777aa2a51d6a8b8d3a25a93fe2be27d218316b6fd077103d2901f7561b4983a27a27fde895b80fdc6ee67089f
+  metadata.gz: 506e52dce7c474998c8bc4b9afa9f5140349e8e2eed2eed7cccbacac0bd9d9f41528b234f1b99ad8407674791471368ee5b99d93b7ab058522311f2642006a20
+  data.tar.gz: 5c17e387f0d67a21d1351cf4e1e69beaa7beecdf5b9f8011e16bd740e990902abf32c54b02257cdead9c296241557571608b6004446cca5d429675fda07ff61a

data/README.md

@@ -9,7 +9,7 @@ Purplelight is published on RubyGems: [purplelight on RubyGems](https://rubygems
 Add to your Gemfile:
 
 ```ruby
-gem 'purplelight', '~> 0.1.4'
+gem 'purplelight', '~> 0.1.5'
 ```
 
 Or install directly:

data/lib/purplelight/partitioner.rb

@@ -11,7 +11,86 @@ module Purplelight
   class Partitioner
     # Builds contiguous _id range filters for N partitions.
     # For ObjectId _id, we sample quantiles to split into near-equal document counts.
-    def self.object_id_partitions(collection:, query:, partitions:)
+    def self.object_id_partitions(collection:, query:, partitions:, mode: nil, telemetry: nil)
+      # Choose planning mode: :timestamp (fast), :cursor (legacy)
+      chosen_mode = (mode || ENV['PL_PARTITIONER_MODE'] || :timestamp).to_sym
+      telemetry ||= (defined?(Telemetry) ? Telemetry::NULL : nil)
+
+      return cursor_sampling_partitions(collection: collection, query: query, partitions: partitions) if chosen_mode == :cursor
+
+      timestamp_partitions(collection: collection, query: query, partitions: partitions, telemetry: telemetry)
+    end
+
+    def self.simple_ranges(collection:, query:, partitions:)
+      # Split by _id quantiles using min/max endpoints
+      min_id = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(1).first&.dig('_id')
+      max_id = collection.find(query || {}).projection(_id: 1).sort(_id: -1).limit(1).first&.dig('_id')
+      return [{ filter: query || {}, sort: { _id: 1 } }] if min_id.nil? || max_id.nil?
+
+      # Create numeric-ish interpolation by sampling
+      ids = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(partitions - 1).to_a.map { |d| d['_id'] }
+      boundaries = [min_id] + ids + [max_id]
+      ranges = []
+      boundaries.each_cons(2) do |a, b|
+        ranges << build_range(a, b)
+      end
+
+      ranges.map do |r|
+        filter = query ? query.dup : {}
+        filter['_id'] = r
+        { filter: filter, sort: { _id: 1 }, hint: { _id: 1 } }
+      end
+    end
+
+    # Faster planning using ObjectId timestamps: O(partitions) indexed lookups
+    def self.timestamp_partitions(collection:, query:, partitions:, telemetry: nil)
+      t_minmax = telemetry&.start(:plan_minmax_time)
+      min_id = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(1).first&.dig('_id')
+      max_id = collection.find(query || {}).projection(_id: 1).sort(_id: -1).limit(1).first&.dig('_id')
+      telemetry&.finish(:plan_minmax_time, t_minmax)
+
+      return [{ filter: query || {}, sort: { _id: 1 } }] if min_id.nil? || max_id.nil?
+
+      min_ts = min_id.respond_to?(:generation_time) ? min_id.generation_time.to_i : nil
+      max_ts = max_id.respond_to?(:generation_time) ? max_id.generation_time.to_i : nil
+
+      # Fallback to cursor sampling if _id isn't an ObjectId
+      return cursor_sampling_partitions(collection: collection, query: query, partitions: partitions) if min_ts.nil? || max_ts.nil? || max_ts <= min_ts
+
+      step = [(max_ts - min_ts) / partitions, 1].max
+      inner_boundaries = []
+      t_boundaries = telemetry&.start(:plan_boundary_queries_time)
+      1.upto(partitions - 1) do |i|
+        target_ts = min_ts + (step * i)
+        candidate = BSON::ObjectId.from_time(Time.at(target_ts))
+        f = query ? query.dup : {}
+        f['_id'] = { '$gt' => candidate }
+        b = collection.find(f).projection(_id: 1).sort(_id: 1).hint(_id: 1).limit(1).first&.dig('_id')
+        inner_boundaries << b if b
+      end
+      telemetry&.finish(:plan_boundary_queries_time, t_boundaries)
+
+      # Build ranges: first range has nil lower bound to include min_id,
+      # middle ranges are (prev, current], and last is (last, +inf)
+      ranges = []
+      t_ranges = telemetry&.start(:plan_ranges_build_time)
+      prev = nil
+      inner_boundaries.each do |b|
+        ranges << build_range(prev, b)
+        prev = b
+      end
+      ranges << build_range(prev, nil)
+      telemetry&.finish(:plan_ranges_build_time, t_ranges)
+
+      ranges.map do |r|
+        filter = query ? query.dup : {}
+        filter['_id'] = r
+        { filter: filter, sort: { _id: 1 }, hint: { _id: 1 } }
+      end
+    end
+
+    # Legacy cursor sampling planner
+    def self.cursor_sampling_partitions(collection:, query:, partitions:)
       # Ensure sort order for sampling
       base_query = collection.find(query || {}, {}.merge(sort: { _id: 1 }))
 
@@ -49,27 +128,6 @@ module Purplelight
       end
     end
 
-    def self.simple_ranges(collection:, query:, partitions:)
-      # Split by _id quantiles using min/max endpoints
-      min_id = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(1).first&.dig('_id')
-      max_id = collection.find(query || {}).projection(_id: 1).sort(_id: -1).limit(1).first&.dig('_id')
-      return [{ filter: query || {}, sort: { _id: 1 } }] if min_id.nil? || max_id.nil?
-
-      # Create numeric-ish interpolation by sampling
-      ids = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(partitions - 1).to_a.map { |d| d['_id'] }
-      boundaries = [min_id] + ids + [max_id]
-      ranges = []
-      boundaries.each_cons(2) do |a, b|
-        ranges << build_range(a, b)
-      end
-
-      ranges.map do |r|
-        filter = query ? query.dup : {}
-        filter['_id'] = r
-        { filter: filter, sort: { _id: 1 }, hint: { _id: 1 } }
-      end
-    end
-
     def self.build_range(from_id, to_id)
       if from_id && to_id
         { '$gt' => from_id, '$lte' => to_id }

data/lib/purplelight/snapshot.rb

@@ -10,6 +10,7 @@ require_relative 'writer_csv'
 require_relative 'writer_parquet'
 require_relative 'manifest'
 require_relative 'errors'
+require_relative 'telemetry'
 
 module Purplelight
   # Snapshot orchestrates partition planning, parallel reads, and writing.
@@ -37,7 +38,7 @@ module Purplelight
                    resume: { enabled: true, state_path: nil, overwrite_incompatible: false },
                    sharding: { mode: :by_size, part_bytes: DEFAULTS[:rotate_bytes], prefix: nil },
                    logger: nil, on_progress: nil, read_concern: DEFAULTS[:read_concern], read_preference: DEFAULTS[:read_preference],
-                   no_cursor_timeout: DEFAULTS[:no_cursor_timeout])
+                   no_cursor_timeout: DEFAULTS[:no_cursor_timeout], telemetry: nil)
       @client = client
       @collection = client[collection]
       @output = output
@@ -60,6 +61,10 @@ module Purplelight
       @no_cursor_timeout = no_cursor_timeout
 
       @running = true
+      @telemetry_enabled = telemetry ? telemetry.enabled? : (ENV['PL_TELEMETRY'] == '1')
+      @telemetry = telemetry || (
+        @telemetry_enabled ? Telemetry.new(enabled: true) : Telemetry::NULL
+      )
     end
 
     # rubocop:disable Naming/PredicateMethod
@@ -90,8 +95,10 @@ module Purplelight
       manifest.ensure_partitions!(@partitions)
 
       # Plan partitions
+      t_plan = @telemetry.start(:partition_plan_time)
       partition_filters = Partitioner.object_id_partitions(collection: @collection, query: @query,
-                                                           partitions: @partitions)
+                                                           partitions: @partitions, telemetry: @telemetry)
+      @telemetry.finish(:partition_plan_time, t_plan)
 
       # Reader queue
       queue = ByteQueue.new(max_bytes: @queue_size_bytes)
@@ -116,12 +123,17 @@ module Purplelight
       # Start reader threads
       readers = partition_filters.each_with_index.map do |pf, idx|
         Thread.new do
-          read_partition(idx: idx, filter_spec: pf, queue: queue, batch_size: @batch_size, manifest: manifest)
+          local_telemetry = @telemetry_enabled ? Telemetry.new(enabled: true) : Telemetry::NULL
+          read_partition(idx: idx, filter_spec: pf, queue: queue, batch_size: @batch_size, manifest: manifest, telemetry: local_telemetry)
+          # Merge per-thread telemetry
+          @telemetry.merge!(local_telemetry) if @telemetry_enabled
         end
       end
 
       # Writer loop
+      writer_telemetry = @telemetry_enabled ? Telemetry.new(enabled: true) : Telemetry::NULL
       writer_thread = Thread.new do
+        Thread.current[:pl_telemetry] = writer_telemetry if @telemetry_enabled
         loop do
           batch = queue.pop
           break if batch.nil?
@@ -146,8 +158,22 @@ module Purplelight
       readers.each(&:join)
       queue.close
       writer_thread.join
+      @telemetry.merge!(writer_telemetry) if @telemetry_enabled
       @running = false
       progress_thread.join
+      if @telemetry_enabled
+        total = @telemetry.timers.values.sum
+        breakdown = @telemetry.timers
+                              .sort_by { |_k, v| -v }
+                              .map { |k, v| [k, v, total.zero? ? 0 : ((v / total) * 100.0)] }
+        if @logger
+          @logger.info('Telemetry (seconds and % of timed work):')
+          breakdown.each { |k, v, pct| @logger.info("  #{k}: #{v.round(3)}s (#{pct.round(1)}%)") }
+        else
+          puts 'Telemetry (seconds and % of timed work):'
+          breakdown.each { |k, v, pct| puts "  #{k}: #{v.round(3)}s (#{pct.round(1)}%)" }
+        end
+      end
       true
     end
     # rubocop:enable Naming/PredicateMethod
@@ -167,7 +193,7 @@ module Purplelight
       [dir, prefix]
     end
 
-    def read_partition(idx:, filter_spec:, queue:, batch_size:, manifest:)
+    def read_partition(idx:, filter_spec:, queue:, batch_size:, manifest:, telemetry: Telemetry::NULL)
       filter = filter_spec[:filter]
       sort = filter_spec[:sort] || { _id: 1 }
       hint = @hint || filter_spec[:hint] || { _id: 1 }
@@ -202,25 +228,32 @@ module Purplelight
         cursor.each do |doc|
           last_id = doc['_id']
           doc = @mapper.call(doc) if @mapper
+          t_ser = telemetry.start(:serialize_time)
           if encode_lines
             line = "#{JSON.generate(doc)}\n"
+            telemetry.finish(:serialize_time, t_ser)
             bytes = line.bytesize
             buffer << line
           else
             # For CSV/Parquet keep raw docs to allow schema/row building
             bytes = (JSON.generate(doc).bytesize + 1)
+            telemetry.finish(:serialize_time, t_ser)
             buffer << doc
           end
           buffer_bytes += bytes
           next unless buffer.length >= batch_size || buffer_bytes >= 1_000_000
 
+          t_q = telemetry.start(:queue_wait_time)
           queue.push(buffer, bytes: buffer_bytes)
+          telemetry.finish(:queue_wait_time, t_q)
           manifest.update_partition_checkpoint!(idx, last_id)
           buffer = []
           buffer_bytes = 0
         end
         unless buffer.empty?
+          t_q = telemetry.start(:queue_wait_time)
           queue.push(buffer, bytes: buffer_bytes)
+          telemetry.finish(:queue_wait_time, t_q)
           manifest.update_partition_checkpoint!(idx, last_id)
           buffer = []
           buffer_bytes = 0

data/lib/purplelight/telemetry.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Purplelight
+  # Lightweight, low-overhead timing and counters with a ticket API.
+  class Telemetry
+    def initialize(enabled: true)
+      @enabled = enabled
+      @counters = Hash.new(0)
+      @timers = Hash.new(0.0)
+      @mutex = Mutex.new
+    end
+
+    def enabled?
+      @enabled
+    end
+
+    # Start a timer. Returns a ticket (Float) when enabled, or nil when disabled.
+    def start(_key)
+      return nil unless @enabled
+
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    # Finish a timer using a ticket from start. No-ops if ticket is nil.
+    def finish(key, ticket)
+      return unless @enabled && ticket
+
+      dt = Process.clock_gettime(Process::CLOCK_MONOTONIC) - ticket
+      @timers[key] += dt
+    end
+
+    def add(key, count = 1)
+      return unless @enabled
+
+      @counters[key] += count
+    end
+
+    def merge!(other)
+      return self unless @enabled
+
+      other.counters.each { |k, v| @counters[k] += v }
+      other.timers.each { |k, v| @timers[k] += v }
+      self
+    end
+
+    attr_reader :counters, :timers
+
+    # A disabled singleton for zero overhead checks if needed.
+    NULL = new(enabled: false)
+  end
+end

data/lib/purplelight/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Purplelight
-  VERSION = '0.1.4'
+  VERSION = '0.1.5'
 end

data/lib/purplelight/writer_csv.rb

@@ -81,8 +81,10 @@ module Purplelight
 
       @csv&.flush
       if @io
+        t = Thread.current[:pl_telemetry]&.start(:rotate_time)
         finalize_current_part!
         @io.close
+        Thread.current[:pl_telemetry]&.finish(:rotate_time, t)
       end
       @closed = true
     end
@@ -122,8 +124,10 @@ module Purplelight
     def rotate!
       return unless @io
 
+      t = Thread.current[:pl_telemetry]&.start(:rotate_time)
       finalize_current_part!
       @io.close
+      Thread.current[:pl_telemetry]&.finish(:rotate_time, t)
       @io = nil
       @csv = nil
       ensure_open!

data/lib/purplelight/writer_jsonl.rb

@@ -106,7 +106,9 @@ module Purplelight
     end
 
     def write_buffer(buffer)
+      t = Thread.current[:pl_telemetry]&.start(:write_time)
       @io.write(buffer)
+      Thread.current[:pl_telemetry]&.finish(:write_time, t)
       @bytes_written += buffer.bytesize
       rotate_if_needed
     end
@@ -114,8 +116,10 @@ module Purplelight
     def rotate!
       return unless @io
 
+      t = Thread.current[:pl_telemetry]&.start(:rotate_time)
       finalize_current_part!
       @io.close
+      Thread.current[:pl_telemetry]&.finish(:rotate_time, t)
       @io = nil
       ensure_open!
     end

data/lib/purplelight/writer_parquet.rb

@@ -44,8 +44,13 @@ module Purplelight
 
       ensure_open!
       unless @buffer_docs.empty?
+        t_tbl = Thread.current[:pl_telemetry]&.start(:parquet_table_build_time)
         table = build_table(@buffer_docs)
+        Thread.current[:pl_telemetry]&.finish(:parquet_table_build_time, t_tbl)
+
+        t_w = Thread.current[:pl_telemetry]&.start(:parquet_write_time)
         write_table(table, @writer_path, append: false)
+        Thread.current[:pl_telemetry]&.finish(:parquet_write_time, t_w)
       end
       finalize_current_part!
       @closed = true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: purplelight
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Alexander Nicholson
@@ -13,44 +13,62 @@ dependencies:
   name: csv
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.3.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.3.5
 - !ruby/object:Gem::Dependency
   name: logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: 1.7.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: 1.7.0
 - !ruby/object:Gem::Dependency
   name: mongo
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.21'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.19'
+        version: 2.21.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.21'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.19'
+        version: 2.21.3
 description: High-throughput, resumable snapshots of MongoDB collections with partitioning,
   multi-threaded readers, and size-based sharded outputs.
 email:
@@ -70,15 +88,16 @@ files:
 - lib/purplelight/partitioner.rb
 - lib/purplelight/queue.rb
 - lib/purplelight/snapshot.rb
+- lib/purplelight/telemetry.rb
 - lib/purplelight/version.rb
 - lib/purplelight/writer_csv.rb
 - lib/purplelight/writer_jsonl.rb
 - lib/purplelight/writer_parquet.rb
+homepage: https://github.com/alexandernicholson/purplelight
 licenses:
 - MIT
 metadata:
   rubygems_mfa_required: 'true'
-  homepage_uri: https://github.com/alexandernicholson/purplelight
   source_code_uri: https://github.com/alexandernicholson/purplelight
   changelog_uri: https://github.com/alexandernicholson/purplelight/releases
 rdoc_options: []