purplelight 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5964231e634be4a743207679e349623c275e0b20771b492bbc54c4261238e352
-  data.tar.gz: 82448e1f4b5ffb8e9846938653b16a4d7008aa29e722d9417863b80362370168
+  metadata.gz: a650fdd2113129b151396a1a90a83a6f1ede97eb5c34c60e028eb7639d5cc4fd
+  data.tar.gz: 1ab1bc421ddaf1c457639ae2ac5968245c6141b8504897ab762a49bb69c51a69
 SHA512:
-  metadata.gz: 4b8ac2ba82501978bcd1ae1b9db888f7dab387de76a7e473c5b0993a526398a727a029a0597a64c77bbbed976980e6adaad7a08f7e3967c9cfe6c7afa2d996ac
-  data.tar.gz: 15f536cfc05a0b70f7cdeb6bc7985e56b2e6dad6b2db53922e7aab12a821b8bb1294217390ff00c095b2cacb46c23e027d24d90a083e9e415a0df6e26fec9b59
+  metadata.gz: 506e52dce7c474998c8bc4b9afa9f5140349e8e2eed2eed7cccbacac0bd9d9f41528b234f1b99ad8407674791471368ee5b99d93b7ab058522311f2642006a20
+  data.tar.gz: 5c17e387f0d67a21d1351cf4e1e69beaa7beecdf5b9f8011e16bd740e990902abf32c54b02257cdead9c296241557571608b6004446cca5d429675fda07ff61a

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Alexander Nicholson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -9,7 +9,7 @@ Purplelight is published on RubyGems: [purplelight on RubyGems](https://rubygems
 Add to your Gemfile:
 
 ```ruby
-gem 'purplelight', '~> 0.1.3'
+gem 'purplelight', '~> 0.1.5'
 ```
 
 Or install directly:

data/lib/purplelight/partitioner.rb

@@ -11,7 +11,86 @@ module Purplelight
   class Partitioner
     # Builds contiguous _id range filters for N partitions.
     # For ObjectId _id, we sample quantiles to split into near-equal document counts.
-    def self.object_id_partitions(collection:, query:, partitions:)
+    def self.object_id_partitions(collection:, query:, partitions:, mode: nil, telemetry: nil)
+      # Choose planning mode: :timestamp (fast), :cursor (legacy)
+      chosen_mode = (mode || ENV['PL_PARTITIONER_MODE'] || :timestamp).to_sym
+      telemetry ||= (defined?(Telemetry) ? Telemetry::NULL : nil)
+
+      return cursor_sampling_partitions(collection: collection, query: query, partitions: partitions) if chosen_mode == :cursor
+
+      timestamp_partitions(collection: collection, query: query, partitions: partitions, telemetry: telemetry)
+    end
+
+    def self.simple_ranges(collection:, query:, partitions:)
+      # Split by _id quantiles using min/max endpoints
+      min_id = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(1).first&.dig('_id')
+      max_id = collection.find(query || {}).projection(_id: 1).sort(_id: -1).limit(1).first&.dig('_id')
+      return [{ filter: query || {}, sort: { _id: 1 } }] if min_id.nil? || max_id.nil?
+
+      # Create numeric-ish interpolation by sampling
+      ids = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(partitions - 1).to_a.map { |d| d['_id'] }
+      boundaries = [min_id] + ids + [max_id]
+      ranges = []
+      boundaries.each_cons(2) do |a, b|
+        ranges << build_range(a, b)
+      end
+
+      ranges.map do |r|
+        filter = query ? query.dup : {}
+        filter['_id'] = r
+        { filter: filter, sort: { _id: 1 }, hint: { _id: 1 } }
+      end
+    end
+
+    # Faster planning using ObjectId timestamps: O(partitions) indexed lookups
+    def self.timestamp_partitions(collection:, query:, partitions:, telemetry: nil)
+      t_minmax = telemetry&.start(:plan_minmax_time)
+      min_id = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(1).first&.dig('_id')
+      max_id = collection.find(query || {}).projection(_id: 1).sort(_id: -1).limit(1).first&.dig('_id')
+      telemetry&.finish(:plan_minmax_time, t_minmax)
+
+      return [{ filter: query || {}, sort: { _id: 1 } }] if min_id.nil? || max_id.nil?
+
+      min_ts = min_id.respond_to?(:generation_time) ? min_id.generation_time.to_i : nil
+      max_ts = max_id.respond_to?(:generation_time) ? max_id.generation_time.to_i : nil
+
+      # Fallback to cursor sampling if _id isn't an ObjectId
+      return cursor_sampling_partitions(collection: collection, query: query, partitions: partitions) if min_ts.nil? || max_ts.nil? || max_ts <= min_ts
+
+      step = [(max_ts - min_ts) / partitions, 1].max
+      inner_boundaries = []
+      t_boundaries = telemetry&.start(:plan_boundary_queries_time)
+      1.upto(partitions - 1) do |i|
+        target_ts = min_ts + (step * i)
+        candidate = BSON::ObjectId.from_time(Time.at(target_ts))
+        f = query ? query.dup : {}
+        f['_id'] = { '$gt' => candidate }
+        b = collection.find(f).projection(_id: 1).sort(_id: 1).hint(_id: 1).limit(1).first&.dig('_id')
+        inner_boundaries << b if b
+      end
+      telemetry&.finish(:plan_boundary_queries_time, t_boundaries)
+
+      # Build ranges: first range has nil lower bound to include min_id,
+      # middle ranges are (prev, current], and last is (last, +inf)
+      ranges = []
+      t_ranges = telemetry&.start(:plan_ranges_build_time)
+      prev = nil
+      inner_boundaries.each do |b|
+        ranges << build_range(prev, b)
+        prev = b
+      end
+      ranges << build_range(prev, nil)
+      telemetry&.finish(:plan_ranges_build_time, t_ranges)
+
+      ranges.map do |r|
+        filter = query ? query.dup : {}
+        filter['_id'] = r
+        { filter: filter, sort: { _id: 1 }, hint: { _id: 1 } }
+      end
+    end
+
+    # Legacy cursor sampling planner
+    def self.cursor_sampling_partitions(collection:, query:, partitions:)
       # Ensure sort order for sampling
       base_query = collection.find(query || {}, {}.merge(sort: { _id: 1 }))
 
@@ -49,27 +128,6 @@ module Purplelight
       end
     end
 
-    def self.simple_ranges(collection:, query:, partitions:)
-      # Split by _id quantiles using min/max endpoints
-      min_id = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(1).first&.dig('_id')
-      max_id = collection.find(query || {}).projection(_id: 1).sort(_id: -1).limit(1).first&.dig('_id')
-      return [{ filter: query || {}, sort: { _id: 1 } }] if min_id.nil? || max_id.nil?
-
-      # Create numeric-ish interpolation by sampling
-      ids = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(partitions - 1).to_a.map { |d| d['_id'] }
-      boundaries = [min_id] + ids + [max_id]
-      ranges = []
-      boundaries.each_cons(2) do |a, b|
-        ranges << build_range(a, b)
-      end
-
-      ranges.map do |r|
-        filter = query ? query.dup : {}
-        filter['_id'] = r
-        { filter: filter, sort: { _id: 1 }, hint: { _id: 1 } }
-      end
-    end
-
     def self.build_range(from_id, to_id)
       if from_id && to_id
         { '$gt' => from_id, '$lte' => to_id }

data/lib/purplelight/snapshot.rb

@@ -10,6 +10,7 @@ require_relative 'writer_csv'
 require_relative 'writer_parquet'
 require_relative 'manifest'
 require_relative 'errors'
+require_relative 'telemetry'
 
 module Purplelight
   # Snapshot orchestrates partition planning, parallel reads, and writing.
@@ -37,7 +38,7 @@ module Purplelight
                    resume: { enabled: true, state_path: nil, overwrite_incompatible: false },
                    sharding: { mode: :by_size, part_bytes: DEFAULTS[:rotate_bytes], prefix: nil },
                    logger: nil, on_progress: nil, read_concern: DEFAULTS[:read_concern], read_preference: DEFAULTS[:read_preference],
-                   no_cursor_timeout: DEFAULTS[:no_cursor_timeout])
+                   no_cursor_timeout: DEFAULTS[:no_cursor_timeout], telemetry: nil)
       @client = client
       @collection = client[collection]
       @output = output
@@ -60,6 +61,10 @@ module Purplelight
       @no_cursor_timeout = no_cursor_timeout
 
       @running = true
+      @telemetry_enabled = telemetry ? telemetry.enabled? : (ENV['PL_TELEMETRY'] == '1')
+      @telemetry = telemetry || (
+        @telemetry_enabled ? Telemetry.new(enabled: true) : Telemetry::NULL
+      )
     end
 
     # rubocop:disable Naming/PredicateMethod
@@ -90,8 +95,10 @@ module Purplelight
       manifest.ensure_partitions!(@partitions)
 
       # Plan partitions
+      t_plan = @telemetry.start(:partition_plan_time)
       partition_filters = Partitioner.object_id_partitions(collection: @collection, query: @query,
-                                                           partitions: @partitions)
+                                                           partitions: @partitions, telemetry: @telemetry)
+      @telemetry.finish(:partition_plan_time, t_plan)
 
       # Reader queue
       queue = ByteQueue.new(max_bytes: @queue_size_bytes)
@@ -116,12 +123,17 @@ module Purplelight
       # Start reader threads
       readers = partition_filters.each_with_index.map do |pf, idx|
         Thread.new do
-          read_partition(idx: idx, filter_spec: pf, queue: queue, batch_size: @batch_size, manifest: manifest)
+          local_telemetry = @telemetry_enabled ? Telemetry.new(enabled: true) : Telemetry::NULL
+          read_partition(idx: idx, filter_spec: pf, queue: queue, batch_size: @batch_size, manifest: manifest, telemetry: local_telemetry)
+          # Merge per-thread telemetry
+          @telemetry.merge!(local_telemetry) if @telemetry_enabled
         end
       end
 
       # Writer loop
+      writer_telemetry = @telemetry_enabled ? Telemetry.new(enabled: true) : Telemetry::NULL
       writer_thread = Thread.new do
+        Thread.current[:pl_telemetry] = writer_telemetry if @telemetry_enabled
         loop do
           batch = queue.pop
           break if batch.nil?
@@ -146,8 +158,22 @@ module Purplelight
       readers.each(&:join)
       queue.close
       writer_thread.join
+      @telemetry.merge!(writer_telemetry) if @telemetry_enabled
       @running = false
       progress_thread.join
+      if @telemetry_enabled
+        total = @telemetry.timers.values.sum
+        breakdown = @telemetry.timers
+                              .sort_by { |_k, v| -v }
+                              .map { |k, v| [k, v, total.zero? ? 0 : ((v / total) * 100.0)] }
+        if @logger
+          @logger.info('Telemetry (seconds and % of timed work):')
+          breakdown.each { |k, v, pct| @logger.info("  #{k}: #{v.round(3)}s (#{pct.round(1)}%)") }
+        else
+          puts 'Telemetry (seconds and % of timed work):'
+          breakdown.each { |k, v, pct| puts "  #{k}: #{v.round(3)}s (#{pct.round(1)}%)" }
+        end
+      end
       true
     end
     # rubocop:enable Naming/PredicateMethod
@@ -167,7 +193,7 @@ module Purplelight
       [dir, prefix]
     end
 
-    def read_partition(idx:, filter_spec:, queue:, batch_size:, manifest:)
+    def read_partition(idx:, filter_spec:, queue:, batch_size:, manifest:, telemetry: Telemetry::NULL)
       filter = filter_spec[:filter]
       sort = filter_spec[:sort] || { _id: 1 }
       hint = @hint || filter_spec[:hint] || { _id: 1 }
@@ -202,25 +228,32 @@ module Purplelight
         cursor.each do |doc|
           last_id = doc['_id']
           doc = @mapper.call(doc) if @mapper
+          t_ser = telemetry.start(:serialize_time)
           if encode_lines
-            line = "#{Oj.dump(doc, mode: :compat)}\n"
+            line = "#{JSON.generate(doc)}\n"
+            telemetry.finish(:serialize_time, t_ser)
             bytes = line.bytesize
             buffer << line
           else
             # For CSV/Parquet keep raw docs to allow schema/row building
-            bytes = (Oj.dump(doc, mode: :compat).bytesize + 1)
+            bytes = (JSON.generate(doc).bytesize + 1)
+            telemetry.finish(:serialize_time, t_ser)
             buffer << doc
           end
           buffer_bytes += bytes
           next unless buffer.length >= batch_size || buffer_bytes >= 1_000_000
 
+          t_q = telemetry.start(:queue_wait_time)
           queue.push(buffer, bytes: buffer_bytes)
+          telemetry.finish(:queue_wait_time, t_q)
           manifest.update_partition_checkpoint!(idx, last_id)
           buffer = []
           buffer_bytes = 0
         end
         unless buffer.empty?
+          t_q = telemetry.start(:queue_wait_time)
           queue.push(buffer, bytes: buffer_bytes)
+          telemetry.finish(:queue_wait_time, t_q)
           manifest.update_partition_checkpoint!(idx, last_id)
           buffer = []
           buffer_bytes = 0

data/lib/purplelight/telemetry.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Purplelight
+  # Lightweight, low-overhead timing and counters with a ticket API.
+  class Telemetry
+    def initialize(enabled: true)
+      @enabled = enabled
+      @counters = Hash.new(0)
+      @timers = Hash.new(0.0)
+      @mutex = Mutex.new
+    end
+
+    def enabled?
+      @enabled
+    end
+
+    # Start a timer. Returns a ticket (Float) when enabled, or nil when disabled.
+    def start(_key)
+      return nil unless @enabled
+
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
+    # Finish a timer using a ticket from start. No-ops if ticket is nil.
+    def finish(key, ticket)
+      return unless @enabled && ticket
+
+      dt = Process.clock_gettime(Process::CLOCK_MONOTONIC) - ticket
+      @timers[key] += dt
+    end
+
+    def add(key, count = 1)
+      return unless @enabled
+
+      @counters[key] += count
+    end
+
+    def merge!(other)
+      return self unless @enabled
+
+      other.counters.each { |k, v| @counters[k] += v }
+      other.timers.each { |k, v| @timers[k] += v }
+      self
+    end
+
+    attr_reader :counters, :timers
+
+    # A disabled singleton for zero overhead checks if needed.
+    NULL = new(enabled: false)
+  end
+end

data/lib/purplelight/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Purplelight
-  VERSION = '0.1.3'
+  VERSION = '0.1.5'
 end

data/lib/purplelight/writer_csv.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'csv'
-require 'oj'
+require 'json'
 require 'zlib'
 require 'fileutils'
 
@@ -81,8 +81,10 @@ module Purplelight
 
       @csv&.flush
       if @io
+        t = Thread.current[:pl_telemetry]&.start(:rotate_time)
         finalize_current_part!
         @io.close
+        Thread.current[:pl_telemetry]&.finish(:rotate_time, t)
       end
       @closed = true
     end
@@ -122,8 +124,10 @@ module Purplelight
     def rotate!
       return unless @io
 
+      t = Thread.current[:pl_telemetry]&.start(:rotate_time)
       finalize_current_part!
       @io.close
+      Thread.current[:pl_telemetry]&.finish(:rotate_time, t)
       @io = nil
       @csv = nil
       ensure_open!
@@ -173,7 +177,7 @@ module Purplelight
       val = doc[key] || doc[key.to_sym]
       case val
       when Hash, Array
-        Oj.dump(val, mode: :compat)
+        JSON.generate(val)
       else
         val
       end

data/lib/purplelight/writer_jsonl.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'oj'
+require 'json'
 require 'zlib'
 require 'fileutils'
 
@@ -44,7 +44,7 @@ module Purplelight
       buffer = if array_of_docs.first.is_a?(String)
                  array_of_docs.join
                else
-                 array_of_docs.map { |doc| "#{Oj.dump(doc, mode: :compat)}\n" }.join
+                 array_of_docs.map { |doc| "#{JSON.generate(doc)}\n" }.join
                end
       rows = array_of_docs.size
       write_buffer(buffer)
@@ -106,7 +106,9 @@ module Purplelight
     end
 
     def write_buffer(buffer)
+      t = Thread.current[:pl_telemetry]&.start(:write_time)
       @io.write(buffer)
+      Thread.current[:pl_telemetry]&.finish(:write_time, t)
       @bytes_written += buffer.bytesize
       rotate_if_needed
     end
@@ -114,8 +116,10 @@ module Purplelight
     def rotate!
       return unless @io
 
+      t = Thread.current[:pl_telemetry]&.start(:rotate_time)
       finalize_current_part!
       @io.close
+      Thread.current[:pl_telemetry]&.finish(:rotate_time, t)
       @io = nil
       ensure_open!
     end

data/lib/purplelight/writer_parquet.rb

@@ -44,8 +44,13 @@ module Purplelight
 
       ensure_open!
       unless @buffer_docs.empty?
+        t_tbl = Thread.current[:pl_telemetry]&.start(:parquet_table_build_time)
         table = build_table(@buffer_docs)
+        Thread.current[:pl_telemetry]&.finish(:parquet_table_build_time, t_tbl)
+
+        t_w = Thread.current[:pl_telemetry]&.start(:parquet_write_time)
         write_table(table, @writer_path, append: false)
+        Thread.current[:pl_telemetry]&.finish(:parquet_write_time, t_w)
       end
       finalize_current_part!
       @closed = true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: purplelight
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Alexander Nicholson
@@ -13,58 +13,62 @@ dependencies:
   name: csv
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.3.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.3.5
 - !ruby/object:Gem::Dependency
   name: logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: 1.7.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: 1.7.0
 - !ruby/object:Gem::Dependency
   name: mongo
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.19'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
+        version: '2.21'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.19'
-- !ruby/object:Gem::Dependency
-  name: oj
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3.16'
+        version: 2.21.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.21'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '3.16'
+        version: 2.21.3
 description: High-throughput, resumable snapshots of MongoDB collections with partitioning,
   multi-threaded readers, and size-based sharded outputs.
 email:
@@ -74,6 +78,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE
 - README.md
 - Rakefile
 - bin/purplelight
@@ -83,15 +88,16 @@ files:
 - lib/purplelight/partitioner.rb
 - lib/purplelight/queue.rb
 - lib/purplelight/snapshot.rb
+- lib/purplelight/telemetry.rb
 - lib/purplelight/version.rb
 - lib/purplelight/writer_csv.rb
 - lib/purplelight/writer_jsonl.rb
 - lib/purplelight/writer_parquet.rb
+homepage: https://github.com/alexandernicholson/purplelight
 licenses:
 - MIT
 metadata:
   rubygems_mfa_required: 'true'
-  homepage_uri: https://github.com/alexandernicholson/purplelight
   source_code_uri: https://github.com/alexandernicholson/purplelight
   changelog_uri: https://github.com/alexandernicholson/purplelight/releases
 rdoc_options: []