purplelight 0.1.6 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 941ef04ede346a29c0afe19a02d69bd9a25d08ce241e21c6c47960498f4a42c6
-  data.tar.gz: c34b089d2842082f5d4be60c96d58e208e079f92712d397726a964a7361ae895
+  metadata.gz: 89114bd20a65a5a398be619718ae9a92e535d8118d9f928c61735da9a11cb5aa
+  data.tar.gz: 372372e6f5efe1cedd30033d661523fcb45a986c20e6e614ca46ca69def97e70
 SHA512:
-  metadata.gz: 8d666cb565283e6410fb0412d8ed369db18b0eaa0e80c573d677295910319b809396e99fca6627214d15942d531c9ac378c7d5a850842a7124d2ff4c9b03d7d8
-  data.tar.gz: 9188cd0b55e0d3e54fb2d209b85eeeacfb1c0d8daa73b1ace0fbb3bf4a2af865d6dd5c125f1e487458f096077a70cc93e7c0191c1222f6edd58701487f5479da
+  metadata.gz: 17103c062f0c6002ee53fbb1c2eed3179fb0df582c0dd99ef5100f8a26dbb0c56432f02f9a7f62c49bd201a05d414d7797c0e32044551522eb9d625ba9c179bf
+  data.tar.gz: e64bfd67ce31afae0c2209eaf058c731c331c63562f8e23de5e1f86a16a4fd3c8033f5938b7a509b3348daa8f4349afe926acb5e32fba8c7882996bd0d435616

data/README.md

@@ -9,7 +9,7 @@ Purplelight is published on RubyGems: [purplelight on RubyGems](https://rubygems
 Add to your Gemfile:
 
 ```ruby
-gem 'purplelight', '~> 0.1.6'
+gem 'purplelight', '~> 0.1.8'
 ```
 
 Or install directly:

data/bin/purplelight

@@ -4,6 +4,7 @@
 require 'optparse'
 require 'json'
 require 'mongo'
+require 'time'
 require_relative '../lib/purplelight'
 
 options = {
@@ -32,14 +33,25 @@ parser = OptionParser.new do |opts|
   opts.on('--partitions N', Integer, 'Number of partitions') { |v| options[:partitions] = v }
   opts.on('--batch-size N', Integer, 'Mongo batch size (default 2000)') { |v| options[:batch_size] = v }
   opts.on('--by-size BYTES', Integer, 'Shard by size (bytes); default 268435456') do |v|
-    options[:sharding] = { mode: :by_size, part_bytes: v }
+    options[:sharding] ||= {}
+    options[:sharding][:mode] = :by_size
+    options[:sharding][:part_bytes] = v
+  end
+  opts.on('--single-file', 'Write a single output file') do
+    options[:sharding] ||= {}
+    options[:sharding][:mode] = :single_file
   end
-  opts.on('--single-file', 'Write a single output file') { options[:sharding] = { mode: :single_file } }
   opts.on('--prefix NAME', 'Output file prefix') do |v|
     options[:sharding] ||= {}
     options[:sharding][:prefix] = v
   end
-  opts.on('-q', '--query JSON', 'Filter query as JSON') { |v| options[:query] = JSON.parse(v) }
+  opts.on('-q', '--query JSON', 'Filter query as JSON (Extended JSON supported)') do |v|
+    # Prefer BSON Extended JSON to support $date, $oid, etc.
+    options[:query] = BSON::ExtJSON.parse(v)
+  rescue StandardError
+    # Fallback to plain JSON for compatibility
+    options[:query] = JSON.parse(v)
+  end
   opts.on('--projection JSON', 'Projection as JSON (e.g., {"_id":1,"field":1})') { |v| options[:projection] = JSON.parse(v) }
   opts.on('--read-preference MODE',
           'Read preference mode: primary|primary_preferred|secondary|secondary_preferred|nearest') do |v|

data/lib/purplelight/partitioner.rb

@@ -27,13 +27,23 @@ module Purplelight
       max_id = collection.find(query || {}).projection(_id: 1).sort(_id: -1).limit(1).first&.dig('_id')
       return [{ filter: query || {}, sort: { _id: 1 } }] if min_id.nil? || max_id.nil?
 
-      # Create numeric-ish interpolation by sampling
-      ids = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(partitions - 1).to_a.map { |d| d['_id'] }
-      boundaries = [min_id] + ids + [max_id]
+      # Create contiguous ranges using ascending inner boundaries.
+      # We intentionally skip the very first _id so the first range includes the smallest document.
+      inner_boundaries = collection.find(query || {})
+                                   .projection(_id: 1)
+                                   .sort(_id: 1)
+                                   .skip(1)
+                                   .limit([partitions - 1, 0].max)
+                                   .to_a
+                                   .map { |d| d['_id'] }
+
       ranges = []
-      boundaries.each_cons(2) do |a, b|
-        ranges << build_range(a, b)
+      prev = nil
+      inner_boundaries.each do |b|
+        ranges << build_range(prev, b)
+        prev = b
       end
+      ranges << build_range(prev, nil)
 
       ranges.map do |r|
         filter = query ? query.dup : {}
@@ -54,7 +64,7 @@ module Purplelight
       min_ts = min_id.respond_to?(:generation_time) ? min_id.generation_time.to_i : nil
       max_ts = max_id.respond_to?(:generation_time) ? max_id.generation_time.to_i : nil
 
-      # Fallback to cursor sampling if _id isn't an ObjectId
+      # Fallback to cursor sampling if _id isn't anObjectId
       return cursor_sampling_partitions(collection: collection, query: query, partitions: partitions) if min_ts.nil? || max_ts.nil? || max_ts <= min_ts
 
       step = [(max_ts - min_ts) / partitions, 1].max

data/lib/purplelight/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Purplelight
-  VERSION = '0.1.6'
+  VERSION = '0.1.8'
 end

data/lib/purplelight/writer_csv.rb

@@ -31,7 +31,7 @@ module Purplelight
       @logger = logger
       @manifest = manifest
       env_level = ENV['PL_ZSTD_LEVEL']&.to_i
-      @compression_level = (env_level && env_level > 0 ? env_level : nil)
+      @compression_level = (env_level&.positive? ? env_level : nil)
       @single_file = single_file
 
       @columns = columns&.map(&:to_s)
@@ -123,8 +123,8 @@ module Purplelight
         @io.flush if @io.respond_to?(:flush)
       end
 
-      def method_missing(method_name, *args, &block)
-        @io.send(method_name, *args, &block)
+      def method_missing(method_name, *, &)
+        @io.send(method_name, *, &)
       end
 
       def respond_to_missing?(method_name, include_private = false)
@@ -202,7 +202,7 @@ module Purplelight
     def determine_effective_compression(requested)
       case requested.to_s
       when 'zstd'
-        ((defined?(ZSTDS) || (Object.const_defined?(:Zstd) && defined?(::Zstd::StreamWriter))) ? :zstd : :gzip)
+        (defined?(ZSTDS) || (Object.const_defined?(:Zstd) && defined?(::Zstd::StreamWriter)) ? :zstd : :gzip)
       when 'none'
         :none
       else

data/lib/purplelight/writer_jsonl.rb

@@ -30,7 +30,7 @@ module Purplelight
       @logger = logger
       @manifest = manifest
       env_level = ENV['PL_ZSTD_LEVEL']&.to_i
-      @compression_level = compression_level || (env_level && env_level > 0 ? env_level : nil)
+      @compression_level = compression_level || (env_level&.positive? ? env_level : nil)
 
       @part_index = nil
       @io = nil
@@ -44,15 +44,15 @@ module Purplelight
         level_disp = @compression_level || (ENV['PL_ZSTD_LEVEL']&.to_i if @effective_compression.to_s == 'zstd')
         @logger.info("WriterJSONL using compression='#{@effective_compression}' level='#{level_disp || 'default'}'")
       end
-      if @effective_compression.to_s != @compression.to_s
-        @logger&.warn("requested compression '#{@compression}' not available; using '#{@effective_compression}'")
-      end
+      return unless @effective_compression.to_s != @compression.to_s
+
+      @logger&.warn("requested compression '#{@compression}' not available; using '#{@effective_compression}'")
     end
 
     def write_many(batch)
       ensure_open!
 
-      chunk_threshold = (ENV['PL_WRITE_CHUNK_BYTES']&.to_i || (8 * 1024 * 1024))
+      chunk_threshold = ENV['PL_WRITE_CHUNK_BYTES']&.to_i || (8 * 1024 * 1024)
       total_bytes = 0
       rows = 0
 
@@ -200,7 +200,7 @@ module Purplelight
     def determine_effective_compression(requested)
       case requested.to_s
       when 'zstd'
-        ((defined?(ZSTDS) || (Object.const_defined?(:Zstd) && defined?(::Zstd::StreamWriter))) ? :zstd : :gzip)
+        (defined?(ZSTDS) || (Object.const_defined?(:Zstd) && defined?(::Zstd::StreamWriter)) ? :zstd : :gzip)
       when 'none'
         :none
       else

data/lib/purplelight/writer_parquet.rb

@@ -116,9 +116,9 @@ module Purplelight
     def next_part_path
       ext = 'parquet'
       filename = if @single_file
-                   format('%<prefix>s.%<ext>s', prefix: @prefix, ext: ext)
+                   "#{@prefix}.#{ext}"
                  else
-                   format('%<prefix}s-part-%<seq>06d.%<ext>s', prefix: @prefix, seq: @file_seq, ext: ext)
+                   format('%<prefix>s-part-%<seq>06d.%<ext>s', prefix: @prefix, seq: @file_seq, ext: ext)
                  end
       File.join(@directory, filename)
     end
@@ -132,7 +132,11 @@ module Purplelight
     end
 
     def extract_value(doc, key)
-      doc[key] || doc[key.to_sym]
+      value = doc[key] || doc[key.to_sym]
+      # Normalize common MongoDB/BSON types to Parquet-friendly values
+      return value.to_s if defined?(BSON) && value.is_a?(BSON::ObjectId)
+
+      value
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: purplelight
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.8
 platform: ruby
 authors:
 - Alexander Nicholson