purplelight 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 941ef04ede346a29c0afe19a02d69bd9a25d08ce241e21c6c47960498f4a42c6
-  data.tar.gz: c34b089d2842082f5d4be60c96d58e208e079f92712d397726a964a7361ae895
+  metadata.gz: 6e3771629528ecab067d858491ec4c5de06a8c88c90c64b56dcdb8658c3a6a89
+  data.tar.gz: 4c0427564bf04b5dc3da6b3af97bfd4e5dc0625b2faa0dd8acec84a81bd1c145
 SHA512:
-  metadata.gz: 8d666cb565283e6410fb0412d8ed369db18b0eaa0e80c573d677295910319b809396e99fca6627214d15942d531c9ac378c7d5a850842a7124d2ff4c9b03d7d8
-  data.tar.gz: 9188cd0b55e0d3e54fb2d209b85eeeacfb1c0d8daa73b1ace0fbb3bf4a2af865d6dd5c125f1e487458f096077a70cc93e7c0191c1222f6edd58701487f5479da
+  metadata.gz: 871cf3c04dcaa017d1023ff79135a5050254158a7ba20145abb04c39887125c08eedfb8cae4d16ddd3c4fd0acb383ceb9c11efff943b497ff036b68643dbea79
+  data.tar.gz: 8a5e7fe7a1913939966df61173380e6ed853f45bec2ef90315410a6d36d6341c07d04ff62d200379629b2366cf68296c7c291b5060d0a080dc3015175f1e6780

data/README.md

@@ -9,7 +9,7 @@ Purplelight is published on RubyGems: [purplelight on RubyGems](https://rubygems
 Add to your Gemfile:
 
 ```ruby
-gem 'purplelight', '~> 0.1.6'
+gem 'purplelight', '~> 0.1.7'
 ```
 
 Or install directly:

data/bin/purplelight

@@ -4,6 +4,7 @@
 require 'optparse'
 require 'json'
 require 'mongo'
+require 'time'
 require_relative '../lib/purplelight'
 
 options = {
@@ -39,7 +40,15 @@ parser = OptionParser.new do |opts|
     options[:sharding] ||= {}
     options[:sharding][:prefix] = v
   end
-  opts.on('-q', '--query JSON', 'Filter query as JSON') { |v| options[:query] = JSON.parse(v) }
+  opts.on('-q', '--query JSON', 'Filter query as JSON (Extended JSON supported)') do |v|
+    begin
+      # Prefer BSON Extended JSON to support $date, $oid, etc.
+      options[:query] = BSON::ExtJSON.parse(v)
+    rescue StandardError
+      # Fallback to plain JSON for compatibility
+      options[:query] = JSON.parse(v)
+    end
+  end
   opts.on('--projection JSON', 'Projection as JSON (e.g., {"_id":1,"field":1})') { |v| options[:projection] = JSON.parse(v) }
   opts.on('--read-preference MODE',
           'Read preference mode: primary|primary_preferred|secondary|secondary_preferred|nearest') do |v|

data/lib/purplelight/partitioner.rb

@@ -27,13 +27,23 @@ module Purplelight
       max_id = collection.find(query || {}).projection(_id: 1).sort(_id: -1).limit(1).first&.dig('_id')
       return [{ filter: query || {}, sort: { _id: 1 } }] if min_id.nil? || max_id.nil?
 
-      # Create numeric-ish interpolation by sampling
-      ids = collection.find(query || {}).projection(_id: 1).sort(_id: 1).limit(partitions - 1).to_a.map { |d| d['_id'] }
-      boundaries = [min_id] + ids + [max_id]
+      # Create contiguous ranges using ascending inner boundaries.
+      # We intentionally skip the very first _id so the first range includes the smallest document.
+      inner_boundaries = collection.find(query || {})
+                                   .projection(_id: 1)
+                                   .sort(_id: 1)
+                                   .skip(1)
+                                   .limit([partitions - 1, 0].max)
+                                   .to_a
+                                   .map { |d| d['_id'] }
+
       ranges = []
-      boundaries.each_cons(2) do |a, b|
-        ranges << build_range(a, b)
+      prev = nil
+      inner_boundaries.each do |b|
+        ranges << build_range(prev, b)
+        prev = b
       end
+      ranges << build_range(prev, nil)
 
       ranges.map do |r|
         filter = query ? query.dup : {}
@@ -54,7 +64,7 @@ module Purplelight
       min_ts = min_id.respond_to?(:generation_time) ? min_id.generation_time.to_i : nil
       max_ts = max_id.respond_to?(:generation_time) ? max_id.generation_time.to_i : nil
 
-      # Fallback to cursor sampling if _id isn't an ObjectId
+      # Fallback to cursor sampling if _id isn't anObjectId
       return cursor_sampling_partitions(collection: collection, query: query, partitions: partitions) if min_ts.nil? || max_ts.nil? || max_ts <= min_ts
 
       step = [(max_ts - min_ts) / partitions, 1].max

data/lib/purplelight/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Purplelight
-  VERSION = '0.1.6'
+  VERSION = '0.1.7'
 end

data/lib/purplelight/writer_parquet.rb

@@ -116,9 +116,9 @@ module Purplelight
     def next_part_path
       ext = 'parquet'
       filename = if @single_file
-                   format('%<prefix>s.%<ext>s', prefix: @prefix, ext: ext)
+                   "#{@prefix}.#{ext}"
                  else
-                   format('%<prefix}s-part-%<seq>06d.%<ext>s', prefix: @prefix, seq: @file_seq, ext: ext)
+                   format('%<prefix>s-part-%<seq>06d.%<ext>s', prefix: @prefix, seq: @file_seq, ext: ext)
                  end
       File.join(@directory, filename)
     end
@@ -132,7 +132,12 @@ module Purplelight
     end
 
     def extract_value(doc, key)
-      doc[key] || doc[key.to_sym]
+      value = doc[key] || doc[key.to_sym]
+      # Normalize common MongoDB/BSON types to Parquet-friendly values
+      if defined?(BSON) && value.is_a?(BSON::ObjectId)
+        return value.to_s
+      end
+      value
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: purplelight
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Alexander Nicholson