purplelight 0.1.10 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f0f51fd601a59915a2a022831663fd4f2468e781b68b96f59d396359be49adbc
-  data.tar.gz: c899a18e7ce390bfc05f832dd32248aa8cbdc7b43bccf86197350e2c7929e7a6
+  metadata.gz: 80f6e48231b485750fc65529ada74a07758befba9b324b58ef372f077305a144
+  data.tar.gz: 4d7eed034f90155d2686da45a76caa73928cbe2d080d2031539830f6f4399cfb
 SHA512:
-  metadata.gz: f6546911873ed22865b9d4cdd2cc62d855ab3b991030808d8f49f3e054727a406b80c7dc43c518a450915152f2934dcba180d53bf75807c540eef893b3ca50b8
-  data.tar.gz: 5e7176eec64956388e72fd3d894db12e006a18edd4aade7eaf13b144381802932d7207ffefac6ad06157c03363f41acec4ca997871fb8abe8efc9e06e2238804
+  metadata.gz: 63f49c8dab688ec5cd922304b472ee448aaada1a3e2c113b7a4ddbe2092f3a3d2a83e1fe396066e87971514d8069831486c1a5ad972e604807c6c3289efd8e31
+  data.tar.gz: 6a281a23a0abf3244045b3e99af606881f3f85bab7eeabd7b4ca94e36c823d2405d7e85b60048ec62e5a7ace63fceddd268e185a831398cc7dbd00c213198961

data/README.md

@@ -9,7 +9,7 @@ Purplelight is published on RubyGems: [purplelight on RubyGems](https://rubygems
 Add to your Gemfile:
 
 ```ruby
-gem 'purplelight', '~> 0.1.10'
+gem 'purplelight', '~> 0.1.11'
 ```
 
 Or install directly:
@@ -144,6 +144,29 @@ Purplelight.snapshot(
 )
 ```
 
+Parquet multi-part (rows-based rotation):
+
+```ruby
+Purplelight.snapshot(
+  client: client,
+  collection: 'users',
+  output: '/data/exports',
+  format: :parquet,
+  # Any mode other than :single_file enables multi-part filenames for Parquet
+  sharding: { mode: :by_size, prefix: 'users_parquet_parts' },
+  # Split into multiple .parquet files, limiting rows per file
+  parquet_max_rows: 100_000,
+  # Optional: Parquet row group size (rows)
+  parquet_row_group: 10_000,
+  resume: { enabled: true }
+)
+```
+
+Notes for Parquet:
+- Parquet multi-part sizing is controlled by rows via `parquet_max_rows`.
+- `--rotate-mb` / `part_bytes` do not affect Parquet part size; they apply to JSONL/CSV.
+- Use `sharding: { mode: :single_file }` to force a single `.parquet` file.
+
 ### Environment variables (optional)
 
 CLI flags take precedence, but these environment variables can set sensible defaults:
@@ -199,6 +222,8 @@ bundle exec bin/purplelight \
 Notes:
 - Compression backend selection order is: requested format → `zstd-ruby` → `zstds` → `gzip`.
 - `--single-file` and `--by-size` update only the sharding mode/params and preserve any provided `--prefix`.
+- Parquet multi-part sizing is programmatic via `parquet_max_rows`; there is no CLI flag for it.
+- To increase concurrent connections, set `maxPoolSize` on your Mongo URI (used by `--uri`), e.g., `mongodb://.../?maxPoolSize=32`. A good starting point is `maxPoolSize >= --partitions`.
 
 ### Architecture
 
@@ -236,6 +261,7 @@ Key points:
 - **Rotation size**: larger (512MB–1GB) reduces finalize overhead for many parts. CLI: `--rotate-mb` (and/or `--by-size`).
 - **JSONL chunking**: tune builder write chunk size for throughput. CLI: `--write-chunk-mb`.
 - **Parquet row groups**: choose a row group size that fits downstream readers. CLI: `--parquet-row-group`.
+- **Parquet parts (rows)**: split Parquet outputs by rows with `parquet_max_rows` (programmatic API). Set `sharding.mode` to anything other than `:single_file` to enable multi-part filenames.
 - **Read preference**: offload to secondaries or tagged analytics nodes when available. CLI: `--read-preference`, `--read-tags`.
 - **Read concern**: pick an appropriate level for consistency/latency trade-offs. CLI: `--read-concern`.
 - **Cursor timeout**: for very long scans, leave `noCursorTimeout` enabled. CLI: `--no-cursor-timeout true|false`.

data/lib/purplelight/snapshot.rb

@@ -39,7 +39,8 @@ module Purplelight
                    sharding: { mode: :by_size, part_bytes: DEFAULTS[:rotate_bytes], prefix: nil },
                    logger: nil, on_progress: nil, read_concern: DEFAULTS[:read_concern], read_preference: DEFAULTS[:read_preference],
                    no_cursor_timeout: DEFAULTS[:no_cursor_timeout], telemetry: nil,
-                   compression_level: nil, writer_threads: 1, write_chunk_bytes: nil, parquet_row_group: nil)
+                   compression_level: nil, writer_threads: 1, write_chunk_bytes: nil, parquet_row_group: nil,
+                   parquet_max_rows: nil)
       @client = client
       @collection = client[collection]
       @output = output
@@ -64,6 +65,7 @@ module Purplelight
       @writer_threads = writer_threads || 1
       @write_chunk_bytes = write_chunk_bytes
       @parquet_row_group = parquet_row_group
+      @parquet_max_rows = parquet_max_rows
 
       @running = true
       @telemetry_enabled = telemetry ? telemetry.enabled? : (ENV['PL_TELEMETRY'] == '1')
@@ -106,6 +108,7 @@ module Purplelight
                             compression_level: @compression_level || (ENV['PL_ZSTD_LEVEL']&.to_i if @compression.to_s == 'zstd') || ENV['PL_ZSTD_LEVEL']&.to_i,
                             write_chunk_bytes: @write_chunk_bytes || ENV['PL_WRITE_CHUNK_BYTES']&.to_i,
                             parquet_row_group: @parquet_row_group || ENV['PL_PARQUET_ROW_GROUP']&.to_i,
+                            parquet_max_rows: @parquet_max_rows,
                             sharding: @sharding,
                             resume_overwrite_incompatible: @resume && @resume[:overwrite_incompatible] ? true : false,
                             telemetry: @telemetry_enabled
@@ -134,7 +137,8 @@ module Purplelight
                  single_file = @sharding && @sharding[:mode].to_s == 'single_file'
                  row_group = @parquet_row_group || ENV['PL_PARQUET_ROW_GROUP']&.to_i || WriterParquet::DEFAULT_ROW_GROUP_SIZE
                  WriterParquet.new(directory: dir, prefix: prefix, compression: @compression, logger: @logger,
-                                   manifest: manifest, single_file: single_file, row_group_size: row_group)
+                          manifest: manifest, single_file: single_file, row_group_size: row_group,
+                          rotate_rows: @parquet_max_rows)
                else
                  raise ArgumentError, "format not implemented: #{@format}"
                end

data/lib/purplelight/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Purplelight
-  VERSION = '0.1.10'
+  VERSION = '0.1.11'
 end

data/lib/purplelight/writer_parquet.rb

@@ -15,7 +15,7 @@ module Purplelight
     DEFAULT_ROW_GROUP_SIZE = 10_000
 
     def initialize(directory:, prefix:, compression: :zstd, row_group_size: DEFAULT_ROW_GROUP_SIZE, logger: nil,
-                   manifest: nil, single_file: true, schema: nil)
+                   manifest: nil, single_file: true, schema: nil, rotate_rows: nil)
       @directory = directory
       @prefix = prefix
       @compression = compression
@@ -24,11 +24,13 @@ module Purplelight
       @manifest = manifest
       @single_file = single_file
       @schema = schema
+      @rotate_rows = rotate_rows
 
       @closed = false
       @file_seq = 0
       @part_index = nil
       @pq_writer = nil
+      @rows_in_current_file = 0
 
       ensure_dependencies!
       reset_buffers
@@ -44,9 +46,8 @@ module Purplelight
     def close
       return if @closed
 
-      ensure_open!
       flush_all_row_groups
-      finalize_current_part!
+      finalize_current_part! if @writer_path
       @closed = true
     end
 
@@ -70,6 +71,7 @@ module Purplelight
       FileUtils.mkdir_p(@directory)
       @writer_path = next_part_path
       @part_index = @manifest&.open_part!(@writer_path) if @manifest
+      @rows_in_current_file = 0
     end
 
     # No-op; we now write once on close for simplicity
@@ -108,6 +110,7 @@ module Purplelight
     end
 
     def finalize_current_part!
+      return if @writer_path.nil?
       if @pq_writer
         @pq_writer.close
         @pq_writer = nil
@@ -115,6 +118,8 @@ module Purplelight
       @manifest&.complete_part!(index: @part_index, checksum: nil)
       @file_seq += 1 unless @single_file
       @writer_path = nil
+      @part_index = nil
+      @rows_in_current_file = 0
     end
 
     def next_part_path
@@ -147,6 +152,7 @@ module Purplelight
       return if @buffer_docs.empty?
 
       while @buffer_docs.length >= @row_group_size
+        ensure_open!
         group = @buffer_docs.shift(@row_group_size)
         t_tbl = Thread.current[:pl_telemetry]&.start(:parquet_table_build_time)
         table = build_table(group)
@@ -155,6 +161,8 @@ module Purplelight
         t_w = Thread.current[:pl_telemetry]&.start(:parquet_write_time)
         write_table(table, @writer_path, append: true)
         Thread.current[:pl_telemetry]&.finish(:parquet_write_time, t_w)
+        @rows_in_current_file += group.length
+        maybe_rotate!
       end
     end
 
@@ -171,9 +179,21 @@ module Purplelight
       Thread.current[:pl_telemetry]&.finish(:parquet_table_build_time, t_tbl)
 
       t_w = Thread.current[:pl_telemetry]&.start(:parquet_write_time)
+      ensure_open!
       write_table(table, @writer_path, append: true)
       Thread.current[:pl_telemetry]&.finish(:parquet_write_time, t_w)
       @buffer_docs.clear
+      @rows_in_current_file += table.n_rows if table.respond_to?(:n_rows)
+      @rows_in_current_file += @buffer_docs.length unless table.respond_to?(:n_rows)
+      maybe_rotate!
+    end
+
+    def maybe_rotate!
+      return if @single_file
+      return unless @rotate_rows && @rows_in_current_file >= @rotate_rows
+
+      finalize_current_part!
+      # Next write will open a new part
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: purplelight
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.11
 platform: ruby
 authors:
 - Alexander Nicholson