purplelight 0.1.9 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b87960253dbd1ab6aae3b60dc790068d851f3798b124c23451bdae96734d6d67
-  data.tar.gz: b1eab05f8580a282b836da8eddb5dfe964ef6cb90a94300304ecd0426f786998
+  metadata.gz: 80f6e48231b485750fc65529ada74a07758befba9b324b58ef372f077305a144
+  data.tar.gz: 4d7eed034f90155d2686da45a76caa73928cbe2d080d2031539830f6f4399cfb
 SHA512:
-  metadata.gz: 7bff1db0acebc6416b7dd484fe882947bc74927a6833e99a0fec64d03203babfbf625f44c6a8d6c29cab31a6bc7ccae31de3a7d0b55283d073053a21515faeb3
-  data.tar.gz: b56bd93e12571aafe2ab47a1dc087d3429c4a15a731d50159552fbe70a0f63b40ee2d44fb23bf27752045df9f6e146376af906a00afdfada7e068420a4012925
+  metadata.gz: 63f49c8dab688ec5cd922304b472ee448aaada1a3e2c113b7a4ddbe2092f3a3d2a83e1fe396066e87971514d8069831486c1a5ad972e604807c6c3289efd8e31
+  data.tar.gz: 6a281a23a0abf3244045b3e99af606881f3f85bab7eeabd7b4ca94e36c823d2405d7e85b60048ec62e5a7ace63fceddd268e185a831398cc7dbd00c213198961

data/README.md

@@ -9,7 +9,7 @@ Purplelight is published on RubyGems: [purplelight on RubyGems](https://rubygems
 Add to your Gemfile:
 
 ```ruby
-gem 'purplelight', '~> 0.1.9'
+gem 'purplelight', '~> 0.1.11'
 ```
 
 Or install directly:
@@ -138,10 +138,44 @@ Purplelight.snapshot(
   output: '/data/exports',
   format: :parquet,
   sharding: { mode: :single_file, prefix: 'users' },
+  # Optional: tune row group size
+  # parquet_row_group: 50_000,
   resume: { enabled: true }
 )
 ```
 
+Parquet multi-part (rows-based rotation):
+
+```ruby
+Purplelight.snapshot(
+  client: client,
+  collection: 'users',
+  output: '/data/exports',
+  format: :parquet,
+  # Any mode other than :single_file enables multi-part filenames for Parquet
+  sharding: { mode: :by_size, prefix: 'users_parquet_parts' },
+  # Split into multiple .parquet files, limiting rows per file
+  parquet_max_rows: 100_000,
+  # Optional: Parquet row group size (rows)
+  parquet_row_group: 10_000,
+  resume: { enabled: true }
+)
+```
+
+Notes for Parquet:
+- Parquet multi-part sizing is controlled by rows via `parquet_max_rows`.
+- `--rotate-mb` / `part_bytes` do not affect Parquet part size; they apply to JSONL/CSV.
+- Use `sharding: { mode: :single_file }` to force a single `.parquet` file.
+
+### Environment variables (optional)
+
+CLI flags take precedence, but these environment variables can set sensible defaults:
+
+- `PL_ZSTD_LEVEL`: default zstd compression level used by writers.
+- `PL_WRITE_CHUNK_BYTES`: JSONL join/write chunk size in bytes.
+- `PL_PARQUET_ROW_GROUP`: default Parquet row group size (rows).
+- `PL_TELEMETRY`: set to `1` to enable telemetry by default.
+
 ### CLI
 
 ```bash
@@ -149,9 +183,48 @@ bundle exec bin/purplelight \
   --uri "$MONGO_URL" \
   --db mydb --collection users \
   --output /data/exports \
-  --format jsonl --partitions 8 --by-size $((256*1024*1024)) --prefix users
+  --format jsonl --partitions 8 --by-size $((256*1024*1024)) --prefix users \
+  --queue-mb 512 --rotate-mb 512 --compression zstd --compression-level 6 \
+  --read-preference secondary --read-tags nodeType=ANALYTICS,region=EAST \
+  --read-concern majority --no-cursor-timeout true
 ```
 
+#### CLI options (reference)
+
+- `--uri URI` (required): Mongo connection string.
+- `--db NAME` (required): Database name.
+- `--collection NAME` (required): Collection name.
+- `--output PATH` (required): Output directory or file path.
+- `--format FORMAT`: `jsonl|csv|parquet` (default `jsonl`).
+- `--compression NAME`: `zstd|gzip|none` (default `zstd`).
+- `--compression-level N`: Compression level (zstd or gzip; writer-specific defaults if omitted).
+- `--partitions N`: Number of reader partitions (defaults to ≥4 and ≤32 based on cores).
+- `--batch-size N`: Mongo batch size (default 2000).
+- `--queue-mb MB`: In-memory queue size in MB (default 256).
+- `--rotate-mb MB`: Target rotate size for JSONL/CSV parts in MB (default 256). For multi-part outputs, pairs well with `--by-size`.
+- `--by-size BYTES`: Plan size-based sharding for multi-part outputs.
+- `--single-file`: Single output file (CSV/Parquet; JSONL remains multi-part).
+- `--prefix NAME`: Output filename prefix (defaults to collection name when output is a directory).
+- `--query JSON`: Filter as JSON/Extended JSON (supports `$date`, `$oid`, etc.).
+- `--projection JSON`: Projection as JSON, e.g. `{"_id":1,"email":1}`.
+- `--read-preference MODE`: `primary|primary_preferred|secondary|secondary_preferred|nearest`.
+- `--read-tags key=value[,key=value...]`: Tag sets for node pinning.
+- `--read-concern LEVEL`: `majority|local|linearizable|available|snapshot`.
+- `--no-cursor-timeout BOOL`: Toggle `noCursorTimeout` (default true).
+- `--parquet-row-group N`: Parquet row group size (rows).
+- `--write-chunk-mb MB`: JSONL encode/write chunk size before enqueueing.
+- `--writer-threads N` (experimental): Number of writer threads (JSONL only).
+- `--telemetry on|off`: Force enable/disable telemetry output.
+- `--resume-overwrite-incompatible`: Overwrite an existing incompatible manifest to safely resume anew.
+- `--dry-run`: Print effective read preference JSON and exit (no snapshot).
+- `--version`, `--help`: Utility commands.
+
+Notes:
+- Compression backend selection order is: requested format → `zstd-ruby` → `zstds` → `gzip`.
+- `--single-file` and `--by-size` update only the sharding mode/params and preserve any provided `--prefix`.
+- Parquet multi-part sizing is programmatic via `parquet_max_rows`; there is no CLI flag for it.
+- To increase concurrent connections, set `maxPoolSize` on your Mongo URI (used by `--uri`), e.g., `mongodb://.../?maxPoolSize=32`. A good starting point is `maxPoolSize >= --partitions`.
+
 ### Architecture
 
 ```mermaid
@@ -181,19 +254,29 @@ Key points:
 
 ### Tuning for performance
 
-- Partitions: start with `2 × cores` (default). Increase gradually if reads are underutilized; too high can add overhead.
-- Batch size: 2k–10k usually works well. Larger batches reduce cursor roundtrips, but can raise latency/memory.
-- Queue size: increase to 256–512MB to reduce backpressure on readers for fast disks.
-- Compression: use `:zstd` for good ratio; for max speed, try `:gzip` with low level.
-- Rotation size: larger (512MB–1GB) reduces finalize overhead for many parts.
-- Read preference: offload to secondaries or tagged analytics nodes when available.
+- **Partitions**: start with `2 × cores` (default). Increase gradually if reads are underutilized; too high can add overhead. CLI: `--partitions`.
+- **Batch size**: 2k–10k usually works well. Larger batches reduce cursor roundtrips, but can raise latency/memory. CLI: `--batch-size`.
+- **Queue size**: increase to 256–512MB to reduce backpressure on readers for fast disks. CLI: `--queue-mb`.
+- **Compression**: prefer `zstd`; adjust level to balance speed/ratio. CLI: `--compression zstd --compression-level N`. For max speed, try `--compression gzip --compression-level 1`.
+- **Rotation size**: larger (512MB–1GB) reduces finalize overhead for many parts. CLI: `--rotate-mb` (and/or `--by-size`).
+- **JSONL chunking**: tune builder write chunk size for throughput. CLI: `--write-chunk-mb`.
+- **Parquet row groups**: choose a row group size that fits downstream readers. CLI: `--parquet-row-group`.
+- **Parquet parts (rows)**: split Parquet outputs by rows with `parquet_max_rows` (programmatic API). Set `sharding.mode` to anything other than `:single_file` to enable multi-part filenames.
+- **Read preference**: offload to secondaries or tagged analytics nodes when available. CLI: `--read-preference`, `--read-tags`.
+- **Read concern**: pick an appropriate level for consistency/latency trade-offs. CLI: `--read-concern`.
+- **Cursor timeout**: for very long scans, leave `noCursorTimeout` enabled. CLI: `--no-cursor-timeout true|false`.
+- **Telemetry**: enable to inspect timing breakdowns; disable for minimal output. CLI: `--telemetry on|off`.
 
 Benchmarking (optional):
 
 ```bash
-# 1M docs benchmark with tunables
+# 1M docs benchmark with tunables (JSONL)
 BENCH=1 BENCH_PARTITIONS=16 BENCH_BATCH_SIZE=8000 BENCH_QUEUE_MB=512 BENCH_ROTATE_MB=512 BENCH_COMPRESSION=gzip \
   bundle exec rspec spec/benchmark_perf_spec.rb --format doc
+
+# Parquet benchmark (requires Arrow/Parquet)
+BENCH=1 BENCH_FORMAT=parquet BENCH_PARQUET_ROW_GROUP=50000 BENCH_PARTITIONS=16 BENCH_BATCH_SIZE=8000 \
+  bundle exec rspec spec/benchmark_perf_spec.rb --format doc
 ```
 
 ### Read preference and node pinning
@@ -262,3 +345,8 @@ Benchmark results:
 Finished in 14.02 seconds (files took 0.31974 seconds to load)
 1 example, 0 failures
 ```
+
+Additional BENCH variables:
+
+- `BENCH_FORMAT`: `jsonl|parquet` (default `jsonl`).
+- `BENCH_PARQUET_ROW_GROUP`: Parquet row group size (rows), e.g. `50000`.

data/lib/purplelight/snapshot.rb

@@ -39,7 +39,8 @@ module Purplelight
                    sharding: { mode: :by_size, part_bytes: DEFAULTS[:rotate_bytes], prefix: nil },
                    logger: nil, on_progress: nil, read_concern: DEFAULTS[:read_concern], read_preference: DEFAULTS[:read_preference],
                    no_cursor_timeout: DEFAULTS[:no_cursor_timeout], telemetry: nil,
-                   compression_level: nil, writer_threads: 1, write_chunk_bytes: nil, parquet_row_group: nil)
+                   compression_level: nil, writer_threads: 1, write_chunk_bytes: nil, parquet_row_group: nil,
+                   parquet_max_rows: nil)
       @client = client
       @collection = client[collection]
       @output = output
@@ -64,6 +65,7 @@ module Purplelight
       @writer_threads = writer_threads || 1
       @write_chunk_bytes = write_chunk_bytes
       @parquet_row_group = parquet_row_group
+      @parquet_max_rows = parquet_max_rows
 
       @running = true
       @telemetry_enabled = telemetry ? telemetry.enabled? : (ENV['PL_TELEMETRY'] == '1')
@@ -106,6 +108,7 @@ module Purplelight
                             compression_level: @compression_level || (ENV['PL_ZSTD_LEVEL']&.to_i if @compression.to_s == 'zstd') || ENV['PL_ZSTD_LEVEL']&.to_i,
                             write_chunk_bytes: @write_chunk_bytes || ENV['PL_WRITE_CHUNK_BYTES']&.to_i,
                             parquet_row_group: @parquet_row_group || ENV['PL_PARQUET_ROW_GROUP']&.to_i,
+                            parquet_max_rows: @parquet_max_rows,
                             sharding: @sharding,
                             resume_overwrite_incompatible: @resume && @resume[:overwrite_incompatible] ? true : false,
                             telemetry: @telemetry_enabled
@@ -134,7 +137,8 @@ module Purplelight
                  single_file = @sharding && @sharding[:mode].to_s == 'single_file'
                  row_group = @parquet_row_group || ENV['PL_PARQUET_ROW_GROUP']&.to_i || WriterParquet::DEFAULT_ROW_GROUP_SIZE
                  WriterParquet.new(directory: dir, prefix: prefix, compression: @compression, logger: @logger,
-                                   manifest: manifest, single_file: single_file, row_group_size: row_group)
+                          manifest: manifest, single_file: single_file, row_group_size: row_group,
+                          rotate_rows: @parquet_max_rows)
                else
                  raise ArgumentError, "format not implemented: #{@format}"
                end

data/lib/purplelight/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Purplelight
-  VERSION = '0.1.9'
+  VERSION = '0.1.11'
 end

data/lib/purplelight/writer_parquet.rb

@@ -15,7 +15,7 @@ module Purplelight
     DEFAULT_ROW_GROUP_SIZE = 10_000
 
     def initialize(directory:, prefix:, compression: :zstd, row_group_size: DEFAULT_ROW_GROUP_SIZE, logger: nil,
-                   manifest: nil, single_file: true, schema: nil)
+                   manifest: nil, single_file: true, schema: nil, rotate_rows: nil)
       @directory = directory
       @prefix = prefix
       @compression = compression
@@ -24,10 +24,13 @@ module Purplelight
       @manifest = manifest
       @single_file = single_file
       @schema = schema
+      @rotate_rows = rotate_rows
 
       @closed = false
       @file_seq = 0
       @part_index = nil
+      @pq_writer = nil
+      @rows_in_current_file = 0
 
       ensure_dependencies!
       reset_buffers
@@ -36,23 +39,15 @@ module Purplelight
     def write_many(array_of_docs)
       ensure_open!
       array_of_docs.each { |doc| @buffer_docs << doc }
+      flush_row_groups_if_needed
       @manifest&.add_progress_to_part!(index: @part_index, rows_delta: array_of_docs.length, bytes_delta: 0)
     end
 
     def close
       return if @closed
 
-      ensure_open!
-      unless @buffer_docs.empty?
-        t_tbl = Thread.current[:pl_telemetry]&.start(:parquet_table_build_time)
-        table = build_table(@buffer_docs)
-        Thread.current[:pl_telemetry]&.finish(:parquet_table_build_time, t_tbl)
-
-        t_w = Thread.current[:pl_telemetry]&.start(:parquet_write_time)
-        write_table(table, @writer_path, append: false)
-        Thread.current[:pl_telemetry]&.finish(:parquet_write_time, t_w)
-      end
-      finalize_current_part!
+      flush_all_row_groups
+      finalize_current_part! if @writer_path
       @closed = true
     end
 
@@ -76,6 +71,7 @@ module Purplelight
       FileUtils.mkdir_p(@directory)
       @writer_path = next_part_path
       @part_index = @manifest&.open_part!(@writer_path) if @manifest
+      @rows_in_current_file = 0
     end
 
     # No-op; we now write once on close for simplicity
@@ -92,25 +88,38 @@ module Purplelight
     end
 
     def write_table(table, path, append: false) # rubocop:disable Lint/UnusedMethodArgument
-      # Prefer Arrow's save with explicit parquet format; compression defaults per build.
-      if table.respond_to?(:save)
-        table.save(path, format: :parquet)
+      # Stream via ArrowFileWriter when available to avoid building huge tables
+      if defined?(Parquet::ArrowFileWriter)
+        unless @pq_writer
+          @pq_writer = Parquet::ArrowFileWriter.open(table.schema, path)
+        end
+        # Prefer passing row_group_size; fallback to single-arg for older APIs
+        begin
+          @pq_writer.write_table(table, @row_group_size)
+        rescue ArgumentError
+          @pq_writer.write_table(table)
+        end
         return
       end
-      # Fallback to red-parquet writer
-      if defined?(Parquet::ArrowFileWriter)
-        writer = Parquet::ArrowFileWriter.open(table.schema, path)
-        writer.write_table(table)
-        writer.close
+      # Fallback to one-shot save when streaming API is not available
+      if table.respond_to?(:save)
+        table.save(path, format: :parquet)
         return
       end
       raise 'Parquet writer not available in this environment'
     end
 
     def finalize_current_part!
+      return if @writer_path.nil?
+      if @pq_writer
+        @pq_writer.close
+        @pq_writer = nil
+      end
       @manifest&.complete_part!(index: @part_index, checksum: nil)
       @file_seq += 1 unless @single_file
       @writer_path = nil
+      @part_index = nil
+      @rows_in_current_file = 0
     end
 
     def next_part_path
@@ -138,5 +147,53 @@ module Purplelight
 
       value
     end
+
+    def flush_row_groups_if_needed
+      return if @buffer_docs.empty?
+
+      while @buffer_docs.length >= @row_group_size
+        ensure_open!
+        group = @buffer_docs.shift(@row_group_size)
+        t_tbl = Thread.current[:pl_telemetry]&.start(:parquet_table_build_time)
+        table = build_table(group)
+        Thread.current[:pl_telemetry]&.finish(:parquet_table_build_time, t_tbl)
+
+        t_w = Thread.current[:pl_telemetry]&.start(:parquet_write_time)
+        write_table(table, @writer_path, append: true)
+        Thread.current[:pl_telemetry]&.finish(:parquet_write_time, t_w)
+        @rows_in_current_file += group.length
+        maybe_rotate!
+      end
+    end
+
+    def flush_all_row_groups
+      return if @buffer_docs.empty?
+
+      # Flush any full groups first
+      flush_row_groups_if_needed
+      return if @buffer_docs.empty?
+
+      # Flush remaining as a final smaller group
+      t_tbl = Thread.current[:pl_telemetry]&.start(:parquet_table_build_time)
+      table = build_table(@buffer_docs)
+      Thread.current[:pl_telemetry]&.finish(:parquet_table_build_time, t_tbl)
+
+      t_w = Thread.current[:pl_telemetry]&.start(:parquet_write_time)
+      ensure_open!
+      write_table(table, @writer_path, append: true)
+      Thread.current[:pl_telemetry]&.finish(:parquet_write_time, t_w)
+      @buffer_docs.clear
+      @rows_in_current_file += table.n_rows if table.respond_to?(:n_rows)
+      @rows_in_current_file += @buffer_docs.length unless table.respond_to?(:n_rows)
+      maybe_rotate!
+    end
+
+    def maybe_rotate!
+      return if @single_file
+      return unless @rotate_rows && @rows_in_current_file >= @rotate_rows
+
+      finalize_current_part!
+      # Next write will open a new part
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: purplelight
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.11
 platform: ruby
 authors:
 - Alexander Nicholson