purplelight 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b87960253dbd1ab6aae3b60dc790068d851f3798b124c23451bdae96734d6d67
-  data.tar.gz: b1eab05f8580a282b836da8eddb5dfe964ef6cb90a94300304ecd0426f786998
+  metadata.gz: f0f51fd601a59915a2a022831663fd4f2468e781b68b96f59d396359be49adbc
+  data.tar.gz: c899a18e7ce390bfc05f832dd32248aa8cbdc7b43bccf86197350e2c7929e7a6
 SHA512:
-  metadata.gz: 7bff1db0acebc6416b7dd484fe882947bc74927a6833e99a0fec64d03203babfbf625f44c6a8d6c29cab31a6bc7ccae31de3a7d0b55283d073053a21515faeb3
-  data.tar.gz: b56bd93e12571aafe2ab47a1dc087d3429c4a15a731d50159552fbe70a0f63b40ee2d44fb23bf27752045df9f6e146376af906a00afdfada7e068420a4012925
+  metadata.gz: f6546911873ed22865b9d4cdd2cc62d855ab3b991030808d8f49f3e054727a406b80c7dc43c518a450915152f2934dcba180d53bf75807c540eef893b3ca50b8
+  data.tar.gz: 5e7176eec64956388e72fd3d894db12e006a18edd4aade7eaf13b144381802932d7207ffefac6ad06157c03363f41acec4ca997871fb8abe8efc9e06e2238804

data/README.md

@@ -9,7 +9,7 @@ Purplelight is published on RubyGems: [purplelight on RubyGems](https://rubygems
 Add to your Gemfile:
 
 ```ruby
-gem 'purplelight', '~> 0.1.9'
+gem 'purplelight', '~> 0.1.10'
 ```
 
 Or install directly:
@@ -138,10 +138,21 @@ Purplelight.snapshot(
   output: '/data/exports',
   format: :parquet,
   sharding: { mode: :single_file, prefix: 'users' },
+  # Optional: tune row group size
+  # parquet_row_group: 50_000,
   resume: { enabled: true }
 )
 ```
 
+### Environment variables (optional)
+
+CLI flags take precedence, but these environment variables can set sensible defaults:
+
+- `PL_ZSTD_LEVEL`: default zstd compression level used by writers.
+- `PL_WRITE_CHUNK_BYTES`: JSONL join/write chunk size in bytes.
+- `PL_PARQUET_ROW_GROUP`: default Parquet row group size (rows).
+- `PL_TELEMETRY`: set to `1` to enable telemetry by default.
+
 ### CLI
 
 ```bash
@@ -149,9 +160,46 @@ bundle exec bin/purplelight \
   --uri "$MONGO_URL" \
   --db mydb --collection users \
   --output /data/exports \
-  --format jsonl --partitions 8 --by-size $((256*1024*1024)) --prefix users
+  --format jsonl --partitions 8 --by-size $((256*1024*1024)) --prefix users \
+  --queue-mb 512 --rotate-mb 512 --compression zstd --compression-level 6 \
+  --read-preference secondary --read-tags nodeType=ANALYTICS,region=EAST \
+  --read-concern majority --no-cursor-timeout true
 ```
 
+#### CLI options (reference)
+
+- `--uri URI` (required): Mongo connection string.
+- `--db NAME` (required): Database name.
+- `--collection NAME` (required): Collection name.
+- `--output PATH` (required): Output directory or file path.
+- `--format FORMAT`: `jsonl|csv|parquet` (default `jsonl`).
+- `--compression NAME`: `zstd|gzip|none` (default `zstd`).
+- `--compression-level N`: Compression level (zstd or gzip; writer-specific defaults if omitted).
+- `--partitions N`: Number of reader partitions (defaults to ≥4 and ≤32 based on cores).
+- `--batch-size N`: Mongo batch size (default 2000).
+- `--queue-mb MB`: In-memory queue size in MB (default 256).
+- `--rotate-mb MB`: Target rotate size for JSONL/CSV parts in MB (default 256). For multi-part outputs, pairs well with `--by-size`.
+- `--by-size BYTES`: Plan size-based sharding for multi-part outputs.
+- `--single-file`: Single output file (CSV/Parquet; JSONL remains multi-part).
+- `--prefix NAME`: Output filename prefix (defaults to collection name when output is a directory).
+- `--query JSON`: Filter as JSON/Extended JSON (supports `$date`, `$oid`, etc.).
+- `--projection JSON`: Projection as JSON, e.g. `{"_id":1,"email":1}`.
+- `--read-preference MODE`: `primary|primary_preferred|secondary|secondary_preferred|nearest`.
+- `--read-tags key=value[,key=value...]`: Tag sets for node pinning.
+- `--read-concern LEVEL`: `majority|local|linearizable|available|snapshot`.
+- `--no-cursor-timeout BOOL`: Toggle `noCursorTimeout` (default true).
+- `--parquet-row-group N`: Parquet row group size (rows).
+- `--write-chunk-mb MB`: JSONL encode/write chunk size before enqueueing.
+- `--writer-threads N` (experimental): Number of writer threads (JSONL only).
+- `--telemetry on|off`: Force enable/disable telemetry output.
+- `--resume-overwrite-incompatible`: Overwrite an existing incompatible manifest to safely resume anew.
+- `--dry-run`: Print effective read preference JSON and exit (no snapshot).
+- `--version`, `--help`: Utility commands.
+
+Notes:
+- Compression backend selection order is: requested format → `zstd-ruby` → `zstds` → `gzip`.
+- `--single-file` and `--by-size` update only the sharding mode/params and preserve any provided `--prefix`.
+
 ### Architecture
 
 ```mermaid
@@ -181,19 +229,28 @@ Key points:
 
 ### Tuning for performance
 
-- Partitions: start with `2 × cores` (default). Increase gradually if reads are underutilized; too high can add overhead.
-- Batch size: 2k–10k usually works well. Larger batches reduce cursor roundtrips, but can raise latency/memory.
-- Queue size: increase to 256–512MB to reduce backpressure on readers for fast disks.
-- Compression: use `:zstd` for good ratio; for max speed, try `:gzip` with low level.
-- Rotation size: larger (512MB–1GB) reduces finalize overhead for many parts.
-- Read preference: offload to secondaries or tagged analytics nodes when available.
+- **Partitions**: start with `2 × cores` (default). Increase gradually if reads are underutilized; too high can add overhead. CLI: `--partitions`.
+- **Batch size**: 2k–10k usually works well. Larger batches reduce cursor roundtrips, but can raise latency/memory. CLI: `--batch-size`.
+- **Queue size**: increase to 256–512MB to reduce backpressure on readers for fast disks. CLI: `--queue-mb`.
+- **Compression**: prefer `zstd`; adjust level to balance speed/ratio. CLI: `--compression zstd --compression-level N`. For max speed, try `--compression gzip --compression-level 1`.
+- **Rotation size**: larger (512MB–1GB) reduces finalize overhead for many parts. CLI: `--rotate-mb` (and/or `--by-size`).
+- **JSONL chunking**: tune builder write chunk size for throughput. CLI: `--write-chunk-mb`.
+- **Parquet row groups**: choose a row group size that fits downstream readers. CLI: `--parquet-row-group`.
+- **Read preference**: offload to secondaries or tagged analytics nodes when available. CLI: `--read-preference`, `--read-tags`.
+- **Read concern**: pick an appropriate level for consistency/latency trade-offs. CLI: `--read-concern`.
+- **Cursor timeout**: for very long scans, leave `noCursorTimeout` enabled. CLI: `--no-cursor-timeout true|false`.
+- **Telemetry**: enable to inspect timing breakdowns; disable for minimal output. CLI: `--telemetry on|off`.
 
 Benchmarking (optional):
 
 ```bash
-# 1M docs benchmark with tunables
+# 1M docs benchmark with tunables (JSONL)
 BENCH=1 BENCH_PARTITIONS=16 BENCH_BATCH_SIZE=8000 BENCH_QUEUE_MB=512 BENCH_ROTATE_MB=512 BENCH_COMPRESSION=gzip \
   bundle exec rspec spec/benchmark_perf_spec.rb --format doc
+
+# Parquet benchmark (requires Arrow/Parquet)
+BENCH=1 BENCH_FORMAT=parquet BENCH_PARQUET_ROW_GROUP=50000 BENCH_PARTITIONS=16 BENCH_BATCH_SIZE=8000 \
+  bundle exec rspec spec/benchmark_perf_spec.rb --format doc
 ```
 
 ### Read preference and node pinning
@@ -262,3 +319,8 @@ Benchmark results:
 Finished in 14.02 seconds (files took 0.31974 seconds to load)
 1 example, 0 failures
 ```
+
+Additional BENCH variables:
+
+- `BENCH_FORMAT`: `jsonl|parquet` (default `jsonl`).
+- `BENCH_PARQUET_ROW_GROUP`: Parquet row group size (rows), e.g. `50000`.

data/lib/purplelight/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Purplelight
-  VERSION = '0.1.9'
+  VERSION = '0.1.10'
 end

data/lib/purplelight/writer_parquet.rb

@@ -28,6 +28,7 @@ module Purplelight
       @closed = false
       @file_seq = 0
       @part_index = nil
+      @pq_writer = nil
 
       ensure_dependencies!
       reset_buffers
@@ -36,6 +37,7 @@ module Purplelight
     def write_many(array_of_docs)
       ensure_open!
       array_of_docs.each { |doc| @buffer_docs << doc }
+      flush_row_groups_if_needed
       @manifest&.add_progress_to_part!(index: @part_index, rows_delta: array_of_docs.length, bytes_delta: 0)
     end
 
@@ -43,15 +45,7 @@ module Purplelight
       return if @closed
 
       ensure_open!
-      unless @buffer_docs.empty?
-        t_tbl = Thread.current[:pl_telemetry]&.start(:parquet_table_build_time)
-        table = build_table(@buffer_docs)
-        Thread.current[:pl_telemetry]&.finish(:parquet_table_build_time, t_tbl)
-
-        t_w = Thread.current[:pl_telemetry]&.start(:parquet_write_time)
-        write_table(table, @writer_path, append: false)
-        Thread.current[:pl_telemetry]&.finish(:parquet_write_time, t_w)
-      end
+      flush_all_row_groups
       finalize_current_part!
       @closed = true
     end
@@ -92,22 +86,32 @@ module Purplelight
     end
 
     def write_table(table, path, append: false) # rubocop:disable Lint/UnusedMethodArgument
-      # Prefer Arrow's save with explicit parquet format; compression defaults per build.
-      if table.respond_to?(:save)
-        table.save(path, format: :parquet)
+      # Stream via ArrowFileWriter when available to avoid building huge tables
+      if defined?(Parquet::ArrowFileWriter)
+        unless @pq_writer
+          @pq_writer = Parquet::ArrowFileWriter.open(table.schema, path)
+        end
+        # Prefer passing row_group_size; fallback to single-arg for older APIs
+        begin
+          @pq_writer.write_table(table, @row_group_size)
+        rescue ArgumentError
+          @pq_writer.write_table(table)
+        end
         return
       end
-      # Fallback to red-parquet writer
-      if defined?(Parquet::ArrowFileWriter)
-        writer = Parquet::ArrowFileWriter.open(table.schema, path)
-        writer.write_table(table)
-        writer.close
+      # Fallback to one-shot save when streaming API is not available
+      if table.respond_to?(:save)
+        table.save(path, format: :parquet)
         return
       end
       raise 'Parquet writer not available in this environment'
     end
 
     def finalize_current_part!
+      if @pq_writer
+        @pq_writer.close
+        @pq_writer = nil
+      end
       @manifest&.complete_part!(index: @part_index, checksum: nil)
       @file_seq += 1 unless @single_file
       @writer_path = nil
@@ -138,5 +142,38 @@ module Purplelight
 
       value
     end
+
+    def flush_row_groups_if_needed
+      return if @buffer_docs.empty?
+
+      while @buffer_docs.length >= @row_group_size
+        group = @buffer_docs.shift(@row_group_size)
+        t_tbl = Thread.current[:pl_telemetry]&.start(:parquet_table_build_time)
+        table = build_table(group)
+        Thread.current[:pl_telemetry]&.finish(:parquet_table_build_time, t_tbl)
+
+        t_w = Thread.current[:pl_telemetry]&.start(:parquet_write_time)
+        write_table(table, @writer_path, append: true)
+        Thread.current[:pl_telemetry]&.finish(:parquet_write_time, t_w)
+      end
+    end
+
+    def flush_all_row_groups
+      return if @buffer_docs.empty?
+
+      # Flush any full groups first
+      flush_row_groups_if_needed
+      return if @buffer_docs.empty?
+
+      # Flush remaining as a final smaller group
+      t_tbl = Thread.current[:pl_telemetry]&.start(:parquet_table_build_time)
+      table = build_table(@buffer_docs)
+      Thread.current[:pl_telemetry]&.finish(:parquet_table_build_time, t_tbl)
+
+      t_w = Thread.current[:pl_telemetry]&.start(:parquet_write_time)
+      write_table(table, @writer_path, append: true)
+      Thread.current[:pl_telemetry]&.finish(:parquet_write_time, t_w)
+      @buffer_docs.clear
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: purplelight
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Alexander Nicholson