purplelight 0.1.12 → 0.1.16

data/benchmark/microbench.rb

@@ -0,0 +1,344 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'fileutils'
+require 'json'
+require 'stringio'
+require 'tmpdir'
+require 'time'
+require 'purplelight'
+require_relative 'harness'
+
+DOCS = Array.new(200) do |index|
+  {
+    '_id' => BSON::ObjectId.new,
+    'index' => index,
+    'active' => index.even?,
+    'tags' => %w[alpha beta],
+    'nested' => { 'value' => index }
+  }
+end.freeze
+PARQUET_DOCS = Array.new(10_000) do |index|
+  {
+    '_id' => BSON::ObjectId.new,
+    'index' => index,
+    'active' => index.even?,
+    'tags' => %w[alpha beta],
+    'nested' => { 'value' => index }
+  }
+end.freeze
+ENCODED_JSONL = DOCS.map { |document| "#{JSON.generate(document)}\n" }.join * 10
+ENCODED_BATCH = Purplelight::WriterJSONL::EncodedBatch.new(
+  data: ENCODED_JSONL,
+  rows: DOCS.length * 10,
+  bytes: ENCODED_JSONL.bytesize
+)
+PARTITION_DOCS = Array.new(5_001) { |index| { '_id' => index } }.freeze
+pipeline_random = Random.new(12_345)
+PIPELINE_DOCS = Array.new(20_000) do |index|
+  object_id = BSON::ObjectId.new.tap(&:to_s)
+  { '_id' => object_id, 'index' => index, 'payload' => pipeline_random.bytes(512).unpack1('H*') }
+end.freeze
+
+FakeCollection = Data.define(:name)
+FakeClient = Class.new do
+  def [](name)
+    FakeCollection.new(name.to_s)
+  end
+end
+EmptyCursor = Class.new do
+  def projection(*) = self
+  def sort(*) = self
+  def limit(*) = self
+  def first = nil
+  def each = nil
+end
+
+EmptyCollection = Data.define(:name) do
+  def find(*) = EmptyCursor.new
+  def estimated_document_count = 0
+end
+
+EmptyClient = Class.new do
+  def [](name) = EmptyCollection.new(name.to_s)
+end
+
+RawCursor = Class.new do
+  def initialize(documents)
+    @documents = documents
+  end
+
+  def each(&) = @documents.each(&)
+end
+
+RawCollection = Data.define(:name, :documents) do
+  def find(*) = RawCursor.new(documents)
+end
+
+CollectionClient = Data.define(:collection) do
+  def [](*) = collection
+end
+
+PipelineCursor = Class.new do
+  include Enumerable
+
+  def initialize(documents)
+    @documents = documents
+    @direction = 1
+    @limit = nil
+  end
+
+  def projection(*) = self
+  def hint(*) = self
+  def batch_size(*) = self
+  def no_cursor_timeout = self
+  def skip(*) = self
+
+  def sort(specification)
+    @direction = specification[:_id] || specification['_id'] || 1
+    self
+  end
+
+  def limit(count)
+    @limit = count
+    self
+  end
+
+  def to_a
+    documents = @direction.negative? ? @documents.reverse : @documents
+    @limit ? documents.first(@limit) : documents
+  end
+
+  def first = to_a.first
+  def each(&) = to_a.each(&)
+end
+
+PipelineCollection = Data.define(:name, :documents) do
+  def find(*) = PipelineCursor.new(documents)
+  def estimated_document_count = documents.length
+end
+
+SinkQueue = Class.new do
+  def push(*) = nil
+end
+
+SinkManifest = Class.new do
+  def partition_checkpoint(*) = nil
+  def update_partition_checkpoint!(*) = nil
+  def mark_partition_complete!(*) = nil
+end
+
+PlannerCursor = Class.new do
+  include Enumerable
+
+  def initialize(documents)
+    @documents = documents
+  end
+
+  def projection(*) = self
+  def batch_size(*) = self
+  def no_cursor_timeout = self
+  def each(&) = @documents.each(&)
+end
+
+PlannerCollection = Data.define(:documents) do
+  def find(*) = PlannerCursor.new(documents)
+  def estimated_document_count = 10_001
+end
+
+harness = Purplelight::Microbench::Harness.new
+
+Dir.mktmpdir('purplelight-microbench') do |directory|
+  sequence = 0
+
+  harness.register('byte_queue_round_trip', paths: :queue, iterations: 20_000) do
+    queue = Purplelight::ByteQueue.new(max_bytes: 1024)
+    queue.push(sequence, bytes: 8)
+    queue.pop
+  end
+  harness.register('byte_queue_bulk_drain', paths: :queue, iterations: 10) do
+    queue = Purplelight::ByteQueue.new(max_bytes: 50_000)
+    5_000.times { queue.push(sequence, bytes: 8) }
+    5_000.times { queue.pop }
+  end
+
+  telemetry = Purplelight::Telemetry.new
+  harness.register('telemetry_counter_and_timer', paths: :telemetry, iterations: 50_000) do
+    ticket = telemetry.start(:operation)
+    telemetry.add(:documents)
+    telemetry.finish(:operation, ticket)
+  end
+
+  harness.register('telemetry_construction', paths: :telemetry, iterations: 100_000) do
+    Purplelight::Telemetry.new
+  end
+
+  harness.register('manifest_progress', paths: :manifest, iterations: 250) do
+    sequence += 1
+    path = File.join(directory, "manifest-#{sequence}.json")
+    manifest = Purplelight::Manifest.new(path:)
+    manifest.configure!(collection: 'records', format: :jsonl, compression: :none,
+                        query_digest: Purplelight::Manifest.query_digest({}, nil))
+    manifest.ensure_partitions!(1)
+    index = manifest.open_part!("part-#{sequence}")
+    manifest.add_progress_to_part!(index:, rows_delta: 200, bytes_delta: 4096)
+    manifest.update_partition_checkpoint!(0, sequence)
+    manifest.mark_partition_complete!(0)
+    manifest.complete_part!(index:)
+  end
+
+  harness.register('partition_range_planning', paths: :partitioner, iterations: 100_000) do
+    Purplelight::Partitioner.build_range(sequence, sequence + 1)
+  end
+
+  planner_collection = PlannerCollection.new(PARTITION_DOCS)
+  harness.register('partition_cursor_sampling', paths: :partitioner, iterations: 10) do
+    Purplelight::Partitioner.cursor_sampling_partitions(collection: planner_collection, query: nil, partitions: 2)
+  end
+
+  client = FakeClient.new
+  harness.register('snapshot_configuration', paths: :snapshot, iterations: 10_000) do
+    snapshot = Purplelight::Snapshot.new(client:, collection: :records, output: directory,
+                                         format: :jsonl, compression: :none, partitions: 1)
+    snapshot.send(:resolve_output, directory, :jsonl)
+  end
+  empty_client = EmptyClient.new
+  harness.register('snapshot_empty_export', paths: :snapshot, iterations: 20) do
+    sequence += 1
+    output = File.join(directory, "snapshot-#{sequence}")
+    Purplelight::Snapshot.new(client: empty_client, collection: :records, output:, format: :jsonl,
+                              compression: :none, partitions: 1, resume: { enabled: false }).run
+  end
+
+  raw_collection = RawCollection.new('records', DOCS)
+  raw_snapshot = Purplelight::Snapshot.new(client: CollectionClient.new(raw_collection), collection: :records,
+                                           output: directory, format: :csv, compression: :none, batch_size: DOCS.length,
+                                           partitions: 1, resume: { enabled: false })
+  sink_queue = SinkQueue.new
+  sink_manifest = SinkManifest.new
+  harness.register('snapshot_raw_batching', paths: :snapshot, iterations: 100) do
+    raw_snapshot.send(:read_partition, idx: 0, filter_spec: { filter: {} }, queue: sink_queue,
+                                       batch_size: DOCS.length, manifest: sink_manifest)
+  end
+
+  pipeline_collection = PipelineCollection.new('pipeline', PIPELINE_DOCS)
+  pipeline_client = CollectionClient.new(pipeline_collection)
+  harness.register('snapshot_jsonl_serial_pipeline', paths: %i[manifest queue snapshot writer_jsonl], iterations: 3) do
+    sequence += 1
+    output = File.join(directory, "pipeline-serial-#{sequence}.jsonl")
+    Purplelight::Snapshot.new(client: pipeline_client, collection: :pipeline, output:, format: :jsonl,
+                              compression: :zstd, batch_size: 250, partitions: 1, writer_threads: 1,
+                              resume: { enabled: false }).run
+  end
+
+  harness.register('snapshot_jsonl_pipeline', paths: %i[manifest queue snapshot writer_jsonl], iterations: 3) do
+    sequence += 1
+    output = File.join(directory, "pipeline-#{sequence}.jsonl")
+    Purplelight::Snapshot.new(client: pipeline_client, collection: :pipeline, output:, format: :jsonl,
+                              compression: :zstd, batch_size: 250, partitions: 1, writer_threads: 2,
+                              resume: { enabled: false }).run
+  end
+
+  harness.register('jsonl_batch_write', paths: :writer_jsonl, iterations: 100) do
+    sequence += 1
+    writer = Purplelight::WriterJSONL.new(directory:, prefix: "jsonl-#{sequence}", compression: :none)
+    writer.write_many(DOCS)
+    writer.close
+  end
+
+  harness.register('jsonl_zstd_batch_write', paths: :writer_jsonl, iterations: 50) do
+    sequence += 1
+    writer = Purplelight::WriterJSONL.new(directory:, prefix: "jsonl-zstd-#{sequence}", compression: :zstd)
+    writer.write_many(DOCS)
+    writer.close
+  end
+
+  accounting_writer = Purplelight::WriterJSONL.new(
+    directory:, prefix: 'accounting', compression: :none, rotate_bytes: nil
+  )
+  accounting_io = StringIO.new
+  accounting_writer.instance_variable_set(:@io, accounting_io)
+  harness.register('jsonl_preencoded_accounting', paths: :writer_jsonl, iterations: 10_000) do
+    accounting_io.truncate(0)
+    accounting_io.rewind
+    accounting_writer.write_many(ENCODED_BATCH)
+  end
+
+  harness.register('csv_batch_write', paths: :writer_csv, iterations: 100) do
+    sequence += 1
+    writer = Purplelight::WriterCSV.new(directory:, prefix: "csv-#{sequence}", compression: :none,
+                                        single_file: true)
+    writer.write_many(DOCS)
+    writer.close
+  end
+
+  harness.register('csv_zstd_batch_write', paths: :writer_csv, iterations: 50) do
+    sequence += 1
+    writer = Purplelight::WriterCSV.new(directory:, prefix: "csv-zstd-#{sequence}", compression: :zstd,
+                                        single_file: true)
+    writer.write_many(DOCS)
+    writer.close
+  end
+
+  harness.register('csv_gzip_batch_write', paths: :writer_csv, iterations: 50) do
+    sequence += 1
+    writer = Purplelight::WriterCSV.new(directory:, prefix: "csv-gzip-#{sequence}", compression: :gzip,
+                                        compression_level: 1, single_file: true)
+    writer.write_many(DOCS)
+    writer.close
+  end
+  raw_csv_io = StringIO.new
+  counting_io_class = Purplelight::WriterCSV::CountingIO
+  has_callback = counting_io_class.instance_method(:initialize).parameters.any? { |_, name| name == :on_write }
+  counting_io = if has_callback
+                  counting_io_class.new(raw_csv_io, on_write: ->(*) {})
+                else
+                  counting_io_class.new(raw_csv_io)
+                end
+  harness.register('csv_counting_io_write', paths: :writer_csv, iterations: 100_000) do
+    counting_io.write('payload')
+  end
+  csv_serializer = Purplelight::WriterCSV.new(directory:, prefix: 'csv-serializer', compression: :none,
+                                              columns: DOCS.first.keys)
+  csv_serialized_row = +''
+  harness.register('csv_row_serialization', paths: :writer_csv, iterations: 25_000) do
+    csv_serialized_row.clear
+    csv_serializer.send(:append_csv_document, csv_serialized_row, DOCS.first)
+  end
+
+  parquet_buffer_writer = Purplelight::WriterParquet.new(directory:, prefix: 'parquet-buffer', compression: :none,
+                                                         row_group_size: 1_000_000, single_file: true)
+  harness.register('parquet_buffer_append', paths: :writer_parquet, iterations: 5_000) do
+    parquet_buffer_writer.write_many(DOCS)
+    parquet_buffer_writer.instance_variable_get(:@buffer_docs).clear
+  end
+
+  object_ids = DOCS.map { |document| document.fetch('_id') }.freeze
+  harness.register('parquet_object_id_array', paths: :writer_parquet, iterations: 1_000) do
+    parquet_buffer_writer.send(:build_object_id_array, object_ids)
+  end
+
+  harness.register('parquet_batch_write', paths: :writer_parquet, iterations: 10) do
+    sequence += 1
+    writer = Purplelight::WriterParquet.new(directory:, prefix: "parquet-#{sequence}", compression: :zstd,
+                                            row_group_size: 100, single_file: true)
+    writer.write_many(DOCS)
+    writer.close
+  end
+
+  harness.register('parquet_row_group_write', paths: :writer_parquet, iterations: 2) do
+    sequence += 1
+    writer = Purplelight::WriterParquet.new(directory:, prefix: "parquet-row-groups-#{sequence}",
+                                            compression: :zstd, row_group_size: 1_000, single_file: true)
+    writer.write_many(PARQUET_DOCS)
+    writer.close
+  end
+
+  baseline_index = ARGV.index('--baseline')
+  output_index = ARGV.index('--output')
+  report = harness.run(baseline_path: baseline_index && ARGV.fetch(baseline_index + 1))
+  rendered = JSON.pretty_generate(report)
+  puts rendered
+  File.write(ARGV.fetch(output_index + 1), "#{rendered}\n") if output_index
+end

data/benchmark/profile.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'fileutils'
+require 'optparse'
+require 'stackprof'
+
+mode = :cpu
+OptionParser.new do |parser|
+  parser.on('--mode MODE', %w[cpu object], 'StackProf sampling mode') { |value| mode = value.to_sym }
+end.parse!
+
+root = File.expand_path('..', __dir__)
+profile_directory = File.join(root, 'tmp', 'profiles')
+FileUtils.mkdir_p(profile_directory)
+dump_path = File.join(profile_directory, "#{mode}.dump")
+flamegraph_path = File.join(profile_directory, "#{mode}-flamegraph.html")
+benchmark_output = File.join(profile_directory, "#{mode}-benchmark.json")
+interval = mode == :object ? 100 : 1_000
+
+original_arguments = ARGV.dup
+ARGV.replace(['--output', benchmark_output])
+StackProf.run(mode:, interval:, raw: true, out: dump_path) do
+  load File.join(__dir__, 'microbench.rb')
+end
+ARGV.replace(original_arguments)
+
+File.open(flamegraph_path, 'w') do |file|
+  StackProf::Report.from_file(dump_path).print_d3_flamegraph(file)
+end
+
+puts "StackProf dump: #{dump_path}"
+puts "Flamegraph: #{flamegraph_path}"

data/bin/purplelight

@@ -25,6 +25,7 @@ options = {
   writer_threads: nil,
   write_chunk_bytes: nil,
   parquet_row_group: nil,
+  parquet_max_rows: nil,
   telemetry_flag: nil,
   read_concern: nil,
   no_cursor_timeout: nil,
@@ -58,9 +59,10 @@ parser = OptionParser.new do |opts|
     options[:sharding] ||= {}
     options[:sharding][:prefix] = v
   end
-  opts.on('--writer-threads N', Integer, 'Number of writer threads (experimental, JSONL only)') { |v| options[:writer_threads] = v }
+  opts.on('--writer-threads N', Integer, 'Number of concurrent JSONL writer threads') { |v| options[:writer_threads] = v }
   opts.on('--write-chunk-mb MB', Integer, 'JSONL encode/write chunk size in MB') { |v| options[:write_chunk_bytes] = v * 1024 * 1024 }
   opts.on('--parquet-row-group N', Integer, 'Parquet row group size (rows)') { |v| options[:parquet_row_group] = v }
+  opts.on('--parquet-max-rows N', Integer, 'Parquet max rows per part (multi-part only)') { |v| options[:parquet_max_rows] = v }
   opts.on('-q', '--query JSON', 'Filter query as JSON (Extended JSON supported)') do |v|
     # Prefer BSON Extended JSON to support $date, $oid, etc.
     options[:query] = BSON::ExtJSON.parse(v)
@@ -159,6 +161,7 @@ snapshot_args[:compression_level] = options[:compression_level] if options[:comp
 snapshot_args[:writer_threads] = options[:writer_threads] if options[:writer_threads]
 snapshot_args[:write_chunk_bytes] = options[:write_chunk_bytes] if options[:write_chunk_bytes]
 snapshot_args[:parquet_row_group] = options[:parquet_row_group] if options[:parquet_row_group]
+snapshot_args[:parquet_max_rows] = options[:parquet_max_rows] if options[:parquet_max_rows]
 
 # telemetry env override
 if options[:telemetry_flag]
@@ -169,6 +172,7 @@ end
 ENV['PL_ZSTD_LEVEL'] = options[:compression_level].to_s if options[:compression_level]
 ENV['PL_WRITE_CHUNK_BYTES'] = options[:write_chunk_bytes].to_s if options[:write_chunk_bytes]
 ENV['PL_PARQUET_ROW_GROUP'] = options[:parquet_row_group].to_s if options[:parquet_row_group]
+# No env default for parquet_max_rows; it is snapshot-argument only
 ENV['PL_WRITER_THREADS'] = options[:writer_threads].to_s if options[:writer_threads]
 
 ok = Purplelight.snapshot(**snapshot_args)

data/lib/purplelight/manifest.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'bson'
 require 'json'
 require 'time'
 require 'securerandom'
@@ -13,7 +14,7 @@ module Purplelight
   # counts so interrupted runs can resume safely and completed runs are
   # reproducible. Methods are thread-safe where mutation occurs.
   class Manifest
-    DEFAULT_VERSION = 1
+    DEFAULT_VERSION = 2
 
     attr_reader :path, :data
 
@@ -24,6 +25,9 @@ module Purplelight
 
     def initialize(path:, data: nil)
       @path = path
+      @directory = File.dirname(path)
+      @temporary_path = "#{path}.tmp"
+      FileUtils.mkdir_p(@directory)
       @data = data || {
         'version' => DEFAULT_VERSION,
         'run_id' => SecureRandom.uuid,
@@ -33,11 +37,12 @@ module Purplelight
         'compression' => nil,
         'query_digest' => nil,
         'options' => {},
+        'partition_filters' => nil,
         'parts' => [],
         'partitions' => []
       }
       @mutex = Mutex.new
-      @last_save_at = Time.now
+      @last_save_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end
 
     def self.load(path)
@@ -46,23 +51,25 @@ module Purplelight
     end
 
     def save!
-      dir = File.dirname(path)
-      FileUtils.mkdir_p(dir)
-      tmp = "#{path}.tmp"
-      File.write(tmp, JSON.pretty_generate(@data))
-      FileUtils.mv(tmp, path)
-    end
-
-    def configure!(collection:, format:, compression:, query_digest:, options: {})
-      @data['collection'] = collection
-      @data['format'] = format.to_s
-      @data['compression'] = compression.to_s
-      @data['query_digest'] = query_digest
-      @data['options'] = options
-      save!
+      File.write(@temporary_path, JSON.pretty_generate(@data))
+      File.rename(@temporary_path, path)
+    end
+
+    def configure!(collection:, format:, compression:, query_digest:, options: {}, partition_count: nil)
+      @mutex.synchronize do
+        @data['collection'] = collection
+        @data['format'] = format.to_s
+        @data['compression'] = compression.to_s
+        @data['query_digest'] = query_digest
+        @data['options'] = options
+        normalize_partition_data!(partition_count) if partition_count
+        save!
+      end
     end
 
     def compatible_with?(collection:, format:, compression:, query_digest:)
+      return false unless @data['version'] == DEFAULT_VERSION
+
       @data['collection'] == collection &&
         @data['format'] == format.to_s &&
         @data['compression'] == compression.to_s &&
@@ -72,9 +79,7 @@ module Purplelight
     def ensure_partitions!(count)
       @mutex.synchronize do
         if @data['partitions'].empty?
-          @data['partitions'] = Array.new(count) do |i|
-            { 'index' => i, 'last_id_exclusive' => nil, 'completed' => false }
-          end
+          normalize_partition_data!(count)
           save!
         end
       end
@@ -83,8 +88,8 @@ module Purplelight
     def update_partition_checkpoint!(index, last_id_exclusive)
       @mutex.synchronize do
         part = @data['partitions'][index]
-        part['last_id_exclusive'] = last_id_exclusive
-        save!
+        part['last_id_exclusive'] = serialize_checkpoint(last_id_exclusive)
+        save_maybe!
       end
     end
 
@@ -132,10 +137,51 @@ module Purplelight
       @data['partitions']
     end
 
+    def partition_checkpoint(index)
+      raw_checkpoint = @data['partitions'][index]&.fetch('last_id_exclusive', nil)
+      BSON::ExtJSON.parse_obj(raw_checkpoint) if raw_checkpoint
+    end
+
+    def partition_filters
+      raw_filters = @data['partition_filters']
+      return unless raw_filters
+
+      BSON::ExtJSON.parse_obj(raw_filters).map do |filter_spec|
+        {
+          filter: filter_spec[:filter] || filter_spec['filter'],
+          sort: filter_spec[:sort] || filter_spec['sort'],
+          hint: filter_spec[:hint] || filter_spec['hint']
+        }
+      end
+    end
+
+    def configure_partition_filters!(filters)
+      @mutex.synchronize do
+        @data['partition_filters'] = filters.as_extended_json
+      end
+    end
+
     private
 
+    def serialize_checkpoint(value)
+      case value
+      when nil, String, Numeric, true, false
+        value
+      else
+        value.as_extended_json
+      end
+    end
+
+    def normalize_partition_data!(count)
+      return unless @data['partitions'].empty?
+
+      @data['partitions'] = Array.new(count) do |index|
+        { 'index' => index, 'last_id_exclusive' => nil, 'completed' => false }
+      end
+    end
+
     def save_maybe!(interval_seconds: 2.0)
-      now = Time.now
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       return unless (now - @last_save_at) >= interval_seconds
 
       save!

data/lib/purplelight/partitioner.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'mongo'
+require_relative 'telemetry'
 
 module Purplelight
   # Partitioner builds MongoDB range filters to split work across workers.
@@ -14,7 +15,7 @@ module Purplelight
     def self.object_id_partitions(collection:, query:, partitions:, mode: nil, telemetry: nil)
       # Choose planning mode: :timestamp (fast), :cursor (legacy)
       chosen_mode = (mode || ENV['PL_PARTITIONER_MODE'] || :timestamp).to_sym
-      telemetry ||= (defined?(Telemetry) ? Telemetry::NULL : nil)
+      telemetry ||= Telemetry::NULL
 
       return cursor_sampling_partitions(collection: collection, query: query, partitions: partitions) if chosen_mode == :cursor
 
@@ -112,11 +113,16 @@ module Purplelight
       step = [total / partitions, 1].max
       boundaries = []
       cursor = base_query.projection(_id: 1).batch_size(1_000).no_cursor_timeout
-      i = 0
+      index = 0
+      next_boundary = 0
       cursor.each do |doc|
-        boundaries << doc['_id'] if (i % step).zero?
-        i += 1
-        break if boundaries.size >= partitions
+        if index == next_boundary
+          boundaries << doc['_id']
+          break if boundaries.size >= partitions
+
+          next_boundary += step
+        end
+        index += 1
       end
 
       ranges = []

data/lib/purplelight/queue.rb

@@ -6,6 +6,7 @@ module Purplelight
     def initialize(max_bytes: 128 * 1024 * 1024)
       @max_bytes = max_bytes
       @queue = []
+      @sizes = []
       @bytes = 0
       @closed = false
       @mutex = Mutex.new
@@ -16,8 +17,12 @@ module Purplelight
       @mutex.synchronize do
         raise 'queue closed' if @closed
 
-        @cv.wait(@mutex) while (@bytes + bytes) > @max_bytes
-        @queue << [item, bytes]
+        while !@queue.empty? && (@bytes + bytes) > @max_bytes
+          @cv.wait(@mutex)
+          raise 'queue closed' if @closed
+        end
+        @queue << item
+        @sizes << bytes
         @bytes += bytes
         @cv.broadcast
       end
@@ -30,7 +35,8 @@ module Purplelight
 
           @cv.wait(@mutex)
         end
-        item, bytes = @queue.shift
+        item = @queue.shift
+        bytes = @sizes.shift
         @bytes -= bytes
         @cv.broadcast
         item