purplelight 0.1.18 → 0.1.19

data/benchmark/microbench.rb

@@ -47,6 +47,7 @@ PARQUET_READ_DOCS = Array.new(10_000) do |index|
     'score' => index * 0.25,
     'nullable_score' => (nullable ? nil : index * 0.5),
     'category' => "category-#{index % 8}",
+    'symbol' => :"symbol-#{index % 8}",
     'unique_text' => format('row-%<index>08d-value-%<encoded>08x',
                             index:, encoded: index * 2_654_435_761),
     'utf8_text' => "日本語-#{index % 32}",

data/lib/purplelight/queue.rb

@@ -43,8 +43,13 @@ module Purplelight
       end
     end
 
-    def close
+    def close(discard: false)
       @mutex.synchronize do
+        if discard
+          @queue.clear
+          @sizes.clear
+          @bytes = 0
+        end
         @closed = true
         @cv.broadcast
       end

data/lib/purplelight/snapshot.rb

@@ -131,6 +131,12 @@ module Purplelight
 
       # Reader queue
       queue = ByteQueue.new(max_bytes: @queue_size_bytes)
+      failure_mutex = Mutex.new
+      worker_failure = nil
+      record_failure = lambda do |error|
+        failure_mutex.synchronize { worker_failure ||= error }
+        queue.close(discard: true)
+      end
 
       # Writer
       writer_count = @format == :jsonl ? [@writer_threads.to_i, 1].max : 1
@@ -163,9 +169,15 @@ module Purplelight
       readers = partition_filters.each_with_index.map do |pf, idx|
         Thread.new do
           local_telemetry = @telemetry_enabled ? Telemetry.new(enabled: true) : Telemetry::NULL
-          read_partition(idx: idx, filter_spec: pf, queue: queue, batch_size: @batch_size, manifest: manifest, telemetry: local_telemetry)
-          # Merge per-thread telemetry
-          @telemetry.merge!(local_telemetry) if @telemetry_enabled
+          begin
+            read_partition(idx: idx, filter_spec: pf, queue: queue, batch_size: @batch_size,
+                           manifest: manifest, telemetry: local_telemetry)
+          rescue StandardError => e
+            record_failure.call(e)
+          ensure
+            # Merge per-thread telemetry
+            @telemetry.merge!(local_telemetry) if @telemetry_enabled
+          end
         end
       end
 
@@ -182,8 +194,14 @@ module Purplelight
 
             worker_writer.write_many(batch)
           end
+        rescue StandardError => e
+          record_failure.call(e)
         ensure
-          worker_writer.close
+          begin
+            worker_writer.close
+          rescue StandardError => e
+            record_failure.call(e)
+          end
         end
       end
 
@@ -212,6 +230,9 @@ module Purplelight
       else
         @running = false
       end
+      worker_error = failure_mutex.synchronize { worker_failure }
+      raise worker_error if worker_error
+
       if @telemetry_enabled
         total = @telemetry.timers.values.sum
         breakdown = @telemetry.timers

data/lib/purplelight/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Purplelight
-  VERSION = '0.1.18'
+  VERSION = '0.1.19'
 end

data/lib/purplelight/writer_parquet.rb

@@ -118,7 +118,7 @@ module Purplelight
         return build_string_array(values)
       end
       return build_string_array(values) if data_type.is_a?(Arrow::StringDataType) ||
-                                           first.is_a?(String) || first.is_a?(Hash)
+                                           first.is_a?(String) || first.is_a?(Symbol) || first.is_a?(Hash)
       if (data_type.is_a?(Arrow::ListDataType) || first.is_a?(Array)) &&
          values.all? { |value| value.nil? || value.is_a?(Array) } &&
          (data_type || values.any? { |value| value && !value.empty? })
@@ -317,13 +317,14 @@ module Purplelight
 
     def low_cardinality_strings?(values)
       first = values.find { |value| !value.nil? }
-      return false unless first.is_a?(String)
+      string_like = first.is_a?(String) || first.is_a?(Symbol)
+      return false unless string_like
 
       distinct = {}
       count = 0
       values.each do |value|
         next if value.nil?
-        return false unless value.is_a?(String)
+        return false unless value.is_a?(String) || value.is_a?(Symbol)
 
         count += 1
         distinct[value] = true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: purplelight
 version: !ruby/object:Gem::Version
-  version: 0.1.18
+  version: 0.1.19
 platform: ruby
 authors:
 - Alexander Nicholson