synthra 0.1.0

data/lib/synthra/performance_mode.rb

@@ -0,0 +1,364 @@
+# frozen_string_literal: true
+
+require "etc"
+
+module Synthra
+  # Performance Mode
+  #
+  # High-performance data generation for millions of records.
+  # Uses parallel processing, memory pooling, and native acceleration.
+  #
+  # @example Generate millions of records
+  #   Synthra::PerformanceMode.generate(schema, count: 1_000_000)
+  #
+  # @example With progress callback
+  #   Synthra::PerformanceMode.generate(schema, count: 10_000_000) do |progress|
+  #     puts "Generated #{progress[:current]} / #{progress[:total]} (#{progress[:percent]}%)"
+  #   end
+  #
+  module PerformanceMode
+    # Default configuration
+    DEFAULT_CONFIG = {
+      batch_size: 10_000,          # Records per batch
+      threads: nil,                 # nil = auto (CPU cores)
+      memory_limit_mb: 512,         # Max memory per batch
+      use_native: true,             # Use Rust engine if available
+      progress_interval: 100_000,   # Report progress every N records
+      gc_interval: 500_000          # Force GC every N records
+    }.freeze
+
+    class << self
+      # Generate records in high-performance mode
+      #
+      # @param schema [Schema] schema to generate
+      # @param count [Integer] number of records
+      # @param options [Hash] generation options
+      # @yield [Hash] progress updates
+      # @return [Array, Enumerator] generated records or enumerator
+      #
+      def generate(schema, count:, **options, &progress_callback)
+        config = DEFAULT_CONFIG.merge(options)
+        config[:threads] ||= optimal_thread_count
+        
+        if config[:use_native] && native_available?
+          generate_native(schema, count, config, &progress_callback)
+        else
+          generate_parallel(schema, count, config, &progress_callback)
+        end
+      end
+
+      # Stream records without loading all into memory
+      #
+      # @param schema [Schema] schema to generate
+      # @param count [Integer] number of records
+      # @param options [Hash] generation options
+      # @return [Enumerator] record enumerator
+      #
+      def stream(schema, count:, **options)
+        config = DEFAULT_CONFIG.merge(options)
+        
+        Enumerator.new do |yielder|
+          generated = 0
+          last_gc = 0
+          seed = options[:seed] || Random.new_seed
+
+          while generated < count
+            batch_size = [config[:batch_size], count - generated].min
+            batch_seed = seed + generated
+
+            batch = schema.generate_many(
+              batch_size,
+              seed: batch_seed,
+              mode: options[:mode] || :random
+            )
+
+            batch.each { |record| yielder << record }
+            generated += batch_size
+
+            # Memory management — trigger on the gc_interval *delta*, not exact divisibility, so a
+            # batch_size that doesn't evenly divide gc_interval can't skip every GC.
+            if generated - last_gc >= config[:gc_interval]
+              GC.start
+              last_gc = generated
+            end
+          end
+        end
+      end
+
+      # Write records directly to file (streaming)
+      #
+      # @param schema [Schema] schema to generate
+      # @param count [Integer] number of records
+      # @param output [String, IO] file path or IO object
+      # @param format [Symbol] output format (:json, :ndjson, :csv)
+      # @param options [Hash] generation options
+      #
+      def to_file(schema, count:, output:, format: :ndjson, **options, &progress_callback)
+        config = DEFAULT_CONFIG.merge(options)
+        
+        io = output.is_a?(String) ? File.open(output, "w") : output
+        generated = 0
+        start_time = Time.now
+        
+        begin
+          # Write header for CSV
+          if format == :csv
+            io.puts schema.fields.map(&:name).join(",")
+          end
+          
+          # JSON array start
+          if format == :json
+            io.puts "["
+          end
+          
+          stream(schema, count: count, **options).each_with_index do |record, index|
+            case format
+            when :json
+              io.print "  "
+              io.print JSON.generate(record)
+              io.puts(index < count - 1 ? "," : "")
+            when :ndjson
+              io.puts JSON.generate(record)
+            when :csv
+              io.puts schema.fields.map { |f| format_csv_value(record[f.name]) }.join(",")
+            end
+            
+            generated += 1
+            
+            if progress_callback && generated % config[:progress_interval] == 0
+              elapsed = Time.now - start_time
+              rate = safe_rate(generated, elapsed)
+              progress_callback.call({
+                current: generated,
+                total: count,
+                percent: (generated * 100.0 / count).round(1),
+                rate: rate,
+                elapsed: elapsed.round(1),
+                eta: safe_eta(count - generated, rate)
+              })
+            end
+          end
+          
+          # JSON array end
+          if format == :json
+            io.puts "]"
+          end
+        ensure
+          io.close if output.is_a?(String)
+        end
+        
+        final_elapsed = Time.now - start_time
+        {
+          records: generated,
+          elapsed: final_elapsed,
+          rate: safe_rate(generated, final_elapsed)
+        }
+      end
+
+      # Benchmark generation performance
+      #
+      # @param schema [Schema] schema to benchmark
+      # @param counts [Array<Integer>] record counts to test
+      # @return [Hash] benchmark results
+      #
+      def benchmark(schema, counts: [1000, 10_000, 100_000])
+        results = {}
+        
+        counts.each do |count|
+          start_time = Time.now
+          records = schema.generate_many(count, mode: :random)
+          elapsed = Time.now - start_time
+          
+          results[count] = {
+            elapsed: elapsed.round(3),
+            rate: safe_rate(count, elapsed),
+            memory_mb: estimate_memory_usage
+          }
+          
+          # Force GC between benchmarks
+          records = nil
+          GC.start
+        end
+        
+        results
+      end
+
+      # Get system capabilities
+      #
+      # @return [Hash] system info
+      #
+      def system_info
+        {
+          cpu_cores: Etc.nprocessors,
+          ruby_version: RUBY_VERSION,
+          native_available: native_available?,
+          optimal_threads: optimal_thread_count,
+          recommended_batch_size: recommended_batch_size
+        }
+      end
+
+      private
+
+      def native_available?
+        defined?(Synthra::Native) && Synthra::Native.respond_to?(:generate_batch)
+      rescue StandardError
+        false
+      end
+
+      def optimal_thread_count
+        # Leave one core for the main thread
+        [Etc.nprocessors - 1, 1].max
+      end
+
+      def recommended_batch_size
+        # Based on available memory
+        available_mb = begin
+          # Try to get actual available memory
+          if RUBY_PLATFORM.include?("darwin")
+            `sysctl -n hw.memsize`.to_i / 1024 / 1024 / 4
+          else
+            1024
+          end
+        rescue StandardError
+          1024
+        end
+        
+        [available_mb * 100, 100_000].min
+      end
+
+      def generate_native(schema, count, config, &progress_callback)
+        require_relative "native" if defined?(Synthra::Native)
+        
+        results = []
+        generated = 0
+        start_time = Time.now
+        seed = config[:seed] || Random.new_seed
+        
+        batches = (count.to_f / config[:batch_size]).ceil
+        
+        batches.times do |batch_num|
+          batch_count = [config[:batch_size], count - generated].min
+          batch_seed = seed + (batch_num * config[:batch_size])
+          
+          batch = Synthra::Native.generate_batch(
+            schema.to_native_spec,
+            batch_count,
+            batch_seed,
+            config[:threads]
+          )
+          
+          results.concat(batch)
+          generated += batch_count
+          
+          if progress_callback && generated % config[:progress_interval] == 0
+            report_progress(progress_callback, generated, count, start_time)
+          end
+        end
+        
+        results
+      rescue StandardError
+        # Fallback to parallel mode
+        generate_parallel(schema, count, config, &progress_callback)
+      end
+
+      def generate_parallel(schema, count, config, &progress_callback)
+        results = []
+        generated = 0
+        start_time = Time.now
+        seed = config[:seed] || Random.new_seed
+        mutex = Mutex.new
+        
+        threads = config[:threads]
+        records_per_thread = (count.to_f / threads).ceil
+        
+        worker_threads = threads.times.map do |thread_num|
+          Thread.new do
+            thread_start = thread_num * records_per_thread
+            thread_count = [records_per_thread, count - thread_start].min
+            next [] if thread_count <= 0
+            
+            thread_seed = seed + thread_start
+            
+            schema.generate_many(
+              thread_count,
+              seed: thread_seed,
+              mode: config[:mode] || :random
+            )
+          end
+        end
+        
+        worker_threads.each do |thread|
+          batch = thread.value
+          mutex.synchronize do
+            results.concat(batch)
+            generated = results.length
+            
+            if progress_callback && generated % config[:progress_interval] < batch.length
+              report_progress(progress_callback, generated, count, start_time)
+            end
+          end
+        end
+        
+        results
+      end
+
+      # Records/sec, guarded: sub-millisecond runs on fast hardware give elapsed == 0.0, and
+      # records / 0.0 is Float::INFINITY whose #round raises FloatDomainError. Return 0 instead.
+      def safe_rate(records, elapsed)
+        elapsed.positive? ? (records / elapsed).round(0) : 0
+      end
+
+      # Seconds remaining, guarded against a 0 rate (which would otherwise divide by zero).
+      def safe_eta(remaining, rate)
+        rate.positive? ? (remaining / rate.to_f).round(1) : 0.0
+      end
+
+      def report_progress(callback, current, total, start_time)
+        elapsed = Time.now - start_time
+        rate = safe_rate(current, elapsed)
+
+        callback.call({
+          current: current,
+          total: total,
+          percent: (current * 100.0 / total).round(1),
+          rate: rate,
+          elapsed: elapsed.round(1),
+          eta: safe_eta(total - current, rate)
+        })
+      end
+
+      def format_csv_value(value)
+        case value
+        when nil
+          ""
+        when String
+          value.include?(",") || value.include?('"') ? "\"#{value.gsub('"', '""')}\"" : value
+        when Array
+          "\"#{value.to_json}\""
+        when Hash
+          "\"#{value.to_json.gsub('"', '""')}\""
+        else
+          value.to_s
+        end
+      end
+
+      def estimate_memory_usage
+        # Try to get memory usage, fallback to 0 if not available
+        if defined?(ObjectSpace) && ObjectSpace.respond_to?(:memsize_of_all)
+          (ObjectSpace.memsize_of_all / 1024.0 / 1024.0).round(2)
+        elsif RUBY_PLATFORM.include?("darwin") || RUBY_PLATFORM.include?("linux")
+          begin
+            # Use ps command as fallback
+            pid = Process.pid
+            mem = `ps -o rss= -p #{pid}`.strip.to_i
+            (mem / 1024.0).round(2)
+          rescue StandardError
+            0.0
+          end
+        else
+          0.0
+        end
+      end
+    end
+  end
+end