synthra 0.1.0

data/ext/synthra_native/Cargo.toml

@@ -0,0 +1,46 @@
+[package]
+name = "synthra_native"
+version = "0.1.0"
+edition = "2021"
+authors = ["Synthra Team"]
+license = "MIT"
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+# Ruby bindings
+magnus = "0.8"
+rb-sys = "0.9"
+
+# Fake data generation (Rust equivalent of Ruby Faker)
+# Use re-exported rand from fake to avoid version conflicts
+fake = { version = "4", features = [
+    "derive",
+    "chrono",
+    "uuid",
+    "random_color",
+    "http"
+]}
+
+# Parallel processing - use all CPU cores
+rayon = "1.8"
+
+# JSON handling
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+
+# Date/time
+chrono = "0.4"
+
+# UUID
+uuid = { version = "1.0", features = ["v4", "v1", "serde"] }
+
+[features]
+default = []
+
+[profile.release]
+opt-level = 3
+lto = true
+codegen-units = 1

data/ext/synthra_native/extconf.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "mkmf"
+
+# Write a no-op Makefile so `gem install` / `bundle install` succeeds WITHOUT building the
+# Rust extension — the gem then runs in pure-Ruby mode. This is what lets the ext source ship
+# in the gem while still installing cleanly where Rust isn't available (CI, plain installs).
+def skip_native_build(reason)
+  warn "[synthra] native extension skipped: #{reason} (pure-Ruby mode)"
+  File.write("Makefile", <<~MAKEFILE)
+    .PHONY: all clean install
+    all:
+    \t@echo "Native extension skipped (#{reason})"
+    clean:
+    \t@echo "Nothing to clean"
+    install:
+    \t@echo "Native extension not installed"
+  MAKEFILE
+end
+
+cargo_on_path = ENV["PATH"].to_s.split(File::PATH_SEPARATOR).any? do |dir|
+  File.executable?(File.join(dir, "cargo"))
+end
+
+if cargo_on_path
+  begin
+    require "rb_sys/mkmf"
+
+    create_rust_makefile("synthra/synthra_native") do |r|
+      # Use release mode for best performance
+      r.profile = ENV.fetch("RB_SYS_CARGO_PROFILE", :release).to_sym
+
+      # Enable parallel feature if available
+      r.extra_cargo_args = ["--features", "parallel"] if ENV["PARALLEL"]
+    end
+  rescue LoadError
+    skip_native_build("rb_sys gem not available")
+  end
+else
+  skip_native_build("Rust/cargo not found")
+end

data/ext/synthra_native/src/generator.rs

@@ -0,0 +1,397 @@
+//! Generation engine for Synthra
+//!
+//! Handles parsing schema specs from Ruby and generating data efficiently.
+//! Uses Rayon for parallel processing to utilize all CPU cores.
+
+use fake::rand::rngs::StdRng;
+use fake::rand::{Rng, SeedableRng};
+use magnus::{Error, IntoValue, RArray, RHash, Ruby, Symbol, TryConvert, Value};
+use rayon::prelude::*;
+use serde_json::Map;
+use std::fs::File;
+use std::io::{BufWriter, Write};
+
+use crate::types::{generate_value, Locale};
+
+/// Field specification parsed from Ruby
+#[derive(Clone, Debug)]
+pub struct FieldSpec {
+    pub name: String,
+    pub type_name: String,
+    pub args: serde_json::Value,
+    pub optional: bool,
+    pub nullable: bool,
+}
+
+/// Native generator engine
+pub struct NativeGenerator {
+    fields: Vec<FieldSpec>,
+    seed: Option<u64>,
+    locale: Locale,
+}
+
+impl NativeGenerator {
+    /// Create generator from Ruby schema specification
+    ///
+    /// Expected format: Array of [name, type_name, args_json, optional, nullable]
+    pub fn from_ruby(schema_spec: RArray, seed: Option<u64>, locale: Option<String>) -> Result<Self, Error> {
+        let mut fields = Vec::new();
+
+        for i in 0..schema_spec.len() {
+            let item: Value = schema_spec.entry(i as isize)?;
+            let field_array = RArray::try_convert(item)?;
+
+            let name: String = field_array.entry(0)?;
+            let type_name: String = field_array.entry(1)?;
+            let args_json: String = field_array.entry(2)?;
+            let optional: bool = field_array.entry::<Value>(3)
+                .map(|v| bool::try_convert(v).unwrap_or(false))
+                .unwrap_or(false);
+            let nullable: bool = field_array.entry::<Value>(4)
+                .map(|v| bool::try_convert(v).unwrap_or(false))
+                .unwrap_or(false);
+
+            let args: serde_json::Value = serde_json::from_str(&args_json)
+                .unwrap_or(serde_json::Value::Object(Map::new()));
+
+            fields.push(FieldSpec {
+                name,
+                type_name,
+                args,
+                optional,
+                nullable,
+            });
+        }
+
+        let loc = locale
+            .map(|s| Locale::from_string(&s))
+            .unwrap_or(Locale::EN);
+
+        Ok(Self {
+            fields,
+            seed,
+            locale: loc,
+        })
+    }
+
+    /// Generate records and return as Ruby Array of Hashes
+    /// Uses parallel processing for large batches
+    pub fn generate_to_ruby(&self, ruby: &Ruby, count: usize) -> Result<RArray, Error> {
+        // For small batches, use single-threaded (less overhead)
+        if count < 1000 {
+            return self.generate_to_ruby_sequential(ruby, count);
+        }
+
+        // Generate records in parallel
+        let records: Vec<_> = (0..count)
+            .into_par_iter()
+            .map(|i| {
+                let mut rng = self.create_rng_for_index(i);
+                self.generate_record(&mut rng, i)
+            })
+            .collect();
+
+        // Convert to Ruby (must be sequential due to Ruby GIL)
+        let result = ruby.ary_new_capa(count);
+        for record in records {
+            let hash = self.json_map_to_ruby_hash(ruby, &record)?;
+            result.push(hash)?;
+        }
+
+        Ok(result)
+    }
+
+    /// Sequential generation for small batches
+    fn generate_to_ruby_sequential(&self, ruby: &Ruby, count: usize) -> Result<RArray, Error> {
+        let result = ruby.ary_new_capa(count);
+        let mut rng = self.create_rng(0);
+
+        for i in 0..count {
+            let record = self.generate_record(&mut rng, i);
+            let hash = self.json_map_to_ruby_hash(ruby, &record)?;
+            result.push(hash)?;
+        }
+
+        Ok(result)
+    }
+
+    /// Generate a single record to Ruby Hash
+    pub fn generate_one_to_ruby(&self, ruby: &Ruby) -> Result<Value, Error> {
+        let mut rng = self.create_rng(0);
+        let record = self.generate_record(&mut rng, 0);
+        let hash = self.json_map_to_ruby_hash(ruby, &record)?;
+        Ok(hash.into_value())
+    }
+
+    /// Generate records directly to file (most efficient)
+    /// Uses parallel generation for large batches
+    pub fn generate_to_file(
+        &self,
+        count: usize,
+        path: &str,
+        format: &str,
+    ) -> std::io::Result<usize> {
+        // For very large batches, use chunked parallel approach
+        if count >= 10_000 {
+            return self.generate_to_file_parallel(count, path, format);
+        }
+
+        // Small batches: sequential is fine
+        let file = File::create(path)?;
+        let mut writer = BufWriter::with_capacity(8 * 1024 * 1024, file); // 8MB buffer
+        let mut rng = self.create_rng(0);
+
+        match format {
+            "csv" => {
+                let headers: Vec<&str> = self.fields.iter().map(|f| f.name.as_str()).collect();
+                writeln!(writer, "{}", headers.join(","))?;
+                for i in 0..count {
+                    let record = self.generate_record(&mut rng, i);
+                    writeln!(writer, "{}", self.record_to_csv_line(&record))?;
+                }
+            }
+            "json" => {
+                write!(writer, "[")?;
+                for i in 0..count {
+                    if i > 0 {
+                        write!(writer, ",")?;
+                    }
+                    let record = self.generate_record(&mut rng, i);
+                    write!(writer, "{}", serde_json::to_string(&record).unwrap())?;
+                }
+                writeln!(writer, "]")?;
+            }
+            _ => {
+                // Default: JSONL
+                for i in 0..count {
+                    let record = self.generate_record(&mut rng, i);
+                    writeln!(writer, "{}", serde_json::to_string(&record).unwrap())?;
+                }
+            }
+        }
+
+        writer.flush()?;
+        Ok(count)
+    }
+
+    /// Parallel file generation for large batches
+    /// Uses chunked parallel approach to balance memory and throughput
+    fn generate_to_file_parallel(
+        &self,
+        count: usize,
+        path: &str,
+        format: &str,
+    ) -> std::io::Result<usize> {
+        let file = File::create(path)?;
+        let mut writer = BufWriter::with_capacity(64 * 1024 * 1024, file); // 64MB buffer
+
+        // Process in manageable chunks to control memory usage
+        // Large enough for parallelism, small enough to avoid memory issues
+        let num_threads = rayon::current_num_threads();
+        let records_per_chunk = 100_000; // 100K records per chunk
+        let num_chunks = (count + records_per_chunk - 1) / records_per_chunk;
+
+        match format {
+            "csv" => {
+                // Write header
+                let headers: Vec<&str> = self.fields.iter().map(|f| f.name.as_str()).collect();
+                writeln!(writer, "{}", headers.join(","))?;
+
+                for chunk_idx in 0..num_chunks {
+                    let start = chunk_idx * records_per_chunk;
+                    let end = (start + records_per_chunk).min(count);
+
+                    // Generate this chunk in parallel to a buffer
+                    let chunk_buffer: String = (start..end)
+                        .into_par_iter()
+                        .map(|i| {
+                            let mut rng = self.create_rng_for_index(i);
+                            let record = self.generate_record(&mut rng, i);
+                            format!("{}\n", self.record_to_csv_line(&record))
+                        })
+                        .collect();
+
+                    write!(writer, "{}", chunk_buffer)?;
+                }
+            }
+            "json" => {
+                write!(writer, "[")?;
+                let mut first = true;
+
+                for chunk_idx in 0..num_chunks {
+                    let start = chunk_idx * records_per_chunk;
+                    let end = (start + records_per_chunk).min(count);
+
+                    // Generate this chunk in parallel
+                    let json_strings: Vec<String> = (start..end)
+                        .into_par_iter()
+                        .map(|i| {
+                            let mut rng = self.create_rng_for_index(i);
+                            let record = self.generate_record(&mut rng, i);
+                            serde_json::to_string(&record).unwrap()
+                        })
+                        .collect();
+
+                    for json in json_strings {
+                        if !first {
+                            write!(writer, ",")?;
+                        }
+                        first = false;
+                        write!(writer, "{}", json)?;
+                    }
+                }
+                writeln!(writer, "]")?;
+            }
+            _ => {
+                // JSONL - chunked parallel generation
+                for chunk_idx in 0..num_chunks {
+                    let start = chunk_idx * records_per_chunk;
+                    let end = (start + records_per_chunk).min(count);
+
+                    // Generate this chunk in parallel to a single buffer string
+                    let chunk_buffer: String = (start..end)
+                        .into_par_iter()
+                        .map(|i| {
+                            let mut rng = self.create_rng_for_index(i);
+                            let record = self.generate_record(&mut rng, i);
+                            format!("{}\n", serde_json::to_string(&record).unwrap())
+                        })
+                        .collect();
+
+                    write!(writer, "{}", chunk_buffer)?;
+                }
+            }
+        }
+
+        writer.flush()?;
+        Ok(count)
+    }
+
+    /// Convert record to CSV line
+    fn record_to_csv_line(&self, record: &Map<String, serde_json::Value>) -> String {
+        let values: Vec<String> = self
+            .fields
+            .iter()
+            .map(|f| match record.get(&f.name) {
+                Some(serde_json::Value::String(s)) => {
+                    if s.contains(',') || s.contains('"') || s.contains('\n') {
+                        format!("\"{}\"", s.replace('"', "\"\""))
+                    } else {
+                        s.clone()
+                    }
+                }
+                Some(v) => v.to_string(),
+                None => String::new(),
+            })
+            .collect();
+        values.join(",")
+    }
+
+    /// Generate a single record as JSON object
+    fn generate_record(&self, rng: &mut StdRng, index: usize) -> Map<String, serde_json::Value> {
+        let mut record = Map::new();
+
+        for field in &self.fields {
+            // Handle optional fields (skip ~20%)
+            if field.optional && rng.random_bool(0.2) {
+                continue;
+            }
+
+            // Handle nullable fields (null ~10%)
+            if field.nullable && rng.random_bool(0.1) {
+                record.insert(field.name.clone(), serde_json::Value::Null);
+                continue;
+            }
+
+            let value = generate_value(rng, &field.type_name, &field.args, self.locale, index);
+            record.insert(field.name.clone(), value);
+        }
+
+        record
+    }
+
+    /// Create RNG from seed
+    fn create_rng(&self, offset: u64) -> StdRng {
+        match self.seed {
+            Some(seed) => StdRng::seed_from_u64(seed.wrapping_add(offset)),
+            None => {
+                // Generate random seed from system time
+                use std::time::{SystemTime, UNIX_EPOCH};
+                let seed = SystemTime::now()
+                    .duration_since(UNIX_EPOCH)
+                    .map(|d| d.as_nanos() as u64)
+                    .unwrap_or(42);
+                StdRng::seed_from_u64(seed)
+            }
+        }
+    }
+
+    /// Create RNG for a specific record index (used in parallel generation)
+    /// Each index gets a deterministic, unique seed based on base seed + index
+    fn create_rng_for_index(&self, index: usize) -> StdRng {
+        let base_seed = self.seed.unwrap_or_else(|| {
+            use std::time::{SystemTime, UNIX_EPOCH};
+            SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .map(|d| d.as_nanos() as u64)
+                .unwrap_or(42)
+        });
+        // Mix seed with index to ensure each record is unique but deterministic
+        StdRng::seed_from_u64(base_seed.wrapping_mul(31).wrapping_add(index as u64))
+    }
+
+    /// Convert serde_json Map to Ruby Hash
+    fn json_map_to_ruby_hash(
+        &self,
+        ruby: &Ruby,
+        record: &Map<String, serde_json::Value>,
+    ) -> Result<RHash, Error> {
+        let hash = ruby.hash_new();
+
+        for (key, value) in record {
+            let ruby_key = Symbol::new(key);
+            let ruby_value = self.json_to_ruby(ruby, value)?;
+            hash.aset(ruby_key, ruby_value)?;
+        }
+
+        Ok(hash)
+    }
+
+    /// Convert serde_json Value to Ruby Value
+    fn json_to_ruby(&self, ruby: &Ruby, value: &serde_json::Value) -> Result<Value, Error> {
+        match value {
+            serde_json::Value::Null => Ok(ruby.qnil().into_value()),
+            serde_json::Value::Bool(b) => {
+                if *b {
+                    Ok(ruby.qtrue().into_value())
+                } else {
+                    Ok(ruby.qfalse().into_value())
+                }
+            }
+            serde_json::Value::Number(n) => {
+                if let Some(i) = n.as_i64() {
+                    Ok(ruby.integer_from_i64(i).into_value())
+                } else if let Some(f) = n.as_f64() {
+                    Ok(ruby.float_from_f64(f).into_value())
+                } else {
+                    Ok(ruby.qnil().into_value())
+                }
+            }
+            serde_json::Value::String(s) => Ok(ruby.str_new(s).into_value()),
+            serde_json::Value::Array(arr) => {
+                let ruby_arr = ruby.ary_new_capa(arr.len());
+                for item in arr {
+                    ruby_arr.push(self.json_to_ruby(ruby, item)?)?;
+                }
+                Ok(ruby_arr.into_value())
+            }
+            serde_json::Value::Object(obj) => {
+                let ruby_hash = ruby.hash_new();
+                for (k, v) in obj {
+                    ruby_hash.aset(Symbol::new(k), self.json_to_ruby(ruby, v)?)?;
+                }
+                Ok(ruby_hash.into_value())
+            }
+        }
+    }
+}