synthra 0.1.0

data/lib/synthra/native_engine.rb

@@ -0,0 +1,304 @@
+# frozen_string_literal: true
+
+module Synthra
+  # High-performance native engine using Rust + fake-rs
+  #
+  # Provides 100-250x speedup over pure Ruby implementation by using
+  # a native Rust extension with the fake-rs library.
+  #
+  # @example Check availability
+  #   if Synthra::NativeEngine.available?
+  #     puts "Native engine is available!"
+  #   end
+  #
+  # @example Basic usage
+  #   schema = Synthra.parse("User:\n  id: uuid\n  name: name")
+  #   engine = Synthra::NativeEngine.new(schema)
+  #   records = engine.generate_many(1000)
+  #
+  # @example Generate to file
+  #   engine.generate_to_file(10_000_000, "users.jsonl", format: "jsonl")
+  #
+  class NativeEngine
+    class << self
+      # Check if native extension is available
+      #
+      # @return [Boolean] true if the native extension is loaded and functional
+      def available?
+        return @available if defined?(@available)
+
+        @available = begin
+          load_native_extension
+          defined?(Synthra::Native) &&
+            Native.respond_to?(:loaded?) &&
+            Native.loaded?
+        rescue LoadError, NameError
+          false
+        end
+      end
+
+      # Get list of types supported by native engine
+      #
+      # @return [Array<String>] list of type names
+      def available_types
+        return [] unless available?
+
+        Native.available_types
+      end
+
+      # Get list of locales supported by native engine
+      #
+      # @return [Array<String>] list of locale codes
+      def available_locales
+        return [] unless available?
+
+        Native.available_locales
+      end
+
+      # Get native extension version
+      #
+      # @return [String, nil] version string or nil if not available
+      def version
+        return nil unless available?
+
+        Native.version
+      end
+
+      # Get performance information
+      #
+      # @return [String, nil] performance info or nil if not available
+      def performance_info
+        return nil unless available?
+
+        Native.performance_info
+      end
+
+      # Get the current number of threads used for parallel generation
+      #
+      # @return [Integer, nil] thread count or nil if not available
+      def thread_count
+        return nil unless available?
+
+        Native.thread_count
+      end
+
+      # Set the number of threads to use for parallel generation
+      # Must be called before any generation operations.
+      #
+      # @param num_threads [Integer] number of threads (0 = use all cores)
+      # @return [Integer] actual number of threads configured
+      #
+      # @example Limit to 4 threads
+      #   Synthra::NativeEngine.set_threads(4)
+      #
+      # @note Alternatively, set RAYON_NUM_THREADS env var before loading:
+      #   RAYON_NUM_THREADS=4 ruby my_script.rb
+      def set_threads(num_threads)
+        return nil unless available?
+
+        Native.set_threads(num_threads)
+      end
+
+      private
+
+      def load_native_extension
+        require "synthra/synthra_native"
+      rescue LoadError => e
+        # Check if it's a missing extension or missing dependencies
+        if e.message.include?("cannot load such file")
+          raise LoadError, <<~MSG
+            Native extension not compiled. To enable native performance:
+
+            1. Install Rust: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+            2. Install rb_sys: gem install rb_sys
+            3. Compile extension: bundle exec rake compile
+
+            Original error: #{e.message}
+          MSG
+        else
+          raise
+        end
+      end
+    end
+
+    # @return [Synthra::Schema] the schema being used
+    attr_reader :schema
+
+    # @return [String] the locale for data generation
+    attr_reader :locale
+
+    # Initialize native engine with a schema
+    #
+    # @param schema [Synthra::Schema] the schema to generate from
+    # @param locale [String] locale for fake data (en, fr_fr, de_de, ja_jp, etc.)
+    # @param threads [Integer, nil] number of threads (nil/0 = use all cores)
+    # @raise [LoadError] if native extension is not available
+    def initialize(schema, locale: "en", threads: nil)
+      unless self.class.available?
+        raise LoadError, "Native extension not available. " \
+                         "Install Rust and run `bundle exec rake compile`"
+      end
+
+      @schema = schema
+      @locale = locale.to_s
+      @threads = threads
+      @field_specs = build_field_specs
+    end
+
+    # Generate multiple records
+    #
+    # @param count [Integer] number of records to generate
+    # @param seed [Integer, nil] random seed for reproducibility
+    # @param threads [Integer, nil] number of threads (nil/0 = use all cores)
+    # @return [Array<Hash>] array of generated records
+    #
+    # @example Generate with default threads
+    #   engine.generate_many(1000, seed: 42)
+    #
+    # @example Generate with limited threads
+    #   engine.generate_many(1000, seed: 42, threads: 4)
+    def generate_many(count, seed: nil, threads: nil)
+      Native.generate(@field_specs, count, seed, @locale, threads || @threads)
+    end
+
+    # Generate a single record
+    #
+    # @param seed [Integer, nil] random seed for reproducibility
+    # @return [Hash] generated record
+    #
+    # @example
+    #   record = engine.generate(seed: 42)
+    def generate(seed: nil)
+      Native.generate_one(@field_specs, seed, @locale)
+    end
+
+    # Generate records directly to file (most efficient for large datasets)
+    #
+    # @param count [Integer] number of records to generate
+    # @param path [String] output file path
+    # @param seed [Integer, nil] random seed for reproducibility
+    # @param format [String] output format: "jsonl", "json", "csv"
+    # @param threads [Integer, nil] number of threads (nil/0 = use all cores)
+    # @return [Integer] number of records written
+    #
+    # @example Generate JSONL (one JSON per line)
+    #   engine.generate_to_file(10_000_000, "users.jsonl", format: "jsonl")
+    #
+    # @example Generate CSV
+    #   engine.generate_to_file(1_000_000, "users.csv", format: "csv")
+    #
+    # @example Generate with limited threads
+    #   engine.generate_to_file(10_000_000, "users.jsonl", threads: 4)
+    def generate_to_file(count, path, seed: nil, format: "jsonl", threads: nil)
+      Native.generate_to_file(@field_specs, count, path.to_s, seed, @locale, format.to_s, threads || @threads)
+    end
+
+    # Alias for compatibility with other engines
+    alias to_file generate_to_file
+
+    # Stream records (yields each record)
+    #
+    # @param count [Integer] number of records to generate
+    # @param seed [Integer, nil] random seed for reproducibility
+    # @yield [Hash] each generated record
+    # @return [Enumerator] if no block given
+    #
+    # @example
+    #   engine.stream(1000) do |record|
+    #     process(record)
+    #   end
+    def stream(count, seed: nil, &block)
+      return enum_for(:stream, count, seed: seed) unless block_given?
+
+      # For now, generate in batches of 10K for memory efficiency
+      batch_size = 10_000
+      remaining = count
+      offset = 0
+
+      while remaining > 0
+        current_batch = [remaining, batch_size].min
+        batch_seed = seed ? seed + offset : nil
+
+        records = Native.generate(@field_specs, current_batch, batch_seed, @locale, @threads)
+        records.each(&block)
+
+        remaining -= current_batch
+        offset += current_batch
+      end
+    end
+
+    private
+
+    # Build field specs for native engine
+    # Format: [name, type_name, args_json, optional, nullable]
+    def build_field_specs
+      @schema.fields.map do |field|
+        [
+          field.name.to_s,
+          map_type_name(field.type_name),
+          build_args_json(field),
+          field.optional?,
+          field.nullable?
+        ]
+      end
+    end
+
+    # Map DSL type names to native engine type names
+    def map_type_name(type_name)
+      type_str = type_name.to_s
+
+      # Handle type aliases
+      case type_str
+      when "full_name" then "name"
+      when "phone_number" then "phone"
+      when "street_address" then "street"
+      when "zip_code" then "postal_code"
+      when "int", "integer" then "number"
+      when "bool" then "boolean"
+      when "timestamp" then "datetime"
+      else type_str
+      end
+    end
+
+    # Build args JSON for a field
+    def build_args_json(field)
+      args = field.type_args || {}
+
+      # Handle enum values
+      if field.type_name.to_s == "enum" && args[:values]
+        args = args.merge(values: args[:values].map { |v| extract_enum_value(v) })
+      end
+
+      # Handle const value
+      if field.type_name.to_s == "const"
+        args = { value: args[:value] || args.values.first }
+      end
+
+      # Handle number ranges
+      if args[:range]
+        range = args[:range]
+        args = args.merge(min: range.begin, max: range.end)
+      end
+
+      args.to_json
+    end
+
+    # Extract the string value from an enum value (handles AST nodes and hashes)
+    def extract_enum_value(value)
+      case value
+      when Hash
+        value[:value] || value["value"]
+      when String, Symbol
+        value.to_s
+      else
+        # Handle AST nodes like EnumValueNode
+        if value.respond_to?(:value)
+          value.value.to_s
+        else
+          value.to_s
+        end
+      end
+    end
+  end
+end
+

data/lib/synthra/openapi_importer.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "json"
+
+module Synthra
+  # OpenAPI/Swagger Schema Importer
+  #
+  # Converts OpenAPI specifications to Synthra schemas.
+  #
+  # @example CLI usage
+  #   $ synthra import openapi.yaml --output schemas/
+  #
+  # @example Ruby usage
+  #   importer = Synthra::OpenAPIImporter.new("openapi.yaml")
+  #   schemas = importer.import
+  #   schemas.each { |name, dsl| File.write("schemas/#{name}.dsl", dsl) }
+  #
+  class OpenAPIImporter
+    # OpenAPI type to Synthra type mapping
+    TYPE_MAP = {
+      # Strings
+      "string" => "text",
+      "string:email" => "email",
+      "string:uri" => "url",
+      "string:url" => "url",
+      "string:uuid" => "uuid",
+      "string:date" => "date",
+      "string:date-time" => "timestamp",
+      "string:password" => "password",
+      "string:byte" => "text",
+      "string:binary" => "text",
+      "string:hostname" => "domain",
+      "string:ipv4" => "ip",
+      "string:ipv6" => "ipv6",
+
+      # Numbers
+      "integer" => "number",
+      "integer:int32" => "number",
+      "integer:int64" => "number",
+      "number" => "float",
+      "number:float" => "float",
+      "number:double" => "float",
+
+      # Boolean
+      "boolean" => "boolean",
+
+      # Complex
+      "array" => "array",
+      "object" => "object"
+    }.freeze
+
+    attr_reader :spec_path, :spec
+
+    # Create a new importer
+    #
+    # @param spec_path [String] path to OpenAPI spec file
+    #
+    def initialize(spec_path)
+      @spec_path = spec_path
+      @spec = load_spec
+    end
+
+    # Import all schemas from the OpenAPI spec
+    #
+    # @return [Hash<String, String>] schema name => DSL content
+    #
+    def import
+      schemas = {}
+
+      components_schemas.each do |name, definition|
+        dsl = schema_to_dsl(name, definition)
+        schemas[name] = dsl if dsl
+      end
+
+      schemas
+    end
+
+    # Import and write to directory
+    #
+    # @param output_dir [String] output directory
+    # @return [Array<String>] paths to created files
+    #
+    def import_to_directory(output_dir)
+      FileUtils.mkdir_p(output_dir)
+      created = []
+
+      base = File.expand_path(output_dir)
+      import.each do |name, dsl|
+        # SECURITY (path traversal): a schema name from the spec is attacker-controlled. basename
+        # strips any "../" path components and the allowlist drops anything left, so the file can only
+        # land inside output_dir; the expand_path assertion is a belt-and-suspenders backstop.
+        safe = File.basename(to_snake_case(name.to_s)).gsub(/[^a-z0-9_]+/i, "_")
+        # :nocov: defensive backstop — File.basename + the gsub allowlist always leave a non-empty
+        # token for any non-empty schema name, so this empty-fallback is unreachable in practice.
+        safe = "schema" if safe.empty?
+        # :nocov:
+        path = File.join(output_dir, "#{safe}.dsl")
+        # :nocov: defensive backstop — basename already neutralizes "../" traversal, so this
+        # expand_path containment check can never fail for a name that produced a safe basename.
+        unless File.expand_path(path).start_with?(base + File::SEPARATOR)
+          raise Synthra::Errors::SecurityError, "Refusing to write outside #{output_dir}: #{name.inspect}"
+        end
+        # :nocov:
+
+        File.write(path, dsl)
+        created << path
+      end
+
+      created
+    end
+
+    private
+
+    def load_spec
+      content = File.read(@spec_path)
+      
+      if @spec_path.end_with?(".json")
+        JSON.parse(content)
+      else
+        YAML.safe_load(content, permitted_classes: [Date, Time])
+      end
+    end
+
+    def components_schemas
+      @spec.dig("components", "schemas") || 
+        @spec.dig("definitions") || # Swagger 2.0
+        {}
+    end
+
+    def schema_to_dsl(name, definition)
+      return nil unless definition["type"] == "object" || definition["properties"]
+
+      lines = ["#{name}:"]
+      properties = definition["properties"] || {}
+      required = definition["required"] || []
+
+      properties.each do |prop_name, prop_def|
+        type = property_to_type(prop_def)
+        optional = required.include?(prop_name) ? "" : "?"
+        nullable = prop_def["nullable"] ? "?" : ""
+        
+        lines << "  #{prop_name}#{optional}: #{type}#{nullable}"
+      end
+
+      lines.join("\n")
+    end
+
+    def property_to_type(prop_def)
+      type = prop_def["type"]
+      format = prop_def["format"]
+
+      # Handle references
+      if prop_def["$ref"]
+        ref = prop_def["$ref"]
+        return extract_ref_name(ref)
+      end
+
+      # Handle enums
+      if prop_def["enum"]
+        values = prop_def["enum"].join(", ")
+        return "enum(#{values})"
+      end
+
+      # Handle arrays
+      if type == "array"
+        items_type = property_to_type(prop_def["items"] || {})
+        min_items = prop_def["minItems"] || 0
+        max_items = prop_def["maxItems"] || 10
+        return "array(#{items_type}, #{min_items}..#{max_items})"
+      end
+
+      # Handle nested objects
+      if type == "object"
+        if prop_def["properties"]
+          # Inline object - use object type
+          return "object"
+        end
+        return "object"
+      end
+
+      # Check for string constraints
+      if type == "string"
+        # Check for specific patterns
+        if (pattern = prop_def["pattern"])
+          return "regex(\"#{escape_regex(pattern)}\")"
+        end
+        
+        # Check for length constraints
+        min_len = prop_def["minLength"]
+        max_len = prop_def["maxLength"]
+        if min_len || max_len
+          range = "#{min_len || 1}..#{max_len || 100}"
+          return "text(#{range})"
+        end
+      end
+
+      # Check for number constraints
+      if %w[integer number].include?(type)
+        min_val = prop_def["minimum"]
+        max_val = prop_def["maximum"]
+        if min_val || max_val
+          range = "#{min_val || 0}..#{max_val || 1000}"
+          return "number(#{range})"
+        end
+      end
+
+      # Look up in type map
+      key = format ? "#{type}:#{format}" : type
+      TYPE_MAP[key] || TYPE_MAP[type] || "text"
+    end
+
+    def extract_ref_name(ref)
+      # Handle #/components/schemas/User or #/definitions/User
+      ref.split("/").last
+    end
+
+    def escape_regex(pattern)
+      pattern.gsub('"', '\\"')
+    end
+
+    def to_snake_case(str)
+      str.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+         .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+         .downcase
+    end
+  end
+end

data/lib/synthra/output/json_formatter.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Synthra
+  module Output
+
+    # JSON formatter for generated data
+    class JsonFormatter
+
+      # Format a single record
+      #
+      # @param record [Hash] the record
+      # @param pretty [Boolean] pretty-print
+      # @return [String] JSON string
+
+      def format(record, pretty: false)
+        if pretty
+          JSON.pretty_generate(record)
+        else
+          JSON.generate(record)
+        end
+      end
+
+
+      # Format multiple records
+      #
+      # @param records [Array<Hash>] the records
+      # @param pretty [Boolean] pretty-print
+      # @return [String] JSON string
+
+      def format_many(records, pretty: false)
+        if pretty
+          JSON.pretty_generate(records)
+        else
+          JSON.generate(records)
+        end
+      end
+    end
+  end
+end
+

data/lib/synthra/output/ndjson_formatter.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Synthra
+  module Output
+
+    # NDJSON (newline-delimited JSON) formatter
+    class NdjsonFormatter
+
+      # Format a stream to NDJSON
+      #
+      # @param stream [Enumerator] the record stream
+      # @param io [IO] output IO object
+      # @return [void]
+
+      def format_stream(stream, io: $stdout)
+        stream.each do |record|
+          io.puts JSON.generate(record)
+          io.flush if io.respond_to?(:flush)
+        end
+      end
+
+
+      # Format records to NDJSON string
+      #
+      # @param records [Array<Hash>] the records
+      # @return [String] NDJSON string
+
+      def format_many(records)
+        # Append into one buffer instead of allocating an N-element intermediate array of strings.
+        # Identical output to map.join("\n") (no trailing newline). For true O(1) memory, stream
+        # to an IO with #format_stream instead of materialising the whole string.
+        buffer = +""
+        records.each_with_index do |record, i|
+          buffer << "\n" if i.positive?
+          buffer << JSON.generate(record)
+        end
+        buffer
+      end
+
+
+      # Write to file
+      #
+      # @param stream [Enumerator] the record stream
+      # @param path [String] file path
+      # @return [void]
+
+      def write_to_file(stream, path)
+        File.open(path, "w") do |file|
+          format_stream(stream, io: file)
+        end
+      end
+    end
+  end
+end
+