spec_ai 0.1.0

data/lib/spec_ai/cli.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require "thor"
+
+module SpecAi
+  class CLI < Thor
+    class_option :root, type: :string, desc: "Rails application root"
+    class_option :provider, type: :string, desc: "LLM provider (openai, anthropic, google, azure, ollama)"
+    class_option :model, type: :string, desc: "LLM model name"
+    class_option :force, type: :boolean, default: false, aliases: "-f", desc: "Replace an existing spec (default for generate)"
+    class_option :merge, type: :boolean, default: false, desc: "Add examples to an existing spec instead of replacing it"
+    class_option :no_run, type: :boolean, default: false, desc: "Skip running RSpec after generation"
+    class_option :no_fix, type: :boolean, default: false, desc: "Skip the AI repair loop"
+    class_option :verbose, type: :boolean, aliases: "-v", desc: "Show RCrewAI and HTTP logs"
+
+    def self.exit_on_failure?
+      true
+    end
+
+    desc "generate TARGET", "Analyze a Rails model and generate RSpec tests"
+    def generate(target)
+      configure_from_options!
+      result = Pipeline::Generate.new(target, force: replace_spec?, ui: ui).run
+      exit(1) unless result.success?
+    rescue Interrupt
+      ui.error("Interrupted.")
+      exit(130)
+    rescue SpecAi::Error => error
+      ui.error(error.message)
+      exit(1)
+    rescue StandardError => error
+      ui.unexpected_error(error)
+      exit(1)
+    end
+
+    desc "fix PATH", "Run a spec and repair failures with AI"
+    def fix(path)
+      configure_from_options!
+      result = Pipeline::Fix.new(path, ui: ui).run
+      exit(1) unless result.success?
+    rescue Interrupt
+      ui.error("Interrupted.")
+      exit(130)
+    rescue SpecAi::Error => error
+      ui.error(error.message)
+      exit(1)
+    rescue StandardError => error
+      ui.unexpected_error(error)
+      exit(1)
+    end
+
+    desc "analyze TARGET", "Print collected Rails context without calling an LLM"
+    def analyze(target)
+      configure_from_options!
+      resolved = Target.new(target)
+      snapshot = Context::Builder.new(resolved).build
+      ui.header
+      ui.analyzing(resolved.class_name)
+      ui.context_collected(snapshot)
+      ui.snapshot(snapshot)
+    rescue Interrupt
+      ui.error("Interrupted.")
+      exit(130)
+    rescue SpecAi::Error => error
+      ui.error(error.message)
+      exit(1)
+    rescue StandardError => error
+      ui.unexpected_error(error)
+      exit(1)
+    end
+
+    desc "missing", "Find important behavior that is not tested (coming in 0.3)"
+    def missing
+      ui.roadmap("missing", "0.3")
+    end
+
+    desc "improve PATH", "Improve an existing spec (coming in 0.5)"
+    def improve(_path = nil)
+      ui.roadmap("improve", "0.5")
+    end
+
+    map "g" => :generate
+
+    no_commands do
+      def ui
+        @ui ||= UI.new
+      end
+    end
+
+    private
+
+    def configure_from_options!
+      root = File.expand_path(options[:root] || detect_root)
+      SpecAi.reset_configuration!
+      SpecAi.configuration.app_root = root
+      config_file = File.join(root, "config", "spec_ai.yml")
+      SpecAi.configuration.load_from_file!(config_file) if File.file?(config_file)
+      SpecAi.configuration.provider = options[:provider] if options[:provider]
+      SpecAi.configuration.model = options[:model] if options[:model]
+      SpecAi.configuration.verbose = true if options[:verbose]
+      SpecAi.configuration.run_after_generation = false if options[:no_run]
+      SpecAi.configuration.auto_fix = false if options[:no_fix]
+      SpecAi.boot_rails!(root)
+    end
+
+    def replace_spec?
+      return true if options[:force]
+      return false if options[:merge]
+
+      true
+    end
+
+    def detect_root
+      dir = Dir.pwd
+      while dir != File.dirname(dir)
+        return dir if File.file?(File.join(dir, "config", "environment.rb")) ||
+                      File.file?(File.join(dir, "config", "spec_ai.yml"))
+
+        dir = File.dirname(dir)
+      end
+      Dir.pwd
+    end
+  end
+end

data/lib/spec_ai/configuration.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module SpecAi
+  class Configuration
+    ATTRS = %i[
+      provider
+      model
+      temperature
+      app_root
+      testing_framework
+      run_after_generation
+      auto_fix
+      max_repair_attempts
+      include_edge_cases
+      include_security_cases
+      include_authorization
+      verbose
+    ].freeze
+
+    attr_accessor(*ATTRS)
+
+    def initialize
+      @provider = "openai"
+      @model = "gpt-4o"
+      @temperature = 0.1
+      @app_root = Dir.pwd
+      @testing_framework = "rspec"
+      @run_after_generation = true
+      @auto_fix = true
+      @max_repair_attempts = 2
+      @include_edge_cases = true
+      @include_security_cases = true
+      @include_authorization = true
+      @verbose = false
+    end
+
+    def load_from_file!(path)
+      data = YAML.safe_load_file(path, permitted_classes: [Symbol]) || {}
+      apply_hash!(data)
+      self
+    end
+
+    def apply_to_rcrewai!
+      require "rcrewai"
+
+      ollama = provider.to_s == "ollama"
+      RCrewAI.configure(validate: !ollama) do |config|
+        config.llm_provider = provider.to_sym
+        assign_rcrewai(config, :model, model)
+        assign_rcrewai(config, :temperature, temperature)
+        assign_rcrewai(config, "#{provider}_model", model)
+        assign_rcrewai(config, :openai_api_key, ENV["OPENAI_API_KEY"])
+        assign_rcrewai(config, :anthropic_api_key, ENV["ANTHROPIC_API_KEY"])
+        assign_rcrewai(config, :google_api_key, ENV["GOOGLE_API_KEY"] || ENV["GEMINI_API_KEY"])
+        assign_rcrewai(config, :azure_api_key, ENV["AZURE_OPENAI_API_KEY"])
+        assign_rcrewai(config, :base_url, ENV["AZURE_OPENAI_ENDPOINT"]) if provider.to_s == "azure"
+        if ollama
+          assign_rcrewai(config, :base_url, ENV["OLLAMA_URL"] || "http://localhost:11434")
+          assign_rcrewai(config, :api_key, ENV["OLLAMA_API_KEY"] || "ollama")
+        end
+      end
+    end
+
+    private
+
+    def apply_hash!(data)
+      data = stringify_keys(data)
+
+      self.provider = data["provider"] if data["provider"]
+      self.model = data["model"] if data["model"]
+      self.temperature = data["temperature"] if data.key?("temperature")
+      self.verbose = data["verbose"] unless data["verbose"].nil?
+
+      if (testing = data["testing"]).is_a?(Hash)
+        self.testing_framework = testing["framework"] if testing["framework"]
+      end
+
+      if (generation = data["generation"]).is_a?(Hash)
+        self.include_edge_cases = generation["include_edge_cases"] unless generation["include_edge_cases"].nil?
+        self.include_security_cases = generation["include_security_cases"] unless generation["include_security_cases"].nil?
+        self.include_authorization = generation["include_authorization"] unless generation["include_authorization"].nil?
+      end
+
+      return unless (execution = data["execution"]).is_a?(Hash)
+
+      self.run_after_generation = execution["run_after_generation"] unless execution["run_after_generation"].nil?
+      self.auto_fix = execution["auto_fix"] unless execution["auto_fix"].nil?
+      self.max_repair_attempts = execution["max_repair_attempts"] if execution["max_repair_attempts"]
+    end
+
+    def stringify_keys(hash)
+      hash.each_with_object({}) do |(key, value), memo|
+        memo[key.to_s] = value.is_a?(Hash) ? stringify_keys(value) : value
+      end
+    end
+
+    def assign_rcrewai(config, setter, value)
+      return if value.nil? || value.to_s.empty?
+      return unless config.respond_to?("#{setter}=")
+
+      config.public_send("#{setter}=", value)
+    end
+  end
+end

data/lib/spec_ai/context/builder.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module SpecAi
+  module Context
+    class Builder
+      MAX_RELATED = 3
+      MAX_EXCERPT = 1200
+
+      def initialize(target, sandbox: Sandbox.new)
+        @target = target.is_a?(Target) ? target : Target.new(target)
+        @sandbox = sandbox
+      end
+
+      def build
+        model = Analyzers::Model.new(@target, sandbox: @sandbox).analyze
+        Snapshot.new(
+          target: @target,
+          model: model,
+          schema: Analyzers::Schema.new(@target, sandbox: @sandbox).analyze,
+          factories: Analyzers::Factories.new(@target, sandbox: @sandbox).analyze,
+          existing_spec: Analyzers::ExistingSpecs.new(@target, sandbox: @sandbox).analyze,
+          routes: Analyzers::Routes.new(@target, sandbox: @sandbox).analyze,
+          related: related_models(model)
+        )
+      end
+
+      private
+
+      def related_models(model)
+        Array(model[:associations]).first(MAX_RELATED).filter_map do |association|
+          class_name = association[:class_name].to_s
+          next if class_name.empty? || class_name == @target.class_name
+
+          path = Inflector.model_path(class_name)
+          next unless @sandbox.exist?(path)
+
+          excerpt = @sandbox.read(path)
+          excerpt = "#{excerpt[0, MAX_EXCERPT]}\n..." if excerpt.length > MAX_EXCERPT
+          { class_name: class_name, path: path, excerpt: excerpt }
+        end
+      end
+    end
+  end
+end

data/lib/spec_ai/context/snapshot.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module SpecAi
+  module Context
+    class Snapshot
+      attr_reader :target, :model, :schema, :factories, :existing_spec, :routes, :related
+
+      def initialize(target:, model:, schema:, factories:, existing_spec:, routes:, related: [])
+        @target = target
+        @model = model
+        @schema = schema
+        @factories = factories
+        @existing_spec = existing_spec
+        @routes = routes
+        @related = related
+      end
+
+      def has_factory?
+        factories[:present]
+      end
+
+      def devise?
+        model[:source].to_s.match?(/\bdevise\b/) ||
+          Array(schema[:columns]).map(&:to_s).include?("encrypted_password")
+      end
+
+      def known_attributes
+        skip = %w[id created_at updated_at encrypted_password reset_password_token reset_password_sent_at remember_created_at]
+        columns = Array(schema[:columns]).map(&:to_s) - skip
+        columns.reject! { |column| column.end_with?("_id") }
+        from_validations = Array(model[:validations]).flat_map { |item| Array(item[:attributes] || item["attributes"]) }.map(&:to_s)
+        virtual = model[:source].to_s.match?(/\bdevise\b/) ? %w[password password_confirmation] : []
+        (columns + from_validations + virtual).uniq
+      end
+
+      def association_names
+        Array(model[:associations]).map { |item| (item[:name] || item["name"]).to_s }.reject(&:empty?)
+      end
+
+      def validators_for(attribute)
+        name = attribute.to_s
+        return [] if name.empty?
+
+        Array(model[:validations]).each_with_object([]) do |item, memo|
+          attrs = Array(item[:attributes] || item["attributes"]).map(&:to_s)
+          next unless attrs.include?(name)
+
+          memo.concat(Array(item[:validators] || item["validators"]).map(&:to_s))
+        end.uniq
+      end
+
+      def existing_example_names
+        Array(existing_spec[:example_names])
+      end
+
+      def to_h
+        {
+          target: target.class_name,
+          model: model,
+          schema: schema,
+          factories: factories,
+          existing_spec: existing_spec,
+          routes: routes,
+          related: related
+        }
+      end
+
+      def to_prompt
+        sections = []
+        sections << "Target: #{target.class_name} (#{target.model_path})"
+        sections << "Table: #{model[:table_name]}"
+        sections << "Source:\n#{model[:source]}"
+        sections << "Associations:\n#{format_list(model[:associations])}"
+        sections << "Validations:\n#{format_list(model[:validations])}"
+        sections << "Scopes: #{Array(model[:scopes]).join(', ')}"
+        sections << "Enums: #{Array(model[:enums]).join(', ')}"
+        sections << "Schema (#{schema[:path]}):\n#{schema[:excerpt]}" if schema[:excerpt]
+        if factories[:present]
+          sections << "Factory (#{factories[:path]}, :#{factories[:name]}):\n#{factories[:excerpt]}"
+        else
+          sections << "Factory: none found"
+        end
+        if existing_spec[:present]
+          sections << "Existing spec (#{existing_spec[:path]}):\n#{existing_spec[:excerpt]}"
+        else
+          sections << "Existing spec: none (will create #{target.spec_path})"
+        end
+        sections << "Routes:\n#{routes[:excerpt]}" if routes[:excerpt]
+        related.each do |item|
+          sections << "Related #{item[:class_name]} (#{item[:path]}):\n#{item[:excerpt]}"
+        end
+        sections.join("\n\n")
+      end
+
+      def to_s
+        to_prompt
+      end
+
+      private
+
+      def format_list(items)
+        return "(none)" if items.nil? || items.empty?
+
+        items.map { |item| "- #{item.inspect}" }.join("\n")
+      end
+    end
+  end
+end

data/lib/spec_ai/crews/generate_crew.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require "rcrewai"
+
+module SpecAi
+  module Crews
+    class GenerateCrew
+      attr_reader :fallback_notice, :fallback_preview
+
+      def initialize(snapshot, sandbox: Sandbox.new)
+        @snapshot = snapshot
+        @sandbox = sandbox
+      end
+
+      def plan
+        Silence.call { llm_plan }
+      rescue InvalidPlan => error
+        fallback!(error)
+      rescue StandardError => error
+        raise unless recoverable?(error)
+
+        fallback!(error)
+      end
+
+      def fallback?
+        !@fallback_notice.nil?
+      end
+
+      private
+
+      def llm_plan
+        SpecAi.configuration.apply_to_rcrewai!
+        crew = RCrewAI::Crew.new("spec_ai_generate", verbose: SpecAi.configuration.verbose)
+        analyst = build_analyst
+        architect = build_architect
+        crew.add_agent(analyst)
+        crew.add_agent(architect)
+
+        analyze_task = build_analyze_task(analyst)
+        plan_task = build_plan_task(architect, analyze_task)
+        crew.add_task(analyze_task)
+        crew.add_task(plan_task)
+        crew.execute
+        TestPlan.from_task(plan_task)
+      end
+
+      def build_analyst
+        RCrewAI::Agent.new(
+          name: "rails_analyst",
+          role: "Senior Rails Application Analyst",
+          goal: "Collect and summarize Rails model context needed to write accurate tests",
+          backstory: "You know Rails conventions, ActiveRecord, and how schema, factories, and specs fit together.",
+          tools: Toolset.build(@sandbox),
+          verbose: SpecAi.configuration.verbose,
+          max_iterations: 4
+        )
+      end
+
+      def build_architect
+        RCrewAI::Agent.new(
+          name: "test_architect",
+          role: "Rails Test Architect",
+          goal: "Produce a complete structured RSpec test plan for the target model",
+          backstory: "You design focused model specs. You never invent columns or associations that are not in the context.",
+          verbose: SpecAi.configuration.verbose,
+          max_iterations: 3
+        )
+      end
+
+      def build_analyze_task(agent)
+        RCrewAI::Task.new(
+          name: "analyze_model",
+          agent: agent,
+          description: <<~DESC,
+            Analyze #{@snapshot.target.class_name} for test generation.
+
+            Pre-collected context:
+            #{@snapshot.to_prompt}
+
+            Use tools only if you need a missing related file. Summarize:
+            1. Validations and attributes
+            2. Associations and dependent behavior
+            3. Scopes / enums
+            4. Factory coverage
+            5. Gaps in the existing spec
+          DESC
+          expected_output: "A concise analysis of behaviors that should be tested, including gaps in the existing spec."
+        )
+      end
+
+      def build_plan_task(agent, analyze_task)
+        extras = []
+        extras << "Include edge cases." if SpecAi.configuration.include_edge_cases
+        extras << "Include security-relevant cases when the model has tokens, passwords, or roles." if SpecAi.configuration.include_security_cases
+        extras << "Include authorization-adjacent cases when admin/role flags exist." if SpecAi.configuration.include_authorization
+
+        RCrewAI::Task.new(
+          name: "plan_tests",
+          agent: agent,
+          context: [analyze_task],
+          output_schema: TestPlan::SCHEMA,
+          guardrail: method(:plan_guardrail),
+          guardrail_max_retries: 2,
+          description: <<~DESC,
+            Create a structured test plan for #{@snapshot.target.class_name}.
+
+            Rails testing playbook:
+            #{Knowledge::Playbook.text}
+
+            #{extras.join("\n")}
+
+            Return JSON only. Start with { and end with }. No markdown, no prose, no tool traces.
+            Each scenario needs name, category, priority, expected_behavior.
+            Set attribute/value for validations and association/association_macro for associations.
+            Do not duplicate these existing examples: #{@snapshot.existing_example_names.join(', ')}
+          DESC
+          expected_output: "JSON only: an object with target, test_type, and scenarios. No other text."
+        )
+      end
+
+      def plan_guardrail(output)
+        plan = TestPlan.from(output)
+        [true, plan.to_h]
+      rescue InvalidPlan => error
+        [false, error.message]
+      end
+
+      def fallback!(error)
+        plan = SnapshotPlan.build(@snapshot)
+        raise rewrite_error(error) unless plan
+
+        cfg = SpecAi.configuration
+        @fallback_notice = "#{cfg.provider} / #{cfg.model} did not return a valid test plan"
+        @fallback_preview = TestPlan.preview(error.message)
+        plan
+      end
+
+      def rewrite_error(error)
+        return error if error.is_a?(InvalidPlan) && error.message.include?("did not return a valid test plan")
+
+        InvalidPlan.new(TestPlan.unreadable(error.message))
+      end
+
+      def recoverable?(error)
+        return true if error.is_a?(Errno::ECONNREFUSED) || error.is_a?(SocketError)
+
+        name = error.class.name.to_s
+        name.match?(/RCrewAI|Faraday|Timeout|HTTP|OpenAI|Anthropic/i)
+      end
+    end
+  end
+end

data/lib/spec_ai/crews/repair_crew.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "rcrewai"
+
+module SpecAi
+  module Crews
+    class RepairCrew
+      def initialize(snapshot, run_result, spec_contents, previous_plan: nil, sandbox: Sandbox.new)
+        @snapshot = snapshot
+        @run_result = run_result
+        @spec_contents = spec_contents
+        @previous_plan = previous_plan
+        @sandbox = sandbox
+      end
+
+      def plan
+        Silence.call { plan! }
+      rescue InvalidPlan
+        raise
+      rescue StandardError => error
+        raise unless recoverable?(error)
+
+        raise InvalidPlan, TestPlan.unreadable(error.message)
+      end
+
+      private
+
+      def plan!
+        SpecAi.configuration.apply_to_rcrewai!
+        crew = RCrewAI::Crew.new("spec_ai_repair", verbose: SpecAi.configuration.verbose)
+        debugger_agent = build_debugger
+        crew.add_agent(debugger_agent)
+        repair_task = build_repair_task(debugger_agent)
+        crew.add_task(repair_task)
+        crew.execute
+        TestPlan.from_task(repair_task)
+      end
+
+      def build_debugger
+        RCrewAI::Agent.new(
+          name: "qa_debugger",
+          role: "RSpec Failure Debugger",
+          goal: "Fix a structured test plan so generated model specs pass without weakening assertions",
+          backstory: "You read RSpec failures, compare them to Rails source, and correct setup, attributes, and expectations. You never drop coverage just to make a spec pass.",
+          tools: Toolset.build(@sandbox),
+          verbose: SpecAi.configuration.verbose,
+          max_iterations: 4
+        )
+      end
+
+      def build_repair_task(agent)
+        previous = @previous_plan ? JSON.pretty_generate(@previous_plan.to_h) : "(none)"
+
+        RCrewAI::Task.new(
+          name: "repair_plan",
+          agent: agent,
+          output_schema: TestPlan::SCHEMA,
+          guardrail: method(:plan_guardrail),
+          guardrail_max_retries: 2,
+          description: <<~DESC,
+            The generated test failed. Return a corrected structured test plan for #{@snapshot.target.class_name}.
+            Do not return Ruby. Keep useful scenarios; fix setup, attributes, factories, and expectations.
+
+            Rails testing playbook:
+            #{Knowledge::Playbook.text}
+
+            Application context:
+            #{@snapshot.to_prompt}
+
+            Previous plan:
+            #{previous}
+
+            Current spec (#{@snapshot.target.spec_path}):
+            #{@spec_contents}
+
+            RSpec output:
+            #{@run_result.output}
+          DESC
+          expected_output: "A corrected JSON test plan matching the schema."
+        )
+      end
+
+      def plan_guardrail(output)
+        plan = TestPlan.from(output)
+        [true, plan.to_h]
+      rescue InvalidPlan => error
+        [false, error.message]
+      end
+
+      def recoverable?(error)
+        return true if error.is_a?(Errno::ECONNREFUSED) || error.is_a?(SocketError)
+
+        error.class.name.to_s.match?(/RCrewAI|Faraday|Timeout|HTTP|OpenAI|Anthropic/i)
+      end
+    end
+  end
+end

data/lib/spec_ai/crews/silence.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module SpecAi
+  module Crews
+    module Silence
+      module_function
+
+      def call
+        return yield if SpecAi.configuration.verbose
+
+        original = $stdout
+        $stdout = StringIO.new
+        yield
+      ensure
+        $stdout = original if original
+      end
+    end
+  end
+end

data/lib/spec_ai/crews/toolset.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "rcrewai"
+
+module SpecAi
+  module Crews
+    module Toolset
+      def self.build(sandbox = Sandbox.new)
+        [
+          Tools::ReadAppFile.new(sandbox: sandbox),
+          Tools::ReadSchemaTable.new(sandbox: sandbox),
+          Tools::ReadFactory.new(sandbox: sandbox),
+          Tools::ReadExistingSpec.new(sandbox: sandbox),
+          Tools::ListRelatedFiles.new(sandbox: sandbox)
+        ]
+      end
+    end
+  end
+end