spec_ai 0.1.0

data/lib/spec_ai/pipeline/generate.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module SpecAi
+  module Pipeline
+    class Generate
+      def initialize(input, force: false, ui: UI.new, sandbox: Sandbox.new)
+        @input = input
+        @force = force
+        @ui = ui
+        @sandbox = sandbox
+      end
+
+      def run
+        @ui.header
+        target = Target.new(@input, app_root: @sandbox.root)
+        @ui.analyzing(target.class_name)
+        snapshot = Context::Builder.new(target, sandbox: @sandbox).build
+        @ui.context_collected(snapshot)
+        Runners::RspecRunner.new(app_root: @sandbox.root).ensure_available! if SpecAi.configuration.run_after_generation
+        snapshot = prepare_app!(target, snapshot)
+
+        crew = Crews::GenerateCrew.new(snapshot, sandbox: @sandbox)
+        plan = @ui.spin_planning { crew.plan }
+        @ui.llm_fallback(crew.fallback_notice, preview: crew.fallback_preview) if crew.fallback?
+        plan = TestPlan.stabilize(plan, snapshot)
+        @ui.generating(plan)
+
+        spec_path = Generators::RspecGenerator.new(plan, snapshot, force: @force, sandbox: @sandbox).write
+        if @force
+          @ui.writing(relative(spec_path))
+        else
+          @ui.merging(relative(spec_path))
+        end
+
+        unless SpecAi.configuration.run_after_generation
+          @ui.completed(true)
+          return Result.new(target: target, spec_path: spec_path, plan: plan, run: nil)
+        end
+
+        run_and_repair(snapshot, plan, spec_path)
+      end
+
+      private
+
+      def run_and_repair(snapshot, plan, spec_path)
+        runner = Runners::RspecRunner.new(app_root: @sandbox.root)
+        result = @ui.spin_running { runner.run(spec_path) }
+        @ui.rspec_result(result)
+        result = rerun_after_helper_patch(runner, spec_path, result)
+
+        attempts = 0
+        prune_passes = 0
+        max = SpecAi.configuration.max_repair_attempts.to_i
+        while result.repairable? && SpecAi.configuration.auto_fix && attempts < max
+          if prune_passes < 2
+            pruned, dropped = Repair::Prune.call(plan, result)
+            next_plan = pruned || SnapshotPlan.build(snapshot)
+            if dropped.any? && next_plan && next_plan.scenarios.map(&:name) != plan.scenarios.map(&:name)
+              plan = next_plan
+              prune_passes += 1
+              @ui.dropped_unfounded(dropped)
+              spec_path = Generators::RspecGenerator.new(plan, snapshot, force: true, sandbox: @sandbox).write
+              @ui.writing(relative(spec_path))
+              result = @ui.spin_running { runner.run(spec_path) }
+              @ui.rspec_result(result)
+              next
+            end
+          end
+
+          attempts += 1
+          spec_contents = File.exist?(spec_path) ? File.read(spec_path) : @sandbox.read(snapshot.target.spec_path)
+          begin
+            plan = @ui.spin_repairing(attempts, max) do
+              Crews::RepairCrew.new(snapshot, result, spec_contents, previous_plan: plan, sandbox: @sandbox).plan
+            end
+            plan = plan.ground(snapshot) || SnapshotPlan.build(snapshot) || plan
+          rescue InvalidPlan
+            @ui.skip_repair("The repair model did not return a valid test plan. The spec was left as-is with the failures above.")
+            break
+          end
+          spec_path = Generators::RspecGenerator.new(plan, snapshot, force: true, sandbox: @sandbox).write
+          @ui.writing(relative(spec_path))
+          result = @ui.spin_running { runner.run(spec_path) }
+          @ui.rspec_result(result)
+        end
+
+        @ui.completed(result.passed?, result: result)
+        Result.new(target: snapshot.target, spec_path: spec_path, plan: plan, run: result, repair_attempts: attempts)
+      end
+
+      def relative(path)
+        path.to_s.sub(%r{\A#{Regexp.escape(@sandbox.root)}/?}, "")
+      end
+
+      def rerun_after_helper_patch(runner, spec_path, result)
+        return result unless result.failed?
+        return result unless Generators::RspecGenerator.patch_helper!(relative(spec_path), sandbox: @sandbox)
+
+        @ui.writing(relative(spec_path))
+        result = @ui.spin_running { runner.run(spec_path) }
+        @ui.rspec_result(result)
+        result
+      end
+
+      def prepare_app!(target, snapshot)
+        setup = Setup::Ensure.new(snapshot, sandbox: @sandbox).run
+        @ui.setup_actions(setup.actions)
+        raise ConfigurationError, setup.blocking_message if setup.blocking?
+
+        return snapshot unless setup.changed?
+
+        Context::Builder.new(target, sandbox: @sandbox).build
+      end
+    end
+  end
+end

data/lib/spec_ai/pipeline/result.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module SpecAi
+  module Pipeline
+    class Result
+      attr_reader :target, :spec_path, :plan, :run, :repair_attempts
+
+      def initialize(target:, spec_path:, plan:, run:, repair_attempts: 0)
+        @target = target
+        @spec_path = spec_path
+        @plan = plan
+        @run = run
+        @repair_attempts = repair_attempts
+      end
+
+      def success?
+        run.nil? || run.passed?
+      end
+    end
+  end
+end

data/lib/spec_ai/railtie.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "rails/railtie"
+
+module SpecAi
+  class Railtie < ::Rails::Railtie
+    rake_tasks do
+      load File.expand_path("../tasks/spec_ai.rake", __dir__)
+    end
+  end
+end

data/lib/spec_ai/repair/prune.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module SpecAi
+  module Repair
+    class Prune
+      def self.call(plan, result)
+        new(plan, result).call
+      end
+
+      def initialize(plan, result)
+        @plan = plan
+        @result = result
+      end
+
+      def call
+        names = droppable_names
+        return [@plan, []] if names.empty?
+
+        [@plan.without_existing(names), names]
+      end
+
+      private
+
+      def droppable_names
+        failures = @result.report.failures
+        @plan.scenarios.filter_map do |scenario|
+          scenario.name if failures.any? { |failure| matches?(failure, scenario) && droppable?(failure) }
+        end
+      end
+
+      def matches?(failure, scenario)
+        failure.example.to_s.downcase.include?(scenario.name.to_s.downcase)
+      end
+
+      def droppable?(failure)
+        haystack = "#{failure.example} #{failure.snippet} #{failure.detail}"
+        return true if haystack.match?(/expected .+ not to be valid/i)
+        return true if haystack.match?(/nil\.present\?/i)
+        return true if empty_errors?(failure, haystack)
+
+        false
+      end
+
+      def empty_errors?(failure, haystack)
+        haystack.match?(/\[\]\.present\?/) && !failure.example.to_s.match?(/duplicate|unique/i)
+      end
+    end
+  end
+end

data/lib/spec_ai/runners/report.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+module SpecAi
+  module Runners
+    class Report
+      Failure = Struct.new(:example, :snippet, :detail, :hint, keyword_init: true)
+
+      SUMMARY_RE = /(\d+)\s+examples?,\s+(\d+)\s+failures?(?:,\s+(\d+)\s+errors?)?/
+      FAILED_EXAMPLE_RE = /^rspec\s+\S+\s+#\s+(.+)$/
+      LOAD_ERROR_RE = /error occurred (?:outside of examples|while loading)/i
+      MISSING_FILE_RE = /cannot load such file -- (\S+)/
+
+      attr_reader :result
+
+      def initialize(result)
+        @result = result
+      end
+
+      def summary
+        if (match = @result.output.match(SUMMARY_RE))
+          text = "#{match[1]} #{match[1] == '1' ? 'example' : 'examples'}, #{match[2]} #{match[2] == '1' ? 'failure' : 'failures'}"
+          text += ", #{match[3]} #{match[3] == '1' ? 'error' : 'errors'}" if match[3]
+          return text
+        end
+
+        return "spec failed to load" if load_error?
+        return "passed" if @result.passed?
+
+        "examples failed"
+      end
+
+      def load_error?
+        @result.output.match?(LOAD_ERROR_RE) ||
+          @result.output.match?(MISSING_FILE_RE) ||
+          @result.output.match?(/uninitialized constant/) ||
+          (@result.failed? && @result.output.match?(/\b0 examples?, 0 failures?, \d+ errors?/))
+      end
+
+      def failures
+        @failures ||= parse_failures
+      end
+
+      private
+
+      def parse_failures
+        blocks = numbered_failures
+        return blocks unless blocks.empty?
+        return [load_failure] if load_error?
+
+        fallback_examples.map do |example|
+          Failure.new(example: example, snippet: nil, detail: nil, hint: hint_for(example, "", ""))
+        end
+      end
+
+      def load_failure
+        detail = load_error_detail
+        Failure.new(
+          example: "spec failed to load",
+          snippet: nil,
+          detail: detail,
+          hint: hint_for("spec failed to load", "", detail.to_s)
+        )
+      end
+
+      def load_error_detail
+        output = @result.output
+        if (match = output.match(MISSING_FILE_RE))
+          return "cannot load such file -- #{match[1]}"
+        end
+
+        line = output.lines.map(&:strip).find do |entry|
+          entry.match?(LOAD_ERROR_RE) || entry.match?(/uninitialized constant|LoadError/)
+        end
+        line || output.lines.map(&:strip).reject(&:empty?).first
+      end
+
+      def numbered_failures
+        chunks = @result.output.split(/^\s+\d+\)\s+/)
+        chunks.shift
+        chunks.filter_map do |chunk|
+          lines = chunk.lines.map(&:rstrip)
+          example = lines.first.to_s.strip
+          next if example.empty?
+
+          snippet_line = lines.find { |line| line.include?("Failure/Error:") }
+          snippet = snippet_line&.sub(/.*Failure\/Error:\s*/, "")&.strip
+          detail = detail_from(lines)
+          Failure.new(example: example, snippet: snippet, detail: detail, hint: hint_for(example, snippet.to_s, detail.to_s))
+        end
+      end
+
+      def detail_from(lines)
+        start = lines.index { |line| line.include?("Failure/Error:") }
+        return nil unless start
+
+        lines[(start + 1)..].take_while do |line|
+          stripped = line.strip
+          stripped != "" && !stripped.start_with?("#") && !stripped.start_with?("# ./")
+        end.map(&:strip).reject(&:empty?).join(" ")
+      end
+
+      def fallback_examples
+        @result.output.scan(FAILED_EXAMPLE_RE).flatten
+      end
+
+      def hint_for(example, snippet, detail)
+        haystack = "#{example} #{snippet} #{detail}"
+
+        if haystack.match?(/cannot load such file -- (?:spec_helper|rails_helper)/)
+          "RSpec's bootstrap files are missing. Spec AI creates spec/spec_helper.rb and spec/rails_helper.rb when the spec folder is gone."
+        elsif haystack.match?(/uninitialized constant/)
+          "Rails did not load this spec. Model specs need `require \"rails_helper\"`."
+        elsif haystack.match?(/can't find executable rspec|not currently included in the bundle/i)
+          "RSpec is not in the application's Gemfile. Add `gem \"rspec-rails\"` and run `bundle install`."
+        elsif missing_association?(haystack)
+          "The spec expects an association that is not defined on the model."
+        elsif uniqueness_miss?(example, haystack)
+          "The duplicate record was not saved, so uniqueness never ran. Create usually needs the same required fields as a valid record."
+        elsif haystack.match?(/\[\]\.present\?/)
+          "The model did not add an error on this attribute. The spec is testing a rule that is not in the model."
+        elsif missing_required_attrs?(haystack)
+          "The spec built a record without required attributes. Add a FactoryBot factory or set those fields in the example."
+        elsif haystack.match?(/expected .+ not to be valid/i)
+          "The model accepted this value. Devise strips email whitespace and does not require password_confirmation to be present."
+        elsif haystack.match?(/NameError|undefined (?:local variable|method)/i)
+          "The spec referenced a method or variable that does not exist."
+        end
+      end
+
+      def missing_association?(haystack)
+        haystack.match?(/reflect_on_association|nil\.present\?|expected `nil`/i) ||
+          haystack.match?(/has_many|belongs_to|has_one/i) && haystack.match?(/to be truthy, got false/i)
+      end
+
+      def uniqueness_miss?(example, haystack)
+        example.match?(/duplicate|unique/i) && haystack.match?(/\[\]\.present\?|to be truthy, got false/i)
+      end
+
+      def missing_required_attrs?(haystack)
+        haystack.match?(/to be valid/i) && haystack.match?(/can't be blank/i)
+      end
+    end
+  end
+end

data/lib/spec_ai/runners/rspec_runner.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module SpecAi
+  module Runners
+    class RspecRunner
+      MISSING_RSPEC = /
+        can't\ find\ executable\ rspec
+        |rspec-core\ is\ not\ currently\ included\ in\ the\ bundle
+        |Could\ not\ find\ gem\ ['"]rspec
+      /ix
+
+      Result = Struct.new(:exit_status, :stdout, :stderr, :spec_path, keyword_init: true) do
+        def passed?
+          exit_status.to_i.zero?
+        end
+
+        def failed?
+          !passed?
+        end
+
+        def output
+          [stdout, stderr].compact.join("\n")
+        end
+
+        def environment_error?
+          output.match?(MISSING_RSPEC)
+        end
+
+        def repairable?
+          failed? && !environment_error? && !report.load_error?
+        end
+
+        def report
+          @report ||= Report.new(self)
+        end
+      end
+
+      def initialize(app_root: SpecAi.configuration.app_root)
+        @app_root = app_root
+      end
+
+      def ensure_available!
+        return if bundled_rspec?
+
+        raise ConfigurationError, missing_rspec_message
+      end
+
+      def run(spec_path)
+        relative = spec_path.to_s.sub(%r{\A#{Regexp.escape(@app_root)}/?}, "")
+        env = {}
+        gemfile = File.join(@app_root, "Gemfile")
+        env["BUNDLE_GEMFILE"] = gemfile if File.file?(gemfile)
+
+        stdout, stderr, status = Open3.capture3(
+          env,
+          "bundle", "exec", "rspec", relative, "--format", "documentation",
+          chdir: @app_root
+        )
+        result = Result.new(exit_status: status.exitstatus, stdout: stdout, stderr: stderr, spec_path: relative)
+        raise ConfigurationError, missing_rspec_message if result.environment_error?
+
+        result
+      end
+
+      private
+
+      def bundled_rspec?
+        return true if File.executable?(File.join(@app_root, "bin", "rspec"))
+
+        lock = File.join(@app_root, "Gemfile.lock")
+        if File.file?(lock)
+          return true if File.read(lock).match?(/^\s{4}rspec(?:-core|-rails)? \(/)
+        end
+
+        gemfile = File.join(@app_root, "Gemfile")
+        File.file?(gemfile) && File.read(gemfile).match?(/\bgem\s+["']rspec(?:-rails|-core)?["']/)
+      end
+
+      def missing_rspec_message
+        <<~MSG.strip
+          RSpec is not in this application's bundle. Spec AI writes model specs, then runs them with `bundle exec rspec`.
+
+          Add this to the Gemfile:
+
+            group :development, :test do
+              gem "rspec-rails"
+            end
+
+          Then run:
+
+            bundle install
+            bin/rails generate rspec:install
+        MSG
+      end
+    end
+  end
+end

data/lib/spec_ai/sandbox.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module SpecAi
+  class Sandbox
+    ALLOWED_PREFIXES = %w[app/ spec/ db/ config/ test/].freeze
+
+    attr_reader :root
+
+    def initialize(root = SpecAi.configuration.app_root)
+      @root = File.expand_path(root)
+    end
+
+    def read(relative_path)
+      File.read(resolve!(relative_path))
+    end
+
+    def write(relative_path, content)
+      path = resolve!(relative_path)
+      FileUtils.mkdir_p(File.dirname(path))
+      File.write(path, content)
+      path
+    end
+
+    def exist?(relative_path)
+      File.file?(resolve!(relative_path))
+    rescue PathNotAllowed
+      false
+    end
+
+    def glob(pattern)
+      Dir.glob(File.join(@root, pattern)).filter_map do |full|
+        next unless full.start_with?(@root)
+
+        rel = full.delete_prefix(@root).delete_prefix("/")
+        rel if allowed?(rel)
+      end
+    end
+
+    def resolve!(relative_path)
+      rel = normalize(relative_path)
+      raise PathNotAllowed, "path is not allowed: #{relative_path}" unless allowed?(rel)
+
+      full = File.expand_path(rel, @root)
+      unless full == @root || full.start_with?(@root + File::SEPARATOR)
+        raise PathNotAllowed, "path escapes app root: #{relative_path}"
+      end
+
+      full
+    end
+
+    private
+
+    def normalize(relative_path)
+      relative_path.to_s.sub(%r{\A/+}, "")
+    end
+
+    def allowed?(rel)
+      ALLOWED_PREFIXES.any? do |prefix|
+        rel == prefix.delete_suffix("/") || rel.start_with?(prefix)
+      end
+    end
+  end
+end

data/lib/spec_ai/setup/ensure.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module SpecAi
+  module Setup
+    class Ensure
+      Action = Struct.new(:kind, :path, :message, :blocking, keyword_init: true)
+
+      def initialize(snapshot, sandbox: Sandbox.new)
+        @snapshot = snapshot
+        @sandbox = sandbox
+        @actions = []
+      end
+
+      def run
+        ensure_rspec_helpers!
+        ensure_factory_bot!
+        ensure_factory!
+        ensure_factory_syntax!
+        self
+      end
+
+      def actions
+        @actions
+      end
+
+      def changed?
+        @actions.any? { |action| %i[created updated].include?(action.kind) }
+      end
+
+      def blocking?
+        @actions.any?(&:blocking)
+      end
+
+      def blocking_message
+        blocked = @actions.select(&:blocking)
+        return if blocked.empty?
+
+        blocked.map(&:message).join("\n\n")
+      end
+
+      private
+
+      def ensure_rspec_helpers!
+        created = Generators::RspecInstall.new(sandbox: @sandbox).write
+        Array(created).each do |path|
+          @actions << Action.new(
+            kind: :created,
+            path: path,
+            message: "RSpec bootstrap so generated specs can load Rails",
+            blocking: false
+          )
+        end
+        ensure_dot_rspec!
+      end
+
+      def ensure_dot_rspec!
+        path = File.join(@sandbox.root, ".rspec")
+        return if File.file?(path)
+
+        File.write(path, "--require spec_helper\n")
+        @actions << Action.new(
+          kind: :created,
+          path: ".rspec",
+          message: "RSpec default options",
+          blocking: false
+        )
+      end
+
+      def ensure_factory_bot!
+        return if factory_bot_bundled?
+
+        added = add_factory_bot_gem!
+        message = if added
+                    "Added factory_bot_rails to the Gemfile. Run `bundle install`, then re-run spec-ai generate #{@snapshot.target.class_name}."
+                  else
+                    <<~MSG.strip
+                      FactoryBot is not in this application's bundle. Spec AI needs a factory to build valid #{@snapshot.target.class_name} records.
+
+                      Add this to the Gemfile:
+
+                        group :development, :test do
+                          gem "factory_bot_rails"
+                        end
+
+                      Then run `bundle install` and re-run spec-ai generate #{@snapshot.target.class_name}.
+                    MSG
+                  end
+        @actions << Action.new(kind: added ? :updated : :needed, path: "Gemfile", message: message, blocking: true)
+      end
+
+      def ensure_factory!
+        return if @snapshot.has_factory?
+
+        path = Generators::FactoryGenerator.new(@snapshot, sandbox: @sandbox).write
+        relative = path.to_s.sub(%r{\A#{Regexp.escape(@sandbox.root)}/?}, "")
+        @actions << Action.new(
+          kind: :created,
+          path: relative,
+          message: "FactoryBot factory for :#{@snapshot.target.factory_name}",
+          blocking: false
+        )
+      end
+
+      def ensure_factory_syntax!
+        helper = "spec/rails_helper.rb"
+        return unless @sandbox.exist?(helper)
+
+        contents = @sandbox.read(helper)
+        return if contents.match?(/FactoryBot::Syntax::Methods/)
+
+        updated = contents.sub(/RSpec\.configure do \|config\|/) do |match|
+          "#{match}\n  config.include FactoryBot::Syntax::Methods"
+        end
+        return if updated == contents
+
+        @sandbox.write(helper, updated)
+        @actions << Action.new(
+          kind: :updated,
+          path: helper,
+          message: "Enabled FactoryBot::Syntax::Methods so specs can call build/create",
+          blocking: false
+        )
+      end
+
+      def factory_bot_bundled?
+        lock = File.join(@sandbox.root, "Gemfile.lock")
+        if File.file?(lock)
+          return true if File.read(lock).match?(/^\s{4}factory_bot(?:_rails)? \(/)
+        end
+
+        gemfile = File.join(@sandbox.root, "Gemfile")
+        File.file?(gemfile) && File.read(gemfile).match?(/\bgem\s+["']factory_bot(?:_rails)?["']/)
+      end
+
+      def add_factory_bot_gem!
+        path = File.join(@sandbox.root, "Gemfile")
+        return false unless File.file?(path)
+
+        contents = File.read(path)
+        return false if contents.match?(/factory_bot_rails/)
+
+        if contents.match?(/group :development, :test do/)
+          contents = contents.sub(/group :development, :test do\n/) do |match|
+            "#{match}  gem \"factory_bot_rails\"\n"
+          end
+        else
+          contents += <<~RUBY
+
+            group :development, :test do
+              gem "factory_bot_rails"
+            end
+          RUBY
+        end
+        File.write(path, contents)
+        true
+      end
+    end
+  end
+end