agentf 0.3.0

data/lib/agentf/commands/metrics.rb

@@ -0,0 +1,272 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Agentf
+  module Commands
+    class Metrics
+      NAME = "metrics"
+
+      WORKFLOW_METRICS_TAG = "workflow_metric"
+
+      def self.manifest
+        {
+          "name" => NAME,
+          "description" => "Record and summarize workflow quality metrics for provider parity and success tracking.",
+          "commands" => [
+            { "name" => "record_workflow", "type" => "function" },
+            { "name" => "summary", "type" => "function" },
+            { "name" => "provider_parity", "type" => "function" }
+          ]
+        }
+      end
+
+      def initialize(memory: nil, project: nil)
+        @project = project || Agentf.config.project_name
+        @memory = memory || Agentf::Memory::RedisMemory.new(project: @project)
+      end
+
+      def record_workflow(workflow_state)
+        metrics = extract_metrics(workflow_state)
+
+        @memory.store_episode(
+          type: "success",
+          title: metric_title(metrics),
+          description: metric_description(metrics),
+          context: metric_context(metrics),
+          tags: metric_tags(metrics),
+          agent: "WORKFLOW_ENGINE",
+          code_snippet: ""
+        )
+
+        { "status" => "recorded", "metrics" => metrics }
+      rescue StandardError => e
+        { "status" => "error", "error" => e.message }
+      end
+
+      def summary(limit: 100)
+        records = metric_records(limit: limit)
+        return empty_summary if records.empty?
+
+        total = records.length
+        completed = records.count { |m| %w[completed approved].include?(m["status"]) }
+        approved = records.count { |m| m["approved"] == true }
+        failed = records.count { |m| m["status"] == "failed" }
+        security_issue_runs = records.count { |m| m["security_issues"].to_i > 0 }
+
+        {
+          "project" => @project,
+          "total_runs" => total,
+          "completion_rate" => ratio(completed, total),
+          "approval_rate" => ratio(approved, total),
+          "failure_rate" => ratio(failed, total),
+          "security_issue_rate" => ratio(security_issue_runs, total),
+          "avg_agents_executed" => average(records.map { |m| m["agents_executed"].to_i }),
+          "contract_adherence_rate" => ratio(records.count { |m| m["contract_blocked"] != true }, total),
+          "contract_blocked_runs" => records.count { |m| m["contract_blocked"] == true },
+          "policy_violation_rate" => ratio(records.count { |m| m["policy_violation_count"].to_i > 0 }, total),
+          "providers" => provider_breakdown(records),
+          "workflow_types" => workflow_breakdown(records),
+          "top_contract_violations" => top_contract_violations(records)
+        }
+      rescue StandardError => e
+        { "error" => e.message }
+      end
+
+      def provider_parity(limit: 100)
+        records = metric_records(limit: limit)
+        grouped = records.group_by { |m| m["provider"].to_s.upcase }
+
+        opencode = grouped.fetch("OPENCODE", [])
+        copilot = grouped.fetch("COPILOT", [])
+
+        {
+          "project" => @project,
+          "providers_present" => grouped.keys.sort,
+          "opencode_runs" => opencode.length,
+          "copilot_runs" => copilot.length,
+          "completion_rate_gap" => (completion_rate(opencode) - completion_rate(copilot)).round(4),
+          "approval_rate_gap" => metric_gap(opencode, copilot, "approved", expected: true),
+          "security_issue_rate_gap" => security_issue_gap(opencode, copilot),
+          "avg_agents_gap" => average(opencode.map { |m| m["agents_executed"].to_i }) - average(copilot.map { |m| m["agents_executed"].to_i })
+        }
+      rescue StandardError => e
+        { "error" => e.message }
+      end
+
+      private
+
+      def extract_metrics(workflow_state)
+        results = Array(workflow_state["results"])
+        status = infer_status(results)
+
+        {
+          "provider" => workflow_state["provider"],
+          "pack" => workflow_state["pack"],
+          "workflow_type" => workflow_state["workflow_type"],
+          "status" => status,
+          "approved" => reviewer_approved?(results),
+          "agents_executed" => Array(workflow_state["completed_agents"]).length,
+          "error_count" => results.count { |entry| entry.dig("result", "error") },
+          "security_issues" => security_issue_count(results),
+          "contract_blocked" => workflow_state.dig("workflow_contract", "blocked") == true,
+          "contract_violations" => collect_contract_violations(workflow_state),
+          "policy_violation_count" => Array(workflow_state["policy_violations"]).length,
+          "task" => workflow_state["task"].to_s
+        }
+      end
+
+      def collect_contract_violations(workflow_state)
+        Array(workflow_state.dig("workflow_contract", "events"))
+          .flat_map { |event| Array(event["violations"]).map { |v| v["code"] } }
+      end
+
+      def infer_status(results)
+        return "failed" if results.any? { |entry| entry.dig("result", "error") }
+
+        reviewer_approved?(results) ? "approved" : "completed"
+      end
+
+      def reviewer_approved?(results)
+        review = results.find { |entry| entry["agent"] == "REVIEWER" }
+        review&.dig("result", "approved") == true
+      end
+
+      def security_issue_count(results)
+        security_result = results.find { |entry| entry["agent"] == "SECURITY" }
+        issues = security_result&.dig("result", "issues")
+        Array(issues).length
+      end
+
+      def metric_title(metrics)
+        "Workflow metrics: #{metrics['provider']} #{metrics['workflow_type']}"
+      end
+
+      def metric_description(metrics)
+        [
+          "status=#{metrics['status']}",
+          "approved=#{metrics['approved']}",
+          "errors=#{metrics['error_count']}",
+          "security_issues=#{metrics['security_issues']}",
+          "agents=#{metrics['agents_executed']}"
+        ].join(" ")
+      end
+
+      def metric_context(metrics)
+        {
+          "provider" => metrics["provider"],
+          "pack" => metrics["pack"],
+          "workflow_type" => metrics["workflow_type"],
+          "status" => metrics["status"],
+          "approved" => metrics["approved"],
+          "agents_executed" => metrics["agents_executed"],
+          "error_count" => metrics["error_count"],
+          "security_issues" => metrics["security_issues"],
+          "contract_blocked" => metrics["contract_blocked"],
+          "contract_violations" => metrics["contract_violations"],
+          "policy_violation_count" => metrics["policy_violation_count"],
+          "task" => metrics["task"]
+        }.to_json
+      end
+
+      def metric_tags(metrics)
+        [
+          WORKFLOW_METRICS_TAG,
+          "provider:#{metrics['provider'].to_s.downcase}",
+          "workflow:#{metrics['workflow_type']}"
+        ]
+      end
+
+      def top_contract_violations(records)
+        counts = Hash.new(0)
+        records.each do |record|
+          Array(record["contract_violations"]).each { |code| counts[code] += 1 }
+        end
+        counts.sort_by { |(_code, count)| -count }.first(5).to_h
+      end
+
+      def metric_records(limit: 100)
+        memories = @memory.get_recent_memories(limit: limit)
+
+        memories
+          .select { |m| Array(m["tags"]).include?(WORKFLOW_METRICS_TAG) }
+          .filter_map do |m|
+            context = parse_context_json(m["context"])
+            next if context.nil?
+
+            context
+          end
+      end
+
+      def parse_context_json(value)
+        return nil if value.to_s.strip.empty?
+
+        JSON.parse(value)
+      rescue JSON::ParserError
+        nil
+      end
+
+      def empty_summary
+        {
+          "project" => @project,
+          "total_runs" => 0,
+          "completion_rate" => 0.0,
+          "approval_rate" => 0.0,
+          "failure_rate" => 0.0,
+          "security_issue_rate" => 0.0,
+          "avg_agents_executed" => 0.0,
+          "contract_adherence_rate" => 0.0,
+          "contract_blocked_runs" => 0,
+          "policy_violation_rate" => 0.0,
+          "providers" => {},
+          "workflow_types" => {},
+          "top_contract_violations" => {}
+        }
+      end
+
+      def ratio(part, total)
+        return 0.0 if total.to_i <= 0
+
+        (part.to_f / total.to_f).round(4)
+      end
+
+      def average(values)
+        return 0.0 if values.empty?
+
+        (values.sum.to_f / values.length.to_f).round(4)
+      end
+
+      def provider_breakdown(records)
+        records.group_by { |m| m["provider"].to_s.upcase }.transform_values do |items|
+          total = items.length
+          {
+            "runs" => total,
+            "completion_rate" => completion_rate(items),
+            "approval_rate" => ratio(items.count { |m| m["approved"] == true }, total),
+            "security_issue_rate" => ratio(items.count { |m| m["security_issues"].to_i > 0 }, total)
+          }
+        end
+      end
+
+      def completion_rate(records)
+        ratio(records.count { |m| %w[completed approved].include?(m["status"]) }, records.length)
+      end
+
+      def workflow_breakdown(records)
+        records.group_by { |m| m["workflow_type"].to_s }.transform_values(&:length)
+      end
+
+      def metric_gap(a_records, b_records, field, expected:)
+        a_rate = ratio(a_records.count { |m| m[field] == expected }, a_records.length)
+        b_rate = ratio(b_records.count { |m| m[field] == expected }, b_records.length)
+        (a_rate - b_rate).round(4)
+      end
+
+      def security_issue_gap(a_records, b_records)
+        a_rate = ratio(a_records.count { |m| m["security_issues"].to_i > 0 }, a_records.length)
+        b_rate = ratio(b_records.count { |m| m["security_issues"].to_i > 0 }, b_records.length)
+        (a_rate - b_rate).round(4)
+      end
+    end
+  end
+end

data/lib/agentf/commands/security_scanner.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Agentf
+  module Commands
+    # Performs lightweight security scanning for workflows
+    class SecurityScanner
+      NAME = "security"
+
+      def self.manifest
+        {
+          "name" => NAME,
+          "description" => "Scan for secrets, detect prompt injection, and provide recommendations.",
+          "commands" => [
+            { "name" => "scan", "type" => "function" },
+            { "name" => "best_practices", "type" => "function" }
+          ]
+        }
+      end
+
+      DEFAULT_PATTERNS = {
+        "AWS Access Key" => /AKIA[0-9A-Z]{16}/,
+        "Generic API Key" => /(api|secret|token)_?(key|token)?\s*[:=]\s*[A-Za-z0-9_-]{16,}/i,
+        "Private Key" => /-----BEGIN (?:RSA|DSA|EC|OPENSSH) PRIVATE KEY-----/,
+        "Password Assignment" => /password\s*[:=]\s*['\"][^'\"]+['\"]/i
+      }.freeze
+
+      PROMPT_INJECTION_PATTERNS = [
+        /print\s+(?:all\s+)?env/i,
+        /show\s+(?:me\s+)?(?:your|the)\s+environment/i,
+        /exfiltrate/i,
+        /ignore\s+previous\s+instructions/i
+      ].freeze
+
+      BEST_PRACTICES = [
+        "Use secret scanning tools such as Gitleaks or TruffleHog before committing.",
+        "Enable GitHub Secret Scanning Push Protection to block accidental leaks.",
+        "Strip sensitive headers/body content from agent logs before persisting.",
+        "Sandbox agent file-system access and avoid storing raw secrets in episodic memory.",
+        "Harden prompts against injection by refusing to reveal environment variables or credentials."
+      ].freeze
+
+      def initialize(patterns: DEFAULT_PATTERNS)
+        @patterns = patterns
+      end
+
+      def scan(task:, context: {})
+        aggregated_text = ([task] + flatten_context(context)).compact.join("\n")
+
+        issues = detect_secret_patterns(aggregated_text) + detect_prompt_injection(aggregated_text)
+
+        {
+          "issues" => issues,
+          "score" => issues.size,
+          "recommendations" => issues.empty? ? [] : BEST_PRACTICES
+        }
+      end
+
+      def best_practices
+        BEST_PRACTICES
+      end
+
+      private
+
+      def flatten_context(context)
+        case context
+        when Hash
+          context.flat_map { |k, v| [k.to_s, *flatten_context(v)] }
+        when Array
+          context.flat_map { |v| flatten_context(v) }
+        else
+          [context.to_s]
+        end
+      end
+
+      def detect_secret_patterns(text)
+        @patterns.each_with_object([]) do |(label, regex), findings|
+          next unless text.match?(regex)
+
+          findings << {
+            "issue" => "Potential Secret: #{label}",
+            "detail" => "Input matched sensitive pattern #{regex.source}."
+          }
+        end
+      end
+
+      def detect_prompt_injection(text)
+        PROMPT_INJECTION_PATTERNS.each_with_object([]) do |regex, findings|
+          next unless text.match?(regex)
+
+          findings << {
+            "issue" => "Possible Prompt Injection",
+            "detail" => "Detected instruction matching #{regex.source}"
+          }
+        end
+      end
+    end
+  end
+end

data/lib/agentf/commands/tester.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+require "open3"
+require "json"
+require "pathname"
+
+module Agentf
+  module Commands
+    class Tester
+      NAME = "tester"
+
+      def self.manifest
+        {
+          "name" => NAME,
+          "description" => "Generate and run tests for project files.",
+          "commands" => [
+            { "name" => "detect_framework", "type" => "function" },
+            { "name" => "generate_unit_tests", "type" => "function" },
+            { "name" => "run_tests", "type" => "function" }
+          ]
+        }
+      end
+
+      PYTEST_TEMPLATE = <<~RUBY
+        import pytest
+        from %<module>s import %<import_name>s
+
+
+        class Test%<class_name>s:
+            """Tests for %<class_name>s"""
+
+            def test_%<method_name>s_success(self):
+                """Test %<method_name>s with valid input"""
+                # Arrange
+                %<arrange_code>s
+
+                # Act
+                result = %<act_code>s
+
+                # Assert
+                assert result is not None
+                %<assertions>s
+
+            def test_%<method_name>s_invalid_input(self):
+                """Test %<method_name>s with invalid input"""
+                # Arrange
+                %<arrange_code_invalid>s
+
+                # Act & Assert
+                with pytest.raises(%<exception_type>s):
+                    %<act_code_invalid>s
+      RUBY
+
+      RSPEC_TEMPLATE = <<~RUBY
+        require_relative '%<module>s'
+
+        describe %<class_name>s do
+          describe '#%<method_name>s' do
+            it 'works with valid input' do
+              # Arrange
+              %<arrange_code>s
+
+              # Act
+              result = subject.%<method_name>s(input)
+
+              # Assert
+              expect(result).not_to be_nil
+              %<assertions>s
+            end
+
+            it 'raises on invalid input' do
+              # Arrange
+              %<arrange_code_invalid>s
+
+              # Act & Assert
+              expect { subject.%<method_name>s(invalid_input) }.to raise_error(%<exception_type>s)
+            end
+          end
+        end
+      RUBY
+
+      def initialize(base_path: nil)
+        @base_path = base_path || Agentf.config.base_path
+      end
+
+      # Detect testing framework from project
+      def detect_framework(file_path: nil)
+        base = Pathname.new(@base_path)
+
+        # Check for RSpec (Rails/Ruby)
+        if (base / "Gemfile").exist?
+          gemfile = base.join("Gemfile").read
+          return "rspec" if gemfile.include?("rspec")
+        end
+
+        # Check for pytest
+        if (base / "pytest.ini").exist? || (base / "pyproject.toml").exist?
+          pyproject = base.join("pyproject.toml")
+          return "pytest" if pyproject.exist? && pyproject.read.include?("pytest")
+        end
+
+        # Check for Jest/Vitest
+        if (base / "package.json").exist?
+          pkg = JSON.parse((base / "package.json").read)
+          deps = pkg.fetch("dependencies", {}).merge(pkg.fetch("devDependencies", {}))
+          return "vitest" if deps.key?("vitest")
+          return "jest" if deps.key?("jest")
+        end
+
+        # Default based on file extension
+        return "rspec" if file_path&.end_with?(".rb")
+        return "pytest" if file_path&.end_with?(".py")
+        return "jest" if file_path&.end_with?(".js", ".ts", ".jsx", ".tsx")
+
+        "rspec"
+      end
+
+      # Generate unit tests for a source file
+      def generate_unit_tests(source_file, test_class: nil)
+        framework = detect_framework(file_path: source_file)
+        source_path = Pathname.new(@base_path) + source_file
+
+        unless source_path.exist?
+          return Agentf::Tools::TestTemplate.new(
+            test_file: "",
+            test_code: "# Source file not found",
+            framework: framework
+          )
+        end
+
+        # Determine test file path
+        test_file = case framework
+                    when "rspec"
+                      source_file.gsub(%r{^app/(.+)\.rb$}, "spec/\\1_spec.rb")
+                    when "pytest"
+                      source_file.gsub(".py", "_test.py")
+                    when "jest", "vitest"
+                      source_file.sub(/\.(js|ts|jsx|tsx)$/, ".test.\\1")
+                    else
+                      "#{source_file}.test"
+        end
+
+        # Generate test code based on framework
+        test_code = case framework
+                    when "rspec"
+                      generate_rspec(source_file, test_class)
+                    when "pytest"
+                      generate_pytest(source_file, test_class)
+                    else
+                      "# Tests for #{source_file}\n"
+        end
+
+        deps = case framework
+               when "rspec" then ["rspec"]
+               when "pytest" then ["pytest"]
+               else []
+        end
+
+        Agentf::Tools::TestTemplate.new(
+          test_file: test_file,
+          test_code: test_code,
+          framework: framework,
+          dependencies: deps
+        )
+      end
+
+      # Execute test suite
+      def run_tests(test_path: nil, test_file: nil, verbose: true)
+        path = test_path || test_file
+        framework = detect_framework(file_path: path)
+
+        cmd = case framework
+              when "rspec"
+                ["bundle", "exec", "rspec", *(["-f", "documentation"] if verbose), path].compact
+              when "pytest"
+                ["pytest", *(["-v"] if verbose), path].compact
+              when "jest"
+                ["npx", "jest", *(["--verbose"] if verbose), path].compact
+              when "vitest"
+                ["npx", "vitest", *(["--verbose"] if verbose), path].compact
+              end
+
+        stdout, stderr, status = Open3.capture3(*cmd, chdir: @base_path)
+
+        {
+          "passed" => status.success?,
+          "returncode" => status.exitstatus,
+          "stdout" => stdout,
+          "stderr" => stderr,
+          "framework" => framework
+        }
+      rescue StandardError => e
+        { "passed" => false, "error" => e.message, "framework" => framework }
+      end
+
+      private
+
+      def generate_rspec(source_file, test_class)
+        module_name = source_file.gsub(%r{^app/(.+)\.rb$}, "\\1").gsub("/", "::").chomp("_controller")
+        class_name = test_class || module_name.split("::").last
+
+        format(RSPEC_TEMPLATE,
+          module: source_file.gsub(".rb", ""),
+          class_name: class_name,
+          method_name: "my_method",
+          arrange_code: "input = 'valid'",
+          act_code: "described_class.new.method(input)",
+          assertions: "expect(result).to be_truthy",
+          arrange_code_invalid: "invalid_input = nil",
+          act_code_invalid: "described_class.new.method(invalid_input)",
+          exception_type: "StandardError")
+      end
+
+      def generate_pytest(source_file, test_class)
+        module_name = source_file.gsub("/", ".").chomp(".py")
+        class_name = test_class || "TestClass"
+
+        format(PYTEST_TEMPLATE,
+          module: module_name,
+          import_name: class_name,
+          class_name: class_name,
+          method_name: "my_method",
+          arrange_code: "input = something",
+          act_code: "#{class_name}().my_method(input)",
+          assertions: "# assert expected behavior",
+          arrange_code_invalid: "invalid_input = None",
+          act_code_invalid: "#{class_name}().my_method(invalid_input)",
+          exception_type: "ValueError")
+      end
+    end
+  end
+end

data/lib/agentf/commands.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Load command implementations
+require_relative "commands/explorer"
+require_relative "commands/tester"
+require_relative "commands/debugger"
+require_relative "commands/designer"
+require_relative "commands/security_scanner"
+require_relative "commands/memory_reviewer"
+require_relative "commands/metrics"
+require_relative "commands/architecture"
+
+module Agentf
+  module Commands
+    # All commands are loaded above
+  end
+end

data/lib/agentf/context_builder.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Agentf
+  class ContextBuilder
+    def initialize(memory:)
+      @memory = memory
+    end
+
+    def build(agent:, workflow_state:, limit: 8)
+      task_type = workflow_state["workflow_type"]
+      task = workflow_state["task"]
+
+      @memory.get_agent_context(
+        agent: agent,
+        task_type: task_type,
+        query_embedding: simple_embedding(task),
+        limit: limit
+      )
+    rescue StandardError
+      { "agent" => agent, "intent" => [], "memories" => [], "similar_tasks" => [] }
+    end
+
+    private
+
+    def simple_embedding(text)
+      normalized = text.to_s.downcase
+      [
+        normalized.include?("fix") || normalized.include?("bug") ? 1.0 : 0.0,
+        normalized.include?("feature") || normalized.include?("add") ? 1.0 : 0.0,
+        normalized.include?("security") ? 1.0 : 0.0,
+        normalized.length.to_f / 100.0
+      ]
+    end
+  end
+end