a2a-test-framework 0.4.0

data/lib/a2a_test_framework/cli.rb

@@ -0,0 +1,190 @@
+# frozen_string_literal: true
+
+require "optparse"
+require_relative "../a2a_test_framework"
+
+module A2ATestFramework
+  class CLI
+    CATEGORIES = {
+      "discovery" => %w[
+        agent_discovery_test
+        agent_card_caching_test
+        agent_card_signing_test
+      ],
+      "send_message" => %w[
+        send_message_test
+      ],
+      "streaming" => %w[
+        send_streaming_message_test
+        streaming_event_delivery_test
+        subscribe_to_task_test
+      ],
+      "tasks" => %w[
+        get_task_test
+        list_tasks_test
+        cancel_task_test
+      ],
+      "push_notifications" => %w[
+        push_notification_delivery_test
+        create_task_push_notification_config_test
+        get_task_push_notification_config_test
+        list_task_push_notification_configs_test
+        delete_task_push_notification_config_test
+      ],
+      "errors" => %w[
+        error_handling_test
+        error_code_mappings_test
+      ],
+      "versioning" => %w[
+        versioning_test
+        versioning_responsibilities_test
+        extension_versioning_test
+      ],
+      "security" => %w[
+        security_considerations_test
+        protocol_security_test
+        authentication_authorization_test
+        in_task_authorization_test
+      ],
+      "protocol" => %w[
+        protocol_data_model_test
+        json_rpc_binding_test
+        http_rest_binding_test
+        json_field_naming_test
+        custom_binding_test
+        protocol_selection_negotiation_test
+        functional_equivalence_test
+        iana_registrations_test
+      ],
+      "semantics" => %w[
+        context_identifier_semantics_test
+        task_identifier_semantics_test
+        history_length_semantics_test
+        field_presence_optionality_test
+        idempotency_test
+        multi_turn_conversation_test
+        messages_and_artifacts_test
+        timestamps_test
+        service_parameters_test
+        capability_validation_test
+      ],
+      "extended" => %w[
+        get_extended_agent_card_test
+      ],
+    }.freeze
+
+    def initialize(argv)
+      @argv = argv
+      @options = {
+        url: ENV.fetch("A2A_BASE_URL", "http://localhost:9292"),
+        binding: "rest",
+        only: [],
+        files: [],
+      }
+    end
+
+    def run
+      parse_options!
+
+      ENV["A2A_BASE_URL"] = @options[:url]
+
+      files = resolve_test_files
+      if files.empty?
+        $stderr.puts "No test files found matching the given criteria."
+        exit 1
+      end
+
+      require "bundler/setup"
+      require "scampi"
+      Scampi.summary_on_exit
+
+      files.each { |f| load f }
+    end
+
+    private
+
+    def parse_options!
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: a2a-test [options] [test_files...]"
+        opts.separator ""
+        opts.separator "Run A2A protocol conformance tests against an A2A server implementation."
+        opts.separator ""
+        opts.separator "Options:"
+
+        opts.on("--url URL", "Base URL of the A2A server (default: $A2A_BASE_URL or http://localhost:9292)") do |url|
+          @options[:url] = url
+        end
+
+        opts.on("--binding BINDING", "Protocol binding to test: rest, grpc (default: rest)") do |binding|
+          unless %w[rest grpc].include?(binding)
+            $stderr.puts "Error: --binding must be 'rest' or 'grpc'"
+            exit 1
+          end
+          @options[:binding] = binding
+        end
+
+        opts.on("--only CATEGORIES", "Comma-separated list of test categories to run") do |cats|
+          @options[:only] = cats.split(",").map(&:strip)
+          invalid = @options[:only] - CATEGORIES.keys
+          unless invalid.empty?
+            $stderr.puts "Error: unknown categories: #{invalid.join(', ')}"
+            $stderr.puts "Available categories: #{CATEGORIES.keys.join(', ')}"
+            exit 1
+          end
+        end
+
+        opts.on("--list-categories", "List available test categories and exit") do
+          puts "Available test categories:"
+          puts ""
+          CATEGORIES.each do |name, files|
+            puts "  #{name}"
+            files.each { |f| puts "    - #{f}" }
+            puts ""
+          end
+          exit 0
+        end
+
+        opts.on("-v", "--version", "Print version and exit") do
+          puts "a2a-test-framework #{A2ATestFramework::VERSION}"
+          exit 0
+        end
+
+        opts.on("-h", "--help", "Show this help message") do
+          puts opts
+          exit 0
+        end
+      end
+
+      parser.parse!(@argv)
+
+      # Remaining arguments are test file paths
+      @options[:files] = @argv.dup
+    end
+
+    def resolve_test_files
+      # If specific file paths were provided, use those
+      unless @options[:files].empty?
+        return @options[:files].map { |f| File.expand_path(f) }.select { |f| File.exist?(f) }
+      end
+
+      binding_dir = File.join(A2ATestFramework::TESTS_DIR, @options[:binding])
+
+      unless Dir.exist?(binding_dir)
+        $stderr.puts "Error: test directory not found: #{binding_dir}"
+        exit 1
+      end
+
+      if @options[:only].empty?
+        # Run all tests for the binding
+        Dir.glob(File.join(binding_dir, "**/*_test.rb")).sort
+      else
+        # Filter by category
+        test_basenames = @options[:only].flat_map { |cat| CATEGORIES[cat] }.compact.uniq
+        test_basenames.filter_map do |basename|
+          path = File.join(binding_dir, "#{basename}.rb")
+          path if File.exist?(path)
+        end
+      end
+    end
+  end
+end

data/lib/a2a_test_framework/sse_client.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: false
+
+require "net/http"
+require "json"
+require "uri"
+require "timeout"
+
+# Simple SSE client for testing streaming endpoints.
+# Reads SSE events from a chunked HTTP response.
+module SSEClient
+  Event = Struct.new(:data, :event, :id, keyword_init: true)
+
+  # POST to a streaming endpoint and collect SSE events.
+  # Returns an array of Event structs.
+  def self.post_stream(path, body, timeout_seconds: 10, headers: {})
+    collect_events(:post, path, body: body, timeout_seconds: timeout_seconds, headers: headers)
+  end
+
+  # GET a streaming endpoint and collect SSE events.
+  # Returns an array of Event structs.
+  def self.get_stream(path, timeout_seconds: 10, headers: {})
+    collect_events(:get, path, timeout_seconds: timeout_seconds, headers: headers)
+  end
+
+  private
+
+  def self.collect_events(method, path, body: nil, timeout_seconds: 10, headers: {})
+    uri = URI("#{BASE_URL}#{path}")
+    events = []
+
+    Timeout.timeout(timeout_seconds) do
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
+        request = case method
+                  when :post
+                    req = Net::HTTP::Post.new(uri.path)
+                    req["Content-Type"] = "application/json"
+                    req.body = JSON.generate(body) if body
+                    req
+                  when :get
+                    Net::HTTP::Get.new(uri.request_uri)
+                  end
+
+        request["Accept"] = "text/event-stream"
+        headers.each { |k, v| request[k] = v }
+
+        http.request(request) do |response|
+          buffer = ""
+          current_event = nil
+          current_data_lines = []
+
+          response.read_body do |chunk|
+            buffer << chunk
+            while (line_end = buffer.index("\n"))
+              line = buffer.slice!(0..line_end).chomp("\r\n").chomp("\n")
+
+              if line.empty?
+                # Empty line = event boundary
+                if current_data_lines.any?
+                  data_str = current_data_lines.join("\n")
+                  parsed = begin
+                    JSON.parse(data_str)
+                  rescue JSON::ParserError
+                    data_str
+                  end
+                  events << Event.new(
+                    data: parsed,
+                    event: current_event
+                  )
+                end
+                current_event = nil
+                current_data_lines = []
+              elsif line.start_with?("data: ")
+                current_data_lines << line.sub(/\Adata: /, "")
+              elsif line.start_with?("data:")
+                current_data_lines << line.sub(/\Adata:/, "")
+              elsif line.start_with?("event: ")
+                current_event = line.sub(/\Aevent: /, "")
+              elsif line.start_with?("id: ")
+                # ignored for now
+              end
+            end
+          end
+
+          # Handle any remaining data in buffer
+          if current_data_lines.any?
+            data_str = current_data_lines.join("\n")
+            parsed = begin
+              JSON.parse(data_str)
+            rescue JSON::ParserError
+              data_str
+            end
+            events << Event.new(data: parsed, event: current_event)
+          end
+        end
+      end
+    end
+
+    events
+  rescue Timeout::Error
+    events
+  rescue EOFError, IOError, Errno::ECONNRESET
+    events
+  end
+end

data/lib/a2a_test_framework/test_helper.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "scampi"
+require "net/http"
+require "json"
+require "uri"
+require "securerandom"
+require "a2a"
+
+BASE_URL = ENV.fetch("A2A_BASE_URL", "http://localhost:9292")
+
+# --- HTTP Helpers -----------------------------------------------------------
+
+def http_post(path, body, headers: {})
+  uri = URI("#{BASE_URL}#{path}")
+  http = Net::HTTP.new(uri.host, uri.port)
+  http.use_ssl = uri.scheme == "https"
+  http.read_timeout = 30
+  request = Net::HTTP::Post.new(uri.path)
+  request["Content-Type"] = "application/json"
+  headers.each { |k, v| request[k] = v }
+  request.body = JSON.generate(body)
+  http.request(request)
+end
+
+def http_get(path, headers: {})
+  uri = URI("#{BASE_URL}#{path}")
+  http = Net::HTTP.new(uri.host, uri.port)
+  http.use_ssl = uri.scheme == "https"
+  http.read_timeout = 30
+  request = Net::HTTP::Get.new(uri.request_uri)
+  headers.each { |k, v| request[k] = v }
+  http.request(request)
+end
+
+def http_delete(path, headers: {})
+  uri = URI("#{BASE_URL}#{path}")
+  http = Net::HTTP.new(uri.host, uri.port)
+  http.use_ssl = uri.scheme == "https"
+  http.read_timeout = 30
+  request = Net::HTTP::Delete.new(uri.path)
+  headers.each { |k, v| request[k] = v }
+  http.request(request)
+end
+
+def parse_json(response)
+  JSON.parse(response.body)
+end
+
+# --- Request Builders (using A2A Schema objects) ----------------------------
+
+def build_send_message_request(text: "Hello, agent", task_id: nil, context_id: nil, configuration: nil, metadata: nil)
+  message = {
+    "messageId" => SecureRandom.uuid,
+    "role" => "ROLE_USER",
+    "parts" => [{ "text" => text }]
+  }
+  message["taskId"] = task_id if task_id
+  message["contextId"] = context_id if context_id
+
+  A2A::Schema["Send Message Request"].new(
+    message: message,
+    configuration: configuration,
+    metadata: metadata
+  ).to_h
+end
+
+def build_get_task_request(id:, history_length: nil)
+  A2A::Schema["Get Task Request"].new(
+    id: id,
+    history_length: history_length
+  ).to_h
+end
+
+def build_list_tasks_request(context_id: nil, status: nil, page_size: nil, page_token: nil, history_length: nil, include_artifacts: nil)
+  A2A::Schema["List Tasks Request"].new(
+    context_id: context_id,
+    status: status,
+    page_size: page_size,
+    page_token: page_token,
+    history_length: history_length,
+    include_artifacts: include_artifacts
+  ).to_h
+end
+
+def build_cancel_task_request(id:, metadata: nil)
+  A2A::Schema["Cancel Task Request"].new(
+    id: id,
+    metadata: metadata
+  ).to_h
+end
+
+def build_subscribe_to_task_request(id:)
+  A2A::Schema["Subscribe To Task Request"].new(id: id).to_h
+end
+
+def build_push_notification_config(task_id:, url: "https://example.com/webhook", token: nil, authentication: nil)
+  A2A::Schema["Task Push Notification Config"].new(
+    task_id: task_id,
+    url: url,
+    token: token,
+    authentication: authentication
+  ).to_h
+end
+
+def build_get_push_notification_config_request(id:, task_id:)
+  A2A::Schema["Get Task Push Notification Config Request"].new(
+    id: id,
+    task_id: task_id
+  ).to_h
+end
+
+def build_list_push_notification_configs_request(task_id:, page_size: nil, page_token: nil)
+  A2A::Schema["List Task Push Notification Configs Request"].new(
+    task_id: task_id,
+    page_size: page_size,
+    page_token: page_token
+  ).to_h
+end
+
+def build_delete_push_notification_config_request(id:, task_id:)
+  A2A::Schema["Delete Task Push Notification Config Request"].new(
+    id: id,
+    task_id: task_id
+  ).to_h
+end
+
+# --- Lifecycle Helpers ------------------------------------------------------
+
+# Sends a message and returns the task hash from the response.
+def create_task!(text: "Hello, agent")
+  body = build_send_message_request(text: text)
+  response = http_post("/message:send", body)
+  raise "Failed to create task: HTTP #{response.code}" unless response.code.to_i == 200
+  data = parse_json(response)
+  raise "No task in response" unless data["task"]
+  data["task"]
+end
+
+# Retrieves a task by ID.
+def get_task!(id)
+  response = http_get("/tasks/#{id}")
+  raise "Failed to get task: HTTP #{response.code}" unless response.code.to_i == 200
+  parse_json(response)
+end

data/lib/a2a_test_framework/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module A2ATestFramework
+  VERSION = "0.4.0"
+end

data/lib/a2a_test_framework.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "a2a_test_framework/version"
+
+module A2ATestFramework
+  # Root directory of the gem (for locating test files, schemas, etc.)
+  ROOT = File.expand_path("..", __dir__)
+
+  # Directory containing the test files
+  TESTS_DIR = File.join(ROOT, "tests")
+
+  # Directory containing endpoint JSON schemas
+  ENDPOINTS_DIR = File.join(ROOT, "endpoints")
+
+  # Path to the protocol schema bundle
+  SCHEMA_PATH = File.join(ROOT, "a2a.json")
+end

data/tests/grpc/cancel_task_test.rb

@@ -0,0 +1,69 @@
+require "a2a_test_framework/test_helper"
+
+# NOTE: All tests commented out -- gRPC binding not supported by reference server
+
+# require "a2a_test_framework/test_helper"
+
+# # NOTE: All tests commented out -- gRPC binding not supported by reference server
+
+# # describe "A2AService/CancelTask" do
+# #   # --- Successful Cancellation ---
+# #
+# #   describe "when a client sends a CancelTask request for a working task" do
+# #     it "should respond with an updated Task object" do
+# #     end
+# #
+# #     it "should reflect the cancellation in the Task status" do
+# #     end
+# #   end
+# #
+# #   describe "when a client sends a CancelTask request for a task in input_required state" do
+# #     it "should respond with an updated Task object" do
+# #     end
+# #
+# #     it "should reflect the cancellation in the Task status" do
+# #     end
+# #   end
+# #
+# #   describe "when a client sends a CancelTask request for a cancelable task" do
+# #     it "should return an updated Task object with cancellation status" do
+# #     end
+# #   end
+# #
+# #   # --- Error Cases ---
+# #
+# #   describe "when a client sends a CancelTask request for a completed task" do
+# #     it "should respond with a TaskNotCancelableError" do
+# #     end
+# #   end
+# #
+# #   describe "when a client sends a CancelTask request for a failed task" do
+# #     it "should respond with a TaskNotCancelableError" do
+# #     end
+# #   end
+# #
+# #   describe "when a client sends a CancelTask request for an already canceled task" do
+# #     it "should respond with a TaskNotCancelableError" do
+# #     end
+# #   end
+# #
+# #   describe "when a client sends a CancelTask request with a non-existent task ID" do
+# #     it "should respond with a TaskNotFoundError" do
+# #     end
+# #   end
+# #
+# #   describe "when a client sends a CancelTask request for a task not accessible to the client" do
+# #     it "should respond with a TaskNotFoundError" do
+# #     end
+# #   end
+# #
+# #   # --- Idempotency ---
+# #
+# #   describe "when a client sends multiple CancelTask requests for the same task" do
+# #     it "should handle repeated cancellation requests idempotently" do
+# #     end
+# #
+# #     it "should respond with TaskNotCancelableError or TaskNotFoundError on subsequent attempts" do
+# #     end
+# #   end
+# # end

data/tests/grpc/create_task_push_notification_config_test.rb

@@ -0,0 +1,79 @@
+require "a2a_test_framework/test_helper"
+
+# NOTE: All tests commented out -- gRPC binding not supported by reference server
+
+# require "a2a_test_framework/test_helper"
+
+# # NOTE: All tests commented out -- gRPC binding not supported by reference server
+
+# # describe "A2AService/CreateTaskPushNotificationConfig" do
+# #   # --- Successful Creation ---
+# #
+# #   describe "when a client creates a push notification config with a valid webhook URL" do
+# #     it "should respond with a PushNotificationConfig object" do
+# #     end
+# #
+# #     it "should contain an assigned ID in the response" do
+# #     end
+# #   end
+# #
+# #   describe "when a push notification config is created for a task" do
+# #     it "should establish a webhook endpoint for task update notifications" do
+# #     end
+# #
+# #     it "should send HTTP POST requests to the configured webhook URL when task updates occur" do
+# #     end
+# #   end
+# #
+# #   describe "when the task status changes after config creation" do
+# #     it "should send an HTTP POST request to the configured webhook URL" do
+# #     end
+# #
+# #     it "should send the payload as a StreamResponse object" do
+# #     end
+# #   end
+# #
+# #   # --- Configuration Persistence ---
+# #
+# #   describe "when a push notification config exists for a non-terminal task" do
+# #     it "should remain active while the task is in a non-terminal state" do
+# #     end
+# #
+# #     it "should remain active until explicitly deleted" do
+# #     end
+# #   end
+# #
+# #   describe "when a task with push notification config reaches completed state" do
+# #     it "should not require the configuration to persist beyond task completion" do
+# #     end
+# #   end
+# #
+# #   # --- Error Cases ---
+# #
+# #   describe "when the server does not support push notifications" do
+# #     it "should respond with a PushNotificationNotSupportedError" do
+# #     end
+# #   end
+# #
+# #   describe "when a client sends a request with a non-existent task ID" do
+# #     it "should respond with a TaskNotFoundError" do
+# #     end
+# #   end
+# #
+# #   describe "when a client sends a request for a task not accessible to the client" do
+# #     it "should respond with a TaskNotFoundError" do
+# #     end
+# #   end
+# #
+# #   # --- Capability Validation ---
+# #
+# #   describe "when the AgentCard declares pushNotifications capability as true" do
+# #     it "should accept and process the request" do
+# #     end
+# #   end
+# #
+# #   describe "when the AgentCard declares pushNotifications capability as false" do
+# #     it "should respond with a PushNotificationNotSupportedError" do
+# #     end
+# #   end
+# # end

data/tests/grpc/delete_task_push_notification_config_test.rb

@@ -0,0 +1,54 @@
+require "a2a_test_framework/test_helper"
+
+# NOTE: All tests commented out -- gRPC binding not supported by reference server
+
+# require "a2a_test_framework/test_helper"
+
+# # NOTE: All tests commented out -- gRPC binding not supported by reference server
+
+# # describe "A2AService/DeleteTaskPushNotificationConfig" do
+# #   # --- Successful Deletion ---
+# #
+# #   describe "when a client deletes an existing push notification config" do
+# #     it "should respond with a confirmation of deletion" do
+# #     end
+# #
+# #     it "should permanently remove the configuration" do
+# #     end
+# #
+# #     it "should cause subsequent GetTaskPushNotificationConfig requests to fail" do
+# #     end
+# #   end
+# #
+# #   describe "when a task changes status after config deletion" do
+# #     it "should not send notifications to the previously configured webhook URL" do
+# #     end
+# #   end
+# #
+# #   # --- Idempotency ---
+# #
+# #   describe "when a client sends multiple delete requests for the same config" do
+# #     it "should have the same effect as a single delete" do
+# #     end
+# #
+# #     it "should not return an error on the second request" do
+# #     end
+# #   end
+# #
+# #   describe "when a client deletes an already-deleted config" do
+# #     it "should not return an error" do
+# #     end
+# #   end
+# #
+# #   # --- Error Cases ---
+# #
+# #   describe "when the server does not support push notifications" do
+# #     it "should respond with a PushNotificationNotSupportedError" do
+# #     end
+# #   end
+# #
+# #   describe "when a client sends a request with a non-existent task ID" do
+# #     it "should respond with a TaskNotFoundError" do
+# #     end
+# #   end
+# # end