reqcord 0.1.0 → 0.1.1

data/lib/reqcord/generator.rb

@@ -0,0 +1,364 @@
+# frozen_string_literal: true
+
+require "open3"
+require "shellwords"
+require "securerandom"
+
+module Reqcord
+  # Runs the test suite in a subprocess with capture enabled, then turns the
+  # captured exchanges into documentation. Route collection happens here, in a
+  # process that already has the application booted.
+  class Generator
+    def self.call(
+      resources: [],
+      version: nil
+    )
+      new(
+        resources: resources,
+        version: version
+      ).call
+    end
+
+    def initialize(resources:, version:, configuration: Reqcord.configuration)
+      @resources = Array(resources).map(&:to_s)
+      @version = version&.to_s
+      @configuration = configuration
+    end
+
+    def call
+      validate!
+
+      routes = collect_routes
+
+      capture_file = build_capture_file
+
+      run_tests(capture_file)
+
+      exchanges =
+        read_exchanges(capture_file)
+
+      dataset =
+        build_dataset(
+          routes,
+          exchanges
+        )
+
+      report(exchanges, dataset)
+
+      write_outputs(dataset)
+
+      dataset
+    ensure
+      FileUtils.rm_f(capture_file) if capture_file
+    end
+
+    # Paths that were captured but belong to no documented route. Kept so the
+    # run can say why a request did not turn into documentation.
+    attr_reader :unmatched_paths
+
+    # Routes seen in the table but not documentable, by reason (redirect,
+    # mount). Set by collect_routes; exposed so a run can be reconciled.
+    attr_accessor :skipped_routes
+
+    # The command that runs the suite. An explicit `test.command` is used as
+    # written (globs expanded, since no shell is involved); otherwise a runner
+    # is built from `test.paths`: `bin/rails test` when the application has
+    # one, a plain Ruby runner when it does not, `rspec` for request specs.
+    def test_argv
+      explicit = configuration.test_command
+      return expand_globs(Shellwords.split(explicit)) if explicit
+
+      paths = configuration.test_paths
+      return %w[bin/rails test] if paths.empty?
+
+      case configuration.test_framework.to_s
+      when "rspec"
+        ["rspec", *paths]
+      else
+        if configuration.root.join("bin", "rails").exist?
+          ["bin/rails", "test", *paths]
+        else
+          ["ruby", "-Itest", "-e", "ARGV.each { |file| require File.expand_path(file) }", *test_files(paths)]
+        end
+      end
+    end
+
+    # Builds the dataset from exchanges that were captured earlier, without
+    # running the suite again.
+    def build_dataset(routes, exchanges)
+      endpoints = {}
+      @unmatched_paths = []
+
+      # A route answering any verb is documented once per verb a test used.
+      routes.each do |route|
+        endpoints[[route, route.method]] = route.endpoint unless route.any_verb?
+      end
+
+      exchanges.each do |raw_exchange|
+        sanitized =
+          Sanitizers::Sanitizer.call(
+            raw_exchange,
+            configuration: configuration
+          )
+
+        route = find_route(routes, sanitized)
+        request = sanitized.fetch("request")
+
+        unless route
+          @unmatched_paths << "#{request['method']} #{request['path']}"
+          next
+        end
+
+        verb = route.any_verb? ? request.fetch("method").to_s.upcase : route.method
+        endpoint = endpoints[[route, verb]] ||= route.endpoint(method: verb)
+
+        attach_exchange(endpoint, sanitized)
+      end
+
+      routes.select(&:any_verb?).each do |route|
+        next if endpoints.keys.any? { |(seen, _verb)| seen == route }
+
+        endpoints[[route, RouteCollector::ANY]] = route.endpoint
+      end
+
+      Dataset.new(
+        endpoints: Dataset.fold_method_twins(endpoints.values)
+      )
+    end
+
+    private
+
+    attr_reader :resources,
+                :version,
+                :configuration
+
+    # Silence here is the worst outcome: an empty page set looks the same
+    # whether the API has no tests or the capture never ran.
+    def report(exchanges, dataset)
+      if exchanges.empty?
+        Reqcord.warn("no request was captured")
+        Reqcord.warn("  is reqcord in the :test group of your Gemfile, and does `test.command` run integration tests?")
+        return
+      end
+
+      covered = dataset.curl_ready_endpoints.size
+      uncovered = dataset.uncovered_endpoints.size
+      matched = exchanges.size - unmatched_paths.size
+      skipped = skipped_routes || {}
+      skipped_total = skipped.values.sum
+
+      Reqcord.log("captured #{exchanges.size} request(s), #{matched} matched a documented route")
+      Reqcord.log("captured a successful 2xx request for #{covered} of #{dataset.endpoints.size} endpoint(s)")
+
+      # Every route lands in exactly one bucket; the sum is the proof.
+      Reqcord.log(
+        "routes: #{dataset.endpoints.size + skipped_total} = " \
+        "#{covered} documented + #{uncovered} uncovered + #{skipped_total} skipped"
+      )
+
+      if skipped_total.positive?
+        reasons = skipped.map { |reason, count| "#{count} #{reason}" }.join(", ")
+        Reqcord.log("skipped #{skipped_total} route(s) that cannot be documented: #{reasons}")
+      end
+
+      return if unmatched_paths.empty?
+
+      shown = unmatched_paths.uniq.first(5)
+
+      Reqcord.log("#{unmatched_paths.uniq.size} path(s) matched no documented route, for example:")
+      shown.each { |path| Reqcord.log("  #{path}") }
+    end
+
+    FRAMEWORKS = %w[minitest rspec].freeze
+
+    def validate!
+      unless FRAMEWORKS.include?(configuration.test_framework.to_s)
+        raise ConfigurationError,
+              "unsupported test framework: #{configuration.test_framework.inspect}, " \
+              "expected one of #{FRAMEWORKS.join(', ')}"
+      end
+
+      configuration.exporters.each { |name| Exporters.fetch(name) }
+
+      nil
+    end
+
+    def collect_routes
+      collector = RouteCollector.new(
+        resources: resources,
+        version: version,
+        prefix: configuration.route_prefix
+      )
+
+      routes = collector.call
+      @skipped_routes = collector.skipped
+
+      routes
+    end
+
+    # A directory means every *_test.rb (or *_spec.rb) beneath it.
+    def test_files(paths)
+      pattern = configuration.test_framework.to_s == "rspec" ? "*_spec.rb" : "*_test.rb"
+
+      paths.flat_map do |path|
+        if configuration.root.join(path).directory?
+          Dir.glob(File.join(path, "**", pattern), base: configuration.root.to_s).sort
+        else
+          expand_globs([path])
+        end
+      end
+    end
+
+    def expand_globs(args)
+      args.flat_map do |arg|
+        next [arg] unless arg.match?(/[*?\[{]/)
+
+        matches = Dir.glob(arg, base: configuration.root.to_s).sort
+        matches.empty? ? [arg] : matches
+      end
+    end
+
+    def build_capture_file
+      Reqcord.root.join(
+        "tmp",
+        "reqcord",
+        "#{SecureRandom.hex(12)}.ndjson"
+      ).to_s
+    end
+
+    def run_tests(capture_file)
+      FileUtils.mkdir_p(
+        File.dirname(capture_file)
+      )
+
+      env = {
+        "REQCORD_CAPTURE" => "1",
+        "REQCORD_CAPTURE_FILE" => capture_file
+      }
+
+      command = test_argv
+
+      Reqcord.log("running #{Shellwords.join(command)}")
+
+      # Streamed rather than captured: a long suite should print as it runs.
+      status =
+        Open3.popen2e(
+          env,
+          *command,
+          chdir: Reqcord.root.to_s
+        ) do |stdin, output, wait_thread|
+          stdin.close
+
+          output.each_line { |line| $stdout.print(line) }
+
+          wait_thread.value
+        end
+
+      return if status.success?
+
+      raise GenerationError,
+            "Test suite failed while generating Reqcord documentation"
+    end
+
+    def read_exchanges(capture_file)
+      return [] unless File.exist?(capture_file)
+
+      File.readlines(capture_file)
+          .filter_map do |line|
+            line = line.strip
+
+            next if line.empty?
+
+            begin
+              JSON.parse(line)
+            rescue JSON::ParserError
+              Reqcord.warn("skipping malformed capture line")
+              nil
+            end
+          end
+    end
+
+    def find_route(routes, exchange)
+      request = exchange.fetch("request")
+      method = request.fetch("method")
+      path = request.fetch("path")
+
+      routes.find { |route| route.matches?(method, path) }
+    rescue StandardError => e
+      raise GenerationError, "could not match #{method} #{path} against the route table: #{e.message}"
+    end
+
+    def attach_exchange(endpoint, exchange)
+      request_data = exchange.fetch("request")
+      response_data = exchange.fetch("response")
+      source = normalize_source(exchange["source"])
+      status = response_data.fetch("status")
+
+      request =
+        RequestExample.new(
+          name: example_name(source),
+          method: request_data.fetch("method"),
+          # The concrete path, not the route pattern: the cURL must be runnable.
+          path: request_data.fetch("path"),
+          path_params: request_data["path_params"] || {},
+          query_params: request_data["query_params"] || {},
+          headers: request_data["headers"] || {},
+          body: request_data["body"],
+          content_type: request_data["content_type"],
+          response_status: status,
+          source: source
+        )
+
+      response =
+        ResponseExample.new(
+          name: example_name(source),
+          status: status,
+          headers: response_data["headers"] || {},
+          body: response_data["body"],
+          content_type: response_data["content_type"],
+          source: source
+        )
+
+      endpoint.add_exchange(
+        request: request,
+        response: response
+      )
+    end
+
+    # "test_creates_customer" -> "Creates Customer"
+    def example_name(source)
+      test = source["test"].to_s.sub(/\Atest_/, "")
+
+      test.empty? ? nil : Support.titleize(test)
+    end
+
+    def normalize_source(source)
+      source = (source || {}).transform_keys(&:to_s)
+      file = source["file"].to_s
+      prefix = "#{Reqcord.root}/"
+
+      source["file"] = file.delete_prefix(prefix) unless file.empty?
+      source.compact
+    end
+
+    def write_outputs(dataset)
+      output = configuration.output_directory
+
+      FileUtils.mkdir_p(output)
+
+      written = [dataset.write(output.join("dataset.json"))]
+
+      configuration.exporters.each do |name|
+        written.concat(
+          Exporters.fetch(name).call(
+            dataset: dataset,
+            output_dir: output,
+            configuration: configuration
+          )
+        )
+      end
+
+      written
+    end
+  end
+end

data/lib/reqcord/railtie.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Reqcord
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load File.expand_path(
+        "../tasks/reqcord.rake",
+        __dir__
+      )
+    end
+
+    initializer "reqcord.capture" do
+      next unless Reqcord::Capture.enabled?
+
+      require "action_dispatch/testing/integration"
+      require_relative "capture/integration_patch"
+
+      unless ActionDispatch::Integration::Session <
+             Reqcord::Capture::IntegrationPatch
+
+        ActionDispatch::Integration::Session.prepend(
+          Reqcord::Capture::IntegrationPatch
+        )
+      end
+
+      # Capture itself is framework agnostic; only naming the example needs to
+      # know which test framework is running.
+      case Reqcord.configuration.test_framework.to_s
+      when "minitest"
+        require "minitest/test"
+        require_relative "capture/minitest_context"
+
+        unless Minitest::Test < Reqcord::Capture::MinitestContext
+          Minitest::Test.prepend(
+            Reqcord::Capture::MinitestContext
+          )
+        end
+
+      when "rspec"
+        require "rspec/core"
+        require_relative "capture/rspec_context"
+
+        Reqcord::Capture::RSpecContext.install!
+
+      else
+        raise ConfigurationError,
+              "unsupported test framework: #{Reqcord.configuration.test_framework.inspect}"
+      end
+    end
+  end
+end

data/lib/reqcord/renderers/curl.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Reqcord
+  module Renderers
+    class Curl
+      def self.call(request, base_url:)
+        new(
+          request,
+          base_url: base_url
+        ).call
+      end
+
+      def initialize(request, base_url:)
+        @request = request
+        @base_url = base_url.to_s.sub(%r{/$}, "")
+      end
+
+      def call
+        parts = [
+          "curl --request #{request.method}",
+          %(--url "#{url}")
+        ]
+
+        request.headers.each do |key, value|
+          parts << %(--header "#{key}: #{escape_header(value)}")
+        end
+
+        parts << data_argument if request.body?
+
+        parts.join(" \\\n  ")
+      end
+
+      private
+
+      attr_reader :request, :base_url
+
+      def url
+        "#{base_url}#{Payload.path_with_query(request)}"
+      end
+
+      def data_argument
+        "--data '#{shell_single_quote(Payload.raw_body(request))}'"
+      end
+
+      def shell_single_quote(value)
+        value.to_s.gsub("'", %q('"'"'))
+      end
+
+      def escape_header(value)
+        value.to_s
+             .gsub("\\") { "\\\\" }
+             .gsub('"') { '\\"' }
+      end
+    end
+  end
+end

data/lib/reqcord/renderers/payload.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Reqcord
+  module Renderers
+    # The wire-level decisions every output format has to agree on: whether a
+    # request is JSON, how a nested query flattens, what the body looks like
+    # as text. cURL, Postman and any later exporter read these, never their
+    # own copy.
+    module Payload
+      module_function
+
+      def json?(request)
+        request.content_type.to_s.include?("json") ||
+          request.headers.any? do |key, value|
+            key.to_s.casecmp?("Content-Type") && value.to_s.include?("json")
+          end
+      end
+
+      # Rails bracket notation: { filter: { status: "a" }, ids: [1, 2] } becomes
+      # [["filter[status]", "a"], ["ids[]", 1], ["ids[]", 2]].
+      def flatten_query(hash, prefix = nil)
+        hash.flat_map do |key, value|
+          current = prefix ? "#{prefix}[#{key}]" : key.to_s
+
+          case value
+          when Hash
+            flatten_query(value, current)
+          when Array
+            value.flat_map do |item|
+              if item.is_a?(Hash)
+                flatten_query(item, "#{current}[]")
+              else
+                [["#{current}[]", item]]
+              end
+            end
+          else
+            [[current, value]]
+          end
+        end
+      end
+
+      def query_pairs(request)
+        flatten_query(request.query_params)
+      end
+
+      def form_pairs(request)
+        request.body.is_a?(Hash) ? flatten_query(request.body) : []
+      end
+
+      def path_with_query(request)
+        return request.path if request.query_params.empty?
+
+        "#{request.path}?#{URI.encode_www_form(query_pairs(request))}"
+      end
+
+      # The body as it goes on the wire: pretty JSON, a form string, or the
+      # raw text the test sent.
+      def raw_body(request)
+        if json?(request)
+          JSON.pretty_generate(request.body)
+        elsif request.body.is_a?(Hash)
+          URI.encode_www_form(form_pairs(request))
+        else
+          request.body.to_s
+        end
+      end
+    end
+  end
+end

data/lib/reqcord/request_example.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Reqcord
+  class RequestExample
+    ATTRIBUTES = %i[
+      name
+      method
+      path
+      path_params
+      query_params
+      headers
+      body
+      content_type
+      response_status
+      source
+    ].freeze
+
+    attr_accessor(*ATTRIBUTES)
+
+    def initialize(
+      method:,
+      path:,
+      name: nil,
+      path_params: {},
+      query_params: {},
+      headers: {},
+      body: nil,
+      content_type: nil,
+      response_status: nil,
+      source: {}
+    )
+      @name = name
+      @method = method.to_s.upcase
+      @path = path
+      @path_params = path_params || {}
+      @query_params = query_params || {}
+      @headers = headers || {}
+      @body = body
+      @content_type = content_type
+      @response_status = response_status&.to_i
+      @source = source || {}
+    end
+
+    # `method` is the HTTP verb here; this alias keeps call sites that mean the
+    # verb from reading like reflection.
+    def http_method
+      method
+    end
+
+    # A payload of empty containers ({"experience" => {}}) carries nothing to
+    # document, and sending it as --data would only mislead.
+    def body?
+      meaningful?(body)
+    end
+
+    # The request the endpoint page leads with should be one that worked.
+    def successful?
+      response_status.nil? ? false : (200..299).cover?(response_status)
+    end
+
+    # Identical requests captured by several tests are stored once — but the
+    # same request that produced a different status is a different example.
+    # Sanitization can make a right and a wrong password look identical; the
+    # 200 must not be dropped because the 401 was captured first.
+    def signature
+      JSON.generate([method, path, headers, query_params, body, response_status])
+    end
+
+    def to_h
+      {
+        name: name,
+        method: method,
+        path: path,
+        path_params: path_params,
+        query_params: query_params,
+        headers: headers,
+        body: body,
+        content_type: content_type,
+        response_status: response_status,
+        source: source
+      }
+    end
+
+    # Tolerates keys this version does not know, so a dataset written by a
+    # newer Reqcord still loads.
+    def self.from_h(hash)
+      hash = hash.transform_keys(&:to_sym).slice(*ATTRIBUTES)
+
+      new(**hash)
+    end
+
+    private
+
+    def meaningful?(value)
+      case value
+      when nil then false
+      when Hash then value.any? { |_key, nested| meaningful?(nested) }
+      when Array then value.any? { |item| meaningful?(item) }
+      when String then !value.empty?
+      else true
+      end
+    end
+  end
+end

data/lib/reqcord/response_example.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Reqcord
+  class ResponseExample
+    ATTRIBUTES = %i[
+      name
+      status
+      headers
+      body
+      content_type
+      source
+    ].freeze
+
+    attr_accessor(*ATTRIBUTES)
+
+    def initialize(
+      status:,
+      name: nil,
+      headers: {},
+      body: nil,
+      content_type: nil,
+      source: {}
+    )
+      @name = name
+      @status = status.to_i
+      @headers = headers || {}
+      @body = body
+      @content_type = content_type
+      @source = source || {}
+    end
+
+    # "201 Created"
+    def status_text
+      Rack::Utils::HTTP_STATUS_CODES[status] || "Unknown"
+    end
+
+    def title
+      "#{status} #{status_text}"
+    end
+
+    def body?
+      !(body.nil? || (body.respond_to?(:empty?) && body.empty?))
+    end
+
+    def empty_body?
+      !body?
+    end
+
+    def signature
+      JSON.generate([status, body])
+    end
+
+    def to_h
+      {
+        name: name,
+        status: status,
+        headers: headers,
+        body: body,
+        content_type: content_type,
+        source: source
+      }
+    end
+
+    # Tolerates keys this version does not know, so a dataset written by a
+    # newer Reqcord still loads.
+    def self.from_h(hash)
+      hash = hash.transform_keys(&:to_sym).slice(*ATTRIBUTES)
+
+      new(**hash)
+    end
+  end
+end