reqcord 0.1.0 → 0.1.2

data/lib/reqcord/exporters/postman.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+module Reqcord
+  module Exporters
+    # Writes a Postman Collection v2.1 (`postman/collection.json`). Folders
+    # follow the controller namespaces, each documented endpoint is one request
+    # built from the successful captured example, and every captured status
+    # is saved as a response example. Hoppscotch imports the same file.
+    class Postman
+      SCHEMA_URL = "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
+      PLACEHOLDER = /\{\{([A-Za-z0-9_]+)\}\}/
+      BEARER = /\ABearer\s+\{\{([A-Za-z0-9_]+)\}\}\z/i
+
+      def self.call(dataset:, output_dir:, configuration:)
+        new(
+          dataset: dataset,
+          output_dir: output_dir,
+          configuration: configuration
+        ).call
+      end
+
+      def initialize(dataset:, output_dir:, configuration:)
+        @dataset = dataset
+        @output_dir = Pathname(output_dir).join("postman")
+        @configuration = configuration
+      end
+
+      def call
+        FileUtils.mkdir_p(output_dir)
+
+        path = output_dir.join("collection.json")
+        File.write(path, "#{JSON.pretty_generate(collection)}\n")
+
+        [path.to_s]
+      end
+
+      def collection
+        {
+          info: {
+            name: "#{File.basename(configuration.root.to_s)} API",
+            description: "Generated by Reqcord from the application's integration tests.",
+            schema: SCHEMA_URL
+          },
+          item: folders,
+          variable: variables,
+          auth: auth
+        }.compact
+      end
+
+      private
+
+      attr_reader :dataset, :output_dir, :configuration
+
+      # api/v2/customers -> Api > V2 > Customers, requests in the leaf folder.
+      def folders
+        tree = { items: [], children: {} }
+
+        dataset.resources.each do |resource|
+          requests = resource.endpoints.select(&:curl_ready?).map { |endpoint| request_item(endpoint) }
+          next if requests.empty?
+
+          leaf = resource.segments.reduce(tree) do |node, segment|
+            node[:children][segment] ||= { items: [], children: {} }
+          end
+
+          leaf[:items].concat(requests)
+        end
+
+        render_folders(tree)
+      end
+
+      def render_folders(node)
+        node[:children].map do |segment, child|
+          { name: Support.titleize(segment), item: render_folders(child) + child[:items] }
+        end
+      end
+
+      def request_item(endpoint)
+        {
+          name: endpoint.name,
+          request: request_object(endpoint.primary_request_example, endpoint),
+          response: endpoint.responses.map { |response| response_object(response, endpoint) }
+        }
+      end
+
+      def request_object(example, endpoint)
+        {
+          method: example.method,
+          description: description(endpoint),
+          header: headers(example),
+          url: url_object(example),
+          body: body_object(example),
+          auth: request_auth(example)
+        }.compact
+      end
+
+      def description(endpoint)
+        lines = ["#{endpoint.method} #{endpoint.path}", "#{endpoint.controller}##{endpoint.action}"]
+        lines << "Also answers #{endpoint.also_methods.join(', ')}." unless endpoint.also_methods.empty?
+        lines << "Route: #{endpoint.route_name}" if endpoint.route_name
+
+        lines.join("\n")
+      end
+
+      # The bearer credential lives on the collection; a request that carried
+      # it inherits, one that did not says so, so public endpoints replay
+      # exactly as their tests did.
+      def headers(example)
+        example.headers.filter_map do |key, value|
+          next if auth && key.to_s.casecmp?("Authorization")
+
+          { key: key.to_s, value: value.to_s }
+        end
+      end
+
+      def request_auth(example)
+        return nil unless auth
+        return nil if example.headers.any? { |key, _| key.to_s.casecmp?("Authorization") }
+
+        { type: "noauth" }
+      end
+
+      def url_object(example)
+        url = {
+          raw: "{{base_url}}#{Renderers::Payload.path_with_query(example)}",
+          host: ["{{base_url}}"],
+          path: example.path.split("/").reject(&:empty?),
+          query: Renderers::Payload.query_pairs(example).map { |key, value| { key: key, value: value.to_s } }
+        }
+
+        url.delete(:query) if url[:query].empty?
+        url
+      end
+
+      def body_object(example)
+        return nil unless example.body?
+
+        if Renderers::Payload.json?(example)
+          { mode: "raw", raw: Renderers::Payload.raw_body(example), options: { raw: { language: "json" } } }
+        elsif example.body.is_a?(Hash)
+          pairs = Renderers::Payload.form_pairs(example).map { |key, value| { key: key, value: value.to_s, type: "text" } }
+
+          { mode: "urlencoded", urlencoded: pairs }
+        else
+          { mode: "raw", raw: example.body.to_s }
+        end
+      end
+
+      def response_object(response, endpoint)
+        example = response.example
+        request = endpoint.request_examples.find { |candidate| candidate.response_status == response.status } ||
+                  endpoint.primary_request_example
+
+        {
+          name: example.title,
+          originalRequest: request_object(request, endpoint),
+          status: example.status_text,
+          code: response.status,
+          _postman_previewlanguage: example.content_type.to_s.include?("json") ? "json" : "text",
+          header: example.headers.map { |key, value| { key: key.to_s, value: value.to_s } },
+          body: response_body(example)
+        }
+      end
+
+      def response_body(example)
+        return "" unless example.body?
+        return example.body if example.body.is_a?(String)
+
+        JSON.pretty_generate(example.body)
+      end
+
+      # `base_url`, plus every `{{name}}` the sanitizer wrote into the
+      # documented examples, so the collection declares what it expects.
+      def variables
+        names = documented_examples.flat_map do |example|
+          [example.headers, example.body, example.query_params].flat_map do |value|
+            JSON.generate(value).scan(PLACEHOLDER).flatten
+          end
+        end
+
+        names |= (configuration.sanitized_headers.values + configuration.sanitized_body_keys.values)
+                 .flat_map { |value| value.to_s.scan(PLACEHOLDER).flatten }
+
+        [{ key: "base_url", value: configuration.base_url, type: "string" }] +
+          (names - ["base_url"]).uniq.sort.map { |name| { key: name, value: "", type: "string" } }
+      end
+
+      def auth
+        return @auth if defined?(@auth)
+
+        placeholder = documented_examples.filter_map do |example|
+          value = example.headers.find { |key, _| key.to_s.casecmp?("Authorization") }&.last
+          value.to_s[BEARER, 1]
+        end.first
+
+        @auth = placeholder && { type: "bearer", bearer: [{ key: "token", value: "{{#{placeholder}}}", type: "string" }] }
+      end
+
+      def documented_examples
+        @documented_examples ||= dataset.curl_ready_endpoints.flat_map(&:request_examples)
+      end
+    end
+
+    register("postman", Postman)
+  end
+end

data/lib/reqcord/exporters.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Reqcord
+  # Exporters register themselves here, so adding one means adding a file
+  # rather than editing the pipeline.
+  module Exporters
+    class << self
+      def register(name, exporter)
+        registry[name.to_s] = exporter
+      end
+
+      def fetch(name)
+        registry.fetch(name.to_s) do
+          raise ConfigurationError,
+                "unknown exporter #{name.inspect}, expected one of #{names.join(', ')}"
+        end
+      end
+
+      def registered?(name)
+        registry.key?(name.to_s)
+      end
+
+      def names
+        registry.keys.sort
+      end
+
+      def registry
+        @registry ||= {}
+      end
+    end
+  end
+end

data/lib/reqcord/generator.rb

@@ -0,0 +1,364 @@
+# frozen_string_literal: true
+
+require "open3"
+require "shellwords"
+require "securerandom"
+
+module Reqcord
+  # Runs the test suite in a subprocess with capture enabled, then turns the
+  # captured exchanges into documentation. Route collection happens here, in a
+  # process that already has the application booted.
+  class Generator
+    def self.call(
+      resources: [],
+      version: nil
+    )
+      new(
+        resources: resources,
+        version: version
+      ).call
+    end
+
+    def initialize(resources:, version:, configuration: Reqcord.configuration)
+      @resources = Array(resources).map(&:to_s)
+      @version = version&.to_s
+      @configuration = configuration
+    end
+
+    def call
+      validate!
+
+      routes = collect_routes
+
+      capture_file = build_capture_file
+
+      run_tests(capture_file)
+
+      exchanges =
+        read_exchanges(capture_file)
+
+      dataset =
+        build_dataset(
+          routes,
+          exchanges
+        )
+
+      report(exchanges, dataset)
+
+      write_outputs(dataset)
+
+      dataset
+    ensure
+      FileUtils.rm_f(capture_file) if capture_file
+    end
+
+    # Paths that were captured but belong to no documented route. Kept so the
+    # run can say why a request did not turn into documentation.
+    attr_reader :unmatched_paths
+
+    # Routes seen in the table but not documentable, by reason (redirect,
+    # mount). Set by collect_routes; exposed so a run can be reconciled.
+    attr_accessor :skipped_routes
+
+    # The command that runs the suite. An explicit `test.command` is used as
+    # written (globs expanded, since no shell is involved); otherwise a runner
+    # is built from `test.paths`: `bin/rails test` when the application has
+    # one, a plain Ruby runner when it does not, `rspec` for request specs.
+    def test_argv
+      explicit = configuration.test_command
+      return expand_globs(Shellwords.split(explicit)) if explicit
+
+      paths = configuration.test_paths
+      return %w[bin/rails test] if paths.empty?
+
+      case configuration.test_framework.to_s
+      when "rspec"
+        ["rspec", *paths]
+      else
+        if configuration.root.join("bin", "rails").exist?
+          ["bin/rails", "test", *paths]
+        else
+          ["ruby", "-Itest", "-e", "ARGV.each { |file| require File.expand_path(file) }", *test_files(paths)]
+        end
+      end
+    end
+
+    # Builds the dataset from exchanges that were captured earlier, without
+    # running the suite again.
+    def build_dataset(routes, exchanges)
+      endpoints = {}
+      @unmatched_paths = []
+
+      # A route answering any verb is documented once per verb a test used.
+      routes.each do |route|
+        endpoints[[route, route.method]] = route.endpoint unless route.any_verb?
+      end
+
+      exchanges.each do |raw_exchange|
+        sanitized =
+          Sanitizers::Sanitizer.call(
+            raw_exchange,
+            configuration: configuration
+          )
+
+        route = find_route(routes, sanitized)
+        request = sanitized.fetch("request")
+
+        unless route
+          @unmatched_paths << "#{request['method']} #{request['path']}"
+          next
+        end
+
+        verb = route.any_verb? ? request.fetch("method").to_s.upcase : route.method
+        endpoint = endpoints[[route, verb]] ||= route.endpoint(method: verb)
+
+        attach_exchange(endpoint, sanitized)
+      end
+
+      routes.select(&:any_verb?).each do |route|
+        next if endpoints.keys.any? { |(seen, _verb)| seen == route }
+
+        endpoints[[route, RouteCollector::ANY]] = route.endpoint
+      end
+
+      Dataset.new(
+        endpoints: Dataset.fold_method_twins(endpoints.values)
+      )
+    end
+
+    private
+
+    attr_reader :resources,
+                :version,
+                :configuration
+
+    # Silence here is the worst outcome: an empty page set looks the same
+    # whether the API has no tests or the capture never ran.
+    def report(exchanges, dataset)
+      if exchanges.empty?
+        Reqcord.warn("no request was captured")
+        Reqcord.warn("  is reqcord in the :test group of your Gemfile, and does `test.command` run integration tests?")
+        return
+      end
+
+      covered = dataset.curl_ready_endpoints.size
+      uncovered = dataset.uncovered_endpoints.size
+      matched = exchanges.size - unmatched_paths.size
+      skipped = skipped_routes || {}
+      skipped_total = skipped.values.sum
+
+      Reqcord.log("captured #{exchanges.size} request(s), #{matched} matched a documented route")
+      Reqcord.log("captured a successful 2xx request for #{covered} of #{dataset.endpoints.size} endpoint(s)")
+
+      # Every route lands in exactly one bucket; the sum is the proof.
+      Reqcord.log(
+        "routes: #{dataset.endpoints.size + skipped_total} = " \
+        "#{covered} documented + #{uncovered} uncovered + #{skipped_total} skipped"
+      )
+
+      if skipped_total.positive?
+        reasons = skipped.map { |reason, count| "#{count} #{reason}" }.join(", ")
+        Reqcord.log("skipped #{skipped_total} route(s) that cannot be documented: #{reasons}")
+      end
+
+      return if unmatched_paths.empty?
+
+      shown = unmatched_paths.uniq.first(5)
+
+      Reqcord.log("#{unmatched_paths.uniq.size} path(s) matched no documented route, for example:")
+      shown.each { |path| Reqcord.log("  #{path}") }
+    end
+
+    FRAMEWORKS = %w[minitest rspec].freeze
+
+    def validate!
+      unless FRAMEWORKS.include?(configuration.test_framework.to_s)
+        raise ConfigurationError,
+              "unsupported test framework: #{configuration.test_framework.inspect}, " \
+              "expected one of #{FRAMEWORKS.join(', ')}"
+      end
+
+      configuration.exporters.each { |name| Exporters.fetch(name) }
+
+      nil
+    end
+
+    def collect_routes
+      collector = RouteCollector.new(
+        resources: resources,
+        version: version,
+        prefix: configuration.route_prefix
+      )
+
+      routes = collector.call
+      @skipped_routes = collector.skipped
+
+      routes
+    end
+
+    # A directory means every *_test.rb (or *_spec.rb) beneath it.
+    def test_files(paths)
+      pattern = configuration.test_framework.to_s == "rspec" ? "*_spec.rb" : "*_test.rb"
+
+      paths.flat_map do |path|
+        if configuration.root.join(path).directory?
+          Dir.glob(File.join(path, "**", pattern), base: configuration.root.to_s).sort
+        else
+          expand_globs([path])
+        end
+      end
+    end
+
+    def expand_globs(args)
+      args.flat_map do |arg|
+        next [arg] unless arg.match?(/[*?\[{]/)
+
+        matches = Dir.glob(arg, base: configuration.root.to_s).sort
+        matches.empty? ? [arg] : matches
+      end
+    end
+
+    def build_capture_file
+      Reqcord.root.join(
+        "tmp",
+        "reqcord",
+        "#{SecureRandom.hex(12)}.ndjson"
+      ).to_s
+    end
+
+    def run_tests(capture_file)
+      FileUtils.mkdir_p(
+        File.dirname(capture_file)
+      )
+
+      env = {
+        "REQCORD_CAPTURE" => "1",
+        "REQCORD_CAPTURE_FILE" => capture_file
+      }
+
+      command = test_argv
+
+      Reqcord.log("running #{Shellwords.join(command)}")
+
+      # Streamed rather than captured: a long suite should print as it runs.
+      status =
+        Open3.popen2e(
+          env,
+          *command,
+          chdir: Reqcord.root.to_s
+        ) do |stdin, output, wait_thread|
+          stdin.close
+
+          output.each_line { |line| $stdout.print(line) }
+
+          wait_thread.value
+        end
+
+      return if status.success?
+
+      raise GenerationError,
+            "Test suite failed while generating Reqcord documentation"
+    end
+
+    def read_exchanges(capture_file)
+      return [] unless File.exist?(capture_file)
+
+      File.readlines(capture_file)
+          .filter_map do |line|
+            line = line.strip
+
+            next if line.empty?
+
+            begin
+              JSON.parse(line)
+            rescue JSON::ParserError
+              Reqcord.warn("skipping malformed capture line")
+              nil
+            end
+          end
+    end
+
+    def find_route(routes, exchange)
+      request = exchange.fetch("request")
+      method = request.fetch("method")
+      path = request.fetch("path")
+
+      routes.find { |route| route.matches?(method, path) }
+    rescue StandardError => e
+      raise GenerationError, "could not match #{method} #{path} against the route table: #{e.message}"
+    end
+
+    def attach_exchange(endpoint, exchange)
+      request_data = exchange.fetch("request")
+      response_data = exchange.fetch("response")
+      source = normalize_source(exchange["source"])
+      status = response_data.fetch("status")
+
+      request =
+        RequestExample.new(
+          name: example_name(source),
+          method: request_data.fetch("method"),
+          # The concrete path, not the route pattern: the cURL must be runnable.
+          path: request_data.fetch("path"),
+          path_params: request_data["path_params"] || {},
+          query_params: request_data["query_params"] || {},
+          headers: request_data["headers"] || {},
+          body: request_data["body"],
+          content_type: request_data["content_type"],
+          response_status: status,
+          source: source
+        )
+
+      response =
+        ResponseExample.new(
+          name: example_name(source),
+          status: status,
+          headers: response_data["headers"] || {},
+          body: response_data["body"],
+          content_type: response_data["content_type"],
+          source: source
+        )
+
+      endpoint.add_exchange(
+        request: request,
+        response: response
+      )
+    end
+
+    # "test_creates_customer" -> "Creates Customer"
+    def example_name(source)
+      test = source["test"].to_s.sub(/\Atest_/, "")
+
+      test.empty? ? nil : Support.titleize(test)
+    end
+
+    def normalize_source(source)
+      source = (source || {}).transform_keys(&:to_s)
+      file = source["file"].to_s
+      prefix = "#{Reqcord.root}/"
+
+      source["file"] = file.delete_prefix(prefix) unless file.empty?
+      source.compact
+    end
+
+    def write_outputs(dataset)
+      output = configuration.output_directory
+
+      FileUtils.mkdir_p(output)
+
+      written = [dataset.write(output.join("dataset.json"))]
+
+      configuration.exporters.each do |name|
+        written.concat(
+          Exporters.fetch(name).call(
+            dataset: dataset,
+            output_dir: output,
+            configuration: configuration
+          )
+        )
+      end
+
+      written
+    end
+  end
+end

data/lib/reqcord/railtie.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Reqcord
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load File.expand_path(
+        "../tasks/reqcord.rake",
+        __dir__
+      )
+    end
+
+    initializer "reqcord.capture" do
+      next unless Reqcord::Capture.enabled?
+
+      require "action_dispatch/testing/integration"
+      require_relative "capture/integration_patch"
+
+      unless ActionDispatch::Integration::Session <
+             Reqcord::Capture::IntegrationPatch
+
+        ActionDispatch::Integration::Session.prepend(
+          Reqcord::Capture::IntegrationPatch
+        )
+      end
+
+      # Capture itself is framework agnostic; only naming the example needs to
+      # know which test framework is running.
+      case Reqcord.configuration.test_framework.to_s
+      when "minitest"
+        require "minitest/test"
+        require_relative "capture/minitest_context"
+
+        unless Minitest::Test < Reqcord::Capture::MinitestContext
+          Minitest::Test.prepend(
+            Reqcord::Capture::MinitestContext
+          )
+        end
+
+      when "rspec"
+        require "rspec/core"
+        require_relative "capture/rspec_context"
+
+        Reqcord::Capture::RSpecContext.install!
+
+      else
+        raise ConfigurationError,
+              "unsupported test framework: #{Reqcord.configuration.test_framework.inspect}"
+      end
+    end
+  end
+end

data/lib/reqcord/renderers/curl.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Reqcord
+  module Renderers
+    class Curl
+      def self.call(request, base_url:)
+        new(
+          request,
+          base_url: base_url
+        ).call
+      end
+
+      def initialize(request, base_url:)
+        @request = request
+        @base_url = base_url.to_s.sub(%r{/$}, "")
+      end
+
+      def call
+        parts = [
+          "curl --request #{request.method}",
+          %(--url "#{url}")
+        ]
+
+        request.headers.each do |key, value|
+          parts << %(--header "#{key}: #{escape_header(value)}")
+        end
+
+        parts << data_argument if request.body?
+
+        parts.join(" \\\n  ")
+      end
+
+      private
+
+      attr_reader :request, :base_url
+
+      def url
+        "#{base_url}#{Payload.path_with_query(request)}"
+      end
+
+      def data_argument
+        "--data '#{shell_single_quote(Payload.raw_body(request))}'"
+      end
+
+      def shell_single_quote(value)
+        value.to_s.gsub("'", %q('"'"'))
+      end
+
+      def escape_header(value)
+        value.to_s
+             .gsub("\\") { "\\\\" }
+             .gsub('"') { '\\"' }
+      end
+    end
+  end
+end