insights-api-common 3.0.0

data/lib/insights/api/common/open_api.rb

@@ -0,0 +1,2 @@
+require "insights/api/common/open_api/docs"
+require "insights/api/common/open_api/serializer"

data/lib/insights/api/common/open_api/docs.rb

@@ -0,0 +1,54 @@
+require "insights/api/common/open_api/docs/component_collection"
+require "insights/api/common/open_api/docs/object_definition"
+require "insights/api/common/open_api/docs/doc_v3"
+
+module Insights
+  module API
+    module Common
+      module OpenApi
+        class Docs
+          def self.instance
+            @instance ||= new(Dir.glob(Rails.root.join("public", "doc", "openapi*.json")))
+          end
+
+          def initialize(glob)
+            @cache = {}
+            glob.each { |f| load_file(f) }
+          end
+
+          def load_file(file)
+            openapi_spec = JSON.parse(File.read(file))
+            store_doc(DocV3.new(openapi_spec))
+          end
+
+          def store_doc(doc)
+            update_doc_for_version(doc, doc.version.segments[0..1].join("."))
+            update_doc_for_version(doc, doc.version.segments.first.to_s)
+          end
+
+          def update_doc_for_version(doc, version)
+            if @cache[version].nil?
+              @cache[version] = doc
+            else
+              existing_version = @cache[version].version
+              @cache[version] = doc if doc.version > existing_version
+            end
+          end
+
+          def [](version)
+            @cache[version]
+          end
+
+          def routes
+            @routes ||= begin
+              @cache.each_with_object([]) do |(version, doc), routes|
+                next unless /\d+\.\d+/ =~ version # Skip unless major.minor
+                routes.concat(doc.routes)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/insights/api/common/open_api/docs/component_collection.rb

@@ -0,0 +1,67 @@
+module Insights
+  module API
+    module Common
+      module OpenApi
+        class Docs
+          class ComponentCollection < Hash
+            attr_reader :doc
+
+            def initialize(doc, category)
+              @doc = doc
+              @category = category
+            end
+
+            def [](name)
+              super || load_definition(name)
+            end
+
+            def load_definition(name)
+              raw_definition = @doc.content.fetch_path(*@category.split("/"), name)
+              raise ArgumentError, "Failed to find definition for #{name}" unless raw_definition.kind_of?(Hash)
+
+              definition = substitute_regexes(raw_definition)
+              definition = substitute_references(definition)
+              self[name] = ::Insights::API::Common::OpenApi::Docs::ObjectDefinition.new.replace(definition)
+            end
+
+            private
+
+            def substitute_references(object)
+              if object.kind_of?(Array)
+                object.collect { |i| substitute_references(i) }
+              elsif object.kind_of?(Hash)
+                return fetch_ref_value(object["$ref"]) if object.keys == ["$ref"]
+                object.each { |k, v| object[k] = substitute_references(v) }
+              else
+                object
+              end
+            end
+
+            def fetch_ref_value(ref_path)
+              ref_paths = ref_path.split("/")
+              property = ref_paths.last
+              section  = ref_paths[1..-2]
+              public_send(:[], property)
+            end
+
+            def substitute_regexes(object)
+              if object.kind_of?(Array)
+                object.collect { |i| substitute_regexes(i) }
+              elsif object.kind_of?(Hash)
+                object.each_with_object({}) do |(k, v), o|
+                  o[k] = k == "pattern" ? regexp_from_pattern(v) : substitute_regexes(v)
+                end
+              else
+                object
+              end
+            end
+
+            def regexp_from_pattern(pattern)
+              Regexp.new(pattern)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/insights/api/common/open_api/docs/doc_v3.rb

@@ -0,0 +1,102 @@
+require "more_core_extensions/core_ext/hash"
+require "more_core_extensions/core_ext/array"
+require "openapi_parser"
+
+module Insights
+  module API
+    module Common
+      module OpenApi
+        class Docs
+          class DocV3
+            attr_reader :content
+
+            def initialize(content)
+              spec_version = content["openapi"]
+              raise "Unsupported OpenAPI Specification version #{spec_version}" unless spec_version =~ /\A3\..*\z/
+
+              @content = content
+            end
+
+            # Validates data types against OpenAPI schema
+            #
+            # @param http_method [String] POST/PATCH/...
+            # @param request_path [String] i.e. /api/sources/v1.0/sources
+            # @param api_version [String] i.e. "v1.0", has to be part of **request_path**
+            # @param payload [String] JSON if payload_content_type == 'application/json'
+            # @param payload_content_type [String]
+            #
+            # @raise OpenAPIParser::OpenAPIError
+            def validate!(http_method, request_path, api_version, payload, payload_content_type = 'application/json')
+              path = request_path.split(api_version)[1]
+              raise "API version not found in request_path" if path.nil?
+
+              request_operation = validator_doc.request_operation(http_method.to_s.downcase, path)
+              request_operation.validate_request_body(payload_content_type, payload)
+            end
+
+            def validate_parameters!(http_method, request_path, api_version, params)
+              path = request_path.split(api_version)[1]
+              raise "API version not found in request_path" if path.nil?
+
+              request_operation = validator_doc.request_operation(http_method.to_s.downcase, path)
+              return unless request_operation
+
+              request_operation.validate_request_parameter(params, {})
+            end
+
+            def version
+              @version ||= Gem::Version.new(content.fetch_path("info", "version"))
+            end
+
+            def parameters
+              @parameters ||= ::Insights::API::Common::OpenApi::Docs::ComponentCollection.new(self, "components/parameters")
+            end
+
+            def schemas
+              @schemas ||= ::Insights::API::Common::OpenApi::Docs::ComponentCollection.new(self, "components/schemas")
+            end
+
+            def definitions
+              schemas
+            end
+
+            def example_attributes(key)
+              schemas[key]["properties"].each_with_object({}) do |(col, stuff), hash|
+                hash[col] = stuff["example"] if stuff.key?("example")
+              end
+            end
+
+            def base_path
+              @base_path ||= @content.fetch_path("servers", 0, "variables", "basePath", "default")
+            end
+
+            def paths
+              @content["paths"]
+            end
+
+            def to_json(options = nil)
+              content.to_json(options)
+            end
+
+            def routes
+              @routes ||= begin
+                paths.flat_map do |path, hash|
+                  hash.collect do |verb, _details|
+                    p = File.join(base_path, path).gsub(/{\w*}/, ":id")
+                    {:path => p, :verb => verb.upcase}
+                  end
+                end
+              end
+            end
+
+            private
+
+            def validator_doc(opts = { :coerce_value => true, :datetime_coerce_class => DateTime })
+              @validator_doc ||= ::OpenAPIParser.parse(content, opts)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/insights/api/common/open_api/docs/object_definition.rb

@@ -0,0 +1,39 @@
+module Insights
+  module API
+    module Common
+      module OpenApi
+        class Docs
+          class ObjectDefinition < Hash
+            def all_attributes
+              properties.map { |key, val| all_attributes_recursive(key, val) }
+            end
+
+            def read_only_attributes
+              properties.select { |k, v| v["readOnly"] == true }.keys
+            end
+
+            def required_attributes
+              self["required"]
+            end
+
+            def properties
+              self["properties"]
+            end
+
+            private
+
+            def all_attributes_recursive(key, value)
+              if value["properties"]
+                {
+                  key => value["properties"].map { |k, v| all_attributes_recursive(k, v) }
+                }
+              else
+                key
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/insights/api/common/open_api/generator.rb

@@ -0,0 +1,520 @@
+module Insights
+  module API
+    module Common
+      module OpenApi
+        class Generator
+          require 'json'
+          require 'insights/api/common/graphql'
+
+          PARAMETERS_PATH = "/components/parameters".freeze
+          SCHEMAS_PATH = "/components/schemas".freeze
+
+          def path_parts(openapi_path)
+            openapi_path.split("/")[1..-1]
+          end
+
+          # Let's get the latest api version based on the openapi.json routes
+          def api_version
+            @api_version ||= Rails.application.routes.routes.each_with_object([]) do |route, array|
+              matches = ActionDispatch::Routing::RouteWrapper
+                        .new(route)
+                        .path.match(/\A.*\/v(\d+.\d+)\/openapi.json.*\z/)
+              array << matches[1] if matches
+            end.max
+          end
+
+          def rails_routes
+            Rails.application.routes.routes.each_with_object([]) do |route, array|
+              r = ActionDispatch::Routing::RouteWrapper.new(route)
+              next if r.internal? # Don't display rails routes
+              next if r.engine? # Don't care right now...
+
+              array << r
+            end
+          end
+
+          def openapi_file
+            @openapi_file ||= Rails.root.join("public", "doc", "openapi-3-v#{api_version}.0.json").to_s
+          end
+
+          def openapi_contents
+            @openapi_contents ||= begin
+              JSON.parse(File.read(openapi_file))
+            end
+          end
+
+          def initialize
+            app_prefix, app_name = base_path.match(/\A(.*)\/(.*)\/v\d+.\d+\z/).captures
+            ENV['APP_NAME'] = app_name
+            ENV['PATH_PREFIX'] = app_prefix
+            Rails.application.reload_routes!
+          end
+
+          def base_path
+            openapi_contents["servers"].first["variables"]["basePath"]["default"]
+          end
+
+          def applicable_rails_routes
+            rails_routes.select { |i| i.path.start_with?(base_path) }
+          end
+
+          def schemas
+            @schemas ||= {
+              "CollectionLinks"    => {
+                "type"       => "object",
+                "properties" => {
+                  "first" => {
+                    "type" => "string"
+                  },
+                  "last"  => {
+                    "type" => "string"
+                  },
+                  "next"  => {
+                    "type" => "string"
+                  },
+                  "prev"  => {
+                    "type" => "string"
+                  },
+                }
+              },
+              "CollectionMetadata" => {
+                "type"       => "object",
+                "properties" => {
+                  "count"  => {
+                    "type" => "integer"
+                  },
+                  "limit"  => {
+                    "type" => "integer"
+                  },
+                  "offset" => {
+                    "type" => "integer"
+                  }
+                }
+              },
+              "ID"                 => {
+                "type"        => "string",
+                "description" => "ID of the resource",
+                "pattern"     => "^\\d+$",
+                "readOnly"    => true,
+              },
+              "SortByAttribute"    => {
+                "type"        => "string",
+                "description" => "Attribute with optional order to sort the result set by.",
+                "pattern"     => "^[a-z\\-_]+(:asc|:desc)?$"
+              }
+            }
+          end
+
+          def build_schema(klass_name)
+            schemas[klass_name] = openapi_schema(klass_name)
+            "##{SCHEMAS_PATH}/#{klass_name}"
+          end
+
+          def build_schema_error_not_found
+            klass_name = "ErrorNotFound"
+
+            schemas[klass_name] = {
+              "type"       => "object",
+              "properties" => {
+                "errors"  => {
+                  "type"  => "array",
+                  "items" => {
+                    "type"        => "object",
+                    "properties"  => {
+                      "status"    => {
+                        "type"    => "integer",
+                        "example" => 404
+                      },
+                      "detail"    => {
+                        "type"    => "string",
+                        "example" => "Record not found"
+                      }
+                    }
+                  }
+                }
+              }
+            }
+
+            "##{SCHEMAS_PATH}/#{klass_name}"
+          end
+
+          def parameters
+            @parameters ||= {
+              "QueryFilter" => {
+                "in"          => "query",
+                "name"        => "filter",
+                "description" => "Filter for querying collections.",
+                "required"    => false,
+                "style"       => "deepObject",
+                "explode"     => true,
+                "schema"      => {
+                  "type" => "object"
+                }
+              },
+              "QueryLimit"  => {
+                "in"          => "query",
+                "name"        => "limit",
+                "description" => "The numbers of items to return per page.",
+                "required"    => false,
+                "schema"      => {
+                  "type"    => "integer",
+                  "minimum" => 1,
+                  "maximum" => 1000,
+                  "default" => 100
+                }
+              },
+              "QueryOffset" => {
+                "in"          => "query",
+                "name"        => "offset",
+                "description" => "The number of items to skip before starting to collect the result set.",
+                "required"    => false,
+                "schema"      => {
+                  "type"    => "integer",
+                  "minimum" => 0,
+                  "default" => 0
+                }
+              },
+              "QuerySortBy" => {
+                "in"          => "query",
+                "name"        => "sort_by",
+                "description" => "The list of attribute and order to sort the result set by.",
+                "required"    => false,
+                "schema"      => {
+                  "oneOf" => [
+                    { "$ref" => "##{SCHEMAS_PATH}/SortByAttribute" },
+                    { "type" => "array", "items" => { "$ref" => "##{SCHEMAS_PATH}/SortByAttribute" } }
+                  ]
+                }
+              }
+            }
+          end
+
+          def build_parameter(name, value = nil)
+            parameters[name] = value
+            "##{PARAMETERS_PATH}/#{name}"
+          end
+
+          def openapi_schema(klass_name)
+            {
+              "type"                 => "object",
+              "properties"           => openapi_schema_properties(klass_name),
+              "additionalProperties" => false
+            }
+          end
+
+          def openapi_list_description(klass_name, primary_collection)
+            sub_collection = (primary_collection != klass_name)
+            {
+              "summary"     => "List #{klass_name.pluralize}#{" for #{primary_collection}" if sub_collection}",
+              "operationId" => "list#{primary_collection if sub_collection}#{klass_name.pluralize}",
+              "description" => "Returns an array of #{klass_name} objects",
+              "parameters"  => [
+                { "$ref" => "##{PARAMETERS_PATH}/QueryLimit"  },
+                { "$ref" => "##{PARAMETERS_PATH}/QueryOffset" },
+                { "$ref" => "##{PARAMETERS_PATH}/QueryFilter" },
+                { "$ref" => "##{PARAMETERS_PATH}/QuerySortBy" }
+              ],
+              "responses"   => {
+                "200" => {
+                  "description" => "#{klass_name.pluralize} collection",
+                  "content"     => {
+                    "application/json" => {
+                      "schema" => { "$ref" => build_collection_schema(klass_name) }
+                    }
+                  }
+                }
+              }
+            }.tap do |h|
+              h["parameters"] << { "$ref" => build_parameter("ID") } if sub_collection
+
+              next unless sub_collection
+
+              h["responses"]["404"] = {
+                "description" => "Not found",
+                "content"     => {
+                  "application/json" => {
+                    "schema"         => { "$ref" => build_schema_error_not_found }
+                  }
+                }
+              }
+            end
+          end
+
+          def build_collection_schema(klass_name)
+            collection_name = "#{klass_name.pluralize}Collection"
+            schemas[collection_name] = {
+              "type"       => "object",
+              "properties" => {
+                "meta"  => { "$ref" => "##{SCHEMAS_PATH}/CollectionMetadata" },
+                "links" => { "$ref" => "##{SCHEMAS_PATH}/CollectionLinks"    },
+                "data"  => {
+                  "type"  => "array",
+                  "items" => { "$ref" => build_schema(klass_name) }
+                }
+              }
+            }
+
+            "##{SCHEMAS_PATH}/#{collection_name}"
+          end
+
+          def openapi_show_description(klass_name)
+            {
+              "summary"     => "Show an existing #{klass_name}",
+              "operationId" => "show#{klass_name}",
+              "description" => "Returns a #{klass_name} object",
+              "parameters"  => [{ "$ref" => build_parameter("ID") }],
+              "responses"   => {
+                "200" => {
+                  "description" => "#{klass_name} info",
+                  "content"     => {
+                    "application/json" => {
+                      "schema" => { "$ref" => build_schema(klass_name) }
+                    }
+                  }
+                },
+                "404" => {
+                  "description" => "Not found",
+                  "content"     => {
+                    "application/json" => {
+                      "schema"         => { "$ref" => build_schema_error_not_found }
+                    }
+                  }
+                }
+              }
+            }
+          end
+
+          def openapi_destroy_description(klass_name)
+            {
+              "summary"     => "Delete an existing #{klass_name}",
+              "operationId" => "delete#{klass_name}",
+              "description" => "Deletes a #{klass_name} object",
+              "parameters"  => [{ "$ref" => build_parameter("ID") }],
+              "responses"   => {
+                "204" => { "description" => "#{klass_name} deleted" },
+                "404" => {
+                  "description" => "Not found",
+                  "content"     => {
+                    "application/json" => {
+                      "schema"         => { "$ref" => build_schema_error_not_found }
+                    }
+                  }
+                }
+              }
+            }
+          end
+
+          def openapi_create_description(klass_name)
+            {
+              "summary"     => "Create a new #{klass_name}",
+              "operationId" => "create#{klass_name}",
+              "description" => "Creates a #{klass_name} object",
+              "requestBody" => {
+                "content"     => {
+                  "application/json" => {
+                    "schema" => { "$ref" => build_schema(klass_name) }
+                  }
+                },
+                "description" => "#{klass_name} attributes to create",
+                "required"    => true
+              },
+              "responses"   => {
+                "201" => {
+                  "description" => "#{klass_name} creation successful",
+                  "content"     => {
+                    "application/json" => {
+                      "schema" => { "$ref" => build_schema(klass_name) }
+                    }
+                  }
+                }
+              }
+            }
+          end
+
+          def openapi_update_description(klass_name, verb)
+            action = verb == "patch" ? "Update" : "Replace"
+            {
+              "summary"     => "#{action} an existing #{klass_name}",
+              "operationId" => "#{action.downcase}#{klass_name}",
+              "description" => "#{action}s a #{klass_name} object",
+              "parameters"  => [
+                { "$ref" => build_parameter("ID") }
+              ],
+              "requestBody" => {
+                "content"     => {
+                  "application/json" => {
+                    "schema" => { "$ref" => build_schema(klass_name) }
+                  }
+                },
+                "description" => "#{klass_name} attributes to update",
+                "required"    => true
+              },
+              "responses"   => {
+                "204" => { "description" => "Updated, no content" },
+                "400" => { "description" => "Bad request"         },
+                "404" => {
+                  "description" => "Not found",
+                  "content"     => {
+                    "application/json" => {
+                      "schema"         => { "$ref" => build_schema_error_not_found }
+                    }
+                  }
+                }
+              }
+            }
+          end
+
+          def openapi_schema_properties_value(klass_name, model, key, value)
+            if key == model.primary_key
+              {
+                "$ref" => "##{SCHEMAS_PATH}/ID"
+              }
+            elsif key.ends_with?("_id")
+              properties_value = {}
+              properties_value["$ref"] = if generator_read_only_definitions.include?(klass_name)
+                                           # Everything under providers data is read only for now
+                                           "##{SCHEMAS_PATH}/ID"
+                                         else
+                                           openapi_contents.dig(*path_parts(SCHEMAS_PATH), klass_name, "properties", key, "$ref") || "##{SCHEMAS_PATH}/ID"
+                                         end
+              properties_value
+            else
+              properties_value = {
+                "type" => "string"
+              }
+
+              case value.sql_type_metadata.type
+              when :datetime
+                properties_value["format"] = "date-time"
+              when :integer
+                properties_value["type"] = "integer"
+              when :float
+                properties_value["type"] = "number"
+              when :boolean
+                properties_value["type"] = "boolean"
+              when :jsonb
+                properties_value["type"] = "object"
+                ['type', 'items', 'properties', 'additionalProperties'].each do |property_key|
+                  prop = openapi_contents.dig(*path_parts(SCHEMAS_PATH), klass_name, "properties", key, property_key)
+                  properties_value[property_key] = prop unless prop.nil?
+                end
+              end
+
+              # Take existing attrs, that we won't generate
+              ['example', 'format', 'readOnly', 'title', 'description'].each do |property_key|
+                property_value                 = openapi_contents.dig(*path_parts(SCHEMAS_PATH), klass_name, "properties", key, property_key)
+                properties_value[property_key] = property_value if property_value
+              end
+
+              if generator_read_only_definitions.include?(klass_name) || generator_read_only_attributes.include?(key.to_sym)
+                # Everything under providers data is read only for now
+                properties_value['readOnly'] = true
+              end
+
+              properties_value.sort.to_h
+            end
+          end
+
+          def run(graphql = false)
+            new_content = openapi_contents.dup
+            new_content["paths"] = build_paths.sort.to_h
+            new_content["components"] ||= {}
+            new_content["components"]["schemas"]    = schemas.sort.each_with_object({})    { |(name, val), h| h[name] = val || openapi_contents["components"]["schemas"][name]    || {} }
+            new_content["components"]["parameters"] = parameters.sort.each_with_object({}) { |(name, val), h| h[name] = val || openapi_contents["components"]["parameters"][name] || {} }
+            File.write(openapi_file, JSON.pretty_generate(new_content) + "\n")
+            Insights::API::Common::GraphQL::Generator.generate(api_version, new_content) if graphql
+          end
+
+          def openapi_schema_properties(klass_name)
+            model = klass_name.constantize
+            model.columns_hash.map do |key, value|
+              unless (generator_blacklist_allowed_attributes[key.to_sym] || []).include?(klass_name)
+                next if generator_blacklist_attributes.include?(key.to_sym)
+              end
+
+              if generator_blacklist_substitute_attributes.include?(key.to_sym)
+                generator_blacklist_substitute_attributes[key.to_sym]
+              else
+                [key, openapi_schema_properties_value(klass_name, model, key, value)]
+              end
+            end.compact.sort.to_h
+          end
+
+          def generator_blacklist_attributes
+            @generator_blacklist_attributes ||= [
+              :resource_timestamp,
+              :resource_timestamps,
+              :resource_timestamps_max,
+              :tenant_id,
+            ].to_set.freeze
+          end
+
+          def generator_blacklist_allowed_attributes
+            @generator_blacklist_allowed_attributes ||= {}
+          end
+
+          def generator_blacklist_substitute_attributes
+            @generator_blacklist_substitute_attributes ||= {}
+          end
+
+          def generator_read_only_attributes
+            @generator_read_only_attributes ||= [
+              :archived_at,
+              :created_at,
+              :last_seen_at,
+              :updated_at,
+            ].to_set.freeze
+          end
+
+          def generator_read_only_definitions
+            @generator_read_only_definitions ||= [].to_set.freeze
+          end
+
+          def build_paths
+            applicable_rails_routes.each_with_object({}) do |route, expected_paths|
+              without_format = route.path.split("(.:format)").first
+              sub_path = without_format.split(base_path).last.sub(/:[_a-z]*id/, "{id}")
+              klass_name = route.controller.split("/").last.camelize.singularize
+              verb = route.verb.downcase
+              primary_collection = sub_path.split("/")[1].camelize.singularize
+
+              expected_paths[sub_path] ||= {}
+              expected_paths[sub_path][verb] =
+                case route.action
+                when "index"   then openapi_list_description(klass_name, primary_collection)
+                when "show"    then openapi_show_description(klass_name)
+                when "destroy" then openapi_destroy_description(klass_name)
+                when "create"  then openapi_create_description(klass_name)
+                when "update"  then openapi_update_description(klass_name, verb)
+                else handle_custom_route_action(route.action.camelize, verb, primary_collection)
+                end
+
+              next if expected_paths[sub_path][verb]
+
+              # If it's not generic action but a custom method like e.g. `post "order", :to => "service_plans#order"`, we will
+              # try to take existing schema, because the description, summary, etc. are likely to be custom.
+              expected_paths[sub_path][verb] =
+                case verb
+                when "post"
+                  if sub_path == "/graphql" && route.action == "query"
+                    schemas["GraphQLRequest"]  = ::Insights::API::Common::GraphQL.openapi_graphql_request
+                    schemas["GraphQLResponse"] = ::Insights::API::Common::GraphQL.openapi_graphql_response
+                    ::Insights::API::Common::GraphQL.openapi_graphql_description
+                  else
+                    openapi_contents.dig("paths", sub_path, verb) || openapi_create_description(klass_name)
+                  end
+                when "get"
+                  openapi_contents.dig("paths", sub_path, verb) || openapi_show_description(klass_name)
+                else
+                  openapi_contents.dig("paths", sub_path, verb)
+                end
+            end
+          end
+
+          def handle_custom_route_action(_route_action, _verb, _primary_collection)
+          end
+        end
+      end
+    end
+  end
+end