archsight 0.1.2 → 0.1.3

data/lib/archsight/import/handlers/rest_api.rb

@@ -0,0 +1,293 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "net/http"
+require "yaml"
+require "uri"
+require "openssl"
+require_relative "../handler"
+require_relative "../registry"
+require_relative "openapi_schema_parser"
+
+# REST API handler - downloads OpenAPI spec and generates ApplicationInterface and DataObject resources
+#
+# Configuration (passed from rest_api_index handler):
+#   import/config/name - API name (e.g., "compute")
+#   import/config/version - API version (e.g., "6.0")
+#   import/config/visibility - API visibility (e.g., "private")
+#   import/config/specUrl - Full URL to OpenAPI spec (http, https, or file://)
+#   import/config/htmlUrl - Full URL to HTML documentation (optional)
+#   import/config/gate - Release gate (e.g., "GA", "BETA")
+#   import/config/interfaceOutputPath - Output path for ApplicationInterface resources
+#   import/config/dataObjectOutputPath - Output path for DataObject resources
+#
+# Output:
+#   - ApplicationInterface resource with annotations
+#   - DataObject resources extracted from OpenAPI schemas
+class Archsight::Import::Handlers::RestApi < Archsight::Import::Handler
+  def execute
+    @name = config("name")
+    raise "Missing required config: name" unless @name
+
+    @version = config("version", default: "1.0")
+    @visibility = config("visibility", default: "private")
+    @spec_url = config("specUrl")
+    raise "Missing required config: specUrl" unless @spec_url
+
+    @html_url = config("htmlUrl")
+    @gate = config("gate", default: "GA")
+
+    @interface_output_path = config("interfaceOutputPath")
+    @data_object_output_path = config("dataObjectOutputPath")
+
+    # Download and parse OpenAPI spec
+    progress.update("Downloading OpenAPI spec for #{@name}")
+    openapi_doc = fetch_openapi_spec(@spec_url)
+
+    # Parse schemas and generate DataObjects first (needed for interface relations)
+    progress.update("Extracting DataObjects from #{@name} schemas")
+    parser = Archsight::Import::Handlers::OpenAPISchemaParser.new(openapi_doc)
+    parsed_objects = parser.parse
+    data_objects = generate_data_objects(parsed_objects)
+
+    # Generate ApplicationInterface with references to DataObjects
+    progress.update("Generating ApplicationInterface for #{@name}")
+    data_object_names = data_objects.map { |obj| obj.dig("metadata", "name") }
+    interface_resource = generate_interface(openapi_doc, data_object_names: data_object_names)
+
+    # Write outputs
+    if @interface_output_path
+      write_yaml(YAML.dump(interface_resource), filename: nil, sort_key: "#{@name}-interface")
+    else
+      write_yaml(YAML.dump(interface_resource), filename: "#{safe_api_name}-interface.yaml")
+    end
+
+    if data_objects.any?
+      data_yaml = data_objects.map { |r| YAML.dump(r) }.join("\n")
+      if @data_object_output_path
+        # Use shared output path - write each object with a sort key
+        data_objects.each do |obj|
+          write_data_object(obj)
+        end
+      else
+        write_yaml(data_yaml, filename: "#{safe_api_name}-data-objects.yaml")
+      end
+    end
+
+    progress.update("Generated #{data_objects.size} DataObjects for #{@name}")
+
+    write_generates_meta
+  end
+
+  private
+
+  def safe_api_name
+    @name.gsub(/[^a-zA-Z0-9_-]/, "_").downcase
+  end
+
+  def fetch_openapi_spec(url)
+    uri = URI(url)
+
+    case uri.scheme
+    when "file"
+      fetch_file_spec(uri)
+    when "http", "https"
+      fetch_http_spec(uri)
+    else
+      raise "Unsupported URL scheme: #{uri.scheme}. Use http://, https://, or file://"
+    end
+  end
+
+  def fetch_file_spec(uri)
+    # Handle file:// URLs - the host part may be treated as part of the path
+    # e.g., file://lib/path becomes host=lib, path=/path
+    # We need to reconstruct: host + path
+    path = if uri.host && uri.host != "localhost" && !uri.host.empty?
+             "#{uri.host}#{uri.path}"
+           else
+             uri.path
+           end
+
+    raise "File not found: #{path}" unless File.exist?(path)
+
+    content = File.read(path)
+    parse_spec_content(content)
+  end
+
+  def fetch_http_spec(uri)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = uri.scheme == "https"
+    http.open_timeout = 30
+    http.read_timeout = 60
+
+    request = Net::HTTP::Get.new(uri)
+    request["Accept"] = "application/yaml, application/json"
+
+    response = http.request(request)
+
+    case response
+    when Net::HTTPSuccess
+      parse_spec_content(response.body)
+    when Net::HTTPRedirection
+      # Follow redirect
+      fetch_openapi_spec(response["location"])
+    else
+      raise "Failed to fetch OpenAPI spec from #{uri}: #{response.code} #{response.message}"
+    end
+  end
+
+  def parse_spec_content(content)
+    # Try YAML first, then JSON
+    YAML.safe_load(content, permitted_classes: [Date, Time])
+  rescue Psych::SyntaxError
+    JSON.parse(content)
+  end
+
+  def generate_interface(openapi_doc, data_object_names: [])
+    info = openapi_doc["info"] || {}
+    title = info["title"] || @name.capitalize
+    description = info["description"] || ""
+    openapi_version = openapi_doc["openapi"] || "3.0.0"
+
+    # Detect technologies from spec
+    tags = detect_technologies(openapi_doc)
+
+    # Build resource name
+    interface_name = build_interface_name
+
+    annotations = {
+      "architecture/title" => title,
+      "architecture/description" => description.strip,
+      "architecture/openapi" => openapi_version,
+      "architecture/version" => @version,
+      "architecture/status" => @gate,
+      "architecture/visibility" => @visibility,
+      "architecture/tags" => tags.join(","),
+      "architecture/encoding" => "json"
+    }
+    annotations["architecture/documentation"] = @html_url if @html_url
+
+    # Build spec with technology relations and data objects
+    spec = {
+      "servedBy" => {
+        "technologyComponents" => build_technology_components(tags)
+      }
+    }
+
+    # Add relation to DataObjects if any were generated
+    spec["serves"] = { "dataObjects" => data_object_names } if data_object_names.any?
+
+    resource_yaml(
+      kind: "ApplicationInterface",
+      name: interface_name,
+      annotations: annotations,
+      spec: spec
+    )
+  end
+
+  def build_interface_name
+    visibility_prefix = @visibility.split("-").map(&:capitalize).join
+    api_name = @name.split(/[-_]/).map(&:capitalize).join
+    # Handle version that may already have "v" prefix (e.g., "v1" or "1.0")
+    version_str = @version.split(".").first
+    version_str = version_str.sub(/^v/i, "") # Remove leading v if present
+    "#{visibility_prefix}:#{api_name}:v#{version_str}:RestAPI"
+  end
+
+  def detect_technologies(openapi_doc)
+    tags = %w[https rest]
+
+    # Check security schemes
+    security_schemes = openapi_doc.dig("components", "securitySchemes") || {}
+
+    security_schemes.each_value do |scheme|
+      case scheme["type"]
+      when "http"
+        case scheme["scheme"]&.downcase
+        when "bearer"
+          tags << "jwt" if scheme["bearerFormat"]&.downcase == "jwt"
+          tags << "bearer" unless tags.include?("jwt")
+        when "basic"
+          tags << "basic-auth"
+        end
+      when "apiKey"
+        tags << "api-key"
+      when "oauth2"
+        tags << "oauth2"
+      when "openIdConnect"
+        tags << "oidc"
+      end
+    end
+
+    # Check for specific headers or patterns
+    tags << "auth" if openapi_doc["paths"]&.to_s&.include?("Authorization") && tags.none? { |t| t.include?("auth") || t == "jwt" || t == "bearer" }
+
+    tags.uniq
+  end
+
+  def build_technology_components(tags)
+    components = ["HTTPS:REST"]
+
+    tags.each do |tag|
+      case tag
+      when "jwt"
+        components << "AUTH:JWT"
+      when "basic-auth"
+        components << "AUTH:Basic"
+      when "api-key"
+        components << "AUTH:APIKey"
+      when "oauth2"
+        components << "AUTH:OAuth2"
+      when "oidc"
+        components << "AUTH:OIDC"
+      end
+    end
+
+    components.uniq
+  end
+
+  def generate_data_objects(parsed_objects)
+    app_name = @name.split(/[-_]/).map(&:capitalize).join
+
+    parsed_objects.map do |normalized_name, info|
+      object_name = "#{app_name}:#{normalized_name}"
+
+      # Build description with field documentation
+      description_parts = []
+      description_parts << info["description"] if info["description"]
+
+      if info["properties"]&.any?
+        field_docs = Archsight::Import::Handlers::OpenAPISchemaParser.generate_field_docs(info["properties"])
+        description_parts << field_docs unless field_docs.empty?
+      end
+
+      annotations = {
+        "data/application" => app_name,
+        "data/visibility" => @visibility,
+        "architecture/description" => description_parts.join("\n\n"),
+        "generated/variants" => info["original_names"].join(", ")
+      }
+
+      resource_yaml(
+        kind: "DataObject",
+        name: object_name,
+        annotations: annotations,
+        spec: {}
+      )
+    end
+  end
+
+  def write_data_object(obj)
+    # Override the output path for data objects
+    original_output_path = import_resource.annotations["import/outputPath"]
+    import_resource.annotations["import/outputPath"] = @data_object_output_path
+
+    name = obj.dig("metadata", "name")
+    write_yaml(YAML.dump(obj), sort_key: name)
+
+    # Restore original output path
+    import_resource.annotations["import/outputPath"] = original_output_path
+  end
+end
+
+Archsight::Import::Registry.register("rest-api", Archsight::Import::Handlers::RestApi)

data/lib/archsight/import/handlers/rest_api_index.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "net/http"
+require "json"
+require "uri"
+require "openssl"
+require_relative "../handler"
+require_relative "../registry"
+
+# REST API Index handler - fetches API index and generates child Import resources
+#
+# Configuration:
+#   import/config/indexUrl - URL to fetch API index JSON (required)
+#   import/config/baseUrl - Base URL for spec files (optional, derived from indexUrl)
+#   import/config/interfaceOutputPath - Shared output for ApplicationInterface resources
+#   import/config/dataObjectOutputPath - Shared output for DataObject resources
+#   import/config/skipVisibility - Comma-separated visibilities to skip (e.g., "public-preview")
+#   import/config/childCacheTime - Cache time for generated child imports (e.g., "1h", "30m")
+#
+# Output:
+#   Generates Import:RestApi:* resources for each API in the index
+#
+# Expected index format (object with "pages" array):
+#   {
+#     "pages": [
+#       {
+#         "name": "compute",
+#         "version": "v1",
+#         "visibility": "public",
+#         "spec": "/rest-api/public-compute-v1.yaml",
+#         "redoc": "/rest-api/docs/compute/v1/",
+#         "gate": "General-Availability"
+#       },
+#       ...
+#     ]
+#   }
+class Archsight::Import::Handlers::RestApiIndex < Archsight::Import::Handler
+  def execute
+    @index_url = config("indexUrl")
+    raise "Missing required config: indexUrl" unless @index_url
+
+    @base_url = config("baseUrl") || derive_base_url(@index_url)
+    @interface_output_path = config("interfaceOutputPath")
+    @data_object_output_path = config("dataObjectOutputPath")
+    @skip_visibilities = (config("skipVisibility") || "").split(",").map(&:strip).reject(&:empty?)
+    @child_cache_time = config("childCacheTime")
+
+    # Fetch API index
+    progress.update("Fetching API index from #{@index_url}")
+    apis = fetch_index
+
+    if apis.empty?
+      progress.warn("No APIs found in index")
+      return
+    end
+
+    # Filter APIs by visibility
+    original_count = apis.size
+    apis = filter_apis(apis)
+    progress.update("Filtered to #{apis.size} APIs (skipped #{original_count - apis.size} by visibility)") if apis.size < original_count
+
+    # Generate child imports
+    progress.update("Generating #{apis.size} import resources")
+    generate_api_imports(apis)
+
+    write_generates_meta
+  end
+
+  private
+
+  def derive_base_url(url)
+    uri = URI(url)
+    "#{uri.scheme}://#{uri.host}"
+  end
+
+  def fetch_index
+    uri = URI(@index_url)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = uri.scheme == "https"
+    http.open_timeout = 30
+    http.read_timeout = 60
+
+    request = Net::HTTP::Get.new(uri)
+    request["Accept"] = "application/json"
+
+    response = http.request(request)
+
+    case response
+    when Net::HTTPSuccess
+      data = JSON.parse(response.body)
+      # Handle both array format and object with "pages" key
+      data.is_a?(Array) ? data : (data["pages"] || [])
+    when Net::HTTPRedirection
+      # Follow redirect
+      @index_url = response["location"]
+      fetch_index
+    when Net::HTTPUnauthorized
+      raise "API index error: 401 Unauthorized - Check credentials"
+    when Net::HTTPForbidden
+      raise "API index error: 403 Forbidden - Access denied"
+    when Net::HTTPNotFound
+      raise "API index error: 404 Not Found - Index not found at #{@index_url}"
+    else
+      raise "API index error: #{response.code} #{response.message}"
+    end
+  end
+
+  def filter_apis(apis)
+    return apis if @skip_visibilities.empty?
+
+    apis.reject do |api|
+      visibility = api["visibility"]&.downcase
+      @skip_visibilities.any? { |skip| visibility == skip.downcase }
+    end
+  end
+
+  def generate_api_imports(apis)
+    yaml_documents = apis.map do |api|
+      api_name = api["name"]
+      api_version = api["version"] || "v1"
+
+      # Build full URLs from base URL and paths
+      # Support both "spec"/"redoc" (new format) and "specPath"/"redocPath" (legacy)
+      spec_path = api["spec"] || api["specPath"]
+      redoc_path = api["redoc"] || api["redocPath"]
+      spec_url = build_full_url(spec_path)
+      html_url = redoc_path ? build_full_url(redoc_path) : nil
+
+      # Derive visibility from spec path if not explicitly provided
+      visibility = api["visibility"] || derive_visibility_from_path(spec_path)
+
+      # Include visibility in import name to distinguish public/private versions of same API
+      import_name = "Import:RestApi:#{visibility}:#{api_name}:#{api_version}"
+      child_config = {
+        "name" => api_name,
+        "version" => api["version"] || "1.0",
+        "visibility" => visibility,
+        "specUrl" => spec_url,
+        "gate" => api["gate"] || "GA"
+      }
+      child_config["htmlUrl"] = html_url if html_url
+
+      # Build annotations for child import
+      child_annotations = {}
+      child_annotations["import/outputPath"] = @interface_output_path if @interface_output_path
+      child_annotations["import/cacheTime"] = @child_cache_time if @child_cache_time
+      child_annotations["import/config/interfaceOutputPath"] = @interface_output_path if @interface_output_path
+      child_annotations["import/config/dataObjectOutputPath"] = @data_object_output_path if @data_object_output_path
+
+      import_yaml(
+        name: import_name,
+        handler: "rest-api",
+        config: child_config,
+        annotations: child_annotations
+      )
+    end
+
+    # Add self-marker with generated/at for caching
+    yaml_documents << self_marker
+
+    # Write all imports to a single file
+    yaml_content = yaml_documents.map { |doc| YAML.dump(doc) }.join("\n")
+    write_yaml(yaml_content)
+  end
+
+  def build_full_url(path)
+    return path if path.start_with?("http://", "https://", "file://")
+
+    "#{@base_url}#{path}"
+  end
+
+  # Derive visibility from spec path patterns like "/rest-api/public-compute-v1.yaml"
+  # or "/docs/public/compute/v1/"
+  def derive_visibility_from_path(path)
+    return "public" if path&.match?(/\bpublic\b/i)
+    return "private" if path&.match?(/\bprivate\b/i)
+
+    "public" # Default to public for APIs without explicit visibility marker
+  end
+end
+
+Archsight::Import::Registry.register("rest-api-index", Archsight::Import::Handlers::RestApiIndex)

data/lib/archsight/import/progress.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+# Progress reporter for import operations
+#
+# In TTY mode: updates a single line with \r
+# In non-TTY mode (CI): prints each update on a new line
+class Archsight::Import::Progress
+  def initialize(output: $stdout)
+    @output = output
+    @tty = output.respond_to?(:tty?) && output.tty?
+    @last_line_length = 0
+    @current_context = nil
+  end
+
+  def tty?
+    @tty
+  end
+
+  # Set the current context (e.g., "Import:GitLab")
+  def context=(name)
+    @current_context = name
+  end
+
+  # Report progress with optional sub-progress
+  # Examples:
+  #   update("Fetching projects...")
+  #   update("Cloning", current: 5, total: 100)
+  def update(message, current: nil, total: nil)
+    line = build_line(message, current, total)
+
+    if @tty
+      # Clear previous line and write new one
+      clear = "\r#{" " * @last_line_length}\r"
+      @output.print "#{clear}#{line}"
+      @output.flush
+      @last_line_length = line.length
+    else
+      @output.puts line
+    end
+  end
+
+  # Complete the current operation (moves to new line in TTY mode)
+  def complete(message = nil)
+    if message
+      line = build_line(message)
+      if @tty
+        clear = "\r#{" " * @last_line_length}\r"
+        @output.puts "#{clear}#{line}"
+      else
+        @output.puts line
+      end
+    elsif @tty
+      @output.puts
+    end
+    @last_line_length = 0
+  end
+
+  # Report an error
+  def error(message)
+    complete if @tty && @last_line_length.positive?
+    @output.puts "  Error: #{message}"
+  end
+
+  # Report a warning
+  def warn(message)
+    if @tty && @last_line_length.positive?
+      # Save current line, print warning, restore
+      @output.puts
+      @output.puts "  Warning: #{message}"
+      @last_line_length = 0
+    else
+      @output.puts "  Warning: #{message}"
+    end
+  end
+
+  private
+
+  def build_line(message, current = nil, total = nil)
+    parts = []
+    parts << @current_context if @current_context
+    parts << progress_indicator(current, total) if current && total
+    parts << message
+
+    parts.join(" - ")
+  end
+
+  def progress_indicator(current, total)
+    percentage = ((current.to_f / total) * 100).round
+    "[#{current}/#{total} #{percentage}%]"
+  end
+end

data/lib/archsight/import/registry.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "archsight/import"
+
+# Registry of import handlers
+#
+# Handlers register themselves with a name that matches the import/handler annotation.
+# The executor looks up handlers by name to instantiate and execute them.
+module Archsight::Import::Registry
+  @handlers = {}
+
+  class << self
+    # Register a handler class with a name
+    # @param name [String, Symbol] Handler name (matches import/handler annotation)
+    # @param handler_class [Class] Handler class that extends Handler
+    def register(name, handler_class)
+      @handlers[name.to_s] = handler_class
+    end
+
+    # Look up a handler class by name
+    # @param name [String] Handler name
+    # @return [Class, nil] Handler class or nil if not found
+    def [](name)
+      @handlers[name.to_s]
+    end
+
+    # Get handler class for an import resource
+    # @param import_resource [Archsight::Resources::Import] Import resource
+    # @return [Class] Handler class
+    # @raise [UnknownHandlerError] if handler is not registered
+    def handler_for(import_resource)
+      handler_name = import_resource.annotations["import/handler"]
+      handler_class = self[handler_name]
+
+      raise Archsight::Import::UnknownHandlerError, "Unknown import handler: #{handler_name}" unless handler_class
+
+      handler_class
+    end
+
+    # List all registered handler names
+    # @return [Array<String>] Handler names
+    def handlers
+      @handlers.keys
+    end
+
+    # Clear all registered handlers (for testing)
+    def clear!
+      @handlers = {}
+    end
+  end
+end
+
+# Error raised when an unknown handler is requested
+class Archsight::Import::UnknownHandlerError < StandardError; end

data/lib/archsight/import/shared_file_writer.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+# Thread-safe file writer for concurrent import handlers
+#
+# Manages shared output files that multiple handlers can write to.
+# Content is buffered in memory and sorted by key when close_all is called.
+#
+# @example
+#   writer = SharedFileWriter.new
+#   writer.append_yaml("/path/to/output.yaml", yaml_content, sort_key: "Repo:name")
+#   writer.close_all  # Sorts and writes buffered content
+class Archsight::Import::SharedFileWriter
+  def initialize
+    @mutex = Mutex.new
+    @files = {}
+  end
+
+  # Append YAML content to a file (thread-safe, buffered)
+  # Content is sorted by sort_key when close_all is called
+  #
+  # @param path [String] Full path to the output file
+  # @param content [String] YAML content to append
+  # @param sort_key [String, nil] Key for sorting (nil keys go last)
+  def append_yaml(path, content, sort_key: nil)
+    @mutex.synchronize do
+      @files[path] ||= { entries: [], lock: Mutex.new }
+    end
+
+    entry = @files[path]
+    entry[:lock].synchronize do
+      entry[:entries] << { key: sort_key, content: content }
+    end
+  end
+
+  # Close all files - sorts and writes buffered content
+  def close_all
+    @mutex.synchronize do
+      @files.each do |path, entry|
+        write_sorted_file(path, entry[:entries])
+      end
+      @files.clear
+    end
+  end
+
+  private
+
+  def write_sorted_file(path, entries)
+    return if entries.empty?
+
+    FileUtils.mkdir_p(File.dirname(path))
+
+    # Sort by key (nil keys go last)
+    sorted = entries.sort_by { |e| e[:key] || "\xFF" }
+
+    File.open(path, "w") do |file|
+      sorted.each_with_index do |entry, idx|
+        content = entry[:content]
+        # Add document separator if not first and content doesn't have one
+        file.write("---\n") if idx.positive? && !content.start_with?("---")
+        file.write(content)
+        file.write("\n") unless content.end_with?("\n")
+      end
+    end
+  end
+end