archsight 0.1.2 → 0.1.3

data/lib/archsight/import/handler.rb

@@ -0,0 +1,263 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "json"
+require "yaml"
+require "archsight/import"
+require_relative "progress"
+
+# Base class for import handlers
+#
+# Subclasses must implement the #execute method to perform the actual import.
+# Use the helper methods to read configuration, validate environment, and write output.
+class Archsight::Import::Handler
+  attr_reader :import_resource, :database, :resources_dir, :progress, :shared_writer
+
+  # @param import_resource [Archsight::Resources::Import] The import resource to execute
+  # @param database [Archsight::Database] The database instance
+  # @param resources_dir [String] Root resources directory
+  # @param progress [Archsight::Import::Progress] Progress reporter
+  # @param shared_writer [Archsight::Import::SharedFileWriter] Thread-safe file writer for concurrent output
+  def initialize(import_resource, database:, resources_dir:, progress: nil, shared_writer: nil)
+    @import_resource = import_resource
+    @database = database
+    @resources_dir = resources_dir
+    @progress = progress || Archsight::Import::Progress.new
+    @shared_writer = shared_writer
+    @tracked_resources = []
+  end
+
+  # Execute the import. Must be implemented by subclasses.
+  # @raise [NotImplementedError] if not overridden
+  def execute
+    raise NotImplementedError, "#{self.class}#execute must be implemented"
+  end
+
+  # Get a configuration value from import/config/* annotations
+  # @param key [String] Configuration key (without the import/config/ prefix)
+  # @param default [Object, nil] Default value if not set
+  # @return [String, nil] The configuration value
+  def config(key, default: nil)
+    import_resource.annotations["import/config/#{key}"] || default
+  end
+
+  # Get all configuration values as a hash
+  # @return [Hash] Configuration key-value pairs
+  def config_all
+    import_resource.annotations.each_with_object({}) do |(key, value), hash|
+      next unless key.start_with?("import/config/")
+
+      config_key = key.sub("import/config/", "")
+      hash[config_key] = value
+    end
+  end
+
+  # Compute a hash of the import's configuration for cache invalidation
+  # @return [String] 16-character hex hash
+  def compute_config_hash
+    config_data = {
+      handler: import_resource.annotations["import/handler"],
+      config: import_resource.annotations.select { |k, _| k.start_with?("import/config/") }.sort.to_h
+    }
+    Digest::SHA256.hexdigest(config_data.to_json)[0, 16]
+  end
+
+  # Generate a marker Import for this handler with generated/at timestamp and config hash
+  # Used for caching - call at end of execute() to persist the execution timestamp
+  # @return [Hash] Import resource hash ready for YAML serialization
+  def self_marker
+    {
+      "apiVersion" => "architecture/v1alpha1",
+      "kind" => "Import",
+      "metadata" => {
+        "name" => import_resource.name,
+        "annotations" => {
+          "generated/at" => Time.now.utc.iso8601,
+          "generated/configHash" => compute_config_hash
+        }
+      },
+      "spec" => {}
+    }
+  end
+
+  # Write YAML content to the output path
+  # @param content [String] YAML content to write
+  # @param filename [String, nil] Output filename (overrides import/outputPath filename)
+  # @return [String] Path to the written file
+  #
+  # Output location is determined by import/outputPath annotation:
+  #   - Relative to resources_dir (e.g., "generated/repositories.yaml")
+  #   - If filename parameter is provided, it replaces the filename from outputPath
+  #   - If no outputPath, falls back to resources_dir/generated with import name as filename
+  #
+  # When shared_writer is available, uses thread-safe append for concurrent writes.
+  # @param content [String] YAML content to write
+  # @param filename [String, nil] Output filename (overrides import/outputPath filename)
+  # @param sort_key [String, nil] Key for sorting in shared files (default: import name)
+  def write_yaml(content, filename: nil, sort_key: nil)
+    output_path = import_resource.annotations["import/outputPath"]
+
+    full_path = if output_path
+                  base = File.join(resources_dir, output_path)
+                  if filename
+                    # Replace filename portion with provided filename
+                    File.join(File.dirname(base), filename)
+                  else
+                    base
+                  end
+                else
+                  # Fallback to resources_dir/generated
+                  File.join(resources_dir, "generated", filename || "#{safe_filename(import_resource.name)}.yaml")
+                end
+
+    if @shared_writer
+      # Use thread-safe shared writer for concurrent execution
+      # Default sort key is import name for stable output ordering
+      key = sort_key || import_resource.name
+      @shared_writer.append_yaml(full_path, content, sort_key: key)
+    else
+      # Direct write for non-concurrent mode
+      FileUtils.mkdir_p(File.dirname(full_path))
+      File.write(full_path, content)
+    end
+
+    full_path
+  end
+
+  # Generate a resource YAML hash with standard metadata
+  # @param kind [String] Resource kind (e.g., "TechnologyArtifact")
+  # @param name [String] Resource name
+  # @param annotations [Hash] Resource annotations
+  # @param spec [Hash] Resource spec (relations)
+  # @return [Hash] Resource hash ready for YAML serialization
+  def resource_yaml(kind:, name:, annotations: {}, spec: {})
+    @tracked_resources << { kind: kind, name: name }
+    {
+      "apiVersion" => "architecture/v1alpha1",
+      "kind" => kind,
+      "metadata" => {
+        "name" => name,
+        "annotations" => annotations.merge(
+          "generated/script" => import_resource.name,
+          "generated/at" => Time.now.utc.iso8601
+        )
+      },
+      "spec" => spec
+    }
+  end
+
+  # Generate an Import resource YAML hash for child imports
+  # @param name [String] Import resource name
+  # @param handler [String] Handler name
+  # @param config [Hash] Configuration annotations (added as import/config/*)
+  # @param annotations [Hash] Additional annotations (added directly)
+  # @return [Hash] Import resource hash ready for YAML serialization
+  #
+  # NOTE: generated/at is NOT set here - it's only set by the self_marker when
+  # the child import actually executes. This ensures caching works correctly
+  # regardless of file loading order.
+  #
+  # Dependencies are not specified here - they are derived from the `generates`
+  # relation on the parent import (tracked via write_generates_meta).
+  def import_yaml(name:, handler:, config: {}, annotations: {})
+    @tracked_resources << { kind: "Import", name: name }
+    all_annotations = {
+      "import/handler" => handler,
+      "generated/script" => import_resource.name
+    }
+
+    # Add direct annotations (e.g., import/outputPath)
+    all_annotations.merge!(annotations)
+
+    # Add config annotations with prefix
+    config.each do |key, value|
+      all_annotations["import/config/#{key}"] = value.to_s
+    end
+
+    {
+      "apiVersion" => "architecture/v1alpha1",
+      "kind" => "Import",
+      "metadata" => {
+        "name" => name,
+        "annotations" => all_annotations
+      },
+      "spec" => {}
+    }
+  end
+
+  # Convert multiple resource hashes to YAML string with document separators
+  # @param resources [Array<Hash>] Array of resource hashes
+  # @return [String] YAML string with --- separators
+  def resources_to_yaml(resources)
+    resources.map { |r| YAML.dump(r) }.join
+  end
+
+  # Write generates meta record for this Import
+  # Call at end of execute() to persist tracking of generated resources
+  # Appends to the output file rather than overwriting
+  def write_generates_meta
+    return if @tracked_resources.empty?
+
+    meta = generates_meta_record(@tracked_resources)
+    output_path = import_resource.annotations["import/outputPath"]
+
+    full_path = if output_path
+                  File.join(resources_dir, output_path)
+                else
+                  File.join(resources_dir, "generated", "#{safe_filename(import_resource.name)}.yaml")
+                end
+
+    if @shared_writer
+      @shared_writer.append_yaml(full_path, YAML.dump(meta), sort_key: "#{import_resource.name}:generates")
+    else
+      # Append to existing file
+      File.open(full_path, "a") { |f| f.write(YAML.dump(meta)) }
+    end
+  end
+
+  private
+
+  # Create meta record for Import with generates relations
+  # @param resources [Array<Hash>] Array of tracked resources with :kind and :name
+  # @return [Hash] Import resource hash with generates spec
+  def generates_meta_record(resources)
+    grouped = resources.group_by { |r| r[:kind] }
+
+    generates_spec = {}
+    grouped.each do |kind, items|
+      kind_key = kind_to_relation_key(kind)
+      generates_spec[kind_key] = items.map { |r| r[:name] }
+    end
+
+    {
+      "apiVersion" => "architecture/v1alpha1",
+      "kind" => "Import",
+      "metadata" => { "name" => import_resource.name },
+      "spec" => { "generates" => generates_spec }
+    }
+  end
+
+  # Convert resource kind to relation key
+  # @param kind [String] Resource kind (e.g., "TechnologyArtifact")
+  # @return [String] Relation key (e.g., "technologyArtifacts")
+  def kind_to_relation_key(kind)
+    case kind
+    when "TechnologyArtifact" then "technologyArtifacts"
+    when "ApplicationInterface" then "applicationInterfaces"
+    when "DataObject" then "dataObjects"
+    when "Import" then "imports"
+    when "BusinessActor" then "businessActors"
+    else
+      # Fallback: convert CamelCase to camelCase plural
+      "#{kind[0].downcase}#{kind[1..]}s"
+    end
+  end
+
+  # Convert a name to a safe filename
+  # @param name [String] Resource name
+  # @return [String] Safe filename
+  def safe_filename(name)
+    name.gsub(/[^a-zA-Z0-9_-]/, "_")
+  end
+end

data/lib/archsight/import/handlers/github.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "net/http"
+require "json"
+require "uri"
+require "openssl"
+require_relative "../handler"
+require_relative "../registry"
+
+# GitHub handler - lists repositories from a GitHub organization and generates child Import resources
+#
+# Configuration:
+#   import/config/org - GitHub organization name
+#   import/config/repoOutputPath - Output path for repository handler results (e.g., "generated/repositories.yaml")
+#   import/config/childCacheTime - Cache time for generated child imports (e.g., "1h", "30m")
+#
+# Environment:
+#   GITHUB_TOKEN - GitHub Personal Access Token (required)
+#     Create at: https://github.com/settings/tokens
+#     Required scopes: repo (private repos) or public_repo (public only)
+#
+# Output:
+#   Generates Import:Repo:* resources for each repository
+#   The repository handler will clone/sync the actual git repositories
+class Archsight::Import::Handlers::Github < Archsight::Import::Handler
+  PER_PAGE = 100
+
+  def execute
+    @org = config("org")
+    raise "Missing required config: org" unless @org
+
+    @token = ENV.fetch("GITHUB_TOKEN", nil)
+    raise "Missing required environment variable: GITHUB_TOKEN" unless @token
+
+    @repo_output_path = config("repoOutputPath")
+    @child_cache_time = config("childCacheTime")
+    @target_dir = File.join(Dir.home, ".cache", "archsight", "git", "github", @org)
+
+    # Fetch all repositories with pagination
+    progress.update("Fetching repositories from #{@org}")
+    repos = fetch_all_repos
+
+    if repos.empty?
+      progress.warn("No repositories found in #{@org}")
+      return
+    end
+
+    # Generate Import resources for each repository
+    progress.update("Generating #{repos.size} import resources")
+    generate_repository_imports(repos)
+
+    write_generates_meta
+  end
+
+  private
+
+  def fetch_all_repos
+    all_repos = []
+    page = 1
+
+    loop do
+      progress.update("Fetching repositories (page #{page})")
+      batch = fetch_repos_page(page)
+      break if batch.empty?
+
+      all_repos.concat(batch)
+      progress.update("Fetched #{all_repos.size} repositories")
+      break if batch.size < PER_PAGE
+
+      page += 1
+    end
+
+    all_repos
+  end
+
+  def fetch_repos_page(page)
+    data = make_request("/orgs/#{@org}/repos", per_page: PER_PAGE, page: page)
+
+    # Transform API response to match expected format
+    data.map do |repo|
+      {
+        "name" => repo["name"],
+        "isArchived" => repo["archived"],
+        "visibility" => repo["visibility"],
+        "sshUrl" => repo["ssh_url"],
+        "url" => repo["html_url"]
+      }
+    end
+  end
+
+  def make_request(path, params = {})
+    uri = URI("https://api.github.com#{path}")
+    uri.query = URI.encode_www_form(params) unless params.empty?
+
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+
+    request = Net::HTTP::Get.new(uri)
+    request["Authorization"] = "Bearer #{@token}"
+    request["Accept"] = "application/vnd.github+json"
+    request["X-GitHub-Api-Version"] = "2022-11-28"
+
+    response = http.request(request)
+    handle_response(response)
+  end
+
+  def handle_response(response)
+    case response
+    when Net::HTTPSuccess
+      JSON.parse(response.body)
+    when Net::HTTPUnauthorized
+      raise "GitHub API error: 401 Unauthorized - Invalid or expired GITHUB_TOKEN"
+    when Net::HTTPForbidden
+      if response["X-RateLimit-Remaining"] == "0"
+        reset_time = Time.at(response["X-RateLimit-Reset"].to_i)
+        raise "GitHub API error: 403 Rate limit exceeded. Resets at #{reset_time}"
+      end
+      raise "GitHub API error: 403 Forbidden - Check token permissions"
+    when Net::HTTPNotFound
+      raise "GitHub API error: 404 Not Found - Organization '#{@org}' not found or not accessible"
+    else
+      raise "GitHub API error: #{response.code} #{response.message}"
+    end
+  end
+
+  def generate_repository_imports(repos)
+    yaml_documents = repos.map do |repo|
+      repo_name = repo["name"]
+      repo_path = File.join(@target_dir, repo_name)
+      visibility = (repo["visibility"] || "private").downcase
+      git_url = repo["sshUrl"] || repo["url"]
+
+      # Build annotations for child import
+      child_annotations = {}
+      child_annotations["import/outputPath"] = @repo_output_path if @repo_output_path
+      child_annotations["import/cacheTime"] = @child_cache_time if @child_cache_time
+
+      import_yaml(
+        name: "Import:Repo:github:#{@org}:#{repo_name}",
+        handler: "repository",
+        config: {
+          "path" => repo_path,
+          "gitUrl" => git_url,
+          "archived" => repo["isArchived"].to_s,
+          "visibility" => visibility == "public" ? "open-source" : "internal"
+        },
+        annotations: child_annotations
+      )
+    end
+
+    # Add self-marker with generated/at for caching
+    yaml_documents << self_marker
+
+    # Write all imports to a single file with --- separators
+    yaml_content = yaml_documents.map { |doc| YAML.dump(doc) }.join("\n")
+    write_yaml(yaml_content)
+  end
+end
+
+Archsight::Import::Registry.register("github", Archsight::Import::Handlers::Github)

data/lib/archsight/import/handlers/gitlab.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require "open3"
+require "fileutils"
+require "net/http"
+require "json"
+require "uri"
+require "openssl"
+require_relative "../handler"
+require_relative "../registry"
+
+# GitLab handler - lists repositories from a GitLab instance and generates child Import resources
+#
+# Configuration:
+#   import/config/host - GitLab host (e.g., gitlab.company.com)
+#   import/config/exploreGroups - If "true", explore all visible groups (default: false)
+#   import/config/perPage - API pagination page size (default: 100)
+#   import/config/verifySSL - If "false", disable SSL verification (default: true)
+#   import/config/sslFingerprint - SSL certificate fingerprint for pinning (SHA256, colon-separated hex)
+#   import/config/repoOutputPath - Output path for repository handler results (e.g., "generated/repositories.yaml")
+#   import/config/childCacheTime - Cache time for generated child imports (e.g., "1h", "30m")
+#
+# Environment:
+#   GITLAB_TOKEN - GitLab personal access token (required)
+#
+# Output:
+#   Generates Import:Repo:* resources for each repository
+#   The repository handler will clone/sync the actual git repositories (via SSH)
+class Archsight::Import::Handlers::Gitlab < Archsight::Import::Handler
+  def execute
+    @host = config("host")
+    raise "Missing required config: host" unless @host
+
+    @token = ENV.fetch("GITLAB_TOKEN", nil)
+    raise "Missing required environment variable: GITLAB_TOKEN" unless @token
+
+    @repo_output_path = config("repoOutputPath")
+    @child_cache_time = config("childCacheTime")
+
+    @target_dir = File.join(Dir.home, ".cache", "archsight", "git", "gitlab")
+    @explore_groups = config("exploreGroups") == "true"
+    @per_page = config("perPage", default: "100").to_i
+    @verify_ssl = config("verifySSL") != "false"
+    @ssl_fingerprint = config("sslFingerprint")
+
+    # Fetch all projects
+    progress.update("Fetching projects from #{@host}")
+    projects = fetch_all_projects
+
+    if projects.empty?
+      progress.warn("No projects found on GitLab")
+      return
+    end
+
+    # Generate Import resources for each repository
+    progress.update("Generating #{projects.size} import resources")
+    generate_repository_imports(projects)
+
+    write_generates_meta
+  end
+
+  private
+
+  def api_endpoint
+    "https://#{@host}/api/v4"
+  end
+
+  def make_request(path, params = {})
+    uri = URI("#{api_endpoint}#{path}")
+    uri.query = URI.encode_www_form(params) unless params.empty?
+
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+
+    # Configure SSL verification
+    if @ssl_fingerprint
+      # Use certificate pinning - disable default verification, we verify fingerprint manually
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      http.verify_callback = lambda do |_preverify_ok, cert_store|
+        # Get the peer certificate from the chain
+        cert = cert_store.chain&.first
+        return true unless cert # Allow if no cert (will fail later)
+
+        # Compute SHA256 fingerprint
+        fingerprint = OpenSSL::Digest::SHA256.new(cert.to_der).to_s.upcase.scan(/../).join(":")
+        expected = @ssl_fingerprint.upcase
+
+        raise OpenSSL::SSL::SSLError, "Certificate fingerprint mismatch! Expected: #{expected}, Got: #{fingerprint}" if fingerprint != expected
+
+        true
+      end
+    else
+      http.verify_mode = @verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+    end
+
+    request = Net::HTTP::Get.new(uri)
+    request["PRIVATE-TOKEN"] = @token
+
+    response = http.request(request)
+
+    raise "GitLab API error: #{response.code} #{response.message}" unless response.is_a?(Net::HTTPSuccess)
+
+    JSON.parse(response.body)
+  end
+
+  def fetch_all_groups
+    groups = []
+    page = 1
+
+    loop do
+      progress.update("Fetching groups (page #{page})")
+      params = { per_page: @per_page, page: page }
+      params[:all_available] = true if @explore_groups
+
+      batch = make_request("/groups", params)
+      break if batch.empty?
+
+      groups.concat(batch)
+      page += 1
+    end
+
+    groups
+  end
+
+  def fetch_projects_for_group(group_id)
+    projects = []
+    page = 1
+
+    loop do
+      params = {
+        include_subgroups: true,
+        per_page: @per_page,
+        page: page,
+        order_by: "created_at"
+      }
+
+      batch = make_request("/groups/#{group_id}/projects", params)
+      break if batch.empty?
+
+      projects.concat(batch)
+      page += 1
+    end
+
+    projects
+  end
+
+  def fetch_all_projects
+    all_projects = []
+    groups = fetch_all_groups
+
+    groups.each_with_index do |group, idx|
+      progress.update("Fetching projects from #{group["full_path"]}", current: idx + 1, total: groups.size)
+      projects = fetch_projects_for_group(group["id"])
+      all_projects.concat(projects)
+    end
+
+    # Deduplicate by project ID
+    all_projects.uniq { |p| p["id"] }
+  end
+
+  def safe_dir_name(path)
+    path.gsub("/", ".")
+  end
+
+  def git_url_for(project)
+    project["ssh_url_to_repo"]
+  end
+
+  def generate_repository_imports(projects)
+    yaml_documents = projects.map do |project|
+      dir_name = safe_dir_name(project["path_with_namespace"])
+      repo_path = File.join(@target_dir, dir_name)
+      git_url = git_url_for(project)
+
+      # Build annotations for child import
+      child_annotations = {}
+      child_annotations["import/outputPath"] = @repo_output_path if @repo_output_path
+      child_annotations["import/cacheTime"] = @child_cache_time if @child_cache_time
+
+      import_yaml(
+        name: "Import:Repo:gitlab:#{dir_name}",
+        handler: "repository",
+        config: {
+          "path" => repo_path,
+          "gitUrl" => git_url,
+          "archived" => project["archived"].to_s,
+          "visibility" => project["visibility"] || "internal"
+        },
+        annotations: child_annotations
+      )
+    end
+
+    # Add self-marker with generated/at for caching
+    yaml_documents << self_marker
+
+    # Write all imports to a single file with --- separators
+    yaml_content = yaml_documents.map { |doc| YAML.dump(doc) }.join("\n")
+    write_yaml(yaml_content)
+  end
+end
+
+Archsight::Import::Registry.register("gitlab", Archsight::Import::Handlers::Gitlab)