archsight 0.3.2 → 0.4.0

data/lib/archsight/import/handlers/command.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require "open3"
+require "shellwords"
+require "tmpdir"
+require "yaml"
+require_relative "../handler"
+require_relative "../registry"
+
+# Command handler - runs a program of the resources repository as an import, in any language. The program writes its
+# resources itself (usually to ARCHSIGHT_OUTPUT_PATH); archsight adds the self marker, so caching, `import/dependsOn`,
+# `-f` and `-n` work as for any other import.
+#
+# The program runs in the root of the repository that holds the resources directory (the resources directory when it
+# is not in a git repository), with a clean environment: PATH, HOME, LANG and the variables below, plus the variables
+# listed in `import/config/env` (credentials, for example).
+#
+#   ARCHSIGHT_RESOURCES_DIR - the resources directory
+#   ARCHSIGHT_CACHE_DIR     - where the git-based imports clone repositories
+#   ARCHSIGHT_IMPORT_NAME   - the name of the Import
+#   ARCHSIGHT_OUTPUT_PATH   - a file for the YAML resources the program generates; archsight writes them unchanged to
+#                             import/outputPath (generated/<name>.yaml without one), followed by the import's marker
+#
+# The output file belongs to the import: it is replaced on every run and removed when outputPath changes. Documents
+# are not marked as generated; a program that wants that sets `generated/script: <import name>` itself (only on kinds
+# that have the `generated` annotations), while documents that add relations to hand-written resources stay unmarked.
+#
+# Configuration:
+#   import/config/command - the command line (split like a shell does, but run without one; use `sh -c '...'` for
+#                           pipes); a relative program path is relative to the working directory
+#   import/config/env     - names of further environment variables to pass, comma separated
+#   import/config/outputs - files the program writes besides the output path, glob patterns relative to the
+#                           resources directory, comma separated; recorded like Handler#write_file
+#   import/config/workdir - working directory relative to the resources directory (default: the repository root)
+class Archsight::Import::Handlers::Command < Archsight::Import::Handler
+  PASSED_ENV = %w[PATH HOME LANG LC_ALL TMPDIR].freeze
+
+  def execute
+    command = Shellwords.split(config("command").to_s)
+    raise "Missing required config: command" if command.empty?
+
+    Dir.mktmpdir("archsight-command") do |tmp|
+      output_path = File.join(tmp, "output.yaml")
+      progress.update("Running #{command.first}")
+      out, err, status = Open3.capture3(environment(output_path), *command, chdir: working_directory, unsetenv_others: true)
+      raise "Command failed (exit #{status.exitstatus}): #{last_lines(err.empty? ? out : err)}" unless status.success?
+
+      record_declared_outputs
+      write_output(program_output(output_path))
+    end
+  rescue Errno::ENOENT => e
+    raise "Command not found: #{e.message}"
+  end
+
+  private
+
+  # What the program wrote to ARCHSIGHT_OUTPUT_PATH (a temporary file), as written: it must be YAML documents that
+  # are mappings, and stays byte for byte as it is (comments, order, no added annotations)
+  def program_output(output_path)
+    return "" unless File.file?(output_path)
+
+    text = File.read(output_path)
+    YAML.load_stream(text).compact.each do |document|
+      raise "#{import_resource.name}: the command wrote a document that is not a resource" unless document.is_a?(Hash)
+    end
+    text.end_with?("\n") || text.empty? ? text : "#{text}\n"
+  rescue Psych::SyntaxError => e
+    raise "The command wrote invalid YAML: #{e.message}"
+  end
+
+  # The output file is the import's own: written whole (not through the shared writer, which would drop the
+  # unmarked documents when another import flushes the file) and recorded as an output, so a changed outputPath
+  # removes the old file. Recorded before the marker is built, so the marker lists it.
+  def write_output(text)
+    rel_path = Archsight::Import.output_path(import_resource)
+    record_output(rel_path)
+    write_file(rel_path, text + YAML.dump(self_marker))
+  end
+
+  def environment(output_path)
+    passed = PASSED_ENV + config("env").to_s.split(",").map(&:strip).reject(&:empty?)
+    env = passed.to_h { |name| [name, ENV.fetch(name, nil)] }.compact
+    env.merge(
+      "ARCHSIGHT_RESOURCES_DIR" => File.expand_path(resources_dir),
+      "ARCHSIGHT_CACHE_DIR" => cache_root,
+      "ARCHSIGHT_IMPORT_NAME" => import_resource.name,
+      "ARCHSIGHT_OUTPUT_PATH" => File.expand_path(output_path)
+    )
+  end
+
+  def working_directory
+    workdir = config("workdir")
+    if workdir
+      root = File.expand_path(resources_dir)
+      path = File.expand_path(workdir, root)
+      return path if path == root || path.start_with?("#{root}/")
+
+      raise "workdir #{workdir} is outside the resources directory"
+    end
+
+    toplevel, _, status = Open3.capture3("git", "rev-parse", "--show-toplevel", chdir: resources_dir)
+    status.success? ? toplevel.strip : File.expand_path(resources_dir)
+  rescue Errno::ENOENT
+    File.expand_path(resources_dir)
+  end
+
+  def record_declared_outputs
+    patterns = config("outputs").to_s.split(",").map(&:strip).reject(&:empty?)
+    root = File.expand_path(resources_dir)
+    patterns.flat_map { |pattern| Dir.glob(pattern, base: root) }.uniq.each do |file|
+      record_output(file) if File.file?(File.join(root, file))
+    end
+  end
+
+  def last_lines(text)
+    text.to_s.lines.last(5).join.strip
+  end
+end
+
+Archsight::Import::Registry.register("command", Archsight::Import::Handlers::Command)

data/lib/archsight/import/handlers/github.rb

@@ -35,6 +35,9 @@ require_relative "cache_pruner"
 # the API (e.g. a repo that was renamed or deleted), so stale clones don't
 # linger and get picked up by later importers.
 class Archsight::Import::Handlers::Github < Archsight::Import::Handler
+  # generates child imports (see Handler.generates_imports?)
+  def self.generates_imports? = true
+
   include Archsight::Import::Handlers::CachePruner
 
   PER_PAGE = 100
@@ -49,7 +52,7 @@ class Archsight::Import::Handlers::Github < Archsight::Import::Handler
     @repo_output_path = config("repoOutputPath")
     @child_cache_time = config("childCacheTime")
     @default_visibility = config("defaultVisibility", default: "internal")
-    cache_root = config("cacheRoot", default: File.join(Dir.home, ".cache", "archsight", "git"))
+    cache_root = Archsight::Import.cache_root(config("cacheRoot"))
     @target_dir = File.join(cache_root, "github", @org)
 
     # Fetch all repositories with pagination

data/lib/archsight/import/handlers/gitlab.rb

@@ -38,6 +38,9 @@ require_relative "cache_pruner"
 # API (e.g. a project that was renamed, moved, or deleted), so stale clones
 # don't linger and get picked up by later importers.
 class Archsight::Import::Handlers::Gitlab < Archsight::Import::Handler
+  # generates child imports (see Handler.generates_imports?)
+  def self.generates_imports? = true
+
   include Archsight::Import::Handlers::CachePruner
 
   def execute
@@ -51,7 +54,7 @@ class Archsight::Import::Handlers::Gitlab < Archsight::Import::Handler
     @child_cache_time = config("childCacheTime")
     @default_visibility = config("defaultVisibility", default: "internal")
 
-    cache_root = config("cacheRoot", default: File.join(Dir.home, ".cache", "archsight", "git"))
+    cache_root = Archsight::Import.cache_root(config("cacheRoot"))
     @target_dir = File.join(cache_root, "gitlab")
     @explore_groups = config("exploreGroups") == "true"
     @per_page = config("perPage", default: "100").to_i

data/lib/archsight/import/handlers/go_dep_resolver.rb

@@ -20,13 +20,19 @@ require_relative "../registry"
 #
 # Configuration:
 #   import/config/path - Path to the Go repository root (same as GoGrapher)
+#   import/config/artifact - Repository artifact the modules belong to; names the component when two repositories
+#                            declare the same module path (see GoModuleParser#owned_component_name)
 class Archsight::Import::Handlers::GoDepResolver < Archsight::Import::Handler
   include Archsight::Import::Handlers::GoModuleParser
 
   def execute
     @path = config_path("path")
     raise "Missing required config: path" unless @path
-    raise "Directory not found: #{@path}" unless File.directory?(@path)
+
+    unless File.directory?(@path)
+      progress.warn("Skipping, repository not cloned: #{@path}")
+      return
+    end
 
     import_resource.annotations["import/outputPath"] ||= "generated/relationships.yaml"
 
@@ -37,11 +43,13 @@ class Archsight::Import::Handlers::GoDepResolver < Archsight::Import::Handler
     output = +""
 
     modules.each do |rel_dir, mod_name|
+      next if non_runtime_module?(rel_dir) || copied_module?(rel_dir, mod_name, config("artifact"))
+
       mod_dir = rel_dir == "." ? @path : File.join(@path, rel_dir)
       dep_names = compute_deps(mod_dir, mod_name, existing_components)
       next if dep_names.empty?
 
-      comp_name = component_name(mod_name)
+      comp_name = owned_component_name(mod_name, config("artifact"))
       existing = existing_components[comp_name]
       next unless existing
 

data/lib/archsight/import/handlers/go_grapher.rb

@@ -23,8 +23,18 @@ require_relative "../registry"
 #   import/config/nodesep              - Vertical gap between nodes in a column (default: 0.15)
 #   import/config/interface_visibility - Visibility prefix for detected interfaces (default: Private)
 class Archsight::Import::Handlers::GoGrapher < Archsight::Import::Handlers::Grapher
+  # generates child imports (see Handler.generates_imports?)
+  def self.generates_imports? = true
+
   include Archsight::Import::Handlers::GoModuleParser
 
+  # Directories not scanned for Go packages
+  PACKAGE_SKIP_DIRS = %w[vendor testdata node_modules].freeze
+  # Packages below these directories are not meant to be imported by other modules
+  NON_LIBRARY_DIRS = %w[internal cmd hack tools examples example e2e test tests].freeze
+  MAX_ENTRYPOINTS = 10
+  IGNORED_BUILD = %r{^(//go:build\s+ignore\b|//\s*\+build\s+ignore\b)}
+
   OPENAPI_FILENAMES = %w[openapi.yaml openapi.yml openapi.json swagger.yaml swagger.yml swagger.json].freeze
   OPENAPI_SUBDIRS   = %w[api docs spec].freeze
 
@@ -55,7 +65,14 @@ class Archsight::Import::Handlers::GoGrapher < Archsight::Import::Handlers::Grap
     output = +""
 
     modules.each do |rel_dir, mod_name|
-      comp_name = component_name(mod_name)
+      next if non_runtime_module?(rel_dir)
+
+      if copied_module?(rel_dir, mod_name, artifact_name)
+        progress.warn("Skipping copied go.mod at #{rel_dir}: module #{mod_name} belongs to another repository")
+        next
+      end
+
+      comp_name = owned_component_name(mod_name, artifact_name)
       next unless comp_name
 
       mod_dir = rel_dir == "." ? path : File.join(path, rel_dir)
@@ -64,6 +81,9 @@ class Archsight::Import::Handlers::GoGrapher < Archsight::Import::Handlers::Grap
       interface_names = specs.filter_map { |s| interface_name_from_spec(s, visibility: visibility) }
                              .select { |n| existing_interfaces.key?(n) }
 
+      existing = database&.instances_by_kind("ApplicationComponent")&.[](comp_name)
+      facts = curated_component_type?(existing) ? {} : component_facts(path, rel_dir, mod_name)
+
       comp_spec = { "realizedThrough" => { "technologyArtifacts" => [artifact_name] } }
       comp_spec["exposes"] = { "applicationInterfaces" => interface_names } if interface_names.any?
 
@@ -73,7 +93,8 @@ class Archsight::Import::Handlers::GoGrapher < Archsight::Import::Handlers::Grap
       component = untracked_resource_yaml(
         kind: "ApplicationComponent",
         name: comp_name,
-        spec: comp_spec
+        spec: comp_spec,
+        annotations: component_annotations(facts, existing_tags: existing&.annotations&.[]("architecture/tags"))
       )
       output << YAML.dump(component)
     end
@@ -85,7 +106,7 @@ class Archsight::Import::Handlers::GoGrapher < Archsight::Import::Handlers::Grap
     output << YAML.dump(import_yaml(
                           name: resolver_name,
                           handler: "go-dep-resolver",
-                          config: { "path" => path }
+                          config: { "path" => path, "artifact" => artifact_name }
                         ))
 
     output
@@ -130,7 +151,83 @@ class Archsight::Import::Handlers::GoGrapher < Archsight::Import::Handlers::Grap
 
   # ── OpenAPI detection ─────────────────────────────────────────────────────
 
-  def untracked_resource_yaml(kind:, name:, spec: {})
+  # A component type another source set (a person or an earlier import) is kept; this import's own earlier
+  # output is regenerated.
+  def curated_component_type?(existing)
+    return false unless existing&.annotations&.key?("component/type")
+
+    existing.annotations["generated/script"] != import_resource.name
+  end
+
+  # Classify a go.mod module from its packages: with `package main` it is an executable, without it a library,
+  # and a module with both main packages and importable packages is a module.
+  def component_facts(repo_root, rel_dir, _mod_name)
+    mod_dir = rel_dir == "." ? repo_root : File.join(repo_root, rel_dir)
+    mains, libraries = scan_go_packages(mod_dir)
+    type = if mains.empty?
+             "library"
+           elsif libraries.empty?
+             "executable"
+           else
+             "module"
+           end
+
+    { type: type,
+      tags: %w[ecosystem:go packaging:go-module], entrypoints: mains.first(MAX_ENTRYPOINTS) }
+  end
+
+  # @return [Array<Array<String>>] [main package dirs, importable library package dirs], relative to mod_dir
+  def scan_go_packages(mod_dir)
+    mains = []
+    libraries = []
+
+    Find.find(mod_dir) do |path|
+      if File.directory?(path)
+        Find.prune if path != mod_dir && skip_go_dir?(path)
+        next
+      end
+      next unless go_source?(path)
+
+      name = go_package_name(path)
+      next unless name
+
+      dir = File.dirname(path).delete_prefix(mod_dir).delete_prefix("/")
+      dir = "." if dir.empty?
+      if name == "main"
+        mains << dir
+      elsif !dir.split("/").intersect?(NON_LIBRARY_DIRS)
+        libraries << dir
+      end
+    end
+
+    [mains.uniq.sort, libraries.uniq.sort]
+  end
+
+  # Hidden and vendored directories, and nested modules (they are components of their own)
+  def skip_go_dir?(path)
+    base = File.basename(path)
+    PACKAGE_SKIP_DIRS.include?(base) || base.start_with?(".") || File.exist?(File.join(path, "go.mod"))
+  end
+
+  def go_source?(path)
+    path.end_with?(".go") && !path.end_with?("_test.go")
+  end
+
+  # Package name of a Go file, nil for a file that is excluded from normal builds (`//go:build ignore`)
+  def go_package_name(path)
+    head = File.foreach(path, encoding: "utf-8").first(40)
+    return nil if head.any? { |line| line.match?(IGNORED_BUILD) }
+
+    head.each do |line|
+      match = line.match(/\Apackage\s+(\w+)/)
+      return match[1] if match
+    end
+    nil
+  rescue SystemCallError, ArgumentError
+    nil
+  end
+
+  def untracked_resource_yaml(kind:, name:, spec: {}, annotations: {})
     {
       "apiVersion" => "architecture/v1alpha1",
       "kind" => kind,
@@ -139,7 +236,7 @@ class Archsight::Import::Handlers::GoGrapher < Archsight::Import::Handlers::Grap
         "annotations" => {
           "generated/script" => import_resource.name,
           "generated/at" => Time.now.utc.iso8601
-        }
+        }.merge(annotations)
       },
       "spec" => spec
     }

data/lib/archsight/import/handlers/go_module_parser.rb

@@ -7,6 +7,13 @@ require "find"
 # Provides go.mod discovery, module name reading, require parsing, and the
 # component_name convention (strip SCM host, join path segments with ":").
 module Archsight::Import::Handlers::GoModuleParser
+  # Directories whose nested modules hold code that never runs in production: tests, pinned tooling, examples and
+  # benchmarks. Whole path segments only, so `pkg/latest` is not a test directory.
+  NON_RUNTIME_MODULE_DIRS = %w[
+    hack tools examples example bench benchmarks
+    test tests e2e e2etest e2etests systemtest systemtests integrationtest integrationtests fullstack
+  ].freeze
+
   # Discover all Go modules in a repository root.
   # Handles go.work workspaces, single-module repos, and multi-module monorepos
   # (root go.mod plus subdirectory go.mod files without go.work).
@@ -141,4 +148,85 @@ module Archsight::Import::Handlers::GoModuleParser
     parts.shift
     parts.join(":") unless parts.empty?
   end
+
+  # The name of the component that the module of one repository gets. Two repositories may declare the same module
+  # path (a prototype and its successor, a fork, a copy of the module); without care both would emit the same
+  # component and the loader would merge them into one. The repository that owns the plain name is the one the module
+  # path names (see #module_path_of_repository?), then the one that is not archived, then the first by name; the others
+  # get the repository appended (`alice:shop@acme:shop-prototype`).
+  #
+  # The other repositories are found through the existing component of the plain name, so a collision is resolved on
+  # the import after both repositories were imported once. A component without `generated/script` is curated: a
+  # person deliberately lists several repositories, and it keeps the plain name.
+  # @return [String, nil]
+  def owned_component_name(mod_name, artifact_name)
+    base = component_name(mod_name)
+    return base unless base && artifact_name
+
+    others = colliding_artifacts(base, artifact_name)
+    return base if others.empty?
+
+    owner = ([artifact_name] + others).min_by do |name|
+      [module_path_of_repository?(mod_name, name) ? 0 : 1, archived_artifact?(name) ? 1 : 0, name]
+    end
+    owner == artifact_name ? base : "#{base}@#{artifact_name.delete_prefix("Repo:")}"
+  end
+
+  # Whether the module path names the repository: the repository is the module itself or a module below it.
+  # `github.com/acme/library` and `github.com/acme/library/v2/client` belong to `Repo:acme:library` (and to
+  # `Repo:library`, an artifact named after its directory), not to `Repo:acme:app-poc`. The segments the artifact name
+  # and the module path both have must agree, so `github.com/alice/shop` does not name `Repo:acme:shop`.
+  # @return [Boolean]
+  def module_path_of_repository?(mod_name, artifact_name)
+    repo = artifact_name.delete_prefix("Repo:").split(":").reject(&:empty?)
+    path = module_path_segments(mod_name)
+    return false if repo.empty? || path.empty?
+
+    path.each_index.any? do |i|
+      next false if i.zero? && path.length > 1 # the first segment is the owner, never the repository
+
+      overlap = [repo.length, i + 1].min
+      repo.last(overlap) == path[(i + 1 - overlap)..i]
+    end
+  end
+
+  # A module in a subdirectory of `artifact_name` that is a copy of another repository's module: its path does not
+  # name this repository, but it names another repository of the database. It gets no component; the repository it
+  # was copied from has it.
+  # @return [Boolean]
+  def copied_module?(rel_dir, mod_name, artifact_name)
+    return false if rel_dir == "." || artifact_name.nil? || module_path_of_repository?(mod_name, artifact_name)
+
+    artifacts = database&.instances_by_kind("TechnologyArtifact") || {}
+    artifacts.keys.any? { |name| name.start_with?("Repo:") && name != artifact_name && module_path_of_repository?(mod_name, name) }
+  end
+
+  # A nested module below a test, tool, example or benchmark directory (`test/e2e`, `build/tools`,
+  # `internal/systemtest`). It gets no component, since nothing realizes it or depends on it; its packages stay in
+  # the module graph of the repository. `cmd/<name>` and `internal/` hold runtime code and are not matched.
+  # @return [Boolean]
+  def non_runtime_module?(rel_dir)
+    return false if rel_dir == "."
+
+    rel_dir.split("/").intersect?(NON_RUNTIME_MODULE_DIRS)
+  end
+
+  # Path segments of a module without the host and without a major version suffix (`/v2`)
+  def module_path_segments(mod_name)
+    mod_name.split("/").drop(1).grep_v(/\Av\d+\z/)
+  end
+
+  # Repository artifacts other than `artifact_name` that a generated component of this name is realized through
+  def colliding_artifacts(base, artifact_name)
+    existing = database&.instances_by_kind("ApplicationComponent")&.[](base)
+    return [] unless existing&.annotations&.key?("generated/script")
+
+    artifacts = Array(existing.raw.dig("spec", "realizedThrough", "technologyArtifacts"))
+    artifacts.select { |name| name.start_with?("Repo:") && name != artifact_name }.uniq
+  end
+
+  def archived_artifact?(name)
+    artifact = database&.instances_by_kind("TechnologyArtifact")&.[](name)
+    artifact&.annotations&.[]("activity/status") == "archived"
+  end
 end

data/lib/archsight/import/handlers/grapher.rb

@@ -35,6 +35,11 @@ class Archsight::Import::Handlers::Grapher < Archsight::Import::Handler
     []
   end
 
+  # Namespaces of the machine tags (namespace:value) that importers write into architecture/tags.
+  # `entrypoint` is no longer written (see component/entrypoints) but stays listed so a re-import
+  # removes the tags of earlier runs.
+  MACHINE_TAG_NAMESPACES = %w[ecosystem packaging entrypoint linkage].freeze
+
   PALETTE = [
     { fill: "#ddeeff", edge: "#2266cc" },
     { fill: "#ddffd8", edge: "#2a8a1e" },
@@ -51,7 +56,13 @@ class Archsight::Import::Handlers::Grapher < Archsight::Import::Handler
   def execute
     @path = config_path("path")
     raise "Missing required config: path" unless @path
-    raise "Directory not found: #{@path}" unless File.directory?(@path)
+
+    unless File.directory?(@path)
+      # The clone is the repository import's job (it reports its own failure);
+      # no marker is written, so this runs again once the clone exists.
+      progress.warn("Skipping, repository not cloned: #{@path}")
+      return
+    end
 
     # Consolidate all grapher output into a single shared file unless an
     # explicit output path is already set (e.g. via grapherOutputPath on the
@@ -123,6 +134,34 @@ class Archsight::Import::Handlers::Grapher < Archsight::Import::Handler
     raise NotImplementedError, "#{self.class}#collect_packages must be implemented"
   end
 
+  # Hook: subclasses that emit ApplicationComponents describe each module here.
+  # @param _repo_root [String] Absolute path to the repository root
+  # @param _rel_dir [String] Directory of the module relative to the root ("." for the root)
+  # @param _mod_name [String] Module name from discover_modules
+  # @return [Hash] { type: "executable|library|module|...", tags: ["ecosystem:go", ...],
+  #   entrypoints: ["cmd/api", ...] }, empty when unknown
+  def component_facts(_repo_root, _rel_dir, _mod_name)
+    {}
+  end
+
+  # component/type, component/entrypoints and the machine tags (namespace:value) in architecture/tags for the
+  # facts of component_facts.
+  # The tags a person set stay; machine tags of the importer's namespaces from an earlier run are replaced.
+  def component_annotations(facts, existing_tags: nil)
+    annotations = {}
+    annotations["component/type"] = facts[:type] if facts[:type]
+    annotations["component/entrypoints"] = facts[:entrypoints].join(",") if facts[:entrypoints]&.any?
+    annotations["architecture/tags"] = merge_machine_tags(existing_tags, facts[:tags]).join(",") if facts[:tags]&.any?
+    annotations
+  end
+
+  def merge_machine_tags(existing_tags, machine_tags)
+    kept = existing_tags.to_s.split(",").map(&:strip).reject(&:empty?).reject do |tag|
+      MACHINE_TAG_NAMESPACES.include?(tag.split(":", 2).first) && tag.include?(":")
+    end
+    (kept + Array(machine_tags)).uniq
+  end
+
   # ── Module coloring ───────────────────────────────────────────────────────
 
   def build_module_colors(modules)

data/lib/archsight/import/handlers/repository.rb

@@ -22,6 +22,9 @@ require_relative "../team_matcher"
 #   import/config/corporateAffixes - Optional comma-separated corporate username affixes for team matching (e.g., "ionos,1and1")
 #   import/config/grapherOutputPath - Optional output path for language-grapher child imports
 class Archsight::Import::Handlers::Repository < Archsight::Import::Handler
+  # generates child imports (see Handler.generates_imports?)
+  def self.generates_imports? = true
+
   def execute
     @path = config_path("path")
     @git_url = config("gitUrl")

data/lib/archsight/import/handlers/rest_api_index.rb

@@ -43,6 +43,9 @@ require_relative "../registry"
 #     ]
 #   }
 class Archsight::Import::Handlers::RestApiIndex < Archsight::Import::Handler
+  # generates child imports (see Handler.generates_imports?)
+  def self.generates_imports? = true
+
   VALID_GATES = %w[General-Availability Early-Access Development GA EA DEV].freeze
   GATE_TOKEN_RE = /\b(ga|ea|dev)\b/i
 

data/lib/archsight/import/registry.rb

@@ -8,13 +8,25 @@ require "archsight/import"
 # The executor looks up handlers by name to instantiate and execute them.
 module Archsight::Import::Registry
   @handlers = {}
+  @builtin = {}
 
   class << self
     # Register a handler class with a name
     # @param name [String, Symbol] Handler name (matches import/handler annotation)
     # @param handler_class [Class] Handler class that extends Handler
+    # @raise [DuplicateHandlerError] if another class already registered the name
     def register(name, handler_class)
-      @handlers[name.to_s] = handler_class
+      name = name.to_s
+      existing = @handlers[name] || @builtin[name]
+      raise Archsight::Import::DuplicateHandlerError, "Import handler '#{name}' is registered twice (#{existing} and #{handler_class})" if existing && existing != handler_class
+
+      @handlers[name] = handler_class
+      @builtin[name] = handler_class if builtin_caller?(caller_locations(1, 1)&.first)
+    end
+
+    # Register the built-in handlers again after `clear!` (they are only required once per process)
+    def restore_builtin!
+      @handlers = @builtin.merge(@handlers)
     end
 
     # Look up a handler class by name
@@ -70,8 +82,17 @@ module Archsight::Import::Registry
     def clear!
       @handlers = {}
     end
+
+    private
+
+    def builtin_caller?(location)
+      location && File.expand_path(location.path).start_with?("#{File.expand_path("handlers", __dir__)}/")
+    end
   end
 end
 
 # Error raised when an unknown handler is requested
 class Archsight::Import::UnknownHandlerError < StandardError; end
+
+# Error raised when two handler classes register the same name
+class Archsight::Import::DuplicateHandlerError < StandardError; end

data/lib/archsight/import.rb

@@ -5,10 +5,73 @@ module Archsight
   module Import
     module Handlers
     end
+
+    # Where the git-based handlers clone repositories: the handler's own `cacheRoot` setting, else `import.cache_root`
+    # of the user configuration (or ARCHSIGHT_IMPORT_CACHE_ROOT), else ~/.cache/archsight/git
+    # @param override [String, nil] a handler's own setting
+    # @return [String] the directory as configured (a `~` stays, so generated child imports keep the portable form;
+    #   expand it before use)
+    def self.cache_root(override = nil)
+      require "archsight/user_config"
+      override || Archsight::UserConfig.setting("import", "cache_root")&.to_s ||
+        File.join(Dir.home, ".cache", "archsight", "git")
+    end
+
+    # The entries of `import/dependsOn`: import names and glob patterns
+    # @return [Array<String>]
+    def self.declared_dependencies(import)
+      import.annotations["import/dependsOn"].to_s.split(",").map(&:strip).reject(&:empty?)
+    end
+
+    # Absolute path of a file below the resources directory, nil when the path leaves it
+    def self.resources_path(resources_dir, rel_path)
+      root = File.expand_path(resources_dir)
+      path = File.expand_path(rel_path.to_s, root)
+      path.start_with?("#{root}/") ? path : nil
+    end
+
+    # The file an import writes its resources to, relative to the resources directory: `import/outputPath`, else
+    # generated/<name>.yaml (see Handler#write_yaml)
+    def self.output_path(import)
+      import.annotations["import/outputPath"] || File.join("generated", "#{import.name.gsub(/[^a-zA-Z0-9_-]/, "_")}.yaml")
+    end
+
+    # The files an import wrote, relative to the resources directory: its output path and the `generated/outputs`
+    # of its last run. Tells which import a resource of a generated file comes from.
+    def self.output_files(import)
+      [output_path(import), *import.annotations["generated/outputs"].to_s.split(",").map(&:strip)].reject(&:empty?).uniq
+    end
+
+    # Hash of the files `import/config/inputs` matches (glob patterns relative to the resources directory, comma
+    # separated): their paths, sizes and modification times. nil when the import declares no inputs.
+    # @return [String, nil]
+    def self.inputs_hash(import, resources_dir)
+      patterns = import.annotations["import/config/inputs"].to_s.split(",").map(&:strip).reject(&:empty?)
+      return nil if patterns.empty?
+
+      root = File.expand_path(resources_dir)
+      files = patterns.flat_map { |pattern| Dir.glob(pattern, base: root) }.uniq.sort
+      stats = files.filter_map do |file|
+        stat = File.stat(File.join(root, file))
+        "#{file}:#{stat.size}:#{stat.mtime.to_f}" if stat.file?
+      end
+      require "digest"
+      Digest::SHA256.hexdigest(stats.join("\n"))[0, 16]
+    end
+
+    # Whether a `import/dependsOn` entry is a glob pattern (`Import:Repo:*`)
+    def self.pattern?(entry)
+      entry.match?(/[*?\[]/)
+    end
+
+    # Whether an import name matches a `import/dependsOn` entry
+    def self.dependency_match?(entry, name)
+      pattern?(entry) ? File.fnmatch?(entry, name, File::FNM_EXTGLOB) : entry == name
+    end
   end
 end
 
 require_relative "import/registry"
 require_relative "import/handler"
 require_relative "import/executor"
-# Handlers are loaded by CLI's require_import_handlers method
+require_relative "import/handler_loader"