archsight 0.3.3 → 0.4.1

data/lib/archsight/export/confluence/storage.rb

@@ -3,8 +3,7 @@
 require "erb"
 require "uri"
 require "digest"
-require "kramdown"
-require "kramdown-parser-gfm"
+require_relative "../../markdown"
 require_relative "../../assets"
 require_relative "../../diagram"
 require_relative "../../helpers/macros"
@@ -17,6 +16,7 @@ require_relative "drawio"
 require_relative "page_url"
 require_relative "rasterizer"
 require_relative "tables"
+require_relative "analysis_output"
 
 module Archsight
   module Export
@@ -79,8 +79,9 @@ module Archsight
 
         MXFILE = "application/vnd.jgraph.mxfile"
 
-        BANNER = "This page is generated by Archsight from <code>%<source>s</code>. " \
-                 "Changes made here are overwritten by the next export: edit the source instead."
+        HOMEPAGE = "https://github.com/ionos-cloud/archsight"
+        BANNER = "This page is generated by <a href=\"#{HOMEPAGE}\">Archsight</a> from %<source>s. " \
+                 "Changes made here are overwritten by the next export: edit the source instead.".freeze
 
         LANGUAGES = {
           "sh" => "bash", "shell" => "bash", "zsh" => "bash", "yml" => "yaml", "json" => "javascript", "js" => "javascript",
@@ -94,6 +95,8 @@ module Archsight
 
         # @param page_name [String] used to name generated attachments
         # @param source [String] the markdown file, shown in the banner
+        # @param git [Archsight::GitSource::Location, nil] where the file is in its git repository: the banner then
+        #   links it to the file in the web UI of the git host and names the repository
         # @param base_dir [String] directory of the markdown file relative to the resources directory
         # @param resources_dir [String]
         # @param wiki [Archsight::Helpers::WikiLinks] resolves [[Targets]]
@@ -102,8 +105,10 @@ module Archsight
         # @param database [Archsight::Database, nil] lets macros that name pages (pagetree) find them
         # @param diagram_links [#call, nil] resolves a diagram's `resource` references (DiagramLinks); none leaves them inert
         # @param drawio [Boolean] the Confluence has the draw.io app: diagrams go in as draw.io macros, else as images
-        def initialize(page_name:, source:, base_dir:, resources_dir:, wiki:, page_id: nil, base: nil, drawio: false, diagram_links: nil, database: nil)
+        def initialize(page_name:, source:, base_dir:, resources_dir:, wiki:, page_id: nil, base: nil, drawio: false, diagram_links: nil, database: nil,
+                       git: nil)
           @database = database
+          @git = git
           @page_name = page_name
           @source = source
           @base_dir = base_dir
@@ -124,7 +129,7 @@ module Archsight
         # @param header [String] storage format of the page properties, shown top left next to the banner
         # @return [Converted]
         def convert(markdown, toc: false, header: "")
-          document = Kramdown::Document.new(protect(markdown), input: "GFM", auto_ids: false, entity_output: :as_char, smart_quotes: %w[apos apos quot quot])
+          document = Archsight::Markdown.unwrapped(protect(markdown), auto_ids: false, entity_output: :as_char, smart_quotes: %w[apos apos quot quot])
           html, = Converter.convert(document.root, document.options.merge(storage: self))
           Converted.new(body: layout(header, banner, (toc ? %(<ac:structured-macro ac:name="toc" />\n) : "") + html), attachments: @attachments, problems: @problems)
         end
@@ -225,13 +230,45 @@ module Archsight
         end
 
         def banner
-          %(<ac:structured-macro ac:name="info"><ac:rich-text-body><p>#{format(BANNER, source: h(@source))}</p></ac:rich-text-body></ac:structured-macro>\n)
+          %(<ac:structured-macro ac:name="info"><ac:rich-text-body><p>#{format(BANNER, source: banner_source)}</p></ac:rich-text-body></ac:structured-macro>\n)
+        end
+
+        # `<code>path</code>`, in a git repository the path from its root, linked to the file on the git host, and
+        # the repository
+        def banner_source
+          return "<code>#{h(@source)}</code>" unless @git
+
+          file = "<code>#{h(@git.path)}</code>"
+          file = %(<a href="#{h(@git.url)}">#{file}</a>) if @git.url
+          %(#{file} in the repository <a href="#{h(@git.repository_url)}">#{h(@git.repository)}</a>)
         end
 
         # {macros}, [[Target|label]] and ![[View/Name]] become placeholders before kramdown sees them (it would split the
-        # brackets), except in code.
+        # brackets), except in code; embedded descriptions are written in first.
         def protect(markdown)
-          markdown.split(CODE).each_with_index.map { |part, i| i.odd? ? part : protect_text(part) }.join
+          outside_code(embed_descriptions(markdown)) { |text| protect_text(text) }
+        end
+
+        def outside_code(markdown, &)
+          markdown.split(CODE).each_with_index.map { |part, i| i.odd? ? part : yield(part) }.join
+        end
+
+        # The description of a resource embedded on a line of its own (`![[MotivationGoal/Name]]`) is written into the
+        # page: it is text, not live content. Embeds in that description stay notes (they are not expanded again).
+        def embed_descriptions(markdown)
+          outside_code(markdown) do |text|
+            text.gsub(/^[ \t]*!\[\[([^\]|]+)\]\][ \t]*$/) { embedded_description(Regexp.last_match(1).strip) || Regexp.last_match(0) }
+          end
+        end
+
+        def embedded_description(reference)
+          kind, name = reference.split("/", 2)
+          return if name.nil? || Archsight::Helpers::Embeds::LIVE_KINDS.include?(kind) ||
+                    !Archsight::Helpers::Embeds.embeddable_kind?(kind)
+
+          instance = @database&.instances_by_kind(kind)&.[](name)
+          description = instance&.annotations.to_h["architecture/description"].to_s.strip
+          description.empty? ? nil : description
         end
 
         def protect_text(text)
@@ -240,6 +277,7 @@ module Archsight
             xml && placeholder("LINK", xml, plain: (macro.plain(value) if macro.respond_to?(:plain)))
           end
           text = text.gsub(%r{^[ \t]*!\[\[View/([^\]|]+)\]\][ \t]*$}) { view_embed(Regexp.last_match(1).strip) || Regexp.last_match(0) }
+          text = text.gsub(%r{^[ \t]*!\[\[Analysis/([^\]|]+)\]\][ \t]*$}) { analysis_embed(Regexp.last_match(1).strip) || Regexp.last_match(0) }
           text = text.gsub(/!\[\[([^\]|]+)\]\]/) { placeholder("EMBED", embed_note(Regexp.last_match(1).strip)) }
           text.gsub(Archsight::Helpers::WikiLinks::PATTERN) do
             placeholder("LINK", wiki_link(Regexp.last_match(1).strip, Regexp.last_match(2)&.strip))
@@ -256,6 +294,18 @@ module Archsight
           placeholder("TABLE", xml, plain: embed_note("View/#{name}"))
         end
 
+        # An Analysis on a line of its own is run and its result written into the page (the data of the moment, like a
+        # view); an unknown Analysis stays an embed (a note), the linter reports it.
+        def analysis_embed(name)
+          analysis = @database&.instances_by_kind("Analysis")&.[](name)
+          return nil unless analysis
+
+          require_relative "../../analysis"
+          result = Archsight::Analysis::Executor.new(@database).execute(analysis)
+          output = AnalysisOutput.new(link: ->(target, label) { wiki_link(target, label) }, code: ->(source, language) { code(source, language) })
+          placeholder("TABLE", output.xml(name, result), plain: embed_note("Analysis/#{name}"))
+        end
+
         # The XML of a table, or nothing and a problem (the page is not exported) if it cannot be built
         def table_xml(what)
           yield
@@ -358,13 +408,12 @@ module Archsight
           %(<p><ac:image><ri:attachment ri:filename="#{h(filename)}" /></ac:image></p>\n)
         end
 
-        # The diagram file of a draw.io macro and its `.png` preview, which must share the name
+        # The diagram file of a draw.io macro and its `.png` preview, which must share the name. The name carries a hash
+        # of the content: the macro shows revision 1 of the attachment it names, so a changed diagram uploaded under the
+        # old name (as a new version of it) would not be shown. A new name is a new attachment, the old one is removed.
         # @return [String] the name the macro refers to
         def attach_diagram(stem, xml, png)
-          sha = Digest::SHA256.hexdigest(xml)
-          name = stem
-          taken = ->(file, data) { @attachments.key?(file) && @attachments[file].data != data }
-          name = "#{stem}-#{sha[0, 8]}" if taken.call(stem, xml) || taken.call("#{stem}.png", png)
+          name = "#{stem}-#{Digest::SHA256.hexdigest(xml)[0, 8]}"
           @attachments[name] = Attachment.new(filename: name, data: xml, content_type: MXFILE)
           @attachments["#{name}.png"] = Attachment.new(filename: "#{name}.png", data: png, content_type: "image/png")
           name

data/lib/archsight/export/confluence/tables.rb

@@ -1,8 +1,7 @@
 # frozen_string_literal: true
 
 require "erb"
-require "kramdown"
-require "kramdown-parser-gfm"
+require_relative "../../markdown"
 require_relative "../../view_table"
 require_relative "../../helpers/macros"
 require_relative "diagram_links"
@@ -67,7 +66,7 @@ module Archsight
           return "" if text.to_s.strip.empty?
 
           safe = text.gsub(%r{<(?!(?:https?://|mailto:))}, "&lt;")
-          html = Kramdown::Document.new(safe, input: "GFM", entity_output: :as_char, auto_ids: false).to_html.strip
+          html = Archsight::Markdown.unwrapped(safe, entity_output: :as_char, auto_ids: false).to_html.strip
           html.match(%r{\A<p>((?:(?!</?p>).)*)</p>\z}m) ? Regexp.last_match(1) : html
         end
 

data/lib/archsight/git_source.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require "erb"
+require "open3"
+require "pathname"
+require "uri"
+
+module Archsight
+  # Where a file lives in its git repository and its page in the web UI of the git host, for links to the source of
+  # generated content (the banner of an exported Confluence page). Only local git is asked (the repository of the
+  # file, its `origin` remote, the checked out branch), nothing goes over the network.
+  #
+  # The web URL is `https://<host>/<owner>/<repo>` of the remote, without the user, password or token a remote URL
+  # may carry. GitHub files are `/blob/<ref>/<path>`, GitLab files `/-/blob/<ref>/<path>`; on other hosts, and for a
+  # file git does not track, only the repository is linked. The ref is the checked out branch, the commit when the
+  # HEAD is detached (CI).
+  module GitSource
+    # repository: "owner/repo"; repository_url: its web page; path: the file relative to the repository root;
+    # url: the web page of the file, nil if the host is not known
+    Location = Struct.new(:repository, :repository_url, :path, :url, keyword_init: true)
+
+    module_function
+
+    # @param file [String] path of a file
+    # @return [Location, nil] nil if the file is not in a git repository with an `origin` remote
+    def locate(file)
+      dir = File.dirname(File.expand_path(file))
+      root = git(dir, "rev-parse", "--show-toplevel") or return nil
+      remote = git(dir, "remote", "get-url", "origin") or return nil
+      host, repository = host_and_repository(remote)
+      return nil unless host && repository
+
+      path = Pathname.new(File.realpath(file)).relative_path_from(Pathname.new(File.realpath(root))).to_s
+      ref = git(dir, "symbolic-ref", "--short", "-q", "HEAD") || git(dir, "rev-parse", "HEAD")
+      repository_url = "https://#{host}/#{repository}"
+      url = tracked?(root, path) ? file_url(host, repository_url, ref, path) : nil
+      Location.new(repository: repository, repository_url: repository_url, path: path, url: url)
+    rescue SystemCallError, ArgumentError
+      nil
+    end
+
+    # `git@host:owner/repo.git`, `ssh://git@host:22/owner/repo.git`, `https://user:token@host/owner/repo` ->
+    # ["host", "owner/repo"]; the port of an ssh remote is not the port of the web UI and is left out
+    def host_and_repository(remote)
+      if (scp = remote.match(%r{\A(?:[^@/\s]+@)?([^:/\s]+):(?!//)(.+)\z}))
+        host, path = scp.captures
+      else
+        uri = URI.parse(remote)
+        return [nil, nil] unless uri.host && %w[http https ssh git].include?(uri.scheme)
+
+        host = uri.scheme.start_with?("http") && uri.port != uri.default_port ? "#{uri.host}:#{uri.port}" : uri.host
+        path = uri.path
+      end
+      repository = path.delete_prefix("/").delete_suffix("/").delete_suffix(".git")
+      repository.empty? ? [nil, nil] : [host, repository]
+    rescue URI::InvalidURIError
+      [nil, nil]
+    end
+
+    def file_url(host, repository_url, ref, path)
+      blob = if host.match?(/(\A|\.)github\./) then "blob"
+             elsif host.match?(/(\A|\.)gitlab\./) then "-/blob"
+             end
+      return nil unless blob && ref
+
+      "#{repository_url}/#{blob}/#{escape(ref)}/#{escape(path)}"
+    end
+
+    # An untracked or ignored file is not on the git host: no link that leads nowhere
+    def tracked?(root, path) = !git(root, "ls-files", "--", path).nil?
+
+    # each segment escaped, the slashes kept
+    def escape(path) = path.split("/").map { |segment| ERB::Util.url_encode(segment) }.join("/")
+
+    def git(dir, *)
+      out, _err, status = Open3.capture3("git", "-C", dir, *)
+      value = out.strip
+      status.success? && !value.empty? ? value : nil
+    end
+  end
+end

data/lib/archsight/helpers/asset_images.rb

@@ -2,8 +2,7 @@
 
 require "cgi"
 require "erb"
-require "kramdown"
-require "kramdown-parser-gfm"
+require_relative "../markdown"
 require_relative "../assets"
 
 module Archsight
@@ -52,7 +51,7 @@ module Archsight
       # Problems with the image references of a markdown text.
       # @return [Array<Hash>] `{ reference:, status: :outside|:type|:missing, path: }`
       def audit(markdown, base_dir:, resources_dir:)
-        html = Kramdown::Document.new(markdown.to_s, input: "GFM").to_html
+        html = Archsight::Markdown.document(markdown).to_html
         html.scan(IMG).filter_map do |tag|
           reference = attribute(tag, SRC)
           next if reference.nil? || Assets.external?(reference)
@@ -65,7 +64,7 @@ module Archsight
       # The existing .asd files a markdown text embeds (for the linter)
       # @return [Array<Array(String, String)>] normalized asset path and real file
       def asd_files(markdown, base_dir:, resources_dir:)
-        html = Kramdown::Document.new(markdown.to_s, input: "GFM").to_html
+        html = Archsight::Markdown.document(markdown).to_html
         html.scan(IMG).filter_map do |tag|
           reference = attribute(tag, SRC)
           next if reference.nil? || Assets.external?(reference)

data/lib/archsight/helpers/embeds.rb

@@ -4,26 +4,37 @@ require "erb"
 
 module Archsight
   module Helpers
-    # Replaces `![[View/Name]]` and `![[Analysis/Name]]` in rendered HTML by a placeholder for the live content of
-    # that view or analysis. The frontend turns each placeholder into the result list or the analysis result (it
-    # loads and runs them itself, so rendering a page never executes a query or a script):
+    # Replaces `![[Kind/Name]]` in rendered HTML by a placeholder for the content of that resource: the result list of
+    # a View, the result of an Analysis, the description of any other resource (a goal, a capability, ...). The
+    # frontend turns each placeholder into that content (it loads and runs it itself, so rendering a page never
+    # executes a query or a script):
     #
     #   <div class="kind-embed" data-kind="View" data-name="View:Services"><a href="/kinds/View/instances/...">...</a></div>
     #
     # The link inside is what API and MCP consumers (and a browser without script) get. A reference that does not
-    # name an existing View or Analysis renders as a `broken-link` marker. Code (`<pre>`, `<code>`) is left alone.
+    # name an existing resource (or names a page) renders as a `broken-link` marker. Code (`<pre>`, `<code>`) is
+    # left alone.
     class Embeds
-      KINDS = %w[View Analysis].freeze
+      # Embeds with live content; every other kind shows its description
+      LIVE_KINDS = %w[View Analysis].freeze
+      # Pages and menus are not resources with a description
+      NOT_EMBEDDABLE = %w[Page PageMenu].freeze
       PATTERN = /!\[\[([^\]|]+)\]\]/
       PARAGRAPH = %r{<p>\s*(#{PATTERN.source})\s*</p>}
       CODE = %r{(<pre\b.*?</pre>|<code\b.*?</code>)}m
 
       REASONS = {
-        kind: "only #{KINDS.join(" and ")} can be embedded, written View/Name or Analysis/Name",
+        kind: "embed a resource as Kind/Name (a View, an Analysis or the description of any other resource); " \
+              "pages cannot be embedded",
         missing: "no such resource",
         ambiguous: "ambiguous reference"
       }.freeze
 
+      # Whether `![[kind/...]]` can be embedded: a known kind that is not a page
+      def self.embeddable_kind?(kind)
+        !Archsight::Resources[kind].nil? && !NOT_EMBEDDABLE.include?(Archsight::Resources.canonical(kind))
+      end
+
       def initialize(database, resolver: ResourceResolver.new(database))
         @database = database
         @resolver = resolver
@@ -66,7 +77,7 @@ module Archsight
       # @return [Array(Array(String, String), Symbol)] kind and name, and the problem (nil if it is embeddable)
       def check(reference)
         kind, name = reference.split("/", 2)
-        return [[kind, name], :kind] unless name && KINDS.include?(kind)
+        return [[kind, name], :kind] unless name && self.class.embeddable_kind?(kind)
 
         problem = @resolver.call(reference)
         [[kind, name], problem.is_a?(Symbol) ? problem : nil]

data/lib/archsight/helpers/resource_resolver.rb

@@ -32,6 +32,31 @@ module Archsight
         matches.first
       end
 
+      # What an annotation that names another resource (`lifecycle/replacedBy`) refers to. A bare `Name` is a
+      # resource of the same kind as the annotated one, another kind needs `Kind/Name`.
+      # @param value [String] the annotation value
+      # @param kind [String] the kind of the annotated resource
+      # @return [Array(String, String), Symbol] `[kind, name]`, or `:missing`
+      def find_annotated(value, kind)
+        reference = value.to_s.strip
+        found = find(reference.include?("/") ? reference : "#{kind}/#{reference}")
+        found.is_a?(Array) ? found : :missing
+      end
+
+      # Where the annotations of a resource that name another resource point to:
+      # `{ key => page path }`, `nil` for a reference that does not resolve.
+      # @param kind [String] the kind of the annotated resource, which knows the annotation formats
+      # @param annotations [Hash]
+      def annotation_links(kind, annotations)
+        klass = Archsight::Resources[kind]
+        annotations.each_with_object({}) do |(key, value), links|
+          next unless klass&.annotation_format(key) == :resource
+
+          found = find_annotated(value, kind)
+          links[key] = found.is_a?(Array) ? path(*found) : nil
+        end
+      end
+
       private
 
       def in_kind(kind, name)

data/lib/archsight/import/executor.rb

@@ -38,7 +38,9 @@ class Archsight::Import::Executor
   # @param output [IO] Output stream for progress (default: $stdout)
   # @param filter [String, nil] Regex pattern to match import names
   # @param force [Boolean] Whether to bypass cache and re-run all imports
-  def initialize(database:, resources_dir:, verbose: false, max_concurrent: MAX_CONCURRENT, output: $stdout, filter: nil, force: false)
+  # @param deps [Boolean] With a filter, also run the imports the matched ones depend on (false: only the matched ones)
+  def initialize(database:, resources_dir:, verbose: false, max_concurrent: MAX_CONCURRENT, output: $stdout, filter: nil, force: false,
+                 deps: true)
     @database = database
     @resources_dir = resources_dir
     @verbose = verbose
@@ -47,7 +49,10 @@ class Archsight::Import::Executor
     @filter = filter
     @filter_regex = Regexp.new(filter, Regexp::IGNORECASE) if filter
     @force = force
+    @deps = deps
     @executed_this_run = Set.new
+    @ran_this_run = Set.new
+    @handler_problems = {}
     @iteration = 0
     @mutex = Mutex.new
   end
@@ -154,18 +159,24 @@ class Archsight::Import::Executor
     sorted = topological_sort(filtered_imports)
 
     @output.puts "Filter: #{@filter}" if @filter
+    @output.puts "Dependencies: not run (--no-deps)" if only_matches?
 
     sorted.each_with_index do |imp, idx|
       enabled = import_enabled?(imp)
       deps = import_dependency_names(imp)
       deps_str = deps.empty? ? "(no dependencies)" : "depends on: #{deps.join(", ")}"
       enabled_str = enabled ? "" : " [DISABLED]"
-      @output.puts "  #{idx + 1}. #{imp.name}#{enabled_str} #{deps_str}"
+      @output.puts "  #{idx + 1}. #{imp.name}#{enabled_str} #{deps_str}#{unmatched_patterns_note(imp)}"
     end
 
     sorted
   end
 
+  # Names of the imports that executed in this run (not served from the cache)
+  def ran_imports
+    @mutex.synchronize { @ran_this_run.to_a.sort }
+  end
+
   private
 
   # Set up signal handlers for graceful shutdown
@@ -222,7 +233,13 @@ class Archsight::Import::Executor
     @imports_to_run.include?(import.name)
   end
 
-  # Compute the full set of imports to run: filtered imports + all their dependencies
+  # Whether only the imports matching the filter run (`-f` with `--no-deps`)
+  def only_matches?
+    !@deps && !@filter_regex.nil?
+  end
+
+  # Compute the full set of imports to run: filtered imports + all their dependencies (only the filtered ones with
+  # `--no-deps`)
   def compute_imports_to_run
     all_imports = database.instances_by_kind("Import")&.values || []
     imports_by_name = all_imports.to_h { |imp| [imp.name, imp] }
@@ -230,6 +247,7 @@ class Archsight::Import::Executor
     # Start with imports matching the filter
     matching = all_imports.select { |imp| @filter_regex.match?(imp.name) }.map(&:name)
     to_run = Set.new(matching)
+    return to_run unless @deps
 
     # Recursively add dependencies
     queue = matching.dup
@@ -249,9 +267,12 @@ class Archsight::Import::Executor
     to_run
   end
 
-  # Check if all dependencies of an import are satisfied (executed successfully this run)
+  # Check if all dependencies of an import are satisfied (executed successfully this run).
+  # A parent that can never run (no handler or disabled, e.g. the repository was
+  # deleted upstream and only its generated `generates` entry is left) is not
+  # waited for, otherwise its children would deadlock the run.
   def dependencies_satisfied?(import)
-    dep_names = import_dependency_names(import)
+    dep_names = runnable_dependency_names(import)
     return true if dep_names.empty?
 
     @mutex.synchronize do
@@ -261,30 +282,81 @@ class Archsight::Import::Executor
     end
   end
 
-  # Get dependency names for an import by finding parents that generate this import
-  # Dependencies are derived from the inverse of `generates` - if Import A generates Import B,
-  # then B depends on A
+  # With `--no-deps` a dependency outside the run is not waited for: it does not run, so the import uses the model
+  # as it is
+  def runnable_dependency_names(import)
+    imports_by_name = (database.instances_by_kind("Import")&.values || []).to_h { |imp| [imp.name, imp] }
+    import_dependency_names(import).select do |dep_name|
+      parent = imports_by_name[dep_name]
+      parent && import_enabled?(parent) && (!only_matches? || import_matches_filter?(parent))
+    end
+  end
+
+  # Get dependency names for an import: the parents that generate it (the inverse of `generates` - if Import A
+  # generates Import B, then B depends on A) and the imports it declares in `import/dependsOn`
   def import_dependency_names(import)
-    # Find all imports that have this import in their generates.imports
     all_imports = database.instances_by_kind("Import")&.values || []
     parent_imports = all_imports.select do |parent|
-      generated = parent.relations(:generates, :imports)
-      generated&.any? { |gen| (gen.is_a?(String) ? gen : gen.name) == import.name }
+      generated_import_names(parent).include?(import.name)
+    end
+    (parent_imports.map(&:name) + declared_dependency_names(import, all_imports)).uniq
+  end
+
+  # The imports `import/dependsOn` names or matches. For a pattern this also includes the imports that may still
+  # generate a matching import in this run: one whose last run generated a match, and one that never ran but whose
+  # handler generates imports. So `Import:Repo:*` waits for the GitHub import and then for the repositories it found.
+  def declared_dependency_names(import, all_imports = database.instances_by_kind("Import")&.values || [])
+    entries = Archsight::Import.declared_dependencies(import)
+    return [] if entries.empty?
+
+    others = all_imports.reject { |imp| imp.name == import.name }
+    entries.flat_map do |entry|
+      matches = others.select { |imp| Archsight::Import.dependency_match?(entry, imp.name) }
+      matches += others.select { |imp| may_generate_match?(imp, entry) } if Archsight::Import.pattern?(entry)
+      matches.map(&:name)
+    end.uniq
+  end
+
+  # Whether an import generated a match of the pattern in its last run, or may do so on its first one
+  def may_generate_match?(import, pattern)
+    generated = generated_import_names(import)
+    return generated.any? { |name| Archsight::Import.dependency_match?(pattern, name) } if generated.any?
+
+    handler = Archsight::Import::Registry[import.annotations["import/handler"]]
+    !import.annotations["generated/at"] && handler.respond_to?(:generates_imports?) && handler.generates_imports?
+  end
+
+  def generated_import_names(import)
+    (import.relations(:generates, :imports) || []).map { |gen| gen.is_a?(String) ? gen : gen.name }
+  end
+
+  # For the plan: patterns of `import/dependsOn` that match no import yet
+  def unmatched_patterns_note(import)
+    all_names = (database.instances_by_kind("Import")&.keys || []) - [import.name]
+    unmatched = Archsight::Import.declared_dependencies(import).select do |entry|
+      Archsight::Import.pattern?(entry) && all_names.none? { |name| Archsight::Import.dependency_match?(entry, name) }
     end
-    parent_imports.map(&:name)
+    unmatched.empty? ? "" : " (#{unmatched.join(", ")} matches no import yet)"
   end
 
-  # Reload database without verbose output
+  # Reload database without verbose output, and load the handlers the imports declare (`import/handlerPath`)
   def reload_database_quietly!
     original_verbose = database.verbose
     database.verbose = false
     database.reload!
+    load_local_handlers
     # Reset filter cache since new imports may have been discovered
     @imports_to_run = nil if @filter_regex
   ensure
     database.verbose = original_verbose
   end
 
+  # A handler file that does not load fails the imports that use it, not the whole run
+  def load_local_handlers
+    problems = Archsight::Import::HandlerLoader.load(database, resources_dir: resources_dir)
+    @handler_problems = problems.to_h { |problem| [problem.import.name, problem.message] }
+  end
+
   # Reload database and update progress total for newly discovered imports
   def reload_and_update_total!
     reload_database_quietly!
@@ -315,6 +387,8 @@ class Archsight::Import::Executor
           result = execute_single_import(imp, slot_progress)
           if result == :cached
             slot_progress.complete("Cached")
+          elsif result.is_a?(String)
+            slot_progress.complete(result)
           else
             @mutex.synchronize { @total_executed += 1 }
             slot_progress.complete("Done")
@@ -339,7 +413,7 @@ class Archsight::Import::Executor
   end
 
   # Execute a single import (called from thread)
-  # @return [Symbol] :executed, :cached, or raises on error
+  # @return [Symbol, String] :executed, :cached, a message for a skipped import, or raises on error
   def execute_single_import(import, import_progress)
     # Check if import is still fresh based on generated/at and import/cacheTime
     if import_fresh?(import)
@@ -347,6 +421,18 @@ class Archsight::Import::Executor
       return :cached
     end
 
+    problem = @handler_problems[import.name]
+    raise Archsight::Import::UnknownHandlerError, problem if problem
+
+    missing = missing_environment(import)
+    if missing.any?
+      message = "missing environment variable#{"s" if missing.length > 1} #{missing.join(", ")} (import/config/requiredEnv)"
+      raise Archsight::Import::ImportError, message unless import.annotations["import/config/optional"] == "true"
+
+      import_progress.warn("Skipped: #{message}")
+      return "Skipped"
+    end
+
     # Execute the import
     handler_class = Archsight::Import::Registry.handler_for(import)
     handler = handler_class.new(
@@ -356,13 +442,32 @@ class Archsight::Import::Executor
       progress: import_progress,
       shared_writer: @shared_writer
     )
+    Archsight::Import.never_index(handler.cache_root)
     handler.execute
+    remove_stale_outputs(import, handler.owned_outputs)
+    @mutex.synchronize { @ran_this_run.add(import.name) }
     :executed
   end
 
+  # Variables of `import/config/requiredEnv` that are not set (or blank)
+  def missing_environment(import)
+    names = import.annotations["import/config/requiredEnv"].to_s.split(",").map(&:strip).reject(&:empty?)
+    names.select { |name| ENV[name].to_s.strip.empty? }
+  end
+
+  # Delete the files an earlier run of the import wrote (`generated/outputs`) that this run did not write again
+  def remove_stale_outputs(import, written)
+    previous = import.annotations["generated/outputs"].to_s.split(",").map(&:strip).reject(&:empty?)
+    (previous - written).each do |rel_path|
+      path = Archsight::Import.resources_path(resources_dir, rel_path)
+      FileUtils.rm_f(path) if path && File.file?(path)
+    end
+  end
+
   # Check if import is still fresh (cached)
-  # Uses generated/at annotation and import/cacheTime to determine freshness
-  # Also validates that output file exists (if specified) and source file hasn't changed
+  # Uses generated/at annotation and import/cacheTime to determine freshness. The cache is also invalid when the
+  # output file or a recorded output (`generated/outputs`) is missing, the configuration or the files of
+  # `import/config/inputs` changed, or an import it declares in `import/dependsOn` executed in this run.
   def import_fresh?(import)
     # Honor force flag - always re-run when forced
     return false if @force
@@ -377,6 +482,11 @@ class Archsight::Import::Executor
       return false unless File.exist?(full_path)
     end
 
+    return false unless recorded_outputs_exist?(import)
+    return false if dependency_ran?(import)
+    return false if import.annotations["import/config/inputs"] &&
+                    import.annotations["generated/inputsHash"] != Archsight::Import.inputs_hash(import, resources_dir)
+
     generated_at = import.annotations["generated/at"]
     return false if generated_at.nil? || generated_at.empty?
 
@@ -399,6 +509,20 @@ class Archsight::Import::Executor
     false
   end
 
+  def recorded_outputs_exist?(import)
+    import.annotations["generated/outputs"].to_s.split(",").map(&:strip).reject(&:empty?).all? do |rel_path|
+      path = Archsight::Import.resources_path(resources_dir, rel_path)
+      path && File.exist?(path)
+    end
+  end
+
+  # Whether an import declared in `import/dependsOn` executed (not just came from the cache) in this run. The
+  # generated imports of a parent are not affected: they keep their own cache time when the parent runs.
+  def dependency_ran?(import)
+    names = declared_dependency_names(import)
+    @mutex.synchronize { names.any? { |name| @ran_this_run.include?(name) } }
+  end
+
   def parse_duration(str)
     return nil if str.nil?