archsight 0.2.7 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee4785cb42348dcbad1e9731b83a9e9b783f3eebe831dac544a771cff6a01829
-  data.tar.gz: 6223d231b7a01a0eed99ec14ad7bfa5ccad45b41ff4b6e4b4a29bd64946cd184
+  metadata.gz: fca6df89e7a4bbcae73578f3f9bed3bf680dd86561269396a0a4243c84109eed
+  data.tar.gz: 3991aafe49e2f950305e91f92cf93c0b953a808647cdceb09ae01edb2df4d304
 SHA512:
-  metadata.gz: 6c76128de7274e7dc083bae693958f8afad14231120d59c2a3ac169408962f739239c06ec3a30d2b1a34fa19fb555bbe2ee87deaec6464881faad4e992482a3c
-  data.tar.gz: 7b6c865e1ab3287be9e8a23c29b9884cf0e98c0929af47ab577e567c51d3c075ee00b51653191267c8f0fde1032b7ba872221932850092643c5d8db1235fa130
+  metadata.gz: 0e2be76c13cb35664086dc9f5d96675c6c0a54dd11d10803895edac5887428bd7b5d0441c5e419a82454f3532175556c3c398615edabe86b2fe4bf62bc02599b
+  data.tar.gz: 42133f269b0f2b09306da83c16fb311539180814e6cec26b3e8bc4f1a4b5b35d08922175f1bf464cd00c6c86f4d7df9931dbf576d00873ca7d174be2459540fe

data/lib/archsight/import/handler.rb

@@ -115,7 +115,7 @@ class Archsight::Import::Handler
       # Use thread-safe shared writer for concurrent execution
       # Default sort key is import name for stable output ordering
       key = sort_key || import_resource.name
-      @shared_writer.append_yaml(full_path, content, sort_key: key)
+      @shared_writer.append_yaml(full_path, content, sort_key: key, producer: import_resource.name)
     else
       # Direct write for non-concurrent mode
       FileUtils.mkdir_p(File.dirname(full_path))
@@ -209,7 +209,8 @@ class Archsight::Import::Handler
                 end
 
     if @shared_writer
-      @shared_writer.append_yaml(full_path, YAML.dump(meta), sort_key: "#{import_resource.name}:generates")
+      @shared_writer.append_yaml(full_path, YAML.dump(meta), sort_key: "#{import_resource.name}:generates",
+                                                             producer: import_resource.name)
     else
       # Append to existing file
       File.open(full_path, "a") { |f| f.write(YAML.dump(meta)) }

data/lib/archsight/import/handlers/cache_pruner.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Shared helper for import handlers that clone repositories into a local
+# cache directory keyed by name (see github.rb, gitlab.rb). Removes cache
+# subdirectories that no longer correspond to any repo/project returned by
+# the current API listing, so a renamed, moved, or deleted repo doesn't
+# leave a stale clone behind that a later importer could still resolve
+# against under its old name.
+module Archsight::Import::Handlers::CachePruner
+  def prune_stale_cache_entries(target_dir, current_names)
+    return unless File.directory?(target_dir)
+
+    Dir.children(target_dir).each do |entry|
+      next if current_names.include?(entry)
+
+      stale_path = File.join(target_dir, entry)
+      next unless File.directory?(stale_path)
+
+      progress.warn("Pruning stale cache entry: #{entry} (repo renamed, moved, or removed)")
+      FileUtils.rm_rf(stale_path)
+    end
+  end
+end

data/lib/archsight/import/handlers/github.rb

@@ -7,6 +7,7 @@ require "uri"
 require "openssl"
 require_relative "../handler"
 require_relative "../registry"
+require_relative "cache_pruner"
 
 # GitHub handler - lists repositories from a GitHub organization and generates child Import resources
 #
@@ -18,6 +19,7 @@ require_relative "../registry"
 #   import/config/botTeam - Team for bot-only repositories (propagated to child imports)
 #   import/config/corporateAffixes - Comma-separated corporate username affixes for team matching (propagated to child imports)
 #   import/config/defaultVisibility - Default visibility for non-public repos (default: "internal")
+#   import/config/cacheRoot - Override for the local git cache root (default: ~/.cache/archsight/git)
 #
 # Environment:
 #   GITHUB_TOKEN - GitHub Personal Access Token (required)
@@ -27,7 +29,14 @@ require_relative "../registry"
 # Output:
 #   Generates Import:Repo:* resources for each repository
 #   The repository handler will clone/sync the actual git repositories
+#
+# Before generating imports, prunes any cache subdirectory under
+# cacheRoot/github/<org> that no longer corresponds to a repo returned by
+# the API (e.g. a repo that was renamed or deleted), so stale clones don't
+# linger and get picked up by later importers.
 class Archsight::Import::Handlers::Github < Archsight::Import::Handler
+  include Archsight::Import::Handlers::CachePruner
+
   PER_PAGE = 100
 
   def execute
@@ -40,7 +49,8 @@ class Archsight::Import::Handlers::Github < Archsight::Import::Handler
     @repo_output_path = config("repoOutputPath")
     @child_cache_time = config("childCacheTime")
     @default_visibility = config("defaultVisibility", default: "internal")
-    @target_dir = File.join(Dir.home, ".cache", "archsight", "git", "github", @org)
+    cache_root = config("cacheRoot", default: File.join(Dir.home, ".cache", "archsight", "git"))
+    @target_dir = File.join(cache_root, "github", @org)
 
     # Fetch all repositories with pagination
     progress.update("Fetching repositories from #{@org}")
@@ -51,6 +61,8 @@ class Archsight::Import::Handlers::Github < Archsight::Import::Handler
       return
     end
 
+    prune_stale_cache_entries(@target_dir, repos.map { |r| r["name"] })
+
     # Generate Import resources for each repository
     progress.update("Generating #{repos.size} import resources")
     generate_repository_imports(repos)

data/lib/archsight/import/handlers/gitlab.rb

@@ -8,6 +8,7 @@ require "uri"
 require "openssl"
 require_relative "../handler"
 require_relative "../registry"
+require_relative "cache_pruner"
 
 # GitLab handler - lists repositories from a GitLab instance and generates child Import resources
 #
@@ -23,6 +24,7 @@ require_relative "../registry"
 #   import/config/botTeam - Team for bot-only repositories (propagated to child imports)
 #   import/config/corporateAffixes - Comma-separated corporate username affixes for team matching (propagated to child imports)
 #   import/config/defaultVisibility - Default visibility when API returns none (default: "internal")
+#   import/config/cacheRoot - Override for the local git cache root (default: ~/.cache/archsight/git)
 #
 # Environment:
 #   GITLAB_TOKEN - GitLab personal access token (required)
@@ -30,7 +32,14 @@ require_relative "../registry"
 # Output:
 #   Generates Import:Repo:* resources for each repository
 #   The repository handler will clone/sync the actual git repositories (via SSH)
+#
+# Before generating imports, prunes any cache subdirectory under
+# cacheRoot/gitlab that no longer corresponds to a project returned by the
+# API (e.g. a project that was renamed, moved, or deleted), so stale clones
+# don't linger and get picked up by later importers.
 class Archsight::Import::Handlers::Gitlab < Archsight::Import::Handler
+  include Archsight::Import::Handlers::CachePruner
+
   def execute
     @host = config("host")
     raise "Missing required config: host" unless @host
@@ -42,7 +51,8 @@ class Archsight::Import::Handlers::Gitlab < Archsight::Import::Handler
     @child_cache_time = config("childCacheTime")
     @default_visibility = config("defaultVisibility", default: "internal")
 
-    @target_dir = File.join(Dir.home, ".cache", "archsight", "git", "gitlab")
+    cache_root = config("cacheRoot", default: File.join(Dir.home, ".cache", "archsight", "git"))
+    @target_dir = File.join(cache_root, "gitlab")
     @explore_groups = config("exploreGroups") == "true"
     @per_page = config("perPage", default: "100").to_i
     @verify_ssl = config("verifySSL") != "false"
@@ -57,6 +67,8 @@ class Archsight::Import::Handlers::Gitlab < Archsight::Import::Handler
       return
     end
 
+    prune_stale_cache_entries(@target_dir, projects.map { |p| safe_dir_name(p["path_with_namespace"]) })
+
     # Generate Import resources for each repository
     progress.update("Generating #{projects.size} import resources")
     generate_repository_imports(projects)

data/lib/archsight/import/handlers/go_module_parser.rb

@@ -33,20 +33,41 @@ module Archsight::Import::Handlers::GoModuleParser
     else
       root_mod = read_module_name(repo_root)
       modules << [".", root_mod] if root_mod
+      modules.concat(discover_nested_modules(repo_root, root_mod))
+    end
+
+    modules
+  end
+
+  # Scan subdirectories for additional go.mod files (multi-module monorepo
+  # without go.work). Skips a nested go.mod whose declared module path is
+  # unrelated to the repo's own root module - likely a vendored/embedded copy
+  # of a foreign project (e.g. a full committed snapshot of another,
+  # possibly since-renamed, repo) rather than an intentional submodule of
+  # this one - so it doesn't manufacture a cross-repo component/dependency.
+  #
+  # @return [Array<Array<String>>] List of [rel_dir, mod_name] pairs
+  def discover_nested_modules(repo_root, root_mod)
+    modules = []
+
+    Find.find(repo_root) do |path|
+      bn = File.basename(path)
+      Find.prune if File.directory?(path) && %w[vendor testdata .git node_modules].include?(bn)
+      next unless bn == "go.mod"
 
-      # Also scan subdirectories for additional go.mod files (multi-module monorepo without go.work)
-      Find.find(repo_root) do |path|
-        bn = File.basename(path)
-        Find.prune if File.directory?(path) && %w[vendor testdata .git node_modules].include?(bn)
-        next unless bn == "go.mod"
+      mod_dir = File.dirname(path)
+      next if mod_dir == repo_root # already added above
 
-        mod_dir = File.dirname(path)
-        next if mod_dir == repo_root # already added above
+      rel = mod_dir.delete_prefix("#{repo_root}/")
+      name = read_module_name(mod_dir)
+      next unless name
 
-        rel = mod_dir.delete_prefix("#{repo_root}/")
-        name = read_module_name(mod_dir)
-        modules << [rel, name] if name
+      if root_mod && !nested_under_root?(name, root_mod)
+        progress.warn("Skipping foreign nested go.mod at #{rel}: module #{name} is unrelated to root module #{root_mod}")
+        next
       end
+
+      modules << [rel, name]
     end
 
     modules
@@ -91,6 +112,15 @@ module Archsight::Import::Handlers::GoModuleParser
     paths.uniq
   end
 
+  # Whether a nested module's declared path is the root module itself or
+  # genuinely nested under it (a real monorepo submodule), as opposed to an
+  # unrelated/foreign module path (a vendored or embedded copy of another
+  # project) that happens to live in a subdirectory.
+  # @return [Boolean]
+  def nested_under_root?(mod_name, root_mod_name)
+    mod_name == root_mod_name || mod_name.start_with?("#{root_mod_name}/")
+  end
+
   # Return the SCM host+org prefix shared by modules in the same org, e.g. "github.com/ionos-cloud/".
   # @return [String, nil] Prefix with trailing slash, or nil for single-segment names
   def same_origin_prefix(mod_name)

data/lib/archsight/import/handlers/repository.rb

@@ -227,7 +227,10 @@ class Archsight::Import::Handlers::Repository < Archsight::Import::Handler
   private
 
   def run_scc(path)
-    cmd = ["scc", "--exclude-dir", ".git,.hg,.svn,vendor,node_modules", "-f", "json2", "--sort", "name", path]
+    # Plain Text files (.txt, .text) are excluded: they carry no structural code
+    # signal and can dominate scc's cost/effort estimates when a repo happens to
+    # contain large ad hoc text blobs (e.g. datasets, doc dumps).
+    cmd = ["scc", "--exclude-dir", ".git,.hg,.svn,vendor,node_modules", "-x", "txt,text", "-f", "json2", "--sort", "name", path]
 
     out, err, status = Open3.capture3(*cmd)
     raise "scc failed: #{cmd.join(" ")}\n#{err}" unless status.success?

data/lib/archsight/import/handlers/rest_api_index.rb

@@ -18,6 +18,13 @@ require_relative "../registry"
 #   import/config/skipVisibility - Comma-separated visibilities to skip (e.g., "public-preview")
 #   import/config/childCacheTime - Cache time for generated child imports (e.g., "1h", "30m")
 #
+# APIs whose "gate" value isn't a recognized status (General-Availability,
+# Early-Access, Development, or their GA/EA/DEV aliases) are skipped, since
+# they can't be represented by the ApplicationInterface status annotation.
+# A gate value that embeds a recognized GA/EA/DEV token alongside extra
+# segments (e.g. "ea.ionosc") is tolerated by extracting that token instead
+# of being rejected outright.
+#
 # Output:
 #   Generates Import:RestApi:* resources for each API in the index
 #
@@ -36,6 +43,9 @@ require_relative "../registry"
 #     ]
 #   }
 class Archsight::Import::Handlers::RestApiIndex < Archsight::Import::Handler
+  VALID_GATES = %w[General-Availability Early-Access Development GA EA DEV].freeze
+  GATE_TOKEN_RE = /\b(ga|ea|dev)\b/i
+
   def execute
     @index_url = config("indexUrl")
     raise "Missing required config: indexUrl" unless @index_url
@@ -55,10 +65,10 @@ class Archsight::Import::Handlers::RestApiIndex < Archsight::Import::Handler
       return
     end
 
-    # Filter APIs by visibility
+    # Filter APIs by visibility and gate
     original_count = apis.size
     apis = filter_apis(apis)
-    progress.update("Filtered to #{apis.size} APIs (skipped #{original_count - apis.size} by visibility)") if apis.size < original_count
+    progress.update("Filtered to #{apis.size} APIs (skipped #{original_count - apis.size} by visibility/gate)") if apis.size < original_count
 
     # Generate child imports
     progress.update("Generating #{apis.size} import resources")
@@ -107,14 +117,33 @@ class Archsight::Import::Handlers::RestApiIndex < Archsight::Import::Handler
   end
 
   def filter_apis(apis)
-    return apis if @skip_visibilities.empty?
-
     apis.reject do |api|
       visibility = api["visibility"]&.downcase
-      @skip_visibilities.any? { |skip| visibility == skip.downcase }
+      skipped_by_visibility = @skip_visibilities.any? { |skip| visibility == skip.downcase }
+      skipped_by_visibility || !valid_gate?(api["gate"])
     end
   end
 
+  def valid_gate?(gate)
+    return true if gate.nil? # falls back to "GA" default in generate_api_imports
+
+    !resolve_gate(gate).nil?
+  end
+
+  # Resolve a gate value to a recognized stage (a VALID_GATES entry, matched
+  # as given, or an extracted GA/EA/DEV token), tolerating extra trailing
+  # segments some index entries carry (e.g. an "ionosc" variant suffix like
+  # "ea.ionosc") by pulling the first recognized stage token out of an
+  # otherwise-unrecognized value rather than rejecting it outright.
+  # @return [String, nil] Resolved gate, or nil if no recognized stage found
+  def resolve_gate(gate)
+    return nil if gate.nil?
+    return gate if VALID_GATES.any? { |valid| gate.casecmp?(valid) }
+
+    match = GATE_TOKEN_RE.match(gate)
+    match && match[1].upcase
+  end
+
   def generate_api_imports(apis)
     yaml_documents = apis.map do |api|
       api_name = api["name"]
@@ -137,7 +166,7 @@ class Archsight::Import::Handlers::RestApiIndex < Archsight::Import::Handler
         "version" => api["version"] || "1.0",
         "visibility" => visibility,
         "specUrl" => spec_url,
-        "gate" => api["gate"] || "GA"
+        "gate" => resolve_gate(api["gate"]) || "GA"
       }
       child_config["htmlUrl"] = html_url if html_url
 

data/lib/archsight/import/shared_file_writer.rb

@@ -1,16 +1,23 @@
 # frozen_string_literal: true
 
 require "fileutils"
+require "yaml"
 
 # Thread-safe file writer for concurrent import handlers
 #
 # Manages shared output files that multiple handlers can write to.
 # Content is buffered in memory and sorted by key when close_all is called.
 #
+# When a file is flushed, any document already on disk whose producer isn't
+# among this flush's fresh producers is carried forward unchanged - this is
+# what lets a cached (skipped) import's previously-generated output, or
+# another task's output from an earlier iteration of the same run, survive
+# being flushed to a shared output file instead of getting silently wiped.
+#
 # @example
 #   writer = SharedFileWriter.new
-#   writer.append_yaml("/path/to/output.yaml", yaml_content, sort_key: "Repo:name")
-#   writer.close_all  # Sorts and writes buffered content
+#   writer.append_yaml("/path/to/output.yaml", yaml_content, sort_key: "Repo:name", producer: "Import:Repo:name")
+#   writer.close_all  # Sorts and writes buffered content, merged with existing content
 class Archsight::Import::SharedFileWriter
   def initialize
     @mutex = Mutex.new
@@ -23,14 +30,17 @@ class Archsight::Import::SharedFileWriter
   # @param path [String] Full path to the output file
   # @param content [String] YAML content to append
   # @param sort_key [String, nil] Key for sorting (nil keys go last)
-  def append_yaml(path, content, sort_key: nil)
+  # @param producer [String, nil] Name of the Import task that produced this
+  #   content, used to decide what existing on-disk content this entry may
+  #   supersede when the file is flushed
+  def append_yaml(path, content, sort_key: nil, producer: nil)
     @mutex.synchronize do
       @files[path] ||= { entries: [], lock: Mutex.new }
     end
 
     entry = @files[path]
     entry[:lock].synchronize do
-      entry[:entries] << { key: sort_key, content: content }
+      entry[:entries] << { key: sort_key, content: content, producer: producer }
     end
   end
 
@@ -51,8 +61,11 @@ class Archsight::Import::SharedFileWriter
 
     FileUtils.mkdir_p(File.dirname(path))
 
+    fresh_producers = entries.filter_map { |e| e[:producer] }.uniq
+    all_entries = preserved_entries(path, fresh_producers) + entries
+
     # Sort by key (nil keys go last)
-    sorted = entries.sort_by { |e| e[:key] || "\xFF" }
+    sorted = all_entries.sort_by { |e| e[:key] || "\xFF" }
 
     File.open(path, "w") do |file|
       sorted.each_with_index do |entry, idx|
@@ -64,4 +77,35 @@ class Archsight::Import::SharedFileWriter
       end
     end
   end
+
+  # Documents already on disk whose producer isn't among this flush's fresh
+  # producers are carried forward unchanged, keyed by their own producer so
+  # a producer that does write fresh content later in the same run replaces
+  # its own prior output rather than duplicating it. Content is kept as its
+  # original raw text (not re-dumped), so anything preserved is byte-for-byte
+  # identical to what was already on disk.
+  def preserved_entries(path, fresh_producers)
+    return [] unless File.exist?(path)
+
+    split_yaml_documents(File.read(path)).filter_map do |raw|
+      doc = YAML.safe_load(raw, permitted_classes: [Time])
+      next nil unless doc.is_a?(Hash)
+
+      producer = document_producer(doc)
+      next nil if producer.nil? || fresh_producers.include?(producer)
+
+      { key: producer, content: raw, producer: producer }
+    rescue Psych::SyntaxError
+      nil
+    end
+  end
+
+  def document_producer(doc)
+    doc.dig("metadata", "annotations", "generated/script") ||
+      (doc["kind"] == "Import" ? doc.dig("metadata", "name") : nil)
+  end
+
+  def split_yaml_documents(text)
+    text.split(/^---\s*$/).map(&:strip).reject(&:empty?)
+  end
 end

data/lib/archsight/version.rb

@@ -4,5 +4,5 @@
 # Do not edit manually.
 
 module Archsight
-  VERSION = "0.2.7"
+  VERSION = "0.2.9"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: archsight
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.2.9
 platform: ruby
 authors:
 - Vincent Landgraf
@@ -264,6 +264,7 @@ files:
 - lib/archsight/import/executor.rb
 - lib/archsight/import/git_analytics.rb
 - lib/archsight/import/handler.rb
+- lib/archsight/import/handlers/cache_pruner.rb
 - lib/archsight/import/handlers/cpp_grapher.rb
 - lib/archsight/import/handlers/crystal_grapher.rb
 - lib/archsight/import/handlers/elixir_grapher.rb