archsight 0.2.3 → 0.2.5

data/lib/archsight/import/handlers/python_grapher.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+require "json"
+require "open3"
+require_relative "grapher"
+require_relative "../registry"
+
+# PythonGrapher handler - analyses a Python repository and generates a GraphViz
+# DOT graph of its package/module structure, stored as architecture/modules on
+# the TechnologyArtifact so it can be rendered in the frontend.
+#
+# Uses static AST analysis (python3 stdlib only — no external Python packages
+# required). Package paths are normalised to "/" separators internally so they
+# are compatible with the generic Grapher layout engine.
+#
+# Configuration:
+#   import/config/path     - Path to the Python repository root
+#   import/config/ranksep  - Horizontal gap between rank columns (default: 0.6)
+#   import/config/nodesep  - Vertical gap between nodes in a column (default: 0.15)
+class Archsight::Import::Handlers::PythonGrapher < Archsight::Import::Handlers::Grapher
+  def self.language_name = "python"
+
+  def self.applicable?(path)
+    File.exist?(File.join(path, "__init__.py")) ||
+      File.exist?(File.join(path, "pyproject.toml")) ||
+      File.exist?(File.join(path, "setup.py")) ||
+      Dir.glob(File.join(path, "*/__init__.py")).any?
+  end
+
+  # Inline Python3 script — scans a single package directory with stdlib ast.
+  # Argv: <pkg_root_dir> <pkg_name>
+  # Stdout: JSON object mapping slash-separated module paths to arrays of deps.
+  #
+  # "from pkg import name" is resolved to "pkg/name" when that submodule exists
+  # on disk, so that intra-package submodule imports produce correct edges.
+  PYTHON_SCANNER = <<~PYTHON
+    import ast, os, sys, json
+
+    def collect_all_mods(pkg_root, pkg_name):
+        mods = set()
+        for dirpath, dirs, files in os.walk(pkg_root):
+            dirs[:] = sorted(d for d in dirs if d != '__pycache__' and not d.startswith('.'))
+            for f in files:
+                if not f.endswith('.py'):
+                    continue
+                rel = os.path.relpath(os.path.join(dirpath, f), pkg_root)
+                parts = rel.replace(os.sep, '/').split('/')
+                parts[-1] = parts[-1][:-3]
+                if parts[-1] == '__init__':
+                    parts = parts[:-1]
+                elif parts[-1] == '__main__':
+                    parts[-1] = 'main'
+                mods.add(pkg_name if not parts else pkg_name + '/' + '/'.join(parts))
+        return mods
+
+    def mod_from_path(path, pkg_root, pkg_name):
+        rel = os.path.relpath(path, pkg_root)
+        parts = rel.replace(os.sep, '/').split('/')
+        parts[-1] = parts[-1][:-3]
+        if parts[-1] == '__init__':
+            parts = parts[:-1]
+        elif parts[-1] == '__main__':
+            parts[-1] = 'main'
+        return pkg_name if not parts else pkg_name + '/' + '/'.join(parts)
+
+    def resolve_relative_base(current_mod, level, module_name):
+        """Resolve a relative import to an absolute slash-path.
+
+        level=1 means same package (go up one component from the module name),
+        level=2 means parent package, etc.
+        """
+        parts = current_mod.split('/')
+        # Strip `level` trailing components to get the anchor package
+        base_parts = parts[:-level] if level <= len(parts) else []
+        if module_name:
+            base_parts = base_parts + module_name.replace('.', '/').split('/')
+        return '/'.join(base_parts) if base_parts else None
+
+    def resolve_from_import(base, names, all_mods):
+        resolved = []
+        for alias in names:
+            sub = base + '/' + alias.name
+            resolved.append(sub if sub in all_mods else base)
+        return resolved or [base]
+
+    def scan_imports(path, current_mod, pkg_name, all_mods):
+        try:
+            tree = ast.parse(open(path, encoding='utf-8', errors='replace').read())
+        except SyntaxError:
+            return []
+        out = []
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Import):
+                for alias in node.names:
+                    name = alias.name.replace('.', '/')
+                    if name == pkg_name or name.startswith(pkg_name + '/'):
+                        out.append(name)
+            elif isinstance(node, ast.ImportFrom):
+                if node.level == 0:
+                    if node.module is None:
+                        continue
+                    base = node.module.replace('.', '/')
+                else:
+                    base = resolve_relative_base(current_mod, node.level, node.module)
+                    if base is None:
+                        continue
+                if base == pkg_name or base.startswith(pkg_name + '/'):
+                    out.extend(resolve_from_import(base, node.names, all_mods))
+        return out
+
+    def is_trivial_init(path):
+        """True if __init__.py has no function/class defs and no non-dunder assignments."""
+        try:
+            tree = ast.parse(open(path, encoding='utf-8', errors='replace').read())
+        except SyntaxError:
+            return True
+        for node in ast.iter_child_nodes(tree):
+            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+                return False
+            if isinstance(node, ast.Assign):
+                targets = [t.id for t in node.targets if isinstance(t, ast.Name)]
+                if any(not t.startswith('__') for t in targets):
+                    return False
+        return True
+
+    pkg_root, pkg_name = sys.argv[1], sys.argv[2]
+    all_mods = collect_all_mods(pkg_root, pkg_name)
+    edges = {}
+    init_file_mods = {}
+    for dirpath, dirs, files in os.walk(pkg_root):
+        dirs[:] = sorted(d for d in dirs if d != '__pycache__' and not d.startswith('.'))
+        for f in files:
+            if not f.endswith('.py'):
+                continue
+            p = os.path.join(dirpath, f)
+            mod = mod_from_path(p, pkg_root, pkg_name)
+            if f == '__init__.py':
+                init_file_mods[mod] = p
+            imps = scan_imports(p, mod, pkg_name, all_mods)
+            if mod not in edges:
+                edges[mod] = []
+            edges[mod].extend(i for i in imps if i != mod)
+    for mod, fpath in init_file_mods.items():
+        if not edges.get(mod) and is_trivial_init(fpath):
+            edges.pop(mod, None)
+    print(json.dumps(edges))
+  PYTHON
+
+  # Scans a list of root-level Python scripts for imports from known packages.
+  # Argv[1]: JSON {"paths": [...], "packages": [...], "all_mods": [...]}
+  # Stdout:  JSON {"<pkg>/main": ["<dep>", ...]}
+  ROOT_SCANNER = <<~PYTHON
+    import ast, sys, json
+
+    def extract_imports(path, pkg_names, all_mods):
+        try:
+            tree = ast.parse(open(path, encoding='utf-8', errors='replace').read())
+        except (SyntaxError, OSError):
+            return []
+        out = []
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Import):
+                for alias in node.names:
+                    base = alias.name.replace('.', '/')
+                    if any(base == p or base.startswith(p + '/') for p in pkg_names):
+                        out.append(base)
+            elif isinstance(node, ast.ImportFrom):
+                if node.level == 0 and node.module:
+                    base = node.module.replace('.', '/')
+                    if any(base == p or base.startswith(p + '/') for p in pkg_names):
+                        sub = base + '/' + node.names[0].name if node.names else base
+                        out.append(sub if sub in all_mods else base)
+        return list(dict.fromkeys(out))
+
+    cfg = json.loads(sys.argv[1])
+    pkg_names = cfg['packages']
+    all_mods = set(cfg.get('all_mods', []))
+    result = {}
+    for path in cfg['paths']:
+        deps = extract_imports(path, pkg_names, all_mods)
+        if not deps:
+            continue
+        main_pkg = next((p for dep in deps for p in pkg_names if dep == p or dep.startswith(p + '/')), None)
+        if not main_pkg:
+            continue
+        result['main'] = list(dict.fromkeys(result.get('main', []) + deps))
+    print(json.dumps(result))
+  PYTHON
+
+  def wrap_single_module?
+    true
+  end
+
+  SKIP_DIRS = %w[test tests docs doc examples example vendor .git __pycache__ dist build
+                 node_modules .tox .venv venv env].freeze
+
+  private
+
+  # ── Module discovery ─────────────────────────────────────────────────────
+
+  def discover_modules(repo_root)
+    # If the root itself is a Python package, treat it as a single module.
+    return [[".", File.basename(repo_root)]] if File.exist?(File.join(repo_root, "__init__.py"))
+
+    modules = []
+    Dir.each_child(repo_root) do |entry|
+      next if SKIP_DIRS.include?(entry) || entry.start_with?(".")
+
+      dir = File.join(repo_root, entry)
+      next unless File.directory?(dir) && File.exist?(File.join(dir, "__init__.py"))
+
+      modules << [entry, entry]
+    end
+
+    modules.sort_by { |rel, _| rel }
+  end
+
+  # ── Package collection ────────────────────────────────────────────────────
+
+  def collect_packages(repo_root, modules, _prefix)
+    all_pkgs = {}
+
+    modules.each do |rel_dir, mod_name|
+      mod_dir = rel_dir == "." ? repo_root : File.join(repo_root, rel_dir)
+      out, err, status = Open3.capture3("python3", "-c", PYTHON_SCANNER, mod_dir, mod_name)
+
+      unless status.success?
+        progress.warn("Skipping #{rel_dir}: #{err.lines.first.to_s.strip}")
+        next
+      end
+
+      JSON.parse(out).each do |pkg, deps|
+        all_pkgs[pkg] ||= []
+        all_pkgs[pkg].concat(deps)
+      end
+    end
+
+    root_scripts = find_root_python_scripts(repo_root)
+    if root_scripts.any?
+      pkg_names = modules.map { |_, mod_name| mod_name }
+      config = { "paths" => root_scripts, "packages" => pkg_names,
+                 "all_mods" => all_pkgs.keys }.to_json
+      out, _err, status = Open3.capture3("python3", "-c", ROOT_SCANNER, config)
+      if status.success?
+        JSON.parse(out).each do |pkg, deps|
+          all_pkgs[pkg] ||= []
+          all_pkgs[pkg].concat(deps)
+        end
+      end
+    end
+
+    all_pkgs
+  end
+
+  def find_root_python_scripts(repo_root)
+    Dir.each_child(repo_root).filter_map do |f|
+      path = File.join(repo_root, f)
+      next unless File.file?(path) && !f.end_with?(".py")
+
+      begin
+        first_line = File.open(path, &:readline).strip
+        path if first_line.match?(/python/)
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end
+
+Archsight::Import::Registry.register("python-grapher", Archsight::Import::Handlers::PythonGrapher)

data/lib/archsight/import/handlers/repository.rb

@@ -20,6 +20,7 @@ require_relative "../team_matcher"
 #   import/config/fallbackTeam - Optional team name when no contributor match found
 #   import/config/botTeam - Optional team name for bot-only repositories
 #   import/config/corporateAffixes - Optional comma-separated corporate username affixes for team matching (e.g., "ionos,1and1")
+#   import/config/grapherOutputPath - Optional output path for language-grapher child imports
 class Archsight::Import::Handlers::Repository < Archsight::Import::Handler
   def execute
     @path = config("path")
@@ -35,19 +36,10 @@ class Archsight::Import::Handlers::Repository < Archsight::Import::Handler
           return
         end
       rescue StandardError => e
-        # Access denied or other git errors - create minimal artifact
-        if access_denied_error?(e.message)
-          progress.update("Access denied - creating minimal artifact")
-          write_minimal_artifact(
-            status: "inaccessible",
-            reason: "Repository not accessible",
-            error: e.message,
-            visibility: "private"
-          )
-          write_generates_meta
-          return
-        end
-        raise
+        raise unless access_denied_error?(e.message)
+
+        write_inaccessible_artifact(e.message)
+        return
       end
     end
 
@@ -80,9 +72,14 @@ class Archsight::Import::Handlers::Repository < Archsight::Import::Handler
     progress.update("Generating resource")
     resource = build_technology_artifact(@path, scc_data, git_data, team_result, license_data)
 
-    # Write output with self-marker for caching
-    yaml_content = YAML.dump(resource) + YAML.dump(self_marker)
-    write_yaml(yaml_content)
+    # Write output: artifact + child imports for every applicable language grapher + self-marker
+    artifact_name = resource["metadata"]["name"]
+    docs = [resource]
+    Archsight::Import::Registry.handlers_for(@path).each do |handler_class|
+      docs << grapher_import(artifact_name, handler_class)
+    end
+    docs << self_marker
+    write_yaml(docs.map { |d| YAML.dump(d) }.join)
 
     write_generates_meta
   end
@@ -159,6 +156,17 @@ class Archsight::Import::Handlers::Repository < Archsight::Import::Handler
     first_line.length > 100 ? "#{first_line[0, 97]}..." : first_line
   end
 
+  def write_inaccessible_artifact(error_message)
+    progress.update("Access denied - creating minimal artifact")
+    write_minimal_artifact(
+      status: "inaccessible",
+      reason: "Repository not accessible",
+      error: error_message,
+      visibility: "private"
+    )
+    write_generates_meta
+  end
+
   # Check if error message indicates access denied
   def access_denied_error?(message)
     return false if message.nil?
@@ -211,7 +219,7 @@ class Archsight::Import::Handlers::Repository < Archsight::Import::Handler
   private
 
   def run_scc(path)
-    cmd = ["scc", "-f", "json2", "--sort", "name", path]
+    cmd = ["scc", "--exclude-dir", ".git,.hg,.svn,vendor,node_modules", "-f", "json2", "--sort", "name", path]
 
     out, err, status = Open3.capture3(*cmd)
     raise "scc failed: #{cmd.join(" ")}\n#{err}" unless status.success?
@@ -453,6 +461,22 @@ class Archsight::Import::Handlers::Repository < Archsight::Import::Handler
     annotations
   end
 
+  def grapher_import(artifact_name, handler_class)
+    lang = handler_class.language_name
+    handler_name = Archsight::Import::Registry.name_for(handler_class)
+    child_name = "Import:#{lang.capitalize}Grapher:#{artifact_name.delete_prefix("Repo:")}"
+    child_annotations = {}
+    if (output_path = config("grapherOutputPath"))
+      child_annotations["import/outputPath"] = output_path
+    end
+    import_yaml(
+      name: child_name,
+      handler: handler_name,
+      config: { "path" => @path },
+      annotations: child_annotations
+    )
+  end
+
   def build_deployment_annotations(git_data)
     annotations = {}
 

data/lib/archsight/import/handlers/ruby_grapher.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+require_relative "grapher"
+require_relative "../registry"
+
+# RubyGrapher handler - analyses a Ruby repository and generates a GraphViz DOT
+# graph of its gem/package structure, stored as architecture/modules on
+# the TechnologyArtifact so it can be rendered in the frontend.
+#
+# Uses static regex analysis of require/require_relative statements.
+# Package paths are normalised to "/" separators internally so they are
+# compatible with the generic Grapher layout engine.
+#
+# Configuration:
+#   import/config/path     - Path to the Ruby repository root
+#   import/config/ranksep  - Horizontal gap between rank columns (default: 0.6)
+#   import/config/nodesep  - Vertical gap between nodes in a column (default: 0.15)
+class Archsight::Import::Handlers::RubyGrapher < Archsight::Import::Handlers::Grapher
+  def self.language_name = "ruby"
+
+  def self.applicable?(path)
+    File.exist?(File.join(path, "Gemfile")) ||
+      Dir.glob(File.join(path, "*.gemspec")).any? ||
+      Dir.glob(File.join(path, "*/*.gemspec")).any?
+  end
+
+  def wrap_single_module?
+    true
+  end
+
+  SKIP_DIRS = %w[test spec tests vendor .git node_modules tmp log coverage .bundle
+                 pkg doc docs generated].freeze
+
+  # Packages with more than this many path components are folded into their ancestor.
+  # Ruby gems use lib/<gem>/<feature>.rb, so depth 2 gives one level of features.
+  MAX_PKG_DEPTH = 2
+
+  private
+
+  # ── Module discovery ─────────────────────────────────────────────────────
+
+  def discover_modules(repo_root)
+    # Root-level gemspecs → single-gem repo.
+    # Use the actual top-level directory in lib/ as the module name so that
+    # package paths (which are lib-relative) match without re-prefixing.
+    root_gemspecs = Dir.glob(File.join(repo_root, "*.gemspec"))
+    if root_gemspecs.any?
+      return root_gemspecs.map do |path|
+        lib = File.join(repo_root, "lib")
+        mod_name = lib_top_dir(lib) || gemspec_name(path) || File.basename(path, ".gemspec")
+        [".", mod_name]
+      end
+    end
+
+    # Subdirectory gemspecs → monorepo
+    sub_gemspecs = Dir.glob(File.join(repo_root, "*/*.gemspec")).reject do |p|
+      SKIP_DIRS.any? { |d| p.split("/").include?(d) }
+    end
+    if sub_gemspecs.any?
+      return sub_gemspecs.sort.map do |path|
+        rel_dir = File.dirname(path).delete_prefix("#{repo_root}/")
+        lib = File.join(repo_root, rel_dir, "lib")
+        mod_name = lib_top_dir(lib) || gemspec_name(path) || File.basename(path, ".gemspec")
+        [rel_dir, mod_name]
+      end
+    end
+
+    # Fallback: Gemfile without gemspec
+    lib = File.join(repo_root, "lib")
+    [[".", lib_top_dir(lib) || File.basename(repo_root)]]
+  end
+
+  # ── Package collection ────────────────────────────────────────────────────
+
+  def collect_packages(repo_root, modules, _prefix)
+    # Build a map of lib dirs for cross-module require resolution:
+    # mod_name => absolute lib dir
+    lib_dirs = modules.filter_map do |rel_dir, mod_name|
+      abs_lib = File.join(rel_dir == "." ? repo_root : File.join(repo_root, rel_dir), "lib")
+      [mod_name, abs_lib] if Dir.exist?(abs_lib)
+    end.to_h
+
+    all_pkgs = {}
+
+    modules.each do |_rel_dir, mod_name| # rubocop:disable Style/HashEachMethods
+      lib_dir = lib_dirs[mod_name]
+      next unless lib_dir
+
+      scan_lib_dir(lib_dir, mod_name, lib_dirs, all_pkgs)
+    end
+
+    all_pkgs
+  end
+
+  # ── Scanning helpers ──────────────────────────────────────────────────────
+
+  def scan_lib_dir(lib_dir, mod_name, lib_dirs, all_pkgs)
+    safe_glob(File.join(lib_dir, "**", "*.rb")).each do |rb_file|
+      rel_parts = rb_file.delete_prefix("#{lib_dir}/").split("/")
+      next if rel_parts.any? { |p| SKIP_DIRS.include?(p) }
+
+      pkg = file_to_pkg(rb_file, lib_dir, mod_name)
+      pkg = cap_depth(pkg, mod_name)
+      all_pkgs[pkg] ||= []
+
+      extract_deps(rb_file, lib_dir, mod_name, lib_dirs).each do |dep|
+        dep = cap_depth(dep, mod_name)
+        next if dep == pkg || all_pkgs[pkg].include?(dep)
+
+        all_pkgs[pkg] << dep
+      end
+    end
+  end
+
+  def extract_deps(rb_file, lib_dir, mod_name, lib_dirs)
+    content = File.read(rb_file, encoding: "utf-8")
+    deps = []
+
+    content.scan(/^\s*require\s+["']([^"']+)["']/) do |(req)|
+      dep = resolve_require(req, lib_dirs)
+      deps << dep if dep
+    end
+
+    content.scan(/^\s*require_relative\s+["']([^"']+)["']/) do |(req)|
+      dep = resolve_require_relative(req, rb_file, lib_dir, mod_name)
+      deps << dep if dep
+    end
+
+    deps.uniq
+  rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError
+    []
+  end
+
+  # Resolve an absolute `require` path against all known lib dirs.
+  def resolve_require(req, lib_dirs)
+    lib_dirs.each do |mod_name, lib_dir|
+      rb_path = File.join(lib_dir, "#{req}.rb")
+      return file_to_pkg(rb_path, lib_dir, mod_name) if File.exist?(rb_path)
+
+      # require 'mod_name' or require 'mod_name/sub' with no .rb — treat as pkg path
+      return req if Dir.exist?(File.join(lib_dir, req))
+    end
+    nil
+  end
+
+  # Resolve a `require_relative` path relative to the current file.
+  def resolve_require_relative(req, rb_file, lib_dir, mod_name)
+    base_dir = File.dirname(rb_file)
+    expanded = File.expand_path(req, base_dir)
+
+    rb_path = expanded.end_with?(".rb") ? expanded : "#{expanded}.rb"
+    return file_to_pkg(rb_path, lib_dir, mod_name) if File.exist?(rb_path)
+
+    # Directory index file: foo/foo.rb
+    if Dir.exist?(expanded)
+      index = File.join(expanded, "#{File.basename(expanded)}.rb")
+      return file_to_pkg(index, lib_dir, mod_name) if File.exist?(index)
+    end
+
+    nil
+  end
+
+  # Convert an absolute .rb path to a slash-separated package path.
+  # Uses the full lib-relative path (without .rb) so that flat files like
+  # lib/gem/database.rb become their own package (gem/database) rather than
+  # collapsing into the root gem package. cap_depth then folds deeper paths.
+  def file_to_pkg(rb_path, lib_dir, mod_name)
+    rel = rb_path.delete_prefix("#{lib_dir}/").delete_suffix(".rb")
+    rel == mod_name ? mod_name : rel
+  end
+
+  # Fold packages deeper than MAX_PKG_DEPTH levels into their ancestor.
+  def cap_depth(pkg, mod_name)
+    suffix = pkg.delete_prefix("#{mod_name}/")
+    return mod_name if suffix == pkg # pkg == mod_name exactly
+
+    parts = suffix.split("/")
+    return pkg if parts.length <= MAX_PKG_DEPTH - 1
+
+    "#{mod_name}/#{parts.first(MAX_PKG_DEPTH - 1).join("/")}"
+  end
+
+  # Returns the single top-level directory inside lib/, which is the gem name
+  # as used on the filesystem (underscores, not hyphens).
+  def lib_top_dir(lib_dir)
+    return nil unless Dir.exist?(lib_dir)
+
+    dirs = Dir.children(lib_dir).select { |e| File.directory?(File.join(lib_dir, e)) }
+    dirs.length == 1 ? dirs.first : nil
+  end
+
+  def gemspec_name(gemspec_path)
+    content = begin
+      File.read(gemspec_path, encoding: "utf-8")
+    rescue StandardError
+      ""
+    end
+    match = content.match(/\.name\s*=\s*["']([^"']+)["']/)
+    match&.captures&.first
+  end
+end
+
+Archsight::Import::Registry.register("ruby-grapher", Archsight::Import::Handlers::RubyGrapher)