bmad-module-skill-forge 2.2.0 → 4.0.0

package/src/shared/scripts/skf-scan-manifests.py

@@ -17,9 +17,11 @@ the layout looks like a monorepo.
      ecosystem table this script implements.
 
   2. **skf-analyze-source / scan-project.md §2** — Finds the same set of
-     manifest files in the broader project-scan pass. Service-config files
-     (Dockerfile, docker-compose.yml) stay LLM-driven; only the manifest
-     enumeration is extracted here.
+     manifest files in the broader project-scan pass: its `candidates[]`
+     are the candidate boundaries, its `services[]` lists the Docker,
+     compose and serverless files, whose folders are candidate boundaries
+     too, and each manifest's `version` is the version a brief records
+     (step-auto-scope and generate-briefs read it).
 
 Pulling this into a deterministic script gives both stages identical
 manifest-set semantics (no LLM drift on what counts as a manifest), a stable
@@ -27,63 +29,173 @@ JSON envelope for the LLM to consume, and a single place to evolve the
 ecosystem matrix as new languages land.
 
 Subcommand:
-  scan <root> [--ecosystems=auto]
-      Walk `<root>` (depth 0–1 plus common monorepo packages/* paths) for
-      recognised manifests, parse each to extract `production` deps only,
-      and emit:
+  scan <root> [--ecosystems=auto] [--include-dev] [--workspaces-file <file>]
+      Walk `<root>` (every depth, pruning the excluded and hidden
+      directories) for recognised manifests, parse each one and emit:
         {
           "manifests": [
             {
               "path": "<rel-from-root, forward-slash>",
               "ecosystem": "<name>",
-              "deps": [{"name": "...", "version": "...|null"}]
+              "name": "<the package's own name>" | null,
+              "private": true | false | null,
+              "version": "<the package's own version>" | null,
+              "version_dynamic": true | false,
+              "description": "<the package's own description>" | null,
+              "deps": [{"name": "...", "version": "...|null"}],
+              "internal_deps": ["<name of another scanned manifest>", ...]
             },
             ...
           ],
-          "total_unique": N,    // unique dep names across all manifests
-          "monorepo": <bool>,    // >1 manifest of same ecosystem at non-overlapping depths
-          "warnings": ["..."]    // optional: only present if any warning was emitted
+          "total_unique": N,          // unique runtime dep names across all manifests
+          "monorepo": <bool>,         // >1 manifest of same ecosystem at non-overlapping depths
+          "folders": [                // each folder holding a manifest ("." for <root>), sorted
+            {"path": "apps/web", "names": ["web"]},   // the package names its manifests give
+            ...
+          ],
+          "services": ["deploy/docker-compose.yml", ...],   // each service definition, sorted
+          "candidates": [             // the candidate boundaries, sorted by path
+            {"path": "apps/web", "names": ["web"], "manifests": ["apps/web/package.json"],
+             "services": ["apps/web/Dockerfile"], "own_source_files": N,
+             "workspace_member": true | false | null},
+            ...
+          ],
+          "umbrella_candidates": [
+            {"path": "...", "name": "...", "ecosystem": "...",
+             "internal_dep_count": N, "member_count": M},
+            ...
+          ],
+          "searched_filenames": ["package.json", ...],   // every manifest name looked for
+          "warnings": ["..."]         // optional: only present if any warning was emitted
         }
       Default `--ecosystems=auto` detects all supported ecosystems. The flag
-      is currently a placeholder for future filtering — `auto` is the only
+      is currently a placeholder for future filtering: `auto` is the only
       accepted value today, but its presence keeps the CLI shape stable.
 
-Supported ecosystems (manifest filename → ecosystem):
-  - package.json                    → npm           (npm/pnpm/yarn share this)
-  - pyproject.toml                  → python        (pip/poetry/pdm)
-  - requirements.txt                → python
-  - setup.py                        → python
-  - setup.cfg                       → python
-  - Pipfile                         → python
-  - Cargo.toml                      → rust
-  - go.mod                          → go
-  - pom.xml                         → maven         (java/kotlin)
-  - build.gradle                    → gradle
-  - build.gradle.kts                → gradle
-  - Gemfile                         → ruby
-  - composer.json                   → composer
-  - Package.swift                   → swift
+      `--include-dev` also lists each manifest's development dependencies in
+      its `deps[]`, each tagged `"scope": "dev"` (a dependency with no
+      `scope` is a runtime one; a name the manifest already lists as runtime
+      is not repeated), and adds `total_unique_dev`: the unique names that
+      appear only as dev dependencies. Without the flag the output holds
+      runtime dependencies only, as before.
+
+      `--workspaces-file` reads the JSON skf-detect-workspaces.py printed
+      for the same root (its `--snapshot` output too): each candidate's
+      `workspace_member` says whether its folder is one of that JSON's
+      `workspaces[]` paths. Without it `workspace_member` is null.
+
+Manifest fields:
+  name           the name the manifest gives its own package: package.json,
+                 composer.json and pyproject.toml ([project] or
+                 [tool.poetry]) `name`, setup.py and setup.cfg `name`, Cargo
+                 [package] `name`, the go.mod `module` path, pom.xml
+                 `groupId:artifactId` (the group inherited from <parent> when
+                 absent), Package.swift `Package(name:)`; null for
+                 requirements.txt, Pipfile, Gemfile and Gradle build scripts,
+                 which name no package.
+  private        true when the manifest says the package is not published:
+                 package.json `"private": true`, Cargo `publish = false` (or
+                 `[]`), the `Private :: Do Not Upload` classifier or Poetry
+                 `package-mode = false`, pom.xml `maven.deploy.skip`; false
+                 when the manifest could set such a flag and does not; null
+                 when it cannot tell (go.mod, composer.json, Package.swift,
+                 the formats that name no package, a Cargo virtual workspace
+                 or a Cargo `publish` inherited from the workspace).
+  version        the version the manifest gives its own package, as it is
+                 written (no semver check): package.json and composer.json
+                 `version`, pyproject.toml `[project]` (or `[tool.poetry]`)
+                 `version`, setup.py and setup.cfg `version` when it is a
+                 literal, Cargo `[package] version` (a `version.workspace =
+                 true` read from the `[workspace.package] version` of the
+                 nearest Cargo.toml above it that has one), pom.xml
+                 `<version>` (the parent's when absent), a Gradle build
+                 script's `version = '...'`; null when it gives none (go.mod,
+                 Package.swift and the formats that name no package).
+  version_dynamic true when the manifest says the version is computed at
+                 build time, so it is not in the manifest: pyproject.toml
+                 `dynamic = ["version"]`, a setup.cfg `attr:` or `file:`
+                 version, a setup.py `version=` that is not a literal.
+  description    the one-line description the manifest gives its package:
+                 package.json and composer.json `description`,
+                 pyproject.toml `[project]` (or `[tool.poetry]`)
+                 `description`, setup.py and setup.cfg `description` when
+                 it is a literal, Cargo `[package] description` (null for
+                 one inherited from the workspace), pom.xml `<description>`,
+                 a Gradle build script's `description = '...'`; else null.
+  internal_deps  the runtime dependencies that name another scanned manifest
+                 of the same ecosystem (the workspace members this one uses),
+                 as those members spell their names, sorted. Names compare
+                 as the ecosystem does (PEP 503 for Python, `-` and `_` alike
+                 for Rust, case-insensitive for Composer and Swift), and a
+                 pom.xml `${project.groupId}` resolves to its own group.
+
+services lists the service definitions the walk finds, by path from the
+root: a Docker, compose or serverless file (is_service_definition, which
+skf-disqualify-candidates.py's `service_definition` signal reads too).
+
+candidates lists the folders a project scan proposes as boundaries (Candidate
+boundaries in skf-analyze-source's unit-detection-heuristics.md names this
+list), each folder of `folders[]` and each folder that holds a `services[]`
+file, less those a workspace only gathers: a folder is left out when it
+holds no source file of its own (`own_source_files`, the source files under
+it outside every other such folder below it) and another such folder lies
+below it. So a workspace root such as a private package.json that lists
+`workspaces`, or a Cargo virtual workspace, with no code of its own is left
+out, and so is `.` unless it holds code of its own or nothing else is
+proposed. A source file is one skf-detect-language.py's is_source_file()
+counts (the sibling in this folder, which skf-detect-workspaces.py's snapshot
+reads too). Each record gives the folder's `names` (as `folders[]`), the
+`manifests` and `services` files directly in it, `own_source_files` and
+`workspace_member` (see --workspaces-file).
+
+umbrella_candidates lists the published manifests (named, `private` not
+true) whose internal_deps cover at least 2 of the other published members of
+their ecosystem, and at least half of them: a root or facade package that
+depends on the members it would re-export. A private package (an example or
+an app) is never a candidate, since a facade is the public surface. Sorted by
+internal_dep_count (highest first), then path. It is a signal for the
+cohesion judgment, not a verdict.
+
+Supported ecosystems (manifest filename, then ecosystem): the
+`MANIFEST_ECOSYSTEMS` table below. `references/manifest-patterns.md` in
+skf-create-stack-skill documents it, and test/test-skf-scan-manifests.py
+fails when the two drift.
 
 Parsing approach: JSON manifests via stdlib `json`, TOML via stdlib
 `tomllib` (3.11+); text manifests (requirements.txt, Pipfile, setup.py,
 setup.cfg, go.mod, Gemfile, pom.xml, build.gradle*, Package.swift) via
-ad-hoc regex extraction. Production deps only — devDependencies /
-[tool.poetry.dev-dependencies] / require-dev are intentionally skipped to
-keep the consumer-facing dep list focused on runtime libraries. Unparseable
-manifests emit a top-level `warnings[]` entry and the manifest still appears
-in `manifests[]` with whatever deps were salvageable (possibly empty).
+ad-hoc regex extraction. `deps[]` holds production dependencies by default,
+so the consumer-facing list stays focused on runtime libraries. The
+development sections read with `--include-dev`: package.json
+`devDependencies`; pyproject.toml `[dependency-groups]`, the
+`[project.optional-dependencies]` extras named dev, develop, development,
+test, tests, testing, lint, docs or doc, `[tool.poetry.dev-dependencies]`,
+every `[tool.poetry.group.<name>.dependencies]`,
+`[tool.pdm.dev-dependencies]` and `[tool.uv] dev-dependencies`; setup.py and
+setup.cfg `tests_require` and those same extras; Pipfile `[dev-packages]`;
+Cargo `[dev-dependencies]`; pom.xml `<scope>test</scope>`; Gradle `test*`
+and `androidTest*` configurations; Gemfile gems inside a `group
+:development` or `:test` block; composer.json `require-dev`. requirements.txt,
+go.mod and Package.swift have no development section. Unparseable manifests
+emit a top-level `warnings[]` entry and the manifest still appears in
+`manifests[]` with whatever deps were salvageable (possibly empty).
 
 Exit codes:
-  0  — success (including: zero manifests found — emit empty result)
-  1  — user error (bad root path, root not a directory)
+  0  success (including: zero manifests found, which emits an empty result)
+  1  user error (bad root path, root not a directory, a --workspaces-file
+     that cannot be read as JSON), or skf-detect-language.py missing beside
+     this script
 
 CLI example:
   uv run skf-scan-manifests.py scan /path/to/repo
+  uv run skf-scan-manifests.py scan /path/to/repo --include-dev
+  uv run skf-scan-manifests.py scan /path/to/repo --workspaces-file snapshot.json
 """
 
 from __future__ import annotations
 
 import argparse
+import importlib.util
 import json
 import re
 import sys
@@ -101,6 +213,9 @@ from typing import Iterable
 # (we test each name in order, but every match is recorded). Lowercase compare
 # is NOT used — manifest filenames are case-sensitive on POSIX, and Windows
 # CI normalises case during the FS walk anyway.
+# The same order is `searched_filenames`. The Supported Ecosystems table in
+# skf-create-stack-skill's references/manifest-patterns.md mirrors this dict
+# (test/test-skf-scan-manifests.py keeps the two equal).
 MANIFEST_ECOSYSTEMS: dict[str, str] = {
     "package.json": "npm",
     "pyproject.toml": "python",
@@ -119,7 +234,8 @@ MANIFEST_ECOSYSTEMS: dict[str, str] = {
 }
 
 # Directories the scan must never descend into. Mirrors
-# references/manifest-patterns.md "Scan Exclusion Patterns".
+# references/manifest-patterns.md "Scan Exclusion Patterns"
+# (test/test-skf-scan-manifests.py keeps the two equal).
 EXCLUDED_DIRS: frozenset[str] = frozenset(
     {
         "node_modules",
@@ -156,13 +272,56 @@ def _is_excluded_dir(name: str) -> bool:
     return False
 
 
+# A service definition: a Docker, compose or serverless file, by its
+# lower-cased name, and a `Dockerfile.<x>` or `<x>.dockerfile` variant.
+SERVICE_DEFINITION_NAMES: frozenset[str] = frozenset({
+    "dockerfile", "containerfile", "docker-compose.yml", "docker-compose.yaml",
+    "compose.yml", "compose.yaml", "serverless.yml", "serverless.yaml",
+})
+
+
+def is_service_definition(name: str) -> bool:
+    """True for the file name of a service definition (SERVICE_DEFINITION_NAMES)."""
+    lname = name.lower()
+    return (
+        lname in SERVICE_DEFINITION_NAMES
+        or lname.startswith("dockerfile.")
+        or lname.endswith(".dockerfile")
+    )
+
+
+_DETECT_LANGUAGE = None
+
+
+def _is_source_file(name: str) -> bool:
+    """skf-detect-language.py's is_source_file(), the sibling beside this
+    script, loaded once. Raises ImportError when it cannot be loaded."""
+    global _DETECT_LANGUAGE
+    if _DETECT_LANGUAGE is None:
+        path = Path(__file__).resolve().parent / "skf-detect-language.py"
+        spec = importlib.util.spec_from_file_location("skf_detect_language", path)
+        if spec is None or spec.loader is None or not path.is_file():
+            raise ImportError(f"cannot load {path.name} beside {Path(__file__).name}")
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        _DETECT_LANGUAGE = module
+    return _DETECT_LANGUAGE.is_source_file(name)
+
+
 def find_manifests(root: Path) -> list[Path]:
     """Walk `root` returning every recognised manifest file.
 
     Excludes `EXCLUDED_DIRS` and all hidden directories. Returns absolute
     paths in stable sorted order so caller output is reproducible.
     """
+    return _walk(root)[0]
+
+
+def _walk(root: Path, sources: list[str] | None = None) -> tuple[list[Path], list[Path]]:
+    """(manifests, service definitions) under `root`, as find_manifests
+    walks; each source file's path from `root` joins `sources` when given."""
     found: list[Path] = []
+    services: list[Path] = []
     # Iterative BFS so we can prune at the directory level.
     stack: list[Path] = [root]
     while stack:
@@ -179,10 +338,15 @@ def find_manifests(root: Path) -> list[Path]:
                     stack.append(entry)
                 elif entry.is_file() and entry.name in MANIFEST_ECOSYSTEMS:
                     found.append(entry)
+                elif entry.is_file() and is_service_definition(entry.name):
+                    services.append(entry)
+                elif sources is not None and entry.is_file() and _is_source_file(entry.name):
+                    sources.append(entry.relative_to(root).as_posix())
             except OSError:
                 continue
     found.sort()
-    return found
+    services.sort()
+    return found, services
 
 
 # --------------------------------------------------------------------------
@@ -583,23 +747,672 @@ PARSERS = {
 }
 
 
+# --------------------------------------------------------------------------
+# Development dependencies (--include-dev)
+# --------------------------------------------------------------------------
+
+
+# Optional-dependency extras read as development dependencies.
+DEV_EXTRAS: frozenset[str] = frozenset(
+    {"dev", "develop", "development", "test", "tests", "testing", "lint", "docs", "doc"}
+)
+
+
+def _table_deps(
+    table: object, *, skip_python: bool = False, star_as_none: bool = False
+) -> list[Dep]:
+    """Deps from a TOML or JSON table of name -> version string or spec table.
+
+    `skip_python` drops Poetry's `python` constraint; `star_as_none` reads a
+    `*` version as no version, as Pipfile does.
+    """
+    if not isinstance(table, dict):
+        return []
+    deps: list[Dep] = []
+    for name, spec in table.items():
+        if not isinstance(name, str) or (skip_python and name.lower() == "python"):
+            continue
+        if isinstance(spec, dict):
+            spec = spec.get("version")
+        version = spec if isinstance(spec, str) else None
+        if star_as_none and version == "*":
+            version = None
+        deps.append({"name": name, "version": version})
+    return deps
+
+
+def _requirement_list(entries: object) -> list[Dep]:
+    """Deps from a list of PEP 508 strings (tables such as include-group skipped)."""
+    if not isinstance(entries, list):
+        return []
+    deps: list[Dep] = []
+    for entry in entries:
+        if isinstance(entry, str):
+            parsed = _parse_pep508_or_pip(entry)
+            if parsed:
+                deps.append(parsed)
+    return deps
+
+
+def dev_package_json(text: str) -> list[Dep]:
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError:
+        return []
+    return _table_deps(data.get("devDependencies")) if isinstance(data, dict) else []
+
+
+def dev_pyproject_toml(text: str) -> list[Dep]:
+    try:
+        data = tomllib.loads(text)
+    except tomllib.TOMLDecodeError:
+        return []
+    deps: list[Dep] = []
+    groups = data.get("dependency-groups")
+    if isinstance(groups, dict):
+        for entries in groups.values():
+            deps.extend(_requirement_list(entries))
+    project = data.get("project")
+    extras = project.get("optional-dependencies") if isinstance(project, dict) else None
+    if isinstance(extras, dict):
+        for extra, entries in extras.items():
+            if isinstance(extra, str) and extra.lower() in DEV_EXTRAS:
+                deps.extend(_requirement_list(entries))
+    tool = data.get("tool") if isinstance(data.get("tool"), dict) else {}
+    poetry = tool.get("poetry")
+    if isinstance(poetry, dict):
+        deps.extend(_table_deps(poetry.get("dev-dependencies"), skip_python=True))
+        poetry_groups = poetry.get("group")
+        if isinstance(poetry_groups, dict):
+            for group in poetry_groups.values():
+                if isinstance(group, dict):
+                    deps.extend(_table_deps(group.get("dependencies"), skip_python=True))
+    pdm = tool.get("pdm")
+    if isinstance(pdm, dict) and isinstance(pdm.get("dev-dependencies"), dict):
+        for entries in pdm["dev-dependencies"].values():
+            deps.extend(_requirement_list(entries))
+    uv = tool.get("uv")
+    if isinstance(uv, dict):
+        deps.extend(_requirement_list(uv.get("dev-dependencies")))
+    return deps
+
+
+def dev_setup_py(text: str) -> list[Dep]:
+    deps: list[Dep] = []
+    m = re.search(r"tests_require\s*=\s*\[([^\]]*)\]", text, re.DOTALL)
+    if m:
+        for match in re.finditer(r"""(['"])([^'"]+)\1""", m.group(1)):
+            parsed = _parse_pep508_or_pip(match.group(2).strip())
+            if parsed:
+                deps.append(parsed)
+    extras = re.search(r"extras_require\s*=\s*\{(.*?)\}", text, re.DOTALL)
+    if extras:
+        for group in re.finditer(
+            r"""(['"])([\w.-]+)\1\s*:\s*\[([^\]]*)\]""", extras.group(1)
+        ):
+            if group.group(2).lower() not in DEV_EXTRAS:
+                continue
+            for match in re.finditer(r"""(['"])([^'"]+)\1""", group.group(3)):
+                parsed = _parse_pep508_or_pip(match.group(2).strip())
+                if parsed:
+                    deps.append(parsed)
+    return deps
+
+
+def _cfg_section(text: str, name: str) -> str:
+    sec = re.search(
+        rf"^\[{re.escape(name)}\][ \t]*$(.*?)(?=^\[|\Z)", text, re.MULTILINE | re.DOTALL
+    )
+    return sec.group(1) if sec else ""
+
+
+def _cfg_values(value: str) -> list[Dep]:
+    deps: list[Dep] = []
+    for line in value.splitlines():
+        parsed = _parse_pep508_or_pip(line.strip())
+        if parsed:
+            deps.append(parsed)
+    return deps
+
+
+# `key = value` in a setup.cfg section, with its indented continuation lines.
+_CFG_OPTION = re.compile(
+    r"^(?P<key>[\w.-]+)[ \t]*=[ \t]*(?P<value>.*(?:\n[ \t]+\S.*)*)", re.MULTILINE
+)
+
+
+def dev_setup_cfg(text: str) -> list[Dep]:
+    deps: list[Dep] = []
+    for m in _CFG_OPTION.finditer(_cfg_section(text, "options")):
+        if m.group("key") == "tests_require":
+            deps.extend(_cfg_values(m.group("value")))
+    for m in _CFG_OPTION.finditer(_cfg_section(text, "options.extras_require")):
+        if m.group("key").lower() in DEV_EXTRAS:
+            deps.extend(_cfg_values(m.group("value")))
+    return deps
+
+
+def dev_pipfile(text: str) -> list[Dep]:
+    try:
+        data = tomllib.loads(text)
+    except tomllib.TOMLDecodeError:
+        return []
+    return _table_deps(data.get("dev-packages"), star_as_none=True)
+
+
+def dev_cargo_toml(text: str) -> list[Dep]:
+    try:
+        data = tomllib.loads(text)
+    except tomllib.TOMLDecodeError:
+        return []
+    return _table_deps(data.get("dev-dependencies"))
+
+
+def dev_pom_xml(text: str) -> list[Dep]:
+    deps: list[Dep] = []
+    for block in re.finditer(r"<dependency>(.*?)</dependency>", text, re.DOTALL):
+        body = block.group(1)
+        scope_m = re.search(r"<scope>\s*(.*?)\s*</scope>", body)
+        if not scope_m or scope_m.group(1).strip().lower() != "test":
+            continue
+        gid = re.search(r"<groupId>\s*(.*?)\s*</groupId>", body)
+        aid = re.search(r"<artifactId>\s*(.*?)\s*</artifactId>", body)
+        ver = re.search(r"<version>\s*(.*?)\s*</version>", body)
+        if gid and aid:
+            deps.append(
+                {
+                    "name": f"{gid.group(1).strip()}:{aid.group(1).strip()}",
+                    "version": ver.group(1).strip() if ver else None,
+                }
+            )
+    return deps
+
+
+# Whitespace or `(` must follow the configuration, so the `"test"` inside
+# `testImplementation(kotlin("test"))` never starts a match.
+_GRADLE_DEV_DEP = re.compile(
+    r"""\b(?:test|androidTest)[A-Za-z]*
+        (?:\s+|\s*\(\s*)
+        (?:['"])
+        (?P<coord>[^'"\n]+)
+        (?:['"])
+        """,
+    re.VERBOSE,
+)
+
+
+def dev_gradle(text: str) -> list[Dep]:
+    deps: list[Dep] = []
+    for m in _GRADLE_DEV_DEP.finditer(text):
+        parts = m.group("coord").split(":")
+        if len(parts) >= 2:
+            deps.append(
+                {"name": f"{parts[0]}:{parts[1]}", "version": parts[2] if len(parts) >= 3 else None}
+            )
+    return deps
+
+
+def dev_gemfile(text: str) -> list[Dep]:
+    """Gems inside `group :development` / `group :test` blocks."""
+    deps: list[Dep] = []
+    depth = 0
+    for raw_line in text.splitlines():
+        line = raw_line.split("#", 1)[0]
+        if re.search(r"group\s+:(development|test)\b", line):
+            depth += 1
+            continue
+        if depth > 0:
+            if re.search(r"\bend\b", line):
+                depth -= 1
+                continue
+            m = _GEMFILE_LINE.match(line)
+            if m:
+                deps.append({"name": m.group("name"), "version": m.group("version")})
+    return deps
+
+
+def dev_composer_json(text: str) -> list[Dep]:
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError:
+        return []
+    if not isinstance(data, dict):
+        return []
+    return [
+        d
+        for d in _table_deps(data.get("require-dev"))
+        if d["name"].lower() != "php" and not d["name"].startswith("ext-")
+    ]
+
+
+DEV_PARSERS = {
+    "package.json": dev_package_json,
+    "pyproject.toml": dev_pyproject_toml,
+    "setup.py": dev_setup_py,
+    "setup.cfg": dev_setup_cfg,
+    "Pipfile": dev_pipfile,
+    "Cargo.toml": dev_cargo_toml,
+    "pom.xml": dev_pom_xml,
+    "build.gradle": dev_gradle,
+    "build.gradle.kts": dev_gradle,
+    "Gemfile": dev_gemfile,
+    "composer.json": dev_composer_json,
+}
+
+
+# --------------------------------------------------------------------------
+# Package identity: the manifest's own name and publish flag
+# --------------------------------------------------------------------------
+
+
+PRIVATE_CLASSIFIER = "Private :: Do Not Upload"
+
+Identity = tuple  # (name: str | None, private: bool | None)
+
+
+def identity_package_json(text: str) -> Identity:
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError:
+        return None, None
+    if not isinstance(data, dict):
+        return None, None
+    name = data.get("name") if isinstance(data.get("name"), str) and data.get("name") else None
+    return name, data.get("private") is True
+
+
+def identity_pyproject_toml(text: str) -> Identity:
+    try:
+        data = tomllib.loads(text)
+    except tomllib.TOMLDecodeError:
+        return None, None
+    project = data.get("project") if isinstance(data.get("project"), dict) else {}
+    tool = data.get("tool") if isinstance(data.get("tool"), dict) else {}
+    poetry = tool.get("poetry") if isinstance(tool.get("poetry"), dict) else {}
+    name = project.get("name") or poetry.get("name")
+    name = name if isinstance(name, str) and name else None
+    if not project and not poetry:
+        return None, None
+    classifiers = [
+        c
+        for source in (project.get("classifiers"), poetry.get("classifiers"))
+        if isinstance(source, list)
+        for c in source
+    ]
+    private = PRIVATE_CLASSIFIER in classifiers or poetry.get("package-mode") is False
+    return name, private
+
+
+def identity_setup_py(text: str) -> Identity:
+    call = re.search(r"\bsetup\s*\((.*)", text, re.DOTALL)
+    m = re.search(r"""\bname\s*=\s*(['"])([^'"]+)\1""", call.group(1)) if call else None
+    if not m:
+        return None, None
+    return m.group(2), PRIVATE_CLASSIFIER in text
+
+
+def identity_setup_cfg(text: str) -> Identity:
+    metadata = _cfg_section(text, "metadata")
+    m = re.search(r"^name[ \t]*=[ \t]*(\S+)", metadata, re.MULTILINE)
+    if not m:
+        return None, None
+    return m.group(1), PRIVATE_CLASSIFIER in metadata
+
+
+def identity_cargo_toml(text: str) -> Identity:
+    try:
+        data = tomllib.loads(text)
+    except tomllib.TOMLDecodeError:
+        return None, None
+    package = data.get("package")
+    if not isinstance(package, dict):
+        return None, None  # a virtual workspace manifest names no package
+    name = package.get("name") if isinstance(package.get("name"), str) else None
+    publish = package.get("publish")
+    if isinstance(publish, dict):
+        return name, None  # `publish.workspace = true`: set by the workspace root
+    return name, publish is False or publish == []
+
+
+def identity_go_mod(text: str) -> Identity:
+    m = re.search(r"""^\s*module\s+["`]?([^\s"`]+)""", text, re.MULTILINE)
+    return (m.group(1) if m else None), None
+
+
+# Sections of a pom.xml whose <groupId>/<artifactId> are not the project's own.
+_POM_FOREIGN_SECTIONS = (
+    "parent",
+    "dependencies",
+    "dependencyManagement",
+    "build",
+    "profiles",
+    "reporting",
+    "modules",
+)
+
+
+def identity_pom_xml(text: str) -> Identity:
+    text = re.sub(r"<!--.*?-->", "", text, flags=re.DOTALL)
+    parent = re.search(r"<parent>(.*?)</parent>", text, re.DOTALL)
+    own = text
+    for tag in _POM_FOREIGN_SECTIONS:
+        own = re.sub(rf"<{tag}>.*?</{tag}>", "", own, flags=re.DOTALL)
+    aid = re.search(r"<artifactId>\s*(.*?)\s*</artifactId>", own)
+    if not aid:
+        return None, None
+    gid = re.search(r"<groupId>\s*(.*?)\s*</groupId>", own)
+    if not gid and parent:
+        gid = re.search(r"<groupId>\s*(.*?)\s*</groupId>", parent.group(1))
+    name = f"{gid.group(1)}:{aid.group(1)}" if gid else aid.group(1)
+    skip = re.search(r"<maven\.deploy\.skip>\s*true\s*</maven\.deploy\.skip>", own)
+    return name, skip is not None
+
+
+def identity_composer_json(text: str) -> Identity:
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError:
+        return None, None
+    if not isinstance(data, dict):
+        return None, None
+    name = data.get("name")
+    return (name if isinstance(name, str) and name else None), None
+
+
+def identity_package_swift(text: str) -> Identity:
+    m = re.search(r'\bPackage\s*\(\s*name\s*:\s*"([^"]+)"', text)
+    return (m.group(1) if m else None), None
+
+
+# --------------------------------------------------------------------------
+# Package version: the version the manifest gives its own package
+# --------------------------------------------------------------------------
+
+
+# (version, how): `how` is "dynamic" for a version computed at build time,
+# "workspace" for a Cargo `version.workspace = true`, else None.
+Version = tuple
+
+
+def _literal(value: object) -> str | None:
+    return value.strip() if isinstance(value, str) and value.strip() else None
+
+
+def version_package_json(text: str) -> Version:
+    data = json.loads(text)
+    return (_literal(data.get("version")) if isinstance(data, dict) else None), None
+
+
+def version_pyproject_toml(text: str) -> Version:
+    data = tomllib.loads(text)
+    project = data.get("project") if isinstance(data.get("project"), dict) else {}
+    tool = data.get("tool") if isinstance(data.get("tool"), dict) else {}
+    poetry = tool.get("poetry") if isinstance(tool.get("poetry"), dict) else {}
+    version = _literal(project.get("version")) or _literal(poetry.get("version"))
+    dynamic = isinstance(project.get("dynamic"), list) and "version" in project["dynamic"]
+    return version, ("dynamic" if dynamic and version is None else None)
+
+
+def version_setup_py(text: str) -> Version:
+    call = re.search(r"\bsetup\s*\((.*)", text, re.DOTALL)
+    if not call:
+        return None, None
+    m = re.search(r"""\bversion\s*=\s*(['"])([^'"]+)\1""", call.group(1))
+    if m:
+        return m.group(2).strip(), None
+    return None, ("dynamic" if re.search(r"\bversion\s*=", call.group(1)) else None)
+
+
+def version_setup_cfg(text: str) -> Version:
+    m = re.search(r"^version[ \t]*=[ \t]*(\S+)", _cfg_section(text, "metadata"), re.MULTILINE)
+    if not m:
+        return None, None
+    return (None, "dynamic") if m.group(1).startswith(("attr:", "file:")) else (m.group(1), None)
+
+
+def version_cargo_toml(text: str) -> Version:
+    package = tomllib.loads(text).get("package")
+    if not isinstance(package, dict):
+        return None, None
+    version = package.get("version")
+    if isinstance(version, dict) and version.get("workspace") is True:
+        return None, "workspace"
+    return _literal(version), None
+
+
+def cargo_workspace_version(text: str) -> str | None:
+    """The `[workspace.package] version` a Cargo workspace root gives its members."""
+    workspace = tomllib.loads(text).get("workspace")
+    package = workspace.get("package") if isinstance(workspace, dict) else None
+    return _literal(package.get("version")) if isinstance(package, dict) else None
+
+
+def version_pom_xml(text: str) -> Version:
+    text = re.sub(r"<!--.*?-->", "", text, flags=re.DOTALL)
+    parent = re.search(r"<parent>(.*?)</parent>", text, re.DOTALL)
+    own = text
+    for tag in _POM_FOREIGN_SECTIONS:
+        own = re.sub(rf"<{tag}>.*?</{tag}>", "", own, flags=re.DOTALL)
+    m = re.search(r"<version>\s*(.*?)\s*</version>", own)
+    if not m and parent:
+        m = re.search(r"<version>\s*(.*?)\s*</version>", parent.group(1))
+    return (m.group(1) or None) if m else None, None
+
+
+def version_gradle(text: str) -> Version:
+    m = re.search(r"""^\s*version\s*=\s*['"]([^'"]+)['"]""", text, re.MULTILINE)
+    return (m.group(1) if m else None), None
+
+
+VERSION_READERS = {
+    "package.json": version_package_json,
+    "pyproject.toml": version_pyproject_toml,
+    "setup.py": version_setup_py,
+    "setup.cfg": version_setup_cfg,
+    "Cargo.toml": version_cargo_toml,
+    "pom.xml": version_pom_xml,
+    "build.gradle": version_gradle,
+    "build.gradle.kts": version_gradle,
+    "composer.json": version_package_json,
+}
+
+
+# --------------------------------------------------------------------------
+# Package description: the one line the manifest gives its own package
+# --------------------------------------------------------------------------
+
+
+def description_package_json(text: str) -> str | None:
+    data = json.loads(text)
+    return _literal(data.get("description")) if isinstance(data, dict) else None
+
+
+def description_pyproject_toml(text: str) -> str | None:
+    data = tomllib.loads(text)
+    project = data.get("project") if isinstance(data.get("project"), dict) else {}
+    tool = data.get("tool") if isinstance(data.get("tool"), dict) else {}
+    poetry = tool.get("poetry") if isinstance(tool.get("poetry"), dict) else {}
+    return _literal(project.get("description")) or _literal(poetry.get("description"))
+
+
+def description_setup_py(text: str) -> str | None:
+    call = re.search(r"\bsetup\s*\((.*)", text, re.DOTALL)
+    m = re.search(r"""\bdescription\s*=\s*(['"])([^'"]+)\1""", call.group(1)) if call else None
+    return _literal(m.group(2)) if m else None
+
+
+def description_setup_cfg(text: str) -> str | None:
+    m = re.search(r"^description[ \t]*=[ \t]*(.+)$", _cfg_section(text, "metadata"), re.MULTILINE)
+    return _literal(m.group(1)) if m and not m.group(1).startswith(("attr:", "file:")) else None
+
+
+def description_cargo_toml(text: str) -> str | None:
+    package = tomllib.loads(text).get("package")
+    return _literal(package.get("description")) if isinstance(package, dict) else None
+
+
+def description_pom_xml(text: str) -> str | None:
+    own = re.sub(r"<!--.*?-->", "", text, flags=re.DOTALL)
+    for tag in _POM_FOREIGN_SECTIONS:
+        own = re.sub(rf"<{tag}>.*?</{tag}>", "", own, flags=re.DOTALL)
+    m = re.search(r"<description>\s*(.*?)\s*</description>", own, re.DOTALL)
+    return _literal(" ".join(m.group(1).split())) if m else None
+
+
+def description_gradle(text: str) -> str | None:
+    m = re.search(r"""^\s*description\s*=\s*['"]([^'"]+)['"]""", text, re.MULTILINE)
+    return _literal(m.group(1)) if m else None
+
+
+DESCRIPTION_READERS = {
+    "package.json": description_package_json,
+    "pyproject.toml": description_pyproject_toml,
+    "setup.py": description_setup_py,
+    "setup.cfg": description_setup_cfg,
+    "Cargo.toml": description_cargo_toml,
+    "pom.xml": description_pom_xml,
+    "build.gradle": description_gradle,
+    "build.gradle.kts": description_gradle,
+    "composer.json": description_package_json,
+}
+
+
+IDENTITY_READERS = {
+    "package.json": identity_package_json,
+    "pyproject.toml": identity_pyproject_toml,
+    "setup.py": identity_setup_py,
+    "setup.cfg": identity_setup_cfg,
+    "Cargo.toml": identity_cargo_toml,
+    "go.mod": identity_go_mod,
+    "pom.xml": identity_pom_xml,
+    "composer.json": identity_composer_json,
+    "Package.swift": identity_package_swift,
+}
+
+
 # --------------------------------------------------------------------------
 # Aggregation
 # --------------------------------------------------------------------------
 
 
-def _parse_manifest(path: Path) -> tuple[list[Dep], list[str]]:
-    """Read and parse a single manifest file. Returns (deps, warnings)."""
+def _describe_manifest(path: Path, include_dev: bool) -> tuple[dict, list[str]]:
+    """Read one manifest once: its identity, runtime deps and (optionally)
+    dev deps tagged `scope: dev`. Returns (record fields, warnings)."""
+    record: dict = {"name": None, "private": None, "version": None, "version_how": None,
+                    "workspace_version": None, "description": None, "deps": []}
     parser = PARSERS.get(path.name)
     if parser is None:
-        return [], [f"{path.name}: no parser registered"]
+        return record, [f"{path.name}: no parser registered"]
     text = _safe_read_text(path)
     if text is None:
-        return [], [f"{path.name}: file unreadable"]
+        return record, [f"{path.name}: file unreadable"]
+    try:
+        deps, warnings = parser(text)
+    except Exception as exc:  # noqa: BLE001 (best-effort parsing)
+        deps, warnings = [], [f"{path.name}: parser raised {type(exc).__name__}: {exc}"]
+    # identity and dev readers are best-effort too; the runtime parser has
+    # already reported a malformed file
+    reader = IDENTITY_READERS.get(path.name)
+    try:
+        record["name"], record["private"] = reader(text) if reader else (None, None)
+    except Exception:  # noqa: BLE001
+        pass
+    version_reader = VERSION_READERS.get(path.name)
     try:
-        return parser(text)
-    except Exception as exc:  # noqa: BLE001 — best-effort parsing
-        return [], [f"{path.name}: parser raised {type(exc).__name__}: {exc}"]
+        if version_reader:
+            record["version"], record["version_how"] = version_reader(text)
+        if path.name == "Cargo.toml":
+            record["workspace_version"] = cargo_workspace_version(text)
+    except Exception:  # noqa: BLE001
+        pass
+    description_reader = DESCRIPTION_READERS.get(path.name)
+    try:
+        record["description"] = description_reader(text) if description_reader else None
+    except Exception:  # noqa: BLE001
+        pass
+    if include_dev and path.name in DEV_PARSERS:
+        try:
+            dev = DEV_PARSERS[path.name](text)
+        except Exception:  # noqa: BLE001
+            dev = []
+        runtime = {d["name"] for d in deps}
+        for dep in dev:
+            if dep["name"] not in runtime:
+                runtime.add(dep["name"])
+                deps.append({**dep, "scope": "dev"})
+    record["deps"] = deps
+    return record, warnings
+
+
+def _member_key(ecosystem: str, name: str) -> str:
+    """How an ecosystem compares package names."""
+    if ecosystem == "python":
+        return re.sub(r"[-_.]+", "-", name).lower()
+    if ecosystem == "rust":
+        return name.lower().replace("_", "-")
+    if ecosystem in ("composer", "swift"):
+        return name.lower()
+    return name
+
+
+def _add_internal_deps(manifests: list[dict]) -> None:
+    """Set each manifest's `internal_deps`: its runtime deps that name another
+    scanned manifest of the same ecosystem, spelled as that member names itself."""
+    members: dict[str, dict[str, str]] = {}
+    for m in manifests:
+        if m["name"]:
+            members.setdefault(m["ecosystem"], {}).setdefault(
+                _member_key(m["ecosystem"], m["name"]), m["name"]
+            )
+    for m in manifests:
+        ecosystem = m["ecosystem"]
+        known = members.get(ecosystem, {})
+        own = _member_key(ecosystem, m["name"]) if m["name"] else None
+        # a pom.xml names sibling modules as ${project.groupId}:<artifact>
+        group = None
+        if ecosystem == "maven" and m["name"] and ":" in m["name"]:
+            group = m["name"].split(":", 1)[0]
+        internal: set[str] = set()
+        for dep in m["deps"]:
+            name = dep.get("name")
+            if not isinstance(name, str) or dep.get("scope") == "dev":
+                continue
+            if group:
+                for token in ("${project.groupId}", "${pom.groupId}", "${groupId}"):
+                    name = name.replace(token, group)
+            key = _member_key(ecosystem, name)
+            if key != own and key in known:
+                internal.add(known[key])
+        m["internal_deps"] = sorted(internal)
+
+
+def _umbrella_candidates(manifests: list[dict]) -> list[dict]:
+    """Published manifests whose internal_deps cover at least 2, and at least
+    half, of the other published members of their ecosystem."""
+    published: dict[str, set[str]] = {}
+    for m in manifests:
+        if m["name"] and m["private"] is not True:
+            published.setdefault(m["ecosystem"], set()).add(_member_key(m["ecosystem"], m["name"]))
+    candidates: list[dict] = []
+    for m in manifests:
+        if not m["name"] or m["private"] is True:
+            continue
+        ecosystem = m["ecosystem"]
+        own = _member_key(ecosystem, m["name"])
+        pool = published.get(ecosystem, set()) - {own}
+        covered = {_member_key(ecosystem, n) for n in m["internal_deps"]} & pool
+        if len(covered) >= 2 and 2 * len(covered) >= len(pool):
+            candidates.append(
+                {
+                    "path": m["path"],
+                    "name": m["name"],
+                    "ecosystem": ecosystem,
+                    "internal_dep_count": len(covered),
+                    "member_count": len(pool),
+                }
+            )
+    candidates.sort(key=lambda c: (-c["internal_dep_count"], c["path"]))
+    return candidates
 
 
 def _is_monorepo(manifests: list[dict]) -> bool:
@@ -641,37 +1454,135 @@ def _path_is_ancestor(a: str, b: str) -> bool:
     return b.startswith(a + "/")
 
 
-def scan(root: Path) -> dict:
-    """Run a full manifest scan rooted at `root`."""
-    paths = find_manifests(root)
+def _folders(manifests: list[dict]) -> list[dict]:
+    """Each folder holding a manifest ("." for the root), with the names its manifests give."""
+    by_folder: dict[str, set[str]] = {}
+    for m in manifests:
+        folder = m["path"].rsplit("/", 1)[0] if "/" in m["path"] else "."
+        names = by_folder.setdefault(folder, set())
+        if isinstance(m["name"], str) and m["name"]:
+            names.add(m["name"])
+    return [{"path": folder, "names": sorted(names)} for folder, names in sorted(by_folder.items())]
+
+
+def _folder_of(rel: str) -> str:
+    """The folder of a path relative to the scan root, "." for the root itself."""
+    return rel.rsplit("/", 1)[0] if "/" in rel else "."
+
+
+def _candidates(manifests: list[dict], services: list[str], sources: list[str],
+                members: set[str] | None) -> list[dict]:
+    """The candidate boundaries: see "candidates" in the module docstring.
+    `members` is the workspace paths --workspaces-file gives, else None."""
+    folders: dict[str, dict] = {}
+
+    def folder_entry(path: str) -> dict:
+        return folders.setdefault(
+            _folder_of(path), {"names": set(), "manifests": [], "services": []}
+        )
+
+    for m in manifests:
+        entry = folder_entry(m["path"])
+        entry["manifests"].append(m["path"])
+        if isinstance(m["name"], str) and m["name"]:
+            entry["names"].add(m["name"])
+    for path in services:
+        folder_entry(path)["services"].append(path)
+    own = dict.fromkeys(folders, 0)
+    for path in sources:
+        folder = _folder_of(path)
+        while folder not in folders and folder != ".":
+            folder = _folder_of(folder)
+        if folder in folders:
+            own[folder] += 1
+    out: list[dict] = []
+    for folder in sorted(folders):
+        below = any(
+            other != folder and (folder == "." or other.startswith(folder + "/"))
+            for other in folders
+        )
+        if own[folder] == 0 and below:
+            continue
+        entry = folders[folder]
+        out.append({
+            "path": folder,
+            "names": sorted(entry["names"]),
+            "manifests": entry["manifests"],
+            "services": entry["services"],
+            "own_source_files": own[folder],
+            "workspace_member": None if members is None else folder in members,
+        })
+    return out
+
+
+def scan(root: Path, include_dev: bool = False, workspaces: set[str] | None = None) -> dict:
+    """Run a full manifest scan rooted at `root`.
+
+    With `include_dev`, each manifest's development dependencies join its
+    `deps[]` tagged `scope: dev`, and `total_unique_dev` is added.
+    `workspaces` is the workspace paths that set each candidate's
+    `workspace_member` (None leaves it null). Raises ImportError when
+    skf-detect-language.py does not sit beside this script.
+    """
+    sources: list[str] = []
+    paths, services = _walk(root, sources)
     manifests: list[dict] = []
     warnings: list[str] = []
+    # the folder of a Cargo workspace root: the version its members inherit
+    workspace_versions: dict[str, str] = {}
+    inheriting: list[dict] = []
 
     for path in paths:
-        deps, warns = _parse_manifest(path)
+        record, warns = _describe_manifest(path, include_dev)
         rel = path.relative_to(root).as_posix()
         for w in warns:
             warnings.append(f"{rel}: {w}")
-        manifests.append(
-            {
-                "path": rel,
-                "ecosystem": MANIFEST_ECOSYSTEMS[path.name],
-                "deps": deps,
-            }
-        )
+        manifest = {
+            "path": rel,
+            "ecosystem": MANIFEST_ECOSYSTEMS[path.name],
+            "name": record["name"],
+            "private": record["private"],
+            "version": record["version"],
+            "version_dynamic": record["version_how"] == "dynamic",
+            "description": record["description"],
+            "deps": record["deps"],
+        }
+        manifests.append(manifest)
+        if record["workspace_version"]:
+            workspace_versions[_folder_of(rel)] = record["workspace_version"]
+        if record["version_how"] == "workspace":
+            inheriting.append(manifest)
+    for manifest in inheriting:
+        folder = _folder_of(manifest["path"])
+        while True:
+            if folder in workspace_versions:
+                manifest["version"] = workspace_versions[folder]
+                break
+            if folder == ".":
+                break
+            folder = _folder_of(folder)
+    _add_internal_deps(manifests)
 
     unique_names: set[str] = set()
+    dev_names: set[str] = set()
     for m in manifests:
         for dep in m["deps"]:
             name = dep.get("name")
             if isinstance(name, str) and name and name != "<unparsable>":
-                unique_names.add(name)
+                (dev_names if dep.get("scope") == "dev" else unique_names).add(name)
 
     result: dict = {
         "manifests": manifests,
         "total_unique": len(unique_names),
-        "monorepo": _is_monorepo(manifests),
     }
+    if include_dev:
+        result["total_unique_dev"] = len(dev_names - unique_names)
+    result["monorepo"] = _is_monorepo(manifests)
+    result["folders"] = _folders(manifests)
+    result["services"] = [path.relative_to(root).as_posix() for path in services]
+    result["candidates"] = _candidates(manifests, result["services"], sources, workspaces)
+    result["umbrella_candidates"] = _umbrella_candidates(manifests)
+    result["searched_filenames"] = list(MANIFEST_ECOSYSTEMS)
     if warnings:
         result["warnings"] = warnings
     return result
@@ -693,11 +1604,30 @@ def _cmd_scan(args: argparse.Namespace) -> int:
             file=sys.stderr,
         )
         return 1
+    workspaces = None
+    if args.workspaces_file is not None:
+        try:
+            data = json.loads(Path(args.workspaces_file).read_text(encoding="utf-8-sig"))
+        except (OSError, UnicodeDecodeError, ValueError) as exc:
+            print(
+                f"error: cannot read --workspaces-file {args.workspaces_file}: {exc}",
+                file=sys.stderr,
+            )
+            return 1
+        listed = data.get("workspaces") if isinstance(data, dict) else None
+        workspaces = {
+            w["path"].strip("/").removeprefix("./")
+            for w in (listed if isinstance(listed, list) else [])
+            if isinstance(w, dict) and isinstance(w.get("path"), str)
+        }
     try:
-        result = scan(root)
+        result = scan(root, include_dev=args.include_dev, workspaces=workspaces)
     except OSError as exc:
         print(f"error: filesystem error during scan: {exc}", file=sys.stderr)
         return 1
+    except ImportError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
     json.dump(result, sys.stdout, indent=2)
     sys.stdout.write("\n")
     return 0
@@ -723,6 +1653,17 @@ def _build_parser() -> argparse.ArgumentParser:
         default="auto",
         help="ecosystem filter (currently only 'auto' is accepted)",
     )
+    p_scan.add_argument(
+        "--include-dev",
+        action="store_true",
+        help="also list development dependencies, each tagged scope: dev",
+    )
+    p_scan.add_argument(
+        "--workspaces-file",
+        metavar="PATH",
+        help="the JSON skf-detect-workspaces.py printed for the same root: sets each "
+             "candidate's workspace_member",
+    )
     p_scan.set_defaults(func=_cmd_scan)
 
     return parser