ecoportal-api-graphql 1.3.12 → 1.3.13

data/.ai-assistance/skills/project-self-docs/scripts/self_docs_scan.py

@@ -0,0 +1,378 @@
+#!/usr/bin/env python3
+"""
+self_docs_scan.py -- Project self-docs OBSERVE stage (deterministic, zero-LLM core).
+
+Skill (semantic refresh + Act): skills-library/project-self-docs/SKILL.md.
+
+Runs FROM A TARGET REPO (the repo documenting itself), not from ep-ai-standards. It does the
+DETERMINISTIC half of self-documentation so no interactive Claude tokens are spent on facts the
+filesystem already knows:
+  - detect the repo type using the same signals as skills-library/discovery-manifest.yaml
+  - collect a structural fingerprint (top-level dirs, key config files, git head/branch/remote,
+    commit-activity clusters, existing .ai-assistance/code specs, CI files, doc files)
+  - compute a stable content hash per self-doc so refreshes are idempotent and the changelog
+    only records REAL changes
+  - emit a structured JSON "observation" the skill consumes to fill the self-doc set
+
+It does NOT write project prose, summarise purpose, or judge architecture -- that semantic work is
+the LLM skill's job (batched to Gemini per the token-frugality rule), gated by human review before
+any egress. This scanner is the thermometer; the skill is the (human-gated) actuator.
+
+HARD CONSTRAINTS:
+  - READ-ONLY by default. It never writes into the target repo except the self-doc index/changelog
+    under --write, and even then only to the self-docs directory (default docs/self-docs/).
+  - NO absolute paths are emitted into any committed artefact: paths are repo-relative.
+  - NO Confluence / network egress. Publish target is UNDECIDED (see SKILL.md); this script only
+    ever touches the local filesystem.
+
+Usage:
+  python self_docs_scan.py                       # human summary of the observation (target = cwd)
+  python self_docs_scan.py --json                # structured observation to stdout
+  python self_docs_scan.py --target <path>       # observe a different LOCAL repo root (convenience
+                                                 # only -- NEVER to generate another repo's self-docs
+                                                 # from here; run the skill in that repo's own session)
+  python self_docs_scan.py --out docs/self-docs  # self-docs dir (default docs/self-docs)
+  python self_docs_scan.py --write               # (re)write the machine index + changelog only
+"""
+import argparse
+import hashlib
+import json
+import os
+import re
+import subprocess
+import sys
+from datetime import datetime, timezone
+
+SCHEMA_VERSION = "1.1"
+
+# Self-doc set: stable filenames the skill fills in. Keeping this list here (not only in the
+# template) lets the deterministic scanner report which docs already exist and which are missing,
+# and lets the index/changelog be generated without the LLM.
+SELF_DOC_SET = [
+    ("OVERVIEW.md", "What the project is, who it serves, current priority, discovery & access"),
+    ("ARCHITECTURE.md", "Top-level structure, key components, how they fit"),
+    ("CONVENTIONS.md", "Coding style, branch naming, commit style, test conventions"),
+    ("INTEGRATIONS.md", "External services + data + permissions, upstream/downstream repos"),
+    ("STATUS.md", "Active work, migration state, known risks, open questions"),
+    ("COMPLIANCE.md", "ISO-27001/audit, data classes + PII, vendors, AI content, leak controls"),
+    ("OPERATIONS.md", "Infrastructure, public exposure + Cloudflare, DevOps audit, usage KPIs"),
+]
+
+# Detection signals mirror skills-library/discovery-manifest.yaml. Order matters: first match wins.
+# (file, ...) => all must exist; file_any => any of the globs; not_file => must be absent.
+REPO_TYPE_RULES = [
+    ("aws-cdk", {"file": ["cdk.json"]}),
+    ("browser-extension", {"file": ["manifest.json"], "file_any": ["package.json", "Gruntfile.js", "webpack.config.js"]}),
+    ("ruby-gem", {"file_any": ["*.gemspec"]}),
+    ("rails-app", {"file": ["config/application.rb"]}),
+    ("ruby-scripts", {"file": ["Gemfile"], "not_file": ["*.gemspec", "config/application.rb"]}),
+    ("typescript-lib", {"file": ["package.json"], "not_file": ["cdk.json", "manifest.json"]}),
+]
+
+KEY_CONFIG_FILES = [
+    "package.json", "Gemfile", "cdk.json", "manifest.json", "go.mod", "Cargo.toml",
+    "pyproject.toml", "requirements.txt", "tsconfig.json", ".eslintrc", ".eslintrc.json",
+    ".eslintrc.js", ".prettierrc", ".rubocop.yml", ".editorconfig",
+]
+CI_FILES = [".gitlab-ci.yml", "Jenkinsfile", "Makefile"]
+
+
+def _iso_now():
+    return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def _exists(root, rel):
+    return os.path.exists(os.path.join(root, rel))
+
+
+def _glob_any(root, patterns):
+    import fnmatch
+    try:
+        top = os.listdir(root)
+    except OSError:
+        return False
+    for pat in patterns:
+        if any(fnmatch.fnmatch(name, pat) for name in top):
+            return True
+    return False
+
+
+def detect_repo_type(root):
+    for type_name, rule in REPO_TYPE_RULES:
+        ok = True
+        for f in rule.get("file", []):
+            if not _exists(root, f):
+                ok = False
+                break
+        if ok and rule.get("file_any"):
+            ok = _glob_any(root, rule["file_any"])
+        if ok and rule.get("not_file"):
+            for f in rule["not_file"]:
+                if "*" in f:
+                    if _glob_any(root, [f]):
+                        ok = False
+                        break
+                elif _exists(root, f):
+                    ok = False
+                    break
+        if ok:
+            return type_name
+    return "unknown"
+
+
+def _git(root, args):
+    try:
+        out = subprocess.run(
+            ["git", "-C", root] + args,
+            capture_output=True, text=True, timeout=15,
+        )
+        return out.stdout.strip() if out.returncode == 0 else ""
+    except (OSError, subprocess.SubprocessError):
+        return ""
+
+
+def git_facts(root):
+    branch = _git(root, ["rev-parse", "--abbrev-ref", "HEAD"])
+    head = _git(root, ["rev-parse", "--short", "HEAD"])
+    remote = _git(root, ["config", "--get", "remote.origin.url"])
+    # Real repo name from the remote, never a local folder name that may be renamed.
+    repo_name = ""
+    if remote and "/" in remote:
+        repo_name = re.sub(r"\.git$", "", remote.rstrip("/").split("/")[-1])
+    # Activity clusters: which top-level dirs churn most in the last 50 commits.
+    log = _git(root, ["log", "-50", "--name-only", "--pretty=format:"])
+    clusters = {}
+    for line in log.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        top = line.split("/")[0] if "/" in line else line
+        clusters[top] = clusters.get(top, 0) + 1
+    top_clusters = sorted(clusters.items(), key=lambda kv: kv[1], reverse=True)[:6]
+    return {
+        "branch": branch,
+        "head": head,
+        "repo_name": repo_name,
+        "has_remote": bool(remote),
+        "activity_clusters": [{"path": k, "touches": v} for k, v in top_clusters],
+    }
+
+
+def structure(root):
+    dirs, files = [], []
+    try:
+        for name in sorted(os.listdir(root)):
+            if name.startswith(".git"):
+                continue
+            full = os.path.join(root, name)
+            if os.path.isdir(full):
+                dirs.append(name)
+            else:
+                files.append(name)
+    except OSError:
+        pass
+    return {
+        "top_level_dirs": dirs,
+        "key_config_files": [f for f in KEY_CONFIG_FILES if _exists(root, f)],
+        "ci_files": [f for f in CI_FILES if _exists(root, f)],
+        "has_readme": _exists(root, "README.md") or _exists(root, "README"),
+        "has_changelog": _exists(root, "CHANGELOG.md"),
+    }
+
+
+def existing_context(root):
+    """Reusable inputs that already exist -- the skill composes from these, not from scratch."""
+    ctx = {"code_specs": [], "repo_context": None, "conventions": None}
+    code_dir = os.path.join(root, ".ai-assistance", "code")
+    if os.path.isdir(code_dir):
+        for dirpath, _, filenames in os.walk(code_dir):
+            for fn in filenames:
+                if fn.endswith(".md"):
+                    rel = os.path.relpath(os.path.join(dirpath, fn), root).replace("\\", "/")
+                    ctx["code_specs"].append(rel)
+    ctx["code_specs"].sort()
+    local = os.path.join(root, ".ai-assistance", "local")
+    if os.path.isfile(os.path.join(local, "repo-context.md")):
+        ctx["repo_context"] = ".ai-assistance/local/repo-context.md"
+    if os.path.isfile(os.path.join(local, "conventions.md")):
+        ctx["conventions"] = ".ai-assistance/local/conventions.md"
+    return ctx
+
+
+def _content_hash(path):
+    """Hash the self-doc BODY (below the frontmatter) so index/changelog record real changes,
+    not just a re-run timestamp. Missing file -> None."""
+    if not os.path.isfile(path):
+        return None
+    with open(path, encoding="utf-8", errors="replace") as fh:
+        text = fh.read()
+    body = text
+    if text.startswith("---"):
+        end = text.find("\n---", 3)
+        if end != -1:
+            body = text[end + 4:]
+    # Normalise line endings + trailing whitespace so cross-platform re-runs are stable.
+    norm = "\n".join(line.rstrip() for line in body.replace("\r\n", "\n").split("\n")).strip()
+    return hashlib.sha256(norm.encode("utf-8")).hexdigest()[:16]
+
+
+def self_doc_status(root, out_rel):
+    out_dir = os.path.join(root, out_rel)
+    docs = []
+    for fname, purpose in SELF_DOC_SET:
+        path = os.path.join(out_dir, fname)
+        docs.append({
+            "file": f"{out_rel}/{fname}".replace("\\", "/"),
+            "purpose": purpose,
+            "exists": os.path.isfile(path),
+            "content_hash": _content_hash(path),
+        })
+    return docs
+
+
+def observe(root, out_rel):
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "observed_at": _iso_now(),
+        "repo_type": detect_repo_type(root),
+        "git": git_facts(root),
+        "structure": structure(root),
+        "existing_context": existing_context(root),
+        "self_doc_set": self_doc_status(root, out_rel),
+        "out_dir": out_rel.replace("\\", "/"),
+    }
+
+
+# == index + changelog (deterministic writers) ======================================
+
+def _read_json(path, default):
+    if not os.path.isfile(path):
+        return default
+    try:
+        with open(path, encoding="utf-8") as fh:
+            return json.load(fh)
+    except (OSError, ValueError):
+        return default
+
+
+def write_index_and_changelog(root, out_rel, obs):
+    """Rewrite the machine-readable index and append changelog deltas -- deterministically.
+
+    The index (self-docs-index.json) is the machine surface a central hub consumes: schema
+    version, repo identity, and a hash per doc. The changelog (CHANGES.jsonl) appends one line
+    per doc whose content hash CHANGED since the previous index -- so the hub sees real deltas,
+    and a no-op re-run appends nothing (idempotent)."""
+    out_dir = os.path.join(root, out_rel)
+    os.makedirs(out_dir, exist_ok=True)
+    index_path = os.path.join(out_dir, "self-docs-index.json")
+    changes_path = os.path.join(out_dir, "CHANGES.jsonl")
+
+    prev = _read_json(index_path, {})
+    prev_hashes = {d["file"]: d.get("content_hash") for d in prev.get("docs", [])}
+
+    docs = obs["self_doc_set"]
+    now = obs["observed_at"]
+    deltas = []
+    for d in docs:
+        new_h = d["content_hash"]
+        old_h = prev_hashes.get(d["file"])
+        if new_h != old_h:
+            deltas.append({
+                "ts": now,
+                "file": d["file"],
+                "change": ("created" if old_h is None and new_h is not None
+                           else "removed" if new_h is None
+                           else "updated"),
+                "from_hash": old_h,
+                "to_hash": new_h,
+            })
+
+    index = {
+        "schema_version": SCHEMA_VERSION,
+        "generated_at": now,
+        "repo_name": obs["git"].get("repo_name") or "",
+        "repo_type": obs["repo_type"],
+        "head": obs["git"].get("head", ""),
+        "docs": [
+            {"file": d["file"], "purpose": d["purpose"],
+             "exists": d["exists"], "content_hash": d["content_hash"]}
+            for d in docs
+        ],
+    }
+    with open(index_path, "w", encoding="utf-8") as fh:
+        json.dump(index, fh, indent=2)
+        fh.write("\n")
+
+    if deltas:
+        with open(changes_path, "a", encoding="utf-8") as fh:
+            for delta in deltas:
+                fh.write(json.dumps(delta) + "\n")
+
+    return {"index": f"{out_rel}/self-docs-index.json".replace("\\", "/"),
+            "changelog": f"{out_rel}/CHANGES.jsonl".replace("\\", "/"),
+            "deltas_appended": len(deltas)}
+
+
+def _human(obs):
+    g = obs["git"]
+    print(f"self-docs observation  (schema {obs['schema_version']})")
+    print(f"  repo_type : {obs['repo_type']}")
+    print(f"  repo_name : {g.get('repo_name') or '(no remote)'}  @ {g.get('head') or '?'} "
+          f"on {g.get('branch') or '?'}")
+    print(f"  out_dir   : {obs['out_dir']}")
+    print("  top-level dirs:", ", ".join(obs["structure"]["top_level_dirs"]) or "(none)")
+    if obs["structure"]["key_config_files"]:
+        print("  config    :", ", ".join(obs["structure"]["key_config_files"]))
+    if g.get("activity_clusters"):
+        hot = ", ".join(f"{c['path']}({c['touches']})" for c in g["activity_clusters"])
+        print("  hot paths :", hot)
+    ec = obs["existing_context"]
+    reuse = []
+    if ec["repo_context"]:
+        reuse.append("ai-discovery repo-context.md")
+    if ec["conventions"]:
+        reuse.append("ai-discovery conventions.md")
+    if ec["code_specs"]:
+        reuse.append(f"{len(ec['code_specs'])} code-specs")
+    print("  reuse from:", ", ".join(reuse) or "(none -- run ai-discovery/code-specs first)")
+    print("  self-doc set:")
+    for d in obs["self_doc_set"]:
+        state = "present" if d["exists"] else "MISSING"
+        print(f"    [{state:7}] {d['file']}  ({d['content_hash'] or '-'})")
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Deterministic project self-docs observation")
+    ap.add_argument("--target", default=".",
+                    help="LOCAL repo root to observe (default: cwd). Convenience only -- do NOT use to "
+                         "generate ANOTHER repo's self-docs from here; run the skill in that repo's own "
+                         "session (self-docs are owned + generated by the repo that documents itself).")
+    ap.add_argument("--out", default="docs/self-docs", help="self-docs dir (default: docs/self-docs)")
+    ap.add_argument("--json", action="store_true", help="emit the structured observation")
+    ap.add_argument("--write", action="store_true",
+                    help="(re)write the machine index + changelog deltas only")
+    args = ap.parse_args()
+
+    root = os.path.abspath(args.target)
+    if not os.path.isdir(root):
+        print(f"[self-docs] target not a directory: {root}", file=sys.stderr)
+        return 2
+
+    obs = observe(root, args.out)
+
+    if args.write:
+        result = write_index_and_changelog(root, args.out, obs)
+        obs["_write_result"] = result
+
+    if args.json:
+        print(json.dumps(obs, indent=2))
+    else:
+        _human(obs)
+        if args.write:
+            r = obs["_write_result"]
+            print(f"  wrote index -> {r['index']}; changelog deltas appended: {r['deltas_appended']}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

data/.ai-assistance/standards-version.json

@@ -6,17 +6,17 @@
   ],
   "deferred": [],
   "installed-components": {
+    "skills/ai-instructions": "0.0.0",
     "skills/code-specs": "0.0.0",
-    "skills/ep-ai-manager": "2.1.0",
-    "skills/dep-graph": "0.0.0",
     "skills/corporate-policies": "1.0.0",
-    "skills/ai-instructions": "0.0.0",
-    "skills/spec-generation": "0.0.0",
+    "skills/dep-graph": "0.0.0",
+    "skills/ep-ai-manager": "2.2.0",
     "skills/gemini-assist": "0.0.0",
-    "skills/ruby-scripting": "0.1.0",
-    "skills/refactor": "0.0.0",
-    "skills/project-cycle": "0.0.0",
     "skills/graphql-schema-analysis": "0.0.0",
-    "skills/procedural-memory": "0.2.0"
+    "skills/project-cycle": "0.0.0",
+    "skills/project-self-docs": "0.2.0",
+    "skills/refactor": "0.0.0",
+    "skills/ruby-scripting": "0.1.0",
+    "skills/spec-generation": "0.0.0"
   }
-}
+}