@passioncode-ai/fabric-agent-adapter 0.80.2 → 0.82.0

package/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/scripts/fabric_source_check.py

@@ -44,6 +44,12 @@ shorter than three characters ignored, matched without regard to case. A list in
 is refused: it would travel with the source. The summary says whether Git chose the files or every file was read.
 Under a locale whose file-system encoding is not UTF-8 the check restarts itself in Python's UTF-8 mode, so a name,
 an argument and a file are read in one encoding. Standard library only.
+
+Under a source manifest (DEC-0149, FAA-261): when ROOT holds `fabric.source.json`, the manifest decides what is source
+and the contract's tree check reads it (fabric_source_tree.py, held to the contract's vectors): blocking findings exit
+1, warnings alone (`email`, `link-outside`, `no-license`, `unused-exclude`, `excluded-by-fold`) exit 0, a source file
+not read exits 2; each finding names its path, line and column, never the text found. `.fabric-source-ignore` and
+`--exclude` are not read there.
 """
 # Example URLs write their scheme in angle brackets: read `<https>` as `https` (FAA-212).
 
@@ -65,6 +71,28 @@ from urllib.parse import unquote
 from typing import Dict, Iterable, List, NoReturn, Optional, Set, Tuple
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+
+def _refuse_sibling(step: str, missing: BaseException) -> NoReturn:
+    """A sibling module that did not load: name it and exit 2, never a traceback and never exit 1 (a finding)."""
+    # the exception's own text is never printed: an OSError's carries a path, which may be under the owner's home
+    why = "a module beside this file refused to load"
+    try:
+        here, sibling = Path(__file__).resolve().parent, step + ".py"
+        file = getattr(missing, "filename", None) if isinstance(missing, OSError) else None
+        why = ("%s must sit beside this file" % sibling if not (here / sibling).is_file()
+               else "%s beside this file cannot be read" % sibling
+               if isinstance(file, (str, bytes)) and Path(os.fsdecode(file)).resolve() == here / sibling
+               else "%s beside this file is from another version: copy it with this one" % sibling
+               if isinstance(missing, (ImportError, SyntaxError))
+               else "%s beside this file refused to load (%s): copy the scripts folder whole, from one version"
+               % (sibling, type(missing).__name__))
+    except Exception:  # naming the cause must never be what fails
+        pass
+    sys.stderr.write("fabric_source_check.py: %s\n" % why)
+    sys.exit(2)
+
+
 _WROTE_BYTECODE, sys.dont_write_bytecode = sys.dont_write_bytecode, True  # a copy inside the checked tree adds no .pyc
 _STEP = "fabric_secrets"  # the sibling being loaded, so a refusal names it whatever the exception says
 try:
@@ -72,26 +100,13 @@ try:
     _STEP = "fabric_source_ignore"
     from fabric_source_ignore import (  # noqa: E402,F401 - how the ignore lines are read, beside this file
         IGNORE_FILE, CheckError, _as_bytes, _glob, _ignored, _line, _readable_glob, ignore_globs)
+    _STEP = "fabric_source_restart"
+    from fabric_source_restart import _interpreter, _interpreter_flags, _keep_console, _utf8_restart  # noqa: E402,F401
     _STEP = "fabric_source_homes"
     from fabric_source_homes import (  # noqa: E402,F401 - how a declared home is read, beside this file
         HOME_ENTRY, _NAME_JOIN, _NAME_START, _SEP, _dashed_home, _name_parts, _owner_homes, _spellings)
 except Exception as _missing:  # a copy without its siblings, or mixed from two versions: a refusal, never exit 1
-    # the exception's own text is never printed: an OSError's carries a path, which may be under the owner's home
-    _why = "a module beside this file refused to load"
-    try:
-        _HERE, _sibling = Path(__file__).resolve().parent, _STEP + ".py"
-        _file = getattr(_missing, "filename", None) if isinstance(_missing, OSError) else None
-        _why = ("%s must sit beside this file" % _sibling if not (_HERE / _sibling).is_file()
-                else "%s beside this file cannot be read" % _sibling
-                if isinstance(_file, (str, bytes)) and Path(os.fsdecode(_file)).resolve() == _HERE / _sibling
-                else "%s beside this file is from another version: copy it with this one" % _sibling
-                if isinstance(_missing, (ImportError, SyntaxError))
-                else "%s beside this file refused to load (%s): copy the scripts folder whole, from one version"
-                % (_sibling, type(_missing).__name__))
-    except Exception:  # naming the cause must never be what fails
-        pass
-    sys.stderr.write("fabric_source_check.py: %s\n" % _why)
-    sys.exit(2)
+    _refuse_sibling(_STEP, _missing)
 finally:
     sys.dont_write_bytecode = _WROTE_BYTECODE
 
@@ -127,13 +142,16 @@ PRIVATE_KEY = re.compile(r"-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----
 JWK_PRIVATE = re.compile(r"\\?[\"']d\\?[\"'][ \t]*:[ \t]*\\?[\"']([A-Za-z0-9_-]{43,})")  # JSON, escaped, Python
 #: Words a documented or synthetic credential carries (AWS's `...EXAMPLE`, `sk-test-not-a-key`); a random key does not.
 SYNTHETIC = ("example", "not-a-key", "notakey", "fake", "dummy", "placeholder", "redacted")
-# The detector's provider-key shapes without its first alternative, `^bearer\s.+`, which reads a whole header value: in
-# a file it would take prose that starts with the word and hide a key later on the same line. A bearer token is looked
-# for as `_BEARER_INSIDE` does: the word, then a run of 12 or more with a digit.
-_BEARER_FIRST = "(^bearer["
-if not _secrets.SECRET_VALUE.pattern.startswith(_BEARER_FIRST):  # not an assert: `python -O` would drop it
-    raise ImportError("fabric_secrets.SECRET_VALUE changed its first branch; update fabric_source_check.py with it")
-PROVIDER_KEY = re.compile("(" + _secrets.SECRET_VALUE.pattern.split("|", 1)[1], _secrets.SECRET_VALUE.flags)
+# This check's own key shapes, the detector's before DEC-0151 (unbounded, ASCII case-insensitive), apart from the shared
+# SECRET_VALUE for parity with the contract's tree check (KEY_SHAPE_SOURCES, `src/source-rules.ts` at the pin; CT-207,
+# FAA-258). Not the detector's `^bearer\s.+`, which reads a whole header value and would take prose that starts with the
+# word: a bearer token is looked for as `_BEARER_INSIDE` does, the word, then a run of 12 or more with a digit.
+KEY_SHAPE_SOURCES = ("sk-[A-Za-z0-9_-]{16,}", "gh[pousr]_[A-Za-z0-9]{20,}", "xox[abposr]-[A-Za-z0-9-]{10,}", "AKIA[0-9A-Z]{16}", "glpat-[A-Za-z0-9_-]{20,}")
+PROVIDER_KEY = re.compile("(" + "|".join(KEY_SHAPE_SOURCES) + ")", re.IGNORECASE | re.ASCII)
+#: A URL parameter's value read as the contract's tree check reads it (`URL_SECRET_VALUE` in `src/source-rules.ts`): the
+#: detector's whole-value `^bearer` branch and this check's own unbounded shapes, never the shared DEC-0151 detector.
+URL_SECRET_VALUE = re.compile("(^bearer[" + _secrets.JS_SPACE + "]" + _secrets._NOT_LINE_END + "+|" + "|".join(KEY_SHAPE_SOURCES) + ")",
+                              re.IGNORECASE | re.ASCII)
 #: Files that are credentials by their name: pack's own rule (fabric_package.SECRET_FILE and SECRET_FILE_OK, held
 #: equal by test_source_check), and the kit's token files (`service.token`, fabric_auth; `board.credential`, fabric_hub).
 _JS_DOT = "[^\n\r\u2028\u2029]"
@@ -216,7 +234,7 @@ def _url_secret(url: str, depth: int = 0) -> bool:
                 if not _real(value):
                     continue
                 if (_secrets.credential_name(name) and _secrets._long_secret(value)) or \
-                        _secrets.JWT_SHAPE.search(value) or _secrets.SECRET_VALUE.search(value):
+                        _secrets.has_jwt(value) or URL_SECRET_VALUE.search(value):
                     return True
     return False
 
@@ -269,7 +287,7 @@ def _secret_in(line: str) -> Optional[str]:
         return "a private key block"
     if any(_real(m.group(1)) for m in JWK_PRIVATE.finditer(line)):
         return "a private key (a JWK's `d`)"
-    if any(_real(m.group(0)) for m in _secrets.JWT_SHAPE.finditer(line)):
+    if any(_real(line[a:b]) for a, b in _secrets.jwt_spans(line)):  # one pass (FAA-313)
         return "a token shaped like a JWT"
     # The detector reads one configuration value; across a line, `sk-` needs a word start of its own, or every name
     # with a segment ending in `sk` (`task-management-service`) would read as a key.
@@ -399,13 +417,13 @@ def _instance_finding(text: str, entries: List[str]) -> Optional[str]:
 
 
 def _alternatives(pattern: "re.Pattern[str]") -> List[Tuple["re.Pattern[str]", ...]]:
-    """Each provider-key alternative of the contract's pattern as (its starts, its prefix, a run of its class, least
+    """Each provider-key alternative of this check's pattern as (its starts, its prefix, a run of its class, least
     length, open-ended, the whole alternative), so a key is read where its own class ends, not where another's does."""
     found = []
     for alt in pattern.pattern[1:-1].split("|"):
         part = re.fullmatch(r"(.+?)(\[[^\]]+\])\{(\d+)(,?)\}", alt)
         if part is None:  # not an assert: `python -O` would drop it
-            raise ImportError("fabric_secrets.SECRET_VALUE has a key shape this check cannot read: %s" % alt[:12])
+            raise ImportError("KEY_SHAPE_SOURCES has a key shape this check cannot read: %s" % alt[:12])
         prefix, cls, least, open_ended = part.groups()
         found.append((re.compile("(?=%s)" % prefix, pattern.flags), re.compile(prefix, pattern.flags),
                       re.compile(cls + "+", pattern.flags), int(least), bool(open_ended), re.compile(alt, pattern.flags)))
@@ -468,7 +486,7 @@ def _secret_spans(text: str) -> List[Tuple[int, int]]:
 
 def _raw_secret_spans(text: str) -> List[Tuple[int, int]]:
     spans = [m.span() for m in PRIVATE_KEY.finditer(text)]
-    spans += [m.span() for m in _secrets.JWT_SHAPE.finditer(text) if _real(m.group(0))]
+    spans += [(a, b) for a, b in _secrets.jwt_spans(text) if _real(text[a:b])]  # one pass (FAA-313)
     spans += [m.span(1) for m in JWK_PRIVATE.finditer(text) if _real(m.group(1))]
     spans += list(_provider_keys(text))
     spans += [m.span() for m in _overlapping(_secrets._BEARER_INSIDE, text) if _real(m.group(0))]
@@ -714,6 +732,15 @@ def check(root: Path, entries: List[str], globs: List[str], every_file: bool = F
         files, mode = _walk_files(root, unreadable, PACK_REFUSES), "walk"
     elif not files:  # not a checkout, Git refused it, or a folder Git ignores or holds nothing of: read what is there
         files, mode = _walk_files(root, unreadable, WALK_SKIP, dependencies), "walk"
+    if os.path.lexists(str(root / "fabric.source.json")):  # DEC-0149: the manifest decides (fabric_source_tree.py)
+        wrote, sys.dont_write_bytecode = sys.dont_write_bytecode, True  # a copy inside the tree adds no .pyc
+        try:
+            import fabric_source_tree  # noqa: E402 - beside this file; imported here, since it imports this one
+        except Exception as missing:  # the same named-sibling refusal as the imports at the top (exit 2)
+            _refuse_sibling("fabric_source_tree", missing)
+        finally:
+            sys.dont_write_bytecode = wrote
+        return fabric_source_tree.check_folder(root, files, mode, entries, unreadable, dependencies)
     findings: List[Dict[str, object]] = []
     read = 0
     skipped = {"binary": 0, "unreadable": len(unreadable), "excluded": 0, "nested": 0, "dependencies": len(dependencies)}
@@ -892,6 +919,8 @@ def run(argv: Optional[List[str]] = None) -> int:
     except CheckError as exc:
         print("fabric_source_check.py: %s" % exc, file=sys.stderr)
         return 2
+    if result.get("manifest"):  # under fabric.source.json: its own report (fabric_source_tree.print_run)
+        return sys.modules["fabric_source_tree"].print_run(result, entries, list_path is not None, args.json, _print)
     findings = result["findings"]
     skipped = result["skipped"]
     listed = sum(1 for e in entries if e and not e.startswith(HOME_ENTRY))
@@ -926,55 +955,6 @@ def run(argv: Optional[List[str]] = None) -> int:
     return 0 if complete else 2
 
 
-def _utf8_restart(encoding: str, utf8_mode: int, executable: str, argv: List[str]) -> Optional[List[bytes]]:
-    """The command that restarts the check in Python's UTF-8 mode, or None when it needs none. Under a locale whose
-    file-system encoding is not UTF-8 (ISO-8859-1, KOI8-R, a C locale with UTF-8 mode off), a name, an argument and a
-    file's text would each be read in its own encoding; in UTF-8 mode all three are, and a byte that is not UTF-8 stays
-    that byte. Windows names are UTF-16 and need none. The arguments go back as the bytes they came in. An empty list:
-    a restart is needed and no interpreter is known (the caller exits 2 rather than read in two encodings)."""
-    if os.name == "nt" or utf8_mode or encoding.lower().replace("_", "-") in ("utf-8", "utf8"):
-        return None
-    if not executable:
-        return []
-    return [os.fsencode(executable), b"-X", b"utf8"] + [os.fsencode(a) for a in argv]
-
-
-def _interpreter() -> str:
-    """The running interpreter: Linux's `/proc/self/exe`, which no PATH or working folder can stand in for (Python 3.9
-    may take a `./python3` of the working folder as sys.executable), else sys.executable, which may be empty."""
-    try:
-        return os.readlink("/proc/self/exe")
-    except (OSError, AttributeError):
-        return sys.executable or ""
-
-
-def _interpreter_flags(flags: object, xoptions: Dict[str, object]) -> List[str]:
-    """The interpreter's own options, rebuilt where sys.orig_argv is missing (Python 3.9), so a restart keeps `-I`."""
-    out: List[str] = []
-    if getattr(flags, "isolated", 0):
-        out.append("-I")
-    else:
-        out += ["-E"] * bool(getattr(flags, "ignore_environment", 0)) + ["-s"] * bool(getattr(flags, "no_user_site", 0))
-    out += ["-S"] * bool(getattr(flags, "no_site", 0)) + ["-B"] * bool(getattr(flags, "dont_write_bytecode", 0))
-    if getattr(flags, "optimize", 0):
-        out.append("-" + "O" * int(getattr(flags, "optimize")))
-    for key, value in xoptions.items():
-        if key not in ("utf8", "fabric_console"):
-            out += ["-X", key if value is True else "%s=%s" % (key, value)]
-    return out
-
-
-def _keep_console(xoptions: Dict[str, object]) -> None:
-    """After a restart, write the console's own encoding again (the parent named it in `-X fabric_console=`)."""
-    console = xoptions.get("fabric_console")
-    if isinstance(console, str):
-        for stream in (sys.stdout, sys.stderr):
-            try:
-                stream.reconfigure(encoding=console, errors="backslashreplace")  # type: ignore[union-attr]
-            except (AttributeError, LookupError, ValueError):
-                pass
-
-
 if __name__ == "__main__":
     _keep_console(getattr(sys, "_xoptions", {}))
     _argv = (list(sys.orig_argv[1:]) if hasattr(sys, "orig_argv")

package/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/scripts/fabric_source_restart.py

@@ -0,0 +1,56 @@
+"""How fabric_source_check.py restarts itself in Python's UTF-8 mode under a locale whose file-system encoding is not
+UTF-8, and keeps the console's encoding and the interpreter's own options across the restart. Standard library only."""
+from __future__ import annotations
+
+import os
+import sys
+from typing import Dict, List, Optional
+
+
+def _utf8_restart(encoding: str, utf8_mode: int, executable: str, argv: List[str]) -> Optional[List[bytes]]:
+    """The command that restarts the check in Python's UTF-8 mode, or None when it needs none. Under a locale whose
+    file-system encoding is not UTF-8 (ISO-8859-1, KOI8-R, a C locale with UTF-8 mode off), a name, an argument and a
+    file's text would each be read in its own encoding; in UTF-8 mode all three are, and a byte that is not UTF-8 stays
+    that byte. Windows names are UTF-16 and need none. The arguments go back as the bytes they came in. An empty list:
+    a restart is needed and no interpreter is known (the caller exits 2 rather than read in two encodings)."""
+    if os.name == "nt" or utf8_mode or encoding.lower().replace("_", "-") in ("utf-8", "utf8"):
+        return None
+    if not executable:
+        return []
+    return [os.fsencode(executable), b"-X", b"utf8"] + [os.fsencode(a) for a in argv]
+
+
+def _interpreter() -> str:
+    """The running interpreter: Linux's `/proc/self/exe`, which no PATH or working folder can stand in for (Python 3.9
+    may take a `./python3` of the working folder as sys.executable), else sys.executable, which may be empty."""
+    try:
+        return os.readlink("/proc/self/exe")
+    except (OSError, AttributeError):
+        return sys.executable or ""
+
+
+def _interpreter_flags(flags: object, xoptions: Dict[str, object]) -> List[str]:
+    """The interpreter's own options, rebuilt where sys.orig_argv is missing (Python 3.9), so a restart keeps `-I`."""
+    out: List[str] = []
+    if getattr(flags, "isolated", 0):
+        out.append("-I")
+    else:
+        out += ["-E"] * bool(getattr(flags, "ignore_environment", 0)) + ["-s"] * bool(getattr(flags, "no_user_site", 0))
+    out += ["-S"] * bool(getattr(flags, "no_site", 0)) + ["-B"] * bool(getattr(flags, "dont_write_bytecode", 0))
+    if getattr(flags, "optimize", 0):
+        out.append("-" + "O" * int(getattr(flags, "optimize")))
+    for key, value in xoptions.items():
+        if key not in ("utf8", "fabric_console"):
+            out += ["-X", key if value is True else "%s=%s" % (key, value)]
+    return out
+
+
+def _keep_console(xoptions: Dict[str, object]) -> None:
+    """After a restart, write the console's own encoding again (the parent named it in `-X fabric_console=`)."""
+    console = xoptions.get("fabric_console")
+    if isinstance(console, str):
+        for stream in (sys.stdout, sys.stderr):
+            try:
+                stream.reconfigure(encoding=console, errors="backslashreplace")  # type: ignore[union-attr]
+            except (AttributeError, LookupError, ValueError):
+                pass