@passioncode-ai/passioncode 0.2.57 → 0.2.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +2 -2
- package/bin/passioncode.js +4 -3
- package/family.json +1 -1
- package/package.json +1 -1
- package/payload/.claude-plugin/marketplace.json +2 -2
- package/payload/manifest.json +8 -8
- package/payload/plugins/fabric-agent-adapter/.claude-plugin/plugin.json +1 -1
- package/payload/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/SKILL.md +16 -2
- package/payload/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/references/source-and-instance.md +154 -0
- package/payload/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/scripts/fabric_source_check.py +995 -0
- package/payload/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/scripts/fabric_source_homes.py +72 -0
- package/payload/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/scripts/fabric_source_ignore.py +185 -0
- package/payload/plugins/fabric-agent-adapter/skills/building-fabric-services/SKILL.md +3 -2
- package/payload/plugins/fabric-agent-adapter/skills/building-fabric-services/references/package.md +22 -0
- package/payload/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/kit.json +91 -1
- package/payload/plugins/fabric-agent-adapter/skills/creating-fabric-agents/SKILL.md +14 -5
- package/payload/plugins/fabric-agent-adapter/skills/creating-fabric-agents/references/source-and-instance.md +154 -0
- package/payload/plugins/fabric-agent-adapter/skills/working-with-fabric/SKILL.md +1 -1
- package/payload/plugins/fabric-agent-adapter/skills/working-with-fabric/scripts/fabric_session_env.py +1 -1
- package/payload/plugins/passioncode/.claude-plugin/plugin.json +1 -1
|
@@ -0,0 +1,995 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Find instance data in an agent's source tree, without repeating it (FAA-188 part 1).
|
|
3
|
+
|
|
4
|
+
An agent's source is what can be handed over (code, skills, prompts, schemas, examples); its instance is what belongs
|
|
5
|
+
to one installation (records, logs, keys, the owner's names and paths, its configuration). This check reads the files
|
|
6
|
+
the source would carry and reports, by path, line and kind, each place where the instance shows through:
|
|
7
|
+
|
|
8
|
+
- `home-path`: an absolute path inside a person's home folder whose name is ASCII: `/Users/<name>/`, `/home/<name>/`,
|
|
9
|
+
`/var/home/<name>/`, `C:\\Users\\<name>\\` (any case, either separator, a JSON-escaped backslash), the dashed form a
|
|
10
|
+
tool makes of such a path (`-Users-<name>-`, `C--Users-<name>-`, DEC-0105, its name up to the next dash), Git Bash's
|
|
11
|
+
`/c/Users/<name>`, WSL's `/mnt/c/Users/<name>` and the URL-encoded ones (`%2FUsers%2F<name>`, `%5CUsers%5C<name>`); a neutral name (`example`, `user`,
|
|
12
|
+
`runner`, a container's or a cloud machine's user such as `node` or `ubuntu`), a one-letter name or a placeholder
|
|
13
|
+
(`<name>`, `$USER`) is not a finding;
|
|
14
|
+
- `secret`: a value shaped like a credential by the contract's detector (fabric_secrets.py, DEC-0044), read line by
|
|
15
|
+
line: a provider key starting a word, a JWT, a bearer token, or a long run mixing upper case, lower case and digits
|
|
16
|
+
assigned to a name that can only carry a credential (a bare long run, such as an integrity hash, is not; a whole
|
|
17
|
+
Ed25519 signature under a JSON `"sig"`, `"signature"` or `"signatures"` member, as `sign` writes it, is public); or a private key: a PEM, OpenSSH or PGP
|
|
18
|
+
block, or a JWK's `d`; a URL that carries a password or a credential in its query; a file that is a credential by
|
|
19
|
+
its name (pack's rule: `.env`, `*.pem`, `*.key`, `.npmrc` ...) or a kit credential file (`*.token`, `board.credential`). A value that says it is synthetic (`EXAMPLE`, `not-a-key`, `fake`, `dummy`) is
|
|
20
|
+
not a finding;
|
|
21
|
+
- `instance-string`: a string from the owner's list of instance strings (names, accounts, hosts), kept outside the tree.
|
|
22
|
+
|
|
23
|
+
A file's path is read as a line is (line 0). It never prints a value it found, a line that holds one, an entry of the
|
|
24
|
+
owner's list or the folder it checks: a finding names its path (an entry of the list, a home folder's name or a
|
|
25
|
+
credential-shaped value in the path printed as `<line N>`, `<home>` or `<secret>`), line, kind and rule, and an
|
|
26
|
+
instance string by its line in the owner's list. It changes nothing. Exit 0: every file was read and none holds a
|
|
27
|
+
finding; 1: at least one finding; 2: the check could not run, or a file was not read: an unreadable file, a nested
|
|
28
|
+
repository, or a binary one the ignore file does not name as source. A file not read is listed by its path.
|
|
29
|
+
|
|
30
|
+
fabric_source_check.py check [ROOT] [--instance-strings FILE] [--exclude GLOB ...] [--every-file] [--json]
|
|
31
|
+
|
|
32
|
+
The files read are those Git would commit (`git ls-files --cached --others --exclude-standard`), as they are in the
|
|
33
|
+
work tree (a staged copy that differs is not read). With `--every-file`, every file but `.git`, `__pycache__`,
|
|
34
|
+
`.DS_Store` and `*.pyc`, which `pack` refuses as litter (what `pack` reads). When ROOT is not in a Git checkout or Git lists nothing in it, every file outside `.git`,
|
|
35
|
+
`__pycache__`, `node_modules`, virtual environments and caches; the dependency folders it leaves are named (so too
|
|
36
|
+
when no `git` is found in PATH's absolute folders, or with PATH unset in `/bin` and `/usr/bin`; on Windows with PATH
|
|
37
|
+
unset the walk is used). A committed symbolic link is checked by the path it names, never followed. A binary file (a NUL byte in
|
|
38
|
+
its first 8 KiB) is not read; it is listed. Globs in ROOT's `.fabric-source-ignore` (one per line, `#`
|
|
39
|
+
comments) and `--exclude` leave matching paths out. The owner's list is `--instance-strings`, else the file
|
|
40
|
+
`FABRIC_INSTANCE_STRINGS` names, else `passioncode-fabric/instance-strings.txt` in the user's configuration folder
|
|
41
|
+
(`$XDG_CONFIG_HOME` when it is absolute, else `~/.config`; `%APPDATA%` on Windows) when it exists: one string per
|
|
42
|
+
line, a line starting with `#` a comment, `home: <name>` declaring the owner's home name (never neutral), strings
|
|
43
|
+
shorter than three characters ignored, matched without regard to case. A list inside ROOT
|
|
44
|
+
is refused: it would travel with the source. The summary says whether Git chose the files or every file was read.
|
|
45
|
+
Under a locale whose file-system encoding is not UTF-8 the check restarts itself in Python's UTF-8 mode, so a name,
|
|
46
|
+
an argument and a file are read in one encoding. Standard library only.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
# #region source-check — docs: plugins/fabric-agent-adapter/skills/creating-fabric-agents/references/source-and-instance.md#the-check
|
|
50
|
+
|
|
51
|
+
from __future__ import annotations
|
|
52
|
+
|
|
53
|
+
import argparse
|
|
54
|
+
import bisect
|
|
55
|
+
import json
|
|
56
|
+
import os
|
|
57
|
+
from pathlib import Path
|
|
58
|
+
import re
|
|
59
|
+
import stat
|
|
60
|
+
import subprocess
|
|
61
|
+
import sys
|
|
62
|
+
import unicodedata
|
|
63
|
+
from urllib.parse import unquote
|
|
64
|
+
from typing import Dict, Iterable, List, NoReturn, Optional, Set, Tuple
|
|
65
|
+
|
|
66
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
67
|
+
_WROTE_BYTECODE, sys.dont_write_bytecode = sys.dont_write_bytecode, True # a copy inside the checked tree adds no .pyc
|
|
68
|
+
_STEP = "fabric_secrets" # the sibling being loaded, so a refusal names it whatever the exception says
|
|
69
|
+
try:
|
|
70
|
+
import fabric_secrets as _secrets # noqa: E402 - the contract's credential detector (DEC-0044), beside this file
|
|
71
|
+
_STEP = "fabric_source_ignore"
|
|
72
|
+
from fabric_source_ignore import ( # noqa: E402,F401 - how the ignore lines are read, beside this file
|
|
73
|
+
IGNORE_FILE, CheckError, _as_bytes, _glob, _ignored, _line, _readable_glob, ignore_globs)
|
|
74
|
+
_STEP = "fabric_source_homes"
|
|
75
|
+
from fabric_source_homes import ( # noqa: E402,F401 - how a declared home is read, beside this file
|
|
76
|
+
HOME_ENTRY, _NAME_JOIN, _NAME_START, _SEP, _dashed_home, _name_parts, _owner_homes, _spellings)
|
|
77
|
+
except Exception as _missing: # a copy without its siblings, or mixed from two versions: a refusal, never exit 1
|
|
78
|
+
# the exception's own text is never printed: an OSError's carries a path, which may be under the owner's home
|
|
79
|
+
_why = "a module beside this file refused to load"
|
|
80
|
+
try:
|
|
81
|
+
_HERE, _sibling = Path(__file__).resolve().parent, _STEP + ".py"
|
|
82
|
+
_file = getattr(_missing, "filename", None) if isinstance(_missing, OSError) else None
|
|
83
|
+
_why = ("%s must sit beside this file" % _sibling if not (_HERE / _sibling).is_file()
|
|
84
|
+
else "%s beside this file cannot be read" % _sibling
|
|
85
|
+
if isinstance(_file, (str, bytes)) and Path(os.fsdecode(_file)).resolve() == _HERE / _sibling
|
|
86
|
+
else "%s beside this file is from another version: copy it with this one" % _sibling
|
|
87
|
+
if isinstance(_missing, (ImportError, SyntaxError))
|
|
88
|
+
else "%s beside this file refused to load (%s): copy the scripts folder whole, from one version"
|
|
89
|
+
% (_sibling, type(_missing).__name__))
|
|
90
|
+
except Exception: # naming the cause must never be what fails
|
|
91
|
+
pass
|
|
92
|
+
sys.stderr.write("fabric_source_check.py: %s\n" % _why)
|
|
93
|
+
sys.exit(2)
|
|
94
|
+
finally:
|
|
95
|
+
sys.dont_write_bytecode = _WROTE_BYTECODE
|
|
96
|
+
|
|
97
|
+
LIST_NAME = "instance-strings.txt"
|
|
98
|
+
#: The folders `pack` refuses to carry (fabric_package.JUNK); --every-file leaves them and the files below unread.
|
|
99
|
+
PACK_REFUSES = {".git", "__pycache__"}
|
|
100
|
+
WALK_SKIP = {".git", "node_modules", "__pycache__", ".venv", "venv", ".tox", ".mypy_cache", ".pytest_cache"}
|
|
101
|
+
#: Names that stand for nobody in a home path: placeholders, CI runners and the system's shared folders.
|
|
102
|
+
NEUTRAL_HOMES = {"example", "examples", "user", "username", "you", "me", "name", "someone", "runner", "shared", "public",
|
|
103
|
+
"default", "guest", "linuxbrew", "alice", "bob", "operator", "owner", "person", "your-name", "yourname",
|
|
104
|
+
# the users of container images, cloud machines and hosted CI
|
|
105
|
+
"node", "ubuntu", "ec2-user", "runneradmin", "admin", "vagrant", "debian", "centos", "app", "builder",
|
|
106
|
+
# dev containers and hosted workspaces, and the reference's own fixture id
|
|
107
|
+
"vscode", "codespace", "circleci", "gitpod", "jovyan", "coder", "example-agent"}
|
|
108
|
+
HOME_PATHS = (
|
|
109
|
+
re.compile(r"(?<![A-Za-z0-9_.-])(?:/var)?/home/([A-Za-z0-9][A-Za-z0-9._-]*)", re.ASCII),
|
|
110
|
+
# macOS writes `/Users` capitalised; `/users/...` in code is a web route
|
|
111
|
+
re.compile(r"(?<![A-Za-z0-9_.-])/Users/([A-Za-z0-9][A-Za-z0-9._-]*)", re.ASCII),
|
|
112
|
+
# Git Bash's `/c/Users/<name>` and WSL's `/mnt/c/Users/<name>`
|
|
113
|
+
re.compile(r"(?<![A-Za-z0-9_.-])(?:/mnt)?/[A-Za-z]/Users/([A-Za-z0-9][A-Za-z0-9._-]*)", re.ASCII),
|
|
114
|
+
re.compile(r"(?<![A-Za-z0-9_.-])[A-Za-z]:%sUsers%s([A-Za-z0-9][A-Za-z0-9._-]*)" % (_SEP, _SEP), re.ASCII | re.IGNORECASE),
|
|
115
|
+
# DEC-0105's dashed form starts a name; after a dash it is a flag or a CSS variable (`--home-dir`)
|
|
116
|
+
# (on Windows a drive comes first: `C--Users-<name>-proj`; Silverblue's, WSL's and Git Bash's homes dash to
|
|
117
|
+
# `-var-home-<name>`, `-mnt-c-Users-<name>` and `-c-Users-<name>`)
|
|
118
|
+
re.compile(r"(?:^|[^A-Za-z0-9-])(?:-Users|-home|-var-home|(?:-mnt)?-[A-Za-z]-Users)-([A-Za-z0-9][A-Za-z0-9._]*)",
|
|
119
|
+
re.ASCII),
|
|
120
|
+
re.compile(r"(?<![A-Za-z0-9])[A-Za-z]--Users-([A-Za-z0-9][A-Za-z0-9._]*)", re.ASCII | re.IGNORECASE),
|
|
121
|
+
re.compile(r"%2[Ff](?:Users|home)%2[Ff]([A-Za-z0-9][A-Za-z0-9._-]*)", re.ASCII), # `%2Fusers%2F` is a route
|
|
122
|
+
re.compile(r"%5CUsers%5C([A-Za-z0-9][A-Za-z0-9._-]*)", re.ASCII | re.IGNORECASE),
|
|
123
|
+
)
|
|
124
|
+
PRIVATE_KEY = re.compile(r"-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----") # PEM, OpenSSH, PGP
|
|
125
|
+
#: A private JWK's `d` (`sign --key` takes an Ed25519 one): 32 bytes or more in base64url.
|
|
126
|
+
JWK_PRIVATE = re.compile(r"\\?[\"']d\\?[\"'][ \t]*:[ \t]*\\?[\"']([A-Za-z0-9_-]{43,})") # JSON, escaped, Python
|
|
127
|
+
#: Words a documented or synthetic credential carries (AWS's `...EXAMPLE`, `sk-test-not-a-key`); a random key does not.
|
|
128
|
+
SYNTHETIC = ("example", "not-a-key", "notakey", "fake", "dummy", "placeholder", "redacted")
|
|
129
|
+
# The detector's provider-key shapes without its first alternative, `^bearer\s.+`, which reads a whole header value: in
|
|
130
|
+
# a file it would take prose that starts with the word and hide a key later on the same line. A bearer token is looked
|
|
131
|
+
# for as `_BEARER_INSIDE` does: the word, then a run of 12 or more with a digit.
|
|
132
|
+
_BEARER_FIRST = "(^bearer["
|
|
133
|
+
if not _secrets.SECRET_VALUE.pattern.startswith(_BEARER_FIRST): # not an assert: `python -O` would drop it
|
|
134
|
+
raise ImportError("fabric_secrets.SECRET_VALUE changed its first branch; update fabric_source_check.py with it")
|
|
135
|
+
PROVIDER_KEY = re.compile("(" + _secrets.SECRET_VALUE.pattern.split("|", 1)[1], _secrets.SECRET_VALUE.flags)
|
|
136
|
+
#: Files that are credentials by their name: pack's own rule (fabric_package.SECRET_FILE and SECRET_FILE_OK, held
|
|
137
|
+
#: equal by test_source_check), and the kit's token files (`service.token`, fabric_auth; `board.credential`, fabric_hub).
|
|
138
|
+
_JS_DOT = "[^\n\r\u2028\u2029]"
|
|
139
|
+
SECRET_FILE = re.compile(r"\.env(\.%s+)?|%s*\.(pem|key|p12|pfx|jks|keystore)|id_(rsa|dsa|ecdsa|ed25519)|\.(npmrc|netrc|pypirc)"
|
|
140
|
+
% (_JS_DOT, _JS_DOT))
|
|
141
|
+
SECRET_FILE_OK = re.compile(r"\.env\.(example|sample|template)")
|
|
142
|
+
_PACK_JUNK_FILE = re.compile(r"\.DS_Store|%s*\.pyc" % _JS_DOT) # held to fabric_package.JUNK by test_source_check
|
|
143
|
+
TOKEN_FILE = re.compile(r".+\.(token|credential)") # the kit's service.token and board.credential
|
|
144
|
+
#: An absolute URL in a line, read whole by `_url_secret`: a password in it, a credential in its query.
|
|
145
|
+
URL = re.compile(r"[A-Za-z][A-Za-z0-9+.-]{0,63}://[^\s\"'<>`]+") # a bounded scheme: linear, and `${X:-pg://..}`
|
|
146
|
+
#: `NAME = value`, `NAME: value` or `"name": "value"`: the long-run rule applies only where the name carries a credential.
|
|
147
|
+
#: The name is bounded and starts a word, so a long line is read in linear time.
|
|
148
|
+
ASSIGNED = re.compile(r"(?<![A-Za-z0-9_.-])([A-Za-z_][A-Za-z0-9_.-]{0,63})[\"']?[ \t]{0,8}[:=][ \t]{0,8}[\"']?"
|
|
149
|
+
r"((?:[A-Za-z0-9+/_-]|%[0-9A-Fa-f]{2}){40,}={0,2})") # `=` only as padding; `%2B` read decoded
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _home_names(text: str, owner: Iterable[str] = ()) -> List[Tuple[int, int]]:
|
|
153
|
+
"""The spans of the names of real people's home folders in a text (neutral and one-letter names left out, unless
|
|
154
|
+
the owner's list declares the name as the owner's home)."""
|
|
155
|
+
owner = set(owner)
|
|
156
|
+
spans = []
|
|
157
|
+
for pattern in HOME_PATHS:
|
|
158
|
+
for match in pattern.finditer(text):
|
|
159
|
+
name = match.group(1).lower()
|
|
160
|
+
if name in owner or (len(name) > 1 and name not in NEUTRAL_HOMES): # one letter (`/Users/a`): a placeholder
|
|
161
|
+
spans.append(match.span(1))
|
|
162
|
+
for pattern in (p for name in sorted(owner) for p in _dashed_home(name)): # declared: found whatever it reads as
|
|
163
|
+
spans += [match.span(1) for match in pattern.finditer(text)]
|
|
164
|
+
return spans
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _real(token: str) -> bool:
|
|
168
|
+
lowered = token.lower()
|
|
169
|
+
return not any(word in lowered for word in SYNTHETIC)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
#: An Ed25519 signature is public: `sign` writes one into fabric.package.json (`"sig"`), and every signed record carries
|
|
173
|
+
#: one. Only a whole one (64 bytes, 86 base64url characters) is: a storage URL's `sig=` (an HMAC) grants access.
|
|
174
|
+
PUBLIC_NAMES = {"sig", "signature", "signatures"}
|
|
175
|
+
_ED25519_SIGNATURE = re.compile(r"[A-Za-z0-9_-]{86}")
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _credential(name: str, value: str, quoted: bool) -> bool:
|
|
179
|
+
"""A credential's name, but a whole Ed25519 signature as a JSON member (`"sig": "..."`, as `sign` writes it)."""
|
|
180
|
+
if quoted and name.lower() in PUBLIC_NAMES and _ED25519_SIGNATURE.fullmatch(value):
|
|
181
|
+
return False
|
|
182
|
+
return _secrets.credential_name(name)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
_COLON_AFTER = re.compile(r"[ \t]*:")
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _quoted(text: str, match: "re.Match[str]") -> bool:
|
|
189
|
+
"""`"name": value`: a JSON member, the name quoted on both sides and `:` after it (never `"sig=...`)."""
|
|
190
|
+
start, end = match.start(1), match.end(1)
|
|
191
|
+
# matched in place: a copy of the rest of the line for each member made a minified JSON line quadratic
|
|
192
|
+
return start > 0 and text[start - 1] == '"' and text[end:end + 1] == '"' and _COLON_AFTER.match(text, end + 1) is not None
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _url_secret(url: str, depth: int = 0) -> bool:
|
|
196
|
+
"""A URL that carries a credential, as a source tree holds one. The contract's URL rule (LC-21) reads one value a
|
|
197
|
+
launch names, where any `token=` is one; across a tree it would read every documented `?token=abc` and `user:pw@`.
|
|
198
|
+
So: a password of 8 or more mixing letters and digits, or a query or fragment value that, percent-decoded, is shaped
|
|
199
|
+
like a credential (a provider key, a JWT, a long random run under a credential's name)."""
|
|
200
|
+
url = url.rstrip(".,;)]}")
|
|
201
|
+
if not _secrets._SCHEME.match(url):
|
|
202
|
+
return False
|
|
203
|
+
password, query, fragment = _secrets._url_parts(url)
|
|
204
|
+
# what says it is synthetic is the value, never the host or the path (`examples.blob.core.windows.net`)
|
|
205
|
+
if len(password) >= 8 and re.search(r"[0-9]", password) and re.search(r"[A-Za-z]", password) and _real(unquote(password)):
|
|
206
|
+
return True
|
|
207
|
+
for part in (query, fragment):
|
|
208
|
+
for name, raws in _secrets.raw_values(part).items():
|
|
209
|
+
for raw in raws:
|
|
210
|
+
value = unquote(raw)
|
|
211
|
+
if depth < 2 and "://" in value and _url_secret(value, depth + 1): # `?next=https%3A%2F%2F...sig%3D`
|
|
212
|
+
return True # read before the synthetic filter: an inner URL is judged by its own values
|
|
213
|
+
if any(True for _ in _provider_keys(value)): # each key by its own word, as on a line
|
|
214
|
+
return True
|
|
215
|
+
if not _real(value):
|
|
216
|
+
continue
|
|
217
|
+
if (_secrets.credential_name(name) and _secrets._long_secret(value)) or \
|
|
218
|
+
_secrets.JWT_SHAPE.search(value) or _secrets.SECRET_VALUE.search(value):
|
|
219
|
+
return True
|
|
220
|
+
return False
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
#: The JSON escapes tools write for `&`, `=` and `/` (Go's encoding/json writes `\u0026`, PHP's json_encode `\/`).
|
|
224
|
+
_JSON_ESCAPES = re.compile(r"\\(?:u0026|u003[dD]|/)")
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
_UNESCAPED = {"u0026": "&", "u003d": "=", "u003D": "=", "/": "/"}
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _json_unescaped(line: str) -> str:
|
|
231
|
+
return _JSON_ESCAPES.sub(lambda m: _UNESCAPED[m.group(0)[1:]], line)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _unescaped_with_origin(line: str) -> Tuple[str, List[Tuple[int, int]]]:
|
|
235
|
+
"""The line with JSON escapes undone, and for each of its characters the raw span it came from."""
|
|
236
|
+
out: List[str] = []
|
|
237
|
+
origin: List[Tuple[int, int]] = []
|
|
238
|
+
last = 0
|
|
239
|
+
for m in _JSON_ESCAPES.finditer(line):
|
|
240
|
+
out.extend(line[last:m.start()])
|
|
241
|
+
origin.extend((i, i + 1) for i in range(last, m.start()))
|
|
242
|
+
out.append(_UNESCAPED[m.group(0)[1:]])
|
|
243
|
+
origin.append(m.span())
|
|
244
|
+
last = m.end()
|
|
245
|
+
out.extend(line[last:])
|
|
246
|
+
origin.extend((i, i + 1) for i in range(last, len(line)))
|
|
247
|
+
return "".join(out), origin
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _secret_finding(line: str) -> Optional[str]:
|
|
251
|
+
"""`_secret_in`, on the line as written and, when it holds a JSON escape of `&`, `=` or `/`, as JSON reads it."""
|
|
252
|
+
found = _secret_in(line)
|
|
253
|
+
if found is None and "\\" in line:
|
|
254
|
+
unescaped = _json_unescaped(line)
|
|
255
|
+
if unescaped != line:
|
|
256
|
+
found = _secret_in(unescaped)
|
|
257
|
+
return found
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
#: A whole Ed25519 signature as `sign` writes it: public, so no rule reads it (not even a provider key's shape in it).
|
|
261
|
+
_PUBLIC_SIGNATURE = re.compile(r"\"(?:sig|signature|signatures)\"[ \t]*:[ \t]*\"[A-Za-z0-9_-]{86}\"", re.IGNORECASE)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _secret_in(line: str) -> Optional[str]:
|
|
265
|
+
if '"' in line:
|
|
266
|
+
line = _PUBLIC_SIGNATURE.sub('"sig": ""', line)
|
|
267
|
+
if PRIVATE_KEY.search(line):
|
|
268
|
+
return "a private key block"
|
|
269
|
+
if any(_real(m.group(1)) for m in JWK_PRIVATE.finditer(line)):
|
|
270
|
+
return "a private key (a JWK's `d`)"
|
|
271
|
+
if any(_real(m.group(0)) for m in _secrets.JWT_SHAPE.finditer(line)):
|
|
272
|
+
return "a token shaped like a JWT"
|
|
273
|
+
# The detector reads one configuration value; across a line, `sk-` needs a word start of its own, or every name
|
|
274
|
+
# with a segment ending in `sk` (`task-management-service`) would read as a key.
|
|
275
|
+
if any(True for _ in _provider_keys(line)):
|
|
276
|
+
return "a value shaped like a provider key"
|
|
277
|
+
if any(_real(m.group(0)) for m in _overlapping(_secrets._BEARER_INSIDE, line)):
|
|
278
|
+
return "a bearer token"
|
|
279
|
+
if "://" in line and any(_url_secret(m.group(0)) for m in URL.finditer(line)):
|
|
280
|
+
return "a URL that carries a credential"
|
|
281
|
+
for m in (_assigned(line) if ":" in line or "=" in line else ()):
|
|
282
|
+
name, value = m.group(1), m.group(2)
|
|
283
|
+
if _credential(name, value, _quoted(line, m)) and _secrets._long_secret(unquote(value)) and _real(unquote(value)):
|
|
284
|
+
return "a long random value assigned to a credential's name"
|
|
285
|
+
return None
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _assigned(line: str) -> Iterable["re.Match[str]"]:
|
|
289
|
+
"""ASSIGNED's matches, each search starting again inside the last value: `id=<run>/TOKEN=<value>` reads both."""
|
|
290
|
+
pos = 0
|
|
291
|
+
while True:
|
|
292
|
+
m = ASSIGNED.search(line, pos)
|
|
293
|
+
if m is None:
|
|
294
|
+
return
|
|
295
|
+
yield m
|
|
296
|
+
pos = m.start(2) if m.start(2) > pos else m.end()
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _fold_map(text: str) -> Tuple[str, List[int], List[int]]:
|
|
300
|
+
"""Text as an entry is compared, with where each of its characters came from: each letter with the marks after it
|
|
301
|
+
is casefolded, then composed (NFC), so `é` typed in the list finds a name a file system stored decomposed (macOS
|
|
302
|
+
names made through its file-system representation are) and `ß` finds `ss`. Folding changes lengths both ways, so a
|
|
303
|
+
match is mapped back to the original characters through the starts and ends, never by its offsets."""
|
|
304
|
+
out: List[str] = []
|
|
305
|
+
starts: List[int] = []
|
|
306
|
+
ends: List[int] = []
|
|
307
|
+
run = 0
|
|
308
|
+
last = text[:1] # the last character of the piece composed so far: what the next one may compose with
|
|
309
|
+
for k in range(1, len(text) + 1):
|
|
310
|
+
if k < len(text):
|
|
311
|
+
joined = unicodedata.normalize("NFC", last + text[k])
|
|
312
|
+
if unicodedata.combining(text[k]) or len(joined) < len(last) + 1: # a mark, or a jamo or vowel sign
|
|
313
|
+
last = joined[-1:]
|
|
314
|
+
continue
|
|
315
|
+
last = text[k]
|
|
316
|
+
piece = unicodedata.normalize("NFC", text[run:k].casefold())
|
|
317
|
+
out.append(piece)
|
|
318
|
+
starts += [run] * len(piece)
|
|
319
|
+
ends += [k] * len(piece)
|
|
320
|
+
run = k
|
|
321
|
+
return "".join(out), starts, ends
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _fold(text: str) -> str:
|
|
325
|
+
return _fold_map(text)[0]
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
#: `home: <name>`: one ASCII name (a letter or digit first; letters, digits, `.`, `_`, `-`, `'` and spaces inside, as
|
|
329
|
+
#: a Windows profile may hold), a horizontal space (a space, a tab, a Unicode space) around the colon; any other white
|
|
330
|
+
#: space there is refused. The name is checked as ASCII apart: IGNORECASE alone would
|
|
331
|
+
#: let `[A-Za-z]` take U+017F and U+212A.
|
|
332
|
+
_GAP = "[ \t\u00a0\u1680\u2000-\u200a\u202f\u205f\u3000]*" # a horizontal space only: never a control or a separator
|
|
333
|
+
_HOME_LINE = re.compile(r"home%s:%s([A-Za-z0-9](?:[A-Za-z0-9._' -]*[A-Za-z0-9._'-])?)" % (_GAP, _GAP), re.IGNORECASE)
|
|
334
|
+
_HOME_KEY = re.compile(r"home\s*:", re.IGNORECASE) # any white space, so a control or a separator is refused
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def read_list(path: Path) -> List[str]:
|
|
338
|
+
"""The owner's list, one item per line: an entry casefolded, a declared home (`home: <name>`) as HOME_ENTRY and the
|
|
339
|
+
name lowered, or '' for a comment, a blank or too short a line."""
|
|
340
|
+
entries = []
|
|
341
|
+
# at CRLF, CR or LF: a list saved with CR alone keeps its lines, and a FF or U+2028 starts none
|
|
342
|
+
for raw in re.split(r"\r\n|\r|\n", path.read_bytes().decode("utf-8-sig")):
|
|
343
|
+
# a byte-order mark or a zero-width character at a line's start (two lists joined) is no part of the line
|
|
344
|
+
if HOME_ENTRY in raw: # never read as another name: a NUL is no character an owner types
|
|
345
|
+
raise CheckError("line %d of the owner's list holds a NUL character: save it as plain text" % (len(entries) + 1))
|
|
346
|
+
text = raw.strip().lstrip("\ufeff\u200b\u2060").strip() # no meaning at a start
|
|
347
|
+
# a joiner (U+200C, U+200D) may belong to a string, never to `home:`: a home line is read or refused with every
|
|
348
|
+
# format character (Unicode Cf: a joiner, a direction mark, a soft hyphen) taken out, wherever it stands
|
|
349
|
+
bare = "".join(c for c in text if unicodedata.category(c) != "Cf").strip()
|
|
350
|
+
home = _HOME_LINE.fullmatch(bare)
|
|
351
|
+
# the name as written: a format character inside it changes the name, so that line is refused, not read as another
|
|
352
|
+
# (looked for after the key's colon, so the letters of `home` cannot stand in for a name such as `me`)
|
|
353
|
+
if home and home.group(1).isascii() and home.group(1) in text.split(":", 1)[1]:
|
|
354
|
+
entries.append(HOME_ENTRY + home.group(1).lower())
|
|
355
|
+
continue
|
|
356
|
+
if home and home.group(1).isascii(): # visible as a home name, but an invisible character sits inside it
|
|
357
|
+
raise CheckError("line %d of the owner's list names a home that holds an invisible format character (a soft "
|
|
358
|
+
"hyphen, a zero-width space or a direction mark, which an editor may not show): delete it "
|
|
359
|
+
"and write the name again" % (len(entries) + 1))
|
|
360
|
+
if _HOME_KEY.match(bare): # meant as a home and not one: never kept as a string that cannot match
|
|
361
|
+
raise CheckError("line %d of the owner's list starts with `home` and a colon but is not one ASCII home name "
|
|
362
|
+
"and nothing after it (a letter or digit first, then letters, digits, spaces, `.`, `_`, `-`, "
|
|
363
|
+
"`'`): a note goes on a line of its own, a path is written as its home name only, and a name "
|
|
364
|
+
"outside ASCII, or holding another character, is listed as a plain string (a typographic "
|
|
365
|
+
"apostrophe the folder's name does not hold is written as '; a folder whose name holds one is "
|
|
366
|
+
"listed as a plain string with it kept and declared too with `home:` written with ')"
|
|
367
|
+
% (len(entries) + 1))
|
|
368
|
+
# a comment or a blank is judged without its invisible characters too, as a home line is (one rule per line)
|
|
369
|
+
# a string keeps a joiner between its letters (an emoji needs one); one at either end belongs to no string
|
|
370
|
+
edge = [i for i, c in enumerate(text) if not (c.isspace() or unicodedata.category(c) == "Cf")]
|
|
371
|
+
text = text[edge[0]:edge[-1] + 1] if edge else "" # ends taken in one slice, spaces between marks too
|
|
372
|
+
entries.append("" if not bare or bare.startswith("#") or len(text) < 3 else _fold(text))
|
|
373
|
+
return entries
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _entry_spans(text: str, entries: List[str]) -> List[Tuple[int, int, int]]:
|
|
377
|
+
strings = [(n, e) for n, e in enumerate(entries, 1) if e and not e.startswith(HOME_ENTRY)]
|
|
378
|
+
if not strings:
|
|
379
|
+
return []
|
|
380
|
+
if text.isascii(): # casefolding ASCII is lowering it, one character for one: no map to build
|
|
381
|
+
folded, starts, ends = text.lower(), None, None
|
|
382
|
+
else:
|
|
383
|
+
folded, starts, ends = _fold_map(text)
|
|
384
|
+
spans = []
|
|
385
|
+
for number, entry in strings:
|
|
386
|
+
start = folded.find(entry)
|
|
387
|
+
while start != -1: # back to the original characters: every one a matched character came from
|
|
388
|
+
end = start + len(entry)
|
|
389
|
+
spans.append((start, end, number) if starts is None or ends is None
|
|
390
|
+
else (starts[start], ends[end - 1], number))
|
|
391
|
+
start = folded.find(entry, start + 1)
|
|
392
|
+
return spans
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _instance_finding(text: str, entries: List[str]) -> Optional[str]:
|
|
396
|
+
spans = _entry_spans(text, entries)
|
|
397
|
+
return "a string from the owner's list (line %d)" % spans[0][2] if spans else None
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _alternatives(pattern: "re.Pattern[str]") -> List[Tuple["re.Pattern[str]", ...]]:
|
|
401
|
+
"""Each provider-key alternative of the contract's pattern as (its starts, its prefix, a run of its class, least
|
|
402
|
+
length, open-ended, the whole alternative), so a key is read where its own class ends, not where another's does."""
|
|
403
|
+
found = []
|
|
404
|
+
for alt in pattern.pattern[1:-1].split("|"):
|
|
405
|
+
part = re.fullmatch(r"(.+?)(\[[^\]]+\])\{(\d+)(,?)\}", alt)
|
|
406
|
+
if part is None: # not an assert: `python -O` would drop it
|
|
407
|
+
raise ImportError("fabric_secrets.SECRET_VALUE has a key shape this check cannot read: %s" % alt[:12])
|
|
408
|
+
prefix, cls, least, open_ended = part.groups()
|
|
409
|
+
found.append((re.compile("(?=%s)" % prefix, pattern.flags), re.compile(prefix, pattern.flags),
|
|
410
|
+
re.compile(cls + "+", pattern.flags), int(least), bool(open_ended), re.compile(alt, pattern.flags)))
|
|
411
|
+
return found
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
_KEY_SHAPES = _alternatives(PROVIDER_KEY)
|
|
415
|
+
_ASCII_LOWER = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _provider_keys(text: str) -> Iterable[Tuple[int, int]]:
|
|
419
|
+
"""The spans of provider keys that start a word and do not say they are synthetic, each read where it starts, so a
|
|
420
|
+
key a dropped run swallowed is found too (`task-runner-sk-<key>`, `sk-<16>_ghp_<key>_fake`). Every start is read
|
|
421
|
+
once, by its own class's run end and the next synthetic word in the line: O(n log n) on any line."""
|
|
422
|
+
if not PROVIDER_KEY.search(text):
|
|
423
|
+
return
|
|
424
|
+
lowered = text.translate(_ASCII_LOWER) # a key's characters are ASCII, and this keeps every offset
|
|
425
|
+
synthetic = [(len(word), [m.start() for m in re.finditer("(?=%s)" % re.escape(word), lowered)]) for word in SYNTHETIC]
|
|
426
|
+
for starts, prefix, run, least, open_ended, whole in _KEY_SHAPES:
|
|
427
|
+
runs = None
|
|
428
|
+
for m in starts.finditer(text):
|
|
429
|
+
p = m.start()
|
|
430
|
+
if p and text[p - 1].isalnum():
|
|
431
|
+
continue
|
|
432
|
+
if open_ended:
|
|
433
|
+
if runs is None:
|
|
434
|
+
runs = [r.span() for r in run.finditer(text)]
|
|
435
|
+
run_starts = [a for a, _ in runs]
|
|
436
|
+
q = p + len(prefix.match(text, p).group(0))
|
|
437
|
+
i = bisect.bisect_right(run_starts, q) - 1
|
|
438
|
+
if i < 0 or runs[i][1] - q < least:
|
|
439
|
+
continue
|
|
440
|
+
end = runs[i][1]
|
|
441
|
+
else:
|
|
442
|
+
key = whole.match(text, p)
|
|
443
|
+
if key is None:
|
|
444
|
+
continue
|
|
445
|
+
end = key.end()
|
|
446
|
+
if not any(k < len(at) and at[k] + n <= end
|
|
447
|
+
for n, at in synthetic for k in (bisect.bisect_left(at, p),)):
|
|
448
|
+
yield p, end
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _overlapping(pattern: "re.Pattern[str]", text: str) -> Iterable["re.Match[str]"]:
|
|
452
|
+
"""Matches that may start inside the last one (a second `Bearer` inside the first token's run gets its own span)."""
|
|
453
|
+
pos = 0
|
|
454
|
+
while pos <= len(text):
|
|
455
|
+
m = pattern.search(text, pos)
|
|
456
|
+
if m is None:
|
|
457
|
+
return
|
|
458
|
+
yield m
|
|
459
|
+
pos = m.start() + 1
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _secret_spans(text: str) -> List[Tuple[int, int]]:
|
|
463
|
+
"""The spans of credential-shaped values in a text, as `_secret_in` reads them (a public signature is no span)."""
|
|
464
|
+
public = [m.span() for m in _PUBLIC_SIGNATURE.finditer(text)] if '"' in text else []
|
|
465
|
+
return [(a, b) for a, b in _raw_secret_spans(text) if not any(p <= a and b <= q for p, q in public)]
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _raw_secret_spans(text: str) -> List[Tuple[int, int]]:
|
|
469
|
+
spans = [m.span() for m in PRIVATE_KEY.finditer(text)]
|
|
470
|
+
spans += [m.span() for m in _secrets.JWT_SHAPE.finditer(text) if _real(m.group(0))]
|
|
471
|
+
spans += [m.span(1) for m in JWK_PRIVATE.finditer(text) if _real(m.group(1))]
|
|
472
|
+
spans += list(_provider_keys(text))
|
|
473
|
+
spans += [m.span() for m in _overlapping(_secrets._BEARER_INSIDE, text) if _real(m.group(0))]
|
|
474
|
+
spans += [m.span() for m in URL.finditer(text) if _url_secret(m.group(0))]
|
|
475
|
+
spans += [m.span(2) for m in _assigned(text)
|
|
476
|
+
if _credential(m.group(1), m.group(2), _quoted(text, m)) and _secrets._long_secret(unquote(m.group(2)))
|
|
477
|
+
and _real(unquote(m.group(2)))]
|
|
478
|
+
return spans
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def redact(path: str, entries: List[str], seen: Optional["_Seen"] = None) -> str:
|
|
482
|
+
"""A path as it may be printed: an owner's string as `<line N>` (its line in the owner's list), a real home
|
|
483
|
+
folder's name as `<home>`, and each segment in which `_secret_finding` finds a credential as `<secret>`, whole
|
|
484
|
+
(a span inside a segment can miss a second value the first one runs into). While the segments left still hold a
|
|
485
|
+
credential together (`https:\\/\\/app:<pw>@db/x`), the segments that finding's spans touch are masked, else the
|
|
486
|
+
last one left, so what is printed holds no finding and its first folders still name the place. With `seen`, what the
|
|
487
|
+
check found elsewhere in the tree is masked too: a home name at every occurrence, and a segment holding a run of 8
|
|
488
|
+
characters of a found value as `<secret>`."""
|
|
489
|
+
spans, offset = [], 0
|
|
490
|
+
for segment in path.split("/"):
|
|
491
|
+
spans.append((offset, offset + len(segment)))
|
|
492
|
+
offset += len(segment) + 1
|
|
493
|
+
masked = [bool(_secret_finding(path[a:b])) for a, b in spans]
|
|
494
|
+
# every segment a found value's span touches, on the path and with JSON escapes undone (mapped back), so a value
|
|
495
|
+
# that runs across a `/` prints no part of itself
|
|
496
|
+
touched = list(_secret_spans(path))
|
|
497
|
+
if "\\" in path:
|
|
498
|
+
unescaped, origin = _unescaped_with_origin(path)
|
|
499
|
+
if unescaped != path:
|
|
500
|
+
touched += [(origin[a][0], origin[b - 1][1]) for a, b in _secret_spans(unescaped) if b > a]
|
|
501
|
+
for a, b in touched:
|
|
502
|
+
for i, (s, e) in enumerate(spans):
|
|
503
|
+
if s < b and a < e:
|
|
504
|
+
masked[i] = True
|
|
505
|
+
|
|
506
|
+
def rest() -> str:
|
|
507
|
+
# a masked segment keeps its length, so the spans read in what is left are offsets in the path
|
|
508
|
+
return "/".join(" " * (b - a) if m else path[a:b] for (a, b), m in zip(spans, masked))
|
|
509
|
+
|
|
510
|
+
# the guard: while what is left still holds a credential, mask the segments its spans touch, else the last one left
|
|
511
|
+
for _ in range(len(spans)):
|
|
512
|
+
left = rest()
|
|
513
|
+
if not _secret_finding(left):
|
|
514
|
+
break
|
|
515
|
+
in_left = _secret_spans(left) # once a pass: read per segment, a long path cost the cube of its folders
|
|
516
|
+
if "\\" in left: # a finding only the unescaped reading sees has its spans there, mapped back
|
|
517
|
+
unescaped, origin = _unescaped_with_origin(left)
|
|
518
|
+
if unescaped != left:
|
|
519
|
+
in_left += [(origin[a][0], origin[b - 1][1]) for a, b in _secret_spans(unescaped) if b > a]
|
|
520
|
+
hit = [i for i, (s, e) in enumerate(spans) if not masked[i] and any(s < b and a < e for a, b in in_left)]
|
|
521
|
+
if not hit:
|
|
522
|
+
hit = [max(i for i, m in enumerate(masked) if not m)]
|
|
523
|
+
for i in hit:
|
|
524
|
+
masked[i] = True
|
|
525
|
+
if seen is not None and seen.grams: # a run of a value found elsewhere in the tree, across a `/` too
|
|
526
|
+
for k in range(len(path) - 7):
|
|
527
|
+
if path[k:k + 8] in seen.grams:
|
|
528
|
+
for i, (a, b) in enumerate(spans):
|
|
529
|
+
if a < k + 8 and k < b:
|
|
530
|
+
masked[i] = True
|
|
531
|
+
secret_segments = [span for span, m in zip(spans, masked) if m]
|
|
532
|
+
marks = [(s, e, "<line %d>" % n) for s, e, n in _entry_spans(path, entries)]
|
|
533
|
+
found = _home_names(path, _owner_homes(entries))
|
|
534
|
+
marks += [(s, e, "<home>") for s, e in found]
|
|
535
|
+
# every other occurrence of a found name, searched in the path itself: a lowered copy can change its length (U+0130),
|
|
536
|
+
# and Unicode case matching also takes the letters that fold to ASCII ones (U+212A to `k`, U+017F to `s`); the names
|
|
537
|
+
# found elsewhere in the tree are one compiled pattern, so the cost does not grow with names times paths
|
|
538
|
+
# a declared home of three or more letters and digits is the owner's own name: masked wherever a printed path holds
|
|
539
|
+
# it, its parts joined by none or any run of the separators a tool writes (a space, `.`, `_`, `-`, `+`, and their
|
|
540
|
+
# percent-encodings), found or not
|
|
541
|
+
known = {path[s:e] for s, e in found} | (seen.names if seen is not None else set())
|
|
542
|
+
for declared in _owner_homes(entries):
|
|
543
|
+
parts = _name_parts(declared)
|
|
544
|
+
if sum(len(part) for part in parts) < 3 and len(parts) > 1: # a short name found: each spelling with a separator
|
|
545
|
+
spaced = (_NAME_JOIN + "+").join(re.escape(part) for part in parts)
|
|
546
|
+
if any(re.fullmatch(spaced, name, re.IGNORECASE) for name in known):
|
|
547
|
+
marks += [(m.start(), m.end(), "<home>") for m in re.finditer(spaced, path, re.IGNORECASE)]
|
|
548
|
+
if sum(len(part) for part in parts) >= 3:
|
|
549
|
+
body = (_NAME_JOIN + "*").join(re.escape(part) for part in parts)
|
|
550
|
+
# found in the tree: masked wherever it stands; unfound: a whole word only, so `home: app` leaves `mapping`
|
|
551
|
+
# readable (a percent-encoded or escaped separator before it starts a word too)
|
|
552
|
+
if not any(re.fullmatch(body, name, re.IGNORECASE) for name in known):
|
|
553
|
+
body = _NAME_START + body + "(?![A-Za-z0-9])"
|
|
554
|
+
marks += [(m.start(), m.end(), "<home>") for m in re.finditer(body, path, re.IGNORECASE)]
|
|
555
|
+
for name in _spellings({path[s:e] for s, e in found}, _owner_homes(entries)):
|
|
556
|
+
marks += [(m.start(), m.end(), "<home>") for m in re.finditer(re.escape(name), path, re.IGNORECASE)]
|
|
557
|
+
if seen is not None and seen.names:
|
|
558
|
+
marks += [(m.start(1), m.end(1), "<home>") for m in seen.names_pattern().finditer(path)]
|
|
559
|
+
marks += [(s, e, "<secret>") for s, e in secret_segments]
|
|
560
|
+
out, last = [], 0
|
|
561
|
+
for start, end, label in sorted(marks):
|
|
562
|
+
if start < last: # overlapping marks: the first covers the union, so no part of either is printed
|
|
563
|
+
last = max(last, end)
|
|
564
|
+
continue
|
|
565
|
+
out += [path[last:start], label]
|
|
566
|
+
last = end
|
|
567
|
+
return "".join(out) + path[last:]
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def _git() -> Optional[str]:
|
|
571
|
+
"""`git` from PATH's absolute folders only. On Windows CreateProcess looks in the current folder first (and
|
|
572
|
+
shutil.which there puts the current folder first too), and on POSIX an empty or relative PATH element (a trailing
|
|
573
|
+
`:`) is the current folder: a tree checked from inside could run its own `git`."""
|
|
574
|
+
name = "git.exe" if os.name == "nt" else "git" # never a .cmd or .bat: cmd.exe would read the arguments
|
|
575
|
+
# with PATH unset, POSIX's default folders (`/bin`, `/usr/bin`); Windows' default (`.;C:\\bin`) is a folder any
|
|
576
|
+
# user may create, so there no PATH means no git and the walk
|
|
577
|
+
folders = os.get_exec_path() if os.name != "nt" else os.environ.get("PATH", "").split(os.pathsep)
|
|
578
|
+
for folder in folders:
|
|
579
|
+
if folder and os.path.isabs(folder):
|
|
580
|
+
candidate = os.path.join(folder, name)
|
|
581
|
+
if os.path.isfile(candidate) and (os.name == "nt" or os.access(candidate, os.X_OK)):
|
|
582
|
+
return candidate
|
|
583
|
+
return None
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
_GIT_LOCATION = frozenset(("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_COMMON_DIR",
|
|
587
|
+
"GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_NAMESPACE", "GIT_PREFIX"))
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
def _git_files(root: Path) -> Optional[List[str]]:
|
|
591
|
+
git = _git()
|
|
592
|
+
if git is None:
|
|
593
|
+
return None
|
|
594
|
+
try:
|
|
595
|
+
# `core.fsmonitor` off: `--others` would run a command the checked tree's own configuration names. Empty, not
|
|
596
|
+
# `false`: Git before 2.36 reads the value as a hook and would run a program named `false` from PATH
|
|
597
|
+
# the repository is found from ROOT: a Git location the caller exports (every hook in a linked worktree gets
|
|
598
|
+
# GIT_DIR and GIT_INDEX_FILE) would list another repository's files and leave ROOT's own unread
|
|
599
|
+
env = {k: v for k, v in os.environ.items() if k.upper() not in _GIT_LOCATION}
|
|
600
|
+
result = subprocess.run([git, "-c", "core.fsmonitor=", "-C", str(root), "ls-files", "-z", "--cached",
|
|
601
|
+
"--others", "--exclude-standard"], capture_output=True, timeout=60, env=env)
|
|
602
|
+
except (OSError, subprocess.SubprocessError):
|
|
603
|
+
return None
|
|
604
|
+
if result.returncode != 0:
|
|
605
|
+
return None
|
|
606
|
+
# Git for Windows writes a lone surrogate in a name as WTF-8: read back as that character, it is the name os.walk
|
|
607
|
+
# gives; elsewhere a name is bytes, decoded as os.walk decodes it. `_name_bytes` turns either back into Git's bytes
|
|
608
|
+
names = set()
|
|
609
|
+
for raw in result.stdout.split(b"\0"): # each name on its own: one name that is not UTF-8 changes no other
|
|
610
|
+
if not raw:
|
|
611
|
+
continue
|
|
612
|
+
try:
|
|
613
|
+
# POSIX: the file system's own encoding, so the name is the one os.walk and open() use in any locale
|
|
614
|
+
names.add(raw.decode("utf-8", "surrogatepass") if os.name == "nt" else os.fsdecode(raw))
|
|
615
|
+
except UnicodeDecodeError:
|
|
616
|
+
names.add(raw.decode("utf-8", "surrogateescape"))
|
|
617
|
+
return sorted(names)
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def _walk_files(root: Path, unreadable: List[str], skip: Iterable[str] = WALK_SKIP,
|
|
621
|
+
left: Optional[List[str]] = None) -> List[str]:
|
|
622
|
+
"""Every file under root but the folders in skip; each folder skipped is named in left."""
|
|
623
|
+
found = []
|
|
624
|
+
skip = set(skip)
|
|
625
|
+
|
|
626
|
+
def failed(error: OSError) -> None:
|
|
627
|
+
unreadable.append(Path(str(error.filename or root)).relative_to(root).as_posix())
|
|
628
|
+
|
|
629
|
+
for folder, dirs, files in os.walk(root, onerror=failed):
|
|
630
|
+
if left is not None:
|
|
631
|
+
left += [Path(folder, d).relative_to(root).as_posix() for d in dirs
|
|
632
|
+
if d in skip and d not in PACK_REFUSES] # `.git` and `__pycache__`: --every-file leaves them too
|
|
633
|
+
dirs[:] = [d for d in dirs if d not in skip]
|
|
634
|
+
# os.walk lists a link to a folder with the folders and does not follow it: check it as Git would, by its target
|
|
635
|
+
files = files + [d for d in dirs if os.path.islink(os.path.join(folder, d))]
|
|
636
|
+
for name in files:
|
|
637
|
+
if name in skip and name == ".git": # a worktree's or a submodule's `.git` file: never carried, never source
|
|
638
|
+
continue
|
|
639
|
+
if skip == PACK_REFUSES and _PACK_JUNK_FILE.fullmatch(name): # --every-file: litter pack refuses, never carried
|
|
640
|
+
continue
|
|
641
|
+
found.append(Path(folder, name).relative_to(root).as_posix())
|
|
642
|
+
return sorted(found)
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def _lines(data: bytes) -> Iterable[Tuple[int, str]]:
|
|
646
|
+
"""Lines as an editor numbers them: split at LF, a trailing CR dropped (str.splitlines also splits at FF, VT...)."""
|
|
647
|
+
for number, line in enumerate(data.decode("utf-8", "replace").split("\n"), 1):
|
|
648
|
+
yield number, line[:-1] if line.endswith("\r") else line
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def _text_findings(text: str, entries: List[str], seen: Optional["_Seen"] = None) -> List[Tuple[str, str]]:
|
|
652
|
+
found = []
|
|
653
|
+
homes = _home_names(text, _owner_homes(entries))
|
|
654
|
+
if homes:
|
|
655
|
+
found.append(("home-path", "a path inside a person's home folder"))
|
|
656
|
+
if seen is not None:
|
|
657
|
+
seen.names.update(_spellings({text[a:b] for a, b in homes}, _owner_homes(entries)))
|
|
658
|
+
secret = _secret_finding(text)
|
|
659
|
+
if secret:
|
|
660
|
+
found.append(("secret", secret))
|
|
661
|
+
if seen is not None:
|
|
662
|
+
seen.add_values(text)
|
|
663
|
+
rule = _instance_finding(text, entries)
|
|
664
|
+
if rule:
|
|
665
|
+
found.append(("instance-string", rule))
|
|
666
|
+
return found
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
class _Seen:
|
|
670
|
+
"""What the check found anywhere in the tree, so that no printed path repeats it: a file named after its owner, or
|
|
671
|
+
after the key it holds, prints with the name as `<home>` and the segment as `<secret>`."""
|
|
672
|
+
|
|
673
|
+
def __init__(self) -> None:
|
|
674
|
+
self.names: Set[str] = set()
|
|
675
|
+
self.grams: Set[str] = set() # every 8-character run of a found value, the whole of it
|
|
676
|
+
self._names_pattern: Optional["re.Pattern[str]"] = None
|
|
677
|
+
|
|
678
|
+
def add_values(self, text: str) -> None:
|
|
679
|
+
sources = [(text, _secret_spans(text))]
|
|
680
|
+
if "\\" in text:
|
|
681
|
+
unescaped = _json_unescaped(text)
|
|
682
|
+
if unescaped != text:
|
|
683
|
+
sources.append((unescaped, _secret_spans(unescaped)))
|
|
684
|
+
for source, spans in sources:
|
|
685
|
+
# overlapping spans (each start of `sk-sk-sk-...` runs to the end) merged first: their runs are the same,
|
|
686
|
+
# and reading each span apart cost the square of the line
|
|
687
|
+
merged: List[List[int]] = []
|
|
688
|
+
for a, b in sorted(spans):
|
|
689
|
+
if merged and a <= merged[-1][1]:
|
|
690
|
+
merged[-1][1] = max(merged[-1][1], b)
|
|
691
|
+
else:
|
|
692
|
+
merged.append([a, b])
|
|
693
|
+
for a, b in merged:
|
|
694
|
+
value = source[a:b]
|
|
695
|
+
self.grams.update(value[i:i + 8] for i in range(max(0, len(value) - 7)))
|
|
696
|
+
|
|
697
|
+
def names_pattern(self) -> "re.Pattern[str]":
|
|
698
|
+
"""Every name found, longest first, as one pattern: read once the tree is read, when no name is added."""
|
|
699
|
+
if self._names_pattern is None:
|
|
700
|
+
alternatives = sorted(self.names, key=len, reverse=True)
|
|
701
|
+
# a lookahead reads every start, so two names that overlap in a path are both marked
|
|
702
|
+
self._names_pattern = re.compile("(?=(%s))" % "|".join(re.escape(n) for n in alternatives), re.IGNORECASE)
|
|
703
|
+
return self._names_pattern
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
def check(root: Path, entries: List[str], globs: List[str], every_file: bool = False) -> Dict[str, object]:
|
|
707
|
+
"""Read the tree and return {files, skipped, findings}; a finding is {path, line, kind, rule}, its path redacted."""
|
|
708
|
+
unreadable: List[str] = []
|
|
709
|
+
files = None if every_file else _git_files(root)
|
|
710
|
+
mode = "git"
|
|
711
|
+
dependencies: List[str] = [] # dependency folders the walk leaves unread: named, so a clean result says so
|
|
712
|
+
if every_file: # what `pack` reads: every file, ignored or not; `.git` and `__pycache__` it refuses to pack
|
|
713
|
+
files, mode = _walk_files(root, unreadable, PACK_REFUSES), "walk"
|
|
714
|
+
elif not files: # not a checkout, Git refused it, or a folder Git ignores or holds nothing of: read what is there
|
|
715
|
+
files, mode = _walk_files(root, unreadable, WALK_SKIP, dependencies), "walk"
|
|
716
|
+
findings: List[Dict[str, object]] = []
|
|
717
|
+
read = 0
|
|
718
|
+
skipped = {"binary": 0, "unreadable": len(unreadable), "excluded": 0, "nested": 0, "dependencies": len(dependencies)}
|
|
719
|
+
# paths are redacted once the whole tree is read, with what was found anywhere in it (`_Seen`)
|
|
720
|
+
not_read: List[Dict[str, str]] = [{"path": p, "why": "unreadable"} for p in unreadable]
|
|
721
|
+
seen = _Seen()
|
|
722
|
+
|
|
723
|
+
def add(path: str, line: int, kind: str, rule: str) -> None:
|
|
724
|
+
findings.append({"path": path, "line": line, "kind": kind, "rule": rule})
|
|
725
|
+
|
|
726
|
+
for path in files:
|
|
727
|
+
if _ignored(path, globs, root):
|
|
728
|
+
skipped["excluded"] += 1
|
|
729
|
+
continue
|
|
730
|
+
for kind, rule in _text_findings(path, entries, seen): # a name carries instance data as a line does
|
|
731
|
+
add(path, 0, kind, rule + " in the file's path")
|
|
732
|
+
name = path.rsplit("/", 1)[-1]
|
|
733
|
+
if (SECRET_FILE.fullmatch(name) and not SECRET_FILE_OK.fullmatch(name)) or TOKEN_FILE.fullmatch(name):
|
|
734
|
+
add(path, 0, "secret", "a file that is a credential by its name")
|
|
735
|
+
full = root / path
|
|
736
|
+
# os.path.islink, not Path.is_symlink: on Python 3.9 the latter raises on a folder it cannot search
|
|
737
|
+
if any(os.path.islink(str(root / parent)) for parent in Path(path).parents if str(parent) != "."):
|
|
738
|
+
# a tracked folder replaced by a link: the file Git lists is not here, and the link is never followed
|
|
739
|
+
skipped["unreadable"] += 1
|
|
740
|
+
not_read.append({"path": path, "why": "under a symbolic link"})
|
|
741
|
+
continue
|
|
742
|
+
try:
|
|
743
|
+
if full.is_symlink(): # Git commits the path a link names: check it, never follow it
|
|
744
|
+
read += 1
|
|
745
|
+
for kind, rule in _text_findings(os.readlink(full), entries, seen):
|
|
746
|
+
add(path, 0, kind, rule + " in the link's target")
|
|
747
|
+
continue
|
|
748
|
+
kind = os.lstat(str(full)).st_mode
|
|
749
|
+
if stat.S_ISDIR(kind): # Git lists a submodule or a nested repository as one path: its files are not read
|
|
750
|
+
skipped["nested"] += 1
|
|
751
|
+
not_read.append({"path": path, "why": "a nested repository"})
|
|
752
|
+
continue
|
|
753
|
+
if not stat.S_ISREG(kind): # a socket, a pipe: nothing a source carries
|
|
754
|
+
continue
|
|
755
|
+
with open(full, "rb") as handle:
|
|
756
|
+
data = handle.read()
|
|
757
|
+
except FileNotFoundError: # Git lists it and the work tree lacks it: deleted, or outside a sparse checkout
|
|
758
|
+
skipped["unreadable"] += 1
|
|
759
|
+
not_read.append({"path": path,
|
|
760
|
+
"why": "not in the work tree" if mode == "git" else "unreadable"})
|
|
761
|
+
continue
|
|
762
|
+
except OSError:
|
|
763
|
+
skipped["unreadable"] += 1
|
|
764
|
+
not_read.append({"path": path, "why": "unreadable"})
|
|
765
|
+
continue
|
|
766
|
+
if b"\0" in data[:8192]:
|
|
767
|
+
skipped["binary"] += 1
|
|
768
|
+
not_read.append({"path": path, "why": "binary"})
|
|
769
|
+
continue
|
|
770
|
+
read += 1
|
|
771
|
+
for number, line in _lines(data):
|
|
772
|
+
for kind, rule in _text_findings(line, entries, seen):
|
|
773
|
+
add(path, number, kind, rule)
|
|
774
|
+
printed: Dict[str, str] = {} # one redaction per path, however many findings it has
|
|
775
|
+
|
|
776
|
+
def shown(path: str) -> str:
|
|
777
|
+
if path not in printed:
|
|
778
|
+
printed[path] = redact(path, entries, seen)
|
|
779
|
+
return printed[path]
|
|
780
|
+
|
|
781
|
+
for entry in findings:
|
|
782
|
+
entry["path"] = shown(str(entry["path"]))
|
|
783
|
+
for entry in not_read:
|
|
784
|
+
entry["path"] = shown(entry["path"])
|
|
785
|
+
return {"mode": mode, "files": read, "skipped": skipped, "notRead": not_read, "findings": findings,
|
|
786
|
+
"dependencies": [shown(d) for d in dependencies]}
|
|
787
|
+
|
|
788
|
+
|
|
789
|
+
def _absolute(path: str, platform: str) -> bool:
|
|
790
|
+
return bool(re.match(r"[A-Za-z]:[\\/]|[\\/]{2}", path)) if platform == "win32" else path.startswith("/")
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
def default_list(platform: Optional[str] = None, env: Optional[Dict[str, str]] = None) -> Path:
|
|
794
|
+
"""Where the owner's list is kept when none is named: the user's configuration folder, never a project's."""
|
|
795
|
+
platform = platform or sys.platform
|
|
796
|
+
env = os.environ if env is None else env
|
|
797
|
+
# a relative value is ignored, as the XDG spec says (a quoted `~/.config` is relative too): never "no list"
|
|
798
|
+
if platform == "win32":
|
|
799
|
+
appdata = env.get("APPDATA", "")
|
|
800
|
+
base = Path(appdata) if appdata and _absolute(appdata, platform) else Path.home() / "AppData" / "Roaming"
|
|
801
|
+
else:
|
|
802
|
+
xdg = env.get("XDG_CONFIG_HOME", "")
|
|
803
|
+
base = Path(xdg) if xdg.startswith("/") else Path.home() / ".config"
|
|
804
|
+
return base / "passioncode-fabric" / LIST_NAME
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
def _list_path(given: Optional[str]) -> Tuple[Optional[Path], bool]:
|
|
808
|
+
"""The owner's list and whether it must be read (a named or an existing default list; else no list)."""
|
|
809
|
+
named = given or os.environ.get("FABRIC_INSTANCE_STRINGS")
|
|
810
|
+
if named:
|
|
811
|
+
try:
|
|
812
|
+
return Path(named).expanduser(), True
|
|
813
|
+
except (RuntimeError, KeyError): # `~user` with no such user (KeyError on 3.9): never "no finding"
|
|
814
|
+
raise CheckError("the owner's list cannot be read")
|
|
815
|
+
try:
|
|
816
|
+
default = default_list()
|
|
817
|
+
except (RuntimeError, KeyError): # no HOME and no account entry (a container's arbitrary user)
|
|
818
|
+
raise CheckError("there is no home folder to find the owner's list in: set an absolute XDG_CONFIG_HOME (APPDATA on "
|
|
819
|
+
"Windows) or name the list with --instance-strings")
|
|
820
|
+
try:
|
|
821
|
+
os.stat(str(default))
|
|
822
|
+
except (FileNotFoundError, NotADirectoryError):
|
|
823
|
+
# a link to a list, its folder or the configuration folder on a volume not mounted: there, but not reachable
|
|
824
|
+
if any(os.path.lexists(str(p)) and not os.path.exists(str(p)) for p in (default, *default.parents)):
|
|
825
|
+
raise CheckError("the owner's list in your configuration folder cannot be read")
|
|
826
|
+
return None, False
|
|
827
|
+
except OSError: # a list that is there but cannot be reached is not "no list": that would fail open
|
|
828
|
+
raise CheckError("the owner's list cannot be read")
|
|
829
|
+
return default, True
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
class _Parser(argparse.ArgumentParser):
|
|
833
|
+
def error(self, message: str) -> NoReturn: # argparse would repeat the arguments it could not read
|
|
834
|
+
self.print_usage(sys.stderr)
|
|
835
|
+
self.exit(2, "fabric_source_check.py: the arguments cannot be read; see the usage above\n")
|
|
836
|
+
|
|
837
|
+
|
|
838
|
+
def _print(text: str) -> None:
|
|
839
|
+
"""Print a line the console may not encode (a name in another script, a byte Git kept as a surrogate)."""
|
|
840
|
+
encoding = getattr(sys.stdout, "encoding", None) or "utf-8"
|
|
841
|
+
print(text.encode(encoding, "backslashreplace").decode(encoding, "replace"))
|
|
842
|
+
|
|
843
|
+
|
|
844
|
+
def run(argv: Optional[List[str]] = None) -> int:
|
|
845
|
+
parser = _Parser(prog="fabric_source_check.py", description="Find instance data in an agent's source.")
|
|
846
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
847
|
+
cmd = sub.add_parser("check", help="read the source tree and report instance data without repeating it")
|
|
848
|
+
cmd.add_argument("root", nargs="?", default=".")
|
|
849
|
+
cmd.add_argument("--instance-strings", metavar="FILE")
|
|
850
|
+
cmd.add_argument("--exclude", action="append", default=[], metavar="GLOB")
|
|
851
|
+
cmd.add_argument("--json", action="store_true")
|
|
852
|
+
cmd.add_argument("--every-file", action="store_true", help="read every file, not only what Git would commit")
|
|
853
|
+
try:
|
|
854
|
+
args = parser.parse_args(argv)
|
|
855
|
+
except SystemExit as exc: # --help, or arguments argparse refused: the exit code, as the command line gives it
|
|
856
|
+
return exc.code if isinstance(exc.code, int) else 2
|
|
857
|
+
try:
|
|
858
|
+
try:
|
|
859
|
+
root = Path(args.root).expanduser().resolve()
|
|
860
|
+
is_folder = root.is_dir()
|
|
861
|
+
except (OSError, RuntimeError, KeyError): # a parent it cannot search, a symlink loop, `~` with no home
|
|
862
|
+
raise CheckError("the folder cannot be read")
|
|
863
|
+
if not is_folder:
|
|
864
|
+
raise CheckError("ROOT is not a folder")
|
|
865
|
+
list_path, _ = _list_path(args.instance_strings) # may raise CheckError
|
|
866
|
+
entries: List[str] = []
|
|
867
|
+
if list_path is not None:
|
|
868
|
+
try:
|
|
869
|
+
resolved = list_path.resolve()
|
|
870
|
+
except (OSError, RuntimeError): # RuntimeError: Python 3.9's symlink loop
|
|
871
|
+
raise CheckError("the owner's list cannot be read")
|
|
872
|
+
if resolved == root or root in resolved.parents:
|
|
873
|
+
raise CheckError("the owner's list is inside the tree it checks, where it would travel with the source; "
|
|
874
|
+
"keep it in your configuration folder (passioncode-fabric/%s)" % LIST_NAME)
|
|
875
|
+
try:
|
|
876
|
+
present = resolved.is_file()
|
|
877
|
+
except OSError:
|
|
878
|
+
raise CheckError("the owner's list cannot be read")
|
|
879
|
+
if not present:
|
|
880
|
+
raise CheckError("the owner's list named by --instance-strings or FABRIC_INSTANCE_STRINGS does not exist or is "
|
|
881
|
+
"not a regular file"
|
|
882
|
+
if args.instance_strings or os.environ.get("FABRIC_INSTANCE_STRINGS")
|
|
883
|
+
else "the owner's list in your configuration folder is not a file")
|
|
884
|
+
try:
|
|
885
|
+
entries = read_list(resolved)
|
|
886
|
+
except UnicodeDecodeError:
|
|
887
|
+
raise CheckError("the owner's list is not UTF-8")
|
|
888
|
+
except OSError:
|
|
889
|
+
raise CheckError("the owner's list cannot be read")
|
|
890
|
+
result = check(root, entries, ignore_globs(root, args.exclude), args.every_file)
|
|
891
|
+
except CheckError as exc:
|
|
892
|
+
print("fabric_source_check.py: %s" % exc, file=sys.stderr)
|
|
893
|
+
return 2
|
|
894
|
+
findings = result["findings"]
|
|
895
|
+
skipped = result["skipped"]
|
|
896
|
+
listed = sum(1 for e in entries if e and not e.startswith(HOME_ENTRY))
|
|
897
|
+
homes = sum(1 for e in entries if e.startswith(HOME_ENTRY))
|
|
898
|
+
# a binary file is not read either: it is complete only when every one is named as source in the ignore file
|
|
899
|
+
complete = not (skipped["unreadable"] or skipped["binary"] or skipped["nested"]) # type: ignore[index]
|
|
900
|
+
result.update({"instanceStrings": listed, "ownerHomes": homes, "instanceList": list_path is not None, "blocking": bool(findings),
|
|
901
|
+
"complete": complete})
|
|
902
|
+
if args.json:
|
|
903
|
+
_print(json.dumps(result, indent=2))
|
|
904
|
+
else:
|
|
905
|
+
for f in findings: # type: ignore[union-attr]
|
|
906
|
+
_print("%s:%s: %s (%s)" % (f["path"], f["line"], f["kind"], f["rule"]))
|
|
907
|
+
for n in result["notRead"]: # type: ignore[union-attr]
|
|
908
|
+
_print("%s: not read (%s)" % (n["path"], n["why"]))
|
|
909
|
+
for folder in result["dependencies"]: # type: ignore[union-attr]
|
|
910
|
+
_print("%s: not read (a dependency folder; --every-file reads it)" % folder)
|
|
911
|
+
if skipped["dependencies"]: # type: ignore[index]
|
|
912
|
+
_print("%d dependency folder(s) not read (node_modules, virtual environments, caches); --every-file reads "
|
|
913
|
+
"them" % skipped["dependencies"]) # type: ignore[index]
|
|
914
|
+
_print("%d file(s) read (%s), %d finding(s); not read: %d binary, %d unreadable, %d nested repository; "
|
|
915
|
+
"%d left out by name; %d instance string(s) listed%s%s"
|
|
916
|
+
% (result["files"], "what Git would commit" if result["mode"] == "git" else
|
|
917
|
+
"every file, as pack reads them" if args.every_file else "every file but dependency folders, Git not used",
|
|
918
|
+
len(findings), skipped["binary"], skipped["unreadable"], skipped["nested"], # type: ignore[index]
|
|
919
|
+
skipped["excluded"], listed, # type: ignore[index]
|
|
920
|
+
"" if listed or homes else (" (the owner's list holds no string of three or more characters)"
|
|
921
|
+
if list_path else " (no owner's list)"),
|
|
922
|
+
"; %d home name(s) declared" % homes if homes else ""))
|
|
923
|
+
if findings:
|
|
924
|
+
return 1
|
|
925
|
+
return 0 if complete else 2
|
|
926
|
+
|
|
927
|
+
|
|
928
|
+
def _utf8_restart(encoding: str, utf8_mode: int, executable: str, argv: List[str]) -> Optional[List[bytes]]:
|
|
929
|
+
"""The command that restarts the check in Python's UTF-8 mode, or None when it needs none. Under a locale whose
|
|
930
|
+
file-system encoding is not UTF-8 (ISO-8859-1, KOI8-R, a C locale with UTF-8 mode off), a name, an argument and a
|
|
931
|
+
file's text would each be read in its own encoding; in UTF-8 mode all three are, and a byte that is not UTF-8 stays
|
|
932
|
+
that byte. Windows names are UTF-16 and need none. The arguments go back as the bytes they came in. An empty list:
|
|
933
|
+
a restart is needed and no interpreter is known (the caller exits 2 rather than read in two encodings)."""
|
|
934
|
+
if os.name == "nt" or utf8_mode or encoding.lower().replace("_", "-") in ("utf-8", "utf8"):
|
|
935
|
+
return None
|
|
936
|
+
if not executable:
|
|
937
|
+
return []
|
|
938
|
+
return [os.fsencode(executable), b"-X", b"utf8"] + [os.fsencode(a) for a in argv]
|
|
939
|
+
|
|
940
|
+
|
|
941
|
+
def _interpreter() -> str:
|
|
942
|
+
"""The running interpreter: Linux's `/proc/self/exe`, which no PATH or working folder can stand in for (Python 3.9
|
|
943
|
+
may take a `./python3` of the working folder as sys.executable), else sys.executable, which may be empty."""
|
|
944
|
+
try:
|
|
945
|
+
return os.readlink("/proc/self/exe")
|
|
946
|
+
except (OSError, AttributeError):
|
|
947
|
+
return sys.executable or ""
|
|
948
|
+
|
|
949
|
+
|
|
950
|
+
def _interpreter_flags(flags: object, xoptions: Dict[str, object]) -> List[str]:
|
|
951
|
+
"""The interpreter's own options, rebuilt where sys.orig_argv is missing (Python 3.9), so a restart keeps `-I`."""
|
|
952
|
+
out: List[str] = []
|
|
953
|
+
if getattr(flags, "isolated", 0):
|
|
954
|
+
out.append("-I")
|
|
955
|
+
else:
|
|
956
|
+
out += ["-E"] * bool(getattr(flags, "ignore_environment", 0)) + ["-s"] * bool(getattr(flags, "no_user_site", 0))
|
|
957
|
+
out += ["-S"] * bool(getattr(flags, "no_site", 0)) + ["-B"] * bool(getattr(flags, "dont_write_bytecode", 0))
|
|
958
|
+
if getattr(flags, "optimize", 0):
|
|
959
|
+
out.append("-" + "O" * int(getattr(flags, "optimize")))
|
|
960
|
+
for key, value in xoptions.items():
|
|
961
|
+
if key not in ("utf8", "fabric_console"):
|
|
962
|
+
out += ["-X", key if value is True else "%s=%s" % (key, value)]
|
|
963
|
+
return out
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def _keep_console(xoptions: Dict[str, object]) -> None:
|
|
967
|
+
"""After a restart, write the console's own encoding again (the parent named it in `-X fabric_console=`)."""
|
|
968
|
+
console = xoptions.get("fabric_console")
|
|
969
|
+
if isinstance(console, str):
|
|
970
|
+
for stream in (sys.stdout, sys.stderr):
|
|
971
|
+
try:
|
|
972
|
+
stream.reconfigure(encoding=console, errors="backslashreplace") # type: ignore[union-attr]
|
|
973
|
+
except (AttributeError, LookupError, ValueError):
|
|
974
|
+
pass
|
|
975
|
+
|
|
976
|
+
|
|
977
|
+
if __name__ == "__main__":
|
|
978
|
+
_keep_console(getattr(sys, "_xoptions", {}))
|
|
979
|
+
_argv = (list(sys.orig_argv[1:]) if hasattr(sys, "orig_argv")
|
|
980
|
+
else _interpreter_flags(sys.flags, getattr(sys, "_xoptions", {})) + sys.argv)
|
|
981
|
+
_restart = _utf8_restart(sys.getfilesystemencoding(), sys.flags.utf8_mode, _interpreter(), _argv)
|
|
982
|
+
if _restart is not None:
|
|
983
|
+
try:
|
|
984
|
+
if not _restart:
|
|
985
|
+
raise OSError("no interpreter")
|
|
986
|
+
# the output keeps the console's encoding, as before the restart: an -X option survives -I and -E
|
|
987
|
+
_console = getattr(sys.stdout, "encoding", None) or "ascii"
|
|
988
|
+
os.execv(_restart[0], _restart[:3] + [b"-X", os.fsencode("fabric_console=" + _console)] + _restart[3:])
|
|
989
|
+
except OSError:
|
|
990
|
+
sys.stderr.write("fabric_source_check.py: could not restart in Python's UTF-8 mode under this locale; run it "
|
|
991
|
+
"as `python3 -X utf8 fabric_source_check.py ...`\n")
|
|
992
|
+
sys.exit(2)
|
|
993
|
+
sys.exit(run())
|
|
994
|
+
|
|
995
|
+
# #endregion source-check
|