@passioncode-ai/passioncode 0.2.57 → 0.2.59

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. package/CHANGELOG.md +19 -0
  2. package/README.md +2 -2
  3. package/bin/passioncode.js +4 -3
  4. package/family.json +1 -1
  5. package/package.json +1 -1
  6. package/payload/.claude-plugin/marketplace.json +2 -2
  7. package/payload/manifest.json +8 -8
  8. package/payload/plugins/fabric-agent-adapter/.claude-plugin/plugin.json +1 -1
  9. package/payload/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/SKILL.md +16 -2
  10. package/payload/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/references/source-and-instance.md +154 -0
  11. package/payload/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/scripts/fabric_source_check.py +995 -0
  12. package/payload/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/scripts/fabric_source_homes.py +72 -0
  13. package/payload/plugins/fabric-agent-adapter/skills/adapting-projects-to-fabric/scripts/fabric_source_ignore.py +185 -0
  14. package/payload/plugins/fabric-agent-adapter/skills/building-fabric-services/SKILL.md +3 -2
  15. package/payload/plugins/fabric-agent-adapter/skills/building-fabric-services/references/package.md +22 -0
  16. package/payload/plugins/fabric-agent-adapter/skills/building-fabric-services/scripts/kit.json +91 -1
  17. package/payload/plugins/fabric-agent-adapter/skills/creating-fabric-agents/SKILL.md +14 -5
  18. package/payload/plugins/fabric-agent-adapter/skills/creating-fabric-agents/references/source-and-instance.md +154 -0
  19. package/payload/plugins/fabric-agent-adapter/skills/working-with-fabric/SKILL.md +1 -1
  20. package/payload/plugins/fabric-agent-adapter/skills/working-with-fabric/scripts/fabric_session_env.py +1 -1
  21. package/payload/plugins/passioncode/.claude-plugin/plugin.json +1 -1
@@ -0,0 +1,995 @@
1
+ #!/usr/bin/env python3
2
+ """Find instance data in an agent's source tree, without repeating it (FAA-188 part 1).
3
+
4
+ An agent's source is what can be handed over (code, skills, prompts, schemas, examples); its instance is what belongs
5
+ to one installation (records, logs, keys, the owner's names and paths, its configuration). This check reads the files
6
+ the source would carry and reports, by path, line and kind, each place where the instance shows through:
7
+
8
+ - `home-path`: an absolute path inside a person's home folder whose name is ASCII: `/Users/<name>/`, `/home/<name>/`,
9
+ `/var/home/<name>/`, `C:\\Users\\<name>\\` (any case, either separator, a JSON-escaped backslash), the dashed form a
10
+ tool makes of such a path (`-Users-<name>-`, `C--Users-<name>-`, DEC-0105, its name up to the next dash), Git Bash's
11
+ `/c/Users/<name>`, WSL's `/mnt/c/Users/<name>` and the URL-encoded ones (`%2FUsers%2F<name>`, `%5CUsers%5C<name>`); a neutral name (`example`, `user`,
12
+ `runner`, a container's or a cloud machine's user such as `node` or `ubuntu`), a one-letter name or a placeholder
13
+ (`<name>`, `$USER`) is not a finding;
14
+ - `secret`: a value shaped like a credential by the contract's detector (fabric_secrets.py, DEC-0044), read line by
15
+ line: a provider key starting a word, a JWT, a bearer token, or a long run mixing upper case, lower case and digits
16
+ assigned to a name that can only carry a credential (a bare long run, such as an integrity hash, is not; a whole
17
+ Ed25519 signature under a JSON `"sig"`, `"signature"` or `"signatures"` member, as `sign` writes it, is public); or a private key: a PEM, OpenSSH or PGP
18
+ block, or a JWK's `d`; a URL that carries a password or a credential in its query; a file that is a credential by
19
+ its name (pack's rule: `.env`, `*.pem`, `*.key`, `.npmrc` ...) or a kit credential file (`*.token`, `board.credential`). A value that says it is synthetic (`EXAMPLE`, `not-a-key`, `fake`, `dummy`) is
20
+ not a finding;
21
+ - `instance-string`: a string from the owner's list of instance strings (names, accounts, hosts), kept outside the tree.
22
+
23
+ A file's path is read as a line is (line 0). It never prints a value it found, a line that holds one, an entry of the
24
+ owner's list or the folder it checks: a finding names its path (an entry of the list, a home folder's name or a
25
+ credential-shaped value in the path printed as `<line N>`, `<home>` or `<secret>`), line, kind and rule, and an
26
+ instance string by its line in the owner's list. It changes nothing. Exit 0: every file was read and none holds a
27
+ finding; 1: at least one finding; 2: the check could not run, or a file was not read: an unreadable file, a nested
28
+ repository, or a binary one the ignore file does not name as source. A file not read is listed by its path.
29
+
30
+ fabric_source_check.py check [ROOT] [--instance-strings FILE] [--exclude GLOB ...] [--every-file] [--json]
31
+
32
+ The files read are those Git would commit (`git ls-files --cached --others --exclude-standard`), as they are in the
33
+ work tree (a staged copy that differs is not read). With `--every-file`, every file but `.git`, `__pycache__`,
34
+ `.DS_Store` and `*.pyc`, which `pack` refuses as litter (what `pack` reads). When ROOT is not in a Git checkout or Git lists nothing in it, every file outside `.git`,
35
+ `__pycache__`, `node_modules`, virtual environments and caches; the dependency folders it leaves are named (so too
36
+ when no `git` is found in PATH's absolute folders, or with PATH unset in `/bin` and `/usr/bin`; on Windows with PATH
37
+ unset the walk is used). A committed symbolic link is checked by the path it names, never followed. A binary file (a NUL byte in
38
+ its first 8 KiB) is not read; it is listed. Globs in ROOT's `.fabric-source-ignore` (one per line, `#`
39
+ comments) and `--exclude` leave matching paths out. The owner's list is `--instance-strings`, else the file
40
+ `FABRIC_INSTANCE_STRINGS` names, else `passioncode-fabric/instance-strings.txt` in the user's configuration folder
41
+ (`$XDG_CONFIG_HOME` when it is absolute, else `~/.config`; `%APPDATA%` on Windows) when it exists: one string per
42
+ line, a line starting with `#` a comment, `home: <name>` declaring the owner's home name (never neutral), strings
43
+ shorter than three characters ignored, matched without regard to case. A list inside ROOT
44
+ is refused: it would travel with the source. The summary says whether Git chose the files or every file was read.
45
+ Under a locale whose file-system encoding is not UTF-8 the check restarts itself in Python's UTF-8 mode, so a name,
46
+ an argument and a file are read in one encoding. Standard library only.
47
+ """
48
+
49
+ # #region source-check — docs: plugins/fabric-agent-adapter/skills/creating-fabric-agents/references/source-and-instance.md#the-check
50
+
51
+ from __future__ import annotations
52
+
53
+ import argparse
54
+ import bisect
55
+ import json
56
+ import os
57
+ from pathlib import Path
58
+ import re
59
+ import stat
60
+ import subprocess
61
+ import sys
62
+ import unicodedata
63
+ from urllib.parse import unquote
64
+ from typing import Dict, Iterable, List, NoReturn, Optional, Set, Tuple
65
+
66
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
67
+ _WROTE_BYTECODE, sys.dont_write_bytecode = sys.dont_write_bytecode, True # a copy inside the checked tree adds no .pyc
68
+ _STEP = "fabric_secrets" # the sibling being loaded, so a refusal names it whatever the exception says
69
+ try:
70
+ import fabric_secrets as _secrets # noqa: E402 - the contract's credential detector (DEC-0044), beside this file
71
+ _STEP = "fabric_source_ignore"
72
+ from fabric_source_ignore import ( # noqa: E402,F401 - how the ignore lines are read, beside this file
73
+ IGNORE_FILE, CheckError, _as_bytes, _glob, _ignored, _line, _readable_glob, ignore_globs)
74
+ _STEP = "fabric_source_homes"
75
+ from fabric_source_homes import ( # noqa: E402,F401 - how a declared home is read, beside this file
76
+ HOME_ENTRY, _NAME_JOIN, _NAME_START, _SEP, _dashed_home, _name_parts, _owner_homes, _spellings)
77
+ except Exception as _missing: # a copy without its siblings, or mixed from two versions: a refusal, never exit 1
78
+ # the exception's own text is never printed: an OSError's carries a path, which may be under the owner's home
79
+ _why = "a module beside this file refused to load"
80
+ try:
81
+ _HERE, _sibling = Path(__file__).resolve().parent, _STEP + ".py"
82
+ _file = getattr(_missing, "filename", None) if isinstance(_missing, OSError) else None
83
+ _why = ("%s must sit beside this file" % _sibling if not (_HERE / _sibling).is_file()
84
+ else "%s beside this file cannot be read" % _sibling
85
+ if isinstance(_file, (str, bytes)) and Path(os.fsdecode(_file)).resolve() == _HERE / _sibling
86
+ else "%s beside this file is from another version: copy it with this one" % _sibling
87
+ if isinstance(_missing, (ImportError, SyntaxError))
88
+ else "%s beside this file refused to load (%s): copy the scripts folder whole, from one version"
89
+ % (_sibling, type(_missing).__name__))
90
+ except Exception: # naming the cause must never be what fails
91
+ pass
92
+ sys.stderr.write("fabric_source_check.py: %s\n" % _why)
93
+ sys.exit(2)
94
+ finally:
95
+ sys.dont_write_bytecode = _WROTE_BYTECODE
96
+
97
+ LIST_NAME = "instance-strings.txt"
98
+ #: The folders `pack` refuses to carry (fabric_package.JUNK); --every-file leaves them and the files below unread.
99
+ PACK_REFUSES = {".git", "__pycache__"}
100
+ WALK_SKIP = {".git", "node_modules", "__pycache__", ".venv", "venv", ".tox", ".mypy_cache", ".pytest_cache"}
101
+ #: Names that stand for nobody in a home path: placeholders, CI runners and the system's shared folders.
102
+ NEUTRAL_HOMES = {"example", "examples", "user", "username", "you", "me", "name", "someone", "runner", "shared", "public",
103
+ "default", "guest", "linuxbrew", "alice", "bob", "operator", "owner", "person", "your-name", "yourname",
104
+ # the users of container images, cloud machines and hosted CI
105
+ "node", "ubuntu", "ec2-user", "runneradmin", "admin", "vagrant", "debian", "centos", "app", "builder",
106
+ # dev containers and hosted workspaces, and the reference's own fixture id
107
+ "vscode", "codespace", "circleci", "gitpod", "jovyan", "coder", "example-agent"}
108
+ HOME_PATHS = (
109
+ re.compile(r"(?<![A-Za-z0-9_.-])(?:/var)?/home/([A-Za-z0-9][A-Za-z0-9._-]*)", re.ASCII),
110
+ # macOS writes `/Users` capitalised; `/users/...` in code is a web route
111
+ re.compile(r"(?<![A-Za-z0-9_.-])/Users/([A-Za-z0-9][A-Za-z0-9._-]*)", re.ASCII),
112
+ # Git Bash's `/c/Users/<name>` and WSL's `/mnt/c/Users/<name>`
113
+ re.compile(r"(?<![A-Za-z0-9_.-])(?:/mnt)?/[A-Za-z]/Users/([A-Za-z0-9][A-Za-z0-9._-]*)", re.ASCII),
114
+ re.compile(r"(?<![A-Za-z0-9_.-])[A-Za-z]:%sUsers%s([A-Za-z0-9][A-Za-z0-9._-]*)" % (_SEP, _SEP), re.ASCII | re.IGNORECASE),
115
+ # DEC-0105's dashed form starts a name; after a dash it is a flag or a CSS variable (`--home-dir`)
116
+ # (on Windows a drive comes first: `C--Users-<name>-proj`; Silverblue's, WSL's and Git Bash's homes dash to
117
+ # `-var-home-<name>`, `-mnt-c-Users-<name>` and `-c-Users-<name>`)
118
+ re.compile(r"(?:^|[^A-Za-z0-9-])(?:-Users|-home|-var-home|(?:-mnt)?-[A-Za-z]-Users)-([A-Za-z0-9][A-Za-z0-9._]*)",
119
+ re.ASCII),
120
+ re.compile(r"(?<![A-Za-z0-9])[A-Za-z]--Users-([A-Za-z0-9][A-Za-z0-9._]*)", re.ASCII | re.IGNORECASE),
121
+ re.compile(r"%2[Ff](?:Users|home)%2[Ff]([A-Za-z0-9][A-Za-z0-9._-]*)", re.ASCII), # `%2Fusers%2F` is a route
122
+ re.compile(r"%5CUsers%5C([A-Za-z0-9][A-Za-z0-9._-]*)", re.ASCII | re.IGNORECASE),
123
+ )
124
+ PRIVATE_KEY = re.compile(r"-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----") # PEM, OpenSSH, PGP
125
+ #: A private JWK's `d` (`sign --key` takes an Ed25519 one): 32 bytes or more in base64url.
126
+ JWK_PRIVATE = re.compile(r"\\?[\"']d\\?[\"'][ \t]*:[ \t]*\\?[\"']([A-Za-z0-9_-]{43,})") # JSON, escaped, Python
127
+ #: Words a documented or synthetic credential carries (AWS's `...EXAMPLE`, `sk-test-not-a-key`); a random key does not.
128
+ SYNTHETIC = ("example", "not-a-key", "notakey", "fake", "dummy", "placeholder", "redacted")
129
+ # The detector's provider-key shapes without its first alternative, `^bearer\s.+`, which reads a whole header value: in
130
+ # a file it would take prose that starts with the word and hide a key later on the same line. A bearer token is looked
131
+ # for as `_BEARER_INSIDE` does: the word, then a run of 12 or more with a digit.
132
+ _BEARER_FIRST = "(^bearer["
133
+ if not _secrets.SECRET_VALUE.pattern.startswith(_BEARER_FIRST): # not an assert: `python -O` would drop it
134
+ raise ImportError("fabric_secrets.SECRET_VALUE changed its first branch; update fabric_source_check.py with it")
135
+ PROVIDER_KEY = re.compile("(" + _secrets.SECRET_VALUE.pattern.split("|", 1)[1], _secrets.SECRET_VALUE.flags)
136
+ #: Files that are credentials by their name: pack's own rule (fabric_package.SECRET_FILE and SECRET_FILE_OK, held
137
+ #: equal by test_source_check), and the kit's token files (`service.token`, fabric_auth; `board.credential`, fabric_hub).
138
+ _JS_DOT = "[^\n\r\u2028\u2029]"
139
+ SECRET_FILE = re.compile(r"\.env(\.%s+)?|%s*\.(pem|key|p12|pfx|jks|keystore)|id_(rsa|dsa|ecdsa|ed25519)|\.(npmrc|netrc|pypirc)"
140
+ % (_JS_DOT, _JS_DOT))
141
+ SECRET_FILE_OK = re.compile(r"\.env\.(example|sample|template)")
142
+ _PACK_JUNK_FILE = re.compile(r"\.DS_Store|%s*\.pyc" % _JS_DOT) # held to fabric_package.JUNK by test_source_check
143
+ TOKEN_FILE = re.compile(r".+\.(token|credential)") # the kit's service.token and board.credential
144
+ #: An absolute URL in a line, read whole by `_url_secret`: a password in it, a credential in its query.
145
+ URL = re.compile(r"[A-Za-z][A-Za-z0-9+.-]{0,63}://[^\s\"'<>`]+") # a bounded scheme: linear, and `${X:-pg://..}`
146
+ #: `NAME = value`, `NAME: value` or `"name": "value"`: the long-run rule applies only where the name carries a credential.
147
+ #: The name is bounded and starts a word, so a long line is read in linear time.
148
+ ASSIGNED = re.compile(r"(?<![A-Za-z0-9_.-])([A-Za-z_][A-Za-z0-9_.-]{0,63})[\"']?[ \t]{0,8}[:=][ \t]{0,8}[\"']?"
149
+ r"((?:[A-Za-z0-9+/_-]|%[0-9A-Fa-f]{2}){40,}={0,2})") # `=` only as padding; `%2B` read decoded
150
+
151
+
152
+ def _home_names(text: str, owner: Iterable[str] = ()) -> List[Tuple[int, int]]:
153
+ """The spans of the names of real people's home folders in a text (neutral and one-letter names left out, unless
154
+ the owner's list declares the name as the owner's home)."""
155
+ owner = set(owner)
156
+ spans = []
157
+ for pattern in HOME_PATHS:
158
+ for match in pattern.finditer(text):
159
+ name = match.group(1).lower()
160
+ if name in owner or (len(name) > 1 and name not in NEUTRAL_HOMES): # one letter (`/Users/a`): a placeholder
161
+ spans.append(match.span(1))
162
+ for pattern in (p for name in sorted(owner) for p in _dashed_home(name)): # declared: found whatever it reads as
163
+ spans += [match.span(1) for match in pattern.finditer(text)]
164
+ return spans
165
+
166
+
167
+ def _real(token: str) -> bool:
168
+ lowered = token.lower()
169
+ return not any(word in lowered for word in SYNTHETIC)
170
+
171
+
172
+ #: An Ed25519 signature is public: `sign` writes one into fabric.package.json (`"sig"`), and every signed record carries
173
+ #: one. Only a whole one (64 bytes, 86 base64url characters) is: a storage URL's `sig=` (an HMAC) grants access.
174
+ PUBLIC_NAMES = {"sig", "signature", "signatures"}
175
+ _ED25519_SIGNATURE = re.compile(r"[A-Za-z0-9_-]{86}")
176
+
177
+
178
+ def _credential(name: str, value: str, quoted: bool) -> bool:
179
+ """A credential's name, but a whole Ed25519 signature as a JSON member (`"sig": "..."`, as `sign` writes it)."""
180
+ if quoted and name.lower() in PUBLIC_NAMES and _ED25519_SIGNATURE.fullmatch(value):
181
+ return False
182
+ return _secrets.credential_name(name)
183
+
184
+
185
+ _COLON_AFTER = re.compile(r"[ \t]*:")
186
+
187
+
188
+ def _quoted(text: str, match: "re.Match[str]") -> bool:
189
+ """`"name": value`: a JSON member, the name quoted on both sides and `:` after it (never `"sig=...`)."""
190
+ start, end = match.start(1), match.end(1)
191
+ # matched in place: a copy of the rest of the line for each member made a minified JSON line quadratic
192
+ return start > 0 and text[start - 1] == '"' and text[end:end + 1] == '"' and _COLON_AFTER.match(text, end + 1) is not None
193
+
194
+
195
+ def _url_secret(url: str, depth: int = 0) -> bool:
196
+ """A URL that carries a credential, as a source tree holds one. The contract's URL rule (LC-21) reads one value a
197
+ launch names, where any `token=` is one; across a tree it would read every documented `?token=abc` and `user:pw@`.
198
+ So: a password of 8 or more mixing letters and digits, or a query or fragment value that, percent-decoded, is shaped
199
+ like a credential (a provider key, a JWT, a long random run under a credential's name)."""
200
+ url = url.rstrip(".,;)]}")
201
+ if not _secrets._SCHEME.match(url):
202
+ return False
203
+ password, query, fragment = _secrets._url_parts(url)
204
+ # what says it is synthetic is the value, never the host or the path (`examples.blob.core.windows.net`)
205
+ if len(password) >= 8 and re.search(r"[0-9]", password) and re.search(r"[A-Za-z]", password) and _real(unquote(password)):
206
+ return True
207
+ for part in (query, fragment):
208
+ for name, raws in _secrets.raw_values(part).items():
209
+ for raw in raws:
210
+ value = unquote(raw)
211
+ if depth < 2 and "://" in value and _url_secret(value, depth + 1): # `?next=https%3A%2F%2F...sig%3D`
212
+ return True # read before the synthetic filter: an inner URL is judged by its own values
213
+ if any(True for _ in _provider_keys(value)): # each key by its own word, as on a line
214
+ return True
215
+ if not _real(value):
216
+ continue
217
+ if (_secrets.credential_name(name) and _secrets._long_secret(value)) or \
218
+ _secrets.JWT_SHAPE.search(value) or _secrets.SECRET_VALUE.search(value):
219
+ return True
220
+ return False
221
+
222
+
223
+ #: The JSON escapes tools write for `&`, `=` and `/` (Go's encoding/json writes `\u0026`, PHP's json_encode `\/`).
224
+ _JSON_ESCAPES = re.compile(r"\\(?:u0026|u003[dD]|/)")
225
+
226
+
227
+ _UNESCAPED = {"u0026": "&", "u003d": "=", "u003D": "=", "/": "/"}
228
+
229
+
230
+ def _json_unescaped(line: str) -> str:
231
+ return _JSON_ESCAPES.sub(lambda m: _UNESCAPED[m.group(0)[1:]], line)
232
+
233
+
234
+ def _unescaped_with_origin(line: str) -> Tuple[str, List[Tuple[int, int]]]:
235
+ """The line with JSON escapes undone, and for each of its characters the raw span it came from."""
236
+ out: List[str] = []
237
+ origin: List[Tuple[int, int]] = []
238
+ last = 0
239
+ for m in _JSON_ESCAPES.finditer(line):
240
+ out.extend(line[last:m.start()])
241
+ origin.extend((i, i + 1) for i in range(last, m.start()))
242
+ out.append(_UNESCAPED[m.group(0)[1:]])
243
+ origin.append(m.span())
244
+ last = m.end()
245
+ out.extend(line[last:])
246
+ origin.extend((i, i + 1) for i in range(last, len(line)))
247
+ return "".join(out), origin
248
+
249
+
250
+ def _secret_finding(line: str) -> Optional[str]:
251
+ """`_secret_in`, on the line as written and, when it holds a JSON escape of `&`, `=` or `/`, as JSON reads it."""
252
+ found = _secret_in(line)
253
+ if found is None and "\\" in line:
254
+ unescaped = _json_unescaped(line)
255
+ if unescaped != line:
256
+ found = _secret_in(unescaped)
257
+ return found
258
+
259
+
260
+ #: A whole Ed25519 signature as `sign` writes it: public, so no rule reads it (not even a provider key's shape in it).
261
+ _PUBLIC_SIGNATURE = re.compile(r"\"(?:sig|signature|signatures)\"[ \t]*:[ \t]*\"[A-Za-z0-9_-]{86}\"", re.IGNORECASE)
262
+
263
+
264
+ def _secret_in(line: str) -> Optional[str]:
265
+ if '"' in line:
266
+ line = _PUBLIC_SIGNATURE.sub('"sig": ""', line)
267
+ if PRIVATE_KEY.search(line):
268
+ return "a private key block"
269
+ if any(_real(m.group(1)) for m in JWK_PRIVATE.finditer(line)):
270
+ return "a private key (a JWK's `d`)"
271
+ if any(_real(m.group(0)) for m in _secrets.JWT_SHAPE.finditer(line)):
272
+ return "a token shaped like a JWT"
273
+ # The detector reads one configuration value; across a line, `sk-` needs a word start of its own, or every name
274
+ # with a segment ending in `sk` (`task-management-service`) would read as a key.
275
+ if any(True for _ in _provider_keys(line)):
276
+ return "a value shaped like a provider key"
277
+ if any(_real(m.group(0)) for m in _overlapping(_secrets._BEARER_INSIDE, line)):
278
+ return "a bearer token"
279
+ if "://" in line and any(_url_secret(m.group(0)) for m in URL.finditer(line)):
280
+ return "a URL that carries a credential"
281
+ for m in (_assigned(line) if ":" in line or "=" in line else ()):
282
+ name, value = m.group(1), m.group(2)
283
+ if _credential(name, value, _quoted(line, m)) and _secrets._long_secret(unquote(value)) and _real(unquote(value)):
284
+ return "a long random value assigned to a credential's name"
285
+ return None
286
+
287
+
288
+ def _assigned(line: str) -> Iterable["re.Match[str]"]:
289
+ """ASSIGNED's matches, each search starting again inside the last value: `id=<run>/TOKEN=<value>` reads both."""
290
+ pos = 0
291
+ while True:
292
+ m = ASSIGNED.search(line, pos)
293
+ if m is None:
294
+ return
295
+ yield m
296
+ pos = m.start(2) if m.start(2) > pos else m.end()
297
+
298
+
299
+ def _fold_map(text: str) -> Tuple[str, List[int], List[int]]:
300
+ """Text as an entry is compared, with where each of its characters came from: each letter with the marks after it
301
+ is casefolded, then composed (NFC), so `é` typed in the list finds a name a file system stored decomposed (macOS
302
+ names made through its file-system representation are) and `ß` finds `ss`. Folding changes lengths both ways, so a
303
+ match is mapped back to the original characters through the starts and ends, never by its offsets."""
304
+ out: List[str] = []
305
+ starts: List[int] = []
306
+ ends: List[int] = []
307
+ run = 0
308
+ last = text[:1] # the last character of the piece composed so far: what the next one may compose with
309
+ for k in range(1, len(text) + 1):
310
+ if k < len(text):
311
+ joined = unicodedata.normalize("NFC", last + text[k])
312
+ if unicodedata.combining(text[k]) or len(joined) < len(last) + 1: # a mark, or a jamo or vowel sign
313
+ last = joined[-1:]
314
+ continue
315
+ last = text[k]
316
+ piece = unicodedata.normalize("NFC", text[run:k].casefold())
317
+ out.append(piece)
318
+ starts += [run] * len(piece)
319
+ ends += [k] * len(piece)
320
+ run = k
321
+ return "".join(out), starts, ends
322
+
323
+
324
+ def _fold(text: str) -> str:
325
+ return _fold_map(text)[0]
326
+
327
+
328
+ #: `home: <name>`: one ASCII name (a letter or digit first; letters, digits, `.`, `_`, `-`, `'` and spaces inside, as
329
+ #: a Windows profile may hold), a horizontal space (a space, a tab, a Unicode space) around the colon; any other white
330
+ #: space there is refused. The name is checked as ASCII apart: IGNORECASE alone would
331
+ #: let `[A-Za-z]` take U+017F and U+212A.
332
+ _GAP = "[ \t\u00a0\u1680\u2000-\u200a\u202f\u205f\u3000]*" # a horizontal space only: never a control or a separator
333
+ _HOME_LINE = re.compile(r"home%s:%s([A-Za-z0-9](?:[A-Za-z0-9._' -]*[A-Za-z0-9._'-])?)" % (_GAP, _GAP), re.IGNORECASE)
334
+ _HOME_KEY = re.compile(r"home\s*:", re.IGNORECASE) # any white space, so a control or a separator is refused
335
+
336
+
337
+ def read_list(path: Path) -> List[str]:
338
+ """The owner's list, one item per line: an entry casefolded, a declared home (`home: <name>`) as HOME_ENTRY and the
339
+ name lowered, or '' for a comment, a blank or too short a line."""
340
+ entries = []
341
+ # at CRLF, CR or LF: a list saved with CR alone keeps its lines, and a FF or U+2028 starts none
342
+ for raw in re.split(r"\r\n|\r|\n", path.read_bytes().decode("utf-8-sig")):
343
+ # a byte-order mark or a zero-width character at a line's start (two lists joined) is no part of the line
344
+ if HOME_ENTRY in raw: # never read as another name: a NUL is no character an owner types
345
+ raise CheckError("line %d of the owner's list holds a NUL character: save it as plain text" % (len(entries) + 1))
346
+ text = raw.strip().lstrip("\ufeff\u200b\u2060").strip() # no meaning at a start
347
+ # a joiner (U+200C, U+200D) may belong to a string, never to `home:`: a home line is read or refused with every
348
+ # format character (Unicode Cf: a joiner, a direction mark, a soft hyphen) taken out, wherever it stands
349
+ bare = "".join(c for c in text if unicodedata.category(c) != "Cf").strip()
350
+ home = _HOME_LINE.fullmatch(bare)
351
+ # the name as written: a format character inside it changes the name, so that line is refused, not read as another
352
+ # (looked for after the key's colon, so the letters of `home` cannot stand in for a name such as `me`)
353
+ if home and home.group(1).isascii() and home.group(1) in text.split(":", 1)[1]:
354
+ entries.append(HOME_ENTRY + home.group(1).lower())
355
+ continue
356
+ if home and home.group(1).isascii(): # visible as a home name, but an invisible character sits inside it
357
+ raise CheckError("line %d of the owner's list names a home that holds an invisible format character (a soft "
358
+ "hyphen, a zero-width space or a direction mark, which an editor may not show): delete it "
359
+ "and write the name again" % (len(entries) + 1))
360
+ if _HOME_KEY.match(bare): # meant as a home and not one: never kept as a string that cannot match
361
+ raise CheckError("line %d of the owner's list starts with `home` and a colon but is not one ASCII home name "
362
+ "and nothing after it (a letter or digit first, then letters, digits, spaces, `.`, `_`, `-`, "
363
+ "`'`): a note goes on a line of its own, a path is written as its home name only, and a name "
364
+ "outside ASCII, or holding another character, is listed as a plain string (a typographic "
365
+ "apostrophe the folder's name does not hold is written as '; a folder whose name holds one is "
366
+ "listed as a plain string with it kept and declared too with `home:` written with ')"
367
+ % (len(entries) + 1))
368
+ # a comment or a blank is judged without its invisible characters too, as a home line is (one rule per line)
369
+ # a string keeps a joiner between its letters (an emoji needs one); one at either end belongs to no string
370
+ edge = [i for i, c in enumerate(text) if not (c.isspace() or unicodedata.category(c) == "Cf")]
371
+ text = text[edge[0]:edge[-1] + 1] if edge else "" # ends taken in one slice, spaces between marks too
372
+ entries.append("" if not bare or bare.startswith("#") or len(text) < 3 else _fold(text))
373
+ return entries
374
+
375
+
376
+ def _entry_spans(text: str, entries: List[str]) -> List[Tuple[int, int, int]]:
377
+ strings = [(n, e) for n, e in enumerate(entries, 1) if e and not e.startswith(HOME_ENTRY)]
378
+ if not strings:
379
+ return []
380
+ if text.isascii(): # casefolding ASCII is lowering it, one character for one: no map to build
381
+ folded, starts, ends = text.lower(), None, None
382
+ else:
383
+ folded, starts, ends = _fold_map(text)
384
+ spans = []
385
+ for number, entry in strings:
386
+ start = folded.find(entry)
387
+ while start != -1: # back to the original characters: every one a matched character came from
388
+ end = start + len(entry)
389
+ spans.append((start, end, number) if starts is None or ends is None
390
+ else (starts[start], ends[end - 1], number))
391
+ start = folded.find(entry, start + 1)
392
+ return spans
393
+
394
+
395
+ def _instance_finding(text: str, entries: List[str]) -> Optional[str]:
396
+ spans = _entry_spans(text, entries)
397
+ return "a string from the owner's list (line %d)" % spans[0][2] if spans else None
398
+
399
+
400
+ def _alternatives(pattern: "re.Pattern[str]") -> List[Tuple["re.Pattern[str]", ...]]:
401
+ """Each provider-key alternative of the contract's pattern as (its starts, its prefix, a run of its class, least
402
+ length, open-ended, the whole alternative), so a key is read where its own class ends, not where another's does."""
403
+ found = []
404
+ for alt in pattern.pattern[1:-1].split("|"):
405
+ part = re.fullmatch(r"(.+?)(\[[^\]]+\])\{(\d+)(,?)\}", alt)
406
+ if part is None: # not an assert: `python -O` would drop it
407
+ raise ImportError("fabric_secrets.SECRET_VALUE has a key shape this check cannot read: %s" % alt[:12])
408
+ prefix, cls, least, open_ended = part.groups()
409
+ found.append((re.compile("(?=%s)" % prefix, pattern.flags), re.compile(prefix, pattern.flags),
410
+ re.compile(cls + "+", pattern.flags), int(least), bool(open_ended), re.compile(alt, pattern.flags)))
411
+ return found
412
+
413
+
414
+ _KEY_SHAPES = _alternatives(PROVIDER_KEY)
415
+ _ASCII_LOWER = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz")
416
+
417
+
418
+ def _provider_keys(text: str) -> Iterable[Tuple[int, int]]:
419
+ """The spans of provider keys that start a word and do not say they are synthetic, each read where it starts, so a
420
+ key a dropped run swallowed is found too (`task-runner-sk-<key>`, `sk-<16>_ghp_<key>_fake`). Every start is read
421
+ once, by its own class's run end and the next synthetic word in the line: O(n log n) on any line."""
422
+ if not PROVIDER_KEY.search(text):
423
+ return
424
+ lowered = text.translate(_ASCII_LOWER) # a key's characters are ASCII, and this keeps every offset
425
+ synthetic = [(len(word), [m.start() for m in re.finditer("(?=%s)" % re.escape(word), lowered)]) for word in SYNTHETIC]
426
+ for starts, prefix, run, least, open_ended, whole in _KEY_SHAPES:
427
+ runs = None
428
+ for m in starts.finditer(text):
429
+ p = m.start()
430
+ if p and text[p - 1].isalnum():
431
+ continue
432
+ if open_ended:
433
+ if runs is None:
434
+ runs = [r.span() for r in run.finditer(text)]
435
+ run_starts = [a for a, _ in runs]
436
+ q = p + len(prefix.match(text, p).group(0))
437
+ i = bisect.bisect_right(run_starts, q) - 1
438
+ if i < 0 or runs[i][1] - q < least:
439
+ continue
440
+ end = runs[i][1]
441
+ else:
442
+ key = whole.match(text, p)
443
+ if key is None:
444
+ continue
445
+ end = key.end()
446
+ if not any(k < len(at) and at[k] + n <= end
447
+ for n, at in synthetic for k in (bisect.bisect_left(at, p),)):
448
+ yield p, end
449
+
450
+
451
+ def _overlapping(pattern: "re.Pattern[str]", text: str) -> Iterable["re.Match[str]"]:
452
+ """Matches that may start inside the last one (a second `Bearer` inside the first token's run gets its own span)."""
453
+ pos = 0
454
+ while pos <= len(text):
455
+ m = pattern.search(text, pos)
456
+ if m is None:
457
+ return
458
+ yield m
459
+ pos = m.start() + 1
460
+
461
+
462
+ def _secret_spans(text: str) -> List[Tuple[int, int]]:
463
+ """The spans of credential-shaped values in a text, as `_secret_in` reads them (a public signature is no span)."""
464
+ public = [m.span() for m in _PUBLIC_SIGNATURE.finditer(text)] if '"' in text else []
465
+ return [(a, b) for a, b in _raw_secret_spans(text) if not any(p <= a and b <= q for p, q in public)]
466
+
467
+
468
+ def _raw_secret_spans(text: str) -> List[Tuple[int, int]]:
469
+ spans = [m.span() for m in PRIVATE_KEY.finditer(text)]
470
+ spans += [m.span() for m in _secrets.JWT_SHAPE.finditer(text) if _real(m.group(0))]
471
+ spans += [m.span(1) for m in JWK_PRIVATE.finditer(text) if _real(m.group(1))]
472
+ spans += list(_provider_keys(text))
473
+ spans += [m.span() for m in _overlapping(_secrets._BEARER_INSIDE, text) if _real(m.group(0))]
474
+ spans += [m.span() for m in URL.finditer(text) if _url_secret(m.group(0))]
475
+ spans += [m.span(2) for m in _assigned(text)
476
+ if _credential(m.group(1), m.group(2), _quoted(text, m)) and _secrets._long_secret(unquote(m.group(2)))
477
+ and _real(unquote(m.group(2)))]
478
+ return spans
479
+
480
+
481
+ def redact(path: str, entries: List[str], seen: Optional["_Seen"] = None) -> str:
482
+ """A path as it may be printed: an owner's string as `<line N>` (its line in the owner's list), a real home
483
+ folder's name as `<home>`, and each segment in which `_secret_finding` finds a credential as `<secret>`, whole
484
+ (a span inside a segment can miss a second value the first one runs into). While the segments left still hold a
485
+ credential together (`https:\\/\\/app:<pw>@db/x`), the segments that finding's spans touch are masked, else the
486
+ last one left, so what is printed holds no finding and its first folders still name the place. With `seen`, what the
487
+ check found elsewhere in the tree is masked too: a home name at every occurrence, and a segment holding a run of 8
488
+ characters of a found value as `<secret>`."""
489
+ spans, offset = [], 0
490
+ for segment in path.split("/"):
491
+ spans.append((offset, offset + len(segment)))
492
+ offset += len(segment) + 1
493
+ masked = [bool(_secret_finding(path[a:b])) for a, b in spans]
494
+ # every segment a found value's span touches, on the path and with JSON escapes undone (mapped back), so a value
495
+ # that runs across a `/` prints no part of itself
496
+ touched = list(_secret_spans(path))
497
+ if "\\" in path:
498
+ unescaped, origin = _unescaped_with_origin(path)
499
+ if unescaped != path:
500
+ touched += [(origin[a][0], origin[b - 1][1]) for a, b in _secret_spans(unescaped) if b > a]
501
+ for a, b in touched:
502
+ for i, (s, e) in enumerate(spans):
503
+ if s < b and a < e:
504
+ masked[i] = True
505
+
506
+ def rest() -> str:
507
+ # a masked segment keeps its length, so the spans read in what is left are offsets in the path
508
+ return "/".join(" " * (b - a) if m else path[a:b] for (a, b), m in zip(spans, masked))
509
+
510
+ # the guard: while what is left still holds a credential, mask the segments its spans touch, else the last one left
511
+ for _ in range(len(spans)):
512
+ left = rest()
513
+ if not _secret_finding(left):
514
+ break
515
+ in_left = _secret_spans(left) # once a pass: read per segment, a long path cost the cube of its folders
516
+ if "\\" in left: # a finding only the unescaped reading sees has its spans there, mapped back
517
+ unescaped, origin = _unescaped_with_origin(left)
518
+ if unescaped != left:
519
+ in_left += [(origin[a][0], origin[b - 1][1]) for a, b in _secret_spans(unescaped) if b > a]
520
+ hit = [i for i, (s, e) in enumerate(spans) if not masked[i] and any(s < b and a < e for a, b in in_left)]
521
+ if not hit:
522
+ hit = [max(i for i, m in enumerate(masked) if not m)]
523
+ for i in hit:
524
+ masked[i] = True
525
+ if seen is not None and seen.grams: # a run of a value found elsewhere in the tree, across a `/` too
526
+ for k in range(len(path) - 7):
527
+ if path[k:k + 8] in seen.grams:
528
+ for i, (a, b) in enumerate(spans):
529
+ if a < k + 8 and k < b:
530
+ masked[i] = True
531
+ secret_segments = [span for span, m in zip(spans, masked) if m]
532
+ marks = [(s, e, "<line %d>" % n) for s, e, n in _entry_spans(path, entries)]
533
+ found = _home_names(path, _owner_homes(entries))
534
+ marks += [(s, e, "<home>") for s, e in found]
535
+ # every other occurrence of a found name, searched in the path itself: a lowered copy can change its length (U+0130),
536
+ # and Unicode case matching also takes the letters that fold to ASCII ones (U+212A to `k`, U+017F to `s`); the names
537
+ # found elsewhere in the tree are one compiled pattern, so the cost does not grow with names times paths
538
+ # a declared home of three or more letters and digits is the owner's own name: masked wherever a printed path holds
539
+ # it, its parts joined by none or any run of the separators a tool writes (a space, `.`, `_`, `-`, `+`, and their
540
+ # percent-encodings), found or not
541
+ known = {path[s:e] for s, e in found} | (seen.names if seen is not None else set())
542
+ for declared in _owner_homes(entries):
543
+ parts = _name_parts(declared)
544
+ if sum(len(part) for part in parts) < 3 and len(parts) > 1: # a short name found: each spelling with a separator
545
+ spaced = (_NAME_JOIN + "+").join(re.escape(part) for part in parts)
546
+ if any(re.fullmatch(spaced, name, re.IGNORECASE) for name in known):
547
+ marks += [(m.start(), m.end(), "<home>") for m in re.finditer(spaced, path, re.IGNORECASE)]
548
+ if sum(len(part) for part in parts) >= 3:
549
+ body = (_NAME_JOIN + "*").join(re.escape(part) for part in parts)
550
+ # found in the tree: masked wherever it stands; unfound: a whole word only, so `home: app` leaves `mapping`
551
+ # readable (a percent-encoded or escaped separator before it starts a word too)
552
+ if not any(re.fullmatch(body, name, re.IGNORECASE) for name in known):
553
+ body = _NAME_START + body + "(?![A-Za-z0-9])"
554
+ marks += [(m.start(), m.end(), "<home>") for m in re.finditer(body, path, re.IGNORECASE)]
555
+ for name in _spellings({path[s:e] for s, e in found}, _owner_homes(entries)):
556
+ marks += [(m.start(), m.end(), "<home>") for m in re.finditer(re.escape(name), path, re.IGNORECASE)]
557
+ if seen is not None and seen.names:
558
+ marks += [(m.start(1), m.end(1), "<home>") for m in seen.names_pattern().finditer(path)]
559
+ marks += [(s, e, "<secret>") for s, e in secret_segments]
560
+ out, last = [], 0
561
+ for start, end, label in sorted(marks):
562
+ if start < last: # overlapping marks: the first covers the union, so no part of either is printed
563
+ last = max(last, end)
564
+ continue
565
+ out += [path[last:start], label]
566
+ last = end
567
+ return "".join(out) + path[last:]
568
+
569
+
570
+ def _git() -> Optional[str]:
571
+ """`git` from PATH's absolute folders only. On Windows CreateProcess looks in the current folder first (and
572
+ shutil.which there puts the current folder first too), and on POSIX an empty or relative PATH element (a trailing
573
+ `:`) is the current folder: a tree checked from inside could run its own `git`."""
574
+ name = "git.exe" if os.name == "nt" else "git" # never a .cmd or .bat: cmd.exe would read the arguments
575
+ # with PATH unset, POSIX's default folders (`/bin`, `/usr/bin`); Windows' default (`.;C:\\bin`) is a folder any
576
+ # user may create, so there no PATH means no git and the walk
577
+ folders = os.get_exec_path() if os.name != "nt" else os.environ.get("PATH", "").split(os.pathsep)
578
+ for folder in folders:
579
+ if folder and os.path.isabs(folder):
580
+ candidate = os.path.join(folder, name)
581
+ if os.path.isfile(candidate) and (os.name == "nt" or os.access(candidate, os.X_OK)):
582
+ return candidate
583
+ return None
584
+
585
+
586
+ _GIT_LOCATION = frozenset(("GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", "GIT_COMMON_DIR",
587
+ "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_NAMESPACE", "GIT_PREFIX"))
588
+
589
+
590
+ def _git_files(root: Path) -> Optional[List[str]]:
591
+ git = _git()
592
+ if git is None:
593
+ return None
594
+ try:
595
+ # `core.fsmonitor` off: `--others` would run a command the checked tree's own configuration names. Empty, not
596
+ # `false`: Git before 2.36 reads the value as a hook and would run a program named `false` from PATH
597
+ # the repository is found from ROOT: a Git location the caller exports (every hook in a linked worktree gets
598
+ # GIT_DIR and GIT_INDEX_FILE) would list another repository's files and leave ROOT's own unread
599
+ env = {k: v for k, v in os.environ.items() if k.upper() not in _GIT_LOCATION}
600
+ result = subprocess.run([git, "-c", "core.fsmonitor=", "-C", str(root), "ls-files", "-z", "--cached",
601
+ "--others", "--exclude-standard"], capture_output=True, timeout=60, env=env)
602
+ except (OSError, subprocess.SubprocessError):
603
+ return None
604
+ if result.returncode != 0:
605
+ return None
606
+ # Git for Windows writes a lone surrogate in a name as WTF-8: read back as that character, it is the name os.walk
607
+ # gives; elsewhere a name is bytes, decoded as os.walk decodes it. `_name_bytes` turns either back into Git's bytes
608
+ names = set()
609
+ for raw in result.stdout.split(b"\0"): # each name on its own: one name that is not UTF-8 changes no other
610
+ if not raw:
611
+ continue
612
+ try:
613
+ # POSIX: the file system's own encoding, so the name is the one os.walk and open() use in any locale
614
+ names.add(raw.decode("utf-8", "surrogatepass") if os.name == "nt" else os.fsdecode(raw))
615
+ except UnicodeDecodeError:
616
+ names.add(raw.decode("utf-8", "surrogateescape"))
617
+ return sorted(names)
618
+
619
+
620
+ def _walk_files(root: Path, unreadable: List[str], skip: Iterable[str] = WALK_SKIP,
621
+ left: Optional[List[str]] = None) -> List[str]:
622
+ """Every file under root but the folders in skip; each folder skipped is named in left."""
623
+ found = []
624
+ skip = set(skip)
625
+
626
+ def failed(error: OSError) -> None:
627
+ unreadable.append(Path(str(error.filename or root)).relative_to(root).as_posix())
628
+
629
+ for folder, dirs, files in os.walk(root, onerror=failed):
630
+ if left is not None:
631
+ left += [Path(folder, d).relative_to(root).as_posix() for d in dirs
632
+ if d in skip and d not in PACK_REFUSES] # `.git` and `__pycache__`: --every-file leaves them too
633
+ dirs[:] = [d for d in dirs if d not in skip]
634
+ # os.walk lists a link to a folder with the folders and does not follow it: check it as Git would, by its target
635
+ files = files + [d for d in dirs if os.path.islink(os.path.join(folder, d))]
636
+ for name in files:
637
+ if name in skip and name == ".git": # a worktree's or a submodule's `.git` file: never carried, never source
638
+ continue
639
+ if skip == PACK_REFUSES and _PACK_JUNK_FILE.fullmatch(name): # --every-file: litter pack refuses, never carried
640
+ continue
641
+ found.append(Path(folder, name).relative_to(root).as_posix())
642
+ return sorted(found)
643
+
644
+
645
+ def _lines(data: bytes) -> Iterable[Tuple[int, str]]:
646
+ """Lines as an editor numbers them: split at LF, a trailing CR dropped (str.splitlines also splits at FF, VT...)."""
647
+ for number, line in enumerate(data.decode("utf-8", "replace").split("\n"), 1):
648
+ yield number, line[:-1] if line.endswith("\r") else line
649
+
650
+
651
+ def _text_findings(text: str, entries: List[str], seen: Optional["_Seen"] = None) -> List[Tuple[str, str]]:
652
+ found = []
653
+ homes = _home_names(text, _owner_homes(entries))
654
+ if homes:
655
+ found.append(("home-path", "a path inside a person's home folder"))
656
+ if seen is not None:
657
+ seen.names.update(_spellings({text[a:b] for a, b in homes}, _owner_homes(entries)))
658
+ secret = _secret_finding(text)
659
+ if secret:
660
+ found.append(("secret", secret))
661
+ if seen is not None:
662
+ seen.add_values(text)
663
+ rule = _instance_finding(text, entries)
664
+ if rule:
665
+ found.append(("instance-string", rule))
666
+ return found
667
+
668
+
669
+ class _Seen:
670
+ """What the check found anywhere in the tree, so that no printed path repeats it: a file named after its owner, or
671
+ after the key it holds, prints with the name as `<home>` and the segment as `<secret>`."""
672
+
673
+ def __init__(self) -> None:
674
+ self.names: Set[str] = set()
675
+ self.grams: Set[str] = set() # every 8-character run of a found value, the whole of it
676
+ self._names_pattern: Optional["re.Pattern[str]"] = None
677
+
678
+ def add_values(self, text: str) -> None:
679
+ sources = [(text, _secret_spans(text))]
680
+ if "\\" in text:
681
+ unescaped = _json_unescaped(text)
682
+ if unescaped != text:
683
+ sources.append((unescaped, _secret_spans(unescaped)))
684
+ for source, spans in sources:
685
+ # overlapping spans (each start of `sk-sk-sk-...` runs to the end) merged first: their runs are the same,
686
+ # and reading each span apart cost the square of the line
687
+ merged: List[List[int]] = []
688
+ for a, b in sorted(spans):
689
+ if merged and a <= merged[-1][1]:
690
+ merged[-1][1] = max(merged[-1][1], b)
691
+ else:
692
+ merged.append([a, b])
693
+ for a, b in merged:
694
+ value = source[a:b]
695
+ self.grams.update(value[i:i + 8] for i in range(max(0, len(value) - 7)))
696
+
697
+ def names_pattern(self) -> "re.Pattern[str]":
698
+ """Every name found, longest first, as one pattern: read once the tree is read, when no name is added."""
699
+ if self._names_pattern is None:
700
+ alternatives = sorted(self.names, key=len, reverse=True)
701
+ # a lookahead reads every start, so two names that overlap in a path are both marked
702
+ self._names_pattern = re.compile("(?=(%s))" % "|".join(re.escape(n) for n in alternatives), re.IGNORECASE)
703
+ return self._names_pattern
704
+
705
+
706
+ def check(root: Path, entries: List[str], globs: List[str], every_file: bool = False) -> Dict[str, object]:
707
+ """Read the tree and return {files, skipped, findings}; a finding is {path, line, kind, rule}, its path redacted."""
708
+ unreadable: List[str] = []
709
+ files = None if every_file else _git_files(root)
710
+ mode = "git"
711
+ dependencies: List[str] = [] # dependency folders the walk leaves unread: named, so a clean result says so
712
+ if every_file: # what `pack` reads: every file, ignored or not; `.git` and `__pycache__` it refuses to pack
713
+ files, mode = _walk_files(root, unreadable, PACK_REFUSES), "walk"
714
+ elif not files: # not a checkout, Git refused it, or a folder Git ignores or holds nothing of: read what is there
715
+ files, mode = _walk_files(root, unreadable, WALK_SKIP, dependencies), "walk"
716
+ findings: List[Dict[str, object]] = []
717
+ read = 0
718
+ skipped = {"binary": 0, "unreadable": len(unreadable), "excluded": 0, "nested": 0, "dependencies": len(dependencies)}
719
+ # paths are redacted once the whole tree is read, with what was found anywhere in it (`_Seen`)
720
+ not_read: List[Dict[str, str]] = [{"path": p, "why": "unreadable"} for p in unreadable]
721
+ seen = _Seen()
722
+
723
+ def add(path: str, line: int, kind: str, rule: str) -> None:
724
+ findings.append({"path": path, "line": line, "kind": kind, "rule": rule})
725
+
726
+ for path in files:
727
+ if _ignored(path, globs, root):
728
+ skipped["excluded"] += 1
729
+ continue
730
+ for kind, rule in _text_findings(path, entries, seen): # a name carries instance data as a line does
731
+ add(path, 0, kind, rule + " in the file's path")
732
+ name = path.rsplit("/", 1)[-1]
733
+ if (SECRET_FILE.fullmatch(name) and not SECRET_FILE_OK.fullmatch(name)) or TOKEN_FILE.fullmatch(name):
734
+ add(path, 0, "secret", "a file that is a credential by its name")
735
+ full = root / path
736
+ # os.path.islink, not Path.is_symlink: on Python 3.9 the latter raises on a folder it cannot search
737
+ if any(os.path.islink(str(root / parent)) for parent in Path(path).parents if str(parent) != "."):
738
+ # a tracked folder replaced by a link: the file Git lists is not here, and the link is never followed
739
+ skipped["unreadable"] += 1
740
+ not_read.append({"path": path, "why": "under a symbolic link"})
741
+ continue
742
+ try:
743
+ if full.is_symlink(): # Git commits the path a link names: check it, never follow it
744
+ read += 1
745
+ for kind, rule in _text_findings(os.readlink(full), entries, seen):
746
+ add(path, 0, kind, rule + " in the link's target")
747
+ continue
748
+ kind = os.lstat(str(full)).st_mode
749
+ if stat.S_ISDIR(kind): # Git lists a submodule or a nested repository as one path: its files are not read
750
+ skipped["nested"] += 1
751
+ not_read.append({"path": path, "why": "a nested repository"})
752
+ continue
753
+ if not stat.S_ISREG(kind): # a socket, a pipe: nothing a source carries
754
+ continue
755
+ with open(full, "rb") as handle:
756
+ data = handle.read()
757
+ except FileNotFoundError: # Git lists it and the work tree lacks it: deleted, or outside a sparse checkout
758
+ skipped["unreadable"] += 1
759
+ not_read.append({"path": path,
760
+ "why": "not in the work tree" if mode == "git" else "unreadable"})
761
+ continue
762
+ except OSError:
763
+ skipped["unreadable"] += 1
764
+ not_read.append({"path": path, "why": "unreadable"})
765
+ continue
766
+ if b"\0" in data[:8192]:
767
+ skipped["binary"] += 1
768
+ not_read.append({"path": path, "why": "binary"})
769
+ continue
770
+ read += 1
771
+ for number, line in _lines(data):
772
+ for kind, rule in _text_findings(line, entries, seen):
773
+ add(path, number, kind, rule)
774
+ printed: Dict[str, str] = {} # one redaction per path, however many findings it has
775
+
776
+ def shown(path: str) -> str:
777
+ if path not in printed:
778
+ printed[path] = redact(path, entries, seen)
779
+ return printed[path]
780
+
781
+ for entry in findings:
782
+ entry["path"] = shown(str(entry["path"]))
783
+ for entry in not_read:
784
+ entry["path"] = shown(entry["path"])
785
+ return {"mode": mode, "files": read, "skipped": skipped, "notRead": not_read, "findings": findings,
786
+ "dependencies": [shown(d) for d in dependencies]}
787
+
788
+
789
+ def _absolute(path: str, platform: str) -> bool:
790
+ return bool(re.match(r"[A-Za-z]:[\\/]|[\\/]{2}", path)) if platform == "win32" else path.startswith("/")
791
+
792
+
793
+ def default_list(platform: Optional[str] = None, env: Optional[Dict[str, str]] = None) -> Path:
794
+ """Where the owner's list is kept when none is named: the user's configuration folder, never a project's."""
795
+ platform = platform or sys.platform
796
+ env = os.environ if env is None else env
797
+ # a relative value is ignored, as the XDG spec says (a quoted `~/.config` is relative too): never "no list"
798
+ if platform == "win32":
799
+ appdata = env.get("APPDATA", "")
800
+ base = Path(appdata) if appdata and _absolute(appdata, platform) else Path.home() / "AppData" / "Roaming"
801
+ else:
802
+ xdg = env.get("XDG_CONFIG_HOME", "")
803
+ base = Path(xdg) if xdg.startswith("/") else Path.home() / ".config"
804
+ return base / "passioncode-fabric" / LIST_NAME
805
+
806
+
807
+ def _list_path(given: Optional[str]) -> Tuple[Optional[Path], bool]:
808
+ """The owner's list and whether it must be read (a named or an existing default list; else no list)."""
809
+ named = given or os.environ.get("FABRIC_INSTANCE_STRINGS")
810
+ if named:
811
+ try:
812
+ return Path(named).expanduser(), True
813
+ except (RuntimeError, KeyError): # `~user` with no such user (KeyError on 3.9): never "no finding"
814
+ raise CheckError("the owner's list cannot be read")
815
+ try:
816
+ default = default_list()
817
+ except (RuntimeError, KeyError): # no HOME and no account entry (a container's arbitrary user)
818
+ raise CheckError("there is no home folder to find the owner's list in: set an absolute XDG_CONFIG_HOME (APPDATA on "
819
+ "Windows) or name the list with --instance-strings")
820
+ try:
821
+ os.stat(str(default))
822
+ except (FileNotFoundError, NotADirectoryError):
823
+ # a link to a list, its folder or the configuration folder on a volume not mounted: there, but not reachable
824
+ if any(os.path.lexists(str(p)) and not os.path.exists(str(p)) for p in (default, *default.parents)):
825
+ raise CheckError("the owner's list in your configuration folder cannot be read")
826
+ return None, False
827
+ except OSError: # a list that is there but cannot be reached is not "no list": that would fail open
828
+ raise CheckError("the owner's list cannot be read")
829
+ return default, True
830
+
831
+
832
+ class _Parser(argparse.ArgumentParser):
833
+ def error(self, message: str) -> NoReturn: # argparse would repeat the arguments it could not read
834
+ self.print_usage(sys.stderr)
835
+ self.exit(2, "fabric_source_check.py: the arguments cannot be read; see the usage above\n")
836
+
837
+
838
+ def _print(text: str) -> None:
839
+ """Print a line the console may not encode (a name in another script, a byte Git kept as a surrogate)."""
840
+ encoding = getattr(sys.stdout, "encoding", None) or "utf-8"
841
+ print(text.encode(encoding, "backslashreplace").decode(encoding, "replace"))
842
+
843
+
844
+ def run(argv: Optional[List[str]] = None) -> int:
845
+ parser = _Parser(prog="fabric_source_check.py", description="Find instance data in an agent's source.")
846
+ sub = parser.add_subparsers(dest="command", required=True)
847
+ cmd = sub.add_parser("check", help="read the source tree and report instance data without repeating it")
848
+ cmd.add_argument("root", nargs="?", default=".")
849
+ cmd.add_argument("--instance-strings", metavar="FILE")
850
+ cmd.add_argument("--exclude", action="append", default=[], metavar="GLOB")
851
+ cmd.add_argument("--json", action="store_true")
852
+ cmd.add_argument("--every-file", action="store_true", help="read every file, not only what Git would commit")
853
+ try:
854
+ args = parser.parse_args(argv)
855
+ except SystemExit as exc: # --help, or arguments argparse refused: the exit code, as the command line gives it
856
+ return exc.code if isinstance(exc.code, int) else 2
857
+ try:
858
+ try:
859
+ root = Path(args.root).expanduser().resolve()
860
+ is_folder = root.is_dir()
861
+ except (OSError, RuntimeError, KeyError): # a parent it cannot search, a symlink loop, `~` with no home
862
+ raise CheckError("the folder cannot be read")
863
+ if not is_folder:
864
+ raise CheckError("ROOT is not a folder")
865
+ list_path, _ = _list_path(args.instance_strings) # may raise CheckError
866
+ entries: List[str] = []
867
+ if list_path is not None:
868
+ try:
869
+ resolved = list_path.resolve()
870
+ except (OSError, RuntimeError): # RuntimeError: Python 3.9's symlink loop
871
+ raise CheckError("the owner's list cannot be read")
872
+ if resolved == root or root in resolved.parents:
873
+ raise CheckError("the owner's list is inside the tree it checks, where it would travel with the source; "
874
+ "keep it in your configuration folder (passioncode-fabric/%s)" % LIST_NAME)
875
+ try:
876
+ present = resolved.is_file()
877
+ except OSError:
878
+ raise CheckError("the owner's list cannot be read")
879
+ if not present:
880
+ raise CheckError("the owner's list named by --instance-strings or FABRIC_INSTANCE_STRINGS does not exist or is "
881
+ "not a regular file"
882
+ if args.instance_strings or os.environ.get("FABRIC_INSTANCE_STRINGS")
883
+ else "the owner's list in your configuration folder is not a file")
884
+ try:
885
+ entries = read_list(resolved)
886
+ except UnicodeDecodeError:
887
+ raise CheckError("the owner's list is not UTF-8")
888
+ except OSError:
889
+ raise CheckError("the owner's list cannot be read")
890
+ result = check(root, entries, ignore_globs(root, args.exclude), args.every_file)
891
+ except CheckError as exc:
892
+ print("fabric_source_check.py: %s" % exc, file=sys.stderr)
893
+ return 2
894
+ findings = result["findings"]
895
+ skipped = result["skipped"]
896
+ listed = sum(1 for e in entries if e and not e.startswith(HOME_ENTRY))
897
+ homes = sum(1 for e in entries if e.startswith(HOME_ENTRY))
898
+ # a binary file is not read either: it is complete only when every one is named as source in the ignore file
899
+ complete = not (skipped["unreadable"] or skipped["binary"] or skipped["nested"]) # type: ignore[index]
900
+ result.update({"instanceStrings": listed, "ownerHomes": homes, "instanceList": list_path is not None, "blocking": bool(findings),
901
+ "complete": complete})
902
+ if args.json:
903
+ _print(json.dumps(result, indent=2))
904
+ else:
905
+ for f in findings: # type: ignore[union-attr]
906
+ _print("%s:%s: %s (%s)" % (f["path"], f["line"], f["kind"], f["rule"]))
907
+ for n in result["notRead"]: # type: ignore[union-attr]
908
+ _print("%s: not read (%s)" % (n["path"], n["why"]))
909
+ for folder in result["dependencies"]: # type: ignore[union-attr]
910
+ _print("%s: not read (a dependency folder; --every-file reads it)" % folder)
911
+ if skipped["dependencies"]: # type: ignore[index]
912
+ _print("%d dependency folder(s) not read (node_modules, virtual environments, caches); --every-file reads "
913
+ "them" % skipped["dependencies"]) # type: ignore[index]
914
+ _print("%d file(s) read (%s), %d finding(s); not read: %d binary, %d unreadable, %d nested repository; "
915
+ "%d left out by name; %d instance string(s) listed%s%s"
916
+ % (result["files"], "what Git would commit" if result["mode"] == "git" else
917
+ "every file, as pack reads them" if args.every_file else "every file but dependency folders, Git not used",
918
+ len(findings), skipped["binary"], skipped["unreadable"], skipped["nested"], # type: ignore[index]
919
+ skipped["excluded"], listed, # type: ignore[index]
920
+ "" if listed or homes else (" (the owner's list holds no string of three or more characters)"
921
+ if list_path else " (no owner's list)"),
922
+ "; %d home name(s) declared" % homes if homes else ""))
923
+ if findings:
924
+ return 1
925
+ return 0 if complete else 2
926
+
927
+
928
+ def _utf8_restart(encoding: str, utf8_mode: int, executable: str, argv: List[str]) -> Optional[List[bytes]]:
929
+ """The command that restarts the check in Python's UTF-8 mode, or None when it needs none. Under a locale whose
930
+ file-system encoding is not UTF-8 (ISO-8859-1, KOI8-R, a C locale with UTF-8 mode off), a name, an argument and a
931
+ file's text would each be read in its own encoding; in UTF-8 mode all three are, and a byte that is not UTF-8 stays
932
+ that byte. Windows names are UTF-16 and need none. The arguments go back as the bytes they came in. An empty list:
933
+ a restart is needed and no interpreter is known (the caller exits 2 rather than read in two encodings)."""
934
+ if os.name == "nt" or utf8_mode or encoding.lower().replace("_", "-") in ("utf-8", "utf8"):
935
+ return None
936
+ if not executable:
937
+ return []
938
+ return [os.fsencode(executable), b"-X", b"utf8"] + [os.fsencode(a) for a in argv]
939
+
940
+
941
+ def _interpreter() -> str:
942
+ """The running interpreter: Linux's `/proc/self/exe`, which no PATH or working folder can stand in for (Python 3.9
943
+ may take a `./python3` of the working folder as sys.executable), else sys.executable, which may be empty."""
944
+ try:
945
+ return os.readlink("/proc/self/exe")
946
+ except (OSError, AttributeError):
947
+ return sys.executable or ""
948
+
949
+
950
+ def _interpreter_flags(flags: object, xoptions: Dict[str, object]) -> List[str]:
951
+ """The interpreter's own options, rebuilt where sys.orig_argv is missing (Python 3.9), so a restart keeps `-I`."""
952
+ out: List[str] = []
953
+ if getattr(flags, "isolated", 0):
954
+ out.append("-I")
955
+ else:
956
+ out += ["-E"] * bool(getattr(flags, "ignore_environment", 0)) + ["-s"] * bool(getattr(flags, "no_user_site", 0))
957
+ out += ["-S"] * bool(getattr(flags, "no_site", 0)) + ["-B"] * bool(getattr(flags, "dont_write_bytecode", 0))
958
+ if getattr(flags, "optimize", 0):
959
+ out.append("-" + "O" * int(getattr(flags, "optimize")))
960
+ for key, value in xoptions.items():
961
+ if key not in ("utf8", "fabric_console"):
962
+ out += ["-X", key if value is True else "%s=%s" % (key, value)]
963
+ return out
964
+
965
+
966
+ def _keep_console(xoptions: Dict[str, object]) -> None:
967
+ """After a restart, write the console's own encoding again (the parent named it in `-X fabric_console=`)."""
968
+ console = xoptions.get("fabric_console")
969
+ if isinstance(console, str):
970
+ for stream in (sys.stdout, sys.stderr):
971
+ try:
972
+ stream.reconfigure(encoding=console, errors="backslashreplace") # type: ignore[union-attr]
973
+ except (AttributeError, LookupError, ValueError):
974
+ pass
975
+
976
+
977
+ if __name__ == "__main__":
978
+ _keep_console(getattr(sys, "_xoptions", {}))
979
+ _argv = (list(sys.orig_argv[1:]) if hasattr(sys, "orig_argv")
980
+ else _interpreter_flags(sys.flags, getattr(sys, "_xoptions", {})) + sys.argv)
981
+ _restart = _utf8_restart(sys.getfilesystemencoding(), sys.flags.utf8_mode, _interpreter(), _argv)
982
+ if _restart is not None:
983
+ try:
984
+ if not _restart:
985
+ raise OSError("no interpreter")
986
+ # the output keeps the console's encoding, as before the restart: an -X option survives -I and -E
987
+ _console = getattr(sys.stdout, "encoding", None) or "ascii"
988
+ os.execv(_restart[0], _restart[:3] + [b"-X", os.fsencode("fabric_console=" + _console)] + _restart[3:])
989
+ except OSError:
990
+ sys.stderr.write("fabric_source_check.py: could not restart in Python's UTF-8 mode under this locale; run it "
991
+ "as `python3 -X utf8 fabric_source_check.py ...`\n")
992
+ sys.exit(2)
993
+ sys.exit(run())
994
+
995
+ # #endregion source-check