claude-dev-env 8.7.0 → 8.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/privacy-hygiene/SKILL.md +1 -0
- package/.agents/skills/pull-request/scripts/pull_request.py +1 -0
- package/.agents/skills/pull-request/scripts/test_pull_request.py +2 -0
- package/.agents/skills/team-advisor/SKILL.md +2 -2
- package/_shared/advisor/reference/identity.md +1 -1
- package/_shared/advisor/reference/spawn-walk-log.md +1 -1
- package/_shared/advisor/reference/warm-up.md +3 -1
- package/_shared/advisor/scripts/config/advisor_scripts_constants/model_tier_run_validator_constants.py +15 -0
- package/_shared/advisor/scripts/model_tier_run_validator.py +20 -3
- package/_shared/advisor/scripts/tests/test_model_tier_run_validator.py +50 -6
- package/docs/references/advisor-tool.md +3 -2
- package/docs/second-claude-account.md +57 -0
- package/hooks/blocking/test_pii_scanner.py +8 -0
- package/hooks/hooks.json +25 -0
- package/hooks/hooks_constants/pii_prevention_constants.py +1 -0
- package/hooks/hooks_constants/skill_loaded_reminder_constants.py +58 -0
- package/hooks/session/skill_loaded_reminder.py +199 -0
- package/hooks/session/test_skill_loaded_reminder.py +206 -0
- package/package.json +1 -1
- package/rules/durable-post-artifacts.md +26 -1
- package/rules/long-horizon-autonomy.md +11 -0
- package/rules/prefer-existing-tools.md +35 -0
- package/rules/review-closure-is-a-check.md +1 -1
- package/scripts/claude_account_choice.py +291 -0
- package/scripts/claude_account_profile.py +371 -0
- package/scripts/claude_chain_usage.py +49 -5
- package/scripts/dev_env_scripts_constants/claude_account_constants.py +181 -0
- package/scripts/dev_env_scripts_constants/gh_artifact_upload_constants.py +27 -0
- package/scripts/dev_env_scripts_constants/private_term_constants.py +60 -0
- package/scripts/dev_env_scripts_constants/test_claude_account_constants.py +46 -0
- package/scripts/dev_env_scripts_constants/test_private_term_constants.py +29 -0
- package/scripts/durable_post_lint.py +47 -1
- package/scripts/durable_post_lint_config/config/constants.py +1 -0
- package/scripts/gh_artifact_upload.py +56 -3
- package/scripts/private_term_scan.py +135 -0
- package/scripts/private_terms.py +87 -0
- package/scripts/repository_checks/config/constants.py +3 -0
- package/scripts/repository_checks/runner.py +7 -0
- package/scripts/repository_checks/tracked_private_terms.py +57 -0
- package/scripts/test_claude_account_choice.py +245 -0
- package/scripts/test_claude_account_profile.py +220 -0
- package/scripts/test_claude_chain_usage.py +40 -0
- package/scripts/test_durable_post_lint.py +90 -0
- package/scripts/tests/test_gh_artifact_upload.py +75 -0
- package/scripts/tests/test_match_exemptions.py +2 -2
- package/scripts/tests/test_private_term_scan.py +176 -0
- package/scripts/tests/test_private_terms.py +59 -0
- package/scripts/tests/test_repository_policy.py +2 -0
- package/scripts/tests/test_runner.py +20 -0
- package/scripts/tests/test_tracked_private_terms.py +71 -0
- package/scripts/tests/test_tracked_secrets.py +1 -1
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Digests of the private organization names this public repository never names.
|
|
2
|
+
|
|
3
|
+
The names stay out of the tree. Each entry holds the SHA-256 digest of one
|
|
4
|
+
name after normalization, which keeps only ASCII letters and digits in lower
|
|
5
|
+
case::
|
|
6
|
+
|
|
7
|
+
"Acme-Widgets, Inc." -> "acmewidgetsinc"
|
|
8
|
+
"pat@acmewidgets.example.com" -> "patacmewidgetsexamplecom"
|
|
9
|
+
|
|
10
|
+
A scan hashes every window of the normalized text whose length matches an
|
|
11
|
+
entry, so any case, spacing, or punctuation of a name matches. To add a name,
|
|
12
|
+
append the digest of its normalized form and its length.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class PrivateTermDigest:
|
|
22
|
+
"""The length and SHA-256 hex digest of one normalized private name."""
|
|
23
|
+
|
|
24
|
+
length: int
|
|
25
|
+
sha256: str
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
ALL_PRIVATE_TERM_DIGESTS: frozenset[PrivateTermDigest] = frozenset(
|
|
29
|
+
{
|
|
30
|
+
PrivateTermDigest(
|
|
31
|
+
length=10,
|
|
32
|
+
sha256="c2a70b3186691a74dece4ca824bec178bfaec83be119be28faedd14aa392233f",
|
|
33
|
+
),
|
|
34
|
+
}
|
|
35
|
+
)
|
|
36
|
+
PRIVATE_TERM_TEXT_ENCODING = "utf-8"
|
|
37
|
+
PRIVATE_TERM_FINDING_CODE = "private-term"
|
|
38
|
+
PRIVATE_TERM_MESSAGE_TEMPLATE = (
|
|
39
|
+
"Line {line_number} names a private organization. "
|
|
40
|
+
"Describe it in general terms and drop any link to it."
|
|
41
|
+
)
|
|
42
|
+
ALL_EVENT_TEXT_FIELDS: tuple[tuple[str, str, str], ...] = (
|
|
43
|
+
("pull_request", "title", "pull request title"),
|
|
44
|
+
("pull_request", "body", "pull request body"),
|
|
45
|
+
("issue", "title", "issue title"),
|
|
46
|
+
("issue", "body", "issue body"),
|
|
47
|
+
("comment", "body", "comment body"),
|
|
48
|
+
("review", "body", "review body"),
|
|
49
|
+
("release", "name", "release name"),
|
|
50
|
+
("release", "body", "release body"),
|
|
51
|
+
)
|
|
52
|
+
FINDING_LINE_TEMPLATE = "{label}: {message}"
|
|
53
|
+
COMMIT_LABEL_TEMPLATE = "commit {short_sha} {part}"
|
|
54
|
+
COMMIT_IDENTITY_PART = "identity"
|
|
55
|
+
COMMIT_MESSAGE_PART = "message"
|
|
56
|
+
SHORT_SHA_LENGTH = 12
|
|
57
|
+
COMMIT_FIELD_SEPARATOR = "\x1f"
|
|
58
|
+
COMMIT_RECORD_SEPARATOR = "\x1e"
|
|
59
|
+
COMMIT_FIELD_SPLIT_LIMIT = 2
|
|
60
|
+
COMMIT_LOG_FORMAT = "--format=%H%x1f%an <%ae> %cn <%ce>%x1f%B%x1e"
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""The second-account thresholds keep the main account the protected one."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dev_env_scripts_constants.claude_account_constants import (
|
|
6
|
+
JSON_ACCOUNT_KEY,
|
|
7
|
+
JSON_CONFIG_DIRECTORY_KEY,
|
|
8
|
+
JSON_METERS_KEY,
|
|
9
|
+
JSON_REASON_KEY,
|
|
10
|
+
JSON_SESSION_RESETS_AT_KEY,
|
|
11
|
+
JSON_SESSION_USED_PERCENT_KEY,
|
|
12
|
+
JSON_WEEKLY_RESETS_AT_KEY,
|
|
13
|
+
JSON_WEEKLY_USED_PERCENT_KEY,
|
|
14
|
+
LAUNCHER_TEXT_TEMPLATE,
|
|
15
|
+
MAIN_SESSION_USED_CEILING_PERCENT,
|
|
16
|
+
MAIN_WEEKLY_USED_CEILING_PERCENT,
|
|
17
|
+
SECOND_SESSION_USED_CEILING_PERCENT,
|
|
18
|
+
SECOND_WEEKLY_USED_CEILING_PERCENT,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_main_ceilings_sit_below_the_second_account_ceilings() -> None:
|
|
23
|
+
assert MAIN_WEEKLY_USED_CEILING_PERCENT < SECOND_WEEKLY_USED_CEILING_PERCENT
|
|
24
|
+
assert MAIN_SESSION_USED_CEILING_PERCENT < SECOND_SESSION_USED_CEILING_PERCENT
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_launcher_names_the_profile_and_passes_every_argument() -> None:
|
|
28
|
+
assert "{profile_home}" in LAUNCHER_TEXT_TEMPLATE
|
|
29
|
+
assert "claude %*" in LAUNCHER_TEXT_TEMPLATE
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_picker_report_keys_never_collide() -> None:
|
|
33
|
+
all_report_keys = [
|
|
34
|
+
JSON_ACCOUNT_KEY,
|
|
35
|
+
JSON_CONFIG_DIRECTORY_KEY,
|
|
36
|
+
JSON_REASON_KEY,
|
|
37
|
+
JSON_METERS_KEY,
|
|
38
|
+
]
|
|
39
|
+
all_meter_keys = [
|
|
40
|
+
JSON_SESSION_USED_PERCENT_KEY,
|
|
41
|
+
JSON_SESSION_RESETS_AT_KEY,
|
|
42
|
+
JSON_WEEKLY_USED_PERCENT_KEY,
|
|
43
|
+
JSON_WEEKLY_RESETS_AT_KEY,
|
|
44
|
+
]
|
|
45
|
+
assert len(set(all_report_keys)) == len(all_report_keys)
|
|
46
|
+
assert len(set(all_meter_keys)) == len(all_meter_keys)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Tests for the private-term digest table."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
from dev_env_scripts_constants.private_term_constants import (
|
|
8
|
+
ALL_EVENT_TEXT_FIELDS,
|
|
9
|
+
ALL_PRIVATE_TERM_DIGESTS,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
_SHA256_HEX_PATTERN = re.compile(r"[0-9a-f]{64}")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_should_ship_only_digests_of_named_lengths() -> None:
|
|
16
|
+
assert ALL_PRIVATE_TERM_DIGESTS
|
|
17
|
+
for each_digest in ALL_PRIVATE_TERM_DIGESTS:
|
|
18
|
+
assert _SHA256_HEX_PATTERN.fullmatch(each_digest.sha256)
|
|
19
|
+
assert each_digest.length > 0
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_should_read_every_text_a_pull_request_issue_comment_review_or_release_carries() -> None:
|
|
23
|
+
assert {each_object for each_object, _field, _label in ALL_EVENT_TEXT_FIELDS} == {
|
|
24
|
+
"pull_request",
|
|
25
|
+
"issue",
|
|
26
|
+
"comment",
|
|
27
|
+
"review",
|
|
28
|
+
"release",
|
|
29
|
+
}
|
|
@@ -12,6 +12,11 @@ from contrast_framing import describe_contrast_framing, find_contrast_framing
|
|
|
12
12
|
from dev_env_scripts_constants.contrast_framing_constants import (
|
|
13
13
|
CONTRAST_FRAMING_FINDING_CODE,
|
|
14
14
|
)
|
|
15
|
+
from dev_env_scripts_constants.private_term_constants import (
|
|
16
|
+
ALL_PRIVATE_TERM_DIGESTS,
|
|
17
|
+
PRIVATE_TERM_FINDING_CODE,
|
|
18
|
+
PRIVATE_TERM_MESSAGE_TEMPLATE,
|
|
19
|
+
)
|
|
15
20
|
from durable_post_lint_config.config.constants import (
|
|
16
21
|
ACTION_PR_CREATE,
|
|
17
22
|
ACTION_PR_EDIT,
|
|
@@ -41,6 +46,7 @@ from durable_post_lint_config.config.constants import (
|
|
|
41
46
|
PATH_ANCHOR_CHARACTER,
|
|
42
47
|
PATH_SEGMENT_START_CHARACTERS,
|
|
43
48
|
RELEASE_BRANCH_PREFIX,
|
|
49
|
+
REPOSITORY_OWNER_SEPARATOR,
|
|
44
50
|
REWRITTEN_RELEASE_BODY_CODE,
|
|
45
51
|
REWRITTEN_RELEASE_BODY_MESSAGE,
|
|
46
52
|
TITLE_NOT_ALLOWED_MESSAGE,
|
|
@@ -48,6 +54,7 @@ from durable_post_lint_config.config.constants import (
|
|
|
48
54
|
VOLATILE_PATH_CODE,
|
|
49
55
|
VOLATILE_PATH_MESSAGE,
|
|
50
56
|
)
|
|
57
|
+
from private_terms import private_term_line_numbers
|
|
51
58
|
|
|
52
59
|
|
|
53
60
|
class DurablePostUsageError(ValueError):
|
|
@@ -153,6 +160,26 @@ def _contrast_framing_findings(text: str) -> list[DurablePostFinding]:
|
|
|
153
160
|
]
|
|
154
161
|
|
|
155
162
|
|
|
163
|
+
def _private_term_findings(
|
|
164
|
+
text: str | None, repository: str | None
|
|
165
|
+
) -> list[DurablePostFinding]:
|
|
166
|
+
if text is None:
|
|
167
|
+
return []
|
|
168
|
+
if repository is not None and private_term_line_numbers(
|
|
169
|
+
repository.split(REPOSITORY_OWNER_SEPARATOR)[0], ALL_PRIVATE_TERM_DIGESTS
|
|
170
|
+
):
|
|
171
|
+
return []
|
|
172
|
+
return [
|
|
173
|
+
DurablePostFinding(
|
|
174
|
+
code=PRIVATE_TERM_FINDING_CODE,
|
|
175
|
+
message=PRIVATE_TERM_MESSAGE_TEMPLATE.format(line_number=each_line_number),
|
|
176
|
+
)
|
|
177
|
+
for each_line_number in private_term_line_numbers(
|
|
178
|
+
text, ALL_PRIVATE_TERM_DIGESTS
|
|
179
|
+
)
|
|
180
|
+
]
|
|
181
|
+
|
|
182
|
+
|
|
156
183
|
def _title_findings(title: str | None) -> list[DurablePostFinding]:
|
|
157
184
|
if title is None:
|
|
158
185
|
return []
|
|
@@ -215,12 +242,29 @@ def lint_durable_post(
|
|
|
215
242
|
title: str | None,
|
|
216
243
|
body_text: str | None,
|
|
217
244
|
head_branch: str | None = None,
|
|
245
|
+
repository: str | None = None,
|
|
218
246
|
) -> tuple[DurablePostFinding, ...]:
|
|
219
|
-
"""Return content findings for one locally valid GitHub post request.
|
|
247
|
+
"""Return content findings for one locally valid GitHub post request.
|
|
248
|
+
|
|
249
|
+
A post to a repository whose owner is a private organization may name that
|
|
250
|
+
organization. Every other post, and a post with no repository given, may not.
|
|
251
|
+
|
|
252
|
+
Args:
|
|
253
|
+
action: The post action, such as ``pr-create`` or ``issue-comment``.
|
|
254
|
+
title: The post title, when the action carries one.
|
|
255
|
+
body_text: The post body, when the action carries one.
|
|
256
|
+
head_branch: The pull request head branch, when known.
|
|
257
|
+
repository: The ``owner/name`` the post targets, when known.
|
|
258
|
+
|
|
259
|
+
Returns:
|
|
260
|
+
Every finding, title findings first.
|
|
261
|
+
"""
|
|
220
262
|
_validate_request_shape(action, title, body_text)
|
|
221
263
|
return (
|
|
222
264
|
*_title_findings(title),
|
|
265
|
+
*_private_term_findings(title, repository),
|
|
223
266
|
*_body_findings(action, body_text, head_branch),
|
|
267
|
+
*_private_term_findings(body_text, repository),
|
|
224
268
|
)
|
|
225
269
|
|
|
226
270
|
|
|
@@ -230,6 +274,7 @@ def _parse_arguments(all_arguments: Sequence[str]) -> argparse.Namespace:
|
|
|
230
274
|
parser.add_argument("--title")
|
|
231
275
|
parser.add_argument("--body-file", type=Path)
|
|
232
276
|
parser.add_argument("--head-branch")
|
|
277
|
+
parser.add_argument("--repository")
|
|
233
278
|
return parser.parse_args(list(all_arguments))
|
|
234
279
|
|
|
235
280
|
|
|
@@ -247,6 +292,7 @@ def main(all_arguments: Sequence[str]) -> int:
|
|
|
247
292
|
title=arguments.title,
|
|
248
293
|
body_text=body_text,
|
|
249
294
|
head_branch=arguments.head_branch,
|
|
295
|
+
repository=arguments.repository,
|
|
250
296
|
)
|
|
251
297
|
except (DurablePostInputError, DurablePostUsageError) as error:
|
|
252
298
|
sys.stderr.write(f"{error}\n")
|
|
@@ -5,8 +5,8 @@ A durable GitHub post (issue, PR, comment, review) must not link a file under a
|
|
|
5
5
|
job scratch directory or worktree, because that scratch is cleaned soon after
|
|
6
6
|
the run while the post lives forever. Binary artifacts belong in a permanent
|
|
7
7
|
place instead. This tool ensures the repo has a prerelease tagged ``artifacts``,
|
|
8
|
-
uploads the
|
|
9
|
-
download URL a post can safely link. The timestamped name keeps each upload a
|
|
8
|
+
shrinks a PNG losslessly with oxipng, uploads the file under a timestamped asset
|
|
9
|
+
name, and prints the permanent download URL a post can safely link. The timestamped name keeps each upload a
|
|
10
10
|
distinct asset. An upload never overwrites an earlier one. A same-name collision
|
|
11
11
|
fails loudly instead of replacing the bytes an existing URL already serves.
|
|
12
12
|
|
|
@@ -28,6 +28,7 @@ import tempfile
|
|
|
28
28
|
from pathlib import Path
|
|
29
29
|
|
|
30
30
|
from dev_env_scripts_constants.gh_artifact_upload_constants import (
|
|
31
|
+
ALL_OXIPNG_LOSSLESS_ARGUMENTS,
|
|
31
32
|
ARTIFACTS_RELEASE_NOTES,
|
|
32
33
|
ARTIFACTS_RELEASE_TAG,
|
|
33
34
|
ARTIFACTS_RELEASE_TITLE,
|
|
@@ -37,7 +38,11 @@ from dev_env_scripts_constants.gh_artifact_upload_constants import (
|
|
|
37
38
|
ASSET_URL_JSON_KEY,
|
|
38
39
|
GH_BINARY_NAME,
|
|
39
40
|
NOTES_FILE_SUFFIX,
|
|
41
|
+
OXIPNG_BINARY_NAME,
|
|
42
|
+
OXIPNG_MISSING_MESSAGE,
|
|
43
|
+
PNG_SUFFIX,
|
|
40
44
|
RELEASE_ASSETS_JSON_KEY,
|
|
45
|
+
SIZE_REPORT_TEMPLATE,
|
|
41
46
|
UTF8_ENCODING,
|
|
42
47
|
)
|
|
43
48
|
|
|
@@ -132,6 +137,49 @@ def timestamped_asset_name(file_path: str) -> str:
|
|
|
132
137
|
)
|
|
133
138
|
|
|
134
139
|
|
|
140
|
+
def _run_oxipng(oxipng_path: str, staged_path: Path) -> None:
|
|
141
|
+
completion = subprocess.run(
|
|
142
|
+
[oxipng_path, *ALL_OXIPNG_LOSSLESS_ARGUMENTS, str(staged_path)],
|
|
143
|
+
capture_output=True,
|
|
144
|
+
text=True,
|
|
145
|
+
encoding=UTF8_ENCODING,
|
|
146
|
+
check=False,
|
|
147
|
+
creationflags=hidden_window_creation_flags(),
|
|
148
|
+
)
|
|
149
|
+
if completion.returncode != 0:
|
|
150
|
+
raise ArtifactUploadError(completion.stderr.strip())
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def optimize_png_losslessly(staged_path: Path) -> None:
|
|
154
|
+
"""Shrink a staged PNG in place with oxipng and report both sizes.
|
|
155
|
+
|
|
156
|
+
::
|
|
157
|
+
|
|
158
|
+
contact_sheet.png: 361105 -> 308782 bytes
|
|
159
|
+
|
|
160
|
+
Oxipng keeps every pixel and every chunk. Other suffixes stay as they are.
|
|
161
|
+
|
|
162
|
+
Args:
|
|
163
|
+
staged_path: The staged copy that the upload sends.
|
|
164
|
+
|
|
165
|
+
Raises:
|
|
166
|
+
ArtifactUploadError: When oxipng is missing or fails.
|
|
167
|
+
"""
|
|
168
|
+
if staged_path.suffix.lower() != PNG_SUFFIX:
|
|
169
|
+
return
|
|
170
|
+
oxipng_path = shutil.which(OXIPNG_BINARY_NAME)
|
|
171
|
+
if oxipng_path is None:
|
|
172
|
+
raise ArtifactUploadError(OXIPNG_MISSING_MESSAGE)
|
|
173
|
+
original_size = staged_path.stat().st_size
|
|
174
|
+
_run_oxipng(oxipng_path, staged_path)
|
|
175
|
+
size_report = SIZE_REPORT_TEMPLATE.format(
|
|
176
|
+
name=staged_path.name,
|
|
177
|
+
original_size=original_size,
|
|
178
|
+
optimized_size=staged_path.stat().st_size,
|
|
179
|
+
)
|
|
180
|
+
print(size_report, file=sys.stderr)
|
|
181
|
+
|
|
182
|
+
|
|
135
183
|
def _uploaded_asset_download_url(repository: str) -> str:
|
|
136
184
|
"""Read the just-uploaded asset's real download URL back from GitHub.
|
|
137
185
|
|
|
@@ -197,6 +245,9 @@ def _uploaded_asset_download_url(repository: str) -> str:
|
|
|
197
245
|
def upload_artifact(file_path: str, repository: str) -> str:
|
|
198
246
|
"""Upload a file to the artifacts release and return its permanent URL.
|
|
199
247
|
|
|
248
|
+
A PNG passes through oxipng on a staged copy first, so the source file
|
|
249
|
+
stays unchanged.
|
|
250
|
+
|
|
200
251
|
Args:
|
|
201
252
|
file_path: The source file to upload.
|
|
202
253
|
repository: The ``owner/repo`` slug.
|
|
@@ -205,7 +256,8 @@ def upload_artifact(file_path: str, repository: str) -> str:
|
|
|
205
256
|
The permanent download URL for the uploaded asset.
|
|
206
257
|
|
|
207
258
|
Raises:
|
|
208
|
-
ArtifactUploadError: When the file is missing
|
|
259
|
+
ArtifactUploadError: When the file is missing, oxipng cannot shrink
|
|
260
|
+
a PNG, or the upload fails.
|
|
209
261
|
"""
|
|
210
262
|
source_path = Path(file_path)
|
|
211
263
|
if not source_path.is_file():
|
|
@@ -215,6 +267,7 @@ def upload_artifact(file_path: str, repository: str) -> str:
|
|
|
215
267
|
with tempfile.TemporaryDirectory() as staging_directory:
|
|
216
268
|
staged_asset_path = Path(staging_directory) / asset_name
|
|
217
269
|
shutil.copyfile(source_path, staged_asset_path)
|
|
270
|
+
optimize_png_losslessly(staged_asset_path)
|
|
218
271
|
completion = _run_gh(
|
|
219
272
|
[
|
|
220
273
|
"release",
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""Fail a GitHub event whose text or commits name a private organization.
|
|
2
|
+
|
|
3
|
+
The workflow ``private-terms.yml`` runs this on pull request, issue, comment,
|
|
4
|
+
review, and release events. Each finding names where the text sits and its
|
|
5
|
+
line number, never the name itself, so the public log stays clean::
|
|
6
|
+
|
|
7
|
+
pull request body: Line 4 names a private organization. ...
|
|
8
|
+
commit 3f2a9c1b77de identity: Line 1 names a private organization. ...
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import json
|
|
15
|
+
import subprocess
|
|
16
|
+
import sys
|
|
17
|
+
from collections.abc import Sequence
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from dev_env_scripts_constants.private_term_constants import (
|
|
21
|
+
ALL_EVENT_TEXT_FIELDS,
|
|
22
|
+
ALL_PRIVATE_TERM_DIGESTS,
|
|
23
|
+
COMMIT_FIELD_SEPARATOR,
|
|
24
|
+
COMMIT_FIELD_SPLIT_LIMIT,
|
|
25
|
+
COMMIT_IDENTITY_PART,
|
|
26
|
+
COMMIT_LABEL_TEMPLATE,
|
|
27
|
+
COMMIT_LOG_FORMAT,
|
|
28
|
+
COMMIT_MESSAGE_PART,
|
|
29
|
+
COMMIT_RECORD_SEPARATOR,
|
|
30
|
+
FINDING_LINE_TEMPLATE,
|
|
31
|
+
PRIVATE_TERM_MESSAGE_TEMPLATE,
|
|
32
|
+
PRIVATE_TERM_TEXT_ENCODING,
|
|
33
|
+
SHORT_SHA_LENGTH,
|
|
34
|
+
)
|
|
35
|
+
from private_terms import private_term_line_numbers
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _findings_for(label: str, text: str) -> list[str]:
|
|
39
|
+
return [
|
|
40
|
+
FINDING_LINE_TEMPLATE.format(
|
|
41
|
+
label=label,
|
|
42
|
+
message=PRIVATE_TERM_MESSAGE_TEMPLATE.format(line_number=each_line_number),
|
|
43
|
+
)
|
|
44
|
+
for each_line_number in private_term_line_numbers(
|
|
45
|
+
text, ALL_PRIVATE_TERM_DIGESTS
|
|
46
|
+
)
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def scan_event(event_path: Path) -> list[str]:
|
|
51
|
+
"""Return findings for the texts a GitHub event payload carries.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
event_path: Path of the event JSON the runner writes.
|
|
55
|
+
|
|
56
|
+
Returns:
|
|
57
|
+
One finding per line that names a private organization.
|
|
58
|
+
"""
|
|
59
|
+
event = json.loads(event_path.read_text(encoding=PRIVATE_TERM_TEXT_ENCODING))
|
|
60
|
+
all_findings: list[str] = []
|
|
61
|
+
for each_object_key, each_field, each_label in ALL_EVENT_TEXT_FIELDS:
|
|
62
|
+
event_object = event.get(each_object_key)
|
|
63
|
+
if not isinstance(event_object, dict):
|
|
64
|
+
continue
|
|
65
|
+
text = event_object.get(each_field)
|
|
66
|
+
if isinstance(text, str):
|
|
67
|
+
all_findings.extend(_findings_for(each_label, text))
|
|
68
|
+
return all_findings
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _commit_findings(record: str) -> list[str]:
|
|
72
|
+
full_sha, identity, message = record.split(
|
|
73
|
+
COMMIT_FIELD_SEPARATOR, COMMIT_FIELD_SPLIT_LIMIT
|
|
74
|
+
)
|
|
75
|
+
short_sha = full_sha[:SHORT_SHA_LENGTH]
|
|
76
|
+
return [
|
|
77
|
+
*_findings_for(
|
|
78
|
+
COMMIT_LABEL_TEMPLATE.format(short_sha=short_sha, part=COMMIT_IDENTITY_PART),
|
|
79
|
+
identity,
|
|
80
|
+
),
|
|
81
|
+
*_findings_for(
|
|
82
|
+
COMMIT_LABEL_TEMPLATE.format(short_sha=short_sha, part=COMMIT_MESSAGE_PART),
|
|
83
|
+
message,
|
|
84
|
+
),
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def scan_commits(repository_root: Path, commit_range: str) -> list[str]:
|
|
89
|
+
"""Return findings for commit messages and identities in a range, oldest first.
|
|
90
|
+
|
|
91
|
+
Args:
|
|
92
|
+
repository_root: Git checkout holding the range.
|
|
93
|
+
commit_range: A ``base..head`` revision range.
|
|
94
|
+
|
|
95
|
+
Returns:
|
|
96
|
+
One finding per commit identity or message line that names a private
|
|
97
|
+
organization.
|
|
98
|
+
"""
|
|
99
|
+
log_text = subprocess.run(
|
|
100
|
+
["git", "-C", str(repository_root), "log", "--reverse", COMMIT_LOG_FORMAT, commit_range],
|
|
101
|
+
check=True,
|
|
102
|
+
capture_output=True,
|
|
103
|
+
encoding=PRIVATE_TERM_TEXT_ENCODING,
|
|
104
|
+
).stdout
|
|
105
|
+
return [
|
|
106
|
+
each_finding
|
|
107
|
+
for each_record in log_text.split(COMMIT_RECORD_SEPARATOR)
|
|
108
|
+
if each_record.strip()
|
|
109
|
+
for each_finding in _commit_findings(each_record.strip())
|
|
110
|
+
]
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def main(all_arguments: Sequence[str]) -> int:
|
|
114
|
+
"""Print every finding and return 1 when any text names a private organization.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
all_arguments: Command-line arguments after the script name.
|
|
118
|
+
|
|
119
|
+
Returns:
|
|
120
|
+
0 for a clean event, 1 when any finding printed.
|
|
121
|
+
"""
|
|
122
|
+
parser = argparse.ArgumentParser(description=__doc__, allow_abbrev=False)
|
|
123
|
+
parser.add_argument("--event-path", type=Path, required=True)
|
|
124
|
+
parser.add_argument("--commit-range")
|
|
125
|
+
arguments = parser.parse_args(list(all_arguments))
|
|
126
|
+
all_findings = scan_event(arguments.event_path)
|
|
127
|
+
if arguments.commit_range:
|
|
128
|
+
all_findings.extend(scan_commits(Path.cwd(), arguments.commit_range))
|
|
129
|
+
for each_finding in all_findings:
|
|
130
|
+
print(each_finding)
|
|
131
|
+
return 1 if all_findings else 0
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
if __name__ == "__main__":
|
|
135
|
+
raise SystemExit(main(sys.argv[1:]))
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Find private organization names in text without holding the names.
|
|
2
|
+
|
|
3
|
+
``private_term_line_numbers`` reports where a name appears and never the
|
|
4
|
+
name, so a caller can print its result in a public log.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import bisect
|
|
10
|
+
import hashlib
|
|
11
|
+
from collections.abc import Collection
|
|
12
|
+
|
|
13
|
+
from dev_env_scripts_constants.private_term_constants import (
|
|
14
|
+
ALL_PRIVATE_TERM_DIGESTS,
|
|
15
|
+
PRIVATE_TERM_TEXT_ENCODING,
|
|
16
|
+
PrivateTermDigest,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _normalized_text_and_line_starts(text: str) -> tuple[str, list[int]]:
|
|
21
|
+
all_normalized_lines: list[str] = []
|
|
22
|
+
all_line_starts: list[int] = []
|
|
23
|
+
offset = 0
|
|
24
|
+
for each_line in text.split("\n"):
|
|
25
|
+
normalized_line = "".join(
|
|
26
|
+
each_character
|
|
27
|
+
for each_character in each_line.lower()
|
|
28
|
+
if each_character.isascii() and each_character.isalnum()
|
|
29
|
+
)
|
|
30
|
+
all_line_starts.append(offset)
|
|
31
|
+
all_normalized_lines.append(normalized_line)
|
|
32
|
+
offset += len(normalized_line)
|
|
33
|
+
return "".join(all_normalized_lines), all_line_starts
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _matching_window_starts(
|
|
37
|
+
normalized_text: str, length: int, all_wanted_digests: set[str]
|
|
38
|
+
) -> list[int]:
|
|
39
|
+
return [
|
|
40
|
+
each_start
|
|
41
|
+
for each_start in range(len(normalized_text) - length + 1)
|
|
42
|
+
if hashlib.sha256(
|
|
43
|
+
normalized_text[each_start : each_start + length].encode(
|
|
44
|
+
PRIVATE_TERM_TEXT_ENCODING
|
|
45
|
+
)
|
|
46
|
+
).hexdigest()
|
|
47
|
+
in all_wanted_digests
|
|
48
|
+
]
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def private_term_line_numbers(
|
|
52
|
+
text: str,
|
|
53
|
+
all_term_digests: Collection[PrivateTermDigest] = ALL_PRIVATE_TERM_DIGESTS,
|
|
54
|
+
) -> tuple[int, ...]:
|
|
55
|
+
"""Return the sorted one-based line numbers where a private name starts.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
text: Text to scan.
|
|
59
|
+
all_term_digests: Digests of the normalized names to find.
|
|
60
|
+
|
|
61
|
+
Returns:
|
|
62
|
+
Each line number once. A name that wraps onto the next line reports
|
|
63
|
+
the line where it starts.
|
|
64
|
+
"""
|
|
65
|
+
normalized_text, all_line_starts = _normalized_text_and_line_starts(text)
|
|
66
|
+
all_found_line_numbers: set[int] = set()
|
|
67
|
+
for each_length in {each_digest.length for each_digest in all_term_digests}:
|
|
68
|
+
all_wanted_digests = {
|
|
69
|
+
each_digest.sha256
|
|
70
|
+
for each_digest in all_term_digests
|
|
71
|
+
if each_digest.length == each_length
|
|
72
|
+
}
|
|
73
|
+
all_found_line_numbers.update(
|
|
74
|
+
bisect.bisect_right(all_line_starts, each_start)
|
|
75
|
+
for each_start in _matching_window_starts(
|
|
76
|
+
normalized_text, each_length, all_wanted_digests
|
|
77
|
+
)
|
|
78
|
+
)
|
|
79
|
+
return tuple(sorted(all_found_line_numbers))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def names_private_term(
|
|
83
|
+
text: str,
|
|
84
|
+
all_term_digests: Collection[PrivateTermDigest] = ALL_PRIVATE_TERM_DIGESTS,
|
|
85
|
+
) -> bool:
|
|
86
|
+
"""Return whether the text names any private organization."""
|
|
87
|
+
return bool(private_term_line_numbers(text, all_term_digests))
|
|
@@ -12,6 +12,7 @@ CHECK_ID_ENV_VAR_DOCUMENTATION = "env-var-documentation"
|
|
|
12
12
|
CHECK_ID_PACKAGE_INVENTORY = "package-inventory"
|
|
13
13
|
CHECK_ID_PYTEST_TESTPATHS = "pytest-testpaths"
|
|
14
14
|
CHECK_ID_TRACKED_PERSONAL_DATA = "tracked-secrets"
|
|
15
|
+
CHECK_ID_TRACKED_PRIVATE_TERMS = "tracked-private-terms"
|
|
15
16
|
|
|
16
17
|
ALL_CHECK_IDS = (
|
|
17
18
|
CHECK_ID_CLAUDE_MD_ORPHANS,
|
|
@@ -19,6 +20,7 @@ ALL_CHECK_IDS = (
|
|
|
19
20
|
CHECK_ID_PACKAGE_INVENTORY,
|
|
20
21
|
CHECK_ID_PYTEST_TESTPATHS,
|
|
21
22
|
CHECK_ID_TRACKED_PERSONAL_DATA,
|
|
23
|
+
CHECK_ID_TRACKED_PRIVATE_TERMS,
|
|
22
24
|
)
|
|
23
25
|
|
|
24
26
|
SEVERITY_BREAKING = "breaking"
|
|
@@ -29,6 +31,7 @@ SEVERITY_BY_CHECK_ID: dict[str, str] = {
|
|
|
29
31
|
CHECK_ID_PACKAGE_INVENTORY: SEVERITY_SMELL,
|
|
30
32
|
CHECK_ID_PYTEST_TESTPATHS: SEVERITY_BREAKING,
|
|
31
33
|
CHECK_ID_TRACKED_PERSONAL_DATA: SEVERITY_BREAKING,
|
|
34
|
+
CHECK_ID_TRACKED_PRIVATE_TERMS: SEVERITY_BREAKING,
|
|
32
35
|
}
|
|
33
36
|
|
|
34
37
|
SUCCESS_EXIT_CODE = 0
|
|
@@ -14,6 +14,9 @@ from repository_checks.env_var_documentation import (
|
|
|
14
14
|
from repository_checks.models import RepositoryCheckReport, RepositoryFinding
|
|
15
15
|
from repository_checks.package_inventory import collect_package_inventory_findings
|
|
16
16
|
from repository_checks.pytest_testpaths import collect_pytest_testpath_findings
|
|
17
|
+
from repository_checks.tracked_private_terms import (
|
|
18
|
+
collect_tracked_private_term_findings,
|
|
19
|
+
)
|
|
17
20
|
from repository_checks.tracked_secrets import collect_tracked_secret_findings
|
|
18
21
|
|
|
19
22
|
RepositoryCollector = Callable[[Path, Sequence[str]], list[RepositoryFinding]]
|
|
@@ -90,4 +93,8 @@ def _all_collectors() -> tuple[tuple[str, RepositoryCollector], ...]:
|
|
|
90
93
|
repository_constants.CHECK_ID_TRACKED_PERSONAL_DATA,
|
|
91
94
|
collect_tracked_secret_findings,
|
|
92
95
|
),
|
|
96
|
+
(
|
|
97
|
+
repository_constants.CHECK_ID_TRACKED_PRIVATE_TERMS,
|
|
98
|
+
collect_tracked_private_term_findings,
|
|
99
|
+
),
|
|
93
100
|
)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Tracked-file check for private organization names."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from dev_env_scripts_constants.private_term_constants import (
|
|
9
|
+
ALL_PRIVATE_TERM_DIGESTS,
|
|
10
|
+
PRIVATE_TERM_MESSAGE_TEMPLATE,
|
|
11
|
+
)
|
|
12
|
+
from policy_lint.config import constants as policy_constants
|
|
13
|
+
from private_terms import private_term_line_numbers
|
|
14
|
+
|
|
15
|
+
from repository_checks.config import constants as repository_constants
|
|
16
|
+
from repository_checks.models import RepositoryFinding
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _read_text_or_none(absolute_path: Path) -> str | None:
|
|
20
|
+
if not absolute_path.is_file():
|
|
21
|
+
return None
|
|
22
|
+
try:
|
|
23
|
+
return absolute_path.read_text(encoding=policy_constants.UTF8_ENCODING)
|
|
24
|
+
except UnicodeDecodeError:
|
|
25
|
+
return None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def collect_tracked_private_term_findings(
|
|
29
|
+
repository_root: Path, all_tracked_paths: Sequence[str]
|
|
30
|
+
) -> list[RepositoryFinding]:
|
|
31
|
+
"""Return one finding for each tracked line that names a private organization.
|
|
32
|
+
|
|
33
|
+
Args:
|
|
34
|
+
repository_root: Git repository root.
|
|
35
|
+
all_tracked_paths: Repository-relative tracked paths.
|
|
36
|
+
|
|
37
|
+
Returns:
|
|
38
|
+
Findings that carry the path and line number, never the name.
|
|
39
|
+
"""
|
|
40
|
+
all_findings: list[RepositoryFinding] = []
|
|
41
|
+
for each_relative_path in all_tracked_paths:
|
|
42
|
+
if (content := _read_text_or_none(repository_root / each_relative_path)) is None:
|
|
43
|
+
continue
|
|
44
|
+
posix_relative_path = each_relative_path.replace(
|
|
45
|
+
repository_constants.WINDOWS_PATH_SEPARATOR, policy_constants.PATH_SEPARATOR
|
|
46
|
+
)
|
|
47
|
+
all_findings.extend(
|
|
48
|
+
RepositoryFinding(
|
|
49
|
+
repository_constants.CHECK_ID_TRACKED_PRIVATE_TERMS,
|
|
50
|
+
posix_relative_path,
|
|
51
|
+
PRIVATE_TERM_MESSAGE_TEMPLATE.format(line_number=each_line_number),
|
|
52
|
+
)
|
|
53
|
+
for each_line_number in private_term_line_numbers(
|
|
54
|
+
content, ALL_PRIVATE_TERM_DIGESTS
|
|
55
|
+
)
|
|
56
|
+
)
|
|
57
|
+
return all_findings
|