@runeditrun/rer 0.2.46 → 0.2.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/global/CHANGELOG.md +5 -0
- package/global/run.mjs +2 -1
- package/onboarding/operator-installer.mjs +1 -1
- package/onboarding/queue.mjs +8 -8
- package/package.json +1 -1
- package/scripts/publish-check/README.md +5 -1
- package/scripts/publish-check/github_refs.py +83 -12
- package/scripts/publish-check/history_scan.py +18 -6
- package/scripts/publish-check/public_hygiene.py +22 -12
- package/scripts/publish-check/tree_scan.py +5 -4
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.47
|
|
4
|
+
|
|
5
|
+
- **The publication scan resolves GitHub repositories in bulk.** Before, the scan looked up each GitHub repository a text names with its own REST call. Now it collects every reference first and resolves them together, in one aliased GraphQL query per 100 repositories, which also draws on GitHub's separate GraphQL budget. A repository counts as public or private only on a complete answer that is not disabled or locked. It counts as missing only when GitHub returns NOT_FOUND for exactly that name. Any other answer for a repository, and any batch that is not a complete HTTP 200 GraphQL document, falls back to the earlier strict REST lookup for that repository. Nothing is cached on disk. A run-oncall scan now makes 1 gh call instead of 19, and its report is byte-identical (#216).
|
|
6
|
+
- **Plainer terminal lines for someone new.** Queue lines now name the seed by its `seed.json` name, else its folder name, and never by its project or package id. `rer run` and the build step run package scripts with `pnpm run --silent`, so pnpm's `> package@version script` banner is gone and the script's own output and errors still show. Step 1 of the Run page now says the installer does not edit your shell files. It also says that the line it may end with puts rer on your PATH now and appends the same setting to your shell's startup file for new terminals (#215).
|
|
7
|
+
|
|
3
8
|
## 0.2.46
|
|
4
9
|
|
|
5
10
|
- **`rer companion` pairs your computer with a Run Edit Run app and runs its change requests here.** The local companion is now a `rer` verb: `map` approves one Git checkout whose `origin` is exactly the app's GitHub repository, `pair` redeems the one-use code from the app's Devices page, `codex-auth` or `claude-auth` sets up a dedicated executor home, `commands` approves setup and verify argv lists, and `run` takes jobs from the app. Each job starts from the exact commit the app names in a separate detached checkout, and the provider runs inside a macOS `sandbox-exec` boundary with no GitHub, Cloudflare or MCP credentials; on other systems `run` refuses before it asks for work. It makes outbound HTTPS requests only and opens no local server. State lives in `~/.rer-local` (`RER_LOCAL_HOME` picks another folder), the device key in a 0600 file. `status` prints the paired device, mapped projects, active job and pending events, never the key. Revoke a device in the app under Settings › Devices (#202).
|
package/global/run.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// rer run SCRIPT [ARGS]: `pnpm run SCRIPT ARGS` in a checkout, with the pnpm
|
|
2
2
|
// and Node it pins. rer keeps both in its own cache, off PATH, so a project's
|
|
3
3
|
// documented `pnpm run ...` works as `rer run ...` right after rer install.
|
|
4
|
+
// --silent drops pnpm's "> package@version script" banner; the script's own output and errors stay.
|
|
4
5
|
// A script that runs Playwright gets the checkout's browser first, on first need.
|
|
5
6
|
import { spawn } from 'node:child_process';
|
|
6
7
|
import { readFile } from 'node:fs/promises';
|
|
@@ -52,7 +53,7 @@ export async function runScript(args) {
|
|
|
52
53
|
const code = await (await import('./browser-runtime.mjs')).browserPreflight(target.root, ['run']);
|
|
53
54
|
if (code !== undefined) return code;
|
|
54
55
|
}
|
|
55
|
-
const child = spawn('pnpm', ['run', ...args], { cwd: target.root, env: process.env, stdio: 'inherit' });
|
|
56
|
+
const child = spawn('pnpm', ['run', '--silent', ...args], { cwd: target.root, env: process.env, stdio: 'inherit' });
|
|
56
57
|
const forward = new Map(['SIGINT', 'SIGTERM', 'SIGHUP'].map(signal => [signal, () => child.kill(signal)]));
|
|
57
58
|
for (const [signal, handler] of forward) process.on(signal, handler);
|
|
58
59
|
try {
|
|
@@ -19,7 +19,7 @@ export function operatorInstaller({ database, features, prepare, prepareData })
|
|
|
19
19
|
keepExisting: true, originKey: 'AUTH_ORIGIN',
|
|
20
20
|
values: ({ origin }) => ({ AUTH_LOCAL: 'true', AUTH_ORIGIN: origin, MAIL_PROVIDER: 'development', MAIL_FROM: 'auth@localhost.test' }),
|
|
21
21
|
},
|
|
22
|
-
build: ['pnpm', ['run', 'build:app']],
|
|
22
|
+
build: ['pnpm', ['run', '--silent', 'build:app']],
|
|
23
23
|
migrate: context => migrateOperator(context, database),
|
|
24
24
|
// Setup uses the local entry for owner-side dependency approval without mail.
|
|
25
25
|
start: developmentTarget,
|
package/onboarding/queue.mjs
CHANGED
|
@@ -6,16 +6,16 @@ const portable = basename(dirname(fileURLToPath(import.meta.url))) === 'base-onb
|
|
|
6
6
|
const queueURL = new URL(portable ? '../base-cli/modules/job-queue/' : '../modules/job-queue/', import.meta.url);
|
|
7
7
|
export const heavyVerbs = new Set(['install', 'setup', 'build', 'verify', 'dev', 'preview', 'deploy', 'release', 'export', 'import', 'installer']);
|
|
8
8
|
export const verificationBoundary = verb => verb === 'release' || verb === 'verify' || verb?.startsWith('verify:');
|
|
9
|
+
// The seed's display name (seed.json `name`, else its folder name) for terminal lines,
|
|
10
|
+
// never its package or project id; the hash tells two checkouts of one seed apart.
|
|
9
11
|
export async function queueLabel(root, verb, phase) {
|
|
10
12
|
root = await realpath(root);
|
|
11
|
-
let
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
return `${project} ${verb}:${phase} [${createHash('sha256').update(root).digest('hex').slice(0, 12)}]`;
|
|
13
|
+
let name = basename(root);
|
|
14
|
+
try {
|
|
15
|
+
const value = JSON.parse(await readFile(join(root, 'seed.json'), 'utf8')).name;
|
|
16
|
+
if (typeof value === 'string' && value.trim() && !/[\p{Cc}]/u.test(value) && value.length <= 80) name = value.trim();
|
|
17
|
+
} catch {}
|
|
18
|
+
return `${name} ${verb}:${phase} [${createHash('sha256').update(root).digest('hex').slice(0, 12)}]`;
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
function shellQuote(value) {
|
package/package.json
CHANGED
|
@@ -11,7 +11,11 @@ entropy or English stopword filters. Both assignment rules go through the same
|
|
|
11
11
|
location-bound Source/module identity classification; provider and environment findings
|
|
12
12
|
remain independent.
|
|
13
13
|
`tree_scan.py` applies the kept text and licence policy. `history_scan.py`
|
|
14
|
-
assigns release, working-tree and historical scopes.
|
|
14
|
+
assigns release, working-tree and historical scopes. Inspection returns each
|
|
15
|
+
text's repository references instead of resolving them; `history_scan.py`
|
|
16
|
+
resolves the whole scan's references at once through `github_refs.py`
|
|
17
|
+
(GraphQL batches, a REST lookup for any repository a batch leaves in doubt) and
|
|
18
|
+
then adds their rules. The Node entry point bounds
|
|
15
19
|
the whole operation and stops its private process group on timeout/cancellation.
|
|
16
20
|
|
|
17
21
|
Git message records carry their kind explicitly. Only commit records enable the
|
|
@@ -200,14 +200,18 @@ def response_constant(value):
|
|
|
200
200
|
raise ValueError('nonstandard API JSON constant')
|
|
201
201
|
|
|
202
202
|
|
|
203
|
-
def
|
|
203
|
+
def response(output):
|
|
204
|
+
"""Split `gh api --include` output into its HTTP status and strict JSON document."""
|
|
204
205
|
headers, body = re.split(r'\r?\n\r?\n', output, maxsplit=1)
|
|
205
206
|
status = re.fullmatch(r'HTTP/(?:1\.[01]|2(?:\.0)?|3(?:\.0)?) (\d{3})(?: [^\r\n]*)?',
|
|
206
207
|
headers.splitlines()[0])
|
|
207
208
|
if not status:
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
209
|
+
raise ValueError('no HTTP status line')
|
|
210
|
+
return int(status.group(1)), json.loads(body, object_pairs_hook=response_object, parse_constant=response_constant)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def visibility(output, exit_code):
|
|
214
|
+
code, document = response(output)
|
|
211
215
|
if not isinstance(document, dict):
|
|
212
216
|
return 'incomplete'
|
|
213
217
|
if code == 200 and exit_code == 0 and type(document.get('private')) is bool:
|
|
@@ -221,22 +225,89 @@ def visibility(output, exit_code):
|
|
|
221
225
|
return 'incomplete'
|
|
222
226
|
|
|
223
227
|
|
|
228
|
+
BATCH = 100
|
|
229
|
+
FIELDS = frozenset({'isPrivate', 'isDisabled', 'isLocked'})
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def bulk_query(repositories):
|
|
233
|
+
"""One aliased GraphQL query; owners and names travel as variables, never as query text."""
|
|
234
|
+
variables, fields, arguments = [], [], []
|
|
235
|
+
for index, repository in enumerate(repositories):
|
|
236
|
+
owner, name = repository.split('/')
|
|
237
|
+
variables.append(f'$o{index}:String!,$n{index}:String!')
|
|
238
|
+
fields.append(f'r{index}:repository(owner:$o{index},name:$n{index}){{{" ".join(sorted(FIELDS))}}}')
|
|
239
|
+
arguments += ['-f', f'o{index}={owner}', '-f', f'n{index}={name}']
|
|
240
|
+
return ['-f', 'query=query(' + ','.join(variables) + '){' + ' '.join(fields) + '}', *arguments]
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def bulk_visibility(repositories, output, exit_code):
|
|
244
|
+
"""Return only the verdicts the batch states without doubt.
|
|
245
|
+
|
|
246
|
+
A repository left out takes today's single REST lookup: a disabled or locked
|
|
247
|
+
repository, an error other than GitHub's own not-found for exactly that name
|
|
248
|
+
(no access through SAML or an IP allow list, a rate limit), or any response
|
|
249
|
+
that is not a complete HTTP 200 GraphQL document for exactly this batch.
|
|
250
|
+
"""
|
|
251
|
+
code, document = response(output)
|
|
252
|
+
aliases = {f'r{index}': repository for index, repository in enumerate(repositories)}
|
|
253
|
+
if code != 200 or not isinstance(document, dict) or not set(document) <= {'data', 'errors'}:
|
|
254
|
+
return {}
|
|
255
|
+
data, errors = document.get('data'), document.get('errors', [])
|
|
256
|
+
# gh exits 1 exactly when the document carries errors.
|
|
257
|
+
if not isinstance(data, dict) or set(data) != set(aliases) or not isinstance(errors, list) or exit_code != (1 if errors else 0):
|
|
258
|
+
return {}
|
|
259
|
+
reported = {}
|
|
260
|
+
for error in errors:
|
|
261
|
+
path = error.get('path') if isinstance(error, dict) else None
|
|
262
|
+
if not (isinstance(path, list) and len(path) == 1 and path[0] in aliases):
|
|
263
|
+
return {}
|
|
264
|
+
reported.setdefault(path[0], []).append(error)
|
|
265
|
+
states = {}
|
|
266
|
+
for alias, repository in aliases.items():
|
|
267
|
+
node, problems = data[alias], reported.get(alias, [])
|
|
268
|
+
if (not problems and isinstance(node, dict) and set(node) == FIELDS and type(node['isPrivate']) is bool and
|
|
269
|
+
node['isDisabled'] is False and node['isLocked'] is False):
|
|
270
|
+
states[repository] = 'private' if node['isPrivate'] else 'public'
|
|
271
|
+
elif (node is None and len(problems) == 1 and set(problems[0]) <= {'type', 'path', 'locations', 'message'} and
|
|
272
|
+
problems[0].get('type') == 'NOT_FOUND' and
|
|
273
|
+
problems[0].get('message') == f"Could not resolve to a Repository with the name '{repository}'."):
|
|
274
|
+
states[repository] = 'missing'
|
|
275
|
+
return states
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def gh(arguments):
|
|
279
|
+
# No disk cache, auth arguments, diagnostic output or response metadata.
|
|
280
|
+
# Disable gh debug logging even if inherited from the calling shell.
|
|
281
|
+
env = {k: v for k, v in os.environ.items() if k not in ('GH_DEBUG', 'DEBUG')}
|
|
282
|
+
env['GH_PROMPT_DISABLED'] = '1'
|
|
283
|
+
response = subprocess.run(['gh', 'api', '--hostname', 'github.com', '--include', *arguments],
|
|
284
|
+
env=env, capture_output=True, timeout=60)
|
|
285
|
+
return response.stdout.decode('utf-8'), response.returncode
|
|
286
|
+
|
|
287
|
+
|
|
224
288
|
class GitHubResolver:
|
|
289
|
+
"""Verdicts for one run only: no disk cache can carry a stale or planted answer."""
|
|
225
290
|
def __init__(self, spool=None):
|
|
226
291
|
self.cache = {}
|
|
227
292
|
|
|
293
|
+
def resolve_all(self, repositories):
|
|
294
|
+
"""Resolve in GraphQL batches; a repository its batch leaves in doubt takes the REST lookup."""
|
|
295
|
+
pending = sorted(set(repositories) - self.cache.keys())
|
|
296
|
+
for start in range(0, len(pending), BATCH):
|
|
297
|
+
batch = pending[start:start + BATCH]
|
|
298
|
+
try:
|
|
299
|
+
states = bulk_visibility(batch, *gh(['graphql', *bulk_query(batch)]))
|
|
300
|
+
except (OSError, subprocess.TimeoutExpired, ValueError, IndexError):
|
|
301
|
+
states = {}
|
|
302
|
+
for repository in batch:
|
|
303
|
+
self.cache[repository] = states[repository] if repository in states else self.resolve(repository)
|
|
304
|
+
return {repository: self.cache[repository] for repository in repositories}
|
|
305
|
+
|
|
228
306
|
def resolve(self, repository):
|
|
229
307
|
if repository in self.cache:
|
|
230
308
|
return self.cache[repository]
|
|
231
|
-
# No disk cache, auth arguments, diagnostic output or response metadata.
|
|
232
|
-
# Disable gh debug logging even if inherited from the calling shell.
|
|
233
|
-
env = {k: v for k, v in os.environ.items() if k not in ('GH_DEBUG', 'DEBUG')}
|
|
234
|
-
env['GH_PROMPT_DISABLED'] = '1'
|
|
235
309
|
try:
|
|
236
|
-
|
|
237
|
-
env=env, capture_output=True, timeout=60)
|
|
238
|
-
output = response.stdout.decode('utf-8')
|
|
239
|
-
result = visibility(output, response.returncode)
|
|
310
|
+
result = visibility(*gh(['repos/' + repository]))
|
|
240
311
|
except (OSError, subprocess.TimeoutExpired, ValueError, IndexError):
|
|
241
312
|
result = 'incomplete'
|
|
242
313
|
self.cache[repository] = result
|
|
@@ -11,6 +11,7 @@ from licenses import EXCEPTIONS, exceptions, license_rules, registry
|
|
|
11
11
|
from reviewed import REVIEWED, load_reviewed
|
|
12
12
|
from report import new_report, Findings, finish
|
|
13
13
|
from tree_scan import context, text_rules, working_files, working_regular
|
|
14
|
+
from public_hygiene import reference_rules
|
|
14
15
|
from content_identities import ContentIdentities, candidate as identity_candidate
|
|
15
16
|
from checksum_maps import ChecksumMaps
|
|
16
17
|
|
|
@@ -36,7 +37,6 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
36
37
|
add(INVALID, 'edition.json', history.commit)
|
|
37
38
|
config['pendingDestination'] = None
|
|
38
39
|
recorded = registry() + exceptions(history.selected(EXCEPTIONS))
|
|
39
|
-
resolver = GitHubResolver()
|
|
40
40
|
public_blobs = {e['oid'] for c in history.public_commits for e in history.entries(c)}
|
|
41
41
|
# A file byte-identical to the same path on the public base is already public.
|
|
42
42
|
public_paths = {e['path']: e['oid'] for e in history.entries(history.public_commit)
|
|
@@ -45,6 +45,8 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
45
45
|
report['history']['newlyPublicCommits'] = [c for c in history.commits if c not in history.public_commits]
|
|
46
46
|
seen_trees, seen_blobs, inspected = set(), set(), {}
|
|
47
47
|
entries_by_commit, dependencies = {}, {}
|
|
48
|
+
# Repository references wait here so that the whole scan resolves them in bulk.
|
|
49
|
+
deferred = []
|
|
48
50
|
release_license_good = not license_rules(history.selected('LICENSE'), recorded)
|
|
49
51
|
for commit in history.commits:
|
|
50
52
|
entries = history.entries(commit)
|
|
@@ -74,11 +76,13 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
74
76
|
local = dependencies[pair] & config_tree['localPaths']
|
|
75
77
|
key = (path, oid, signature, local)
|
|
76
78
|
if key not in inspected:
|
|
77
|
-
inspected[key] = text_rules(history.blob(oid), path, config_tree,
|
|
78
|
-
rules, provenance = inspected[key]
|
|
79
|
+
inspected[key] = text_rules(history.blob(oid), path, config_tree, hashes)
|
|
80
|
+
rules, provenance, references = inspected[key]
|
|
79
81
|
entry_scope = 'pre-existing' if (scope != 'release-tree' and oid in public_blobs) or public_paths.get(path) == oid else scope
|
|
80
82
|
for rule in rules:
|
|
81
83
|
add(rule, path, commit, oid, entry_scope)
|
|
84
|
+
if references:
|
|
85
|
+
deferred.append((references, path, commit, oid, entry_scope))
|
|
82
86
|
if provenance:
|
|
83
87
|
add('provenance.declared-source', path, commit, oid, entry_scope)
|
|
84
88
|
report['history'].update(commits=len(history.commits), trees=len(seen_trees), blobs=len(seen_blobs))
|
|
@@ -94,11 +98,13 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
94
98
|
# their known immutable identity; new bytes have no history identity.
|
|
95
99
|
entry = entries_by_commit[history.commit].get(path)
|
|
96
100
|
oid = entry['oid'] if entry and entry['kind'] == 'blob' and history.blob(entry['oid']) == data else None
|
|
97
|
-
rules, _ = text_rules(data, path, selected_context,
|
|
101
|
+
rules, _, references = text_rules(data, path, selected_context, hashes)
|
|
98
102
|
public_oid = public_paths.get(path)
|
|
99
103
|
scope = 'pre-existing' if public_oid and (public_oid == oid or history.blob(public_oid) == data) else 'working-tree'
|
|
100
104
|
for rule in rules:
|
|
101
105
|
add(rule, path, history.commit, oid, scope)
|
|
106
|
+
if references:
|
|
107
|
+
deferred.append((references, path, history.commit, oid, scope))
|
|
102
108
|
if path == 'LICENSE':
|
|
103
109
|
for rule in license_rules(data, recorded):
|
|
104
110
|
add(rule, path, history.commit, oid, 'working-tree')
|
|
@@ -159,11 +165,13 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
159
165
|
for message_type, commit, data in history.messages():
|
|
160
166
|
path = '@' + message_type + '-message'
|
|
161
167
|
report['history']['tagMessages' if message_type == 'tag' else 'commitMessages'] += 1
|
|
162
|
-
rules, _ = text_rules(data, path, selected_context,
|
|
163
|
-
|
|
168
|
+
rules, _, references = text_rules(data, path, selected_context, hashes,
|
|
169
|
+
commit_message=message_type == 'commit')
|
|
164
170
|
scope = 'pre-existing' if commit in history.public_commits or commit in history.public_tags else 'release-tree' if commit == history.commit else 'newly-public'
|
|
165
171
|
for rule in rules:
|
|
166
172
|
add(rule, path, commit, scope=scope)
|
|
173
|
+
if references:
|
|
174
|
+
deferred.append((references, path, commit, None, scope))
|
|
167
175
|
messages.append(data + b'\n')
|
|
168
176
|
length = (data + b'\n').count(b'\n')
|
|
169
177
|
lines.append((count, count + length, path, commit, scope))
|
|
@@ -172,6 +180,10 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
|
|
|
172
180
|
line = hit.get('StartLine', 0)
|
|
173
181
|
location = next(((p, c, s) for start, end, p, c, s in lines if start <= line < end), ('@message', None, 'newly-public'))
|
|
174
182
|
add(secret_rule(hit['RuleID']), location[0], location[1], scope=location[2], scanner_rule=hit['RuleID'])
|
|
183
|
+
states = GitHubResolver().resolve_all({repository for references, *_ in deferred for repository, _, _ in references})
|
|
184
|
+
for references, path, commit, oid, scope in deferred:
|
|
185
|
+
for rule in reference_rules(references, states):
|
|
186
|
+
add(rule, path, commit, oid, scope)
|
|
175
187
|
except PublicBaseError:
|
|
176
188
|
add('scan.public-base-rejected', '.')
|
|
177
189
|
except InspectionError as error:
|
|
@@ -153,32 +153,33 @@ def without_spans(data, spans):
|
|
|
153
153
|
return b' '.join(kept)
|
|
154
154
|
|
|
155
155
|
|
|
156
|
-
def inspect(data, path, config,
|
|
156
|
+
def inspect(data, path, config, *, commit_message=False):
|
|
157
157
|
chunks = list(text_chunks(data))
|
|
158
158
|
# Image and video samples are not text: they can spell an address or a
|
|
159
159
|
# path by chance. Comments, XMP, tags and other metadata remain text.
|
|
160
160
|
spans = sample_spans(data) or []
|
|
161
161
|
# Names in a C2PA manifest's signing certificates are certificate metadata.
|
|
162
162
|
spans += certificate_spans(data)
|
|
163
|
-
rules, provenance = set(), False
|
|
163
|
+
rules, provenance, references = set(), False, set()
|
|
164
164
|
for start, end, fields in chunks:
|
|
165
165
|
spans.append((start, end))
|
|
166
166
|
for field in fields:
|
|
167
167
|
# A metadata value is text, not a standalone source manifest or
|
|
168
168
|
# trust bundle, even when the containing image has a JSON name.
|
|
169
|
-
field_rules, field_provenance = inspect_text(field, '@png-text', config,
|
|
169
|
+
field_rules, field_provenance, field_references = inspect_text(field, '@png-text', config, binary_content=False)
|
|
170
170
|
rules.update(field_rules)
|
|
171
171
|
provenance |= field_provenance
|
|
172
|
+
references |= field_references
|
|
172
173
|
# Keep the original binary classification for content outside text chunks.
|
|
173
174
|
# CRC-adjacent bytes must not also create a synthetic raw mailbox match.
|
|
174
|
-
raw_rules, raw_provenance = inspect_text(without_spans(data, spans), path, config,
|
|
175
|
-
|
|
175
|
+
raw_rules, raw_provenance, raw_references = inspect_text(without_spans(data, spans), path, config,
|
|
176
|
+
commit_message=commit_message, binary_content=binary(data))
|
|
176
177
|
rules.update(raw_rules)
|
|
177
178
|
provenance |= raw_provenance
|
|
178
|
-
return rules, provenance
|
|
179
|
+
return rules, provenance, frozenset(references | raw_references)
|
|
179
180
|
|
|
180
181
|
|
|
181
|
-
def inspect_text(data, path, config,
|
|
182
|
+
def inspect_text(data, path, config, *, commit_message=False, binary_content=None):
|
|
182
183
|
# NUL removal covers UTF-16/32 ASCII content, matching the secret scanner.
|
|
183
184
|
text = decoded_text(data)
|
|
184
185
|
rules = set()
|
|
@@ -197,21 +198,30 @@ def inspect_text(data, path, config, resolver, *, commit_message=False, binary_c
|
|
|
197
198
|
rules.add('hygiene.email')
|
|
198
199
|
ref_text, provenance = provenance_text(text, path, config.get('declaredSources', frozenset()))
|
|
199
200
|
stated, ambiguous = reference_sets(ref_text, path, config.get('localPaths', frozenset()), config.get('projectIdentity'))
|
|
200
|
-
|
|
201
|
+
references = set()
|
|
202
|
+
for repository in stated | ambiguous:
|
|
201
203
|
# A future source identity does not assert current GitHub visibility.
|
|
202
204
|
# Explicit release destinations still take the strict visibility path.
|
|
203
205
|
if (repository in config.get('intendedPublicRepositories', frozenset()) and
|
|
204
206
|
repository != config.get('pendingDestination')):
|
|
205
207
|
continue
|
|
206
|
-
|
|
208
|
+
references.add((repository, repository in ambiguous, repository == config.get('pendingDestination')))
|
|
209
|
+
return rules, provenance, frozenset(references)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def reference_rules(references, states):
|
|
213
|
+
"""Rules for (repository, ambiguous, pending destination) references once their states are known."""
|
|
214
|
+
rules = set()
|
|
215
|
+
for repository, ambiguous, pending in references:
|
|
216
|
+
state = states[repository]
|
|
207
217
|
# A bare word/word with no GitHub context names a repository only when
|
|
208
218
|
# one exists privately; that no repository answers to it proves nothing.
|
|
209
|
-
if state == 'missing' and
|
|
219
|
+
if state == 'missing' and ambiguous and not pending:
|
|
210
220
|
continue
|
|
211
|
-
if state == 'missing' and
|
|
221
|
+
if state == 'missing' and pending:
|
|
212
222
|
rules.add(PENDING_DESTINATION)
|
|
213
223
|
elif state in ('private', 'missing'):
|
|
214
224
|
rules.add('hygiene.github-' + state)
|
|
215
225
|
elif state != 'public':
|
|
216
226
|
rules.add('scan.github-incomplete')
|
|
217
|
-
return rules
|
|
227
|
+
return rules
|
|
@@ -37,15 +37,16 @@ def context(history, commit, config):
|
|
|
37
37
|
'trustBundles': trust}, invalid
|
|
38
38
|
|
|
39
39
|
|
|
40
|
-
def text_rules(data, path, config,
|
|
41
|
-
|
|
42
|
-
|
|
40
|
+
def text_rules(data, path, config, env_hashes, *, commit_message=False):
|
|
41
|
+
"""Rules, provenance and the repository references whose rules wait for one bulk resolution."""
|
|
42
|
+
rules, provenance, references = inspect(data, path, config, commit_message=commit_message)
|
|
43
|
+
path_rules, _, path_references = inspect(path.encode('utf-8', 'surrogateescape'), '@path', config)
|
|
43
44
|
rules.update(path_rules)
|
|
44
45
|
if env_hashes and (environment_match(data, env_hashes) or environment_match(path.encode(), env_hashes)):
|
|
45
46
|
rules.add('secret.env-match')
|
|
46
47
|
if data.startswith(b'version https://git-lfs.github.com/spec/v1'):
|
|
47
48
|
rules.add('scan.lfs-uninspected')
|
|
48
|
-
return rules, provenance
|
|
49
|
+
return rules, provenance, references | path_references
|
|
49
50
|
|
|
50
51
|
|
|
51
52
|
def working_files(repo):
|