@runeditrun/rer 0.2.45 → 0.2.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,16 +6,16 @@ const portable = basename(dirname(fileURLToPath(import.meta.url))) === 'base-onb
6
6
  const queueURL = new URL(portable ? '../base-cli/modules/job-queue/' : '../modules/job-queue/', import.meta.url);
7
7
  export const heavyVerbs = new Set(['install', 'setup', 'build', 'verify', 'dev', 'preview', 'deploy', 'release', 'export', 'import', 'installer']);
8
8
  export const verificationBoundary = verb => verb === 'release' || verb === 'verify' || verb?.startsWith('verify:');
9
+ // The seed's display name (seed.json `name`, else its folder name) for terminal lines,
10
+ // never its package or project id; the hash tells two checkouts of one seed apart.
9
11
  export async function queueLabel(root, verb, phase) {
10
12
  root = await realpath(root);
11
- let project='project';
12
- for(const file of ['OPERATIONS.json','package.json']){
13
- try{const value=JSON.parse(await readFile(join(root,file),'utf8')),name=value.project??value.name;
14
- if(typeof name==='string'&&/^@?[a-zA-Z0-9_.-]+(?:\/[a-zA-Z0-9_.-]+)?$/.test(name)){project=name;break;}
15
- }catch{}
16
- }
17
-
18
- return `${project} ${verb}:${phase} [${createHash('sha256').update(root).digest('hex').slice(0, 12)}]`;
13
+ let name = basename(root);
14
+ try {
15
+ const value = JSON.parse(await readFile(join(root, 'seed.json'), 'utf8')).name;
16
+ if (typeof value === 'string' && value.trim() && !/[\p{Cc}]/u.test(value) && value.length <= 80) name = value.trim();
17
+ } catch {}
18
+ return `${name} ${verb}:${phase} [${createHash('sha256').update(root).digest('hex').slice(0, 12)}]`;
19
19
  }
20
20
 
21
21
  function shellQuote(value) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runeditrun/rer",
3
- "version": "0.2.45",
3
+ "version": "0.2.47",
4
4
  "description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -11,7 +11,11 @@ entropy or English stopword filters. Both assignment rules go through the same
11
11
  location-bound Source/module identity classification; provider and environment findings
12
12
  remain independent.
13
13
  `tree_scan.py` applies the kept text and licence policy. `history_scan.py`
14
- assigns release, working-tree and historical scopes. The Node entry point bounds
14
+ assigns release, working-tree and historical scopes. Inspection returns each
15
+ text's repository references instead of resolving them; `history_scan.py`
16
+ resolves the whole scan's references at once through `github_refs.py`
17
+ (GraphQL batches, a REST lookup for any repository a batch leaves in doubt) and
18
+ then adds their rules. The Node entry point bounds
15
19
  the whole operation and stops its private process group on timeout/cancellation.
16
20
 
17
21
  Git message records carry their kind explicitly. Only commit records enable the
@@ -200,14 +200,18 @@ def response_constant(value):
200
200
  raise ValueError('nonstandard API JSON constant')
201
201
 
202
202
 
203
- def visibility(output, exit_code):
203
+ def response(output):
204
+ """Split `gh api --include` output into its HTTP status and strict JSON document."""
204
205
  headers, body = re.split(r'\r?\n\r?\n', output, maxsplit=1)
205
206
  status = re.fullmatch(r'HTTP/(?:1\.[01]|2(?:\.0)?|3(?:\.0)?) (\d{3})(?: [^\r\n]*)?',
206
207
  headers.splitlines()[0])
207
208
  if not status:
208
- return 'incomplete'
209
- code = int(status.group(1))
210
- document = json.loads(body, object_pairs_hook=response_object, parse_constant=response_constant)
209
+ raise ValueError('no HTTP status line')
210
+ return int(status.group(1)), json.loads(body, object_pairs_hook=response_object, parse_constant=response_constant)
211
+
212
+
213
+ def visibility(output, exit_code):
214
+ code, document = response(output)
211
215
  if not isinstance(document, dict):
212
216
  return 'incomplete'
213
217
  if code == 200 and exit_code == 0 and type(document.get('private')) is bool:
@@ -221,22 +225,89 @@ def visibility(output, exit_code):
221
225
  return 'incomplete'
222
226
 
223
227
 
228
+ BATCH = 100
229
+ FIELDS = frozenset({'isPrivate', 'isDisabled', 'isLocked'})
230
+
231
+
232
+ def bulk_query(repositories):
233
+ """One aliased GraphQL query; owners and names travel as variables, never as query text."""
234
+ variables, fields, arguments = [], [], []
235
+ for index, repository in enumerate(repositories):
236
+ owner, name = repository.split('/')
237
+ variables.append(f'$o{index}:String!,$n{index}:String!')
238
+ fields.append(f'r{index}:repository(owner:$o{index},name:$n{index}){{{" ".join(sorted(FIELDS))}}}')
239
+ arguments += ['-f', f'o{index}={owner}', '-f', f'n{index}={name}']
240
+ return ['-f', 'query=query(' + ','.join(variables) + '){' + ' '.join(fields) + '}', *arguments]
241
+
242
+
243
+ def bulk_visibility(repositories, output, exit_code):
244
+ """Return only the verdicts the batch states without doubt.
245
+
246
+ A repository left out takes today's single REST lookup: a disabled or locked
247
+ repository, an error other than GitHub's own not-found for exactly that name
248
+ (no access through SAML or an IP allow list, a rate limit), or any response
249
+ that is not a complete HTTP 200 GraphQL document for exactly this batch.
250
+ """
251
+ code, document = response(output)
252
+ aliases = {f'r{index}': repository for index, repository in enumerate(repositories)}
253
+ if code != 200 or not isinstance(document, dict) or not set(document) <= {'data', 'errors'}:
254
+ return {}
255
+ data, errors = document.get('data'), document.get('errors', [])
256
+ # gh exits 1 exactly when the document carries errors.
257
+ if not isinstance(data, dict) or set(data) != set(aliases) or not isinstance(errors, list) or exit_code != (1 if errors else 0):
258
+ return {}
259
+ reported = {}
260
+ for error in errors:
261
+ path = error.get('path') if isinstance(error, dict) else None
262
+ if not (isinstance(path, list) and len(path) == 1 and path[0] in aliases):
263
+ return {}
264
+ reported.setdefault(path[0], []).append(error)
265
+ states = {}
266
+ for alias, repository in aliases.items():
267
+ node, problems = data[alias], reported.get(alias, [])
268
+ if (not problems and isinstance(node, dict) and set(node) == FIELDS and type(node['isPrivate']) is bool and
269
+ node['isDisabled'] is False and node['isLocked'] is False):
270
+ states[repository] = 'private' if node['isPrivate'] else 'public'
271
+ elif (node is None and len(problems) == 1 and set(problems[0]) <= {'type', 'path', 'locations', 'message'} and
272
+ problems[0].get('type') == 'NOT_FOUND' and
273
+ problems[0].get('message') == f"Could not resolve to a Repository with the name '{repository}'."):
274
+ states[repository] = 'missing'
275
+ return states
276
+
277
+
278
+ def gh(arguments):
279
+ # No disk cache, auth arguments, diagnostic output or response metadata.
280
+ # Disable gh debug logging even if inherited from the calling shell.
281
+ env = {k: v for k, v in os.environ.items() if k not in ('GH_DEBUG', 'DEBUG')}
282
+ env['GH_PROMPT_DISABLED'] = '1'
283
+ response = subprocess.run(['gh', 'api', '--hostname', 'github.com', '--include', *arguments],
284
+ env=env, capture_output=True, timeout=60)
285
+ return response.stdout.decode('utf-8'), response.returncode
286
+
287
+
224
288
  class GitHubResolver:
289
+ """Verdicts for one run only: no disk cache can carry a stale or planted answer."""
225
290
  def __init__(self, spool=None):
226
291
  self.cache = {}
227
292
 
293
+ def resolve_all(self, repositories):
294
+ """Resolve in GraphQL batches; a repository its batch leaves in doubt takes the REST lookup."""
295
+ pending = sorted(set(repositories) - self.cache.keys())
296
+ for start in range(0, len(pending), BATCH):
297
+ batch = pending[start:start + BATCH]
298
+ try:
299
+ states = bulk_visibility(batch, *gh(['graphql', *bulk_query(batch)]))
300
+ except (OSError, subprocess.TimeoutExpired, ValueError, IndexError):
301
+ states = {}
302
+ for repository in batch:
303
+ self.cache[repository] = states[repository] if repository in states else self.resolve(repository)
304
+ return {repository: self.cache[repository] for repository in repositories}
305
+
228
306
  def resolve(self, repository):
229
307
  if repository in self.cache:
230
308
  return self.cache[repository]
231
- # No disk cache, auth arguments, diagnostic output or response metadata.
232
- # Disable gh debug logging even if inherited from the calling shell.
233
- env = {k: v for k, v in os.environ.items() if k not in ('GH_DEBUG', 'DEBUG')}
234
- env['GH_PROMPT_DISABLED'] = '1'
235
309
  try:
236
- response = subprocess.run(['gh', 'api', '--hostname', 'github.com', '--include', 'repos/' + repository],
237
- env=env, capture_output=True, timeout=60)
238
- output = response.stdout.decode('utf-8')
239
- result = visibility(output, response.returncode)
310
+ result = visibility(*gh(['repos/' + repository]))
240
311
  except (OSError, subprocess.TimeoutExpired, ValueError, IndexError):
241
312
  result = 'incomplete'
242
313
  self.cache[repository] = result
@@ -11,6 +11,7 @@ from licenses import EXCEPTIONS, exceptions, license_rules, registry
11
11
  from reviewed import REVIEWED, load_reviewed
12
12
  from report import new_report, Findings, finish
13
13
  from tree_scan import context, text_rules, working_files, working_regular
14
+ from public_hygiene import reference_rules
14
15
  from content_identities import ContentIdentities, candidate as identity_candidate
15
16
  from checksum_maps import ChecksumMaps
16
17
 
@@ -36,7 +37,6 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
36
37
  add(INVALID, 'edition.json', history.commit)
37
38
  config['pendingDestination'] = None
38
39
  recorded = registry() + exceptions(history.selected(EXCEPTIONS))
39
- resolver = GitHubResolver()
40
40
  public_blobs = {e['oid'] for c in history.public_commits for e in history.entries(c)}
41
41
  # A file byte-identical to the same path on the public base is already public.
42
42
  public_paths = {e['path']: e['oid'] for e in history.entries(history.public_commit)
@@ -45,6 +45,8 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
45
45
  report['history']['newlyPublicCommits'] = [c for c in history.commits if c not in history.public_commits]
46
46
  seen_trees, seen_blobs, inspected = set(), set(), {}
47
47
  entries_by_commit, dependencies = {}, {}
48
+ # Repository references wait here so that the whole scan resolves them in bulk.
49
+ deferred = []
48
50
  release_license_good = not license_rules(history.selected('LICENSE'), recorded)
49
51
  for commit in history.commits:
50
52
  entries = history.entries(commit)
@@ -74,11 +76,13 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
74
76
  local = dependencies[pair] & config_tree['localPaths']
75
77
  key = (path, oid, signature, local)
76
78
  if key not in inspected:
77
- inspected[key] = text_rules(history.blob(oid), path, config_tree, resolver, hashes)
78
- rules, provenance = inspected[key]
79
+ inspected[key] = text_rules(history.blob(oid), path, config_tree, hashes)
80
+ rules, provenance, references = inspected[key]
79
81
  entry_scope = 'pre-existing' if (scope != 'release-tree' and oid in public_blobs) or public_paths.get(path) == oid else scope
80
82
  for rule in rules:
81
83
  add(rule, path, commit, oid, entry_scope)
84
+ if references:
85
+ deferred.append((references, path, commit, oid, entry_scope))
82
86
  if provenance:
83
87
  add('provenance.declared-source', path, commit, oid, entry_scope)
84
88
  report['history'].update(commits=len(history.commits), trees=len(seen_trees), blobs=len(seen_blobs))
@@ -94,11 +98,13 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
94
98
  # their known immutable identity; new bytes have no history identity.
95
99
  entry = entries_by_commit[history.commit].get(path)
96
100
  oid = entry['oid'] if entry and entry['kind'] == 'blob' and history.blob(entry['oid']) == data else None
97
- rules, _ = text_rules(data, path, selected_context, resolver, hashes)
101
+ rules, _, references = text_rules(data, path, selected_context, hashes)
98
102
  public_oid = public_paths.get(path)
99
103
  scope = 'pre-existing' if public_oid and (public_oid == oid or history.blob(public_oid) == data) else 'working-tree'
100
104
  for rule in rules:
101
105
  add(rule, path, history.commit, oid, scope)
106
+ if references:
107
+ deferred.append((references, path, history.commit, oid, scope))
102
108
  if path == 'LICENSE':
103
109
  for rule in license_rules(data, recorded):
104
110
  add(rule, path, history.commit, oid, 'working-tree')
@@ -159,11 +165,13 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
159
165
  for message_type, commit, data in history.messages():
160
166
  path = '@' + message_type + '-message'
161
167
  report['history']['tagMessages' if message_type == 'tag' else 'commitMessages'] += 1
162
- rules, _ = text_rules(data, path, selected_context, resolver, hashes,
163
- commit_message=message_type == 'commit')
168
+ rules, _, references = text_rules(data, path, selected_context, hashes,
169
+ commit_message=message_type == 'commit')
164
170
  scope = 'pre-existing' if commit in history.public_commits or commit in history.public_tags else 'release-tree' if commit == history.commit else 'newly-public'
165
171
  for rule in rules:
166
172
  add(rule, path, commit, scope=scope)
173
+ if references:
174
+ deferred.append((references, path, commit, None, scope))
167
175
  messages.append(data + b'\n')
168
176
  length = (data + b'\n').count(b'\n')
169
177
  lines.append((count, count + length, path, commit, scope))
@@ -172,6 +180,10 @@ def scan(repo, ref='HEAD', public_base=None, env_file=None, project=None, reposi
172
180
  line = hit.get('StartLine', 0)
173
181
  location = next(((p, c, s) for start, end, p, c, s in lines if start <= line < end), ('@message', None, 'newly-public'))
174
182
  add(secret_rule(hit['RuleID']), location[0], location[1], scope=location[2], scanner_rule=hit['RuleID'])
183
+ states = GitHubResolver().resolve_all({repository for references, *_ in deferred for repository, _, _ in references})
184
+ for references, path, commit, oid, scope in deferred:
185
+ for rule in reference_rules(references, states):
186
+ add(rule, path, commit, oid, scope)
175
187
  except PublicBaseError:
176
188
  add('scan.public-base-rejected', '.')
177
189
  except InspectionError as error:
@@ -153,32 +153,33 @@ def without_spans(data, spans):
153
153
  return b' '.join(kept)
154
154
 
155
155
 
156
- def inspect(data, path, config, resolver, *, commit_message=False):
156
+ def inspect(data, path, config, *, commit_message=False):
157
157
  chunks = list(text_chunks(data))
158
158
  # Image and video samples are not text: they can spell an address or a
159
159
  # path by chance. Comments, XMP, tags and other metadata remain text.
160
160
  spans = sample_spans(data) or []
161
161
  # Names in a C2PA manifest's signing certificates are certificate metadata.
162
162
  spans += certificate_spans(data)
163
- rules, provenance = set(), False
163
+ rules, provenance, references = set(), False, set()
164
164
  for start, end, fields in chunks:
165
165
  spans.append((start, end))
166
166
  for field in fields:
167
167
  # A metadata value is text, not a standalone source manifest or
168
168
  # trust bundle, even when the containing image has a JSON name.
169
- field_rules, field_provenance = inspect_text(field, '@png-text', config, resolver, binary_content=False)
169
+ field_rules, field_provenance, field_references = inspect_text(field, '@png-text', config, binary_content=False)
170
170
  rules.update(field_rules)
171
171
  provenance |= field_provenance
172
+ references |= field_references
172
173
  # Keep the original binary classification for content outside text chunks.
173
174
  # CRC-adjacent bytes must not also create a synthetic raw mailbox match.
174
- raw_rules, raw_provenance = inspect_text(without_spans(data, spans), path, config, resolver,
175
- commit_message=commit_message, binary_content=binary(data))
175
+ raw_rules, raw_provenance, raw_references = inspect_text(without_spans(data, spans), path, config,
176
+ commit_message=commit_message, binary_content=binary(data))
176
177
  rules.update(raw_rules)
177
178
  provenance |= raw_provenance
178
- return rules, provenance
179
+ return rules, provenance, frozenset(references | raw_references)
179
180
 
180
181
 
181
- def inspect_text(data, path, config, resolver, *, commit_message=False, binary_content=None):
182
+ def inspect_text(data, path, config, *, commit_message=False, binary_content=None):
182
183
  # NUL removal covers UTF-16/32 ASCII content, matching the secret scanner.
183
184
  text = decoded_text(data)
184
185
  rules = set()
@@ -197,21 +198,30 @@ def inspect_text(data, path, config, resolver, *, commit_message=False, binary_c
197
198
  rules.add('hygiene.email')
198
199
  ref_text, provenance = provenance_text(text, path, config.get('declaredSources', frozenset()))
199
200
  stated, ambiguous = reference_sets(ref_text, path, config.get('localPaths', frozenset()), config.get('projectIdentity'))
200
- for repository in sorted(stated | ambiguous):
201
+ references = set()
202
+ for repository in stated | ambiguous:
201
203
  # A future source identity does not assert current GitHub visibility.
202
204
  # Explicit release destinations still take the strict visibility path.
203
205
  if (repository in config.get('intendedPublicRepositories', frozenset()) and
204
206
  repository != config.get('pendingDestination')):
205
207
  continue
206
- state = resolver.resolve(repository)
208
+ references.add((repository, repository in ambiguous, repository == config.get('pendingDestination')))
209
+ return rules, provenance, frozenset(references)
210
+
211
+
212
+ def reference_rules(references, states):
213
+ """Rules for (repository, ambiguous, pending destination) references once their states are known."""
214
+ rules = set()
215
+ for repository, ambiguous, pending in references:
216
+ state = states[repository]
207
217
  # A bare word/word with no GitHub context names a repository only when
208
218
  # one exists privately; that no repository answers to it proves nothing.
209
- if state == 'missing' and repository in ambiguous and repository != config.get('pendingDestination'):
219
+ if state == 'missing' and ambiguous and not pending:
210
220
  continue
211
- if state == 'missing' and repository == config.get('pendingDestination'):
221
+ if state == 'missing' and pending:
212
222
  rules.add(PENDING_DESTINATION)
213
223
  elif state in ('private', 'missing'):
214
224
  rules.add('hygiene.github-' + state)
215
225
  elif state != 'public':
216
226
  rules.add('scan.github-incomplete')
217
- return rules, provenance
227
+ return rules
@@ -37,15 +37,16 @@ def context(history, commit, config):
37
37
  'trustBundles': trust}, invalid
38
38
 
39
39
 
40
- def text_rules(data, path, config, resolver, env_hashes, *, commit_message=False):
41
- rules, provenance = inspect(data, path, config, resolver, commit_message=commit_message)
42
- path_rules, _ = inspect(path.encode('utf-8', 'surrogateescape'), '@path', config, resolver)
40
+ def text_rules(data, path, config, env_hashes, *, commit_message=False):
41
+ """Rules, provenance and the repository references whose rules wait for one bulk resolution."""
42
+ rules, provenance, references = inspect(data, path, config, commit_message=commit_message)
43
+ path_rules, _, path_references = inspect(path.encode('utf-8', 'surrogateescape'), '@path', config)
43
44
  rules.update(path_rules)
44
45
  if env_hashes and (environment_match(data, env_hashes) or environment_match(path.encode(), env_hashes)):
45
46
  rules.add('secret.env-match')
46
47
  if data.startswith(b'version https://git-lfs.github.com/spec/v1'):
47
48
  rules.add('scan.lfs-uninspected')
48
- return rules, provenance
49
+ return rules, provenance, references | path_references
49
50
 
50
51
 
51
52
  def working_files(repo):