@runeditrun/rer 0.2.72 → 0.2.74

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/cli/README.md +13 -0
  2. package/cli/adapter.mjs +1 -0
  3. package/cli/contract.mjs +1 -1
  4. package/cli/deploy-owner-input.mjs +5 -2
  5. package/cli/export-summary.mjs +5 -3
  6. package/cli/hosted-export-command.mjs +47 -0
  7. package/cli/hosted-export.mjs +8 -5
  8. package/cli/inspect.mjs +6 -0
  9. package/cli/main.mjs +1 -1
  10. package/cli/output.mjs +1 -1
  11. package/cli/parse.mjs +1 -0
  12. package/cli/portability-worker.mjs +9 -8
  13. package/cli/release.mjs +9 -0
  14. package/cli/transfers.mjs +3 -0
  15. package/global/CHANGELOG.md +18 -0
  16. package/modules/container/build.mjs +2 -1
  17. package/modules/operator-auth/README.md +9 -3
  18. package/modules/operator-auth/dev/index.mjs +1 -1
  19. package/modules/operator-auth/dev-gate.mjs +18 -6
  20. package/modules/operator-auth/index.d.mts +3 -0
  21. package/modules/operator-auth/scheduled/definition.mjs +3 -1
  22. package/modules/operator-auth/scheduled/expiry.mjs +2 -2
  23. package/modules/operator-auth/scheduled/fence.mjs +2 -1
  24. package/modules/operator-auth/scheduled/runner.mjs +13 -5
  25. package/modules/portability/README.md +34 -8
  26. package/modules/portability/backfills.mjs +22 -14
  27. package/modules/portability/bookkeeping.mjs +6 -1
  28. package/modules/portability/bridge.mjs +9 -6
  29. package/modules/portability/import.mjs +7 -4
  30. package/modules/portability/migration-sql.mjs +8 -0
  31. package/modules/portability/migration-statements.mjs +22 -17
  32. package/modules/portability/rebuilds.mjs +106 -0
  33. package/modules/portability/replay-views.mjs +35 -0
  34. package/modules/portability/storage.mjs +9 -2
  35. package/modules/release/produce.mjs +5 -3
  36. package/modules/release/publication-scan.mjs +5 -4
  37. package/package.json +1 -1
  38. package/scripts/publish-check/README.md +6 -1
  39. package/scripts/publish-check/cli.py +49 -5
  40. package/scripts/publish-check/history_hydration.py +4 -1
  41. package/scripts/publish-check/slots.py +67 -0
  42. package/scripts/publish-check/timeout.json +1 -0
  43. package/scripts/publish-check/timeout.mjs +14 -0
  44. package/scripts/publish-check.mjs +27 -8
@@ -17,14 +17,15 @@ export function validatePublicationDestination(source, project, repository) {
17
17
  }
18
18
  }
19
19
 
20
- export function publicationScan(repo, ref, envFile, project, repository, publicBase, annotationFile) {
20
+ export function publicationScan(repo, ref, envFile, project, repository, publicBase, annotationFile, scanTimeoutSeconds) {
21
21
  const started = performance.now();
22
22
  let report;
23
23
  try {
24
24
  report = JSON.parse(execFileSync(process.execPath,
25
25
  [resolve(import.meta.dirname, '../../scripts/publish-check.mjs'), '--repo', repo, '--ref', ref,
26
26
  ...(publicBase ? ['--public-base', publicBase] : []), ...(project != null ? ['--project', project] : []), ...(repository != null ? ['--repository', repository] : []), ...(envFile ? ['--env-file', envFile] : []),
27
- ...(annotationFile ? ['--annotation-file', annotationFile] : [])],
27
+ ...(annotationFile ? ['--annotation-file', annotationFile] : []),
28
+ ...(scanTimeoutSeconds === undefined ? [] : ['--timeout-seconds', String(scanTimeoutSeconds)])],
28
29
  { stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 128 * 1024 * 1024 }));
29
30
  } catch (error) {
30
31
  try { report = JSON.parse(error.stdout); }
@@ -39,7 +40,7 @@ export function publicationScan(repo, ref, envFile, project, repository, publicB
39
40
  // seed. Materialize raw committed bytes, without checkout filters or export
40
41
  // attributes, and mirror every source ref for the existing history policy.
41
42
  // Neither the caller's local files nor verification output enters this tree.
42
- export function sourcePublicationScan(source, temporary, envFile, project, repository, publicBase, tagged, adopted) {
43
+ export function sourcePublicationScan(source, temporary, envFile, project, repository, publicBase, tagged, adopted, scanTimeoutSeconds) {
43
44
  const started = performance.now();
44
45
  const candidate = sourceTree(source, resolve(temporary, 'publication'));
45
46
  git(source.repo, 'clone', '--mirror', '--no-hardlinks', '--', source.repo, resolve(candidate, '.git'));
@@ -61,7 +62,7 @@ export function sourcePublicationScan(source, temporary, envFile, project, repos
61
62
  git(candidate, 'tag', '-d', name);
62
63
  }
63
64
  const stagingSeconds = (performance.now() - started) / 1000;
64
- const report = publicationScan(candidate, source.commit, envFile, project, repository, publicBase, annotationFile);
65
+ const report = publicationScan(candidate, source.commit, envFile, project, repository, publicBase, annotationFile, scanTimeoutSeconds);
65
66
  Object.assign(report.invocation, { stagingSeconds, totalSeconds: (performance.now() - started) / 1000 });
66
67
  return report;
67
68
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runeditrun/rer",
3
- "version": "0.2.72",
3
+ "version": "0.2.74",
4
4
  "description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -33,7 +33,12 @@ text's repository references instead of resolving them; `history_scan.py`
33
33
  resolves the whole scan's references at once through `github_refs.py`
34
34
  (GraphQL batches, a REST lookup for any repository a batch leaves in doubt) and
35
35
  then adds their rules. The Node entry point bounds
36
- the whole operation and stops its private process group on timeout/cancellation.
36
+ the scan after slot acquisition and stops its private process group on
37
+ timeout/cancellation. `slots.py` obtains one per-user `fcntl.flock` slot before
38
+ starting the scanner. The holder watches Node's pipe for abrupt parent death;
39
+ its worker inherits the lock so it cannot outlive its slot. Queueing has its
40
+ own one-hour cap. `timeout.json` supplies the shared default and upper bound
41
+ to Python and the Node/release validator.
37
42
 
38
43
  Git message records carry their kind explicitly. Only commit records enable the
39
44
  bounded public coauthor address rule described in the command guide; a file named
@@ -1,18 +1,56 @@
1
1
  #!/usr/bin/env python3
2
2
  import argparse
3
3
  import json
4
+ import re
5
+ from pathlib import Path
4
6
  from history_scan import scan
7
+ from public_base import parse as parse_public_base, PublicBaseError
8
+ from report import new_report
9
+
10
+
11
+ def flag_name(value):
12
+ # Only an option name may reach diagnostics, never an attached value or
13
+ # control characters from an untrusted command line.
14
+ match = re.match(r'--?[A-Za-z][A-Za-z0-9_-]*', value)
15
+ return match.group() if match else None
16
+
17
+
18
+ TIMEOUT = json.loads(Path(__file__).with_name('timeout.json').read_text())
5
19
 
6
20
 
7
21
  class SafeParser(argparse.ArgumentParser):
22
+ def parse_args(self, args=None, namespace=None):
23
+ parsed, unknown = self.parse_known_args(args, namespace)
24
+ if unknown:
25
+ flags = [name for item in unknown if (name := flag_name(item))]
26
+ self.error('unrecognized arguments' + (': ' + ' '.join(flags) if flags else ''))
27
+ return parsed
28
+
8
29
  def error(self, message):
9
- self.exit(2, 'publish-check invalid arguments\n')
30
+ if message.startswith('ambiguous option: '):
31
+ name = flag_name(message[len('ambiguous option: '):])
32
+ choices = [flag for flag in self._option_string_actions if name and flag.startswith(name)]
33
+ message = 'ambiguous option: ' + (name or '(invalid flag)') + ' could match ' + ', '.join(choices)
34
+ elif ': ignored explicit argument ' in message:
35
+ message = message.split(': ignored explicit argument ', 1)[0] + ': ignored explicit argument'
36
+ report = new_report()
37
+ report['findings'].append(dict(rule='scan.usage', path='.', commit=None, scope='newly-public',
38
+ message=message, validFlags=list(self._option_string_actions)))
39
+ print(json.dumps(report))
40
+ self.exit(2)
41
+
42
+ def print_help(self, file=None):
43
+ # The worker speaks JSON; the public Node entry point renders help.
44
+ print(json.dumps(dict(help=self.format_help())))
10
45
 
11
46
 
12
47
  def seconds(value):
13
- result = float(value)
14
- if not 0 < result <= 7200:
15
- raise argparse.ArgumentTypeError('invalid timeout')
48
+ try:
49
+ result = float(value)
50
+ except ValueError:
51
+ raise argparse.ArgumentTypeError(f"invalid timeout; expected a number greater than 0 and at most {TIMEOUT['max']}") from None
52
+ if not 0 < result <= TIMEOUT['max']:
53
+ raise argparse.ArgumentTypeError(f"invalid timeout; expected a number greater than 0 and at most {TIMEOUT['max']}")
16
54
  return result
17
55
 
18
56
 
@@ -22,8 +60,14 @@ def main():
22
60
  parser.add_argument('--ref', default='HEAD')
23
61
  for option in ('public-base', 'env-file', 'project', 'repository', 'cache-dir', 'annotation-file'):
24
62
  parser.add_argument('--' + option)
25
- parser.add_argument('--timeout-seconds', type=seconds, default=300)
63
+ parser.add_argument('--timeout-seconds', type=seconds, default=TIMEOUT['default'])
26
64
  args = parser.parse_args()
65
+ if args.public_base is not None:
66
+ try:
67
+ parse_public_base(args.public_base, args.repo)
68
+ except PublicBaseError as error:
69
+ if error.code in ('public-base.not-remote-branch', 'public-base.not-remote-url', 'public-base.not-branch-name'):
70
+ parser.error('argument --public-base: expected <url>#<branch> with a remote URL and branch name')
27
71
  report = scan(args.repo, args.ref, args.public_base, args.env_file, args.project, args.repository,
28
72
  cache_dir=args.cache_dir, timeout=args.timeout_seconds, annotation_file=args.annotation_file)
29
73
  print(json.dumps(report))
@@ -38,8 +38,11 @@ def prepare(history):
38
38
  # Explicit object wants cover selected commits fetched outside the
39
39
  # remote's configured branch refspecs too. stdin keeps one batch
40
40
  # even when a full history exceeds the command argument limit.
41
+ # Refetch encourages automatic repacking. Detached maintenance can
42
+ # outlive inspection and write into a caller's disposable repo
43
+ # during cleanup, so hydration must not start it.
41
44
  history.run('fetch', '--quiet', '--refetch', '--no-filter', '--no-tags',
42
- '--no-recurse-submodules', '--no-write-fetch-head', '--stdin', '--', names[0],
45
+ '--no-auto-maintenance', '--no-recurse-submodules', '--no-write-fetch-head', '--stdin', '--', names[0],
43
46
  input=b'\n'.join(before) + b'\n')
44
47
  except InspectionError:
45
48
  raise missing_blob(history, before[0].decode()) from None
@@ -0,0 +1,67 @@
1
+ """Hold a per-user advisory scan slot until the real scan worker exits."""
2
+ import fcntl
3
+ import os
4
+ from pathlib import Path
5
+ import select
6
+ import signal
7
+ import subprocess
8
+ import sys
9
+ import time
10
+
11
+
12
+ def parent_closed(seconds):
13
+ ready, _, _ = select.select([sys.stdin], [], [], max(0, seconds))
14
+ return bool(ready) and not os.read(sys.stdin.fileno(), 1)
15
+
16
+
17
+ def notify(message):
18
+ os.write(3, (message + '\n').encode())
19
+
20
+
21
+ def main(*, wait_seconds=3600):
22
+ # wait_seconds is injectable by the test interpreter, never a user knob.
23
+ raw = os.environ.get('RER_SCAN_SLOTS', '3')
24
+ if not raw.isdecimal() or not 1 <= int(raw) <= 8:
25
+ notify('scan.slot-configuration')
26
+ return 2
27
+ directory = Path.home() / '.cache/rer/scan-slots'
28
+ directory.mkdir(mode=0o700, parents=True, exist_ok=True)
29
+ deadline = time.monotonic() + wait_seconds
30
+ waiting = False
31
+ while True:
32
+ if waiting and time.monotonic() >= deadline:
33
+ notify('scan.slot-timeout')
34
+ return 2
35
+ for number in range(int(raw)):
36
+ fd = os.open(directory / f'slot-{number}', os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
37
+ try:
38
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
39
+ except BlockingIOError:
40
+ os.close(fd)
41
+ continue
42
+ try:
43
+ notify('acquired')
44
+ worker = subprocess.Popen([sys.executable, '-B', str(Path(__file__).with_name('cli.py')),
45
+ *sys.argv[1:]], stdin=subprocess.DEVNULL,
46
+ stderr=subprocess.DEVNULL, pass_fds=(fd,))
47
+ # Node's pipe closes even on SIGKILL. Stop only this private
48
+ # group, so an orphan never consumes CPU or holds a slot.
49
+ while worker.poll() is None:
50
+ if parent_closed(0.2):
51
+ os.killpg(os.getpgrp(), signal.SIGKILL)
52
+ return worker.returncode
53
+ finally:
54
+ os.close(fd)
55
+ if not waiting:
56
+ print('waiting for a scan slot', file=sys.stderr, flush=True)
57
+ waiting = True
58
+ remaining = deadline - time.monotonic()
59
+ if remaining <= 0:
60
+ notify('scan.slot-timeout')
61
+ return 2
62
+ if parent_closed(min(2, remaining)):
63
+ return 2
64
+
65
+
66
+ if __name__ == '__main__':
67
+ raise SystemExit(main())
@@ -0,0 +1 @@
1
+ {"default": 300, "max": 7200}
@@ -0,0 +1,14 @@
1
+ import { readFileSync } from 'node:fs';
2
+
3
+ export const timeoutBounds = Object.freeze(JSON.parse(readFileSync(new URL('./timeout.json', import.meta.url), 'utf8')));
4
+
5
+ // Release requires whole seconds; publish-check also accepts fractional
6
+ // seconds, which its existing deadline test and standalone callers use.
7
+ export function parseTimeoutSeconds(value, { integer = false } = {}) {
8
+ const seconds = Number(value);
9
+ if ((integer && !/^\d+$/.test(String(value))) || !String(value).trim() ||
10
+ !(seconds > 0 && seconds <= timeoutBounds.max)) {
11
+ throw new RangeError(`Scan timeout must be ${integer ? 'integer ' : ''}seconds from ${integer ? '1' : 'above 0'} to ${timeoutBounds.max}.`);
12
+ }
13
+ return seconds;
14
+ }
@@ -4,12 +4,15 @@ import { mkdtempSync, readFileSync, openSync, closeSync, rmSync } from 'node:fs'
4
4
  import { tmpdir } from 'node:os';
5
5
  import { join } from 'node:path';
6
6
  import { runningVersion } from '../cli/version.mjs';
7
+ import { parseTimeoutSeconds, timeoutBounds } from './publish-check/timeout.mjs';
7
8
 
8
9
  const args = process.argv.slice(2);
9
- let seconds = 300;
10
+ let seconds = timeoutBounds.default;
10
11
  for (let i = 0; i < args.length; i++) {
11
12
  const raw = args[i] === '--timeout-seconds' ? args[i + 1] : args[i].startsWith('--timeout-seconds=') ? args[i].slice(18) : null;
12
- if (raw !== null && Number(raw) > 0 && Number(raw) <= 7200) seconds = Number(raw);
13
+ if (raw !== null) {
14
+ try { seconds = parseTimeoutSeconds(raw); } catch { /* Python reports invalid arguments. */ }
15
+ }
13
16
  }
14
17
  const scratch = mkdtempSync(join(tmpdir(), 'publish-check-'));
15
18
  const output = join(scratch, 'report.json');
@@ -28,11 +31,21 @@ process.on('SIGINT', interrupt);
28
31
  process.on('SIGTERM', interrupt);
29
32
  let report, code = 2;
30
33
  try {
31
- child = spawn('python3', ['-B', new URL('./publish-check/cli.py', import.meta.url).pathname, ...args], {
32
- detached: true, stdio: ['ignore', fd, 'ignore'], env: { ...process.env, TMPDIR: scratch },
34
+ child = spawn('python3', ['-B', new URL('./publish-check/slots.py', import.meta.url).pathname, ...args], {
35
+ detached: true, stdio: ['pipe', fd, 'inherit', 'pipe'], env: { ...process.env, TMPDIR: scratch },
33
36
  });
34
37
  closeSync(fd);
35
- timer = setTimeout(() => stop('scan.deadline-exceeded'), seconds * 1000);
38
+ let messages = '';
39
+ child.stdio[3].setEncoding('utf8');
40
+ child.stdio[3].on('data', data => {
41
+ messages += data;
42
+ for (let at; (at = messages.indexOf('\n')) !== -1;) {
43
+ const message = messages.slice(0, at);
44
+ messages = messages.slice(at + 1);
45
+ if (message === 'acquired' && !timer) timer = setTimeout(() => stop('scan.deadline-exceeded'), seconds * 1000);
46
+ else if (message.startsWith('scan.')) reason ??= message;
47
+ }
48
+ });
36
49
  const result = await new Promise(resolve => {
37
50
  child.once('error', () => resolve(null));
38
51
  child.once('close', value => resolve(value));
@@ -55,7 +68,13 @@ if (reason) report = {
55
68
  history: { commits: 0, trees: 0, blobs: 0 },
56
69
  guarantees: { historyScanned: false, visualRecognition: false, environmentChecked: false },
57
70
  };
58
- report.implementation ??= {};
59
- report.implementation.version = runningVersion();
60
- process.stdout.write(JSON.stringify(report) + '\n');
71
+ if (report.help) process.stdout.write(report.help);
72
+ else {
73
+ for (const finding of report.findings) {
74
+ if (finding.rule === 'scan.usage') process.stderr.write(`publish-check: ${finding.message}; see --help\n`);
75
+ }
76
+ report.implementation ??= {};
77
+ report.implementation.version = runningVersion();
78
+ process.stdout.write(JSON.stringify(report) + '\n');
79
+ }
61
80
  process.exitCode = code;