@runeditrun/rer 0.2.72 → 0.2.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/README.md +13 -0
- package/cli/adapter.mjs +1 -0
- package/cli/contract.mjs +1 -1
- package/cli/deploy-owner-input.mjs +5 -2
- package/cli/export-summary.mjs +5 -3
- package/cli/hosted-export-command.mjs +47 -0
- package/cli/hosted-export.mjs +8 -5
- package/cli/inspect.mjs +6 -0
- package/cli/main.mjs +1 -1
- package/cli/output.mjs +1 -1
- package/cli/parse.mjs +1 -0
- package/cli/portability-worker.mjs +9 -8
- package/cli/release.mjs +9 -0
- package/cli/transfers.mjs +3 -0
- package/global/CHANGELOG.md +18 -0
- package/modules/container/build.mjs +2 -1
- package/modules/operator-auth/README.md +9 -3
- package/modules/operator-auth/dev/index.mjs +1 -1
- package/modules/operator-auth/dev-gate.mjs +18 -6
- package/modules/operator-auth/index.d.mts +3 -0
- package/modules/operator-auth/scheduled/definition.mjs +3 -1
- package/modules/operator-auth/scheduled/expiry.mjs +2 -2
- package/modules/operator-auth/scheduled/fence.mjs +2 -1
- package/modules/operator-auth/scheduled/runner.mjs +13 -5
- package/modules/portability/README.md +34 -8
- package/modules/portability/backfills.mjs +22 -14
- package/modules/portability/bookkeeping.mjs +6 -1
- package/modules/portability/bridge.mjs +9 -6
- package/modules/portability/import.mjs +7 -4
- package/modules/portability/migration-sql.mjs +8 -0
- package/modules/portability/migration-statements.mjs +22 -17
- package/modules/portability/rebuilds.mjs +106 -0
- package/modules/portability/replay-views.mjs +35 -0
- package/modules/portability/storage.mjs +9 -2
- package/modules/release/produce.mjs +5 -3
- package/modules/release/publication-scan.mjs +5 -4
- package/package.json +1 -1
- package/scripts/publish-check/README.md +6 -1
- package/scripts/publish-check/cli.py +49 -5
- package/scripts/publish-check/history_hydration.py +4 -1
- package/scripts/publish-check/slots.py +67 -0
- package/scripts/publish-check/timeout.json +1 -0
- package/scripts/publish-check/timeout.mjs +14 -0
- package/scripts/publish-check.mjs +27 -8
|
@@ -17,14 +17,15 @@ export function validatePublicationDestination(source, project, repository) {
|
|
|
17
17
|
}
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
export function publicationScan(repo, ref, envFile, project, repository, publicBase, annotationFile) {
|
|
20
|
+
export function publicationScan(repo, ref, envFile, project, repository, publicBase, annotationFile, scanTimeoutSeconds) {
|
|
21
21
|
const started = performance.now();
|
|
22
22
|
let report;
|
|
23
23
|
try {
|
|
24
24
|
report = JSON.parse(execFileSync(process.execPath,
|
|
25
25
|
[resolve(import.meta.dirname, '../../scripts/publish-check.mjs'), '--repo', repo, '--ref', ref,
|
|
26
26
|
...(publicBase ? ['--public-base', publicBase] : []), ...(project != null ? ['--project', project] : []), ...(repository != null ? ['--repository', repository] : []), ...(envFile ? ['--env-file', envFile] : []),
|
|
27
|
-
...(annotationFile ? ['--annotation-file', annotationFile] : [])
|
|
27
|
+
...(annotationFile ? ['--annotation-file', annotationFile] : []),
|
|
28
|
+
...(scanTimeoutSeconds === undefined ? [] : ['--timeout-seconds', String(scanTimeoutSeconds)])],
|
|
28
29
|
{ stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 128 * 1024 * 1024 }));
|
|
29
30
|
} catch (error) {
|
|
30
31
|
try { report = JSON.parse(error.stdout); }
|
|
@@ -39,7 +40,7 @@ export function publicationScan(repo, ref, envFile, project, repository, publicB
|
|
|
39
40
|
// seed. Materialize raw committed bytes, without checkout filters or export
|
|
40
41
|
// attributes, and mirror every source ref for the existing history policy.
|
|
41
42
|
// Neither the caller's local files nor verification output enters this tree.
|
|
42
|
-
export function sourcePublicationScan(source, temporary, envFile, project, repository, publicBase, tagged, adopted) {
|
|
43
|
+
export function sourcePublicationScan(source, temporary, envFile, project, repository, publicBase, tagged, adopted, scanTimeoutSeconds) {
|
|
43
44
|
const started = performance.now();
|
|
44
45
|
const candidate = sourceTree(source, resolve(temporary, 'publication'));
|
|
45
46
|
git(source.repo, 'clone', '--mirror', '--no-hardlinks', '--', source.repo, resolve(candidate, '.git'));
|
|
@@ -61,7 +62,7 @@ export function sourcePublicationScan(source, temporary, envFile, project, repos
|
|
|
61
62
|
git(candidate, 'tag', '-d', name);
|
|
62
63
|
}
|
|
63
64
|
const stagingSeconds = (performance.now() - started) / 1000;
|
|
64
|
-
const report = publicationScan(candidate, source.commit, envFile, project, repository, publicBase, annotationFile);
|
|
65
|
+
const report = publicationScan(candidate, source.commit, envFile, project, repository, publicBase, annotationFile, scanTimeoutSeconds);
|
|
65
66
|
Object.assign(report.invocation, { stagingSeconds, totalSeconds: (performance.now() - started) / 1000 });
|
|
66
67
|
return report;
|
|
67
68
|
}
|
package/package.json
CHANGED
|
@@ -33,7 +33,12 @@ text's repository references instead of resolving them; `history_scan.py`
|
|
|
33
33
|
resolves the whole scan's references at once through `github_refs.py`
|
|
34
34
|
(GraphQL batches, a REST lookup for any repository a batch leaves in doubt) and
|
|
35
35
|
then adds their rules. The Node entry point bounds
|
|
36
|
-
the
|
|
36
|
+
the scan after slot acquisition and stops its private process group on
|
|
37
|
+
timeout/cancellation. `slots.py` obtains one per-user `fcntl.flock` slot before
|
|
38
|
+
starting the scanner. The holder watches Node's pipe for abrupt parent death;
|
|
39
|
+
its worker inherits the lock so it cannot outlive its slot. Queueing has its
|
|
40
|
+
own one-hour cap. `timeout.json` supplies the shared default and upper bound
|
|
41
|
+
to Python and the Node/release validator.
|
|
37
42
|
|
|
38
43
|
Git message records carry their kind explicitly. Only commit records enable the
|
|
39
44
|
bounded public coauthor address rule described in the command guide; a file named
|
|
@@ -1,18 +1,56 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
import argparse
|
|
3
3
|
import json
|
|
4
|
+
import re
|
|
5
|
+
from pathlib import Path
|
|
4
6
|
from history_scan import scan
|
|
7
|
+
from public_base import parse as parse_public_base, PublicBaseError
|
|
8
|
+
from report import new_report
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def flag_name(value):
|
|
12
|
+
# Only an option name may reach diagnostics, never an attached value or
|
|
13
|
+
# control characters from an untrusted command line.
|
|
14
|
+
match = re.match(r'--?[A-Za-z][A-Za-z0-9_-]*', value)
|
|
15
|
+
return match.group() if match else None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
TIMEOUT = json.loads(Path(__file__).with_name('timeout.json').read_text())
|
|
5
19
|
|
|
6
20
|
|
|
7
21
|
class SafeParser(argparse.ArgumentParser):
|
|
22
|
+
def parse_args(self, args=None, namespace=None):
|
|
23
|
+
parsed, unknown = self.parse_known_args(args, namespace)
|
|
24
|
+
if unknown:
|
|
25
|
+
flags = [name for item in unknown if (name := flag_name(item))]
|
|
26
|
+
self.error('unrecognized arguments' + (': ' + ' '.join(flags) if flags else ''))
|
|
27
|
+
return parsed
|
|
28
|
+
|
|
8
29
|
def error(self, message):
|
|
9
|
-
|
|
30
|
+
if message.startswith('ambiguous option: '):
|
|
31
|
+
name = flag_name(message[len('ambiguous option: '):])
|
|
32
|
+
choices = [flag for flag in self._option_string_actions if name and flag.startswith(name)]
|
|
33
|
+
message = 'ambiguous option: ' + (name or '(invalid flag)') + ' could match ' + ', '.join(choices)
|
|
34
|
+
elif ': ignored explicit argument ' in message:
|
|
35
|
+
message = message.split(': ignored explicit argument ', 1)[0] + ': ignored explicit argument'
|
|
36
|
+
report = new_report()
|
|
37
|
+
report['findings'].append(dict(rule='scan.usage', path='.', commit=None, scope='newly-public',
|
|
38
|
+
message=message, validFlags=list(self._option_string_actions)))
|
|
39
|
+
print(json.dumps(report))
|
|
40
|
+
self.exit(2)
|
|
41
|
+
|
|
42
|
+
def print_help(self, file=None):
|
|
43
|
+
# The worker speaks JSON; the public Node entry point renders help.
|
|
44
|
+
print(json.dumps(dict(help=self.format_help())))
|
|
10
45
|
|
|
11
46
|
|
|
12
47
|
def seconds(value):
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
48
|
+
try:
|
|
49
|
+
result = float(value)
|
|
50
|
+
except ValueError:
|
|
51
|
+
raise argparse.ArgumentTypeError(f"invalid timeout; expected a number greater than 0 and at most {TIMEOUT['max']}") from None
|
|
52
|
+
if not 0 < result <= TIMEOUT['max']:
|
|
53
|
+
raise argparse.ArgumentTypeError(f"invalid timeout; expected a number greater than 0 and at most {TIMEOUT['max']}")
|
|
16
54
|
return result
|
|
17
55
|
|
|
18
56
|
|
|
@@ -22,8 +60,14 @@ def main():
|
|
|
22
60
|
parser.add_argument('--ref', default='HEAD')
|
|
23
61
|
for option in ('public-base', 'env-file', 'project', 'repository', 'cache-dir', 'annotation-file'):
|
|
24
62
|
parser.add_argument('--' + option)
|
|
25
|
-
parser.add_argument('--timeout-seconds', type=seconds, default=
|
|
63
|
+
parser.add_argument('--timeout-seconds', type=seconds, default=TIMEOUT['default'])
|
|
26
64
|
args = parser.parse_args()
|
|
65
|
+
if args.public_base is not None:
|
|
66
|
+
try:
|
|
67
|
+
parse_public_base(args.public_base, args.repo)
|
|
68
|
+
except PublicBaseError as error:
|
|
69
|
+
if error.code in ('public-base.not-remote-branch', 'public-base.not-remote-url', 'public-base.not-branch-name'):
|
|
70
|
+
parser.error('argument --public-base: expected <url>#<branch> with a remote URL and branch name')
|
|
27
71
|
report = scan(args.repo, args.ref, args.public_base, args.env_file, args.project, args.repository,
|
|
28
72
|
cache_dir=args.cache_dir, timeout=args.timeout_seconds, annotation_file=args.annotation_file)
|
|
29
73
|
print(json.dumps(report))
|
|
@@ -38,8 +38,11 @@ def prepare(history):
|
|
|
38
38
|
# Explicit object wants cover selected commits fetched outside the
|
|
39
39
|
# remote's configured branch refspecs too. stdin keeps one batch
|
|
40
40
|
# even when a full history exceeds the command argument limit.
|
|
41
|
+
# Refetch encourages automatic repacking. Detached maintenance can
|
|
42
|
+
# outlive inspection and write into a caller's disposable repo
|
|
43
|
+
# during cleanup, so hydration must not start it.
|
|
41
44
|
history.run('fetch', '--quiet', '--refetch', '--no-filter', '--no-tags',
|
|
42
|
-
'--no-recurse-submodules', '--no-write-fetch-head', '--stdin', '--', names[0],
|
|
45
|
+
'--no-auto-maintenance', '--no-recurse-submodules', '--no-write-fetch-head', '--stdin', '--', names[0],
|
|
43
46
|
input=b'\n'.join(before) + b'\n')
|
|
44
47
|
except InspectionError:
|
|
45
48
|
raise missing_blob(history, before[0].decode()) from None
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Hold a per-user advisory scan slot until the real scan worker exits."""
|
|
2
|
+
import fcntl
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import select
|
|
6
|
+
import signal
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
import time
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def parent_closed(seconds):
|
|
13
|
+
ready, _, _ = select.select([sys.stdin], [], [], max(0, seconds))
|
|
14
|
+
return bool(ready) and not os.read(sys.stdin.fileno(), 1)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def notify(message):
|
|
18
|
+
os.write(3, (message + '\n').encode())
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def main(*, wait_seconds=3600):
|
|
22
|
+
# wait_seconds is injectable by the test interpreter, never a user knob.
|
|
23
|
+
raw = os.environ.get('RER_SCAN_SLOTS', '3')
|
|
24
|
+
if not raw.isdecimal() or not 1 <= int(raw) <= 8:
|
|
25
|
+
notify('scan.slot-configuration')
|
|
26
|
+
return 2
|
|
27
|
+
directory = Path.home() / '.cache/rer/scan-slots'
|
|
28
|
+
directory.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
29
|
+
deadline = time.monotonic() + wait_seconds
|
|
30
|
+
waiting = False
|
|
31
|
+
while True:
|
|
32
|
+
if waiting and time.monotonic() >= deadline:
|
|
33
|
+
notify('scan.slot-timeout')
|
|
34
|
+
return 2
|
|
35
|
+
for number in range(int(raw)):
|
|
36
|
+
fd = os.open(directory / f'slot-{number}', os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
|
|
37
|
+
try:
|
|
38
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
39
|
+
except BlockingIOError:
|
|
40
|
+
os.close(fd)
|
|
41
|
+
continue
|
|
42
|
+
try:
|
|
43
|
+
notify('acquired')
|
|
44
|
+
worker = subprocess.Popen([sys.executable, '-B', str(Path(__file__).with_name('cli.py')),
|
|
45
|
+
*sys.argv[1:]], stdin=subprocess.DEVNULL,
|
|
46
|
+
stderr=subprocess.DEVNULL, pass_fds=(fd,))
|
|
47
|
+
# Node's pipe closes even on SIGKILL. Stop only this private
|
|
48
|
+
# group, so an orphan never consumes CPU or holds a slot.
|
|
49
|
+
while worker.poll() is None:
|
|
50
|
+
if parent_closed(0.2):
|
|
51
|
+
os.killpg(os.getpgrp(), signal.SIGKILL)
|
|
52
|
+
return worker.returncode
|
|
53
|
+
finally:
|
|
54
|
+
os.close(fd)
|
|
55
|
+
if not waiting:
|
|
56
|
+
print('waiting for a scan slot', file=sys.stderr, flush=True)
|
|
57
|
+
waiting = True
|
|
58
|
+
remaining = deadline - time.monotonic()
|
|
59
|
+
if remaining <= 0:
|
|
60
|
+
notify('scan.slot-timeout')
|
|
61
|
+
return 2
|
|
62
|
+
if parent_closed(min(2, remaining)):
|
|
63
|
+
return 2
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
if __name__ == '__main__':
|
|
67
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"default": 300, "max": 7200}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
|
|
3
|
+
export const timeoutBounds = Object.freeze(JSON.parse(readFileSync(new URL('./timeout.json', import.meta.url), 'utf8')));
|
|
4
|
+
|
|
5
|
+
// Release requires whole seconds; publish-check also accepts fractional
|
|
6
|
+
// seconds, which its existing deadline test and standalone callers use.
|
|
7
|
+
export function parseTimeoutSeconds(value, { integer = false } = {}) {
|
|
8
|
+
const seconds = Number(value);
|
|
9
|
+
if ((integer && !/^\d+$/.test(String(value))) || !String(value).trim() ||
|
|
10
|
+
!(seconds > 0 && seconds <= timeoutBounds.max)) {
|
|
11
|
+
throw new RangeError(`Scan timeout must be ${integer ? 'integer ' : ''}seconds from ${integer ? '1' : 'above 0'} to ${timeoutBounds.max}.`);
|
|
12
|
+
}
|
|
13
|
+
return seconds;
|
|
14
|
+
}
|
|
@@ -4,12 +4,15 @@ import { mkdtempSync, readFileSync, openSync, closeSync, rmSync } from 'node:fs'
|
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
6
|
import { runningVersion } from '../cli/version.mjs';
|
|
7
|
+
import { parseTimeoutSeconds, timeoutBounds } from './publish-check/timeout.mjs';
|
|
7
8
|
|
|
8
9
|
const args = process.argv.slice(2);
|
|
9
|
-
let seconds =
|
|
10
|
+
let seconds = timeoutBounds.default;
|
|
10
11
|
for (let i = 0; i < args.length; i++) {
|
|
11
12
|
const raw = args[i] === '--timeout-seconds' ? args[i + 1] : args[i].startsWith('--timeout-seconds=') ? args[i].slice(18) : null;
|
|
12
|
-
if (raw !== null
|
|
13
|
+
if (raw !== null) {
|
|
14
|
+
try { seconds = parseTimeoutSeconds(raw); } catch { /* Python reports invalid arguments. */ }
|
|
15
|
+
}
|
|
13
16
|
}
|
|
14
17
|
const scratch = mkdtempSync(join(tmpdir(), 'publish-check-'));
|
|
15
18
|
const output = join(scratch, 'report.json');
|
|
@@ -28,11 +31,21 @@ process.on('SIGINT', interrupt);
|
|
|
28
31
|
process.on('SIGTERM', interrupt);
|
|
29
32
|
let report, code = 2;
|
|
30
33
|
try {
|
|
31
|
-
child = spawn('python3', ['-B', new URL('./publish-check/
|
|
32
|
-
detached: true, stdio: ['
|
|
34
|
+
child = spawn('python3', ['-B', new URL('./publish-check/slots.py', import.meta.url).pathname, ...args], {
|
|
35
|
+
detached: true, stdio: ['pipe', fd, 'inherit', 'pipe'], env: { ...process.env, TMPDIR: scratch },
|
|
33
36
|
});
|
|
34
37
|
closeSync(fd);
|
|
35
|
-
|
|
38
|
+
let messages = '';
|
|
39
|
+
child.stdio[3].setEncoding('utf8');
|
|
40
|
+
child.stdio[3].on('data', data => {
|
|
41
|
+
messages += data;
|
|
42
|
+
for (let at; (at = messages.indexOf('\n')) !== -1;) {
|
|
43
|
+
const message = messages.slice(0, at);
|
|
44
|
+
messages = messages.slice(at + 1);
|
|
45
|
+
if (message === 'acquired' && !timer) timer = setTimeout(() => stop('scan.deadline-exceeded'), seconds * 1000);
|
|
46
|
+
else if (message.startsWith('scan.')) reason ??= message;
|
|
47
|
+
}
|
|
48
|
+
});
|
|
36
49
|
const result = await new Promise(resolve => {
|
|
37
50
|
child.once('error', () => resolve(null));
|
|
38
51
|
child.once('close', value => resolve(value));
|
|
@@ -55,7 +68,13 @@ if (reason) report = {
|
|
|
55
68
|
history: { commits: 0, trees: 0, blobs: 0 },
|
|
56
69
|
guarantees: { historyScanned: false, visualRecognition: false, environmentChecked: false },
|
|
57
70
|
};
|
|
58
|
-
report.
|
|
59
|
-
|
|
60
|
-
|
|
71
|
+
if (report.help) process.stdout.write(report.help);
|
|
72
|
+
else {
|
|
73
|
+
for (const finding of report.findings) {
|
|
74
|
+
if (finding.rule === 'scan.usage') process.stderr.write(`publish-check: ${finding.message}; see --help\n`);
|
|
75
|
+
}
|
|
76
|
+
report.implementation ??= {};
|
|
77
|
+
report.implementation.version = runningVersion();
|
|
78
|
+
process.stdout.write(JSON.stringify(report) + '\n');
|
|
79
|
+
}
|
|
61
80
|
process.exitCode = code;
|