@runeditrun/rer 0.2.73 → 0.2.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/contract.mjs +1 -1
- package/cli/parse.mjs +1 -0
- package/cli/portability-worker.mjs +6 -7
- package/cli/release.mjs +9 -0
- package/global/CHANGELOG.md +8 -0
- package/modules/container/build.mjs +2 -1
- package/modules/operator-auth/scheduled/definition.mjs +3 -1
- package/modules/operator-auth/scheduled/expiry.mjs +2 -2
- package/modules/operator-auth/scheduled/fence.mjs +2 -1
- package/modules/operator-auth/scheduled/runner.mjs +13 -5
- package/modules/portability/README.md +27 -8
- package/modules/portability/backfills.mjs +22 -14
- package/modules/portability/bridge.mjs +9 -6
- package/modules/portability/migration-sql.mjs +8 -0
- package/modules/portability/migration-statements.mjs +22 -17
- package/modules/portability/rebuilds.mjs +106 -0
- package/modules/portability/replay-views.mjs +35 -0
- package/modules/release/produce.mjs +5 -3
- package/modules/release/publication-scan.mjs +5 -4
- package/package.json +1 -1
- package/scripts/publish-check/README.md +6 -1
- package/scripts/publish-check/cli.py +8 -4
- package/scripts/publish-check/history_hydration.py +4 -1
- package/scripts/publish-check/slots.py +67 -0
- package/scripts/publish-check/timeout.json +1 -0
- package/scripts/publish-check/timeout.mjs +14 -0
- package/scripts/publish-check.mjs +18 -5
package/cli/contract.mjs
CHANGED
|
@@ -26,5 +26,5 @@ export const demoVerb = Object.freeze({ flags: [...commonFlags, '--port', '--hos
|
|
|
26
26
|
// sign the owner in to, so no seed declares it and check does not ask for it.
|
|
27
27
|
export const signinVerb = Object.freeze({ flags: [...commonFlags, '--environment', '--env-file'], effects: 'A one-time sign-in link for the hosted owner, without mail. Uses the Cloudflare credentials that deployed --environment (default production; from the environment or --env-file) to write a single-use grant for the recorded owner into that deployment\'s auth database, then prints the link once to this terminal; it is never saved. The link works once, only for that owner, expires within 15 minutes, and its use is recorded in the owner\'s audit log. Signed in this way, the owner can correct a wrong email in settings.' });
|
|
28
28
|
export const aliases = Object.fromEntries([...Object.keys(verbs).filter(v => v !== 'install'), 'release', 'deps', 'rer'].map(name => [name, name === 'deps' ? 'install' : name === 'rer' ? null : name]));
|
|
29
|
-
export const releaseFlags = ['--repo', '--project', '--repository', '--output', '--ref', '--public-base', '--dry-run', '--local-verify', '--verify-workflow', '--tag', '--clean-history', '--onto', '--env-file', '--allow-placeholder-mark', '--no-queue'];
|
|
29
|
+
export const releaseFlags = ['--repo', '--project', '--repository', '--output', '--ref', '--public-base', '--dry-run', '--local-verify', '--verify-workflow', '--scan-timeout-seconds', '--tag', '--clean-history', '--onto', '--env-file', '--allow-placeholder-mark', '--no-queue'];
|
|
30
30
|
export const requiredHooks = ['preview', 'dependencies', 'build', 'migrate', 'start', 'health', 'owner', 'signIn', 'verify', 'deploy', 'portability', 'state', 'settings', 'input'];
|
package/cli/parse.mjs
CHANGED
|
@@ -17,6 +17,7 @@ export const origin = (host, port) => `http://${host === '::1' ? '[::1]' : host}
|
|
|
17
17
|
export function parse(argv, cwd = process.cwd(), { independent = false, deferTail = false } = {}) {
|
|
18
18
|
const [verb, ...args] = argv;
|
|
19
19
|
if (!verb || verb === '--help') return { help: true };
|
|
20
|
+
// Release validates its own flags, including its integer publication scan timeout.
|
|
20
21
|
if (verb === 'release') return { verb, releaseArgs: args };
|
|
21
22
|
if (verb === 'archive') return { verb, archiveArgs: args };
|
|
22
23
|
// rer demo is rer dev --demo.
|
|
@@ -4,7 +4,7 @@ import { readLegacyArchive } from '../modules/portability/format.mjs';
|
|
|
4
4
|
import { recoverLegacyArchive } from '../modules/portability/legacy-files.mjs';
|
|
5
5
|
import { legacyTargetState, requireLegacyEmpty } from '../modules/portability/legacy-safety.mjs';
|
|
6
6
|
import { migrationReplay } from '../modules/portability/migration-history.mjs';
|
|
7
|
-
import {
|
|
7
|
+
import { migrationStatements } from '../modules/portability/migration-sql.mjs';
|
|
8
8
|
import { appliedMigrations as appliedMigrationNames } from '../modules/portability/bridge.mjs';
|
|
9
9
|
import { join } from 'node:path';
|
|
10
10
|
import { createHash } from 'node:crypto';
|
|
@@ -17,16 +17,15 @@ import { parseJsonc } from './jsonc.mjs';
|
|
|
17
17
|
import { loadPolicy } from './portability-policy.mjs';
|
|
18
18
|
|
|
19
19
|
// The migrations this installation applied, in its ledger's order, with each
|
|
20
|
-
// file's hash, so a restore can place an older archive within them.
|
|
21
|
-
//
|
|
22
|
-
|
|
23
|
-
async function appliedMigrations(root, db, binding) {
|
|
20
|
+
// file's hash, so a restore can place an older archive within them. Reproduce
|
|
21
|
+
// the runner's stored SQL text, then recover SQLite's statement boundaries.
|
|
22
|
+
async function appliedMigrations(root, db, binding, split) {
|
|
24
23
|
const config = parseJsonc(await readFile(join(root, 'wrangler.jsonc'), 'utf8'));
|
|
25
24
|
const directory = join(root, config.d1_databases.find(entry => entry.binding === binding).migrations_dir ?? 'migrations');
|
|
26
25
|
const names = await appliedMigrationNames(db) ?? [];
|
|
27
26
|
return Promise.all(names.map(async name => {
|
|
28
27
|
const bytes = await readFile(join(directory, name));
|
|
29
|
-
return { name, statements:
|
|
28
|
+
return { name, statements: migrationStatements(bytes.toString('utf8'), split), sha256: createHash('sha256').update(bytes).digest('hex') };
|
|
30
29
|
}));
|
|
31
30
|
}
|
|
32
31
|
|
|
@@ -60,7 +59,7 @@ async function run(direction, { root, path, journal, binding, buckets, policy, p
|
|
|
60
59
|
let proxy;
|
|
61
60
|
try {
|
|
62
61
|
proxy = await getPlatformProxy({ configPath, persist: { path: join(root, '.wrangler/state/v3') }, remoteBindings: false });
|
|
63
|
-
const migrations=async()=>migrationReplay(await appliedMigrations(root,proxy.env[binding],binding),policy);
|
|
62
|
+
const migrations=async()=>migrationReplay(await appliedMigrations(root,proxy.env[binding],binding,unstable_splitSqlQuery),policy);
|
|
64
63
|
const options={ db: proxy.env[binding], buckets: Object.fromEntries(Object.entries(buckets).map(([name, binding]) => [name, proxy.env[binding]])), policy, migrations };
|
|
65
64
|
if(direction==='legacy-preflight') {
|
|
66
65
|
await requireLegacyEmpty(options,await legacyTargetState(options.db,options.migrations));
|
package/cli/release.mjs
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { parseTimeoutSeconds, timeoutBounds } from '../scripts/publish-check/timeout.mjs';
|
|
2
|
+
|
|
1
3
|
// Preserve release's accepted parser, producer, receipt and validation exits.
|
|
2
4
|
const releaseHelp = `Usage: rer release --repo DIR --project PUBLISHER/NAME --repository GITHUB_URL --output NEW_DIR [FLAGS]
|
|
3
5
|
|
|
@@ -22,6 +24,9 @@ Without --tag, write a preview. This command never pushes, uploads or deploys.
|
|
|
22
24
|
--verify-workflow NAME_OR_FILE
|
|
23
25
|
Override the seed's declared GitHub verification workflow
|
|
24
26
|
by name or file (case-insensitive).
|
|
27
|
+
--scan-timeout-seconds N
|
|
28
|
+
Publication scan deadline in integer seconds (1..${timeoutBounds.max}).
|
|
29
|
+
Omit to use the scanner default.
|
|
25
30
|
--clean-history Prepare a parentless public export.
|
|
26
31
|
--onto GITHUB_URL#BRANCH
|
|
27
32
|
Prepare a public child of the destination's current branch;
|
|
@@ -52,6 +57,10 @@ export async function dispatchRelease(args) {
|
|
|
52
57
|
for (let index = 0; index < args.length; index++) {
|
|
53
58
|
if(args[index]==='--no-queue'){}
|
|
54
59
|
else if (args[index] === '--tag') options.tag = args[index + 1]?.startsWith('v') ? args[++index] : true;
|
|
60
|
+
else if (args[index] === '--scan-timeout-seconds') {
|
|
61
|
+
try { options.scanTimeoutSeconds = parseTimeoutSeconds(args[++index], { integer: true }); }
|
|
62
|
+
catch (error) { throw Error(`--scan-timeout-seconds: ${error.message}`); }
|
|
63
|
+
}
|
|
55
64
|
else if (args[index] === '--clean-history') options.cleanHistory = true;
|
|
56
65
|
else if (args[index] === '--dry-run') options.dryRun = true;
|
|
57
66
|
else if (args[index] === '--allow-placeholder-mark') options.allowPlaceholderMark = true;
|
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.74
|
|
4
|
+
|
|
5
|
+
- **Restores replay more later migrations (#359).** Archive admission and replay accept row-less targets and row-less tables (restored in the destination's definition), RAISE-only guard triggers, statements that read views, and row-preserving CHECK rebuilds, so retained archives from before those migrations restore exactly (Vector: 121 tables / 23 rows; Penny with its settings and clock replayed).
|
|
6
|
+
- **Publication scans share machine-wide slots (#363).** `publish-check` takes 1 of `RER_SCAN_SLOTS` (default 3) slots per machine and waits for one outside its own deadline, so concurrent scans no longer fail on the 300 s timeout under load; `rer release --scan-timeout-seconds <n>` (1-7200) forwards a longer scan deadline. Defaults are unchanged.
|
|
7
|
+
- **Replay schema identity splits migrations like the runner (#370).** Restore placement splits each applied migration the way the installation runner does, so a `--` comment inside a CREATE TABLE no longer makes a same-version restore refuse with `schema_mismatch`.
|
|
8
|
+
- **Publication scan hydration no longer races its own cleanup (#366).** The history refetch runs with `--no-auto-maintenance`, so a detached git GC cannot write into a partial clone's temporary directory while it is removed.
|
|
9
|
+
- **Scheduled policy door hardening (#369).** The run cap counts only a run's own action audit rows (refusals and expiries no longer use it up), a forged `policy_expired` result is recorded as a failure (only the audited sweeper expires runs), a snapshot whose id does not match its policy is refused, expired runs show as 'Expired run', and crash expiry covers every stale running run atomically with its audit row.
|
|
10
|
+
|
|
3
11
|
## 0.2.73
|
|
4
12
|
|
|
5
13
|
- **Local dev sessions survive OAuth callbacks (#361).** The dev-gate cookie `rer_dev_gate_<port>` is now SameSite=Lax, like the session cookie it guards, so a provider's top-level redirect back to a local connector callback (Xero, Slack) carries the dev session instead of answering 401. The new `gate.admitsSession` is the dev-session check; `gate.admits` (dev sign-in, personas, one-click /signin) still refuses a cookie-only request marked `Sec-Fetch-Site: cross-site`. Only the dev gate (`rer dev`/`rer demo`) changes; production session cookies keep their SameSite.
|
|
@@ -4,6 +4,7 @@ import { createRequire } from 'node:module';
|
|
|
4
4
|
import { pathToFileURL } from 'node:url';
|
|
5
5
|
import { parseJsonc } from '../../cli/jsonc.mjs';
|
|
6
6
|
import { portabilityPlan } from './portability-schema.mjs';
|
|
7
|
+
import { migrationStatements } from '../portability/migration-sql.mjs';
|
|
7
8
|
import { checkBoundary } from './boundary.mjs';
|
|
8
9
|
import { localDurableObjects } from './durable-objects.mjs';
|
|
9
10
|
import { readMigrationRewrites, withRewrites } from '../deploy-kit/migrations.mjs';
|
|
@@ -75,7 +76,7 @@ async function buildImage(root, { output = join(root,'.rer/container/image'), en
|
|
|
75
76
|
const rewrites = await readMigrationRewrites(root);
|
|
76
77
|
const inventory = withRewrites(await containerMigrationInventory(root,{config:workerConfig,configFile}),rewrites);
|
|
77
78
|
const migrations = Object.fromEntries((workerConfig.d1_databases ?? []).map(db => [db.binding,[]]));
|
|
78
|
-
for (const {binding,...migration} of inventory) migrations[binding].push({...migration,statements:
|
|
79
|
+
for (const {binding,...migration} of inventory) migrations[binding].push({...migration,statements:migrationStatements(migration.sql,split)});
|
|
79
80
|
const entry = engine === 'workerd' ? 'dist/server/'+workerConfig.main : 'dist/server/index.mjs';
|
|
80
81
|
await readFile(join(root,entry));
|
|
81
82
|
if (engine === 'node' && workerConfig.kv_namespaces?.length) throw new Error('Node container does not support KV');
|
|
@@ -34,5 +34,7 @@ export async function readPolicy(definitions, id) {
|
|
|
34
34
|
if (!Array.isArray(policies) || new Set(policies.map(policy => policy.id)).size !== policies.length) invalid();
|
|
35
35
|
const input = policies.find(policy => policy.id === id);
|
|
36
36
|
if (!input) throw new AuthError('policy_not_found', 404);
|
|
37
|
-
|
|
37
|
+
const policy = await policyVersion(input);
|
|
38
|
+
if (policy.definition.id !== id) invalid();
|
|
39
|
+
return policy;
|
|
38
40
|
}
|
|
@@ -3,12 +3,12 @@
|
|
|
3
3
|
export const scheduledHandlerLimitMs = 15 * 60000;
|
|
4
4
|
|
|
5
5
|
export async function expireRuns(store, policyId, now) {
|
|
6
|
-
const stale = "policy_id=? AND
|
|
6
|
+
const stale = "policy_id=? AND status='running' AND started_at<?";
|
|
7
7
|
const args = [policyId, now - scheduledHandlerLimitMs];
|
|
8
8
|
// One transaction records each abandoned run exactly once before closing it.
|
|
9
9
|
await store.batch([
|
|
10
10
|
[`INSERT INTO oa_auth_audit(id,actor_id,action,value,created_at)
|
|
11
|
-
SELECT lower(hex(randomblob(16))),'policy:'||policy_id||'@'||version_hash,'policy:expired',
|
|
11
|
+
SELECT lower(hex(randomblob(16))),'policy:'||policy_id||CASE WHEN version_hash IS NULL THEN '' ELSE '@'||version_hash END,'policy:expired',
|
|
12
12
|
json_object('policyId',policy_id,'versionHash',version_hash,'approvalId',approval_id,
|
|
13
13
|
'approvedBy',approved_by,'runId',id,'error','policy_expired'),?
|
|
14
14
|
FROM oa_policy_runs WHERE ${stale}`, [now, ...args]],
|
|
@@ -22,7 +22,8 @@ export function actionFence(approval, runId, action, target, now, maxActions) {
|
|
|
22
22
|
return { sql: '(' + sql + ')', args };
|
|
23
23
|
});
|
|
24
24
|
return { sql: `${approvalGuard.sql} AND EXISTS(SELECT 1 FROM oa_policy_runs WHERE id=? AND approval_id=? AND status='running' AND admitted=1 AND started_at>=?)
|
|
25
|
-
AND (SELECT COUNT(*) FROM oa_auth_audit WHERE
|
|
25
|
+
AND (SELECT COUNT(*) FROM oa_auth_audit WHERE action NOT IN ('policy:refused','policy:expired')
|
|
26
|
+
AND CASE WHEN json_valid(value) THEN json_extract(value,'$.runId') END=?)<?
|
|
26
27
|
AND role!='owner' AND status!='invited' AND id=? AND (${scopes.map(scope => scope.sql).join(' OR ') || '0'})`,
|
|
27
28
|
args: [...approvalGuard.args, runId, approval.id, now - scheduledHandlerLimitMs, runId, maxActions, target.id, ...scopes.flatMap(scope => scope.args)] };
|
|
28
29
|
}
|
|
@@ -4,6 +4,11 @@ import { readPolicy } from './definition.mjs';
|
|
|
4
4
|
import { actionFence, bindPolicyActor } from './fence.mjs';
|
|
5
5
|
import { finishRun } from './history.mjs';
|
|
6
6
|
import { expireRuns } from './expiry.mjs';
|
|
7
|
+
// Expiry is reserved for the audited sweeper, never a callback result.
|
|
8
|
+
function runFailure(error) {
|
|
9
|
+
return error instanceof AuthError && error.code !== 'policy_expired'
|
|
10
|
+
? { status: 'refused', code: error.code } : { status: 'failed', code: 'policy_failed' };
|
|
11
|
+
}
|
|
7
12
|
const actionStatuses = Object.freeze({ 'operators:deactivate': 'inactive', 'operators:reactivate': 'active' });
|
|
8
13
|
|
|
9
14
|
export function policyRunner({ definitions, store, clock, currentApproval, check, rolePolicy }) {
|
|
@@ -23,13 +28,16 @@ export function policyRunner({ definitions, store, clock, currentApproval, check
|
|
|
23
28
|
}
|
|
24
29
|
}
|
|
25
30
|
try {
|
|
31
|
+
await expireRuns(store, policyId, startedAt);
|
|
26
32
|
policy = await readPolicy(definitions, policyId); approval = await currentApproval(policyId);
|
|
27
33
|
await store.run('UPDATE oa_policy_runs SET version_hash=?,approval_id=?,approved_by=? WHERE id=?', [policy.versionHash, approval?.id ?? null, approval?.ownerId ?? null, id]);
|
|
28
34
|
await verify(policy);
|
|
29
|
-
await expireRuns(store, policyId, startedAt);
|
|
30
35
|
const admission = await store.first(`UPDATE oa_policy_runs SET admitted=1 WHERE id=? AND NOT EXISTS(
|
|
31
|
-
SELECT 1 FROM oa_policy_runs WHERE policy_id=? AND id!=? AND admitted=1
|
|
32
|
-
AND (status='running' OR COALESCE(CASE WHEN
|
|
36
|
+
SELECT 1 FROM oa_policy_runs prior WHERE policy_id=? AND id!=? AND admitted=1
|
|
37
|
+
AND (status='running' OR COALESCE(CASE WHEN EXISTS(
|
|
38
|
+
SELECT 1 FROM oa_auth_audit WHERE action='policy:expired'
|
|
39
|
+
AND CASE WHEN json_valid(value) THEN json_extract(value,'$.runId') END=prior.id
|
|
40
|
+
) THEN started_at ELSE finished_at END,started_at)>?)) RETURNING id`,
|
|
33
41
|
[id, policyId, id, startedAt - policy.definition.minIntervalMinutes * 60000]);
|
|
34
42
|
if (!admission) deny('policy_interval', 429);
|
|
35
43
|
let pending = null, actionError = null, committed = 0;
|
|
@@ -52,7 +60,7 @@ export function policyRunner({ definitions, store, clock, currentApproval, check
|
|
|
52
60
|
pending = operation;
|
|
53
61
|
try { return await operation; } catch (error) {
|
|
54
62
|
actionError = error;
|
|
55
|
-
await store.run('INSERT INTO oa_auth_audit(id,actor_id,action,value,created_at) VALUES(?,?,?,?,?)', [crypto.randomUUID(), capability.actor, 'policy:refused', JSON.stringify({ policyId, versionHash: policy.versionHash, approvalId: approval.id, approvedBy: approval.ownerId, runId: id, operatorId, attemptedAction: action, error: error
|
|
63
|
+
await store.run('INSERT INTO oa_auth_audit(id,actor_id,action,value,created_at) VALUES(?,?,?,?,?)', [crypto.randomUUID(), capability.actor, 'policy:refused', JSON.stringify({ policyId, versionHash: policy.versionHash, approvalId: approval.id, approvedBy: approval.ownerId, runId: id, operatorId, attemptedAction: action, error: runFailure(error).code }), clock()]);
|
|
56
64
|
throw error;
|
|
57
65
|
} finally { pending = null; }
|
|
58
66
|
} });
|
|
@@ -61,7 +69,7 @@ export function policyRunner({ definitions, store, clock, currentApproval, check
|
|
|
61
69
|
await finishRun(store, id, null, clock());
|
|
62
70
|
return { id };
|
|
63
71
|
} catch (error) {
|
|
64
|
-
await finishRun(store, id,
|
|
72
|
+
await finishRun(store, id, runFailure(error), clock());
|
|
65
73
|
throw error;
|
|
66
74
|
} finally { open = false; }
|
|
67
75
|
};
|
|
@@ -261,10 +261,11 @@ requires version 2 integrity.
|
|
|
261
261
|
|
|
262
262
|
An archive restores into an installation whose schema differs only by migrations
|
|
263
263
|
applied after the archive was made, when those migrations are additive: new
|
|
264
|
-
tables, new columns that are nullable or have a default, and indexes
|
|
264
|
+
tables, new columns that are nullable or have a default, and indexes, or the
|
|
265
|
+
preserving CHECK rebuild described below. The
|
|
265
266
|
archive's tables restore with their archived columns, later columns take their
|
|
266
|
-
defaults, and later tables stay empty
|
|
267
|
-
|
|
267
|
+
defaults, and later tables stay empty unless their eligible seed or backfill is
|
|
268
|
+
replayed below. This applies to normal v2 import and to backup-only v1 recovery.
|
|
268
269
|
|
|
269
270
|
The Base CLI runners (local Workers and container) pass `createPortability` a
|
|
270
271
|
`migrations` replayer (`migration-history.mjs`, Node only) built from the
|
|
@@ -298,7 +299,8 @@ contain only `CREATE TABLE` (not `AS SELECT`), `CREATE INDEX` and
|
|
|
298
299
|
`ALTER TABLE ... ADD COLUMN` (SQLite allows only constant defaults there) on the
|
|
299
300
|
restorable tables. A later migration with any other statement that names a
|
|
300
301
|
restorable table, such as an `INSERT`, `UPDATE` or `DELETE` that purges or
|
|
301
|
-
backfills rows
|
|
302
|
+
backfills rows, is refused with `schema_mismatch` unless admitted or replayed
|
|
303
|
+
below. Bounded trigger and view creation/removal changes only schema. Statements that
|
|
302
304
|
touch only excluded tables are allowed. A data statement (`INSERT`, `REPLACE`,
|
|
303
305
|
`UPDATE` or `DELETE`, including `INSERT ... SELECT` and a leading `WITH`) is
|
|
304
306
|
judged by the table it writes: when that table is outside the restorable
|
|
@@ -319,12 +321,22 @@ keywords outside table position, function names (also table-valued ones such as
|
|
|
319
321
|
literals, numbers and comments never count. Text Base cannot read to a
|
|
320
322
|
statement boundary, such as an unclosed block comment, is refused. An archived table the archive holds no
|
|
321
323
|
rows of restores empty, so an `UPDATE` or `DELETE` of it changes nothing and is
|
|
322
|
-
allowed
|
|
323
|
-
|
|
324
|
-
is still refused. For archives without a migration record,
|
|
324
|
+
allowed regardless of what it reads: no row can change or fire a trigger. An
|
|
325
|
+
ordinary `INSERT` into it remains refused. For archives without a migration record,
|
|
325
326
|
every migration after the earliest prefix that gives the archive's schema is
|
|
326
327
|
checked, since the archive may predate any of them.
|
|
327
328
|
|
|
329
|
+
A complete five-statement `CREATE TABLE copy AS SELECT` / `DROP TABLE` /
|
|
330
|
+
`CREATE TABLE` / `INSERT ... SELECT` / `DROP TABLE copy` sequence within one
|
|
331
|
+
migration is admitted when every column is copied back unchanged and, for a
|
|
332
|
+
populated archived table, its rowid is copied back too. The copy must be local
|
|
333
|
+
to the sequence. Only CHECK clauses may differ; column declarations, defaults,
|
|
334
|
+
collations, other constraints and table options remain unchanged. Manifest
|
|
335
|
+
counts identify empty tables without another row scan. Restore uses the
|
|
336
|
+
destination definition, so a row rejected by its CHECK fails the whole atomic
|
|
337
|
+
restore; no row is skipped or coerced. Other statements still face the usual
|
|
338
|
+
data classifier, and a proven preserving block is not a competing backfill writer.
|
|
339
|
+
|
|
328
340
|
A backfill of a catalogue table the archive does not carry (one a later
|
|
329
341
|
migration adds, restored empty) is **replayed** instead of refused: an
|
|
330
342
|
`INSERT ... SELECT` (or `INSERT ... VALUES`) into that table. Finalize empties
|
|
@@ -339,13 +351,18 @@ scans the restored rows. Rows the restore normalizes (import safety) are read
|
|
|
339
351
|
as restored. The restore is refused with
|
|
340
352
|
`schema_mismatch:migration_data:<migration>` instead when the archive carries
|
|
341
353
|
the table (even with no rows), when a write to it is not a plain `INSERT` or
|
|
342
|
-
`REPLACE`, reads any table other than archived ones and itself
|
|
354
|
+
`REPLACE`, reads any table other than archived ones and itself (including through
|
|
355
|
+
a view resolved from applied migration history), selects `*` or
|
|
343
356
|
a secret column, when a trigger on it or a foreign key action from it reaches a
|
|
344
357
|
catalogue table, when any other later statement writes it (directly or through
|
|
345
358
|
a trigger) or changes its columns other than by `ADD COLUMN` (allowed only when
|
|
346
359
|
every replay names its columns), or when the statement cannot be bounded.
|
|
347
360
|
A failed or interrupted apply leaves no replayed row; finalizing again replays
|
|
348
361
|
from the start.
|
|
362
|
+
Read-only trigger bodies, including RAISE-only guards, add no write dependency.
|
|
363
|
+
Eligible constant seed INSERTs replay too. Statements confined to tables added
|
|
364
|
+
after the archive retain their existing admission when they need no replay;
|
|
365
|
+
an INSERT reading a view must still resolve entirely to carried tables.
|
|
349
366
|
|
|
350
367
|
The same check runs when the archive's schema identity equals the destination's:
|
|
351
368
|
a later migration that only purges or backfills rows leaves the schema unchanged,
|
|
@@ -646,6 +663,8 @@ modules/portability/ledgers.mjs
|
|
|
646
663
|
modules/portability/migration-statements.mjs
|
|
647
664
|
modules/portability/sql-text.mjs
|
|
648
665
|
modules/portability/backfills.mjs
|
|
666
|
+
modules/portability/rebuilds.mjs
|
|
667
|
+
modules/portability/replay-views.mjs
|
|
649
668
|
modules/portability/legacy-safety.mjs
|
|
650
669
|
modules/portability/export.mjs
|
|
651
670
|
modules/portability/import.mjs
|
|
@@ -2,6 +2,8 @@ import { statement, quote, insist } from './storage.mjs';
|
|
|
2
2
|
import { tokens, folded } from './generated.mjs';
|
|
3
3
|
import { changesData, dataStatement, reachingTables, triggerBodies, schemaChange, statementNames } from './migration-statements.mjs';
|
|
4
4
|
import { splitStatements } from './sql-text.mjs';
|
|
5
|
+
import { preservingRebuilds } from './rebuilds.mjs';
|
|
6
|
+
import { replayViews, viewReads } from './replay-views.mjs';
|
|
5
7
|
|
|
6
8
|
// The data statements of later migrations, judged by the tables they write
|
|
7
9
|
// (bridge.mjs placement). Restored rows never pass through a later migration.
|
|
@@ -10,12 +12,13 @@ import { splitStatements } from './sql-text.mjs';
|
|
|
10
12
|
// writes reach no catalogue table through a trigger or foreign key action
|
|
11
13
|
// (reachingTables) changes no restored row, whatever it reads: admitted.
|
|
12
14
|
// - One whose target is a catalogue table the archive does not carry (restored
|
|
13
|
-
// empty)
|
|
14
|
-
//
|
|
15
|
+
// empty) is replayed at finalize when it inserts seed or backfill rows.
|
|
16
|
+
// Every later write to that table must then be a
|
|
15
17
|
// replayable INSERT: they run in migration order, exactly as written, after
|
|
16
18
|
// the archived rows are inserted and with the destination's triggers still
|
|
17
19
|
// dropped, so the table holds what the live upgrade gave it. A replay may read
|
|
18
|
-
// only archived tables (complete by then)
|
|
20
|
+
// only archived tables (complete by then), views over those tables and the
|
|
21
|
+
// table itself, by named
|
|
19
22
|
// columns that are not secret; no trigger on the table may write a catalogue
|
|
20
23
|
// table; no later statement may write it another way (directly or through a
|
|
21
24
|
// trigger) or change its columns other than by ADD COLUMN.
|
|
@@ -25,14 +28,15 @@ import { splitStatements } from './sql-text.mjs';
|
|
|
25
28
|
// Each chunk the runner's splitter handed over is split again at SQLite's own
|
|
26
29
|
// statement boundaries first (sql-text.mjs), so a trigger glued to the
|
|
27
30
|
// statements after it is judged as the statements it holds.
|
|
28
|
-
// Returns the replays in migration order, or refuses with the first migration
|
|
31
|
+
// Returns the replays in migration order and the proven CHECK rebuilds, or refuses with the first migration
|
|
29
32
|
// (in `unapplied` order) holding a statement that cannot be admitted.
|
|
30
|
-
export function laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets}) {
|
|
33
|
+
export function laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets,manifest}) {
|
|
31
34
|
const split=name=>replay.statements(name).flatMap(splitStatements),fold=list=>list.map(folded);
|
|
32
35
|
const statements=applied.flatMap(split);
|
|
33
36
|
const restorable=fold(catalogue),carried=fold(archived),absent=restorable.filter(name=>!carried.includes(name));
|
|
34
37
|
const reached=reachingTables(statements,carried),writable=reachingTables(statements,restorable);
|
|
35
38
|
const later=unapplied.flatMap(migration=>split(migration).map((sql,index)=>({migration,index,sql,data:dataStatement(sql)})));
|
|
39
|
+
const {admitted,tables:rebuilt}=preservingRebuilds(later,manifest,tables,statements),views=replayViews(statements);
|
|
36
40
|
const known=fold(tables),outside=later.filter(s=>s.data && !writable.includes(s.data.target)),tainted=new Set();
|
|
37
41
|
// Tables an admitted write fills from archived rows (directly, through its
|
|
38
42
|
// target's triggers, or from another such table) hold rows the restore lacks.
|
|
@@ -40,29 +44,33 @@ export function laterData({applied,unapplied,replay,catalogue,archived,empty,tab
|
|
|
40
44
|
for(let grew=true;grew;)grew=outside.filter(s=>s.data.names.some(name=>carried.includes(name) || tainted.has(name))).map(s=>taint(s.data.target)).some(Boolean);
|
|
41
45
|
const refused=new Set(),writes=new Map();
|
|
42
46
|
for(const s of later) {
|
|
43
|
-
if(outside.includes(s))continue;
|
|
47
|
+
if(outside.includes(s) || admitted.has(s))continue;
|
|
44
48
|
s.today=changesData(s.sql,[...reached,...tainted],empty);
|
|
45
49
|
if(s.data && absent.includes(s.data.target)){if(!writes.has(s.data.target))writes.set(s.data.target,[]);writes.get(s.data.target).push(s);}
|
|
46
50
|
else if(s.today)refused.add(s);
|
|
47
51
|
}
|
|
48
52
|
const replays=[];
|
|
49
53
|
for(const [table,list] of writes) {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
const
|
|
54
|
+
// Seed INSERTs into a new table replay too: the rows already present in a
|
|
55
|
+
// fresh destination must be reset and derived from the restored installation.
|
|
56
|
+
const bad=list.filter(s=>!replayable(s,{table,statements,restorable,writable,carried,tables:known,secrets,views}));
|
|
57
|
+
// Writes confined to tables created after the archive retain their
|
|
58
|
+
// admission. Replay eligible seeds, but do not turn a new-table-only
|
|
59
|
+
// upgrade into a refused backfill. View reads always require resolution.
|
|
60
|
+
const required=list.some(s=>s.today || s.data.names.some(name=>views.has(name)));
|
|
61
|
+
if(!required && bad.length)continue;
|
|
62
|
+
const reach=reachingTables(statements,[table]),other=later.filter(s=>!admitted.has(s) && !list.includes(s) && writesAnother(s,table,reach,list));
|
|
53
63
|
if(bad.length || other.length){[...bad,...other].forEach(s=>refused.add(s));continue;}
|
|
54
64
|
replays.push(...list);
|
|
55
65
|
}
|
|
56
66
|
const first=later.find(s=>refused.has(s));
|
|
57
67
|
insist(!first,'schema_mismatch:migration_data:'+first?.migration);
|
|
58
68
|
const named=Object.fromEntries(catalogue.map(name=>[folded(name),name]));
|
|
59
|
-
return later.filter(s=>replays.includes(s)).map(s=>({migration:s.migration,statement:s.index,table:named[s.data.target],sql:s.sql}));
|
|
69
|
+
return {rebuilt,replays:later.filter(s=>replays.includes(s)).map(s=>({migration:s.migration,statement:s.index,table:named[s.data.target],sql:s.sql}))};
|
|
60
70
|
}
|
|
61
|
-
function replayable({data},{table,statements,restorable,writable,carried,tables,secrets}) {
|
|
71
|
+
function replayable({data},{table,statements,restorable,writable,carried,tables,secrets,views}) {
|
|
62
72
|
if(!['insert','replace'].includes(data.verb) || data.star)return false;
|
|
63
|
-
|
|
64
|
-
if(!reads.every(name=>carried.includes(name)))return false;
|
|
65
|
-
if(reads.some(name=>Object.entries(secrets).some(([t,columns])=>folded(t)===name && columns.some(c=>data.words.includes(folded(c))))))return false;
|
|
73
|
+
if(!viewReads(data.names,data.words,{views,tables,carried,table,secrets}))return false;
|
|
66
74
|
if(triggerBodies(statements,table).some(name=>writable.includes(name)))return false;
|
|
67
75
|
// Foreign key actions: a row the replay deletes or rekeys (REPLACE, upsert) must reach no other catalogue table.
|
|
68
76
|
return !reachingTables(statements,restorable.filter(name=>name!==table)).includes(table);
|
|
@@ -3,6 +3,7 @@ import { canonicalJson } from './hash.mjs';
|
|
|
3
3
|
import { tokens, identifier, folded } from './generated.mjs';
|
|
4
4
|
import { laterData } from './backfills.mjs';
|
|
5
5
|
import { destinationLedger } from './ledgers.mjs';
|
|
6
|
+
import { checkRebuild } from './rebuilds.mjs';
|
|
6
7
|
|
|
7
8
|
// Applied product migrations in the ledger's own order (ledgers.mjs). A
|
|
8
9
|
// state-bearing ledger may hold an unacknowledged migration whose effects are
|
|
@@ -51,8 +52,8 @@ async function placement(db,manifest,cat,replay,ledger,history) {
|
|
|
51
52
|
const catalogue=cat.tables.map(t=>t.name),archived=catalogue.filter(name=>manifest.tables.some(t=>t.name===name));
|
|
52
53
|
const empty=archived.filter(name=>!manifest.tables.find(t=>t.name===name).count);
|
|
53
54
|
const tables=(await all(db,"SELECT name FROM sqlite_master WHERE type IN ('table','view')")).map(t=>t.name);
|
|
54
|
-
const replays=laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets:manifest.excludedColumns??{}});
|
|
55
|
-
return {earlier,replays};
|
|
55
|
+
const {replays,rebuilt}=laterData({applied,unapplied,replay,catalogue,archived,empty,tables,secrets:manifest.excludedColumns??{},manifest:manifest.tables});
|
|
56
|
+
return {earlier,replays,rebuilt};
|
|
56
57
|
}
|
|
57
58
|
// An archive without a migration record that carries its source's migration
|
|
58
59
|
// ledger (an old archive from before that ledger became bookkeeping) records
|
|
@@ -84,20 +85,22 @@ export async function sameSchemaMigrations(db,manifest,cat,migrations,history) {
|
|
|
84
85
|
await placement(db,manifest,cat,await migrations(),ledger,history);
|
|
85
86
|
}
|
|
86
87
|
// An archive made before later migrations restores only when those migrations
|
|
87
|
-
// are additive
|
|
88
|
-
//
|
|
88
|
+
// are additive, or a proven row-preserving CHECK rebuild: new tables, new
|
|
89
|
+
// columns that take a default, or indexes (which archives never carry).
|
|
90
|
+
// Anything else stays a schema_mismatch refusal.
|
|
89
91
|
export async function additiveBridge(db,manifest,cat,migrations,history) {
|
|
90
92
|
insist(migrations,'schema_mismatch');
|
|
91
93
|
const replay=await migrations(),ledger=await destinationLedger(db)??[],applied=ledger.map(m=>m.name);
|
|
92
94
|
insist(!manifest.migrations || manifest.migrations.every(name=>applied.includes(name)),'schema_mismatch:migrations');
|
|
93
|
-
const {earlier,replays}=await placement(db,manifest,cat,replay,ledger,history);
|
|
95
|
+
const {earlier,replays,rebuilt}=await placement(db,manifest,cat,replay,ledger,history);
|
|
94
96
|
const columns={};
|
|
95
97
|
for(const table of manifest.tables) {
|
|
96
98
|
const target=cat.tables.find(t=>t.name===table.name),label=':'+table.name;
|
|
97
99
|
insist(target,'schema_mismatch:table'+label);
|
|
98
100
|
const kept=target.columns.slice(0,table.columns.length),added=target.columns.slice(table.columns.length);
|
|
99
101
|
insist(canonicalJson(kept)===canonicalJson(table.columns) && table.rowidAlias===target.rowidAlias && table.dependencies.every(d=>target.dependencies.includes(d)),'schema_mismatch:columns'+label);
|
|
100
|
-
|
|
102
|
+
const rebuiltChecks=rebuilt.has(folded(table.name)) && checkRebuild(table.sql,target.sql);
|
|
103
|
+
if(!added.length){insist(table.sql===target.sql || rebuiltChecks,'schema_mismatch:definition'+label);continue;}
|
|
101
104
|
const declared=await all(db,`PRAGMA table_xinfo(${quote(table.name)})`);
|
|
102
105
|
insist(added.every(c=>!c.pk && !c.generated && (!c.notnull || declared.find(d=>d.name===c.name)?.dflt_value!=null)),'schema_mismatch:column_default'+label);
|
|
103
106
|
insist(appended(table.sql,target.sql,added.map(c=>c.name)),'schema_mismatch:definition'+label);
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { splitStatements } from './sql-text.mjs';
|
|
2
|
+
|
|
3
|
+
// Use the installation's migration runner to reproduce the text it stores in
|
|
4
|
+
// sqlite_master (including its comment handling). Then recover SQLite's own
|
|
5
|
+
// boundaries: Wrangler can glue a CASE ... END trigger to later statements.
|
|
6
|
+
export function migrationStatements(sql, runnerSplit) {
|
|
7
|
+
return runnerSplit(sql).flatMap(splitStatements);
|
|
8
|
+
}
|
|
@@ -16,8 +16,8 @@ import { tableNames, splitStatements } from './sql-text.mjs';
|
|
|
16
16
|
// operator-auth or other excluded tables that no trigger links to restorable ones.
|
|
17
17
|
// `empty` lists the restorable tables the archive holds no rows of: restore
|
|
18
18
|
// leaves them empty, and an UPDATE or DELETE of an empty table changes nothing
|
|
19
|
-
// (no row fires a trigger either),
|
|
20
|
-
//
|
|
19
|
+
// (no row fires a trigger either), regardless of the tables it reads. This
|
|
20
|
+
// exemption requires a bounded data statement whose target can be read.
|
|
21
21
|
// Any other statement naming an empty table still does: an INSERT would leave
|
|
22
22
|
// rows the restore lacks. A chunk holding several statements (sql-text.mjs
|
|
23
23
|
// splitStatements) changes data when any of them does.
|
|
@@ -27,28 +27,30 @@ export function changesData(sql,reached,empty=[]) {
|
|
|
27
27
|
function statementChanges(sql,reached,empty) {
|
|
28
28
|
const list=tokens(sql),words=list.map(t=>t.quoted?undefined:folded(t.text)),kind=kindOf(words);
|
|
29
29
|
if(kind==='create table')return topLevelWords(list,words).includes('as');
|
|
30
|
-
if(
|
|
30
|
+
if(['create index','create view','drop view'].includes(kind))return false;
|
|
31
31
|
if(kind==='alter table' && words.find((w,i)=>i>2 && ['add','rename','drop'].includes(w))==='add')return false;
|
|
32
32
|
if(kind==='create trigger' && createdTrigger(list))return false;
|
|
33
33
|
if(kind==='drop trigger' && droppedTrigger(list))return false;
|
|
34
|
-
const
|
|
35
|
-
|
|
34
|
+
const data=dataStatement(sql);
|
|
35
|
+
if(data && ['update','delete'].includes(data.verb) && empty.map(folded).includes(data.target))return false;
|
|
36
|
+
return names(list,new Set(reached.map(folded)));
|
|
36
37
|
}
|
|
37
38
|
// The restorable tables and every table whose writes reach one through a
|
|
38
39
|
// trigger any of `statements` creates, or through a foreign key action (ON
|
|
39
40
|
// DELETE/UPDATE CASCADE or SET) declared by a reached table: a write to such a
|
|
40
41
|
// table during a later migration changes restorable rows the archive lacks. A
|
|
41
|
-
// trigger counts when its statement names a reached
|
|
42
|
-
//
|
|
43
|
-
//
|
|
42
|
+
// bounded trigger with a writing body counts when its statement names a reached
|
|
43
|
+
// table anywhere; a read-only body (including SELECT RAISE) writes nothing.
|
|
44
|
+
// One this parser cannot bound adds every table name it holds (sql-text.mjs
|
|
45
|
+
// tableNames: never a keyword, function, qualified column or literal).
|
|
44
46
|
export function reachingTables(statements,restorable) {
|
|
45
47
|
const reached=new Set(restorable.map(folded)),parsed=statements.flatMap(splitStatements).map(sql=>tokens(sql)).map(list=>({list,words:list.map(t=>t.quoted?undefined:folded(t.text))}));
|
|
46
|
-
const triggers=parsed.filter(({words})=>kindOf(words)==='create trigger').map(({list})=>({list,
|
|
48
|
+
const triggers=parsed.filter(({words})=>kindOf(words)==='create trigger').map(({list})=>({list,created:createdTrigger(list)})).filter(({created})=>!created || created.writes);
|
|
47
49
|
const actions=parsed.filter(({words})=>['create table','alter table'].includes(kindOf(words)) && words.some((w,i)=>w==='cascade' || (w==='set' && ['delete','update'].includes(words[i-1])))).map(({list})=>referencing(list));
|
|
48
50
|
for(let grew=true;grew;) {
|
|
49
51
|
grew=false;
|
|
50
52
|
const add=name=>{if(!reached.has(name)){reached.add(name);grew=true;}};
|
|
51
|
-
for(const {list,
|
|
53
|
+
for(const {list,created} of triggers)if(names(list,reached))(created===undefined?tableNames(list):[created.on]).forEach(add);
|
|
52
54
|
for(const {table,parents} of actions)if(table===undefined?parents.some(name=>reached.has(name)):reached.has(table))parents.forEach(add);
|
|
53
55
|
}
|
|
54
56
|
return [...reached];
|
|
@@ -115,7 +117,7 @@ function closing(parts,start) {
|
|
|
115
117
|
for(let i=start;i<parts.length;i++){if(parts[i].quoted)continue;if(parts[i].text==='(')depth++;else if(parts[i].text===')' && !--depth)return i+1;}
|
|
116
118
|
return parts.length;
|
|
117
119
|
}
|
|
118
|
-
// The table names in
|
|
120
|
+
// The table names in writing bodies of triggers on `table` that `statements`
|
|
119
121
|
// create, and every one in a trigger this parser cannot bound that names the table.
|
|
120
122
|
export function triggerBodies(statements,table) {
|
|
121
123
|
const result=[];
|
|
@@ -123,7 +125,7 @@ export function triggerBodies(statements,table) {
|
|
|
123
125
|
if(kindOf(list.map(t=>t.quoted?undefined:folded(t.text)))!=='create trigger')continue;
|
|
124
126
|
const created=createdTrigger(list);
|
|
125
127
|
if(!created){if(tableNames(list).includes(table))result.push(...tableNames(list));}
|
|
126
|
-
else if(created.on===table)result.push(...tableNames(atoms(list).slice(created.begin+1)));
|
|
128
|
+
else if(created.on===table && created.writes)result.push(...tableNames(atoms(list).slice(created.begin+1)));
|
|
127
129
|
}
|
|
128
130
|
return result;
|
|
129
131
|
}
|
|
@@ -149,9 +151,11 @@ function atoms(list) {
|
|
|
149
151
|
}
|
|
150
152
|
const semicolon=a=>!a.quoted && a.text===';';
|
|
151
153
|
const word=(a,value)=>a && !a.quoted && folded(a.text)===value;
|
|
152
|
-
// {on} for CREATE [TEMP] TRIGGER ... ON table ... BEGIN statement;
|
|
153
|
-
// nothing after it.
|
|
154
|
-
//
|
|
154
|
+
// {on,begin,writes} for CREATE [TEMP] TRIGGER ... ON table ... BEGIN statement;
|
|
155
|
+
// ... END and nothing after it. Only body DML counts as writing; WHEN reads
|
|
156
|
+
// and the header's INSERT/UPDATE/DELETE event do not. The body ends at the first
|
|
157
|
+
// END that starts a statement; CASE ... END inside a statement does not.
|
|
158
|
+
// Anything else is unbounded.
|
|
155
159
|
function createdTrigger(list) {
|
|
156
160
|
const parts=atoms(list);let depth=0,begin=-1,on;
|
|
157
161
|
for(let i=0;i<parts.length && begin<0;i++) {
|
|
@@ -170,11 +174,12 @@ function createdTrigger(list) {
|
|
|
170
174
|
}else if(!depth && word(a,'begin'))begin=i;
|
|
171
175
|
}
|
|
172
176
|
if(begin<0 || on===undefined)return;
|
|
173
|
-
let start=true,statements=0;
|
|
177
|
+
let start=true,statements=0,writes=false;
|
|
174
178
|
for(let i=begin+1;i<parts.length;i++) {
|
|
175
179
|
const a=parts[i];
|
|
176
|
-
if(start && word(a,'end'))return i===parts.length-1 && statements && !depth?{on,begin}:undefined;
|
|
180
|
+
if(start && word(a,'end'))return i===parts.length-1 && statements && !depth?{on,begin,writes}:undefined;
|
|
177
181
|
if(semicolon(a)){if(start || depth)return;start=true;statements++;continue;}
|
|
182
|
+
if(['insert','replace','update','delete'].some(verb=>word(a,verb)))writes=true;
|
|
178
183
|
if(!a.quoted){if(a.text==='(')depth++;else if(a.text===')' && --depth<0)return;}
|
|
179
184
|
start=false;
|
|
180
185
|
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { tokens, identifier, folded } from './generated.mjs';
|
|
2
|
+
import { statementNames, reachingTables } from './migration-statements.mjs';
|
|
3
|
+
|
|
4
|
+
const word=(t,w)=>t && !t.quoted && folded(t.text)===w;
|
|
5
|
+
const name=t=>t && (t.quoted?t.text[0]!=="'":/^[A-Za-z_\u0080-\uffff][\w$\u0080-\uffff]*$/.test(t.text))?folded(identifier(t)):undefined;
|
|
6
|
+
const same=(a,b)=>JSON.stringify(a)===JSON.stringify(b);
|
|
7
|
+
const lex=sql=>tokens(sql);
|
|
8
|
+
function groups(list) {
|
|
9
|
+
const result=[[]];let depth=0;
|
|
10
|
+
for(const t of list) {
|
|
11
|
+
if(t.text==='(')depth++;
|
|
12
|
+
if(t.text===')')depth--;
|
|
13
|
+
if(t.text===',' && !depth)result.push([]);else result.at(-1).push(t);
|
|
14
|
+
}
|
|
15
|
+
return result;
|
|
16
|
+
}
|
|
17
|
+
// A definition may change CHECKs, while every column declaration, other
|
|
18
|
+
// constraint, collation, default, generated expression and table option stays.
|
|
19
|
+
export function checkRebuild(before,after) {
|
|
20
|
+
const shape=sql=>{
|
|
21
|
+
const list=lex(sql),open=list.findIndex(t=>t.text==='(');let depth=0,close=-1;
|
|
22
|
+
for(let i=open;i<list.length;i++){if(list[i].text==='(')depth++;if(list[i].text===')' && !--depth){close=i;break;}}
|
|
23
|
+
if(open<0 || close<0)return;
|
|
24
|
+
const declarations=groups(list.slice(open+1,close)).map(part=>{
|
|
25
|
+
const kept=[];
|
|
26
|
+
for(let i=0;i<part.length;i++) {
|
|
27
|
+
if(word(part[i],'constraint') && word(part[i+2],'check')){i+=2;}
|
|
28
|
+
if(word(part[i],'check') && part[i+1]?.text==='(') {
|
|
29
|
+
let d=0;i++;
|
|
30
|
+
do {if(part[i].text==='(')d++;if(part[i].text===')')d--;i++;}while(i<part.length && d);
|
|
31
|
+
if(d)return ['unbounded'];
|
|
32
|
+
i--;continue;
|
|
33
|
+
}
|
|
34
|
+
kept.push(part[i]);
|
|
35
|
+
}
|
|
36
|
+
return kept;
|
|
37
|
+
}).filter(part=>part.length);
|
|
38
|
+
return [...declarations.map(part=>part.map(normal)),list.slice(close+1).map(normal)];
|
|
39
|
+
};
|
|
40
|
+
const a=shape(before),b=shape(after);
|
|
41
|
+
return a!==undefined && b!==undefined && same(a,b);
|
|
42
|
+
}
|
|
43
|
+
const normal=t=>t.quoted?(t.text[0]==="'"?t.text:folded(identifier(t))):folded(t.text);
|
|
44
|
+
function declaration(sql) {
|
|
45
|
+
const list=lex(sql);
|
|
46
|
+
if(!word(list[0],'create') || !word(list[1],'table') || !name(list[2]) || list[3]?.text!=='(')return;
|
|
47
|
+
let depth=0,close=-1;
|
|
48
|
+
for(let i=3;i<list.length;i++){if(list[i].text==='(')depth++;if(list[i].text===')' && !--depth){close=i;break;}}
|
|
49
|
+
if(close<0)return;
|
|
50
|
+
const columns=groups(list.slice(4,close)).filter(p=>!['constraint','primary','unique','check','foreign'].some(w=>word(p[0],w))).map(p=>name(p[0]));
|
|
51
|
+
if(columns.some(c=>!c))return;
|
|
52
|
+
return {name:name(list[2]),columns};
|
|
53
|
+
}
|
|
54
|
+
function select(list,columns) {
|
|
55
|
+
if(!word(list[0],'select'))return;
|
|
56
|
+
const from=list.findIndex(t=>word(t,'from'));
|
|
57
|
+
if(from<0 || from!==list.length-2 || !name(list.at(-1)))return;
|
|
58
|
+
const projection=groups(list.slice(1,from)).flatMap(part=>part.length===1 && part[0].text==='*'?columns.map(c=>({source:c,name:c})):
|
|
59
|
+
part.length===1 && name(part[0])?[{source:name(part[0]),name:name(part[0])}]:
|
|
60
|
+
part.length===3 && word(part[1],'as') && name(part[0]) && name(part[2])?[{source:name(part[0]),name:name(part[2])}]:[{source:undefined}]);
|
|
61
|
+
if(projection.some(p=>!p.source))return;
|
|
62
|
+
return {table:name(list.at(-1)),projection};
|
|
63
|
+
}
|
|
64
|
+
const drop=(sql,table)=>{const t=lex(sql);return t.length===3 && word(t[0],'drop') && word(t[1],'table') && name(t[2])===table;};
|
|
65
|
+
function copied(block,table) {
|
|
66
|
+
const [copy,removed,created,insert,cleaned]=block.map(s=>s.sql),head=lex(copy),target=declaration(created);
|
|
67
|
+
if(!target || target.name!==folded(table.name) || !word(head[0],'create') || !word(head[1],'table') || !name(head[2]) || !word(head[3],'as'))return;
|
|
68
|
+
const temporary=name(head[2]),saved=select(head.slice(4),target.columns);
|
|
69
|
+
if(!saved || saved.table!==target.name || temporary===target.name || !drop(removed,target.name) || !drop(cleaned,temporary))return;
|
|
70
|
+
const put=lex(insert);
|
|
71
|
+
if(!word(put[0],'insert') || !word(put[1],'into') || name(put[2])!==target.name)return;
|
|
72
|
+
let at=3,columns=target.columns;
|
|
73
|
+
if(put[at]?.text==='('){const end=put.findIndex((t,i)=>i>at && t.text===')');if(end<0)return;columns=groups(put.slice(at+1,end)).map(p=>p.length===1?name(p[0]):undefined);at=end+1;}
|
|
74
|
+
const restored=select(put.slice(at),saved.projection.map(p=>p.name));
|
|
75
|
+
if(!restored || restored.table!==temporary || restored.projection.length!==columns.length || new Set(columns).size!==columns.length)return;
|
|
76
|
+
const mapping=columns.map((column,i)=>({column,source:saved.projection.find(p=>p.name===restored.projection[i].source)?.source}));
|
|
77
|
+
if(!target.columns.every(c=>mapping.some(m=>m.column===c && m.source===c)))return;
|
|
78
|
+
if(!table.columns.every(c=>target.columns.includes(folded(c.name))) || table.columns.some(c=>c.generated))return;
|
|
79
|
+
// A populated archive requires an explicit rowid round trip, even for a
|
|
80
|
+
// table whose declared key makes its rowid incidental to archive identity.
|
|
81
|
+
if(table.count && !mapping.some(m=>['rowid','_rowid_','oid'].includes(m.column) && ['rowid','_rowid_','oid'].includes(m.source) && !target.columns.includes(m.column) && !target.columns.includes(m.source)))return;
|
|
82
|
+
if(mapping.some(m=>!m.source || (!target.columns.includes(m.column) && !['rowid','_rowid_','oid'].includes(m.column))))return;
|
|
83
|
+
return temporary;
|
|
84
|
+
}
|
|
85
|
+
// Admit only a complete, local copy/drop/create/copy-back/drop sequence. The
|
|
86
|
+
// manifest supplies counts; no restored rows are scanned to classify SQL.
|
|
87
|
+
// Statements outside the block still face the normal data classifier.
|
|
88
|
+
export function preservingRebuilds(later,manifest,known,statements) {
|
|
89
|
+
const admitted=new Set(),tables=new Set();
|
|
90
|
+
for(let i=0;i+4<later.length;i++) {
|
|
91
|
+
const block=later.slice(i,i+5);
|
|
92
|
+
if(block.some(s=>s.migration!==block[0].migration))continue;
|
|
93
|
+
const table=manifest.find(t=>folded(t.name)===declaration(block[2].sql)?.name);
|
|
94
|
+
if(!table)continue;
|
|
95
|
+
const temporary=copied(block,table);
|
|
96
|
+
if(!temporary || known.map(folded).includes(temporary))continue;
|
|
97
|
+
if(later.some(s=>!block.includes(s) && statementNames(s.sql).includes(temporary)))continue;
|
|
98
|
+
const foreignKeys=statements.filter(sql=>{const t=lex(sql);return (word(t[0],'create') || word(t[0],'alter')) && word(t[1],'table');});
|
|
99
|
+
if(table.count && reachingTables(foreignKeys,manifest.filter(t=>t!==table).map(t=>t.name)).includes(folded(table.name)))continue;
|
|
100
|
+
// CHECK-only changes are verified against the archived definition too;
|
|
101
|
+
// dropping/changing a column never earns a definition exemption.
|
|
102
|
+
if(!checkRebuild(table.sql,block[2].sql))continue;
|
|
103
|
+
block.forEach(s=>admitted.add(s));tables.add(folded(table.name));i+=4;
|
|
104
|
+
}
|
|
105
|
+
return {admitted,tables};
|
|
106
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { tokens, identifier, folded } from './generated.mjs';
|
|
2
|
+
import { statementNames, dataStatement } from './migration-statements.mjs';
|
|
3
|
+
|
|
4
|
+
// Resolve the latest view definitions in the applied history, rather than
|
|
5
|
+
// trusting an independently edited sqlite_master view. Nested views resolve
|
|
6
|
+
// recursively; cycles and reads of non-carried tables refuse replay.
|
|
7
|
+
export function replayViews(statements) {
|
|
8
|
+
const views=new Map();
|
|
9
|
+
for(const sql of statements) {
|
|
10
|
+
const list=tokens(sql),words=list.map(t=>t.quoted?undefined:folded(t.text));
|
|
11
|
+
if(!['create','drop'].includes(words[0]) || words[1]!=='view')continue;
|
|
12
|
+
const at=words[2]==='if'?(words[0]==='create'?5:4):2,name=folded(identifier(list[at]));
|
|
13
|
+
if(words[0]==='drop'){views.delete(name);continue;}
|
|
14
|
+
const as=list.findIndex((t,i)=>i>at && !t.quoted && folded(t.text)==='as');
|
|
15
|
+
if(as<0){views.set(name,undefined);continue;}
|
|
16
|
+
const body=sql.slice(list[as].end),data=dataStatement('INSERT INTO rer_view_probe '+body);
|
|
17
|
+
views.set(name,{names:statementNames(body),words:data?.words??[],star:data?.star});
|
|
18
|
+
}
|
|
19
|
+
return views;
|
|
20
|
+
}
|
|
21
|
+
export function viewReads(names,words,{views,tables,carried,table,secrets},seen=new Set()) {
|
|
22
|
+
const reads=[];
|
|
23
|
+
for(const name of names.filter(n=>tables.includes(n) && n!==table)) {
|
|
24
|
+
if(!views.has(name)){if(!carried.includes(name))return false;reads.push(name);continue;}
|
|
25
|
+
const view=views.get(name);
|
|
26
|
+
if(!view || seen.has(name))return false;
|
|
27
|
+
const nested=viewReads(view.names,[...words,...view.words],{views,tables,carried,table,secrets},new Set([...seen,name]));
|
|
28
|
+
if(!nested)return false;
|
|
29
|
+
// A wildcard could expose a secret column through a renamed view output.
|
|
30
|
+
if(view.star && nested.some(n=>Object.entries(secrets).some(([t,columns])=>folded(t)===n && columns.length)))return false;
|
|
31
|
+
reads.push(...nested);
|
|
32
|
+
}
|
|
33
|
+
if(reads.some(name=>Object.entries(secrets).some(([t,columns])=>folded(t)===name && columns.some(c=>words.includes(folded(c))))))return false;
|
|
34
|
+
return reads;
|
|
35
|
+
}
|
|
@@ -12,11 +12,13 @@ import { releasePublicBase } from './public-base.mjs';
|
|
|
12
12
|
import { prepareTagAdoption, confirmTagAdoption, assertTagAdoption } from './tag-adoption.mjs';
|
|
13
13
|
import { placeholderMarks, placeholderMessage } from '../../cli/placeholder-mark.mjs';
|
|
14
14
|
import { cleanupTemporary } from './cleanup.mjs';
|
|
15
|
+
import { parseTimeoutSeconds } from '../../scripts/publish-check/timeout.mjs';
|
|
15
16
|
|
|
16
17
|
// Failure output belongs to the private receipt, not the site preview.
|
|
17
18
|
const withoutOutput = gates => JSON.parse(JSON.stringify(gates, (key, value) => key === 'output' ? undefined : value));
|
|
18
19
|
|
|
19
|
-
export async function release({ repo, ref = 'HEAD', output, project, repository, dryRun = false, tag = false, cleanHistory: clean = false, envFile, onto, publicBase, localVerify = false, allowPlaceholderMark = false, verifyWorkflow }) {
|
|
20
|
+
export async function release({ repo, ref = 'HEAD', output, project, repository, dryRun = false, tag = false, cleanHistory: clean = false, envFile, onto, publicBase, localVerify = false, allowPlaceholderMark = false, verifyWorkflow, scanTimeoutSeconds }) {
|
|
21
|
+
if (scanTimeoutSeconds !== undefined) scanTimeoutSeconds = parseTimeoutSeconds(scanTimeoutSeconds, { integer: true });
|
|
20
22
|
if (!/^[\w.-]+\/[\w.-]+$/.test(project ?? '')) throw Error('--project must be publisher/name');
|
|
21
23
|
if (!/^https:\/\/github\.com\/[\w.-]+\/[\w.-]+(?:\.git)?$/i.test(repository ?? '')) throw Error('--repository must be a credential-free GitHub repository URL');
|
|
22
24
|
if (onto !== undefined && clean) throw Error('--onto and --clean-history are mutually exclusive');
|
|
@@ -60,8 +62,8 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
|
|
|
60
62
|
const exported = destination ? publicChild(source, temporary, destination, publicParent, Boolean(tag))
|
|
61
63
|
: clean ? cleanHistory(source, temporary, Boolean(tag)) : null;
|
|
62
64
|
let report = exported
|
|
63
|
-
? publicationScan(exported.repo, exported.commit, envFile, project, repository, scanBase)
|
|
64
|
-
: sourcePublicationScan(source, temporary, envFile, project, repository, scanBase, Boolean(tag), adopted);
|
|
65
|
+
? publicationScan(exported.repo, exported.commit, envFile, project, repository, scanBase, undefined, scanTimeoutSeconds)
|
|
66
|
+
: sourcePublicationScan(source, temporary, envFile, project, repository, scanBase, Boolean(tag), adopted, scanTimeoutSeconds);
|
|
65
67
|
if (destination) {
|
|
66
68
|
assertPublicTree(exported.repo, exported.commit, source.tree);
|
|
67
69
|
if (report.status === 'PASS' && (report.complete !== true || report.guarantees?.historyScanned !== true ||
|
|
@@ -17,14 +17,15 @@ export function validatePublicationDestination(source, project, repository) {
|
|
|
17
17
|
}
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
export function publicationScan(repo, ref, envFile, project, repository, publicBase, annotationFile) {
|
|
20
|
+
export function publicationScan(repo, ref, envFile, project, repository, publicBase, annotationFile, scanTimeoutSeconds) {
|
|
21
21
|
const started = performance.now();
|
|
22
22
|
let report;
|
|
23
23
|
try {
|
|
24
24
|
report = JSON.parse(execFileSync(process.execPath,
|
|
25
25
|
[resolve(import.meta.dirname, '../../scripts/publish-check.mjs'), '--repo', repo, '--ref', ref,
|
|
26
26
|
...(publicBase ? ['--public-base', publicBase] : []), ...(project != null ? ['--project', project] : []), ...(repository != null ? ['--repository', repository] : []), ...(envFile ? ['--env-file', envFile] : []),
|
|
27
|
-
...(annotationFile ? ['--annotation-file', annotationFile] : [])
|
|
27
|
+
...(annotationFile ? ['--annotation-file', annotationFile] : []),
|
|
28
|
+
...(scanTimeoutSeconds === undefined ? [] : ['--timeout-seconds', String(scanTimeoutSeconds)])],
|
|
28
29
|
{ stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 128 * 1024 * 1024 }));
|
|
29
30
|
} catch (error) {
|
|
30
31
|
try { report = JSON.parse(error.stdout); }
|
|
@@ -39,7 +40,7 @@ export function publicationScan(repo, ref, envFile, project, repository, publicB
|
|
|
39
40
|
// seed. Materialize raw committed bytes, without checkout filters or export
|
|
40
41
|
// attributes, and mirror every source ref for the existing history policy.
|
|
41
42
|
// Neither the caller's local files nor verification output enters this tree.
|
|
42
|
-
export function sourcePublicationScan(source, temporary, envFile, project, repository, publicBase, tagged, adopted) {
|
|
43
|
+
export function sourcePublicationScan(source, temporary, envFile, project, repository, publicBase, tagged, adopted, scanTimeoutSeconds) {
|
|
43
44
|
const started = performance.now();
|
|
44
45
|
const candidate = sourceTree(source, resolve(temporary, 'publication'));
|
|
45
46
|
git(source.repo, 'clone', '--mirror', '--no-hardlinks', '--', source.repo, resolve(candidate, '.git'));
|
|
@@ -61,7 +62,7 @@ export function sourcePublicationScan(source, temporary, envFile, project, repos
|
|
|
61
62
|
git(candidate, 'tag', '-d', name);
|
|
62
63
|
}
|
|
63
64
|
const stagingSeconds = (performance.now() - started) / 1000;
|
|
64
|
-
const report = publicationScan(candidate, source.commit, envFile, project, repository, publicBase, annotationFile);
|
|
65
|
+
const report = publicationScan(candidate, source.commit, envFile, project, repository, publicBase, annotationFile, scanTimeoutSeconds);
|
|
65
66
|
Object.assign(report.invocation, { stagingSeconds, totalSeconds: (performance.now() - started) / 1000 });
|
|
66
67
|
return report;
|
|
67
68
|
}
|
package/package.json
CHANGED
|
@@ -33,7 +33,12 @@ text's repository references instead of resolving them; `history_scan.py`
|
|
|
33
33
|
resolves the whole scan's references at once through `github_refs.py`
|
|
34
34
|
(GraphQL batches, a REST lookup for any repository a batch leaves in doubt) and
|
|
35
35
|
then adds their rules. The Node entry point bounds
|
|
36
|
-
the
|
|
36
|
+
the scan after slot acquisition and stops its private process group on
|
|
37
|
+
timeout/cancellation. `slots.py` obtains one per-user `fcntl.flock` slot before
|
|
38
|
+
starting the scanner. The holder watches Node's pipe for abrupt parent death;
|
|
39
|
+
its worker inherits the lock so it cannot outlive its slot. Queueing has its
|
|
40
|
+
own one-hour cap. `timeout.json` supplies the shared default and upper bound
|
|
41
|
+
to Python and the Node/release validator.
|
|
37
42
|
|
|
38
43
|
Git message records carry their kind explicitly. Only commit records enable the
|
|
39
44
|
bounded public coauthor address rule described in the command guide; a file named
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import argparse
|
|
3
3
|
import json
|
|
4
4
|
import re
|
|
5
|
+
from pathlib import Path
|
|
5
6
|
from history_scan import scan
|
|
6
7
|
from public_base import parse as parse_public_base, PublicBaseError
|
|
7
8
|
from report import new_report
|
|
@@ -14,6 +15,9 @@ def flag_name(value):
|
|
|
14
15
|
return match.group() if match else None
|
|
15
16
|
|
|
16
17
|
|
|
18
|
+
TIMEOUT = json.loads(Path(__file__).with_name('timeout.json').read_text())
|
|
19
|
+
|
|
20
|
+
|
|
17
21
|
class SafeParser(argparse.ArgumentParser):
|
|
18
22
|
def parse_args(self, args=None, namespace=None):
|
|
19
23
|
parsed, unknown = self.parse_known_args(args, namespace)
|
|
@@ -44,9 +48,9 @@ def seconds(value):
|
|
|
44
48
|
try:
|
|
45
49
|
result = float(value)
|
|
46
50
|
except ValueError:
|
|
47
|
-
raise argparse.ArgumentTypeError(
|
|
48
|
-
if not 0 < result <=
|
|
49
|
-
raise argparse.ArgumentTypeError(
|
|
51
|
+
raise argparse.ArgumentTypeError(f"invalid timeout; expected a number greater than 0 and at most {TIMEOUT['max']}") from None
|
|
52
|
+
if not 0 < result <= TIMEOUT['max']:
|
|
53
|
+
raise argparse.ArgumentTypeError(f"invalid timeout; expected a number greater than 0 and at most {TIMEOUT['max']}")
|
|
50
54
|
return result
|
|
51
55
|
|
|
52
56
|
|
|
@@ -56,7 +60,7 @@ def main():
|
|
|
56
60
|
parser.add_argument('--ref', default='HEAD')
|
|
57
61
|
for option in ('public-base', 'env-file', 'project', 'repository', 'cache-dir', 'annotation-file'):
|
|
58
62
|
parser.add_argument('--' + option)
|
|
59
|
-
parser.add_argument('--timeout-seconds', type=seconds, default=
|
|
63
|
+
parser.add_argument('--timeout-seconds', type=seconds, default=TIMEOUT['default'])
|
|
60
64
|
args = parser.parse_args()
|
|
61
65
|
if args.public_base is not None:
|
|
62
66
|
try:
|
|
@@ -38,8 +38,11 @@ def prepare(history):
|
|
|
38
38
|
# Explicit object wants cover selected commits fetched outside the
|
|
39
39
|
# remote's configured branch refspecs too. stdin keeps one batch
|
|
40
40
|
# even when a full history exceeds the command argument limit.
|
|
41
|
+
# Refetch encourages automatic repacking. Detached maintenance can
|
|
42
|
+
# outlive inspection and write into a caller's disposable repo
|
|
43
|
+
# during cleanup, so hydration must not start it.
|
|
41
44
|
history.run('fetch', '--quiet', '--refetch', '--no-filter', '--no-tags',
|
|
42
|
-
'--no-recurse-submodules', '--no-write-fetch-head', '--stdin', '--', names[0],
|
|
45
|
+
'--no-auto-maintenance', '--no-recurse-submodules', '--no-write-fetch-head', '--stdin', '--', names[0],
|
|
43
46
|
input=b'\n'.join(before) + b'\n')
|
|
44
47
|
except InspectionError:
|
|
45
48
|
raise missing_blob(history, before[0].decode()) from None
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Hold a per-user advisory scan slot until the real scan worker exits."""
|
|
2
|
+
import fcntl
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import select
|
|
6
|
+
import signal
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
import time
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def parent_closed(seconds):
|
|
13
|
+
ready, _, _ = select.select([sys.stdin], [], [], max(0, seconds))
|
|
14
|
+
return bool(ready) and not os.read(sys.stdin.fileno(), 1)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def notify(message):
|
|
18
|
+
os.write(3, (message + '\n').encode())
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def main(*, wait_seconds=3600):
|
|
22
|
+
# wait_seconds is injectable by the test interpreter, never a user knob.
|
|
23
|
+
raw = os.environ.get('RER_SCAN_SLOTS', '3')
|
|
24
|
+
if not raw.isdecimal() or not 1 <= int(raw) <= 8:
|
|
25
|
+
notify('scan.slot-configuration')
|
|
26
|
+
return 2
|
|
27
|
+
directory = Path.home() / '.cache/rer/scan-slots'
|
|
28
|
+
directory.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
29
|
+
deadline = time.monotonic() + wait_seconds
|
|
30
|
+
waiting = False
|
|
31
|
+
while True:
|
|
32
|
+
if waiting and time.monotonic() >= deadline:
|
|
33
|
+
notify('scan.slot-timeout')
|
|
34
|
+
return 2
|
|
35
|
+
for number in range(int(raw)):
|
|
36
|
+
fd = os.open(directory / f'slot-{number}', os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
|
|
37
|
+
try:
|
|
38
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
39
|
+
except BlockingIOError:
|
|
40
|
+
os.close(fd)
|
|
41
|
+
continue
|
|
42
|
+
try:
|
|
43
|
+
notify('acquired')
|
|
44
|
+
worker = subprocess.Popen([sys.executable, '-B', str(Path(__file__).with_name('cli.py')),
|
|
45
|
+
*sys.argv[1:]], stdin=subprocess.DEVNULL,
|
|
46
|
+
stderr=subprocess.DEVNULL, pass_fds=(fd,))
|
|
47
|
+
# Node's pipe closes even on SIGKILL. Stop only this private
|
|
48
|
+
# group, so an orphan never consumes CPU or holds a slot.
|
|
49
|
+
while worker.poll() is None:
|
|
50
|
+
if parent_closed(0.2):
|
|
51
|
+
os.killpg(os.getpgrp(), signal.SIGKILL)
|
|
52
|
+
return worker.returncode
|
|
53
|
+
finally:
|
|
54
|
+
os.close(fd)
|
|
55
|
+
if not waiting:
|
|
56
|
+
print('waiting for a scan slot', file=sys.stderr, flush=True)
|
|
57
|
+
waiting = True
|
|
58
|
+
remaining = deadline - time.monotonic()
|
|
59
|
+
if remaining <= 0:
|
|
60
|
+
notify('scan.slot-timeout')
|
|
61
|
+
return 2
|
|
62
|
+
if parent_closed(min(2, remaining)):
|
|
63
|
+
return 2
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
if __name__ == '__main__':
|
|
67
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"default": 300, "max": 7200}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
|
|
3
|
+
export const timeoutBounds = Object.freeze(JSON.parse(readFileSync(new URL('./timeout.json', import.meta.url), 'utf8')));
|
|
4
|
+
|
|
5
|
+
// Release requires whole seconds; publish-check also accepts fractional
|
|
6
|
+
// seconds, which its existing deadline test and standalone callers use.
|
|
7
|
+
export function parseTimeoutSeconds(value, { integer = false } = {}) {
|
|
8
|
+
const seconds = Number(value);
|
|
9
|
+
if ((integer && !/^\d+$/.test(String(value))) || !String(value).trim() ||
|
|
10
|
+
!(seconds > 0 && seconds <= timeoutBounds.max)) {
|
|
11
|
+
throw new RangeError(`Scan timeout must be ${integer ? 'integer ' : ''}seconds from ${integer ? '1' : 'above 0'} to ${timeoutBounds.max}.`);
|
|
12
|
+
}
|
|
13
|
+
return seconds;
|
|
14
|
+
}
|
|
@@ -4,12 +4,15 @@ import { mkdtempSync, readFileSync, openSync, closeSync, rmSync } from 'node:fs'
|
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
6
|
import { runningVersion } from '../cli/version.mjs';
|
|
7
|
+
import { parseTimeoutSeconds, timeoutBounds } from './publish-check/timeout.mjs';
|
|
7
8
|
|
|
8
9
|
const args = process.argv.slice(2);
|
|
9
|
-
let seconds =
|
|
10
|
+
let seconds = timeoutBounds.default;
|
|
10
11
|
for (let i = 0; i < args.length; i++) {
|
|
11
12
|
const raw = args[i] === '--timeout-seconds' ? args[i + 1] : args[i].startsWith('--timeout-seconds=') ? args[i].slice(18) : null;
|
|
12
|
-
if (raw !== null
|
|
13
|
+
if (raw !== null) {
|
|
14
|
+
try { seconds = parseTimeoutSeconds(raw); } catch { /* Python reports invalid arguments. */ }
|
|
15
|
+
}
|
|
13
16
|
}
|
|
14
17
|
const scratch = mkdtempSync(join(tmpdir(), 'publish-check-'));
|
|
15
18
|
const output = join(scratch, 'report.json');
|
|
@@ -28,11 +31,21 @@ process.on('SIGINT', interrupt);
|
|
|
28
31
|
process.on('SIGTERM', interrupt);
|
|
29
32
|
let report, code = 2;
|
|
30
33
|
try {
|
|
31
|
-
child = spawn('python3', ['-B', new URL('./publish-check/
|
|
32
|
-
detached: true, stdio: ['
|
|
34
|
+
child = spawn('python3', ['-B', new URL('./publish-check/slots.py', import.meta.url).pathname, ...args], {
|
|
35
|
+
detached: true, stdio: ['pipe', fd, 'inherit', 'pipe'], env: { ...process.env, TMPDIR: scratch },
|
|
33
36
|
});
|
|
34
37
|
closeSync(fd);
|
|
35
|
-
|
|
38
|
+
let messages = '';
|
|
39
|
+
child.stdio[3].setEncoding('utf8');
|
|
40
|
+
child.stdio[3].on('data', data => {
|
|
41
|
+
messages += data;
|
|
42
|
+
for (let at; (at = messages.indexOf('\n')) !== -1;) {
|
|
43
|
+
const message = messages.slice(0, at);
|
|
44
|
+
messages = messages.slice(at + 1);
|
|
45
|
+
if (message === 'acquired' && !timer) timer = setTimeout(() => stop('scan.deadline-exceeded'), seconds * 1000);
|
|
46
|
+
else if (message.startsWith('scan.')) reason ??= message;
|
|
47
|
+
}
|
|
48
|
+
});
|
|
36
49
|
const result = await new Promise(resolve => {
|
|
37
50
|
child.once('error', () => resolve(null));
|
|
38
51
|
child.once('close', value => resolve(value));
|