@runeditrun/rer 0.2.44 → 0.2.45

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -181,11 +181,17 @@ naming their PIDs.
181
181
 
182
182
  Servers started through `rer` (`rer dev`, `rer preview`) get a 10-server limit and a 12-hour idle timeout
183
183
  unless you set `RER_SERVER_LIMIT`, `RER_SERVER_IDLE_SECONDS` or `--idle-timeout`.
184
+ `RER_SERVER_LIMIT_FILE=PATH` names a file holding the limit as a positive integer,
185
+ read at every start; it overrides `RER_SERVER_LIMIT` and the limits running servers
186
+ recorded, so a fleet's one current value cannot be capped by an older run's
187
+ environment. An unreadable or non-integer file refuses the start, naming the file.
184
188
  `rer` supplies them only to checkouts whose pinned copy applies lower defaults
185
189
  or does not report its own (`rer describe` reports `defaults`), so a current
186
190
  checkout never records a limit you did not set.
187
191
  A start is never refused for machine load alone unless `RER_SERVERS_LOAD_THRESHOLD`
188
- or `RER_JOBS_LOAD_THRESHOLD` is set; memory and disk guards stay.
192
+ or `RER_JOBS_LOAD_THRESHOLD` is set; memory and disk guards stay. The disk guard
193
+ needs 15 GiB free; `RER_SERVERS_DISK_FLOOR_GIB=N` changes it (CI runners may set
194
+ it lower, for example 4; people should not).
189
195
 
190
196
  ## The front door
191
197
 
package/cli/README.md CHANGED
@@ -182,6 +182,13 @@ idle timeout, and they stop with `dev` (explicit stop, idle timeout, Ctrl-C or
182
182
  an application exit). `preview` does not run schedules. A seed that does not
183
183
  want this declares one adapter option: `commandAdapter(lifecycle, { scheduled: false, ... })`.
184
184
 
185
+ Without `--host`, `setup`, `dev` and `preview` on a fresh checkout bind
186
+ `127.0.0.1`; a seed declares another loopback host with the adapter option
187
+ `commandAdapter(lifecycle, { host: 'localhost', ... })` (one of `127.0.0.1`,
188
+ `localhost`, `::1`). Auth needs `localhost` because WebAuthn refuses an IP
189
+ address as its relying-party ID. A saved origin keeps its own host, and the
190
+ front door and `--host` take precedence.
191
+
185
192
  ### Local and production sign-in
186
193
 
187
194
  `setup` claims the local owner and stops its temporary runtime. `dev` prints
@@ -217,6 +224,7 @@ expiry clean up owned descendants before releasing state and capacity.
217
224
  The private registry defaults to the OS user's `~/.rer/servers` independently
218
225
  of checkout and Codex home; `RER_SERVERS_DIR` selects an isolated directory.
219
226
  The combined dev/preview cap is 10 (`RER_SERVER_LIMIT`, positive integer). A start that sets `RER_SERVER_LIMIT` is capped by that value, higher or lower, whatever running servers recorded; a start that sets nothing is capped by the lowest limit a running server's person set explicitly, and a limit recorded by a server that did not set it never lowers the cap.
227
+ `RER_SERVER_LIMIT_FILE` names a file holding one positive integer, read afresh at every start: that value is the limit for the start and overrides this process's `RER_SERVER_LIMIT` and every running server's recorded limit (servers started under the file still count). A fleet uses it so one current value, edited in the file, controls every run, including long-lived ones that still export an older `RER_SERVER_LIMIT`. A file that cannot be read or holds anything but a positive integer refuses the start, naming the file; the limit refusal names the file when it set the cap.
220
228
  Starting reservations count. At the cap a start is refused with a plain message
221
229
  naming each running server, its checkout and its stop command; rer never stops
222
230
  another checkout's server to make room, in any mode. A run that declares itself
package/cli/deploy.mjs CHANGED
@@ -178,7 +178,7 @@ function failureText({ error, step, created, again, retry, secretChange, deploye
178
178
  lines.push(`Resources created this attempt: ${created.join(', ') || 'none'}.`);
179
179
  const migration = migrationFailure(error, again);
180
180
  if (migration) lines.push(redact(migration));
181
- if (code !== 'uncertain-migration-claim') lines.push(recovery({ code, at, resource, again, retry, secretChange }));
181
+ if (code !== 'uncertain-migration-claim' && !error.unconfirmedClaim) lines.push(recovery({ code, at, resource, again, retry, secretChange }));
182
182
  return lines.join('\n');
183
183
  }
184
184
 
package/cli/local.mjs CHANGED
@@ -3,7 +3,7 @@ import { resolve, relative } from 'node:path';
3
3
  import { createInterface } from 'node:readline/promises';
4
4
  import { contained } from './project.mjs';
5
5
  import { shared } from './shared.mjs';
6
- import { origin, loopback } from './parse.mjs';
6
+ import { origin, loopback, listenHost } from './parse.mjs';
7
7
  import { refuse } from './errors.mjs';
8
8
  import { defaultPort, firstFreePort, busyPortMessage } from './port.mjs';
9
9
  import { localSettingKeys } from './settings.mjs';
@@ -22,7 +22,11 @@ export async function selectedOrigin(project, options, { setup = false } = {}) {
22
22
  // A front-door origin (saved, or --door on a fresh setup) serves through the door.
23
23
  const door = await selectDoor(project, options, previous, { vars, savedOrigin: previous?.origin });
24
24
  if (door) return door;
25
- const host = options.host ?? (previous && !doorSite(previous.origin) ? previous.hostname.replace(/^\[|\]$/g, '') : '127.0.0.1');
25
+ // A seed may declare the host a fresh checkout binds (commandAdapter({ host })),
26
+ // such as localhost for a WebAuthn RP ID; a saved origin keeps its own host.
27
+ const declared = project.adapter?.host;
28
+ if (declared !== undefined && !listenHost(declared)) throw refuse('The seed adapter declares host ' + JSON.stringify(String(declared)).slice(0, 80) + '; it must be exactly 127.0.0.1, localhost or ::1.');
29
+ const host = options.host ?? (previous && !doorSite(previous.origin) ? previous.hostname.replace(/^\[|\]$/g, '') : declared ?? '127.0.0.1');
26
30
  const savedPort = previous && !doorSite(previous.origin) ? Number(previous.port || 80) : undefined;
27
31
  // A first setup with no port asked for takes the first free one, never a busy one.
28
32
  let port = options.port ?? savedPort, taken;
package/cli/port.mjs CHANGED
@@ -4,21 +4,20 @@ import { createServer } from 'node:net';
4
4
  import { readJson } from '../modules/job-queue/storage.mjs';
5
5
  import { serversDirectory, stopInstruction } from './server-registry.mjs';
6
6
  import { shared } from './shared.mjs';
7
+ import { refuse } from './errors.mjs';
7
8
 
8
9
  export const defaultPort = 8787;
9
10
 
10
11
  // The first port from `from` that nothing listens on, on any address, and that
11
- // this host can bind. Another checkout's server may listen on every address,
12
- // which a bind to 127.0.0.1 alone does not notice.
13
- export async function firstFreePort(host, from = defaultPort, context) {
14
- const { searchContext } = await shared('process-diagnostics.mjs');
15
- const { probePort } = await shared('port-probe.mjs');
16
- context ??= searchContext({verb:'setup'});
17
- for (let port = from; port <= 65535; port++) {
18
- if ((await probePort(host, port, { context, anyAddress: true, searching: true })).available) return port;
19
- if (performance.now() >= context.deadline) throw Error('Could not find a free port in time (system busy). Run rer setup again.');
20
- }
21
- throw Error(`No free port from ${from} upward.`);
12
+ // this host can bind, among the next `searchCandidates` ports. Another
13
+ // checkout's server may listen on every address, which a bind to 127.0.0.1
14
+ // alone does not notice. Only a port in use is skipped; the search never gives
15
+ // up for time or load, only after every candidate was refused.
16
+ export async function firstFreePort(host, from = defaultPort) {
17
+ const { searchPort } = await shared('port-probe.mjs');
18
+ const found = await searchPort({ host, from, verb: 'setup' });
19
+ if (found.port !== undefined) return found.port;
20
+ throw refuse(`No free port from ${found.range[0]} to ${found.range[1]}: all ${found.tried} ports tried were in use. Stop something that holds one, or run rer setup --port PORT with a free port.`);
22
21
  }
23
22
 
24
23
  // Who holds a port: a server another checkout registered (its path and the
@@ -0,0 +1,21 @@
1
+ import { readFile } from 'node:fs/promises';
2
+ import { resolve } from 'node:path';
3
+ import { refuse } from './errors.mjs';
4
+
5
+ // A fleet's server limit as one current value its newest launcher controls.
6
+ // RER_SERVER_LIMIT_FILE names a file holding a positive integer, read afresh at
7
+ // every start; that value is the limit for the start and overrides this process's
8
+ // RER_SERVER_LIMIT and every running server's recorded limit, so a long-lived run
9
+ // that still exports an older RER_SERVER_LIMIT cannot cap the others. Unset, the
10
+ // start behaves as before. A file that cannot be read or holds anything else
11
+ // refuses the start: there is no silent fallback to another limit.
12
+ export async function serverLimitFile(env = process.env) {
13
+ if (env.RER_SERVER_LIMIT_FILE === undefined) return undefined;
14
+ const path = resolve(env.RER_SERVER_LIMIT_FILE);
15
+ let text;
16
+ try { text = await readFile(path, 'utf8'); }
17
+ catch (error) { throw refuse(`Not started: RER_SERVER_LIMIT_FILE names ${path}, which cannot be read (${error.code ?? error.message}). Create it holding a positive whole number such as 10, or unset RER_SERVER_LIMIT_FILE.`); }
18
+ const value = text.trim();
19
+ if (!/^[1-9][0-9]*$/.test(value) || !Number.isSafeInteger(Number(value))) throw refuse(`Not started: ${path} (RER_SERVER_LIMIT_FILE) must hold one positive whole number such as 10, and holds ${JSON.stringify(value.length > 40 ? `${value.slice(0, 40)}...` : value)}. Fix the file, or unset RER_SERVER_LIMIT_FILE.`);
20
+ return { path, limit: Number(value) };
21
+ }
@@ -9,6 +9,7 @@ import { identity, alive } from '../modules/job-queue/identity.mjs';
9
9
  import { transaction } from '../modules/job-queue/storage.mjs';
10
10
  import { refuse, invalid } from './errors.mjs';
11
11
  import { serverDefaults } from './contract.mjs';
12
+ import { serverLimitFile } from './server-limit-file.mjs';
12
13
  import { availableMemory } from '../modules/job-queue/resources.mjs';
13
14
 
14
15
  export function positive(value, name, fallback) {
@@ -55,11 +56,13 @@ function memoryMessage(counted,available,floor,mode) {
55
56
  const list=counted.length?`Running now:\n${counted.map(listing).join('\n')}\nA dev server holds about 0.5-1.3 GiB and a preview 0.4-0.8 GiB; stopping one with its command above frees that.`:'No dev or preview server is running, so something else is using the memory.';
56
57
  return `${head}\n${list}\nTo start anyway, lower the floor, for example RER_SERVERS_MIN_MEMORY_BYTES=${Math.floor(floor/2)} rer ${mode} (0 turns it off).`;
57
58
  }
58
- function limitMessage(counted,cap,explicit,mode) {
59
+ function limitMessage(counted,cap,explicit,mode,file) {
59
60
  const holders=explicit?[]:counted.filter(item=>item.explicit&&item.limit===cap);
60
61
  const head=`Not started: ${counted.length} dev or preview ${counted.length===1?'server is':'servers are'} running on this computer and the limit is ${cap}.`;
61
62
  const list=`Running now:\n${counted.map(listing).join('\n')}`;
62
- const raise=holders.length
63
+ const raise=file
64
+ ?`The limit is ${cap} because ${file} (RER_SERVER_LIMIT_FILE) says ${cap}; to allow more, raise the number in that file and start again.`
65
+ :holders.length
63
66
  ?`The limit is ${cap} because ${holders.length===1?'a server was':'servers were'} started with RER_SERVER_LIMIT=${cap} (${holders.map(item=>item.checkout).join(', ')}). Stop ${holders.length===1?'it':'them'} and start again, or start everything with a higher RER_SERVER_LIMIT=N.`
64
67
  :explicit?`This start set RER_SERVER_LIMIT=${cap}; to allow more, start it with a higher RER_SERVER_LIMIT=N.`
65
68
  :`To allow more, start this one with a higher limit, for example RER_SERVER_LIMIT=${cap*2} rer ${mode}. Servers started without RER_SERVER_LIMIT allow ${defaultServerLimit}.`;
@@ -78,7 +81,10 @@ function limitMessage(counted,cap,explicit,mode) {
78
81
  // running server's person set explicitly, so a deliberate "RER_SERVER_LIMIT=3"
79
82
  // keeps holding starts that do not say otherwise. A recorded default (including
80
83
  // the old default of 2 in records written before this rule) never lowers the cap
81
- // below the current default.
84
+ // below the current default. When RER_SERVER_LIMIT_FILE names a file holding a
85
+ // positive integer, that value (read at this start) is the cap and overrides both
86
+ // RER_SERVER_LIMIT and every running record's limit: records started under the
87
+ // file are counted, and the limit they recorded lowers nothing.
82
88
  export const defaultServerLimit=serverDefaults.serverLimit;
83
89
  export const serversDirectory=(env=process.env)=>resolve(env.RER_SERVERS_DIR??join((env.HOME || homedir()),'.rer/servers'));
84
90
  export async function serverRegistry(root,{env=process.env,memory=availableMemory}={}) {
@@ -140,12 +146,13 @@ export async function serverRegistry(root,{env=process.env,memory=availableMemor
140
146
  const same=state.running.find(item=>item.checkout===checkout);
141
147
  if(same)return {existing:same};
142
148
  // Journeys are listed but uncounted: they neither use nor lower the cap.
143
- const counted=state.running.filter(item=>!item.journey),cap=explicit?limit:Math.min(limit,...counted.filter(item=>item.explicit).map(item=>item.limit));
144
- if(!journey&&counted.length>=cap)throw refuse(limitMessage(counted,cap,explicit,mode));
149
+ const fileLimit=await serverLimitFile(env),counted=state.running.filter(item=>!item.journey);
150
+ const cap=fileLimit?fileLimit.limit:explicit?limit:Math.min(limit,...counted.filter(item=>item.explicit).map(item=>item.limit));
151
+ if(!journey&&counted.length>=cap)throw refuse(limitMessage(counted,cap,explicit||Boolean(fileLimit),mode,fileLimit?.path));
145
152
  // Unreadable memory is not a reason to refuse; only a reading below the floor is.
146
153
  const available=journey||floor===0?NaN:await memory();
147
154
  if(available<floor)throw refuse(memoryMessage(counted,available,floor,mode));
148
- owned={checkout,mode,journey,generation:randomUUID(),token:randomUUID(),owner:lifetime.info,limit,explicit,startedAt:Number(process.hrtime.bigint()/1000000n),idleTimeout:options.idleTimeout};
155
+ owned={checkout,mode,journey,generation:randomUUID(),token:randomUUID(),owner:lifetime.info,limit:fileLimit?fileLimit.limit:limit,explicit:explicit||Boolean(fileLimit),startedAt:Number(process.hrtime.bigint()/1000000n),idleTimeout:options.idleTimeout};
149
156
  owned.lastActivity=owned.startedAt;owned.active=0;
150
157
  const name=`rer-server-${createHash('sha256').update(owned.generation).digest('hex').slice(0,20)}.sock`;
151
158
  owned.control=join(tmpdir(),name);if(Buffer.byteLength(owned.control)>=100)owned.control=join('/tmp',name);
@@ -5,18 +5,29 @@ import {refuse} from './errors.mjs';
5
5
  // after finite preparation, while its canonical server reservation is held.
6
6
  // The 1-minute-load guard is a fleet's rule, set in the fleet's environment:
7
7
  // without RER_SERVERS_LOAD_THRESHOLD a person's start is never refused for load.
8
+ // The free-disk floor is 15 GiB; a CI or managed build runner on a smaller disk
9
+ // sets RER_SERVERS_DISK_FLOOR_GIB, in GiB with decimals allowed and 0 for no floor.
8
10
  export function servingLoadThreshold(env=process.env){
9
11
  if(env.RER_SERVERS_LOAD_THRESHOLD===undefined)return Infinity;
10
12
  const threshold=Number(env.RER_SERVERS_LOAD_THRESHOLD);
11
13
  if(!Number.isFinite(threshold)||threshold<0)throw refuse('Invalid server load threshold.');
12
14
  return threshold;
13
15
  }
16
+ const gibibyte=1024n**3n;
17
+ export function serversDiskFloor(env=process.env){
18
+ const value=env.RER_SERVERS_DISK_FLOOR_GIB;
19
+ if(value===undefined)return {gib:15,bytes:15n*gibibyte};
20
+ const gib=Number(value);
21
+ if(!/^(\d+\.?\d*|\.\d+)$/.test(value)||!Number.isFinite(gib))throw refuse('Invalid server disk floor. RER_SERVERS_DISK_FLOOR_GIB must be a number of GiB such as 4 or 2.5, 0 for no floor.');
22
+ return {gib,bytes:BigInt(Math.round(gib*Number(gibibyte)))};
23
+ }
14
24
  export async function serverResources(root,{load=()=>loadavg()[0],disk=statfs,env=process.env}={}){
15
- const threshold=servingLoadThreshold(env);
25
+ const threshold=servingLoadThreshold(env),floor=serversDiskFloor(env);
16
26
  const measured=threshold===Infinity?0:await load();
17
27
  if(!Number.isFinite(measured)||measured>threshold)throw refuse(`Not started: this computer is busy. Its 1-minute load is ${Number.isFinite(measured)?measured.toFixed(1):'unknown'} and a server starts only at ${threshold} or below.\nTry again in a few minutes, or start it anyway with a higher threshold, for example RER_SERVERS_LOAD_THRESHOLD=${Math.ceil(Math.max(threshold,Number.isFinite(measured)?measured:0)*2)} rer dev.`);
18
28
  for(const path of new Set([root,tmpdir()])){
19
29
  const state=await disk(path,{bigint:true});
20
- if(state.bavail*state.bsize<15n*1024n**3n)throw refuse('Server startup requires 15 GiB free on checkout and temporary volumes. Recover owned space before retrying.');
30
+ const free=state.bavail*state.bsize;
31
+ if(free<floor.bytes)throw refuse(`Server startup requires ${floor.gib} GiB free on checkout and temporary volumes, and ${path} has ${(Number(free*10n/gibibyte)/10).toFixed(1)} GiB free. Recover owned space before retrying.\nA CI runner on a small disk can lower the floor with RER_SERVERS_DISK_FLOOR_GIB (for example 4).`);
21
32
  }
22
33
  }
@@ -1,5 +1,14 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.2.45
4
+
5
+ - **`RER_SERVER_LIMIT_FILE` sets the server limit from a file.** A start that sees it reads the positive integer in the named file and uses that as the limit, over its own `RER_SERVER_LIMIT` and over the limits running servers recorded, in both the launcher's check and the dispatcher's registry. A fleet's launcher points every run at one file, so a long-lived run that still exports an older `RER_SERVER_LIMIT` can no longer cap the rest. An unreadable file or a non-integer refuses the start naming the file; the limit refusal names it when it set the cap. Without the variable nothing changes (#205).
6
+ - **`rer deploy` retries a migration that D1 rolled back, when a reviewed rewrite replaced its bytes.** If a hosted D1 import fails, D1 rolls it back, but rer's ledger keeps an `applying` claim on the old bytes. Before, a reviewed rewrite in `migrations/rewrites.json` covered only an `applied` row, so `rer deploy` refused with `changed-or-uncertain-migration-history`. Now, holding its deploy state lock, `rer deploy` reruns a claim whose bytes are a declared `rewrites[].from`, using the current bytes. It does so only after the same proof that none of the effects are present, and it prints `Migrations: retried <name>: the earlier attempt was rolled back`. A claim on any other bytes is still refused, and the refusal now gives the exact `wrangler d1 execute` command, filled in, and says when running it is safe. A table the migration rebuilds no longer blocks the retry. The absence check also now fits within D1's limit on compound SELECT terms (#208).
7
+ - **A seed can declare the host a fresh checkout binds.** A seed's `commandAdapter({ host })` names `127.0.0.1`, `localhost` or `::1`, and a fresh checkout binds that host. Any other value is refused. A saved origin keeps its own host, and `--host` and the front door still take precedence. Auth declares `localhost`, because WebAuthn refuses an IP address as a passkey relying-party ID. In package mode Auth no longer needs the `pnpm patch` of `@runeditrun/rer`, which `rer check` SC-13 refused as a patched pin (#204).
8
+ - **`RER_SERVERS_DISK_FLOOR_GIB` sets `rer dev`'s free-disk floor.** The value is a number of GiB (decimals allowed, 0 for no floor). Unset, the floor is 15, as before. An invalid value refuses with exit 4 and names the variable. The disk refusal states the floor it used, the free space it measured and the path it measured, and names the variable. CI or managed build runners on small disks may set it lower, for example 4 (#210).
9
+ - **The hosted demo start page prefetches the console's chunks while the visitor waits.** `rer deploy --demo` writes an ordered list of the seed's hashed `/assets` chunks into the native demo bundle. The root and console layouts come first, then the console's own views, capped at 1 MiB gzip. The start page names them as `<link rel="prefetch">`. The first visit to each console view then finds its chunks already at the edge (Mango's first-visit navigation p50 went from about 380 ms to about 90 ms on a cold colo). The start request, the sign-in redirect and the CSP are unchanged (#207).
10
+ - **`rer dev` and `rer setup` no longer give up on a free port because the computer is busy.** Before, both free-port searches stopped on a wall-clock deadline. Under heavy load, `rer dev` could stop with "Could not find a free port … in time (system busy)" even with free ports ahead. Now both try the next 1000 ports and wait for each bind's own answer. They give up only when every candidate was refused, and the message names the range and the count (#211).
11
+
3
12
  ## 0.2.44
4
13
 
5
14
  - **Under `rer dev`, a seed can link to another local seed over http.** The dependencies module accepts a provider address of `http://127.0.0.1`, `http://[::1]` or `http://localhost`, on any port, for the RER auth preset, but only when the app runs under `rer dev`. That lets a local Balm link to a local Auth. Every other `http` address, and every hosted deploy, still requires https (#196).
package/global/README.md CHANGED
@@ -181,11 +181,17 @@ naming their PIDs.
181
181
 
182
182
  Servers started through `rer` (`rer dev`, `rer preview`) get a 10-server limit and a 12-hour idle timeout
183
183
  unless you set `RER_SERVER_LIMIT`, `RER_SERVER_IDLE_SECONDS` or `--idle-timeout`.
184
+ `RER_SERVER_LIMIT_FILE=PATH` names a file holding the limit as a positive integer,
185
+ read at every start; it overrides `RER_SERVER_LIMIT` and the limits running servers
186
+ recorded, so a fleet's one current value cannot be capped by an older run's
187
+ environment. An unreadable or non-integer file refuses the start, naming the file.
184
188
  `rer` supplies them only to checkouts whose pinned copy applies lower defaults
185
189
  or does not report its own (`rer describe` reports `defaults`), so a current
186
190
  checkout never records a limit you did not set.
187
191
  A start is never refused for machine load alone unless `RER_SERVERS_LOAD_THRESHOLD`
188
- or `RER_JOBS_LOAD_THRESHOLD` is set; memory and disk guards stay.
192
+ or `RER_JOBS_LOAD_THRESHOLD` is set; memory and disk guards stay. The disk guard
193
+ needs 15 GiB free; `RER_SERVERS_DISK_FLOOR_GIB=N` changes it (CI runners may set
194
+ it lower, for example 4; people should not).
189
195
 
190
196
  ## The front door
191
197
 
@@ -8,6 +8,8 @@ import { existsSync } from 'node:fs';
8
8
  import { join } from 'node:path';
9
9
  import { pathToFileURL } from 'node:url';
10
10
  import { liveServers, stopCommand } from '../registry.mjs';
11
+ import { RerError } from '../../cli/errors.mjs';
12
+ import { serverLimitFile } from '../../cli/server-limit-file.mjs';
11
13
  import { LaunchFailure } from './failure.mjs';
12
14
 
13
15
  export const personDefaults = { limit: 10, idleSeconds: 43200 };
@@ -44,7 +46,8 @@ export function personEnvironment(env, args, pin) {
44
46
  const listing = record => ` ${record.mode} on port ${record.port ?? '(starting)'} in ${record.checkout} (PID ${record.owner?.pid})\n stop it with: ${stopCommand(record)}`;
45
47
 
46
48
  // Refuse in seconds, before any install or setup, when the pinned dispatcher
47
- // would refuse the start for the server limit. Returns this checkout's own
49
+ // would refuse the start for the server limit. RER_SERVER_LIMIT_FILE, when set,
50
+ // is the cap alone, as in the dispatcher's registry. Returns this checkout's own
48
51
  // running server when there is one.
49
52
  export async function checkCapacity(checkout, env, pin) {
50
53
  const running = await liveServers(env);
@@ -52,8 +55,9 @@ export async function checkCapacity(checkout, env, pin) {
52
55
  if (own) return { own };
53
56
  if (env.RER_SERVER_JOURNEY === '1') return {};
54
57
  const counted = running.filter(record => !record.journey);
55
- const limit = Number(env.RER_SERVER_LIMIT ?? pin.limit);
56
- const binding = pin.explicitOnly ? counted.filter(record => record.explicit) : counted;
58
+ const file = await serverLimitFile(env).catch(error => { throw error instanceof RerError ? new LaunchFailure(error.exitCode, error.message) : error; });
59
+ const limit = file ? file.limit : Number(env.RER_SERVER_LIMIT ?? pin.limit);
60
+ const binding = file ? [] : pin.explicitOnly ? counted.filter(record => record.explicit) : counted;
57
61
  const cap = Math.min(limit, ...binding.map(record => record.limit));
58
62
  if (counted.length < cap) return {};
59
63
  const holders = cap < limit ? binding.filter(record => record.limit === cap) : [];
@@ -67,6 +71,7 @@ export async function checkCapacity(checkout, env, pin) {
67
71
  throw new LaunchFailure(4, [
68
72
  `Not started: ${counted.length} dev or preview ${counted.length === 1 ? 'server is' : 'servers are'} running on this computer and the limit is ${cap}.`,
69
73
  'Running now:', ...counted.map(listing),
70
- `To make room, stop one with its command above, or start this one with a higher limit, for example RER_SERVER_LIMIT=${cap * 2} rer dev.`,
74
+ file ? `The limit is ${cap} because ${file.path} (RER_SERVER_LIMIT_FILE) says ${cap}. To make room, stop one with its command above, or raise the number in that file.`
75
+ : `To make room, stop one with its command above, or start this one with a higher limit, for example RER_SERVER_LIMIT=${cap * 2} rer dev.`,
71
76
  ].join('\n'));
72
77
  }
@@ -3,8 +3,7 @@ import { execFile } from 'node:child_process';
3
3
  import { stat } from 'node:fs/promises';
4
4
  import { join } from 'node:path';
5
5
  import { promisify } from 'node:util';
6
- import { searchContext } from '../../onboarding/process-diagnostics.mjs';
7
- import { probePort } from '../../onboarding/port-probe.mjs';
6
+ import { probePort, searchPort } from '../../onboarding/port-probe.mjs';
8
7
  import { LaunchFailure } from './failure.mjs';
9
8
 
10
9
  const run = promisify(execFile);
@@ -69,24 +68,17 @@ export async function portTaken(port) {
69
68
  return Boolean((await probePort('127.0.0.1', port, { hosts: ['127.0.0.1', '::1'], anyAddress: true })).listener);
70
69
  }
71
70
 
72
- // The first port from 8787 (RER_DEV_PORT_FROM moves the start) that is not
73
- // `held` (by a registered server, or saved by another checkout that is stopped),
74
- // nothing listens on, binds on both 127.0.0.1 and ::1 (a browser that tries
75
- // ::1 first never reaches another server) and that `claim` secures against
76
- // another launch choosing it at the same time.
71
+ // The first port from 8787 (RER_DEV_PORT_FROM moves the start), among the next
72
+ // `searchCandidates` ports, that is not `held` (by a registered server, or saved
73
+ // by another checkout that is stopped), nothing listens on, binds on both
74
+ // 127.0.0.1 and ::1 (a browser that tries ::1 first never reaches another
75
+ // server) and that `claim` secures against another launch choosing it at the
76
+ // same time. The search is bounded by that count, never by time or load.
77
77
  export async function freePort(ports, claim, env = process.env) {
78
78
  const from = Number(env.RER_DEV_PORT_FROM ?? 8787);
79
79
  if (!Number.isInteger(from) || from < 1 || from > 65535) throw new LaunchFailure(2, 'RER_DEV_PORT_FROM must be a port number.');
80
- const held = new Set(ports.filter(Boolean)), context = searchContext();
81
- for (let port = from; port <= 65535; port++) {
82
- if (held.has(port)) continue;
83
- const result = await probePort('127.0.0.1', port, { context, hosts: ['127.0.0.1', '::1'], anyAddress: true, searching: true });
84
- if (!result.available) {
85
- if (performance.now() >= context.deadline) throw new LaunchFailure(4, `Could not find a free port from ${from} in time (system busy). Run rer dev again or pass rer dev --port PORT with a free port.`);
86
- continue;
87
- }
88
- const claimed = await claim(port);
89
- if (claimed) return { port, ...claimed };
90
- }
91
- throw new LaunchFailure(4, `No free port from ${from} upward. Pass one with rer dev --port PORT.`);
80
+ const held = new Set(ports.filter(Boolean));
81
+ const found = await searchPort({ host: '127.0.0.1', from, hosts: ['127.0.0.1', '::1'], skip: port => held.has(port), claim, verb: 'dev' });
82
+ if (found.port !== undefined) return found;
83
+ throw new LaunchFailure(4, `No free port from ${found.range[0]} to ${found.range[1]}: all ${found.tried} ports tried were in use. Stop something that holds one, or pass rer dev --port PORT with a free port.`);
92
84
  }
@@ -12,6 +12,9 @@ host. See [adoption and deployment](../../export/DEMO.md#hosting-the-demo).
12
12
  outbound requests return to the same visitor, including declared secondary hosts.
13
13
  - `worker.mjs`, `pages.mjs`, `start-script.mjs`: Turnstile start, progress,
14
14
  cancellation and visitor routing. Static assets are served at the edge.
15
+ - `native/warm.mjs`: the start page's prefetch list, built from `dist/client`
16
+ and TanStack Start's route manifest, the console's views first, capped by
17
+ compressed size.
15
18
  - `capacity.mjs`: concurrent visitor admission and operator diagnostics. This
16
19
  is used by the native host as well as retained during legacy upgrades.
17
20
  - `upgrade.mjs`: verifies ownership, removes the old Container application and
@@ -40,7 +40,9 @@ export async function buildDemo(root, output, declaration) {
40
40
  await rm(output, { recursive: true, force: true });
41
41
  await mkdir(join(output, 'app'), { recursive: true });
42
42
  await cp(join(root, 'dist'), join(output, 'app/dist'), { recursive: true });
43
- await writeFile(join(output, 'app/demo.json'), JSON.stringify({ config: workerConfig, migrations }) + '\n');
43
+ // Where the console is mounted orders what the start page prefetches (native/warm.mjs).
44
+ const { operatorPath } = JSON.parse(await readFile(join(root, 'seed.json'), 'utf8').catch(() => '{}')).deploy ?? {};
45
+ await writeFile(join(output, 'app/demo.json'), JSON.stringify({ config: workerConfig, migrations, operatorPath }) + '\n');
44
46
  }
45
47
 
46
48
  if (process.argv[1] === import.meta.filename) {
@@ -1,6 +1,8 @@
1
1
  import { parseCron } from './cron.mjs';
2
2
 
3
3
  export const RUNTIME_FILES = ['config', 'worker', 'visitor', 'loader', 'd1', 'sql', 'r2', 'r2-store', 'queues', 'resources', 'cron', 'bridge', 'alarms'];
4
+ // Hashed build output: served immutable, and what the start page may prefetch.
5
+ export const HASHED_ASSET = /^\/assets\/.*-[\w-]{8,}\.[\w]+$/;
4
6
  export const ISOLATE_FILES = ['bindings', 'namespace', 'r2-client', 'd1', 'sql', 'seed-object'];
5
7
  export function nativeBindings(config) {
6
8
  for (const kind of ['kv_namespaces', 'services', 'workflows', 'ai', 'vectorize', 'hyperdrive', 'analytics_engine_datasets', 'browser', 'images', 'send_email', 'dispatch_namespaces', 'mtls_certificates', 'pipelines', 'secrets_store_secrets', 'unsafe', 'vpc_services', 'vpc_networks', 'ratelimits', 'worker_loaders', 'containers', 'ai_search_namespaces', 'ai_search', 'agent_memory', 'connect', 'artifacts', 'unsafe_hello_world', 'flagship', 'logfwdr', 'version_metadata']) {
@@ -4,6 +4,7 @@ import { createHash } from 'node:crypto';
4
4
  import { collectBuildModules } from '../../deploy-kit/build-modules.mjs';
5
5
  import { collectFiles, moduleType } from '../../deploy-kit/artifact.mjs';
6
6
  import { nativeBindings, ISOLATE_FILES, objectExports } from './config.mjs';
7
+ import { warmList } from './warm.mjs';
7
8
 
8
9
  // The Worker Loader module kind for each upload type deploy-kit gives a build
9
10
  // module, so a demo carries what a normal deploy uploads. Binary kinds travel
@@ -32,7 +33,8 @@ export async function prepareNative(folder) {
32
33
  const assets = await collectFiles(join(app, 'dist/client'));
33
34
  const bundle = { compatibilityDate: config.compatibility_date, compatibilityFlags: [...new Set([...(config.compatibility_flags ?? []), 'nodejs_compat'])], modules,
34
35
  bindings, migrations: manifest.migrations, vars: config.vars ?? {}, crons: config.triggers?.crons ?? [],
35
- assetPaths: assets.filter(file => !file.path.split('/').some(part => part.startsWith('.'))).map(file => '/' + file.path) };
36
+ assetPaths: assets.filter(file => !file.path.split('/').some(part => part.startsWith('.'))).map(file => '/' + file.path),
37
+ warm: (await warmList(assets, root, manifest.operatorPath)).paths };
36
38
  bundle.digest = createHash('sha256').update(JSON.stringify(bundle)).digest('hex');
37
39
  await cp(join(app, 'dist/client'), join(folder, 'assets'), { recursive: true });
38
40
  await writeFile(join(folder, 'modules/demo-host/native/bundle.mjs'), `export default ${JSON.stringify(bundle)};\n`);
@@ -0,0 +1,49 @@
1
+ import { readdir } from 'node:fs/promises';
2
+ import { join } from 'node:path';
3
+ import { pathToFileURL } from 'node:url';
4
+ import { gzipSync } from 'node:zlib';
5
+ import { HASHED_ASSET } from './config.mjs';
6
+
7
+ // The start page prefetches the console's hashed JS and CSS while the visitor
8
+ // waits for the human check and sign-in, so each view's first visit finds its
9
+ // chunks in the browser's cache, and the colo holds them for the next visitor.
10
+ // At most WARM_BYTES of gzip-compressed transfer, the console first: the
11
+ // root and console layout, the console's own views (what its nav links to),
12
+ // its deeper views, the other routes, then the remaining chunks.
13
+ export const WARM_BYTES = 1024 * 1024;
14
+
15
+ const segments = path => path.split('/').filter(Boolean);
16
+ const dynamic = path => segments(path).filter(part => part.startsWith('$')).length;
17
+ const within = (path, root) => path === root || path.startsWith(root.endsWith('/') ? root : root + '/');
18
+
19
+ // TanStack Start's route manifest from the server build: each route's CSS and
20
+ // the chunks it preloads.
21
+ async function startRoutes(server) {
22
+ const directory = join(server, 'assets');
23
+ const file = (await readdir(directory).catch(() => [])).find(name => /^_tanstack-start-manifest_v-[\w-]+\.js$/.test(name));
24
+ if (!file) return {};
25
+ const { tsrStartManifest } = await import(pathToFileURL(join(directory, file)));
26
+ return tsrStartManifest().routes ?? {};
27
+ }
28
+
29
+ function routeOrder(routes, consolePath) {
30
+ const ids = Object.keys(routes).filter(id => id !== '__root__');
31
+ const views = id => within(id, consolePath) && id !== consolePath;
32
+ const nav = id => views(id) && !dynamic(id) && segments(id).length === segments(consolePath).length + 1 && !id.endsWith('_');
33
+ const rank = id => [consolePath, consolePath + '/'].includes(id) ? 0 : nav(id) ? 1 : views(id) ? 2 : 3;
34
+ const sorted = ids.sort((a, b) => rank(a) - rank(b) || dynamic(a) - dynamic(b) || segments(a).length - segments(b).length || a.localeCompare(b));
35
+ return ['__root__', ...sorted].filter(id => routes[id]);
36
+ }
37
+
38
+ /** The ordered, size-capped paths the start page prefetches, from dist/client's files and the server build. */
39
+ export async function warmList(files, server, consolePath = '/app') {
40
+ const sizes = new Map(files.map(file => ['/' + file.path, file]).filter(([path]) => HASHED_ASSET.test(path) && /\.(js|css)$/.test(path))
41
+ .map(([path, file]) => [path, gzipSync(Buffer.from(file.base64, 'base64')).length]));
42
+ const routes = await startRoutes(server);
43
+ const ordered = new Set();
44
+ for (const id of routeOrder(routes, consolePath)) for (const path of [...(routes[id].css ?? []), ...(routes[id].preloads ?? [])]) if (sizes.has(path)) ordered.add(path);
45
+ for (const path of [...sizes.keys()].sort((a, b) => sizes.get(a) - sizes.get(b))) ordered.add(path);
46
+ const warm = []; let bytes = 0;
47
+ for (const path of ordered) if (bytes + sizes.get(path) <= WARM_BYTES) { warm.push(path); bytes += sizes.get(path); }
48
+ return { paths: warm, bytes, of: sizes.size };
49
+ }
@@ -1,5 +1,6 @@
1
1
  import front from '../worker.mjs';
2
2
  import { bundle } from './loader.mjs';
3
+ import { HASHED_ASSET } from './config.mjs';
3
4
  export { HostBridge } from './bridge.mjs';
4
5
  export { DemoAssets, DemoEgress } from './loader.mjs';
5
6
  export { NativeDemo } from './visitor.mjs';
@@ -13,10 +14,11 @@ export default {
13
14
  if (['GET', 'HEAD'].includes(request.method) && assets.has(url.pathname)) {
14
15
  const response = await env.ASSETS.fetch(request);
15
16
  const headers = new Headers(response.headers);
16
- if (/^\/assets\/.*-[\w-]{8,}\.[\w]+$/.test(url.pathname)) headers.set('cache-control', 'public, max-age=31536000, immutable');
17
+ if (HASHED_ASSET.test(url.pathname)) headers.set('cache-control', 'public, max-age=31536000, immutable');
17
18
  headers.set('x-rer-demo-assets', 'edge');
18
19
  return new Response(response.body, { status: response.status, headers });
19
20
  }
20
- return front.fetch(request, env, ctx);
21
+ // The start page prefetches the console's chunks while the visitor waits.
22
+ return front.fetch(request, { ...env, DEMO_WARM: bundle.warm }, ctx);
21
23
  },
22
24
  };
@@ -24,10 +24,15 @@ ol.steps{display:none;list-style:none;margin:22px auto 0;padding:0;width:max-con
24
24
  @media(min-width:600px) and (max-width:1023px){main{width:calc(100% - 64px);max-width:666px;margin:100px auto 48px}.card{padding:36px}h1{font-size:38px;line-height:46px}.intro{font-size:18px;line-height:27px}}
25
25
  @media(max-width:599px){main,.ended main{width:calc(100% - 32px);margin:32px auto}.ended .card{padding:26px 22px}.ended h1{font-size:29px;line-height:36px}.ended .intro{font-size:17px;line-height:25px;margin-bottom:28px}.card{padding:26px 22px;border-radius:16px}.brand{font-size:16px}h1{font-size:34px;line-height:41px;letter-spacing:-.8px}.intro{font-size:17px;line-height:25px}ul{font-size:16px;line-height:25px;margin:22px 0 26px}li{padding-left:22px;margin-bottom:9px}li:before{left:2px;top:9px}.button{font-size:18px;margin-top:22px}.note{font-size:13px;line-height:20px}.steps{font-size:15px}.back{margin-top:24px;font-size:14px}}`;
26
26
 
27
+ // The console's chunks, fetched at idle priority while the visitor waits for
28
+ // the check and sign-in (native/warm.mjs). The browser ignores a failed one,
29
+ // and one with prefetching turned off fetches none.
30
+ const warm = env => (env.DEMO_WARM ?? []).map(path => `<link rel="prefetch" href="${escape(path)}">`).join('');
31
+
27
32
  function html(env, { page, title, body, status = 200, headers = {}, turnstile = false, refresh = false }) {
28
33
  const nonce = crypto.randomUUID().replaceAll('-', ''), site = new URL(env.DEMO_SITE);
29
34
  const policy = `default-src 'none'; style-src 'nonce-${nonce}'; img-src 'self'; form-action 'self'; base-uri 'none'; frame-ancestors 'none'${turnstile ? `; script-src 'nonce-${nonce}' https://challenges.cloudflare.com; frame-src https://challenges.cloudflare.com; connect-src 'self' https://challenges.cloudflare.com` : ''}`;
30
- return new Response(`<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="robots" content="noindex"><title>${escape(title)}</title>${refresh ? '<noscript><meta http-equiv="refresh" content="2;url=/__demo/status"></noscript>' : ''}<style nonce="${nonce}">${STYLE}</style>${turnstile ? `<noscript><style nonce="${nonce}">#note{display:none}</style></noscript><script nonce="${nonce}">${START_SCRIPT}</script><script nonce="${nonce}" src="https://challenges.cloudflare.com/turnstile/v0/api.js" async defer></script>` : ''}</head>
35
+ return new Response(`<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="robots" content="noindex"><title>${escape(title)}</title>${refresh ? '<noscript><meta http-equiv="refresh" content="2;url=/__demo/status"></noscript>' : ''}<style nonce="${nonce}">${STYLE}</style>${turnstile ? `<noscript><style nonce="${nonce}">#note{display:none}</style></noscript><script nonce="${nonce}">${START_SCRIPT}</script><script nonce="${nonce}" src="https://challenges.cloudflare.com/turnstile/v0/api.js" async defer></script>${warm(env)}` : ''}</head>
31
36
  <body class="${page}"><main><section class="card" aria-labelledby="title"><div class="mark" aria-hidden="true"></div><p class="brand">${escape(env.DEMO_NAME)}</p>${body}</section><nav class="back" aria-label="Return"><a href="${escape(site.origin)}/">Back to ${escape(site.host)}</a></nav></main></body></html>`,
32
37
  { status, headers: { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store', 'content-security-policy': policy, 'referrer-policy': 'strict-origin', 'x-content-type-options': 'nosniff', ...headers } });
33
38
  }
@@ -203,7 +203,7 @@ No Single Redirect (dynamic redirect ruleset) permission is needed. Base's refer
203
203
 
204
204
  SQL files have ordered unique names and SHA-256 identities. `d1_migrations` holds controller history with ordered IDs, names, timestamps, SHA-256 hashes and states. Native archives already exclude this installation table. Standard Wrangler name-only inserts remain possible, but their missing hashes/states block subsequent Base deployment pending reviewed adoption. Applied entries must be an unchanged prefix of the desired set. A new migration gets an `applying` claim before its SQL and completion marker are submitted together through the [Cloudflare D1 SQL import API](https://developers.cloudflare.com/api/resources/d1/subresources/database/methods/import/), using the same whole-file protocol as Wrangler to preserve compound trigger bodies and atomic SQL/completion. Uploads use bounded HTTPS R2 destinations, without account credentials; polling is bounded and signed URLs are never diagnostic output. A failed import keeps Cloudflare's own codes and messages on the error (`providerRefusal`, which `rer deploy` prints, with credentials and URLs removed and each message cut to 300 characters) and names the migration it was running.
205
205
 
206
- An import that did not complete leaves its `applying` claim. The next deploy of the same migration bytes runs it again only when it proves none of it applied (`migration-retry.mjs`): the import is atomic, so the migration's effects that would make a second run fail (a CREATE without IF NOT EXISTS, ADD COLUMN, RENAME) and that last to the end of the file must all be absent. Then no attempt committed, and one that commits late makes the re-run fail whole instead of applying twice. A migration with no such effect, or with any of them present, stays refused as `uncertain-migration-claim`; the message names the migration, what is present, and the exact step: undo what is present and deploy again, or confirm by hand and delete or complete its claim with `wrangler d1 execute`. The kit never re-runs a migration whose effects may be present, rolls back user data, or marks an unknown outcome successful. Any other unconfirmed or changed history is still `changed-or-uncertain-migration-history`. Readback checks every final identity.
206
+ An import that did not complete leaves its `applying` claim. The next deploy of the same migration bytes runs it again only when it proves none of it applied (`migration-retry.mjs`): the import is atomic, so the migration's effects that would make a second run fail (a CREATE without IF NOT EXISTS, ADD COLUMN, RENAME) and that last to the end of the file must all be absent. Then no attempt committed, and one that commits late makes the re-run fail whole instead of applying twice. A name the file drops or renames away existed before it ran, so a table it rebuilds under the same name, and that table's indexes, are not checked. A migration with no such effect, or with any of them present, stays refused as `uncertain-migration-claim`; the message names the migration, what is present, and the exact step: undo what is present and deploy again, or confirm by hand and delete or complete its claim with `wrangler d1 execute`. The kit never re-runs a migration whose effects may be present, rolls back user data, or marks an unknown outcome successful. Any other unconfirmed or changed history is still `changed-or-uncertain-migration-history`. Readback checks every final identity.
207
207
 
208
208
  A seed that had to rewrite a migration it already released declares the rewrite
209
209
  in `migrations/rewrites.json`: `{ "version": 1, "rewrites": [{ "migration":
@@ -216,6 +216,20 @@ migration in the set, and `to` is that migration's current bytes; every other
216
216
  change to applied history is still refused. Keep the record for as long as any
217
217
  installation may hold the released digest.
218
218
 
219
+ An `applying` claim on `from`, left when D1 rolled back an import of the released
220
+ bytes, is retried with the current bytes by `rer deploy`, which runs migrations
221
+ inside its `<state file>.lock` (One run at a time), so no other deploy of that
222
+ install is running. The same proof as above must hold for the current bytes, and the
223
+ completion marker moves the claim to the current digest only as that import
224
+ commits. The deploy prints `Migrations: retried <name>: the earlier attempt was
225
+ rolled back (D1 <database>).` A caller without that lock keeps the refusal. An
226
+ `applying` claim on any other bytes stays `changed-or-uncertain-migration-history`,
227
+ and the message gives the `wrangler d1 execute ... DELETE FROM d1_migrations WHERE
228
+ name=... AND sha256=... AND state='applying'` command with the database and
229
+ migration filled in. It is safe once that attempt has ended and no other deploy of
230
+ the environment is running: the claim and the migration commit together, so a claim
231
+ still `applying` means none of it applied.
232
+
219
233
  Completion preserves the original SQL bytes and always starts on a new line.
220
234
  Comment-only migrations are rejected before any provider request or ledger claim.
221
235
  The controller adds a separator only when the final non-comment token is not a
@@ -15,8 +15,10 @@ import { verifyHealth } from './health.mjs';
15
15
  * mode creates owned resources itself first, then passes false. `uploadWithoutReceipt`
16
16
  * re-uploads an identical build when no receipt survived, instead of refusing the no-op.
17
17
  * `declaredSecrets` (default `plan.secrets`) are the secrets the fingerprint names: owner mode
18
- * passes the declaration's, so retiring the spent setup token leaves the build unchanged. */
19
- export async function deploy({api,plan,artifact,artifactSha256,previousReceipt,repeatOnly=false,createResources=!previousReceipt,uploadWithoutReceipt=false,onReceipt,onHealth,readiness={},env={},fetch=globalThis.fetch,healthOptions={},declaredSecrets=plan.secrets}) {
18
+ * passes the declaration's, so retiring the spent setup token leaves the build unchanged.
19
+ * `locked`: the caller holds the install's deploy lock (owner mode), so migrate may retry a
20
+ * claim on bytes a reviewed rewrite replaced. */
21
+ export async function deploy({api,plan,artifact,artifactSha256,previousReceipt,repeatOnly=false,createResources=!previousReceipt,uploadWithoutReceipt=false,onReceipt,onHealth,readiness={},env={},fetch=globalThis.fetch,healthOptions={},declaredSecrets=plan.secrets,locked=false}) {
20
22
  validatePlan(plan); validateArtifact(artifact);
21
23
  await verifyServiceGrants(api, plan);
22
24
  for (const item of plan.d1) validateMigrations(item.migrations);
@@ -43,7 +45,7 @@ export async function deploy({api,plan,artifact,artifactSha256,previousReceipt,r
43
45
  }
44
46
  const migrations = [];
45
47
  // A migration failure names its database, so the owner knows which one to inspect.
46
- for (const item of plan.d1) migrations.push({binding:item.binding,...await migrate(api,resources.d1.find(x => x.binding === item.binding).id,item.migrations,{apply:!unchanged,upgrade:true}).catch(error => { error.resource ??= `d1:${item.name}`; throw error; })});
48
+ for (const item of plan.d1) migrations.push({binding:item.binding,...await migrate(api,resources.d1.find(x => x.binding === item.binding).id,item.migrations,{apply:!unchanged,upgrade:true,locked}).catch(error => { error.resource ??= `d1:${item.name}`; throw error; })});
47
49
  const window = readiness.window ?? readinessWindow({...(healthOptions.now ? {clock:healthOptions.now} : {}), ...(healthOptions.sleep ? {sleep:healthOptions.sleep} : {}), ...readiness});
48
50
  let upload;
49
51
  if (!unchanged) { window.start(); upload = await uploadWorker(api,{plan,artifact,bindings,secrets,migration,fingerprint}); }
@@ -40,6 +40,14 @@ export function migrationFailure(error, again = 'rer deploy') {
40
40
  ? `Some of its changes are present (${present.join(', ')}). Undo them so the database is as it was before ${name}, then run ${again} again: rer checks again and runs it.`
41
41
  : `It creates no table, index, view, trigger or column, so rer cannot check whether it applied. If none of its changes are present, delete its claim with ${run('DELETE FROM d1_migrations')}, then run ${again} again.`} If it did apply in full, record that instead with ${run("UPDATE d1_migrations SET state='applied'")}.`;
42
42
  }
43
+ const unconfirmed = error?.unconfirmedClaim;
44
+ if (typeof unconfirmed?.name === 'string' && /^[a-f0-9]{64}$/.test(unconfirmed.sha256 ?? '')) {
45
+ const name = safe(unconfirmed.name).replaceAll(' ','?');
46
+ const run = `npx wrangler d1 execute ${database ?? '<database>'} --remote --command "DELETE FROM d1_migrations WHERE name='${name}' AND sha256='${unconfirmed.sha256}' AND state='applying'"`;
47
+ return `Migration ${name} in ${database ? `D1 ${database}` : 'its D1 database'} was started on bytes this build does not have (SHA-256 ${unconfirmed.sha256.slice(0,12)}) and never confirmed. ${unconfirmed.rewritten
48
+ ? `A reviewed rewrite in migrations/rewrites.json replaces those bytes; rer deploy retries it under its deploy lock, and this run does not hold one.`
49
+ : `No reviewed rewrite in migrations/rewrites.json replaces those bytes, so rer will not run it again.`} D1 imports a migration and its completion marker as one atomic file, so a claim still applying after that attempt ended means none of it applied. Deleting the claim is safe once that attempt has ended (it failed, or its run was stopped) and no other deploy of this environment is running: ${run}, then run ${again} again. While a deploy may still be running, leave it.`;
50
+ }
43
51
  if (typeof error?.migration === 'string') return `Migration ${safe(error.migration)} did not apply; the next ${again} runs it again once rer proves none of it applied.`;
44
52
  return undefined;
45
53
  }
@@ -44,10 +44,26 @@ export const recorded = (row, migration) => row.name === migration?.name && (row
44
44
  const validPrefix = (rows, migrations) => rows.every((row, index) => recorded(row, migrations[index]) && row.state === 'applied');
45
45
  const same = (a, b) => a.length === b.length && a.every((row, index) => row.name === b[index].name && row.sha256 === b[index].sha256 && row.state === b[index].state);
46
46
 
47
- export async function readLedger(api, id, migrations, query) {
47
+ /** History this build does not match. When it ends in an unconfirmed claim after a
48
+ * valid prefix, the error names that claim (`unconfirmedClaim`) so the refusal can
49
+ * give the exact command that clears it; `rewritten` says a reviewed rewrite
50
+ * replaced the claimed bytes and only a deploy holding the install's lock retries it. */
51
+ function unmatched(rows, migrations) {
52
+ const error = new Error('changed-or-uncertain-migration-history'), last = rows.at(-1);
53
+ if (last?.state === 'applying' && /^[a-f0-9]{64}$/.test(last.sha256 ?? '') && validPrefix(rows.slice(0, -1), migrations)) {
54
+ const next = migrations[rows.length - 1];
55
+ error.unconfirmedClaim = { name:last.name, sha256:last.sha256, rewritten:last.name === next?.name && (next.rewrites ?? []).some(rewrite => rewrite.from === last.sha256) };
56
+ }
57
+ return error;
58
+ }
59
+
60
+ /** `rewritten`: the caller holds the install's deploy lock (owner.mjs withStateLock),
61
+ * so no other deploy of it is running; a claim on bytes a reviewed rewrite replaced
62
+ * is then this build's to retry. */
63
+ export async function readLedger(api, id, migrations, query, { rewritten = false } = {}) {
48
64
  const read = async sql => (await query(api, id, sql))[0].results;
49
65
  const tables = (await read(`SELECT name FROM sqlite_master WHERE type='table' AND name IN ('${legacy}','${ledger}')`)).map(row => row.name);
50
- let old = [], applied = [], transition = '', claimed = false;
66
+ let old = [], applied = [], transition = '', claimed = false, claimedFrom;
51
67
  if (tables.includes(legacy)) {
52
68
  const columns = await read(`SELECT name FROM pragma_table_info('${legacy}')`);
53
69
  assert(columns.length === 3 && columns.every(row => ['name','sha256','state'].includes(row.name)), 'invalid-migration-ledger');
@@ -59,11 +75,15 @@ export async function readLedger(api, id, migrations, query) {
59
75
  assert(['id','name','applied_at'].every(name => columns.includes(name)), 'invalid-migration-ledger');
60
76
  if (columns.includes('sha256') && columns.includes('state')) {
61
77
  applied = await read(`SELECT name, sha256, state FROM ${ledger} ORDER BY id`);
62
- // The last row may claim this build's next migration, exact bytes, whose import
63
- // did not complete; migrate re-runs it only on proof (migration-retry.mjs).
64
- const last = applied.at(-1);
65
- if (!tables.includes(legacy) && last?.state === 'applying' && last.name === migrations[applied.length - 1]?.name && last.sha256 === migrations[applied.length - 1].sha256) claimed = Boolean(applied.pop());
66
- assert(validPrefix(applied, migrations), 'changed-or-uncertain-migration-history');
78
+ // The last row may claim this build's next migration whose import did not
79
+ // complete: its exact bytes, or (`rewritten`) bytes a reviewed rewrite replaced.
80
+ // migrate re-runs it only on proof (migration-retry.mjs).
81
+ const last = applied.at(-1), next = migrations[applied.length - 1];
82
+ if (!tables.includes(legacy) && last?.state === 'applying' && last.name === next?.name) {
83
+ if (last.sha256 === next.sha256) claimed = Boolean(applied.pop());
84
+ else if (rewritten && (next.rewrites ?? []).some(rewrite => rewrite.from === last.sha256)) { claimedFrom = last.sha256; claimed = Boolean(applied.pop()); }
85
+ }
86
+ if (!validPrefix(applied, migrations)) throw unmatched(applied, migrations);
67
87
  if (tables.includes(legacy)) assert(same(applied, old), 'changed-or-uncertain-migration-history');
68
88
  } else {
69
89
  assert(!columns.includes('sha256') && !columns.includes('state'), 'invalid-migration-ledger');
@@ -82,7 +102,7 @@ export async function readLedger(api, id, migrations, query) {
82
102
  applied = old;
83
103
  }
84
104
  if (tables.includes(legacy)) transition += `DROP TABLE ${legacy};`;
85
- return { applied, transition, claimed, exists:tables.includes(ledger) || tables.includes(legacy) };
105
+ return { applied, transition, claimed, claimedFrom, exists:tables.includes(ledger) || tables.includes(legacy) };
86
106
  }
87
107
  function insert(row) {
88
108
  return `INSERT INTO ${ledger}(name,sha256,state) VALUES (${literal(row.name)},${literal(row.sha256)},'applied');\n`;
@@ -8,6 +8,8 @@ import { literal } from './migration-ledger.mjs';
8
8
  // end of the file proves it: its effect absent now means no attempt committed, and
9
9
  // an earlier attempt that commits late makes the re-run fail whole instead of
10
10
  // applying twice. Every such effect must be absent; any present one is refused.
11
+ // A name the file drops or renames away existed before it ran, so an object it
12
+ // makes again under that name (a rebuilt table, its indexes) proves nothing.
11
13
 
12
14
  const word = token => token?.kind === 'word' ? token.value.toUpperCase() : '';
13
15
  const lower = token => token?.kind === 'word' || token?.kind === 'identifier' ? token.value.toLowerCase() : undefined;
@@ -43,11 +45,13 @@ function create(statement) {
43
45
  /** The effects of a migration whose absence proves it never applied. */
44
46
  export function retryGuards(sql) {
45
47
  let guards = [];
48
+ const before = new Set();
49
+ const guard = item => { if (!before.has(item.object) && !before.has(item.table) && !before.has(`${item.table}.${item.column}`)) guards.push(item); };
46
50
  for (const statement of statements(sql)) {
47
51
  const first = word(statement[0]);
48
52
  if (first === 'CREATE') {
49
- const guard = create(statement);
50
- if (guard) guards.push(guard);
53
+ const created = create(statement);
54
+ if (created) guard(created);
51
55
  } else if (first === 'ALTER' && word(statement[1]) === 'TABLE') {
52
56
  const { name:table, next } = named(statement, 2);
53
57
  let index = next;
@@ -55,21 +59,26 @@ export function retryGuards(sql) {
55
59
  if (word(statement[index]) === 'COLUMN') index++;
56
60
  const column = lower(statement[index]);
57
61
  if (!table) guards = [];
58
- else if (action === 'ADD' && column) guards.push({ table, column });
62
+ else if (action === 'ADD' && column) guard({ table, column });
59
63
  else if (action === 'RENAME' && word(statement[next + 1]) === 'TO') {
60
64
  const to = named(statement, next + 2).name;
61
- guards = guards.filter(guard => guard.table !== table);
62
- if (to) guards.push({ type:'table', object:to, table:to });
65
+ guards = guards.filter(item => item.table !== table);
66
+ before.add(table);
67
+ if (to) guard({ type:'table', object:to, table:to });
63
68
  } else if (action === 'RENAME' && column && word(statement[index + 1]) === 'TO' && lower(statement[index + 2])) {
64
- guards = guards.filter(guard => !(guard.table === table && guard.column === column));
65
- guards.push({ table, column:lower(statement[index + 2]) });
66
- } else if (action === 'DROP' && column) guards = guards.filter(guard => !(guard.table === table && guard.column === column));
67
- else guards = [];
69
+ guards = guards.filter(item => !(item.table === table && item.column === column));
70
+ before.add(`${table}.${column}`);
71
+ guard({ table, column:lower(statement[index + 2]) });
72
+ } else if (action === 'DROP' && column) {
73
+ guards = guards.filter(item => !(item.table === table && item.column === column));
74
+ before.add(`${table}.${column}`);
75
+ } else guards = [];
68
76
  } else if (first === 'DROP') {
69
77
  const type = word(statement[1]);
70
78
  const { name } = named(statement, word(statement[2]) === 'IF' ? 4 : 2);
71
79
  // A later drop removes the effect it would prove; an unreadable one proves nothing.
72
- guards = name ? guards.filter(guard => guard.object !== name && !(['TABLE', 'VIEW'].includes(type) && guard.table === name)) : [];
80
+ guards = name ? guards.filter(item => item.object !== name && !(['TABLE', 'VIEW'].includes(type) && item.table === name)) : [];
81
+ if (name) before.add(name);
73
82
  }
74
83
  }
75
84
  return guards;
@@ -78,14 +87,21 @@ export function retryGuards(sql) {
78
87
  const label = guard => guard.column === undefined ? `${guard.type} ${guard.object}` : `column ${guard.table}.${guard.column}`;
79
88
 
80
89
  /** Refuse unless the claimed migration's effects are provably absent: it then
81
- * may run again. `read(sql)` returns one SELECT's rows. */
90
+ * may run again. `read(sql)` returns one SELECT's rows. D1 accepts at most five
91
+ * terms in a compound SELECT, so objects and columns are each one plain query. */
82
92
  export async function assertRetryable(read, migration) {
83
93
  const guards = retryGuards(migration.sql), present = [];
84
- for (let start = 0; start < guards.length; start += 50) {
85
- const part = guards.slice(start, start + 50).map((guard, index) => guard.column === undefined
86
- ? `SELECT ${index} AS guard FROM sqlite_master WHERE type=${literal(guard.type)} AND lower(name)=${literal(guard.object)}`
87
- : `SELECT ${index} AS guard FROM pragma_table_info(${literal(guard.table)}) WHERE lower(name)=${literal(guard.column)}`);
88
- for (const row of await read(part.join(' UNION ALL '))) present.push(label(guards[start + row.guard]));
94
+ const names = list => [...new Set(list)].map(literal).join(',');
95
+ const objects = guards.filter(guard => guard.column === undefined), columns = guards.filter(guard => guard.column !== undefined);
96
+ for (let start = 0; start < objects.length; start += 100) {
97
+ const part = objects.slice(start, start + 100);
98
+ const found = new Set((await read(`SELECT type, lower(name) AS name FROM sqlite_master WHERE lower(name) IN (${names(part.map(guard => guard.object))})`)).map(row => `${row.type} ${row.name}`));
99
+ for (const guard of part) if (found.has(label(guard))) present.push(label(guard));
100
+ }
101
+ for (let start = 0; start < columns.length; start += 100) {
102
+ const part = columns.slice(start, start + 100);
103
+ const found = new Set((await read(`SELECT lower(m.name) AS tbl, lower(p.name) AS col FROM sqlite_master AS m, pragma_table_info(m.name) AS p WHERE m.type='table' AND lower(m.name) IN (${names(part.map(guard => guard.table))})`)).map(row => `column ${row.tbl}.${row.col}`));
104
+ for (const guard of part) if (found.has(label(guard))) present.push(label(guard));
89
105
  }
90
106
  if (!guards.length || present.length) throw Object.assign(new Error('uncertain-migration-claim'), { migrationClaim:{ name:migration.name, present:[...new Set(present)] } });
91
107
  }
@@ -53,9 +53,10 @@ export async function query(api, id, sql) {
53
53
  assert(Array.isArray(rows) && rows.length > 0 && rows.every(x => x.success === true && Array.isArray(x.results)), 'd1-query-failed');
54
54
  return rows;
55
55
  }
56
- export async function migrate(api, id, migrations, { apply = true, upgrade = apply } = {}) {
56
+ /** `locked`: the caller holds the install's deploy lock (owner.mjs withStateLock). */
57
+ export async function migrate(api, id, migrations, { apply = true, upgrade = apply, locked = false } = {}) {
57
58
  validateMigrations(migrations);
58
- const { applied, transition, exists, claimed } = await readLedger(api, id, migrations, query);
59
+ const { applied, transition, exists, claimed, claimedFrom } = await readLedger(api, id, migrations, query, { rewritten:locked });
59
60
  const pending = migrations.slice(applied.length);
60
61
  assert(apply || pending.length === 0,'pending-migrations');
61
62
  if (transition && upgrade) {
@@ -73,12 +74,17 @@ export async function migrate(api, id, migrations, { apply = true, upgrade = app
73
74
  // A second terminator after trailing comments creates an empty D1 statement.
74
75
  // Keep the newline even when terminated: a final line comment must end.
75
76
  const separator = last?.kind === 'symbol' && last.value === ';' ? '' : '; ';
76
- await api.importSql(id,`${migration.sql}\n${separator}UPDATE ${ledger} SET state='applied' WHERE name=${literal(migration.name)} AND sha256=${literal(migration.sha256)};`)
77
+ // A claim on bytes a reviewed rewrite replaced moves to these bytes only when this
78
+ // import commits; an earlier attempt that commits first leaves it applied on those.
79
+ const done = claimedFrom && index === 0
80
+ ? `UPDATE ${ledger} SET sha256=${literal(migration.sha256)}, state='applied' WHERE name=${literal(migration.name)} AND sha256=${literal(claimedFrom)} AND state='applying'`
81
+ : `UPDATE ${ledger} SET state='applied' WHERE name=${literal(migration.name)} AND sha256=${literal(migration.sha256)}`;
82
+ await api.importSql(id,`${migration.sql}\n${separator}${done};`)
77
83
  .catch(error => { error.migration ??= migration.name; throw error; });
78
84
  }
79
85
  if (pending.length) {
80
86
  const final = (await query(api,id,`SELECT name, sha256, state FROM ${ledger} ORDER BY id`))[0].results;
81
87
  assert(final.length === migrations.length && final.every((x,i) => x.state === 'applied' && recorded(x, migrations[i])), 'migration-readback-mismatch');
82
88
  }
83
- return { applied:pending.map(x => x.name), history:migrations.map(({name,sha256}) => ({name,sha256})) };
89
+ return { applied:pending.map(x => x.name), retried:claimed ? [pending[0].name] : [], history:migrations.map(({name,sha256}) => ({name,sha256})) };
84
90
  }
@@ -116,6 +116,7 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
116
116
  api.annotateFailure(error,error.message.startsWith('missing-secret-') ? 'secrets' : undefined);
117
117
  throw error;
118
118
  });
119
+ for (const {binding,retried = []} of receipt.migrations) for (const name of retried) notice(`Migrations: retried ${name}: the earlier attempt was rolled back (D1 ${plan.d1.find(item => item.binding === binding).name}).`);
119
120
  // Optional secrets the Worker does not hold: never supplied, or removed with --unset.
120
121
  const optionalUnset = declaredSecrets.filter(item => item?.optional && (unsetSecrets.includes(item.binding) || retiredSecrets.includes(item.binding))).map(item => item.binding);
121
122
  const result = {ok:true,changed:receipt.changed,worker:target.worker,origin:receipt.origin,versionId:receipt.provider.versionId,created,sentSecrets,...(optionalUnset.length ? {optionalUnset} : {}),retiredSecrets,...(state.ownerClaim ? {ownerClaim:state.ownerClaim} : {}),state:statePath};
@@ -415,7 +415,8 @@ async function deployLocked({api, target, plan, artifact, statePath, env, fetch,
415
415
  return result;
416
416
  }};
417
417
  const receipt = await deploy({api:observed, plan, artifact, artifactSha256:hash(artifact), previousReceipt:state.receipt ?? undefined,
418
- createResources:false, uploadWithoutReceipt:true, env, fetch, healthOptions, readiness, declaredSecrets,
418
+ // Migrations run inside this run's state lock: no other deploy of this install is running.
419
+ createResources:false, uploadWithoutReceipt:true, locked:true, env, fetch, healthOptions, readiness, declaredSecrets,
419
420
  onReceipt: async candidate => { if (candidate.changed || !state.receipt) { state.receipt = candidate; delete state.pendingUpload; delete state.pendingSecret; await save(); } },
420
421
  // Keep each first-health attempt beside the identity it belongs to.
421
422
  onHealth: async candidate => { if (candidate.changed) { state.receipt = candidate; await save(); } }});
@@ -1,6 +1,6 @@
1
1
  import { createServer } from 'node:net';
2
2
  import { setTimeout as delay } from 'node:timers/promises';
3
- import { inspectionContext, portListener, uncheckedInspection } from './process-diagnostics.mjs';
3
+ import { inspectionContext, portListener, searchContext, uncheckedInspection } from './process-diagnostics.mjs';
4
4
 
5
5
  async function bind(host, port) {
6
6
  const socket = createServer();
@@ -38,3 +38,25 @@ export async function probePort(host, port, { context = inspectionContext({ port
38
38
  await delay(Math.min(100, remaining));
39
39
  } while (true);
40
40
  }
41
+
42
+ // How many ports a free-port search tries, counting from its first one. The
43
+ // search is bounded by this count and by nothing else: a busy computer makes
44
+ // each bind slower, and the search waits for every bind's answer.
45
+ export const searchCandidates = 1000;
46
+
47
+ // The first port from `from` that probePort finds available and `claim` secures
48
+ // (claim may be omitted), among the next `searchCandidates` ports. A port in
49
+ // `skip` counts as tried and in use. Each candidate's ownership check has its
50
+ // own load-scaled budget, so no candidate can spend another's. Without a free
51
+ // port, the result says what was tried: { range: [first, last], tried }.
52
+ export async function searchPort({ host, from, hosts, skip = () => false, claim = async () => ({}), verb }) {
53
+ const last = Math.min(65535, from + searchCandidates - 1);
54
+ for (let port = from; port <= last; port++) {
55
+ if (skip(port)) continue;
56
+ const context = searchContext({ port, verb });
57
+ if (!(await probePort(host, port, { context, hosts, anyAddress: true, searching: true })).available) continue;
58
+ const claimed = await claim(port);
59
+ if (claimed) return { port, ...claimed };
60
+ }
61
+ return { range: [from, last], tried: last - from + 1 };
62
+ }
@@ -13,8 +13,9 @@ const exec=promisify(execFile);
13
13
  const lsof=process.platform==='darwin'?'/usr/sbin/lsof':'lsof';
14
14
  // One monotonic budget covers every command in an ownership decision.
15
15
  export const inspectionContext=({budgetMs=15000,...details}={})=>({deadline:performance.now()+budgetMs,budgetMs,...details});
16
- // A free-port search on a busy computer gets longer, still bounded: the budget
17
- // grows with the 1-minute load per CPU, up to four times.
16
+ // The ownership check of one free-port candidate on a busy computer gets longer,
17
+ // still bounded: the budget grows with the 1-minute load per CPU, up to four
18
+ // times. The search itself is bounded by its candidate count, not by this.
18
19
  export const searchContext=(details={})=>inspectionContext({budgetMs:Math.round(15000*Math.min(4,Math.max(1,loadavg()[0]/availableParallelism()))),...details});
19
20
  export const uncheckedInspection=error=>approvedFailure(error)?.ownershipUnchecked===true;
20
21
  const run=async(command,args,context=inspectionContext(),{maxBuffer=1024*1024,noMatch=false,positiveResult}={})=>{
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runeditrun/rer",
3
- "version": "0.2.44",
3
+ "version": "0.2.45",
4
4
  "description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
5
5
  "license": "MIT",
6
6
  "type": "module",