@runeditrun/rer 0.2.66 → 0.2.67

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -57,6 +57,30 @@ emission. HTTP errors contain only a fixed code and numeric status; network,
57
57
  parsing and unexpected errors never expose response text, URLs or key values.
58
58
  429 stops with `RATE_LIMITED`; there is no hidden retry loop.
59
59
 
60
+ ## Credentials in responses
61
+
62
+ Every adapter strips credential values from a parsed response before it emits
63
+ bytes for upload, keeping the record, by the same rules as the seed's import
64
+ door. A field whose name's letters end in a credential word (`apiKey`,
65
+ `httpPassword`, `http_password`, `client_secret`, `X-Api-Key`, `integration_key`,
66
+ `webhook_url`, `secret_key`, `x-functions-key`, `Ocp-Apim-Subscription-Key`,
67
+ `pwd` and the like) or are `X-Auth`, the `value` of a `{name|key, value}` pair
68
+ naming one (`{name: 'Authorization', value}`), or the value of such a
69
+ `[name, value]` pair in a list of them (Sentry's header tuples), becomes
70
+ `"[removed]"`. A plural name (`apiKeys`, `tokens`) counts only when it holds
71
+ text, so a count such as `tokens: 1500` stays. A URL of any scheme keeps a
72
+ marker where each credential was: its user name and password become `removed@`,
73
+ and the value of each credential query or fragment parameter (`api_key`,
74
+ `access_token`, `sig` and the like) becomes `removed`
75
+ (`?api_key=removed&region=eu`, `#access_token=removed`), except the adapter's
76
+ own validated pagination keys (`page_token`). The door drops each marker and
77
+ reports it per record as `CREDENTIAL_REMOVED`, so the owner sees which record
78
+ lost a credential. Null, booleans, empty strings and masks such as `****` are
79
+ not credentials and stay as they are. Names that merely contain such letters
80
+ (`keyword`, `monkey`, `signal`, `design`) are not credential names.
81
+ Pagination is read from the response as received, so a credential-bearing
82
+ pagination link is still refused.
83
+
60
84
  ## Inputs and native coverage
61
85
 
62
86
  * `sentry`: `organization: {id, slug, region: 'us'|'de'}`, `start`, `end`, and
@@ -75,8 +99,8 @@ parsing and unexpected errors never expose response text, URLs or key values.
75
99
  with layers/final coverage, overrides for the window and escalation policies.
76
100
  Schedule listing only discovers IDs and is not duplicated in the bundle.
77
101
  Overrides retain full boundaries with `overflow=true` and add `schedule_id`,
78
- as Oncall's documented bundle requires. Credential-bearing response fields
79
- refuse the export. Incident history and paging configuration are not fetched.
102
+ as Oncall's documented bundle requires. Incident history and paging
103
+ configuration are not fetched.
80
104
  * `calendly`: optional `asOf` timestamp (recommended for a repeatable catch-up).
81
105
  An omitted timestamp is captured at first invocation and retained on resume.
82
106
  Downloads the current user, event types, availability schedules, per-type
@@ -1,4 +1,4 @@
1
- import { object, list, id, timestamp, reference, cursor, assertCount, hash, shape, fail } from './common.mjs';
1
+ import { object, list, id, timestamp, reference, cursor, assertCount, hash, shape, fail, withoutCredentials } from './common.mjs';
2
2
 
3
3
  export const origin = 'https://api.calendly.com';
4
4
  const queries = ['user','count','event_type','min_start_time','page_token'];
@@ -66,7 +66,7 @@ export async function* run(ctx) {
66
66
  const next = s.phase === 0 ? null : nextToken(body, path, s.token);
67
67
  if (next !== null && (next === s.token || s.seen.includes(next))) fail('PAGINATION_CYCLE');
68
68
  const name = `${resource}-${String(s.files.length + 1).padStart(6,'0')}.json`;
69
- const bytes = Buffer.from(JSON.stringify(body) + '\n');
69
+ const bytes = Buffer.from(JSON.stringify(withoutCredentials(body, queries)) + '\n');
70
70
  if (s.files.length >= 600) fail('FILE_LIMIT');
71
71
  s.files.push({ name, resource, page: s.seen.length + 1, size: bytes.length, sha256: hash(bytes), capturedAt: s.asOf });
72
72
  s.token = next;
@@ -46,6 +46,79 @@ export function reference(value, origin, pathname, allowedQuery = []) {
46
46
  if ([...url.searchParams.keys()].some(key => !allowedQuery.includes(key))) fail('INVALID_PAGINATION');
47
47
  return url;
48
48
  }
49
+ // The seed's import door applies the same rules (modules/import-kit, removeCredentials).
50
+ // A field holds a credential when its name's letters end in a credential word
51
+ // (apiKey, httpPassword, http_password, client_secret, X-Api-Key, integration_key,
52
+ // webhook_url, secret_key, x-functions-key, Ocp-Apim-Subscription-Key, db_pwd) or
53
+ // are X-Auth, unless it is empty: null, a boolean (Calendly's `secret` flag),
54
+ // whitespace or a mask such as "****" or "[redacted]". A plural (apiKeys, tokens)
55
+ // holds credentials only as text, so a count such as `tokens: 1500` stays.
56
+ const credentialName = /(?:authorization|credentials?|webhookurl|calurl)$|^xauth$/u;
57
+ const credentialWord = /(?:password|passwd|pwd|secret|token|cookie|(?:api|access|private|auth|client|integration|routing|service|signing|session|refresh|secret|subscription|functions)key)(s?)$/u;
58
+ const placeholder = /^(?:\s*|[*\u2022\u25CF\u00B7x#_.-]{1,64}|[<[({]?\s*(?:redacted|hidden|masked|removed|omitted|none|null|n\/a|placeholder)\s*[>\])}]?)$/iu;
59
+ /** What a stripped credential value becomes; the seed's import door reports it per record. */
60
+ export const REMOVED_CREDENTIAL = '[removed]';
61
+ /** What a stripped credential query or fragment parameter's value becomes
62
+ * (`?api_key=removed`), as a URL's user name and password become `removed@`; the
63
+ * door drops the pair and reports it per record. */
64
+ export const REMOVED_PARAMETER = 'removed';
65
+ const empty = value => value === null || typeof value === 'boolean' || (typeof value === 'string' && placeholder.test(value));
66
+ // Text as a member (`['…']`, `{github: '…'}`); a list of objects is checked field by field.
67
+ const text = value => typeof value === 'string' ? !placeholder.test(value) : !!value && typeof value === 'object' && Object.values(value).some(child => (typeof child === 'string' || Array.isArray(child)) && text(child));
68
+ /** Every percent-encoding layer of a key, decoded byte by byte, as the door inspects it. */
69
+ function decoded(text) {
70
+ for (let depth = 0; depth < 32 && /%[\da-f]{2}/iu.test(text); depth++) text = text.replace(/%([\da-f]{2})/giu, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
71
+ return text;
72
+ }
73
+ /** Whether a value under this name is a credential, by the door's rule. */
74
+ function credential(name, value) {
75
+ const letters = decoded(name).toLowerCase().replace(/[^a-z]/gu, ''), word = credentialWord.exec(letters);
76
+ if (!word && !credentialName.test(letters)) return false;
77
+ return !empty(value) && (!word?.[1] || text(value));
78
+ }
79
+ // The door's credentialQueryKey rule for URL query and fragment parameter names;
80
+ // `sig` (an Azure SAS signature) only as the whole name.
81
+ const credentialParameter = key => /token|secret|key|auth|signature|credential|password|cookie/iu.test(decoded(key)) || /^\s*sig\s*$/iu.test(decoded(key));
82
+ const authority = /^(\s*(?:[a-z][a-z\d+.-]*:)?[\\/]{2})[^\\/?#]*@/iu;
83
+ /** A URL of any scheme with its user name and password replaced by `removed@`,
84
+ * and the value of each credential query or fragment parameter by `removed`,
85
+ * except the adapter's own validated pagination keys, spelled exactly. */
86
+ function withoutUrlCredentials(text, pagination) {
87
+ if (!/^\s*(?:[a-z][a-z\d+.-]*:)?[\\/]{2}|^[/?]/iu.test(text)) return text;
88
+ const result = text.replace(authority, '$1removed@');
89
+ const mark = pairs => pairs.split('&').map(pair => {
90
+ const at = pair.indexOf('='), key = at < 0 ? pair : pair.slice(0, at);
91
+ return at < 0 || pagination.includes(key) || !credentialParameter(key) ? pair : `${key}=${REMOVED_PARAMETER}`;
92
+ }).join('&');
93
+ const hashAt = result.indexOf('#'), end = hashAt < 0 ? result.length : hashAt;
94
+ const fragment = hashAt < 0 ? '' : result.slice(hashAt);
95
+ const marked = fragment.includes('=') ? '#' + mark(fragment.slice(1)) : fragment;
96
+ const queryAt = result.indexOf('?');
97
+ if (queryAt < 0 || queryAt > end) return result.slice(0, end) + marked;
98
+ return result.slice(0, queryAt + 1) + mark(result.slice(queryAt + 1, end)) + marked;
99
+ }
100
+ /** The `[name, value]` pairs of a list holding only such pairs (Sentry's header tuples). */
101
+ const tuples = list => list.length > 0 && list.every(item => Array.isArray(item) && item.length === 2 && typeof item[0] === 'string');
102
+ /** A copy of a parsed vendor response without credential values, made before any
103
+ * byte is emitted for upload: a credential-named field's value, the `value` of a
104
+ * `{name|key, value}` pair naming one (a header pair), or the value of a
105
+ * credential-named `[name, value]` pair in a list of them, becomes
106
+ * REMOVED_CREDENTIAL, so the record stays and the seed can say what was removed.
107
+ * URLs lose credentials as withoutUrlCredentials describes; `pagination` lists
108
+ * the adapter's validated query keys, which stay. */
109
+ export function withoutCredentials(value, pagination = []) {
110
+ if (Array.isArray(value)) {
111
+ const pairs = tuples(value);
112
+ return value.map(child => pairs && credential(child[0], child[1]) ? [child[0], REMOVED_CREDENTIAL] : withoutCredentials(child, pagination));
113
+ }
114
+ if (value && typeof value === 'object') {
115
+ const label = Object.entries(value).find(([key, child]) => /^(?:name|key)$/iu.test(key) && typeof child === 'string');
116
+ const pair = !!label && Object.hasOwn(value, 'value');
117
+ return Object.fromEntries(Object.entries(value).map(([key, child]) =>
118
+ [key, credential(pair && key === 'value' ? label[1] : key, child) ? REMOVED_CREDENTIAL : withoutCredentials(child, pagination)]));
119
+ }
120
+ return typeof value === 'string' ? withoutUrlCredentials(value, pagination) : value;
121
+ }
49
122
  export function assertCount(value, max = 1000) {
50
123
  if (!Number.isSafeInteger(value) || value < 0 || value > max) fail('INVALID_CHECKPOINT');
51
124
  return value;
@@ -1,15 +1,7 @@
1
- import { object, list, id, windowInputs, assertCount, shape, fail } from './common.mjs';
1
+ import { object, list, id, windowInputs, assertCount, shape, fail, withoutCredentials } from './common.mjs';
2
2
 
3
3
  export const origin = 'https://api.pagerduty.com';
4
4
  export const route = url => /^\/(?:users|escalation_policies|schedules(?:\/[a-zA-Z0-9_-]+(?:\/overrides)?)?)$/u.test(url.pathname) && [...url.searchParams.keys()].every(x => ['limit','offset','since','until','overflow'].includes(x));
5
- const credential = /^(?:password|passwd|secret|token|credentials?|authorization|cookie)$|(?:api|access|auth|integration|routing|service|signing|private|client|refresh|session)[_-]?(?:key|secret|token)$|^webhook[_-]?(?:url|secret)$|(?:http|web)[_-]?cal[_-]?url$/iu;
6
- function noCredentials(value) {
7
- if (Array.isArray(value)) value.forEach(noCredentials);
8
- else if (value && typeof value === 'object') for (const [key, child] of Object.entries(value)) {
9
- if (credential.test(key)) fail('CREDENTIAL_IN_RESPONSE');
10
- noCredentials(child);
11
- }
12
- }
13
5
  function advance(body, state) {
14
6
  if (typeof body.more !== 'boolean') fail();
15
7
  if (body.more) {
@@ -40,8 +32,7 @@ export async function* run(ctx) {
40
32
  if (s.schedules.length > 500) fail('RECORD_LIMIT');
41
33
  bytes = '';
42
34
  } else {
43
- noCredentials(body);
44
- bytes = (s.first ? (s.phase === 0 ? '{"users":[' : '],"escalation_policies":[') : ',') + JSON.stringify(body);
35
+ bytes = (s.first ? (s.phase === 0 ? '{"users":[' : '],"escalation_policies":[') : ',') + JSON.stringify(withoutCredentials(body));
45
36
  s.first = false;
46
37
  }
47
38
  if (!advance(body, s)) { s.phase++; s.first = true; }
@@ -49,8 +40,8 @@ export async function* run(ctx) {
49
40
  if (s.index === s.schedules.length) { s.phase = 3; s.index = 0; s.first = true; bytes = s.schedules.length ? '' : '],"schedules":['; }
50
41
  else {
51
42
  const { body } = await ctx.get(`/schedules/${id(s.schedules[s.index])}?${window}`);
52
- object(body); object(body.schedule); noCredentials(body); ctx.records(1);
53
- bytes = (s.first ? '],"schedules":[' : ',') + JSON.stringify(body);
43
+ object(body); object(body.schedule); ctx.records(1);
44
+ bytes = (s.first ? '],"schedules":[' : ',') + JSON.stringify(withoutCredentials(body));
54
45
  s.first = false; s.index++;
55
46
  }
56
47
  } else {
@@ -58,8 +49,8 @@ export async function* run(ctx) {
58
49
  else {
59
50
  const schedule = id(s.schedules[s.index]);
60
51
  const { body } = await ctx.get(`/schedules/${schedule}/overrides?${window}&overflow=true`);
61
- object(body); ctx.records(list(body.overrides).length); noCredentials(body);
62
- bytes = (s.first ? '],"overrides":[' : ',') + JSON.stringify({ ...body, schedule_id: schedule });
52
+ object(body); ctx.records(list(body.overrides).length);
53
+ bytes = (s.first ? '],"overrides":[' : ',') + JSON.stringify(withoutCredentials({ ...body, schedule_id: schedule }));
63
54
  s.first = false;
64
55
  s.index++;
65
56
  }
@@ -1,7 +1,8 @@
1
- import { object, list, id, windowInputs, cursor, assertCount, shape, fail } from './common.mjs';
1
+ import { object, list, id, windowInputs, cursor, assertCount, shape, fail, withoutCredentials } from './common.mjs';
2
2
 
3
3
  export const origins = Object.freeze({ us: 'https://us.sentry.io', de: 'https://de.sentry.io' });
4
- export const route = url => /^\/api\/0\/(?:organizations\/[a-zA-Z0-9_-]+\/(?:projects|releases|issues)|projects\/[a-zA-Z0-9_-]+\/[a-zA-Z0-9_-]+\/events\/[a-fA-F0-9]{32})\/$/u.test(url.pathname) && [...url.searchParams.keys()].every(x => ['per_page','limit','cursor','query','project','start','end','sort'].includes(x));
4
+ const queries = ['per_page','limit','cursor','query','project','start','end','sort'];
5
+ export const route = url => /^\/api\/0\/(?:organizations\/[a-zA-Z0-9_-]+\/(?:projects|releases|issues)|projects\/[a-zA-Z0-9_-]+\/[a-zA-Z0-9_-]+\/events\/[a-fA-F0-9]{32})\/$/u.test(url.pathname) && [...url.searchParams.keys()].every(x => queries.includes(x));
5
6
  function nextCursor(headers) {
6
7
  const link = headers.get('link');
7
8
  if (!link) return null;
@@ -73,7 +74,7 @@ export async function* run(ctx) {
73
74
  if (next !== null && (next === s.cursor || s.seen.includes(next))) fail('PAGINATION_CYCLE');
74
75
  if (++s.page > 100) fail('PAGE_LIMIT');
75
76
  let bytes = s.cursor === null ? (s.collections ? ',' : '') + JSON.stringify({kind,project}).slice(0,-1) + ',"pages":[' : ',';
76
- bytes += JSON.stringify({ cursor: s.cursor, nextCursor: next, response });
77
+ bytes += JSON.stringify({ cursor: s.cursor, nextCursor: next, response: withoutCredentials(response, queries) });
77
78
  s.cursor = next;
78
79
  if (next === null) {
79
80
  bytes += ']}'; s.collections++; s.seen = []; s.page = 0; s.event = 0;
@@ -1,20 +1,21 @@
1
- import { object, list, reference, shape, fail } from './common.mjs';
1
+ import { object, list, reference, shape, fail, withoutCredentials } from './common.mjs';
2
2
 
3
3
  export const origin = 'https://api.uptimerobot.com';
4
- export const route = url => url.pathname === '/v3/monitors' && [...url.searchParams.keys()].every(x => ['limit','offset','cursor','page','page_token'].includes(x));
4
+ const pagination = ['limit','offset','cursor','page','page_token'];
5
+ export const route = url => url.pathname === '/v3/monitors' && [...url.searchParams.keys()].every(x => pagination.includes(x));
5
6
  export async function* run(ctx) {
6
7
  const state = ctx.state ?? { next: '/v3/monitors?limit=200', seen: [] };
7
8
  shape(state, ['next','seen']);
8
9
  if (!Array.isArray(state.seen) || state.seen.length > 600) fail('INVALID_CHECKPOINT');
9
- state.seen.forEach(value => reference(value, origin, /^\/v3\/monitors$/u, ['limit','offset','cursor','page','page_token']));
10
+ state.seen.forEach(value => reference(value, origin, /^\/v3\/monitors$/u, pagination));
10
11
  while (state.next !== null) {
11
- const url = reference(state.next, origin, /^\/v3\/monitors$/u, ['limit','offset','cursor','page','page_token']);
12
+ const url = reference(state.next, origin, /^\/v3\/monitors$/u, pagination);
12
13
  if (state.seen.includes(url.href)) fail('PAGINATION_CYCLE');
13
14
  const { body } = await ctx.get(url.href);
14
15
  object(body); ctx.records(list(body.data).length);
15
16
  if (!Object.hasOwn(body, 'nextLink')) fail();
16
17
  state.seen.push(url.href);
17
- state.next = body.nextLink == null ? null : reference(body.nextLink, origin, /^\/v3\/monitors$/u, ['limit','offset','cursor','page','page_token']).href;
18
- yield ctx.item(JSON.stringify(body) + '\n', state);
18
+ state.next = body.nextLink == null ? null : reference(body.nextLink, origin, /^\/v3\/monitors$/u, pagination).href;
19
+ yield ctx.item(JSON.stringify(withoutCredentials(body, pagination)) + '\n', state);
19
20
  }
20
21
  }
@@ -49,11 +49,14 @@ export async function productionMail(root, declaration, env) {
49
49
  const routes = selected.map(provider => {
50
50
  const credentialKeys = deferred && nativeCredential[provider] && !declaredProviders.has(provider) ? [nativeCredential[provider]]
51
51
  : ['MAIL_API_KEY', ...(nativeCredential[provider] ? [nativeCredential[provider]] : [])];
52
+ // A seed's direct-provider adapter can declare the contract's shared provider
53
+ // binding. Its explicit environment mapping (or live declared secret) still wins.
54
+ if (nativeCredential[provider] && secrets.has('MAIL_PROVIDER_API_KEY')) credentialKeys.push('MAIL_PROVIDER_API_KEY');
52
55
  const declaredCredentials = credentialKeys.filter(key => secrets.has(key));
53
56
  // Only resend and sendgrid have a provider key of their own; cloudflare and rer send with MAIL_API_KEY alone.
54
57
  const providerKeys = (nativeCredential[provider] ? [nativeCredential[provider]] : []).filter(key => !declaredCredentials.includes(key));
55
58
  const credentialText = declaredCredentials.includes('MAIL_API_KEY') && providerKeys.length
56
- ? `MAIL_API_KEY (or ${providerKeys.join(' / ')})`
59
+ ? `${declaredCredentials.join(' or ')} (or ${providerKeys.join(' / ')})`
57
60
  : declaredCredentials.join(' or ') || credentialKeys.join(' or ');
58
61
  return live => {
59
62
  let relaySelected = true;
@@ -1,5 +1,11 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.2.67
4
+
5
+ - **Import door options (#298 with #308).** Seeds choose credential removal modes and a preserve or strict reversal policy when an import is created. Failed applies can be retried, CLI-stripped query and fragment credentials receive per-record notes, more credential shapes are recognized, and a blocked re-baseline is refused at plan time with a named 409.
6
+ - **Release from partial clones (#313).** Hydrate missing history blobs before release reads and tolerate temporary-directory cleanup failures after a saved result.
7
+ - **Mail credentials and large migration histories (#314).** Deploy honors a declared MAIL_PROVIDER_API_KEY for SendGrid and Resend, and scales the bounded provider request budget to pending migrations.
8
+
3
9
  ## 0.2.66
4
10
 
5
11
  - **Seed copies pass the pin-time credential scan (#305).** Credential-shaped test fixtures in seed-vendored Base files are constructed at runtime, preserving their values and assertions. Seed copies and exported references no longer trigger the hosted pin-time credential scan, so seeds can refresh Base past 0.2.61 and pin again.
@@ -2,6 +2,8 @@ import { createHash } from 'node:crypto';
2
2
  import { assert, uuid } from './validation.mjs';
3
3
  import { recordDeployFailure } from './diagnostics.mjs';
4
4
 
5
+ export const D1_IMPORT_POLL_LIMIT = 100;
6
+
5
7
  // Wrangler's remote --file protocol. D1 imports execute the whole file atomically,
6
8
  // including trigger bodies; the query endpoint's semicolon splitting cannot.
7
9
  // Cloudflare's own reason for an import that failed or did not finish, kept on the
@@ -31,7 +33,7 @@ export async function importD1({ envelope, upload, redact = String }, id, sql) {
31
33
  await upload(response.upload_url, bytes, etag);
32
34
  response = await post({ action:'ingest', filename:response.filename, etag });
33
35
  }
34
- for (let polls = 0; polls < 100; polls++) {
36
+ for (let polls = 0; polls < D1_IMPORT_POLL_LIMIT; polls++) {
35
37
  // REST places errors on the envelope; validate nested errors when supplied.
36
38
  if (!(response?.success === true && response.type === 'import' && response.status !== 'error' && (!Object.hasOwn(response, 'errors') || Array.isArray(response.errors) && !response.errors.length))) throw importFailure('d1-import-failed', id, response, redact);
37
39
  if (response.status === 'complete') {
@@ -58,6 +58,7 @@ export async function migrate(api, id, migrations, { apply = true, upgrade = app
58
58
  validateMigrations(migrations);
59
59
  const { applied, pending, transition, exists, claimed, claimedFrom } = await readLedger(api, id, migrations, query, { rewritten:locked });
60
60
  assert(apply || pending.length === 0,'pending-migrations');
61
+ if (apply && pending.length) api.reserveMigrationRequests?.(id, pending.length);
61
62
  if (transition && upgrade) {
62
63
  await api.importSql(id, transition);
63
64
  const verified = await readLedger(api, id, migrations, query);
@@ -64,7 +64,7 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
64
64
  const deadline = AbortSignal.timeout(Math.max(1, Math.ceil(remaining)));
65
65
  return request(input, {...init,signal:init.signal ? AbortSignal.any([init.signal,deadline]) : deadline});
66
66
  };
67
- const api = deploymentDiagnostics(cloudflare({accountId:env.CLOUDFLARE_ACCOUNT_ID,token:env.CLOUDFLARE_API_TOKEN,fetch,...(command === 'remove' ? {maxRequests:Infinity,removal:true} : {})}),{onCreated});
67
+ const api = deploymentDiagnostics(cloudflare({accountId:env.CLOUDFLARE_ACCOUNT_ID,token:env.CLOUDFLARE_API_TOKEN,fetch,budgetMigrations:command === 'deploy',...(command === 'remove' ? {maxRequests:Infinity,removal:true} : {})}),{onCreated});
68
68
  // Removal needs the recorded account/name identity, not a workers.dev route.
69
69
  const target = command === 'remove'
70
70
  ? {accountId:api.accountId,project:declaration.project,environment:values.environment,worker:ownerWorkerName(declaration.project,values.environment)}
@@ -2,8 +2,8 @@ import { providerPages, providerListShape } from './pagination.mjs';
2
2
  import { deleteR2Object } from './r2-object-transport.mjs';
3
3
  import { isR2ObjectDelete } from './r2-objects.mjs';
4
4
  import { recordDeployFailure } from './diagnostics.mjs';
5
- import { importD1, importUploadURL } from './d1-import.mjs';
6
- import { assert, hexId } from './validation.mjs';
5
+ import { D1_IMPORT_POLL_LIMIT, importD1, importUploadURL } from './d1-import.mjs';
6
+ import { assert, hexId, uuid } from './validation.mjs';
7
7
  import { providerResponse, transportOutput, transportReason } from './provider-transport.mjs';
8
8
 
9
9
  // A gateway/server error can arrive after a write. An explicit API refusal cannot.
@@ -28,10 +28,22 @@ export async function boundedJSON(response, limit) {
28
28
  }
29
29
  const redactValues = (text, values) => values.filter(Boolean).flatMap(value=>[value,encodeURIComponent(value),JSON.stringify(value).slice(1,-1)]).sort((a,b)=>b.length-a.length).reduce((value,secret)=>value.split(secret).join('[REDACTED]'),text);
30
30
  /** Only this fixed origin and the explicitly selected account ever receive the token. */
31
- export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, objectDelete = deleteR2Object, timeoutMs = 30000, maxRequests = 500, removal = false }) {
31
+ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, objectDelete = deleteR2Object, timeoutMs = 30000, maxRequests = 500, removal = false, budgetMigrations = false }) {
32
32
  assert(hexId(accountId), 'invalid-cloudflare-account-id');
33
33
  assert(typeof token === 'string' && token.trim() && !/[\r\n]/.test(token), 'invalid-cloudflare-api-token');
34
34
  let requests = 0;
35
+ const migrationBudgets = new Set();
36
+ function reserveMigrationRequests(id, pending) {
37
+ if (!budgetMigrations || migrationBudgets.has(id)) return;
38
+ assert(uuid(id) && Number.isSafeInteger(pending) && pending >= 0, 'invalid-provider-request-budget');
39
+ // Each pending migration can claim, initialize, upload and ingest, then use
40
+ // the bounded import poll protocol. Metadata and transport retries share the
41
+ // original allowance. Reserve validated work only once per database/run.
42
+ const limit = maxRequests + pending * (4 + D1_IMPORT_POLL_LIMIT);
43
+ assert(Number.isSafeInteger(limit), 'invalid-provider-request-budget');
44
+ migrationBudgets.add(id);
45
+ maxRequests = limit;
46
+ }
35
47
  // `zone` scope: the zone-level API (/zones/...), for a run site's zone lookup and redirect rule.
36
48
  async function rawEnvelope(path, { method = 'GET', body, assetToken, zone = false, signal: callerSignal } = {}) {
37
49
  assert(typeof path === 'string' && (isR2ObjectDelete(path, method) || /^\/[a-z0-9/_?=&.,%-]+$/i.test(path) && !path.includes('..') && !path.split('?')[0].includes('%')), 'invalid-provider-path');
@@ -147,6 +159,6 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
147
159
  assert(response.status === 200, 'd1-import-upload-failed');
148
160
  assert(response.headers.get('etag')?.replace(/^"|"$/g, '') === etag, 'd1-import-upload-etag-mismatch');
149
161
  }
150
- return { accountId, envelope, list, importSql(id, sql) { return importD1({envelope,upload,redact:text => redactValues(text,[accountId,token])}, id, sql); }, request:requestResult, workerContent,
162
+ return { accountId, envelope, list, reserveMigrationRequests, importSql(id, sql) { return importD1({envelope,upload,redact:text => redactValues(text,[accountId,token])}, id, sql); }, request:requestResult, workerContent,
151
163
  zoneRequest(path, options = {}) { return requestResult(path, { ...options, zone: true }); } };
152
164
  }
@@ -15,7 +15,12 @@ pnpm rer release --repo /path/to/seed --ref main \
15
15
 
16
16
  The `rer` bin and `node bin/rer.mjs` expose the same command. Node 22+, Python 3,
17
17
  Git, pnpm and tar are required, along with authenticated `gh` for GitHub sources.
18
- Use a full, clean seed clone with its declared package-manager version. The producer reads
18
+ Use a full-history, clean seed clone with its declared package-manager version;
19
+ `--filter=blob:none` is supported, while shallow history is refused. Before
20
+ reading source files, destination validation uses the publication scanner's
21
+ single batched hydration fetch. An unavailable blob reports its object ID and
22
+ committed path with `fetch the full history` guidance, separately from identity
23
+ disagreements. The producer reads
19
24
  committed `seed.json`, `package.json`, `RELEASE.json`, `LICENSE`, `CONTRACT.md` and
20
25
  `AGENTS.md`. Version declarations must agree. `--project` declares the publishing
21
26
  identity; `--repository` is the credential-free public destination URL. Neither is inferred
@@ -33,6 +38,11 @@ URLs; identity comparison ignores case and a terminal `.git` in URLs. Missing,
33
38
  malformed, partial or inconsistent identities fail closed. The selected
34
39
  committed tree supplies the declarations.
35
40
 
41
+ Temporary release directories are removed recursively, including incidental
42
+ Finder metadata. A cleanup failure keeps a completed release successful and
43
+ records a `temporary-cleanup-failed` warning in stderr and the release receipt;
44
+ its directory can then be removed manually.
45
+
36
46
  The platform reads these same HTTPS GitHub URLs, plus bare an owner and repository name separated by a slash for releases that carry it; ordinary release editions still require the URL here.
37
47
 
38
48
  Every publication scan receives the explicit project and destination: source
@@ -0,0 +1,16 @@
1
+ import { rmSync } from 'node:fs';
2
+
3
+ // Finder and other filesystem observers can add metadata while removal walks
4
+ // the directory. Retry those races, but never turn finished artifacts into a
5
+ // failed release merely because temporary files could not be removed.
6
+ export function cleanupTemporary(directory) {
7
+ try {
8
+ rmSync(directory, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
9
+ return null;
10
+ } catch (error) {
11
+ const warning = { code: 'temporary-cleanup-failed', errorCode: error.code ?? 'UNKNOWN',
12
+ directory, message: 'Temporary release cleanup failed; remove the remaining directory manually.' };
13
+ console.warn(`Warning: ${warning.message} (${warning.errorCode}): ${directory}`);
14
+ return warning;
15
+ }
16
+ }
@@ -11,6 +11,7 @@ import { writePublishHandoff } from './publish-handoff.mjs';
11
11
  import { releasePublicBase } from './public-base.mjs';
12
12
  import { prepareTagAdoption, confirmTagAdoption, assertTagAdoption } from './tag-adoption.mjs';
13
13
  import { placeholderMarks, placeholderMessage } from '../../cli/placeholder-mark.mjs';
14
+ import { cleanupTemporary } from './cleanup.mjs';
14
15
 
15
16
  // Failure output belongs to the private receipt, not the site preview.
16
17
  const withoutOutput = gates => JSON.parse(JSON.stringify(gates, (key, value) => key === 'output' ? undefined : value));
@@ -21,13 +22,15 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
21
22
  if (onto !== undefined && clean) throw Error('--onto and --clean-history are mutually exclusive');
22
23
  if (publicBase !== undefined && (onto !== undefined || clean)) throw Error('--public-base selects exact-source history; cannot combine with export history modes');
23
24
  const preview = dryRun || !tag;
25
+ // The shared history reader hydrates promised blobs once, before any source
26
+ // metadata, placeholder inspection or release export reads their contents.
27
+ validatePublicationDestination({ repo, commit: ref }, project, repository);
24
28
  const source = inspectSource(repo, ref);
25
29
  // A release refuses a seed still wearing Base's placeholder mark unless the
26
30
  // caller allows it explicitly; the receipt records that allowance.
27
31
  const files = [...source.entries].filter(([, entry]) => entry.type === 'blob' && ['100644', '100755'].includes(entry.mode)).map(([path]) => path);
28
32
  const marks = await placeholderMarks(source.entries.has('OPERATIONS.json') ? JSON.parse(source.bytes('OPERATIONS.json')).project : project, files, source.bytes);
29
33
  if (marks.length && !allowPlaceholderMark) throw Object.assign(Error(`${placeholderMessage(marks)} Release refused; pass --allow-placeholder-mark to release with it.`), { code: 'placeholder_mark' });
30
- validatePublicationDestination(source, project, repository);
31
34
  releaseMessage(source);
32
35
  if (typeof tag === 'string' && tag !== source.tag) throw Error('Requested tag must match the committed release version');
33
36
  const existing = git(source.repo, 'tag', '--list', source.tag).toString().trim();
@@ -44,6 +47,7 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
44
47
  output = newOutput(source.repo, output);
45
48
  const temporary = mkdtempSync(resolve(tmpdir(), 'rer-release-'));
46
49
  let receipt;
50
+ let cleanupAttempted = false;
47
51
  try {
48
52
  const checkout = resolve(temporary, 'seed');
49
53
  git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, checkout);
@@ -117,17 +121,20 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
117
121
  });
118
122
  }
119
123
  }
120
- if (passed && tag && destination) writePublishHandoff(output, destination, receipt);
121
124
  if (passed && adopted) {
122
125
  assertTagAdoption(source, adopted, { imported: true });
123
126
  if (receipt.tag.object !== adopted.object || receipt.publicExport.tagObject !== adopted.object ||
124
127
  receipt.publicExport.commit !== adopted.commit || receipt.source.commit !== adopted.commit)
125
128
  throw Error('Adopted release tag identity does not match the release receipt');
126
129
  }
130
+ cleanupAttempted = true;
131
+ const warning = cleanupTemporary(temporary);
132
+ if (warning) receipt.warnings = [warning];
133
+ if (passed && tag && destination) writePublishHandoff(output, destination, receipt);
127
134
  writeFileSync(resolve(output, 'release-receipt.json'), json(receipt), { flag: 'wx' });
128
135
  if (passed && destination) {
129
136
  requirePublicDestination(destination);
130
- if (remotePublicHead(exported.repo, destination, source.tag) !== publicParent) throw Error('Public destination changed before handoff');
137
+ if (remotePublicHead(tag ? resolve(output, 'public-git') : source.repo, destination, source.tag) !== publicParent) throw Error('Public destination changed before handoff');
131
138
  if (tag && !dryRun) annotate(source.repo, source, source.commit);
132
139
  }
133
140
  return receipt;
@@ -135,5 +142,5 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
135
142
  // No incomplete directory should be mistaken for a completed export.
136
143
  rmSync(output, { recursive: true, force: true });
137
144
  throw error;
138
- } finally { rmSync(temporary, { recursive: true, force: true }); }
145
+ } finally { if (!cleanupAttempted) cleanupTemporary(temporary); }
139
146
  }
@@ -10,7 +10,11 @@ export function validatePublicationDestination(source, project, repository) {
10
10
  execFileSync('python3', ['-B', resolve(import.meta.dirname, '../../scripts/publish-check/destination.py'),
11
11
  '--repo', source.repo, '--ref', source.commit, '--project', project, '--repository', repository],
12
12
  { stdio: ['ignore', 'pipe', 'pipe'] });
13
- } catch { throw Error('Publication destination must agree with the committed edition and project identities'); }
13
+ } catch (error) {
14
+ if (error.status === 2) throw Error(error.stderr.toString().trim());
15
+ if (error.status === 1) throw Error('Publication destination must agree with the committed edition and project identities');
16
+ throw Error('Publication destination inspection could not run');
17
+ }
14
18
  }
15
19
 
16
20
  export function publicationScan(repo, ref, envFile, project, repository, publicBase, annotationFile) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runeditrun/rer",
3
- "version": "0.2.66",
3
+ "version": "0.2.67",
4
4
  "description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -6,8 +6,11 @@ See [setup, command and scope](../publish-check-README.md).
6
6
  blob reads to system Git (`rev-parse`, `rev-list`, `ls-tree`, `cat-file --batch`).
7
7
  Before any tree inventory asks for blob sizes, `history_hydration.py` counts
8
8
  missing objects and hydrates a partial clone with one `fetch --refetch --no-filter`
9
- from its promisor remote. A failed fetch or remaining missing object makes the
10
- inspection incomplete. `history.hydration` reports the missing counts, fetch
9
+ from its promisor remote. The batch explicitly requests every missing object,
10
+ including selected history outside that remote's branch refspecs. A failed fetch
11
+ or remaining missing object makes the inspection incomplete and names a missing
12
+ blob and its committed path, with `fetch the full history` guidance.
13
+ `history.hydration` reports the missing counts, fetch
11
14
  count and elapsed time; subsequent history reads disable lazy fetching.
12
15
 
13
16
  Reports include implementation version, Python version and a digest of the
@@ -2,6 +2,7 @@
2
2
  import argparse
3
3
  import json
4
4
  import re
5
+ import sys
5
6
  from github_refs import FULL
6
7
  from project_identity import FIELDS, ordinary_document, tree_identity
7
8
 
@@ -115,7 +116,13 @@ if __name__ == '__main__':
115
116
  try:
116
117
  history = History(args.repo, args.ref)
117
118
  destination(history, args.project, args.repository)
118
- except (ValueError, TypeError, UnicodeError, OSError, InspectionError):
119
+ except InspectionError as error:
120
+ print(str(error), file=sys.stderr)
121
+ raise SystemExit(2) from None
122
+ except OSError:
123
+ print('Publication destination inspection could not run', file=sys.stderr)
124
+ raise SystemExit(2) from None
125
+ except (ValueError, TypeError, UnicodeError):
119
126
  raise SystemExit(1) from None
120
127
  finally:
121
128
  if history:
@@ -11,7 +11,7 @@ class InspectionError(Exception):
11
11
 
12
12
  def git_environment():
13
13
  env = {k: v for k, v in os.environ.items() if not k.startswith('GIT_')}
14
- env.update(GIT_TERMINAL_PROMPT='0', GIT_NO_REPLACE_OBJECTS='1')
14
+ env.update(GIT_TERMINAL_PROMPT='0', GIT_NO_REPLACE_OBJECTS='1', GIT_NO_LAZY_FETCH='1')
15
15
  return env
16
16
 
17
17
 
@@ -41,7 +41,6 @@ class History:
41
41
  self.revisions = [self.commit, *public_revisions] if public_base else ['--all', self.commit]
42
42
  from history_hydration import prepare
43
43
  self.hydration = prepare(self)
44
- self.env['GIT_NO_LAZY_FETCH'] = '1'
45
44
  self.commits = self.run('rev-list', *self.revisions).decode().splitlines()
46
45
  self.public_commits = set(self.run('rev-list', *public_revisions).decode().splitlines()) if public_base else set()
47
46
  self.trees = {}
@@ -49,10 +48,10 @@ class History:
49
48
  self.reader = subprocess.Popen(['git', '-C', self.repo, 'cat-file', '--batch'],
50
49
  env=self.env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
51
50
 
52
- def run(self, *args):
51
+ def run(self, *args, input=None):
53
52
  try:
54
53
  result = subprocess.run(['git', '-C', self.repo, '-c', 'core.quotePath=false', *args],
55
- env=self.env, capture_output=True, timeout=120)
54
+ env=self.env, input=input, capture_output=True, timeout=120)
56
55
  if result.returncode:
57
56
  raise InspectionError('history.git-failed')
58
57
  return result.stdout
@@ -89,11 +88,13 @@ class History:
89
88
  self.reader.stdin.flush()
90
89
  header = self.reader.stdout.readline().split()
91
90
  if len(header) != 3 or header[1] != b'blob':
92
- raise InspectionError('history.missing-blob')
91
+ from history_hydration import missing_blob
92
+ raise missing_blob(self, oid)
93
93
  size = int(header[2])
94
94
  data = self.reader.stdout.read(size)
95
95
  if len(data) != size or self.reader.stdout.read(1) != b'\n':
96
- raise InspectionError('history.missing-blob')
96
+ from history_hydration import missing_blob
97
+ raise missing_blob(self, oid)
97
98
  return data
98
99
 
99
100
  def selected(self, path):
@@ -3,26 +3,49 @@ from time import monotonic
3
3
  from git_history import InspectionError
4
4
 
5
5
 
6
+ def missing_blob(history, oid):
7
+ # Sizes need the blob itself. Locate a promised object using tree entries
8
+ # without -l, so a failed fetch cannot trigger another lazy fetch.
9
+ trees = set(history.run('log', '--format=%T', *history.revisions).splitlines())
10
+ for tree in sorted(trees):
11
+ for record in history.run('ls-tree', '-r', '-z', tree.decode()).split(b'\0'):
12
+ metadata, separator, path = record.partition(b'\t')
13
+ if separator and metadata.split()[2].decode() == oid:
14
+ return InspectionError(f'missing blob {oid} at {path.decode(errors="replace")}: fetch the full history')
15
+ return InspectionError(f'missing blob {oid} at <unknown path>: fetch the full history')
16
+
17
+
6
18
  def prepare(history):
7
19
  started = monotonic()
8
20
  def missing():
9
- return sum(line.startswith(b'?') for line in
10
- history.run('rev-list', '--objects', '--missing=print', *history.revisions).splitlines())
21
+ return sorted(line[1:] for line in
22
+ history.run('rev-list', '--objects', '--missing=print', *history.revisions).splitlines()
23
+ if line.startswith(b'?'))
11
24
  before = missing()
12
25
  fetches = 0
13
26
  if before:
14
27
  # A blob:none clone already owns the commits and trees. ls-tree -l and
15
28
  # cat-file would otherwise ask its promisor for one blob at a time.
16
- remotes = history.run('config', '--get-regexp', r'^remote\..*\.promisor$').decode().splitlines()
29
+ try:
30
+ remotes = history.run('config', '--get-regexp', r'^remote\..*\.promisor$').decode().splitlines()
31
+ except InspectionError:
32
+ remotes = []
17
33
  names = [line.rsplit(None, 1)[0][len('remote.'):-len('.promisor')]
18
34
  for line in remotes if line.rsplit(None, 1)[-1].lower() == 'true']
19
35
  if len(names) != 1:
20
- raise InspectionError('history.promisor-ambiguous')
21
- history.run('fetch', '--quiet', '--refetch', '--no-filter', '--no-tags',
22
- '--no-recurse-submodules', '--no-write-fetch-head', '--', names[0])
36
+ raise missing_blob(history, before[0].decode())
37
+ try:
38
+ # Explicit object wants cover selected commits fetched outside the
39
+ # remote's configured branch refspecs too. stdin keeps one batch
40
+ # even when a full history exceeds the command argument limit.
41
+ history.run('fetch', '--quiet', '--refetch', '--no-filter', '--no-tags',
42
+ '--no-recurse-submodules', '--no-write-fetch-head', '--stdin', '--', names[0],
43
+ input=b'\n'.join(before) + b'\n')
44
+ except InspectionError:
45
+ raise missing_blob(history, before[0].decode()) from None
23
46
  fetches = 1
24
- after = missing() if before else 0
47
+ after = missing() if before else []
25
48
  if after:
26
- raise InspectionError('history.missing-blob')
27
- return dict(missingBefore=before, missingAfter=after, fetches=fetches,
49
+ raise missing_blob(history, after[0].decode())
50
+ return dict(missingBefore=len(before), missingAfter=len(after), fetches=fetches,
28
51
  seconds=round(monotonic() - started, 6))