@runeditrun/rer 0.2.65 → 0.2.67
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/business/vendors/README.md +26 -2
- package/cli/business/vendors/calendly.mjs +2 -2
- package/cli/business/vendors/common.mjs +73 -0
- package/cli/business/vendors/pagerduty.mjs +6 -15
- package/cli/business/vendors/sentry.mjs +4 -3
- package/cli/business/vendors/uptimerobot.mjs +7 -6
- package/cli/production-mail.mjs +4 -1
- package/global/CHANGELOG.md +10 -0
- package/modules/deploy-kit/d1-import.mjs +3 -1
- package/modules/deploy-kit/migrations.mjs +1 -0
- package/modules/deploy-kit/owner-cli.mjs +1 -1
- package/modules/deploy-kit/provider.mjs +16 -4
- package/modules/release/README.md +11 -1
- package/modules/release/cleanup.mjs +16 -0
- package/modules/release/produce.mjs +11 -4
- package/modules/release/publication-scan.mjs +5 -1
- package/modules/release/sign.mjs +1 -1
- package/onboarding/redaction.mjs +1 -1
- package/package.json +1 -1
- package/scripts/publish-check/README.md +5 -2
- package/scripts/publish-check/destination.py +8 -1
- package/scripts/publish-check/git_history.py +7 -6
- package/scripts/publish-check/history_hydration.py +32 -9
|
@@ -57,6 +57,30 @@ emission. HTTP errors contain only a fixed code and numeric status; network,
|
|
|
57
57
|
parsing and unexpected errors never expose response text, URLs or key values.
|
|
58
58
|
429 stops with `RATE_LIMITED`; there is no hidden retry loop.
|
|
59
59
|
|
|
60
|
+
## Credentials in responses
|
|
61
|
+
|
|
62
|
+
Every adapter strips credential values from a parsed response before it emits
|
|
63
|
+
bytes for upload, keeping the record, by the same rules as the seed's import
|
|
64
|
+
door. A field whose name's letters end in a credential word (`apiKey`,
|
|
65
|
+
`httpPassword`, `http_password`, `client_secret`, `X-Api-Key`, `integration_key`,
|
|
66
|
+
`webhook_url`, `secret_key`, `x-functions-key`, `Ocp-Apim-Subscription-Key`,
|
|
67
|
+
`pwd` and the like) or are `X-Auth`, the `value` of a `{name|key, value}` pair
|
|
68
|
+
naming one (`{name: 'Authorization', value}`), or the value of such a
|
|
69
|
+
`[name, value]` pair in a list of them (Sentry's header tuples), becomes
|
|
70
|
+
`"[removed]"`. A plural name (`apiKeys`, `tokens`) counts only when it holds
|
|
71
|
+
text, so a count such as `tokens: 1500` stays. A URL of any scheme keeps a
|
|
72
|
+
marker where each credential was: its user name and password become `removed@`,
|
|
73
|
+
and the value of each credential query or fragment parameter (`api_key`,
|
|
74
|
+
`access_token`, `sig` and the like) becomes `removed`
|
|
75
|
+
(`?api_key=removed®ion=eu`, `#access_token=removed`), except the adapter's
|
|
76
|
+
own validated pagination keys (`page_token`). The door drops each marker and
|
|
77
|
+
reports it per record as `CREDENTIAL_REMOVED`, so the owner sees which record
|
|
78
|
+
lost a credential. Null, booleans, empty strings and masks such as `****` are
|
|
79
|
+
not credentials and stay as they are. Names that merely contain such letters
|
|
80
|
+
(`keyword`, `monkey`, `signal`, `design`) are not credential names.
|
|
81
|
+
Pagination is read from the response as received, so a credential-bearing
|
|
82
|
+
pagination link is still refused.
|
|
83
|
+
|
|
60
84
|
## Inputs and native coverage
|
|
61
85
|
|
|
62
86
|
* `sentry`: `organization: {id, slug, region: 'us'|'de'}`, `start`, `end`, and
|
|
@@ -75,8 +99,8 @@ parsing and unexpected errors never expose response text, URLs or key values.
|
|
|
75
99
|
with layers/final coverage, overrides for the window and escalation policies.
|
|
76
100
|
Schedule listing only discovers IDs and is not duplicated in the bundle.
|
|
77
101
|
Overrides retain full boundaries with `overflow=true` and add `schedule_id`,
|
|
78
|
-
as Oncall's documented bundle requires.
|
|
79
|
-
|
|
102
|
+
as Oncall's documented bundle requires. Incident history and paging
|
|
103
|
+
configuration are not fetched.
|
|
80
104
|
* `calendly`: optional `asOf` timestamp (recommended for a repeatable catch-up).
|
|
81
105
|
An omitted timestamp is captured at first invocation and retained on resume.
|
|
82
106
|
Downloads the current user, event types, availability schedules, per-type
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { object, list, id, timestamp, reference, cursor, assertCount, hash, shape, fail } from './common.mjs';
|
|
1
|
+
import { object, list, id, timestamp, reference, cursor, assertCount, hash, shape, fail, withoutCredentials } from './common.mjs';
|
|
2
2
|
|
|
3
3
|
export const origin = 'https://api.calendly.com';
|
|
4
4
|
const queries = ['user','count','event_type','min_start_time','page_token'];
|
|
@@ -66,7 +66,7 @@ export async function* run(ctx) {
|
|
|
66
66
|
const next = s.phase === 0 ? null : nextToken(body, path, s.token);
|
|
67
67
|
if (next !== null && (next === s.token || s.seen.includes(next))) fail('PAGINATION_CYCLE');
|
|
68
68
|
const name = `${resource}-${String(s.files.length + 1).padStart(6,'0')}.json`;
|
|
69
|
-
const bytes = Buffer.from(JSON.stringify(body) + '\n');
|
|
69
|
+
const bytes = Buffer.from(JSON.stringify(withoutCredentials(body, queries)) + '\n');
|
|
70
70
|
if (s.files.length >= 600) fail('FILE_LIMIT');
|
|
71
71
|
s.files.push({ name, resource, page: s.seen.length + 1, size: bytes.length, sha256: hash(bytes), capturedAt: s.asOf });
|
|
72
72
|
s.token = next;
|
|
@@ -46,6 +46,79 @@ export function reference(value, origin, pathname, allowedQuery = []) {
|
|
|
46
46
|
if ([...url.searchParams.keys()].some(key => !allowedQuery.includes(key))) fail('INVALID_PAGINATION');
|
|
47
47
|
return url;
|
|
48
48
|
}
|
|
49
|
+
// The seed's import door applies the same rules (modules/import-kit, removeCredentials).
|
|
50
|
+
// A field holds a credential when its name's letters end in a credential word
|
|
51
|
+
// (apiKey, httpPassword, http_password, client_secret, X-Api-Key, integration_key,
|
|
52
|
+
// webhook_url, secret_key, x-functions-key, Ocp-Apim-Subscription-Key, db_pwd) or
|
|
53
|
+
// are X-Auth, unless it is empty: null, a boolean (Calendly's `secret` flag),
|
|
54
|
+
// whitespace or a mask such as "****" or "[redacted]". A plural (apiKeys, tokens)
|
|
55
|
+
// holds credentials only as text, so a count such as `tokens: 1500` stays.
|
|
56
|
+
const credentialName = /(?:authorization|credentials?|webhookurl|calurl)$|^xauth$/u;
|
|
57
|
+
const credentialWord = /(?:password|passwd|pwd|secret|token|cookie|(?:api|access|private|auth|client|integration|routing|service|signing|session|refresh|secret|subscription|functions)key)(s?)$/u;
|
|
58
|
+
const placeholder = /^(?:\s*|[*\u2022\u25CF\u00B7x#_.-]{1,64}|[<[({]?\s*(?:redacted|hidden|masked|removed|omitted|none|null|n\/a|placeholder)\s*[>\])}]?)$/iu;
|
|
59
|
+
/** What a stripped credential value becomes; the seed's import door reports it per record. */
|
|
60
|
+
export const REMOVED_CREDENTIAL = '[removed]';
|
|
61
|
+
/** What a stripped credential query or fragment parameter's value becomes
|
|
62
|
+
* (`?api_key=removed`), as a URL's user name and password become `removed@`; the
|
|
63
|
+
* door drops the pair and reports it per record. */
|
|
64
|
+
export const REMOVED_PARAMETER = 'removed';
|
|
65
|
+
const empty = value => value === null || typeof value === 'boolean' || (typeof value === 'string' && placeholder.test(value));
|
|
66
|
+
// Text as a member (`['…']`, `{github: '…'}`); a list of objects is checked field by field.
|
|
67
|
+
const text = value => typeof value === 'string' ? !placeholder.test(value) : !!value && typeof value === 'object' && Object.values(value).some(child => (typeof child === 'string' || Array.isArray(child)) && text(child));
|
|
68
|
+
/** Every percent-encoding layer of a key, decoded byte by byte, as the door inspects it. */
|
|
69
|
+
function decoded(text) {
|
|
70
|
+
for (let depth = 0; depth < 32 && /%[\da-f]{2}/iu.test(text); depth++) text = text.replace(/%([\da-f]{2})/giu, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
|
|
71
|
+
return text;
|
|
72
|
+
}
|
|
73
|
+
/** Whether a value under this name is a credential, by the door's rule. */
|
|
74
|
+
function credential(name, value) {
|
|
75
|
+
const letters = decoded(name).toLowerCase().replace(/[^a-z]/gu, ''), word = credentialWord.exec(letters);
|
|
76
|
+
if (!word && !credentialName.test(letters)) return false;
|
|
77
|
+
return !empty(value) && (!word?.[1] || text(value));
|
|
78
|
+
}
|
|
79
|
+
// The door's credentialQueryKey rule for URL query and fragment parameter names;
|
|
80
|
+
// `sig` (an Azure SAS signature) only as the whole name.
|
|
81
|
+
const credentialParameter = key => /token|secret|key|auth|signature|credential|password|cookie/iu.test(decoded(key)) || /^\s*sig\s*$/iu.test(decoded(key));
|
|
82
|
+
const authority = /^(\s*(?:[a-z][a-z\d+.-]*:)?[\\/]{2})[^\\/?#]*@/iu;
|
|
83
|
+
/** A URL of any scheme with its user name and password replaced by `removed@`,
|
|
84
|
+
* and the value of each credential query or fragment parameter by `removed`,
|
|
85
|
+
* except the adapter's own validated pagination keys, spelled exactly. */
|
|
86
|
+
function withoutUrlCredentials(text, pagination) {
|
|
87
|
+
if (!/^\s*(?:[a-z][a-z\d+.-]*:)?[\\/]{2}|^[/?]/iu.test(text)) return text;
|
|
88
|
+
const result = text.replace(authority, '$1removed@');
|
|
89
|
+
const mark = pairs => pairs.split('&').map(pair => {
|
|
90
|
+
const at = pair.indexOf('='), key = at < 0 ? pair : pair.slice(0, at);
|
|
91
|
+
return at < 0 || pagination.includes(key) || !credentialParameter(key) ? pair : `${key}=${REMOVED_PARAMETER}`;
|
|
92
|
+
}).join('&');
|
|
93
|
+
const hashAt = result.indexOf('#'), end = hashAt < 0 ? result.length : hashAt;
|
|
94
|
+
const fragment = hashAt < 0 ? '' : result.slice(hashAt);
|
|
95
|
+
const marked = fragment.includes('=') ? '#' + mark(fragment.slice(1)) : fragment;
|
|
96
|
+
const queryAt = result.indexOf('?');
|
|
97
|
+
if (queryAt < 0 || queryAt > end) return result.slice(0, end) + marked;
|
|
98
|
+
return result.slice(0, queryAt + 1) + mark(result.slice(queryAt + 1, end)) + marked;
|
|
99
|
+
}
|
|
100
|
+
/** The `[name, value]` pairs of a list holding only such pairs (Sentry's header tuples). */
|
|
101
|
+
const tuples = list => list.length > 0 && list.every(item => Array.isArray(item) && item.length === 2 && typeof item[0] === 'string');
|
|
102
|
+
/** A copy of a parsed vendor response without credential values, made before any
|
|
103
|
+
* byte is emitted for upload: a credential-named field's value, the `value` of a
|
|
104
|
+
* `{name|key, value}` pair naming one (a header pair), or the value of a
|
|
105
|
+
* credential-named `[name, value]` pair in a list of them, becomes
|
|
106
|
+
* REMOVED_CREDENTIAL, so the record stays and the seed can say what was removed.
|
|
107
|
+
* URLs lose credentials as withoutUrlCredentials describes; `pagination` lists
|
|
108
|
+
* the adapter's validated query keys, which stay. */
|
|
109
|
+
export function withoutCredentials(value, pagination = []) {
|
|
110
|
+
if (Array.isArray(value)) {
|
|
111
|
+
const pairs = tuples(value);
|
|
112
|
+
return value.map(child => pairs && credential(child[0], child[1]) ? [child[0], REMOVED_CREDENTIAL] : withoutCredentials(child, pagination));
|
|
113
|
+
}
|
|
114
|
+
if (value && typeof value === 'object') {
|
|
115
|
+
const label = Object.entries(value).find(([key, child]) => /^(?:name|key)$/iu.test(key) && typeof child === 'string');
|
|
116
|
+
const pair = !!label && Object.hasOwn(value, 'value');
|
|
117
|
+
return Object.fromEntries(Object.entries(value).map(([key, child]) =>
|
|
118
|
+
[key, credential(pair && key === 'value' ? label[1] : key, child) ? REMOVED_CREDENTIAL : withoutCredentials(child, pagination)]));
|
|
119
|
+
}
|
|
120
|
+
return typeof value === 'string' ? withoutUrlCredentials(value, pagination) : value;
|
|
121
|
+
}
|
|
49
122
|
export function assertCount(value, max = 1000) {
|
|
50
123
|
if (!Number.isSafeInteger(value) || value < 0 || value > max) fail('INVALID_CHECKPOINT');
|
|
51
124
|
return value;
|
|
@@ -1,15 +1,7 @@
|
|
|
1
|
-
import { object, list, id, windowInputs, assertCount, shape, fail } from './common.mjs';
|
|
1
|
+
import { object, list, id, windowInputs, assertCount, shape, fail, withoutCredentials } from './common.mjs';
|
|
2
2
|
|
|
3
3
|
export const origin = 'https://api.pagerduty.com';
|
|
4
4
|
export const route = url => /^\/(?:users|escalation_policies|schedules(?:\/[a-zA-Z0-9_-]+(?:\/overrides)?)?)$/u.test(url.pathname) && [...url.searchParams.keys()].every(x => ['limit','offset','since','until','overflow'].includes(x));
|
|
5
|
-
const credential = /^(?:password|passwd|secret|token|credentials?|authorization|cookie)$|(?:api|access|auth|integration|routing|service|signing|private|client|refresh|session)[_-]?(?:key|secret|token)$|^webhook[_-]?(?:url|secret)$|(?:http|web)[_-]?cal[_-]?url$/iu;
|
|
6
|
-
function noCredentials(value) {
|
|
7
|
-
if (Array.isArray(value)) value.forEach(noCredentials);
|
|
8
|
-
else if (value && typeof value === 'object') for (const [key, child] of Object.entries(value)) {
|
|
9
|
-
if (credential.test(key)) fail('CREDENTIAL_IN_RESPONSE');
|
|
10
|
-
noCredentials(child);
|
|
11
|
-
}
|
|
12
|
-
}
|
|
13
5
|
function advance(body, state) {
|
|
14
6
|
if (typeof body.more !== 'boolean') fail();
|
|
15
7
|
if (body.more) {
|
|
@@ -40,8 +32,7 @@ export async function* run(ctx) {
|
|
|
40
32
|
if (s.schedules.length > 500) fail('RECORD_LIMIT');
|
|
41
33
|
bytes = '';
|
|
42
34
|
} else {
|
|
43
|
-
|
|
44
|
-
bytes = (s.first ? (s.phase === 0 ? '{"users":[' : '],"escalation_policies":[') : ',') + JSON.stringify(body);
|
|
35
|
+
bytes = (s.first ? (s.phase === 0 ? '{"users":[' : '],"escalation_policies":[') : ',') + JSON.stringify(withoutCredentials(body));
|
|
45
36
|
s.first = false;
|
|
46
37
|
}
|
|
47
38
|
if (!advance(body, s)) { s.phase++; s.first = true; }
|
|
@@ -49,8 +40,8 @@ export async function* run(ctx) {
|
|
|
49
40
|
if (s.index === s.schedules.length) { s.phase = 3; s.index = 0; s.first = true; bytes = s.schedules.length ? '' : '],"schedules":['; }
|
|
50
41
|
else {
|
|
51
42
|
const { body } = await ctx.get(`/schedules/${id(s.schedules[s.index])}?${window}`);
|
|
52
|
-
object(body); object(body.schedule);
|
|
53
|
-
bytes = (s.first ? '],"schedules":[' : ',') + JSON.stringify(body);
|
|
43
|
+
object(body); object(body.schedule); ctx.records(1);
|
|
44
|
+
bytes = (s.first ? '],"schedules":[' : ',') + JSON.stringify(withoutCredentials(body));
|
|
54
45
|
s.first = false; s.index++;
|
|
55
46
|
}
|
|
56
47
|
} else {
|
|
@@ -58,8 +49,8 @@ export async function* run(ctx) {
|
|
|
58
49
|
else {
|
|
59
50
|
const schedule = id(s.schedules[s.index]);
|
|
60
51
|
const { body } = await ctx.get(`/schedules/${schedule}/overrides?${window}&overflow=true`);
|
|
61
|
-
object(body); ctx.records(list(body.overrides).length);
|
|
62
|
-
bytes = (s.first ? '],"overrides":[' : ',') + JSON.stringify({ ...body, schedule_id: schedule });
|
|
52
|
+
object(body); ctx.records(list(body.overrides).length);
|
|
53
|
+
bytes = (s.first ? '],"overrides":[' : ',') + JSON.stringify(withoutCredentials({ ...body, schedule_id: schedule }));
|
|
63
54
|
s.first = false;
|
|
64
55
|
s.index++;
|
|
65
56
|
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { object, list, id, windowInputs, cursor, assertCount, shape, fail } from './common.mjs';
|
|
1
|
+
import { object, list, id, windowInputs, cursor, assertCount, shape, fail, withoutCredentials } from './common.mjs';
|
|
2
2
|
|
|
3
3
|
export const origins = Object.freeze({ us: 'https://us.sentry.io', de: 'https://de.sentry.io' });
|
|
4
|
-
|
|
4
|
+
const queries = ['per_page','limit','cursor','query','project','start','end','sort'];
|
|
5
|
+
export const route = url => /^\/api\/0\/(?:organizations\/[a-zA-Z0-9_-]+\/(?:projects|releases|issues)|projects\/[a-zA-Z0-9_-]+\/[a-zA-Z0-9_-]+\/events\/[a-fA-F0-9]{32})\/$/u.test(url.pathname) && [...url.searchParams.keys()].every(x => queries.includes(x));
|
|
5
6
|
function nextCursor(headers) {
|
|
6
7
|
const link = headers.get('link');
|
|
7
8
|
if (!link) return null;
|
|
@@ -73,7 +74,7 @@ export async function* run(ctx) {
|
|
|
73
74
|
if (next !== null && (next === s.cursor || s.seen.includes(next))) fail('PAGINATION_CYCLE');
|
|
74
75
|
if (++s.page > 100) fail('PAGE_LIMIT');
|
|
75
76
|
let bytes = s.cursor === null ? (s.collections ? ',' : '') + JSON.stringify({kind,project}).slice(0,-1) + ',"pages":[' : ',';
|
|
76
|
-
bytes += JSON.stringify({ cursor: s.cursor, nextCursor: next, response });
|
|
77
|
+
bytes += JSON.stringify({ cursor: s.cursor, nextCursor: next, response: withoutCredentials(response, queries) });
|
|
77
78
|
s.cursor = next;
|
|
78
79
|
if (next === null) {
|
|
79
80
|
bytes += ']}'; s.collections++; s.seen = []; s.page = 0; s.event = 0;
|
|
@@ -1,20 +1,21 @@
|
|
|
1
|
-
import { object, list, reference, shape, fail } from './common.mjs';
|
|
1
|
+
import { object, list, reference, shape, fail, withoutCredentials } from './common.mjs';
|
|
2
2
|
|
|
3
3
|
export const origin = 'https://api.uptimerobot.com';
|
|
4
|
-
|
|
4
|
+
const pagination = ['limit','offset','cursor','page','page_token'];
|
|
5
|
+
export const route = url => url.pathname === '/v3/monitors' && [...url.searchParams.keys()].every(x => pagination.includes(x));
|
|
5
6
|
export async function* run(ctx) {
|
|
6
7
|
const state = ctx.state ?? { next: '/v3/monitors?limit=200', seen: [] };
|
|
7
8
|
shape(state, ['next','seen']);
|
|
8
9
|
if (!Array.isArray(state.seen) || state.seen.length > 600) fail('INVALID_CHECKPOINT');
|
|
9
|
-
state.seen.forEach(value => reference(value, origin, /^\/v3\/monitors$/u,
|
|
10
|
+
state.seen.forEach(value => reference(value, origin, /^\/v3\/monitors$/u, pagination));
|
|
10
11
|
while (state.next !== null) {
|
|
11
|
-
const url = reference(state.next, origin, /^\/v3\/monitors$/u,
|
|
12
|
+
const url = reference(state.next, origin, /^\/v3\/monitors$/u, pagination);
|
|
12
13
|
if (state.seen.includes(url.href)) fail('PAGINATION_CYCLE');
|
|
13
14
|
const { body } = await ctx.get(url.href);
|
|
14
15
|
object(body); ctx.records(list(body.data).length);
|
|
15
16
|
if (!Object.hasOwn(body, 'nextLink')) fail();
|
|
16
17
|
state.seen.push(url.href);
|
|
17
|
-
state.next = body.nextLink == null ? null : reference(body.nextLink, origin, /^\/v3\/monitors$/u,
|
|
18
|
-
yield ctx.item(JSON.stringify(body) + '\n', state);
|
|
18
|
+
state.next = body.nextLink == null ? null : reference(body.nextLink, origin, /^\/v3\/monitors$/u, pagination).href;
|
|
19
|
+
yield ctx.item(JSON.stringify(withoutCredentials(body, pagination)) + '\n', state);
|
|
19
20
|
}
|
|
20
21
|
}
|
package/cli/production-mail.mjs
CHANGED
|
@@ -49,11 +49,14 @@ export async function productionMail(root, declaration, env) {
|
|
|
49
49
|
const routes = selected.map(provider => {
|
|
50
50
|
const credentialKeys = deferred && nativeCredential[provider] && !declaredProviders.has(provider) ? [nativeCredential[provider]]
|
|
51
51
|
: ['MAIL_API_KEY', ...(nativeCredential[provider] ? [nativeCredential[provider]] : [])];
|
|
52
|
+
// A seed's direct-provider adapter can declare the contract's shared provider
|
|
53
|
+
// binding. Its explicit environment mapping (or live declared secret) still wins.
|
|
54
|
+
if (nativeCredential[provider] && secrets.has('MAIL_PROVIDER_API_KEY')) credentialKeys.push('MAIL_PROVIDER_API_KEY');
|
|
52
55
|
const declaredCredentials = credentialKeys.filter(key => secrets.has(key));
|
|
53
56
|
// Only resend and sendgrid have a provider key of their own; cloudflare and rer send with MAIL_API_KEY alone.
|
|
54
57
|
const providerKeys = (nativeCredential[provider] ? [nativeCredential[provider]] : []).filter(key => !declaredCredentials.includes(key));
|
|
55
58
|
const credentialText = declaredCredentials.includes('MAIL_API_KEY') && providerKeys.length
|
|
56
|
-
?
|
|
59
|
+
? `${declaredCredentials.join(' or ')} (or ${providerKeys.join(' / ')})`
|
|
57
60
|
: declaredCredentials.join(' or ') || credentialKeys.join(' or ');
|
|
58
61
|
return live => {
|
|
59
62
|
let relaySelected = true;
|
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.67
|
|
4
|
+
|
|
5
|
+
- **Import door options (#298 with #308).** Seeds choose credential removal modes and a preserve or strict reversal policy when an import is created. Failed applies can be retried, CLI-stripped query and fragment credentials receive per-record notes, more credential shapes are recognized, and a blocked re-baseline is refused at plan time with a named 409.
|
|
6
|
+
- **Release from partial clones (#313).** Hydrate missing history blobs before release reads and tolerate temporary-directory cleanup failures after a saved result.
|
|
7
|
+
- **Mail credentials and large migration histories (#314).** Deploy honors a declared MAIL_PROVIDER_API_KEY for SendGrid and Resend, and scales the bounded provider request budget to pending migrations.
|
|
8
|
+
|
|
9
|
+
## 0.2.66
|
|
10
|
+
|
|
11
|
+
- **Seed copies pass the pin-time credential scan (#305).** Credential-shaped test fixtures in seed-vendored Base files are constructed at runtime, preserving their values and assertions. Seed copies and exported references no longer trigger the hosted pin-time credential scan, so seeds can refresh Base past 0.2.61 and pin again.
|
|
12
|
+
|
|
3
13
|
## 0.2.65
|
|
4
14
|
|
|
5
15
|
- **Tag-first patch releases (#302).** rer release --tag adopts an existing remote annotated version tag at the exact source commit after its tag Verify passes. The producer captures one immutable tag object and checks object/commit equality before mutation and RELEASED. The existing release-tag Verify rule remains in force.
|
|
@@ -2,6 +2,8 @@ import { createHash } from 'node:crypto';
|
|
|
2
2
|
import { assert, uuid } from './validation.mjs';
|
|
3
3
|
import { recordDeployFailure } from './diagnostics.mjs';
|
|
4
4
|
|
|
5
|
+
export const D1_IMPORT_POLL_LIMIT = 100;
|
|
6
|
+
|
|
5
7
|
// Wrangler's remote --file protocol. D1 imports execute the whole file atomically,
|
|
6
8
|
// including trigger bodies; the query endpoint's semicolon splitting cannot.
|
|
7
9
|
// Cloudflare's own reason for an import that failed or did not finish, kept on the
|
|
@@ -31,7 +33,7 @@ export async function importD1({ envelope, upload, redact = String }, id, sql) {
|
|
|
31
33
|
await upload(response.upload_url, bytes, etag);
|
|
32
34
|
response = await post({ action:'ingest', filename:response.filename, etag });
|
|
33
35
|
}
|
|
34
|
-
for (let polls = 0; polls <
|
|
36
|
+
for (let polls = 0; polls < D1_IMPORT_POLL_LIMIT; polls++) {
|
|
35
37
|
// REST places errors on the envelope; validate nested errors when supplied.
|
|
36
38
|
if (!(response?.success === true && response.type === 'import' && response.status !== 'error' && (!Object.hasOwn(response, 'errors') || Array.isArray(response.errors) && !response.errors.length))) throw importFailure('d1-import-failed', id, response, redact);
|
|
37
39
|
if (response.status === 'complete') {
|
|
@@ -58,6 +58,7 @@ export async function migrate(api, id, migrations, { apply = true, upgrade = app
|
|
|
58
58
|
validateMigrations(migrations);
|
|
59
59
|
const { applied, pending, transition, exists, claimed, claimedFrom } = await readLedger(api, id, migrations, query, { rewritten:locked });
|
|
60
60
|
assert(apply || pending.length === 0,'pending-migrations');
|
|
61
|
+
if (apply && pending.length) api.reserveMigrationRequests?.(id, pending.length);
|
|
61
62
|
if (transition && upgrade) {
|
|
62
63
|
await api.importSql(id, transition);
|
|
63
64
|
const verified = await readLedger(api, id, migrations, query);
|
|
@@ -64,7 +64,7 @@ export async function main(argv = process.argv.slice(2), env = process.env, {fet
|
|
|
64
64
|
const deadline = AbortSignal.timeout(Math.max(1, Math.ceil(remaining)));
|
|
65
65
|
return request(input, {...init,signal:init.signal ? AbortSignal.any([init.signal,deadline]) : deadline});
|
|
66
66
|
};
|
|
67
|
-
const api = deploymentDiagnostics(cloudflare({accountId:env.CLOUDFLARE_ACCOUNT_ID,token:env.CLOUDFLARE_API_TOKEN,fetch,...(command === 'remove' ? {maxRequests:Infinity,removal:true} : {})}),{onCreated});
|
|
67
|
+
const api = deploymentDiagnostics(cloudflare({accountId:env.CLOUDFLARE_ACCOUNT_ID,token:env.CLOUDFLARE_API_TOKEN,fetch,budgetMigrations:command === 'deploy',...(command === 'remove' ? {maxRequests:Infinity,removal:true} : {})}),{onCreated});
|
|
68
68
|
// Removal needs the recorded account/name identity, not a workers.dev route.
|
|
69
69
|
const target = command === 'remove'
|
|
70
70
|
? {accountId:api.accountId,project:declaration.project,environment:values.environment,worker:ownerWorkerName(declaration.project,values.environment)}
|
|
@@ -2,8 +2,8 @@ import { providerPages, providerListShape } from './pagination.mjs';
|
|
|
2
2
|
import { deleteR2Object } from './r2-object-transport.mjs';
|
|
3
3
|
import { isR2ObjectDelete } from './r2-objects.mjs';
|
|
4
4
|
import { recordDeployFailure } from './diagnostics.mjs';
|
|
5
|
-
import { importD1, importUploadURL } from './d1-import.mjs';
|
|
6
|
-
import { assert, hexId } from './validation.mjs';
|
|
5
|
+
import { D1_IMPORT_POLL_LIMIT, importD1, importUploadURL } from './d1-import.mjs';
|
|
6
|
+
import { assert, hexId, uuid } from './validation.mjs';
|
|
7
7
|
import { providerResponse, transportOutput, transportReason } from './provider-transport.mjs';
|
|
8
8
|
|
|
9
9
|
// A gateway/server error can arrive after a write. An explicit API refusal cannot.
|
|
@@ -28,10 +28,22 @@ export async function boundedJSON(response, limit) {
|
|
|
28
28
|
}
|
|
29
29
|
const redactValues = (text, values) => values.filter(Boolean).flatMap(value=>[value,encodeURIComponent(value),JSON.stringify(value).slice(1,-1)]).sort((a,b)=>b.length-a.length).reduce((value,secret)=>value.split(secret).join('[REDACTED]'),text);
|
|
30
30
|
/** Only this fixed origin and the explicitly selected account ever receive the token. */
|
|
31
|
-
export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, objectDelete = deleteR2Object, timeoutMs = 30000, maxRequests = 500, removal = false }) {
|
|
31
|
+
export function cloudflare({ accountId, token, fetch: request = globalThis.fetch, objectDelete = deleteR2Object, timeoutMs = 30000, maxRequests = 500, removal = false, budgetMigrations = false }) {
|
|
32
32
|
assert(hexId(accountId), 'invalid-cloudflare-account-id');
|
|
33
33
|
assert(typeof token === 'string' && token.trim() && !/[\r\n]/.test(token), 'invalid-cloudflare-api-token');
|
|
34
34
|
let requests = 0;
|
|
35
|
+
const migrationBudgets = new Set();
|
|
36
|
+
function reserveMigrationRequests(id, pending) {
|
|
37
|
+
if (!budgetMigrations || migrationBudgets.has(id)) return;
|
|
38
|
+
assert(uuid(id) && Number.isSafeInteger(pending) && pending >= 0, 'invalid-provider-request-budget');
|
|
39
|
+
// Each pending migration can claim, initialize, upload and ingest, then use
|
|
40
|
+
// the bounded import poll protocol. Metadata and transport retries share the
|
|
41
|
+
// original allowance. Reserve validated work only once per database/run.
|
|
42
|
+
const limit = maxRequests + pending * (4 + D1_IMPORT_POLL_LIMIT);
|
|
43
|
+
assert(Number.isSafeInteger(limit), 'invalid-provider-request-budget');
|
|
44
|
+
migrationBudgets.add(id);
|
|
45
|
+
maxRequests = limit;
|
|
46
|
+
}
|
|
35
47
|
// `zone` scope: the zone-level API (/zones/...), for a run site's zone lookup and redirect rule.
|
|
36
48
|
async function rawEnvelope(path, { method = 'GET', body, assetToken, zone = false, signal: callerSignal } = {}) {
|
|
37
49
|
assert(typeof path === 'string' && (isR2ObjectDelete(path, method) || /^\/[a-z0-9/_?=&.,%-]+$/i.test(path) && !path.includes('..') && !path.split('?')[0].includes('%')), 'invalid-provider-path');
|
|
@@ -147,6 +159,6 @@ export function cloudflare({ accountId, token, fetch: request = globalThis.fetch
|
|
|
147
159
|
assert(response.status === 200, 'd1-import-upload-failed');
|
|
148
160
|
assert(response.headers.get('etag')?.replace(/^"|"$/g, '') === etag, 'd1-import-upload-etag-mismatch');
|
|
149
161
|
}
|
|
150
|
-
return { accountId, envelope, list, importSql(id, sql) { return importD1({envelope,upload,redact:text => redactValues(text,[accountId,token])}, id, sql); }, request:requestResult, workerContent,
|
|
162
|
+
return { accountId, envelope, list, reserveMigrationRequests, importSql(id, sql) { return importD1({envelope,upload,redact:text => redactValues(text,[accountId,token])}, id, sql); }, request:requestResult, workerContent,
|
|
151
163
|
zoneRequest(path, options = {}) { return requestResult(path, { ...options, zone: true }); } };
|
|
152
164
|
}
|
|
@@ -15,7 +15,12 @@ pnpm rer release --repo /path/to/seed --ref main \
|
|
|
15
15
|
|
|
16
16
|
The `rer` bin and `node bin/rer.mjs` expose the same command. Node 22+, Python 3,
|
|
17
17
|
Git, pnpm and tar are required, along with authenticated `gh` for GitHub sources.
|
|
18
|
-
Use a full, clean seed clone with its declared package-manager version
|
|
18
|
+
Use a full-history, clean seed clone with its declared package-manager version;
|
|
19
|
+
`--filter=blob:none` is supported, while shallow history is refused. Before
|
|
20
|
+
reading source files, destination validation uses the publication scanner's
|
|
21
|
+
single batched hydration fetch. An unavailable blob reports its object ID and
|
|
22
|
+
committed path with `fetch the full history` guidance, separately from identity
|
|
23
|
+
disagreements. The producer reads
|
|
19
24
|
committed `seed.json`, `package.json`, `RELEASE.json`, `LICENSE`, `CONTRACT.md` and
|
|
20
25
|
`AGENTS.md`. Version declarations must agree. `--project` declares the publishing
|
|
21
26
|
identity; `--repository` is the credential-free public destination URL. Neither is inferred
|
|
@@ -33,6 +38,11 @@ URLs; identity comparison ignores case and a terminal `.git` in URLs. Missing,
|
|
|
33
38
|
malformed, partial or inconsistent identities fail closed. The selected
|
|
34
39
|
committed tree supplies the declarations.
|
|
35
40
|
|
|
41
|
+
Temporary release directories are removed recursively, including incidental
|
|
42
|
+
Finder metadata. A cleanup failure keeps a completed release successful and
|
|
43
|
+
records a `temporary-cleanup-failed` warning in stderr and the release receipt;
|
|
44
|
+
its directory can then be removed manually.
|
|
45
|
+
|
|
36
46
|
The platform reads these same HTTPS GitHub URLs, plus bare an owner and repository name separated by a slash for releases that carry it; ordinary release editions still require the URL here.
|
|
37
47
|
|
|
38
48
|
Every publication scan receives the explicit project and destination: source
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { rmSync } from 'node:fs';
|
|
2
|
+
|
|
3
|
+
// Finder and other filesystem observers can add metadata while removal walks
|
|
4
|
+
// the directory. Retry those races, but never turn finished artifacts into a
|
|
5
|
+
// failed release merely because temporary files could not be removed.
|
|
6
|
+
export function cleanupTemporary(directory) {
|
|
7
|
+
try {
|
|
8
|
+
rmSync(directory, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
9
|
+
return null;
|
|
10
|
+
} catch (error) {
|
|
11
|
+
const warning = { code: 'temporary-cleanup-failed', errorCode: error.code ?? 'UNKNOWN',
|
|
12
|
+
directory, message: 'Temporary release cleanup failed; remove the remaining directory manually.' };
|
|
13
|
+
console.warn(`Warning: ${warning.message} (${warning.errorCode}): ${directory}`);
|
|
14
|
+
return warning;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
@@ -11,6 +11,7 @@ import { writePublishHandoff } from './publish-handoff.mjs';
|
|
|
11
11
|
import { releasePublicBase } from './public-base.mjs';
|
|
12
12
|
import { prepareTagAdoption, confirmTagAdoption, assertTagAdoption } from './tag-adoption.mjs';
|
|
13
13
|
import { placeholderMarks, placeholderMessage } from '../../cli/placeholder-mark.mjs';
|
|
14
|
+
import { cleanupTemporary } from './cleanup.mjs';
|
|
14
15
|
|
|
15
16
|
// Failure output belongs to the private receipt, not the site preview.
|
|
16
17
|
const withoutOutput = gates => JSON.parse(JSON.stringify(gates, (key, value) => key === 'output' ? undefined : value));
|
|
@@ -21,13 +22,15 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
|
|
|
21
22
|
if (onto !== undefined && clean) throw Error('--onto and --clean-history are mutually exclusive');
|
|
22
23
|
if (publicBase !== undefined && (onto !== undefined || clean)) throw Error('--public-base selects exact-source history; cannot combine with export history modes');
|
|
23
24
|
const preview = dryRun || !tag;
|
|
25
|
+
// The shared history reader hydrates promised blobs once, before any source
|
|
26
|
+
// metadata, placeholder inspection or release export reads their contents.
|
|
27
|
+
validatePublicationDestination({ repo, commit: ref }, project, repository);
|
|
24
28
|
const source = inspectSource(repo, ref);
|
|
25
29
|
// A release refuses a seed still wearing Base's placeholder mark unless the
|
|
26
30
|
// caller allows it explicitly; the receipt records that allowance.
|
|
27
31
|
const files = [...source.entries].filter(([, entry]) => entry.type === 'blob' && ['100644', '100755'].includes(entry.mode)).map(([path]) => path);
|
|
28
32
|
const marks = await placeholderMarks(source.entries.has('OPERATIONS.json') ? JSON.parse(source.bytes('OPERATIONS.json')).project : project, files, source.bytes);
|
|
29
33
|
if (marks.length && !allowPlaceholderMark) throw Object.assign(Error(`${placeholderMessage(marks)} Release refused; pass --allow-placeholder-mark to release with it.`), { code: 'placeholder_mark' });
|
|
30
|
-
validatePublicationDestination(source, project, repository);
|
|
31
34
|
releaseMessage(source);
|
|
32
35
|
if (typeof tag === 'string' && tag !== source.tag) throw Error('Requested tag must match the committed release version');
|
|
33
36
|
const existing = git(source.repo, 'tag', '--list', source.tag).toString().trim();
|
|
@@ -44,6 +47,7 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
|
|
|
44
47
|
output = newOutput(source.repo, output);
|
|
45
48
|
const temporary = mkdtempSync(resolve(tmpdir(), 'rer-release-'));
|
|
46
49
|
let receipt;
|
|
50
|
+
let cleanupAttempted = false;
|
|
47
51
|
try {
|
|
48
52
|
const checkout = resolve(temporary, 'seed');
|
|
49
53
|
git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, checkout);
|
|
@@ -117,17 +121,20 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
|
|
|
117
121
|
});
|
|
118
122
|
}
|
|
119
123
|
}
|
|
120
|
-
if (passed && tag && destination) writePublishHandoff(output, destination, receipt);
|
|
121
124
|
if (passed && adopted) {
|
|
122
125
|
assertTagAdoption(source, adopted, { imported: true });
|
|
123
126
|
if (receipt.tag.object !== adopted.object || receipt.publicExport.tagObject !== adopted.object ||
|
|
124
127
|
receipt.publicExport.commit !== adopted.commit || receipt.source.commit !== adopted.commit)
|
|
125
128
|
throw Error('Adopted release tag identity does not match the release receipt');
|
|
126
129
|
}
|
|
130
|
+
cleanupAttempted = true;
|
|
131
|
+
const warning = cleanupTemporary(temporary);
|
|
132
|
+
if (warning) receipt.warnings = [warning];
|
|
133
|
+
if (passed && tag && destination) writePublishHandoff(output, destination, receipt);
|
|
127
134
|
writeFileSync(resolve(output, 'release-receipt.json'), json(receipt), { flag: 'wx' });
|
|
128
135
|
if (passed && destination) {
|
|
129
136
|
requirePublicDestination(destination);
|
|
130
|
-
if (remotePublicHead(
|
|
137
|
+
if (remotePublicHead(tag ? resolve(output, 'public-git') : source.repo, destination, source.tag) !== publicParent) throw Error('Public destination changed before handoff');
|
|
131
138
|
if (tag && !dryRun) annotate(source.repo, source, source.commit);
|
|
132
139
|
}
|
|
133
140
|
return receipt;
|
|
@@ -135,5 +142,5 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
|
|
|
135
142
|
// No incomplete directory should be mistaken for a completed export.
|
|
136
143
|
rmSync(output, { recursive: true, force: true });
|
|
137
144
|
throw error;
|
|
138
|
-
} finally {
|
|
145
|
+
} finally { if (!cleanupAttempted) cleanupTemporary(temporary); }
|
|
139
146
|
}
|
|
@@ -10,7 +10,11 @@ export function validatePublicationDestination(source, project, repository) {
|
|
|
10
10
|
execFileSync('python3', ['-B', resolve(import.meta.dirname, '../../scripts/publish-check/destination.py'),
|
|
11
11
|
'--repo', source.repo, '--ref', source.commit, '--project', project, '--repository', repository],
|
|
12
12
|
{ stdio: ['ignore', 'pipe', 'pipe'] });
|
|
13
|
-
} catch
|
|
13
|
+
} catch (error) {
|
|
14
|
+
if (error.status === 2) throw Error(error.stderr.toString().trim());
|
|
15
|
+
if (error.status === 1) throw Error('Publication destination must agree with the committed edition and project identities');
|
|
16
|
+
throw Error('Publication destination inspection could not run');
|
|
17
|
+
}
|
|
14
18
|
}
|
|
15
19
|
|
|
16
20
|
export function publicationScan(repo, ref, envFile, project, repository, publicBase, annotationFile) {
|
package/modules/release/sign.mjs
CHANGED
|
@@ -21,7 +21,7 @@ export async function signArtifact(options) {
|
|
|
21
21
|
let privateKey;
|
|
22
22
|
try {
|
|
23
23
|
const bytes = await keyInput(options);
|
|
24
|
-
if (
|
|
24
|
+
if (!new RegExp(['^-----BEGIN ', 'PRIVATE ', String.raw`KEY-----\r?\n[\s\S]+\r?\n-----END `, 'PRIVATE ', String.raw`KEY-----\s*$`].join('')).test(bytes.toString('ascii'))) throw new Error();
|
|
25
25
|
privateKey = createPrivateKey({ key: bytes, format: 'pem', type: 'pkcs8' });
|
|
26
26
|
if (privateKey.asymmetricKeyType !== 'ed25519') throw new Error();
|
|
27
27
|
} catch { refuse('SIGNATURE_INVALID'); }
|
package/onboarding/redaction.mjs
CHANGED
|
@@ -60,7 +60,7 @@ const SCHEME = '(?<![a-z0-9+.-])[a-z][a-z0-9+.-]*://';
|
|
|
60
60
|
const USERINFO = SCHEME + '([^\\s/?#"<>`]*)@';
|
|
61
61
|
const SLASHED_USERINFO = SCHEME + '([^\\s/?#:@"<>`\\[\\]]*:(?![0-9]*(?:[/?#\\s"<>`]|$))(?=(?<password>[^\\s"<>`@]{0,128}))\\k<password>)@';
|
|
62
62
|
// Provider token formats recognisable by their prefix alone.
|
|
63
|
-
const TOKEN_PREFIX = '(?:gh[pousr]_
|
|
63
|
+
const TOKEN_PREFIX = '(?:' + ['gh[pousr]_', 'github_pat_', 'sk-(?:proj-)?', 'sk_live_', 'sk_test_', 'rk_live_', 'rk_test_', 'pk_live_', 'whsec_', 'SG\\.', 'xox[baprs]-', 'AKIA', 'ASIA', 'AIza', 'npm_', 'glpat-', 'pypi-', 'hf_', 're_'].join('|') + ')';
|
|
64
64
|
const KNOWN_TOKEN = '(?<![A-Za-z0-9_])' + TOKEN_PREFIX + '[A-Za-z0-9_./+=-]+';
|
|
65
65
|
const PATTERNS = new RegExp('(?:' + [
|
|
66
66
|
AUTH_HEADER, BARE_AUTH,
|
package/package.json
CHANGED
|
@@ -6,8 +6,11 @@ See [setup, command and scope](../publish-check-README.md).
|
|
|
6
6
|
blob reads to system Git (`rev-parse`, `rev-list`, `ls-tree`, `cat-file --batch`).
|
|
7
7
|
Before any tree inventory asks for blob sizes, `history_hydration.py` counts
|
|
8
8
|
missing objects and hydrates a partial clone with one `fetch --refetch --no-filter`
|
|
9
|
-
from its promisor remote.
|
|
10
|
-
|
|
9
|
+
from its promisor remote. The batch explicitly requests every missing object,
|
|
10
|
+
including selected history outside that remote's branch refspecs. A failed fetch
|
|
11
|
+
or remaining missing object makes the inspection incomplete and names a missing
|
|
12
|
+
blob and its committed path, with `fetch the full history` guidance.
|
|
13
|
+
`history.hydration` reports the missing counts, fetch
|
|
11
14
|
count and elapsed time; subsequent history reads disable lazy fetching.
|
|
12
15
|
|
|
13
16
|
Reports include implementation version, Python version and a digest of the
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import argparse
|
|
3
3
|
import json
|
|
4
4
|
import re
|
|
5
|
+
import sys
|
|
5
6
|
from github_refs import FULL
|
|
6
7
|
from project_identity import FIELDS, ordinary_document, tree_identity
|
|
7
8
|
|
|
@@ -115,7 +116,13 @@ if __name__ == '__main__':
|
|
|
115
116
|
try:
|
|
116
117
|
history = History(args.repo, args.ref)
|
|
117
118
|
destination(history, args.project, args.repository)
|
|
118
|
-
except
|
|
119
|
+
except InspectionError as error:
|
|
120
|
+
print(str(error), file=sys.stderr)
|
|
121
|
+
raise SystemExit(2) from None
|
|
122
|
+
except OSError:
|
|
123
|
+
print('Publication destination inspection could not run', file=sys.stderr)
|
|
124
|
+
raise SystemExit(2) from None
|
|
125
|
+
except (ValueError, TypeError, UnicodeError):
|
|
119
126
|
raise SystemExit(1) from None
|
|
120
127
|
finally:
|
|
121
128
|
if history:
|
|
@@ -11,7 +11,7 @@ class InspectionError(Exception):
|
|
|
11
11
|
|
|
12
12
|
def git_environment():
|
|
13
13
|
env = {k: v for k, v in os.environ.items() if not k.startswith('GIT_')}
|
|
14
|
-
env.update(GIT_TERMINAL_PROMPT='0', GIT_NO_REPLACE_OBJECTS='1')
|
|
14
|
+
env.update(GIT_TERMINAL_PROMPT='0', GIT_NO_REPLACE_OBJECTS='1', GIT_NO_LAZY_FETCH='1')
|
|
15
15
|
return env
|
|
16
16
|
|
|
17
17
|
|
|
@@ -41,7 +41,6 @@ class History:
|
|
|
41
41
|
self.revisions = [self.commit, *public_revisions] if public_base else ['--all', self.commit]
|
|
42
42
|
from history_hydration import prepare
|
|
43
43
|
self.hydration = prepare(self)
|
|
44
|
-
self.env['GIT_NO_LAZY_FETCH'] = '1'
|
|
45
44
|
self.commits = self.run('rev-list', *self.revisions).decode().splitlines()
|
|
46
45
|
self.public_commits = set(self.run('rev-list', *public_revisions).decode().splitlines()) if public_base else set()
|
|
47
46
|
self.trees = {}
|
|
@@ -49,10 +48,10 @@ class History:
|
|
|
49
48
|
self.reader = subprocess.Popen(['git', '-C', self.repo, 'cat-file', '--batch'],
|
|
50
49
|
env=self.env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
|
51
50
|
|
|
52
|
-
def run(self, *args):
|
|
51
|
+
def run(self, *args, input=None):
|
|
53
52
|
try:
|
|
54
53
|
result = subprocess.run(['git', '-C', self.repo, '-c', 'core.quotePath=false', *args],
|
|
55
|
-
env=self.env, capture_output=True, timeout=120)
|
|
54
|
+
env=self.env, input=input, capture_output=True, timeout=120)
|
|
56
55
|
if result.returncode:
|
|
57
56
|
raise InspectionError('history.git-failed')
|
|
58
57
|
return result.stdout
|
|
@@ -89,11 +88,13 @@ class History:
|
|
|
89
88
|
self.reader.stdin.flush()
|
|
90
89
|
header = self.reader.stdout.readline().split()
|
|
91
90
|
if len(header) != 3 or header[1] != b'blob':
|
|
92
|
-
|
|
91
|
+
from history_hydration import missing_blob
|
|
92
|
+
raise missing_blob(self, oid)
|
|
93
93
|
size = int(header[2])
|
|
94
94
|
data = self.reader.stdout.read(size)
|
|
95
95
|
if len(data) != size or self.reader.stdout.read(1) != b'\n':
|
|
96
|
-
|
|
96
|
+
from history_hydration import missing_blob
|
|
97
|
+
raise missing_blob(self, oid)
|
|
97
98
|
return data
|
|
98
99
|
|
|
99
100
|
def selected(self, path):
|
|
@@ -3,26 +3,49 @@ from time import monotonic
|
|
|
3
3
|
from git_history import InspectionError
|
|
4
4
|
|
|
5
5
|
|
|
6
|
+
def missing_blob(history, oid):
|
|
7
|
+
# Sizes need the blob itself. Locate a promised object using tree entries
|
|
8
|
+
# without -l, so a failed fetch cannot trigger another lazy fetch.
|
|
9
|
+
trees = set(history.run('log', '--format=%T', *history.revisions).splitlines())
|
|
10
|
+
for tree in sorted(trees):
|
|
11
|
+
for record in history.run('ls-tree', '-r', '-z', tree.decode()).split(b'\0'):
|
|
12
|
+
metadata, separator, path = record.partition(b'\t')
|
|
13
|
+
if separator and metadata.split()[2].decode() == oid:
|
|
14
|
+
return InspectionError(f'missing blob {oid} at {path.decode(errors="replace")}: fetch the full history')
|
|
15
|
+
return InspectionError(f'missing blob {oid} at <unknown path>: fetch the full history')
|
|
16
|
+
|
|
17
|
+
|
|
6
18
|
def prepare(history):
|
|
7
19
|
started = monotonic()
|
|
8
20
|
def missing():
|
|
9
|
-
return
|
|
10
|
-
|
|
21
|
+
return sorted(line[1:] for line in
|
|
22
|
+
history.run('rev-list', '--objects', '--missing=print', *history.revisions).splitlines()
|
|
23
|
+
if line.startswith(b'?'))
|
|
11
24
|
before = missing()
|
|
12
25
|
fetches = 0
|
|
13
26
|
if before:
|
|
14
27
|
# A blob:none clone already owns the commits and trees. ls-tree -l and
|
|
15
28
|
# cat-file would otherwise ask its promisor for one blob at a time.
|
|
16
|
-
|
|
29
|
+
try:
|
|
30
|
+
remotes = history.run('config', '--get-regexp', r'^remote\..*\.promisor$').decode().splitlines()
|
|
31
|
+
except InspectionError:
|
|
32
|
+
remotes = []
|
|
17
33
|
names = [line.rsplit(None, 1)[0][len('remote.'):-len('.promisor')]
|
|
18
34
|
for line in remotes if line.rsplit(None, 1)[-1].lower() == 'true']
|
|
19
35
|
if len(names) != 1:
|
|
20
|
-
raise
|
|
21
|
-
|
|
22
|
-
|
|
36
|
+
raise missing_blob(history, before[0].decode())
|
|
37
|
+
try:
|
|
38
|
+
# Explicit object wants cover selected commits fetched outside the
|
|
39
|
+
# remote's configured branch refspecs too. stdin keeps one batch
|
|
40
|
+
# even when a full history exceeds the command argument limit.
|
|
41
|
+
history.run('fetch', '--quiet', '--refetch', '--no-filter', '--no-tags',
|
|
42
|
+
'--no-recurse-submodules', '--no-write-fetch-head', '--stdin', '--', names[0],
|
|
43
|
+
input=b'\n'.join(before) + b'\n')
|
|
44
|
+
except InspectionError:
|
|
45
|
+
raise missing_blob(history, before[0].decode()) from None
|
|
23
46
|
fetches = 1
|
|
24
|
-
after = missing() if before else
|
|
47
|
+
after = missing() if before else []
|
|
25
48
|
if after:
|
|
26
|
-
raise
|
|
27
|
-
return dict(missingBefore=before, missingAfter=after, fetches=fetches,
|
|
49
|
+
raise missing_blob(history, after[0].decode())
|
|
50
|
+
return dict(missingBefore=len(before), missingAfter=len(after), fetches=fetches,
|
|
28
51
|
seconds=round(monotonic() - started, 6))
|