@runeditrun/rer 0.2.66 → 0.2.68
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/business/vendors/README.md +38 -2
- package/cli/business/vendors/calendly.mjs +2 -2
- package/cli/business/vendors/common.mjs +94 -0
- package/cli/business/vendors/credential-names.mjs +124 -0
- package/cli/business/vendors/pagerduty.mjs +6 -15
- package/cli/business/vendors/sentry.mjs +4 -3
- package/cli/business/vendors/uptimerobot.mjs +7 -6
- package/cli/import-summary.mjs +7 -0
- package/cli/output.mjs +2 -0
- package/cli/portability-worker.mjs +10 -4
- package/cli/portability.mjs +1 -0
- package/cli/production-mail.mjs +4 -1
- package/global/CHANGELOG.md +14 -0
- package/modules/deploy-kit/d1-import.mjs +3 -1
- package/modules/deploy-kit/migrations.mjs +1 -0
- package/modules/deploy-kit/owner-cli.mjs +1 -1
- package/modules/deploy-kit/provider.mjs +16 -4
- package/modules/portability/README.md +73 -8
- package/modules/portability/bridge.mjs +44 -29
- package/modules/portability/catalogue.mjs +33 -3
- package/modules/portability/files.mjs +9 -3
- package/modules/portability/import-safety.mjs +80 -0
- package/modules/portability/import.mjs +13 -7
- package/modules/portability/index.d.mts +9 -3
- package/modules/portability/index.mjs +1 -1
- package/modules/portability/ledgers.mjs +83 -0
- package/modules/portability/legacy-files.mjs +7 -4
- package/modules/portability/migration-history.mjs +3 -2
- package/modules/portability/migration-statements.mjs +83 -9
- package/modules/release/README.md +25 -7
- package/modules/release/cleanup.mjs +16 -0
- package/modules/release/produce.mjs +11 -4
- package/modules/release/publication-scan.mjs +18 -8
- package/package.json +1 -1
- package/scripts/publish-check/README.md +5 -2
- package/scripts/publish-check/destination.py +8 -1
- package/scripts/publish-check/git_history.py +7 -6
- package/scripts/publish-check/history_hydration.py +32 -9
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { all, insist } from './storage.mjs';
|
|
2
|
+
import { identifier } from './bookkeeping.mjs';
|
|
3
|
+
import { canonicalJson } from './hash.mjs';
|
|
4
|
+
import { validateRow } from './catalogue.mjs';
|
|
5
|
+
|
|
6
|
+
// Migration ledgers, by the runner that writes them: Base deploy and container
|
|
7
|
+
// runners (_rer_deploy_migrations), Wrangler and the Base deploy kit
|
|
8
|
+
// (d1_migrations, tracked or plain) and RER Platform (rer_migrations).
|
|
9
|
+
export const ledgerNames=['_rer_deploy_migrations','d1_migrations','rer_migrations'];
|
|
10
|
+
// Platform's ledger exactly as its runner creates it
|
|
11
|
+
// (platform src/integrations/cloudflare-migrations.ts:33).
|
|
12
|
+
const platformShape=canonicalJson([{name:'name',type:'TEXT',notnull:0,pk:1},{name:'sha256',type:'TEXT',notnull:1,pk:0},{name:'applied_at',type:'TEXT',notnull:1,pk:0}]);
|
|
13
|
+
const shape=columns=>canonicalJson(columns.map(({name,type,notnull,pk})=>({name,type,notnull,pk})));
|
|
14
|
+
// Platform records a file applied in part as <name>#<start>-<end> until the
|
|
15
|
+
// file completes (platform src/integrations/migration-batch.ts:31): its effects
|
|
16
|
+
// are uncertain, like a tracked ledger's 'applying' row.
|
|
17
|
+
const partial=name=>name.includes('#');
|
|
18
|
+
|
|
19
|
+
/** The destination's applied product migrations, [{name,sha256?}] in the ledger's own order, or undefined without a ledger. */
|
|
20
|
+
export async function destinationLedger(db) {
|
|
21
|
+
const found=(await all(db,`SELECT name FROM sqlite_master WHERE type='table' AND name IN (${ledgerNames.map(n=>`'${n}'`).join(',')})`)).map(x=>x.name);
|
|
22
|
+
if(found.includes('_rer_deploy_migrations'))return (await all(db,"SELECT * FROM _rer_deploy_migrations WHERE state='applied' ORDER BY name")).map(entry);
|
|
23
|
+
if(found.includes('d1_migrations')) {
|
|
24
|
+
const columns=(await all(db,'PRAGMA table_info(d1_migrations)')).map(x=>x.name);
|
|
25
|
+
const tracked=columns.includes('state') || columns.includes('sha256');
|
|
26
|
+
if(tracked)insist(columns.includes('state') && columns.includes('sha256'),'schema_mismatch:migration_history',409);
|
|
27
|
+
const rows=await all(db,`SELECT name${tracked?',state,sha256':''} FROM d1_migrations ORDER BY id`);
|
|
28
|
+
if(tracked)insist(rows.every(row=>row.state==='applied'),'schema_mismatch:uncertain_migrations',409);
|
|
29
|
+
return rows.map(entry);
|
|
30
|
+
}
|
|
31
|
+
if(!found.includes('rer_migrations'))return;
|
|
32
|
+
// Platform applies pending files in its plan's order and inserts each ledger
|
|
33
|
+
// row in the same request as the file's SQL (migration-batch.ts:35-38,43-64;
|
|
34
|
+
// cloudflare-migrations.ts:34-44), so rowid order is application order.
|
|
35
|
+
insist(shape(await all(db,'PRAGMA table_info(rer_migrations)'))===platformShape,'schema_mismatch:migration_history',409);
|
|
36
|
+
const rows=await all(db,'SELECT name,sha256 FROM rer_migrations ORDER BY rowid');
|
|
37
|
+
insist(!rows.some(row=>partial(row.name)),'schema_mismatch:uncertain_migrations',409);
|
|
38
|
+
return rows.map(entry);
|
|
39
|
+
}
|
|
40
|
+
const entry=row=>({name:row.name,...(typeof row.sha256==='string'?{sha256:row.sha256}:{})});
|
|
41
|
+
|
|
42
|
+
/** The ledger tables an archive carries (old archives carried rer_migrations before it became bookkeeping). */
|
|
43
|
+
export const archivedLedgerTables=manifest=>manifest.tables.filter(t=>ledgerNames.includes(identifier(t.name)));
|
|
44
|
+
|
|
45
|
+
/** Collects the rows of an archive's ledger tables while its records are read: add(record) each, then ledgers() is {table:rows}, or undefined without one. */
|
|
46
|
+
export function ledgerReader() {
|
|
47
|
+
let ledgers;
|
|
48
|
+
return {
|
|
49
|
+
add(record) {
|
|
50
|
+
if(record.kind==='manifest') {const tables=archivedLedgerTables(record);if(tables.length)ledgers=Object.fromEntries(tables.map(t=>[t.name,[]]));}
|
|
51
|
+
if(record.kind==='part' && ledgers)for(const item of record.items)if(item.kind==='row' && ledgers[item.table])ledgers[item.table].push(item.row);
|
|
52
|
+
},
|
|
53
|
+
ledgers:()=>ledgers,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/** Checks rows a caller read from the archive for its ledger tables ({table:rows}), so uploadPart can bind each to the authenticated parts. */
|
|
57
|
+
export function checkArchivedLedgers(manifest,ledgers) {
|
|
58
|
+
if(ledgers===undefined)return;
|
|
59
|
+
const tables=archivedLedgerTables(manifest);
|
|
60
|
+
insist(ledgers && typeof ledgers==='object' && Object.keys(ledgers).every(name=>tables.some(t=>t.name===name)),'ledger_inventory');
|
|
61
|
+
for(const [name,rows] of Object.entries(ledgers)) {
|
|
62
|
+
const table=tables.find(t=>t.name===name);
|
|
63
|
+
insist(Array.isArray(rows) && rows.length===table.count,'ledger_inventory');
|
|
64
|
+
for(const row of rows)validateRow(table,row,manifest.excludedColumns[name]??[]);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** The applied migrations an archive's own ledger records, [{name,sha256?}]; undefined when it carries no ledger. */
|
|
69
|
+
export function archivedHistory(manifest,ledgers) {
|
|
70
|
+
const tables=archivedLedgerTables(manifest);
|
|
71
|
+
if(!tables.length)return;
|
|
72
|
+
insist(tables.length===1 && ledgers?.[tables[0].name],'schema_mismatch:migration_history');
|
|
73
|
+
const [table]=tables,rows=ledgers[table.name],columns=table.columns.map(c=>c.name);
|
|
74
|
+
if(identifier(table.name)==='rer_migrations') {
|
|
75
|
+
insist(shape(table.columns)===platformShape,'schema_mismatch:migration_history');
|
|
76
|
+
insist(!rows.some(row=>partial(row.name)),'schema_mismatch:uncertain_migrations');
|
|
77
|
+
}else{
|
|
78
|
+
insist(columns.includes('name') && columns.includes('state')===columns.includes('sha256'),'schema_mismatch:migration_history');
|
|
79
|
+
insist(rows.every(row=>row.state===undefined || row.state==='applied'),'schema_mismatch:uncertain_migrations');
|
|
80
|
+
}
|
|
81
|
+
insist(rows.every(row=>typeof row.name==='string') && new Set(rows.map(row=>row.name)).size===rows.length,'schema_mismatch:migration_history');
|
|
82
|
+
return rows.map(entry);
|
|
83
|
+
}
|
|
@@ -6,13 +6,16 @@ import { readLegacyArchive, ArchiveVerifier } from './format.mjs';
|
|
|
6
6
|
import { fileRecords, exportArchive } from './files.mjs';
|
|
7
7
|
import { ensureLegacyAudit, legacyTargetState, requireLegacyEmpty } from './legacy-safety.mjs';
|
|
8
8
|
import { insist, PortabilityError } from './storage.mjs';
|
|
9
|
+
import { ledgerReader } from './ledgers.mjs';
|
|
9
10
|
|
|
10
11
|
export async function recoverLegacyArchive({db,buckets,policy:input,migrations},path,output,transient,acknowledged) {
|
|
11
12
|
insist(acknowledged===true,'legacy_acknowledgement_required',409);
|
|
12
13
|
insist(typeof output==='string' && output!==path,'legacy_output_required',409);
|
|
13
14
|
insist(!(await lstat(output).catch(error=>{if(error.code==='ENOENT')return null;throw error;})),'legacy_output_exists',409);
|
|
14
|
-
// Whole-file verification precedes even staging/audit schema writes.
|
|
15
|
-
|
|
15
|
+
// Whole-file verification precedes even staging/audit schema writes. It also
|
|
16
|
+
// reads the archive's own migration ledger, if any, for placement (bridge.mjs).
|
|
17
|
+
const ledger=ledgerReader();
|
|
18
|
+
for await(const record of fileRecords(path,readLegacyArchive))ledger.add(record);
|
|
16
19
|
const context={db,buckets,config:policy(input),migrations},target=await legacyTargetState(db,migrations);
|
|
17
20
|
await requireLegacyEmpty(context,target);
|
|
18
21
|
let id,manifest,closing,integrity,status,complete=false;
|
|
@@ -26,7 +29,7 @@ export async function recoverLegacyArchive({db,buckets,policy:input,migrations},
|
|
|
26
29
|
event:'legacy_v1_unverified_restore',acknowledgedUnverifiedMetadata:true,
|
|
27
30
|
sourceIdentity:record.sourceIdentity,exportedAt:record.exportedAt,
|
|
28
31
|
recoveryOutput:output,...target,
|
|
29
|
-
});
|
|
32
|
+
},ledger.ledgers());
|
|
30
33
|
// A refused schema writes nothing, not even the audit table.
|
|
31
34
|
await ensureLegacyAudit(db);
|
|
32
35
|
}else if(record.kind==='part') {integrity.add(record);await uploadPart(context,id,record.index,record);}
|
|
@@ -38,6 +41,6 @@ export async function recoverLegacyArchive({db,buckets,policy:input,migrations},
|
|
|
38
41
|
// A failure here must be reported; the durable restore audit remains intact.
|
|
39
42
|
try {
|
|
40
43
|
const recovered=await exportArchive(await createPortability({db,buckets,policy:input}),output,transient);
|
|
41
|
-
return {...recovered,legacyRecovery:true,unverifiedMetadata:true,restoredContentHash:closing.contentHash,...(status.bridged?{bridged:status.bridged}:{})};
|
|
44
|
+
return {...recovered,legacyRecovery:true,unverifiedMetadata:true,restoredContentHash:closing.contentHash,...(status.bridged?{bridged:status.bridged}:{}),...(status.safetyChanges?{safetyChanges:status.safetyChanges}:{})};
|
|
42
45
|
}catch{throw new PortabilityError('legacy_restored_export_failed',500);}
|
|
43
46
|
}
|
|
@@ -4,10 +4,11 @@ import { policy, schemaIdentity } from './catalogue.mjs';
|
|
|
4
4
|
// Replays a chosen set of the destination's migrations, in its applied order,
|
|
5
5
|
// into a scratch database and returns the resulting archive schema identity.
|
|
6
6
|
// Node runners only: restore uses it to place an older archive within the
|
|
7
|
-
// destination's history (see bridge.mjs). plan: [{name,statements}]
|
|
7
|
+
// destination's history (see bridge.mjs). plan: [{name,statements,sha256?}],
|
|
8
|
+
// sha256 being the migration file's hash where the runner read the file.
|
|
8
9
|
export function migrationReplay(plan,input) {
|
|
9
10
|
const config=policy(input);
|
|
10
|
-
return {names:plan.map(migration=>migration.name),statements:name=>plan.find(m=>m.name===name).statements,async schemaAfter(names) {
|
|
11
|
+
return {names:plan.map(migration=>migration.name),statements:name=>plan.find(m=>m.name===name).statements,sha256:name=>plan.find(m=>m.name===name).sha256,async schemaAfter(names) {
|
|
11
12
|
const scratch=new DatabaseSync(':memory:');
|
|
12
13
|
const db={prepare:sql=>{const query=scratch.prepare(sql);let values=[];return {bind(...args){values=args;return this;},async all(){return {results:query.all(...values)};}};}};
|
|
13
14
|
try {
|
|
@@ -3,18 +3,92 @@ import { tokens, identifier, folded } from './generated.mjs';
|
|
|
3
3
|
// Whether one migration statement can change rows a restore carries. Only
|
|
4
4
|
// CREATE TABLE (not AS SELECT), CREATE INDEX and ALTER TABLE ... ADD COLUMN
|
|
5
5
|
// are additive: SQLite refuses ADD COLUMN with a non-constant default, so a
|
|
6
|
-
// restored row takes the same value the migration gave existing rows.
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
6
|
+
// restored row takes the same value the migration gave existing rows.
|
|
7
|
+
// CREATE TRIGGER and DROP TRIGGER are schema-only whatever the body does:
|
|
8
|
+
// neither runs the body, so no row changes when the migration runs, and a
|
|
9
|
+
// restore inserts rows with the destination's triggers dropped and recreates
|
|
10
|
+
// them afterwards (import.mjs finalizeImport), exactly as the live upgrade left
|
|
11
|
+
// the rows that predate the trigger. That holds only for a statement bounded
|
|
12
|
+
// as exactly one trigger statement; any other (INSERT, UPDATE, DELETE, table
|
|
13
|
+
// rebuilds, a trigger glued to a further statement) changes data unless it
|
|
14
|
+
// names none of `reached` (reachingTables), as when it only touches
|
|
15
|
+
// operator-auth or other excluded tables that no trigger links to restorable ones.
|
|
16
|
+
// `empty` lists the restorable tables the archive holds no rows of: restore
|
|
17
|
+
// leaves them empty, and an UPDATE or DELETE of an empty table changes nothing
|
|
18
|
+
// (no row fires a trigger either), so such a statement changes data only if it
|
|
19
|
+
// names another reached table (one it could write, or read into its writes).
|
|
20
|
+
// Any other statement naming an empty table still does: an INSERT would leave
|
|
21
|
+
// rows the restore lacks.
|
|
22
|
+
export function changesData(sql,reached,empty=[]) {
|
|
23
|
+
const list=tokens(sql),words=list.map(t=>t.quoted?undefined:folded(t.text)),kind=kindOf(words);
|
|
13
24
|
if(kind==='create table')return topLevelWords(list,words).includes('as');
|
|
14
25
|
if(kind==='create index')return false;
|
|
15
26
|
if(kind==='alter table' && words.find((w,i)=>i>2 && ['add','rename','drop'].includes(w))==='add')return false;
|
|
16
|
-
|
|
17
|
-
|
|
27
|
+
if(kind==='create trigger' && createdTrigger(list))return false;
|
|
28
|
+
if(kind==='drop trigger' && droppedTrigger(list))return false;
|
|
29
|
+
const skipped=new Set(['update','delete'].includes(words[0])?empty.map(folded):[]);
|
|
30
|
+
return names(list,new Set(reached.map(folded).filter(name=>!skipped.has(name))));
|
|
31
|
+
}
|
|
32
|
+
// The restorable tables and every table whose writes reach one through a
|
|
33
|
+
// trigger any of `statements` creates: a write to such a table during a later
|
|
34
|
+
// migration changes restorable rows the archive lacks. A trigger counts when
|
|
35
|
+
// its statement names a reached table anywhere; one this parser cannot bound
|
|
36
|
+
// adds every name it holds.
|
|
37
|
+
export function reachingTables(statements,restorable) {
|
|
38
|
+
const reached=new Set(restorable.map(folded));
|
|
39
|
+
const triggers=statements.map(sql=>tokens(sql)).filter(list=>kindOf(list.map(t=>t.quoted?undefined:folded(t.text)))==='create trigger')
|
|
40
|
+
.map(list=>({list,on:createdTrigger(list)?.on}));
|
|
41
|
+
for(let grew=true;grew;) {
|
|
42
|
+
grew=false;
|
|
43
|
+
for(const {list,on} of triggers)if(names(list,reached))for(const name of on===undefined?identifiers(list):[on])if(!reached.has(name)){reached.add(name);grew=true;}
|
|
44
|
+
}
|
|
45
|
+
return [...reached];
|
|
46
|
+
}
|
|
47
|
+
const kindOf=words=>words.slice(0,4).filter(w=>!['temp','temporary','unique'].includes(w)).slice(0,2).join(' ');
|
|
48
|
+
const identifiers=list=>list.flatMap(t=>t.quoted?(t.text.startsWith("'")?[]:[folded(identifier(t))]):folded(t.text).split(/[^a-z0-9_$\u0080-\uffff]+/).filter(Boolean));
|
|
49
|
+
const names=(list,reached)=>identifiers(list).some(name=>reached.has(name));
|
|
50
|
+
// Tokens with each unquoted ';' split out, and trailing ones dropped.
|
|
51
|
+
function atoms(list) {
|
|
52
|
+
const result=list.flatMap(t=>t.quoted?[t]:t.text.split(/(;)/).filter(Boolean).map(text=>({text})));
|
|
53
|
+
while(result.length && semicolon(result.at(-1)))result.pop();
|
|
54
|
+
return result;
|
|
55
|
+
}
|
|
56
|
+
const semicolon=a=>!a.quoted && a.text===';';
|
|
57
|
+
const word=(a,value)=>a && !a.quoted && folded(a.text)===value;
|
|
58
|
+
// {on} for CREATE [TEMP] TRIGGER ... ON table ... BEGIN statement; ... END and
|
|
59
|
+
// nothing after it. The body ends at the first END that starts a statement;
|
|
60
|
+
// CASE ... END inside a statement does not. Anything else is unbounded.
|
|
61
|
+
function createdTrigger(list) {
|
|
62
|
+
const parts=atoms(list);let depth=0,begin=-1,on;
|
|
63
|
+
for(let i=0;i<parts.length && begin<0;i++) {
|
|
64
|
+
const a=parts[i];
|
|
65
|
+
if(a.quoted)continue;
|
|
66
|
+
if(a.text===';')return;
|
|
67
|
+
if(a.text==='(')depth++;
|
|
68
|
+
else if(a.text===')'){if(--depth<0)return;}
|
|
69
|
+
else if(!depth && on===undefined && word(a,'on')) {
|
|
70
|
+
const name=[];
|
|
71
|
+
for(let j=i+1;j<parts.length && !['for','when','begin'].some(w=>word(parts[j],w));j++)name.push(parts[j]);
|
|
72
|
+
const last=name.at(-1);
|
|
73
|
+
if(!last || name.length>3)return;
|
|
74
|
+
on=folded(last.quoted?identifier(last):last.text.split('.').at(-1));
|
|
75
|
+
if(!on)return;
|
|
76
|
+
}else if(!depth && word(a,'begin'))begin=i;
|
|
77
|
+
}
|
|
78
|
+
if(begin<0 || on===undefined)return;
|
|
79
|
+
let start=true,statements=0;
|
|
80
|
+
for(let i=begin+1;i<parts.length;i++) {
|
|
81
|
+
const a=parts[i];
|
|
82
|
+
if(start && word(a,'end'))return i===parts.length-1 && statements && !depth?{on}:undefined;
|
|
83
|
+
if(semicolon(a)){if(start || depth)return;start=true;statements++;continue;}
|
|
84
|
+
if(!a.quoted){if(a.text==='(')depth++;else if(a.text===')' && --depth<0)return;}
|
|
85
|
+
start=false;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
// DROP TRIGGER [IF EXISTS] [schema.]name, and nothing after it.
|
|
89
|
+
function droppedTrigger(list) {
|
|
90
|
+
const parts=atoms(list),from=word(parts[2],'if') && word(parts[3],'exists')?4:2,name=parts.slice(from);
|
|
91
|
+
return name.length>=1 && name.length<=3 && name.every(a=>a.quoted?!a.text.startsWith("'"):!/[;(),]/.test(a.text));
|
|
18
92
|
}
|
|
19
93
|
function topLevelWords(list,words) {
|
|
20
94
|
let depth=0;const result=[];
|
|
@@ -15,7 +15,14 @@ pnpm rer release --repo /path/to/seed --ref main \
|
|
|
15
15
|
|
|
16
16
|
The `rer` bin and `node bin/rer.mjs` expose the same command. Node 22+, Python 3,
|
|
17
17
|
Git, pnpm and tar are required, along with authenticated `gh` for GitHub sources.
|
|
18
|
-
Use a full, clean seed clone with its declared package-manager version
|
|
18
|
+
Use a full-history, clean seed clone with its declared package-manager version;
|
|
19
|
+
`--filter=blob:none` is supported, while shallow history is refused. Before
|
|
20
|
+
release work begins, tracked changes and untracked files in the source checkout
|
|
21
|
+
are refused; ignored local files do not become release inputs. Before
|
|
22
|
+
reading source files, destination validation uses the publication scanner's
|
|
23
|
+
single batched hydration fetch. An unavailable blob reports its object ID and
|
|
24
|
+
committed path with `fetch the full history` guidance, separately from identity
|
|
25
|
+
disagreements. The producer reads
|
|
19
26
|
committed `seed.json`, `package.json`, `RELEASE.json`, `LICENSE`, `CONTRACT.md` and
|
|
20
27
|
`AGENTS.md`. Version declarations must agree. `--project` declares the publishing
|
|
21
28
|
identity; `--repository` is the credential-free public destination URL. Neither is inferred
|
|
@@ -33,6 +40,11 @@ URLs; identity comparison ignores case and a terminal `.git` in URLs. Missing,
|
|
|
33
40
|
malformed, partial or inconsistent identities fail closed. The selected
|
|
34
41
|
committed tree supplies the declarations.
|
|
35
42
|
|
|
43
|
+
Temporary release directories are removed recursively, including incidental
|
|
44
|
+
Finder metadata. A cleanup failure keeps a completed release successful and
|
|
45
|
+
records a `temporary-cleanup-failed` warning in stderr and the release receipt;
|
|
46
|
+
its directory can then be removed manually.
|
|
47
|
+
|
|
36
48
|
The platform reads these same HTTPS GitHub URLs, plus bare an owner and repository name separated by a slash for releases that carry it; ordinary release editions still require the URL here.
|
|
37
49
|
|
|
38
50
|
Every publication scan receives the explicit project and destination: source
|
|
@@ -87,9 +99,14 @@ path needs the seed's browser prerequisites and any resources its verification
|
|
|
87
99
|
requires. Routine CI and default verification do not invoke the release-only
|
|
88
100
|
script.
|
|
89
101
|
|
|
90
|
-
Both paths then
|
|
91
|
-
|
|
92
|
-
|
|
102
|
+
Both paths then run publish-check in a fresh disposable clone checked out at
|
|
103
|
+
the selected source commit. Its working tree contains exactly that commit's
|
|
104
|
+
committed files, isolated from the caller's ignored or untracked files and from
|
|
105
|
+
installation, build and verification outputs. The clone retains all source refs
|
|
106
|
+
and full history: committed secrets and historical findings remain subject to
|
|
107
|
+
the publication checks. When the source checkout's origin is a confirmed public
|
|
108
|
+
repository, the scanner uses that source repository's advertised default branch
|
|
109
|
+
and published tags as its public base.
|
|
93
110
|
Already-public findings are classified pre-existing; unpublished content still
|
|
94
111
|
blocks. Other sources retain complete local-ref history inspection.
|
|
95
112
|
A passing step's receipt retains exit status and numeric final summaries from
|
|
@@ -202,9 +219,10 @@ Public export commit and annotated-tag messages contain the version and optional
|
|
|
202
219
|
`RELEASE.json.publicReleaseNotes` only. Private source commit identities stay in
|
|
203
220
|
the private release receipt. The clean export's final annotated tag exists
|
|
204
221
|
before publication inspection, and the checked repository is copied unchanged
|
|
205
|
-
into `public-git/`. Ordinary releases
|
|
206
|
-
|
|
207
|
-
the source tag. Release notes pass the same content rules as the
|
|
222
|
+
into `public-git/`. Ordinary releases stage the exact final annotation in the
|
|
223
|
+
fresh selected-commit checkout and inspect it with the source history before
|
|
224
|
+
creating the source tag. Release notes pass the same content rules as the
|
|
225
|
+
public tree.
|
|
208
226
|
|
|
209
227
|
## Releasing a seed that is already public
|
|
210
228
|
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { rmSync } from 'node:fs';
|
|
2
|
+
|
|
3
|
+
// Finder and other filesystem observers can add metadata while removal walks
|
|
4
|
+
// the directory. Retry those races, but never turn finished artifacts into a
|
|
5
|
+
// failed release merely because temporary files could not be removed.
|
|
6
|
+
export function cleanupTemporary(directory) {
|
|
7
|
+
try {
|
|
8
|
+
rmSync(directory, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
9
|
+
return null;
|
|
10
|
+
} catch (error) {
|
|
11
|
+
const warning = { code: 'temporary-cleanup-failed', errorCode: error.code ?? 'UNKNOWN',
|
|
12
|
+
directory, message: 'Temporary release cleanup failed; remove the remaining directory manually.' };
|
|
13
|
+
console.warn(`Warning: ${warning.message} (${warning.errorCode}): ${directory}`);
|
|
14
|
+
return warning;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
@@ -11,6 +11,7 @@ import { writePublishHandoff } from './publish-handoff.mjs';
|
|
|
11
11
|
import { releasePublicBase } from './public-base.mjs';
|
|
12
12
|
import { prepareTagAdoption, confirmTagAdoption, assertTagAdoption } from './tag-adoption.mjs';
|
|
13
13
|
import { placeholderMarks, placeholderMessage } from '../../cli/placeholder-mark.mjs';
|
|
14
|
+
import { cleanupTemporary } from './cleanup.mjs';
|
|
14
15
|
|
|
15
16
|
// Failure output belongs to the private receipt, not the site preview.
|
|
16
17
|
const withoutOutput = gates => JSON.parse(JSON.stringify(gates, (key, value) => key === 'output' ? undefined : value));
|
|
@@ -21,13 +22,15 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
|
|
|
21
22
|
if (onto !== undefined && clean) throw Error('--onto and --clean-history are mutually exclusive');
|
|
22
23
|
if (publicBase !== undefined && (onto !== undefined || clean)) throw Error('--public-base selects exact-source history; cannot combine with export history modes');
|
|
23
24
|
const preview = dryRun || !tag;
|
|
25
|
+
// The shared history reader hydrates promised blobs once, before any source
|
|
26
|
+
// metadata, placeholder inspection or release export reads their contents.
|
|
27
|
+
validatePublicationDestination({ repo, commit: ref }, project, repository);
|
|
24
28
|
const source = inspectSource(repo, ref);
|
|
25
29
|
// A release refuses a seed still wearing Base's placeholder mark unless the
|
|
26
30
|
// caller allows it explicitly; the receipt records that allowance.
|
|
27
31
|
const files = [...source.entries].filter(([, entry]) => entry.type === 'blob' && ['100644', '100755'].includes(entry.mode)).map(([path]) => path);
|
|
28
32
|
const marks = await placeholderMarks(source.entries.has('OPERATIONS.json') ? JSON.parse(source.bytes('OPERATIONS.json')).project : project, files, source.bytes);
|
|
29
33
|
if (marks.length && !allowPlaceholderMark) throw Object.assign(Error(`${placeholderMessage(marks)} Release refused; pass --allow-placeholder-mark to release with it.`), { code: 'placeholder_mark' });
|
|
30
|
-
validatePublicationDestination(source, project, repository);
|
|
31
34
|
releaseMessage(source);
|
|
32
35
|
if (typeof tag === 'string' && tag !== source.tag) throw Error('Requested tag must match the committed release version');
|
|
33
36
|
const existing = git(source.repo, 'tag', '--list', source.tag).toString().trim();
|
|
@@ -44,6 +47,7 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
|
|
|
44
47
|
output = newOutput(source.repo, output);
|
|
45
48
|
const temporary = mkdtempSync(resolve(tmpdir(), 'rer-release-'));
|
|
46
49
|
let receipt;
|
|
50
|
+
let cleanupAttempted = false;
|
|
47
51
|
try {
|
|
48
52
|
const checkout = resolve(temporary, 'seed');
|
|
49
53
|
git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, checkout);
|
|
@@ -117,17 +121,20 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
|
|
|
117
121
|
});
|
|
118
122
|
}
|
|
119
123
|
}
|
|
120
|
-
if (passed && tag && destination) writePublishHandoff(output, destination, receipt);
|
|
121
124
|
if (passed && adopted) {
|
|
122
125
|
assertTagAdoption(source, adopted, { imported: true });
|
|
123
126
|
if (receipt.tag.object !== adopted.object || receipt.publicExport.tagObject !== adopted.object ||
|
|
124
127
|
receipt.publicExport.commit !== adopted.commit || receipt.source.commit !== adopted.commit)
|
|
125
128
|
throw Error('Adopted release tag identity does not match the release receipt');
|
|
126
129
|
}
|
|
130
|
+
cleanupAttempted = true;
|
|
131
|
+
const warning = cleanupTemporary(temporary);
|
|
132
|
+
if (warning) receipt.warnings = [warning];
|
|
133
|
+
if (passed && tag && destination) writePublishHandoff(output, destination, receipt);
|
|
127
134
|
writeFileSync(resolve(output, 'release-receipt.json'), json(receipt), { flag: 'wx' });
|
|
128
135
|
if (passed && destination) {
|
|
129
136
|
requirePublicDestination(destination);
|
|
130
|
-
if (remotePublicHead(
|
|
137
|
+
if (remotePublicHead(tag ? resolve(output, 'public-git') : source.repo, destination, source.tag) !== publicParent) throw Error('Public destination changed before handoff');
|
|
131
138
|
if (tag && !dryRun) annotate(source.repo, source, source.commit);
|
|
132
139
|
}
|
|
133
140
|
return receipt;
|
|
@@ -135,5 +142,5 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
|
|
|
135
142
|
// No incomplete directory should be mistaken for a completed export.
|
|
136
143
|
rmSync(output, { recursive: true, force: true });
|
|
137
144
|
throw error;
|
|
138
|
-
} finally {
|
|
145
|
+
} finally { if (!cleanupAttempted) cleanupTemporary(temporary); }
|
|
139
146
|
}
|
|
@@ -2,7 +2,7 @@ import { execFileSync } from 'node:child_process';
|
|
|
2
2
|
import { resolve } from 'node:path';
|
|
3
3
|
import { writeFileSync } from 'node:fs';
|
|
4
4
|
import { git } from './source.mjs';
|
|
5
|
-
import { annotate } from './export.mjs';
|
|
5
|
+
import { annotate, sourceTree } from './export.mjs';
|
|
6
6
|
import { runningVersion } from '../../cli/version.mjs';
|
|
7
7
|
|
|
8
8
|
export function validatePublicationDestination(source, project, repository) {
|
|
@@ -10,7 +10,11 @@ export function validatePublicationDestination(source, project, repository) {
|
|
|
10
10
|
execFileSync('python3', ['-B', resolve(import.meta.dirname, '../../scripts/publish-check/destination.py'),
|
|
11
11
|
'--repo', source.repo, '--ref', source.commit, '--project', project, '--repository', repository],
|
|
12
12
|
{ stdio: ['ignore', 'pipe', 'pipe'] });
|
|
13
|
-
} catch
|
|
13
|
+
} catch (error) {
|
|
14
|
+
if (error.status === 2) throw Error(error.stderr.toString().trim());
|
|
15
|
+
if (error.status === 1) throw Error('Publication destination must agree with the committed edition and project identities');
|
|
16
|
+
throw Error('Publication destination inspection could not run');
|
|
17
|
+
}
|
|
14
18
|
}
|
|
15
19
|
|
|
16
20
|
export function publicationScan(repo, ref, envFile, project, repository, publicBase, annotationFile) {
|
|
@@ -32,26 +36,32 @@ export function publicationScan(repo, ref, envFile, project, repository, publicB
|
|
|
32
36
|
}
|
|
33
37
|
|
|
34
38
|
// This is the ordinary release gate, also usable without installing/building a
|
|
35
|
-
// seed.
|
|
36
|
-
//
|
|
39
|
+
// seed. Materialize raw committed bytes, without checkout filters or export
|
|
40
|
+
// attributes, and mirror every source ref for the existing history policy.
|
|
41
|
+
// Neither the caller's local files nor verification output enters this tree.
|
|
37
42
|
export function sourcePublicationScan(source, temporary, envFile, project, repository, publicBase, tagged, adopted) {
|
|
38
43
|
const started = performance.now();
|
|
44
|
+
const candidate = sourceTree(source, resolve(temporary, 'publication'));
|
|
45
|
+
git(source.repo, 'clone', '--mirror', '--no-hardlinks', '--', source.repo, resolve(candidate, '.git'));
|
|
46
|
+
git(candidate, 'config', 'core.bare', 'false');
|
|
47
|
+
git(candidate, 'update-ref', '--no-deref', 'HEAD', source.commit);
|
|
48
|
+
git(candidate, 'read-tree', source.commit);
|
|
39
49
|
let annotationFile;
|
|
40
50
|
if (adopted) {
|
|
41
51
|
annotationFile = resolve(temporary, 'annotation.tag');
|
|
42
52
|
writeFileSync(annotationFile, git(adopted.repo, 'cat-file', 'tag', adopted.object), { mode: 0o600 });
|
|
43
53
|
} else if (tagged) {
|
|
44
|
-
const candidate = resolve(temporary, 'annotation');
|
|
45
|
-
git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, candidate);
|
|
46
|
-
git(candidate, 'checkout', '--detach', source.commit);
|
|
47
54
|
const staged = git(candidate, 'tag', '--list', source.tag).toString().trim();
|
|
48
55
|
const name = staged ? `${source.tag}-publication-candidate` : source.tag;
|
|
49
56
|
annotate(candidate, source, source.commit, name);
|
|
50
57
|
annotationFile = resolve(temporary, 'annotation.tag');
|
|
51
58
|
writeFileSync(annotationFile, git(candidate, 'cat-file', 'tag', `refs/tags/${name}`), { mode: 0o600 });
|
|
59
|
+
// Inspect the new tag once via annotationFile, leaving the mirrored source
|
|
60
|
+
// refs unchanged for the full-history scan.
|
|
61
|
+
git(candidate, 'tag', '-d', name);
|
|
52
62
|
}
|
|
53
63
|
const stagingSeconds = (performance.now() - started) / 1000;
|
|
54
|
-
const report = publicationScan(
|
|
64
|
+
const report = publicationScan(candidate, source.commit, envFile, project, repository, publicBase, annotationFile);
|
|
55
65
|
Object.assign(report.invocation, { stagingSeconds, totalSeconds: (performance.now() - started) / 1000 });
|
|
56
66
|
return report;
|
|
57
67
|
}
|
package/package.json
CHANGED
|
@@ -6,8 +6,11 @@ See [setup, command and scope](../publish-check-README.md).
|
|
|
6
6
|
blob reads to system Git (`rev-parse`, `rev-list`, `ls-tree`, `cat-file --batch`).
|
|
7
7
|
Before any tree inventory asks for blob sizes, `history_hydration.py` counts
|
|
8
8
|
missing objects and hydrates a partial clone with one `fetch --refetch --no-filter`
|
|
9
|
-
from its promisor remote.
|
|
10
|
-
|
|
9
|
+
from its promisor remote. The batch explicitly requests every missing object,
|
|
10
|
+
including selected history outside that remote's branch refspecs. A failed fetch
|
|
11
|
+
or remaining missing object makes the inspection incomplete and names a missing
|
|
12
|
+
blob and its committed path, with `fetch the full history` guidance.
|
|
13
|
+
`history.hydration` reports the missing counts, fetch
|
|
11
14
|
count and elapsed time; subsequent history reads disable lazy fetching.
|
|
12
15
|
|
|
13
16
|
Reports include implementation version, Python version and a digest of the
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import argparse
|
|
3
3
|
import json
|
|
4
4
|
import re
|
|
5
|
+
import sys
|
|
5
6
|
from github_refs import FULL
|
|
6
7
|
from project_identity import FIELDS, ordinary_document, tree_identity
|
|
7
8
|
|
|
@@ -115,7 +116,13 @@ if __name__ == '__main__':
|
|
|
115
116
|
try:
|
|
116
117
|
history = History(args.repo, args.ref)
|
|
117
118
|
destination(history, args.project, args.repository)
|
|
118
|
-
except
|
|
119
|
+
except InspectionError as error:
|
|
120
|
+
print(str(error), file=sys.stderr)
|
|
121
|
+
raise SystemExit(2) from None
|
|
122
|
+
except OSError:
|
|
123
|
+
print('Publication destination inspection could not run', file=sys.stderr)
|
|
124
|
+
raise SystemExit(2) from None
|
|
125
|
+
except (ValueError, TypeError, UnicodeError):
|
|
119
126
|
raise SystemExit(1) from None
|
|
120
127
|
finally:
|
|
121
128
|
if history:
|
|
@@ -11,7 +11,7 @@ class InspectionError(Exception):
|
|
|
11
11
|
|
|
12
12
|
def git_environment():
|
|
13
13
|
env = {k: v for k, v in os.environ.items() if not k.startswith('GIT_')}
|
|
14
|
-
env.update(GIT_TERMINAL_PROMPT='0', GIT_NO_REPLACE_OBJECTS='1')
|
|
14
|
+
env.update(GIT_TERMINAL_PROMPT='0', GIT_NO_REPLACE_OBJECTS='1', GIT_NO_LAZY_FETCH='1')
|
|
15
15
|
return env
|
|
16
16
|
|
|
17
17
|
|
|
@@ -41,7 +41,6 @@ class History:
|
|
|
41
41
|
self.revisions = [self.commit, *public_revisions] if public_base else ['--all', self.commit]
|
|
42
42
|
from history_hydration import prepare
|
|
43
43
|
self.hydration = prepare(self)
|
|
44
|
-
self.env['GIT_NO_LAZY_FETCH'] = '1'
|
|
45
44
|
self.commits = self.run('rev-list', *self.revisions).decode().splitlines()
|
|
46
45
|
self.public_commits = set(self.run('rev-list', *public_revisions).decode().splitlines()) if public_base else set()
|
|
47
46
|
self.trees = {}
|
|
@@ -49,10 +48,10 @@ class History:
|
|
|
49
48
|
self.reader = subprocess.Popen(['git', '-C', self.repo, 'cat-file', '--batch'],
|
|
50
49
|
env=self.env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
|
51
50
|
|
|
52
|
-
def run(self, *args):
|
|
51
|
+
def run(self, *args, input=None):
|
|
53
52
|
try:
|
|
54
53
|
result = subprocess.run(['git', '-C', self.repo, '-c', 'core.quotePath=false', *args],
|
|
55
|
-
env=self.env, capture_output=True, timeout=120)
|
|
54
|
+
env=self.env, input=input, capture_output=True, timeout=120)
|
|
56
55
|
if result.returncode:
|
|
57
56
|
raise InspectionError('history.git-failed')
|
|
58
57
|
return result.stdout
|
|
@@ -89,11 +88,13 @@ class History:
|
|
|
89
88
|
self.reader.stdin.flush()
|
|
90
89
|
header = self.reader.stdout.readline().split()
|
|
91
90
|
if len(header) != 3 or header[1] != b'blob':
|
|
92
|
-
|
|
91
|
+
from history_hydration import missing_blob
|
|
92
|
+
raise missing_blob(self, oid)
|
|
93
93
|
size = int(header[2])
|
|
94
94
|
data = self.reader.stdout.read(size)
|
|
95
95
|
if len(data) != size or self.reader.stdout.read(1) != b'\n':
|
|
96
|
-
|
|
96
|
+
from history_hydration import missing_blob
|
|
97
|
+
raise missing_blob(self, oid)
|
|
97
98
|
return data
|
|
98
99
|
|
|
99
100
|
def selected(self, path):
|
|
@@ -3,26 +3,49 @@ from time import monotonic
|
|
|
3
3
|
from git_history import InspectionError
|
|
4
4
|
|
|
5
5
|
|
|
6
|
+
def missing_blob(history, oid):
|
|
7
|
+
# Sizes need the blob itself. Locate a promised object using tree entries
|
|
8
|
+
# without -l, so a failed fetch cannot trigger another lazy fetch.
|
|
9
|
+
trees = set(history.run('log', '--format=%T', *history.revisions).splitlines())
|
|
10
|
+
for tree in sorted(trees):
|
|
11
|
+
for record in history.run('ls-tree', '-r', '-z', tree.decode()).split(b'\0'):
|
|
12
|
+
metadata, separator, path = record.partition(b'\t')
|
|
13
|
+
if separator and metadata.split()[2].decode() == oid:
|
|
14
|
+
return InspectionError(f'missing blob {oid} at {path.decode(errors="replace")}: fetch the full history')
|
|
15
|
+
return InspectionError(f'missing blob {oid} at <unknown path>: fetch the full history')
|
|
16
|
+
|
|
17
|
+
|
|
6
18
|
def prepare(history):
|
|
7
19
|
started = monotonic()
|
|
8
20
|
def missing():
|
|
9
|
-
return
|
|
10
|
-
|
|
21
|
+
return sorted(line[1:] for line in
|
|
22
|
+
history.run('rev-list', '--objects', '--missing=print', *history.revisions).splitlines()
|
|
23
|
+
if line.startswith(b'?'))
|
|
11
24
|
before = missing()
|
|
12
25
|
fetches = 0
|
|
13
26
|
if before:
|
|
14
27
|
# A blob:none clone already owns the commits and trees. ls-tree -l and
|
|
15
28
|
# cat-file would otherwise ask its promisor for one blob at a time.
|
|
16
|
-
|
|
29
|
+
try:
|
|
30
|
+
remotes = history.run('config', '--get-regexp', r'^remote\..*\.promisor$').decode().splitlines()
|
|
31
|
+
except InspectionError:
|
|
32
|
+
remotes = []
|
|
17
33
|
names = [line.rsplit(None, 1)[0][len('remote.'):-len('.promisor')]
|
|
18
34
|
for line in remotes if line.rsplit(None, 1)[-1].lower() == 'true']
|
|
19
35
|
if len(names) != 1:
|
|
20
|
-
raise
|
|
21
|
-
|
|
22
|
-
|
|
36
|
+
raise missing_blob(history, before[0].decode())
|
|
37
|
+
try:
|
|
38
|
+
# Explicit object wants cover selected commits fetched outside the
|
|
39
|
+
# remote's configured branch refspecs too. stdin keeps one batch
|
|
40
|
+
# even when a full history exceeds the command argument limit.
|
|
41
|
+
history.run('fetch', '--quiet', '--refetch', '--no-filter', '--no-tags',
|
|
42
|
+
'--no-recurse-submodules', '--no-write-fetch-head', '--stdin', '--', names[0],
|
|
43
|
+
input=b'\n'.join(before) + b'\n')
|
|
44
|
+
except InspectionError:
|
|
45
|
+
raise missing_blob(history, before[0].decode()) from None
|
|
23
46
|
fetches = 1
|
|
24
|
-
after = missing() if before else
|
|
47
|
+
after = missing() if before else []
|
|
25
48
|
if after:
|
|
26
|
-
raise
|
|
27
|
-
return dict(missingBefore=before, missingAfter=after, fetches=fetches,
|
|
49
|
+
raise missing_blob(history, after[0].decode())
|
|
50
|
+
return dict(missingBefore=len(before), missingAfter=len(after), fetches=fetches,
|
|
28
51
|
seconds=round(monotonic() - started, 6))
|