@runeditrun/rer 0.2.66 → 0.2.68

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/cli/business/vendors/README.md +38 -2
  2. package/cli/business/vendors/calendly.mjs +2 -2
  3. package/cli/business/vendors/common.mjs +94 -0
  4. package/cli/business/vendors/credential-names.mjs +124 -0
  5. package/cli/business/vendors/pagerduty.mjs +6 -15
  6. package/cli/business/vendors/sentry.mjs +4 -3
  7. package/cli/business/vendors/uptimerobot.mjs +7 -6
  8. package/cli/import-summary.mjs +7 -0
  9. package/cli/output.mjs +2 -0
  10. package/cli/portability-worker.mjs +10 -4
  11. package/cli/portability.mjs +1 -0
  12. package/cli/production-mail.mjs +4 -1
  13. package/global/CHANGELOG.md +14 -0
  14. package/modules/deploy-kit/d1-import.mjs +3 -1
  15. package/modules/deploy-kit/migrations.mjs +1 -0
  16. package/modules/deploy-kit/owner-cli.mjs +1 -1
  17. package/modules/deploy-kit/provider.mjs +16 -4
  18. package/modules/portability/README.md +73 -8
  19. package/modules/portability/bridge.mjs +44 -29
  20. package/modules/portability/catalogue.mjs +33 -3
  21. package/modules/portability/files.mjs +9 -3
  22. package/modules/portability/import-safety.mjs +80 -0
  23. package/modules/portability/import.mjs +13 -7
  24. package/modules/portability/index.d.mts +9 -3
  25. package/modules/portability/index.mjs +1 -1
  26. package/modules/portability/ledgers.mjs +83 -0
  27. package/modules/portability/legacy-files.mjs +7 -4
  28. package/modules/portability/migration-history.mjs +3 -2
  29. package/modules/portability/migration-statements.mjs +83 -9
  30. package/modules/release/README.md +25 -7
  31. package/modules/release/cleanup.mjs +16 -0
  32. package/modules/release/produce.mjs +11 -4
  33. package/modules/release/publication-scan.mjs +18 -8
  34. package/package.json +1 -1
  35. package/scripts/publish-check/README.md +5 -2
  36. package/scripts/publish-check/destination.py +8 -1
  37. package/scripts/publish-check/git_history.py +7 -6
  38. package/scripts/publish-check/history_hydration.py +32 -9
@@ -0,0 +1,83 @@
1
+ import { all, insist } from './storage.mjs';
2
+ import { identifier } from './bookkeeping.mjs';
3
+ import { canonicalJson } from './hash.mjs';
4
+ import { validateRow } from './catalogue.mjs';
5
+
6
+ // Migration ledgers, by the runner that writes them: Base deploy and container
7
+ // runners (_rer_deploy_migrations), Wrangler and the Base deploy kit
8
+ // (d1_migrations, tracked or plain) and RER Platform (rer_migrations).
9
+ export const ledgerNames=['_rer_deploy_migrations','d1_migrations','rer_migrations'];
10
+ // Platform's ledger exactly as its runner creates it
11
+ // (platform src/integrations/cloudflare-migrations.ts:33).
12
+ const platformShape=canonicalJson([{name:'name',type:'TEXT',notnull:0,pk:1},{name:'sha256',type:'TEXT',notnull:1,pk:0},{name:'applied_at',type:'TEXT',notnull:1,pk:0}]);
13
+ const shape=columns=>canonicalJson(columns.map(({name,type,notnull,pk})=>({name,type,notnull,pk})));
14
+ // Platform records a file applied in part as <name>#<start>-<end> until the
15
+ // file completes (platform src/integrations/migration-batch.ts:31): its effects
16
+ // are uncertain, like a tracked ledger's 'applying' row.
17
+ const partial=name=>name.includes('#');
18
+
19
+ /** The destination's applied product migrations, [{name,sha256?}] in the ledger's own order, or undefined without a ledger. */
20
+ export async function destinationLedger(db) {
21
+ const found=(await all(db,`SELECT name FROM sqlite_master WHERE type='table' AND name IN (${ledgerNames.map(n=>`'${n}'`).join(',')})`)).map(x=>x.name);
22
+ if(found.includes('_rer_deploy_migrations'))return (await all(db,"SELECT * FROM _rer_deploy_migrations WHERE state='applied' ORDER BY name")).map(entry);
23
+ if(found.includes('d1_migrations')) {
24
+ const columns=(await all(db,'PRAGMA table_info(d1_migrations)')).map(x=>x.name);
25
+ const tracked=columns.includes('state') || columns.includes('sha256');
26
+ if(tracked)insist(columns.includes('state') && columns.includes('sha256'),'schema_mismatch:migration_history',409);
27
+ const rows=await all(db,`SELECT name${tracked?',state,sha256':''} FROM d1_migrations ORDER BY id`);
28
+ if(tracked)insist(rows.every(row=>row.state==='applied'),'schema_mismatch:uncertain_migrations',409);
29
+ return rows.map(entry);
30
+ }
31
+ if(!found.includes('rer_migrations'))return;
32
+ // Platform applies pending files in its plan's order and inserts each ledger
33
+ // row in the same request as the file's SQL (migration-batch.ts:35-38,43-64;
34
+ // cloudflare-migrations.ts:34-44), so rowid order is application order.
35
+ insist(shape(await all(db,'PRAGMA table_info(rer_migrations)'))===platformShape,'schema_mismatch:migration_history',409);
36
+ const rows=await all(db,'SELECT name,sha256 FROM rer_migrations ORDER BY rowid');
37
+ insist(!rows.some(row=>partial(row.name)),'schema_mismatch:uncertain_migrations',409);
38
+ return rows.map(entry);
39
+ }
40
+ const entry=row=>({name:row.name,...(typeof row.sha256==='string'?{sha256:row.sha256}:{})});
41
+
42
+ /** The ledger tables an archive carries (old archives carried rer_migrations before it became bookkeeping). */
43
+ export const archivedLedgerTables=manifest=>manifest.tables.filter(t=>ledgerNames.includes(identifier(t.name)));
44
+
45
+ /** Collects the rows of an archive's ledger tables while its records are read: add(record) each, then ledgers() is {table:rows}, or undefined without one. */
46
+ export function ledgerReader() {
47
+ let ledgers;
48
+ return {
49
+ add(record) {
50
+ if(record.kind==='manifest') {const tables=archivedLedgerTables(record);if(tables.length)ledgers=Object.fromEntries(tables.map(t=>[t.name,[]]));}
51
+ if(record.kind==='part' && ledgers)for(const item of record.items)if(item.kind==='row' && ledgers[item.table])ledgers[item.table].push(item.row);
52
+ },
53
+ ledgers:()=>ledgers,
54
+ };
55
+ }
56
+ /** Checks rows a caller read from the archive for its ledger tables ({table:rows}), so uploadPart can bind each to the authenticated parts. */
57
+ export function checkArchivedLedgers(manifest,ledgers) {
58
+ if(ledgers===undefined)return;
59
+ const tables=archivedLedgerTables(manifest);
60
+ insist(ledgers && typeof ledgers==='object' && Object.keys(ledgers).every(name=>tables.some(t=>t.name===name)),'ledger_inventory');
61
+ for(const [name,rows] of Object.entries(ledgers)) {
62
+ const table=tables.find(t=>t.name===name);
63
+ insist(Array.isArray(rows) && rows.length===table.count,'ledger_inventory');
64
+ for(const row of rows)validateRow(table,row,manifest.excludedColumns[name]??[]);
65
+ }
66
+ }
67
+
68
+ /** The applied migrations an archive's own ledger records, [{name,sha256?}]; undefined when it carries no ledger. */
69
+ export function archivedHistory(manifest,ledgers) {
70
+ const tables=archivedLedgerTables(manifest);
71
+ if(!tables.length)return;
72
+ insist(tables.length===1 && ledgers?.[tables[0].name],'schema_mismatch:migration_history');
73
+ const [table]=tables,rows=ledgers[table.name],columns=table.columns.map(c=>c.name);
74
+ if(identifier(table.name)==='rer_migrations') {
75
+ insist(shape(table.columns)===platformShape,'schema_mismatch:migration_history');
76
+ insist(!rows.some(row=>partial(row.name)),'schema_mismatch:uncertain_migrations');
77
+ }else{
78
+ insist(columns.includes('name') && columns.includes('state')===columns.includes('sha256'),'schema_mismatch:migration_history');
79
+ insist(rows.every(row=>row.state===undefined || row.state==='applied'),'schema_mismatch:uncertain_migrations');
80
+ }
81
+ insist(rows.every(row=>typeof row.name==='string') && new Set(rows.map(row=>row.name)).size===rows.length,'schema_mismatch:migration_history');
82
+ return rows.map(entry);
83
+ }
@@ -6,13 +6,16 @@ import { readLegacyArchive, ArchiveVerifier } from './format.mjs';
6
6
  import { fileRecords, exportArchive } from './files.mjs';
7
7
  import { ensureLegacyAudit, legacyTargetState, requireLegacyEmpty } from './legacy-safety.mjs';
8
8
  import { insist, PortabilityError } from './storage.mjs';
9
+ import { ledgerReader } from './ledgers.mjs';
9
10
 
10
11
  export async function recoverLegacyArchive({db,buckets,policy:input,migrations},path,output,transient,acknowledged) {
11
12
  insist(acknowledged===true,'legacy_acknowledgement_required',409);
12
13
  insist(typeof output==='string' && output!==path,'legacy_output_required',409);
13
14
  insist(!(await lstat(output).catch(error=>{if(error.code==='ENOENT')return null;throw error;})),'legacy_output_exists',409);
14
- // Whole-file verification precedes even staging/audit schema writes.
15
- for await(const _record of fileRecords(path,readLegacyArchive)){}
15
+ // Whole-file verification precedes even staging/audit schema writes. It also
16
+ // reads the archive's own migration ledger, if any, for placement (bridge.mjs).
17
+ const ledger=ledgerReader();
18
+ for await(const record of fileRecords(path,readLegacyArchive))ledger.add(record);
16
19
  const context={db,buckets,config:policy(input),migrations},target=await legacyTargetState(db,migrations);
17
20
  await requireLegacyEmpty(context,target);
18
21
  let id,manifest,closing,integrity,status,complete=false;
@@ -26,7 +29,7 @@ export async function recoverLegacyArchive({db,buckets,policy:input,migrations},
26
29
  event:'legacy_v1_unverified_restore',acknowledgedUnverifiedMetadata:true,
27
30
  sourceIdentity:record.sourceIdentity,exportedAt:record.exportedAt,
28
31
  recoveryOutput:output,...target,
29
- });
32
+ },ledger.ledgers());
30
33
  // A refused schema writes nothing, not even the audit table.
31
34
  await ensureLegacyAudit(db);
32
35
  }else if(record.kind==='part') {integrity.add(record);await uploadPart(context,id,record.index,record);}
@@ -38,6 +41,6 @@ export async function recoverLegacyArchive({db,buckets,policy:input,migrations},
38
41
  // A failure here must be reported; the durable restore audit remains intact.
39
42
  try {
40
43
  const recovered=await exportArchive(await createPortability({db,buckets,policy:input}),output,transient);
41
- return {...recovered,legacyRecovery:true,unverifiedMetadata:true,restoredContentHash:closing.contentHash,...(status.bridged?{bridged:status.bridged}:{})};
44
+ return {...recovered,legacyRecovery:true,unverifiedMetadata:true,restoredContentHash:closing.contentHash,...(status.bridged?{bridged:status.bridged}:{}),...(status.safetyChanges?{safetyChanges:status.safetyChanges}:{})};
42
45
  }catch{throw new PortabilityError('legacy_restored_export_failed',500);}
43
46
  }
@@ -4,10 +4,11 @@ import { policy, schemaIdentity } from './catalogue.mjs';
4
4
  // Replays a chosen set of the destination's migrations, in its applied order,
5
5
  // into a scratch database and returns the resulting archive schema identity.
6
6
  // Node runners only: restore uses it to place an older archive within the
7
- // destination's history (see bridge.mjs). plan: [{name,statements}].
7
+ // destination's history (see bridge.mjs). plan: [{name,statements,sha256?}],
8
+ // sha256 being the migration file's hash where the runner read the file.
8
9
  export function migrationReplay(plan,input) {
9
10
  const config=policy(input);
10
- return {names:plan.map(migration=>migration.name),statements:name=>plan.find(m=>m.name===name).statements,async schemaAfter(names) {
11
+ return {names:plan.map(migration=>migration.name),statements:name=>plan.find(m=>m.name===name).statements,sha256:name=>plan.find(m=>m.name===name).sha256,async schemaAfter(names) {
11
12
  const scratch=new DatabaseSync(':memory:');
12
13
  const db={prepare:sql=>{const query=scratch.prepare(sql);let values=[];return {bind(...args){values=args;return this;},async all(){return {results:query.all(...values)};}};}};
13
14
  try {
@@ -3,18 +3,92 @@ import { tokens, identifier, folded } from './generated.mjs';
3
3
  // Whether one migration statement can change rows a restore carries. Only
4
4
  // CREATE TABLE (not AS SELECT), CREATE INDEX and ALTER TABLE ... ADD COLUMN
5
5
  // are additive: SQLite refuses ADD COLUMN with a non-constant default, so a
6
- // restored row takes the same value the migration gave existing rows. Any
7
- // other statement (INSERT, UPDATE, DELETE, triggers, table rebuilds) changes
8
- // data unless it names none of the restorable tables, as when it only touches
9
- // operator-auth or other excluded tables.
10
- export function changesData(sql,restorable) {
11
- const list=tokens(sql),words=list.map(t=>t.quoted?undefined:folded(t.text));
12
- const kind=words.slice(0,4).filter(w=>!['temp','temporary','unique'].includes(w)).slice(0,2).join(' ');
6
+ // restored row takes the same value the migration gave existing rows.
7
+ // CREATE TRIGGER and DROP TRIGGER are schema-only whatever the body does:
8
+ // neither runs the body, so no row changes when the migration runs, and a
9
+ // restore inserts rows with the destination's triggers dropped and recreates
10
+ // them afterwards (import.mjs finalizeImport), exactly as the live upgrade left
11
+ // the rows that predate the trigger. That holds only for a statement bounded
12
+ // as exactly one trigger statement; any other (INSERT, UPDATE, DELETE, table
13
+ // rebuilds, a trigger glued to a further statement) changes data unless it
14
+ // names none of `reached` (reachingTables), as when it only touches
15
+ // operator-auth or other excluded tables that no trigger links to restorable ones.
16
+ // `empty` lists the restorable tables the archive holds no rows of: restore
17
+ // leaves them empty, and an UPDATE or DELETE of an empty table changes nothing
18
+ // (no row fires a trigger either), so such a statement changes data only if it
19
+ // names another reached table (one it could write, or read into its writes).
20
+ // Any other statement naming an empty table still does: an INSERT would leave
21
+ // rows the restore lacks.
22
+ export function changesData(sql,reached,empty=[]) {
23
+ const list=tokens(sql),words=list.map(t=>t.quoted?undefined:folded(t.text)),kind=kindOf(words);
13
24
  if(kind==='create table')return topLevelWords(list,words).includes('as');
14
25
  if(kind==='create index')return false;
15
26
  if(kind==='alter table' && words.find((w,i)=>i>2 && ['add','rename','drop'].includes(w))==='add')return false;
16
- const names=new Set(restorable.map(folded));
17
- return list.some(t=>t.quoted ? !t.text.startsWith("'") && names.has(folded(identifier(t))) : t.text.split('.').some(part=>names.has(folded(part))));
27
+ if(kind==='create trigger' && createdTrigger(list))return false;
28
+ if(kind==='drop trigger' && droppedTrigger(list))return false;
29
+ const skipped=new Set(['update','delete'].includes(words[0])?empty.map(folded):[]);
30
+ return names(list,new Set(reached.map(folded).filter(name=>!skipped.has(name))));
31
+ }
32
+ // The restorable tables and every table whose writes reach one through a
33
+ // trigger any of `statements` creates: a write to such a table during a later
34
+ // migration changes restorable rows the archive lacks. A trigger counts when
35
+ // its statement names a reached table anywhere; one this parser cannot bound
36
+ // adds every name it holds.
37
+ export function reachingTables(statements,restorable) {
38
+ const reached=new Set(restorable.map(folded));
39
+ const triggers=statements.map(sql=>tokens(sql)).filter(list=>kindOf(list.map(t=>t.quoted?undefined:folded(t.text)))==='create trigger')
40
+ .map(list=>({list,on:createdTrigger(list)?.on}));
41
+ for(let grew=true;grew;) {
42
+ grew=false;
43
+ for(const {list,on} of triggers)if(names(list,reached))for(const name of on===undefined?identifiers(list):[on])if(!reached.has(name)){reached.add(name);grew=true;}
44
+ }
45
+ return [...reached];
46
+ }
47
+ const kindOf=words=>words.slice(0,4).filter(w=>!['temp','temporary','unique'].includes(w)).slice(0,2).join(' ');
48
+ const identifiers=list=>list.flatMap(t=>t.quoted?(t.text.startsWith("'")?[]:[folded(identifier(t))]):folded(t.text).split(/[^a-z0-9_$\u0080-\uffff]+/).filter(Boolean));
49
+ const names=(list,reached)=>identifiers(list).some(name=>reached.has(name));
50
+ // Tokens with each unquoted ';' split out, and trailing ones dropped.
51
+ function atoms(list) {
52
+ const result=list.flatMap(t=>t.quoted?[t]:t.text.split(/(;)/).filter(Boolean).map(text=>({text})));
53
+ while(result.length && semicolon(result.at(-1)))result.pop();
54
+ return result;
55
+ }
56
+ const semicolon=a=>!a.quoted && a.text===';';
57
+ const word=(a,value)=>a && !a.quoted && folded(a.text)===value;
58
+ // {on} for CREATE [TEMP] TRIGGER ... ON table ... BEGIN statement; ... END and
59
+ // nothing after it. The body ends at the first END that starts a statement;
60
+ // CASE ... END inside a statement does not. Anything else is unbounded.
61
+ function createdTrigger(list) {
62
+ const parts=atoms(list);let depth=0,begin=-1,on;
63
+ for(let i=0;i<parts.length && begin<0;i++) {
64
+ const a=parts[i];
65
+ if(a.quoted)continue;
66
+ if(a.text===';')return;
67
+ if(a.text==='(')depth++;
68
+ else if(a.text===')'){if(--depth<0)return;}
69
+ else if(!depth && on===undefined && word(a,'on')) {
70
+ const name=[];
71
+ for(let j=i+1;j<parts.length && !['for','when','begin'].some(w=>word(parts[j],w));j++)name.push(parts[j]);
72
+ const last=name.at(-1);
73
+ if(!last || name.length>3)return;
74
+ on=folded(last.quoted?identifier(last):last.text.split('.').at(-1));
75
+ if(!on)return;
76
+ }else if(!depth && word(a,'begin'))begin=i;
77
+ }
78
+ if(begin<0 || on===undefined)return;
79
+ let start=true,statements=0;
80
+ for(let i=begin+1;i<parts.length;i++) {
81
+ const a=parts[i];
82
+ if(start && word(a,'end'))return i===parts.length-1 && statements && !depth?{on}:undefined;
83
+ if(semicolon(a)){if(start || depth)return;start=true;statements++;continue;}
84
+ if(!a.quoted){if(a.text==='(')depth++;else if(a.text===')' && --depth<0)return;}
85
+ start=false;
86
+ }
87
+ }
88
+ // DROP TRIGGER [IF EXISTS] [schema.]name, and nothing after it.
89
+ function droppedTrigger(list) {
90
+ const parts=atoms(list),from=word(parts[2],'if') && word(parts[3],'exists')?4:2,name=parts.slice(from);
91
+ return name.length>=1 && name.length<=3 && name.every(a=>a.quoted?!a.text.startsWith("'"):!/[;(),]/.test(a.text));
18
92
  }
19
93
  function topLevelWords(list,words) {
20
94
  let depth=0;const result=[];
@@ -15,7 +15,14 @@ pnpm rer release --repo /path/to/seed --ref main \
15
15
 
16
16
  The `rer` bin and `node bin/rer.mjs` expose the same command. Node 22+, Python 3,
17
17
  Git, pnpm and tar are required, along with authenticated `gh` for GitHub sources.
18
- Use a full, clean seed clone with its declared package-manager version. The producer reads
18
+ Use a full-history, clean seed clone with its declared package-manager version;
19
+ `--filter=blob:none` is supported, while shallow history is refused. Before
20
+ release work begins, tracked changes and untracked files in the source checkout
21
+ are refused; ignored local files do not become release inputs. Before
22
+ reading source files, destination validation uses the publication scanner's
23
+ single batched hydration fetch. An unavailable blob reports its object ID and
24
+ committed path with `fetch the full history` guidance, separately from identity
25
+ disagreements. The producer reads
19
26
  committed `seed.json`, `package.json`, `RELEASE.json`, `LICENSE`, `CONTRACT.md` and
20
27
  `AGENTS.md`. Version declarations must agree. `--project` declares the publishing
21
28
  identity; `--repository` is the credential-free public destination URL. Neither is inferred
@@ -33,6 +40,11 @@ URLs; identity comparison ignores case and a terminal `.git` in URLs. Missing,
33
40
  malformed, partial or inconsistent identities fail closed. The selected
34
41
  committed tree supplies the declarations.
35
42
 
43
+ Temporary release directories are removed recursively, including incidental
44
+ Finder metadata. A cleanup failure keeps a completed release successful and
45
+ records a `temporary-cleanup-failed` warning in stderr and the release receipt;
46
+ its directory can then be removed manually.
47
+
36
48
  The platform reads these same HTTPS GitHub URLs, plus bare an owner and repository name separated by a slash for releases that carry it; ordinary release editions still require the URL here.
37
49
 
38
50
  Every publication scan receives the explicit project and destination: source
@@ -87,9 +99,14 @@ path needs the seed's browser prerequisites and any resources its verification
87
99
  requires. Routine CI and default verification do not invoke the release-only
88
100
  script.
89
101
 
90
- Both paths then scan source history with publish-check. When the checkout's
91
- origin is a confirmed public repository, the scanner uses that source
92
- repository's advertised default branch and published tags as its public base.
102
+ Both paths then run publish-check in a fresh disposable clone checked out at
103
+ the selected source commit. Its working tree contains exactly that commit's
104
+ committed files, isolated from the caller's ignored or untracked files and from
105
+ installation, build and verification outputs. The clone retains all source refs
106
+ and full history: committed secrets and historical findings remain subject to
107
+ the publication checks. When the source checkout's origin is a confirmed public
108
+ repository, the scanner uses that source repository's advertised default branch
109
+ and published tags as its public base.
93
110
  Already-public findings are classified pre-existing; unpublished content still
94
111
  blocks. Other sources retain complete local-ref history inspection.
95
112
  A passing step's receipt retains exit status and numeric final summaries from
@@ -202,9 +219,10 @@ Public export commit and annotated-tag messages contain the version and optional
202
219
  `RELEASE.json.publicReleaseNotes` only. Private source commit identities stay in
203
220
  the private release receipt. The clean export's final annotated tag exists
204
221
  before publication inspection, and the checked repository is copied unchanged
205
- into `public-git/`. Ordinary releases inspect their source history and stage the
206
- exact final annotation in a disposable checkout for inspection before creating
207
- the source tag. Release notes pass the same content rules as the public tree.
222
+ into `public-git/`. Ordinary releases stage the exact final annotation in the
223
+ fresh selected-commit checkout and inspect it with the source history before
224
+ creating the source tag. Release notes pass the same content rules as the
225
+ public tree.
208
226
 
209
227
  ## Releasing a seed that is already public
210
228
 
@@ -0,0 +1,16 @@
1
+ import { rmSync } from 'node:fs';
2
+
3
+ // Finder and other filesystem observers can add metadata while removal walks
4
+ // the directory. Retry those races, but never turn finished artifacts into a
5
+ // failed release merely because temporary files could not be removed.
6
+ export function cleanupTemporary(directory) {
7
+ try {
8
+ rmSync(directory, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
9
+ return null;
10
+ } catch (error) {
11
+ const warning = { code: 'temporary-cleanup-failed', errorCode: error.code ?? 'UNKNOWN',
12
+ directory, message: 'Temporary release cleanup failed; remove the remaining directory manually.' };
13
+ console.warn(`Warning: ${warning.message} (${warning.errorCode}): ${directory}`);
14
+ return warning;
15
+ }
16
+ }
@@ -11,6 +11,7 @@ import { writePublishHandoff } from './publish-handoff.mjs';
11
11
  import { releasePublicBase } from './public-base.mjs';
12
12
  import { prepareTagAdoption, confirmTagAdoption, assertTagAdoption } from './tag-adoption.mjs';
13
13
  import { placeholderMarks, placeholderMessage } from '../../cli/placeholder-mark.mjs';
14
+ import { cleanupTemporary } from './cleanup.mjs';
14
15
 
15
16
  // Failure output belongs to the private receipt, not the site preview.
16
17
  const withoutOutput = gates => JSON.parse(JSON.stringify(gates, (key, value) => key === 'output' ? undefined : value));
@@ -21,13 +22,15 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
21
22
  if (onto !== undefined && clean) throw Error('--onto and --clean-history are mutually exclusive');
22
23
  if (publicBase !== undefined && (onto !== undefined || clean)) throw Error('--public-base selects exact-source history; cannot combine with export history modes');
23
24
  const preview = dryRun || !tag;
25
+ // The shared history reader hydrates promised blobs once, before any source
26
+ // metadata, placeholder inspection or release export reads their contents.
27
+ validatePublicationDestination({ repo, commit: ref }, project, repository);
24
28
  const source = inspectSource(repo, ref);
25
29
  // A release refuses a seed still wearing Base's placeholder mark unless the
26
30
  // caller allows it explicitly; the receipt records that allowance.
27
31
  const files = [...source.entries].filter(([, entry]) => entry.type === 'blob' && ['100644', '100755'].includes(entry.mode)).map(([path]) => path);
28
32
  const marks = await placeholderMarks(source.entries.has('OPERATIONS.json') ? JSON.parse(source.bytes('OPERATIONS.json')).project : project, files, source.bytes);
29
33
  if (marks.length && !allowPlaceholderMark) throw Object.assign(Error(`${placeholderMessage(marks)} Release refused; pass --allow-placeholder-mark to release with it.`), { code: 'placeholder_mark' });
30
- validatePublicationDestination(source, project, repository);
31
34
  releaseMessage(source);
32
35
  if (typeof tag === 'string' && tag !== source.tag) throw Error('Requested tag must match the committed release version');
33
36
  const existing = git(source.repo, 'tag', '--list', source.tag).toString().trim();
@@ -44,6 +47,7 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
44
47
  output = newOutput(source.repo, output);
45
48
  const temporary = mkdtempSync(resolve(tmpdir(), 'rer-release-'));
46
49
  let receipt;
50
+ let cleanupAttempted = false;
47
51
  try {
48
52
  const checkout = resolve(temporary, 'seed');
49
53
  git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, checkout);
@@ -117,17 +121,20 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
117
121
  });
118
122
  }
119
123
  }
120
- if (passed && tag && destination) writePublishHandoff(output, destination, receipt);
121
124
  if (passed && adopted) {
122
125
  assertTagAdoption(source, adopted, { imported: true });
123
126
  if (receipt.tag.object !== adopted.object || receipt.publicExport.tagObject !== adopted.object ||
124
127
  receipt.publicExport.commit !== adopted.commit || receipt.source.commit !== adopted.commit)
125
128
  throw Error('Adopted release tag identity does not match the release receipt');
126
129
  }
130
+ cleanupAttempted = true;
131
+ const warning = cleanupTemporary(temporary);
132
+ if (warning) receipt.warnings = [warning];
133
+ if (passed && tag && destination) writePublishHandoff(output, destination, receipt);
127
134
  writeFileSync(resolve(output, 'release-receipt.json'), json(receipt), { flag: 'wx' });
128
135
  if (passed && destination) {
129
136
  requirePublicDestination(destination);
130
- if (remotePublicHead(exported.repo, destination, source.tag) !== publicParent) throw Error('Public destination changed before handoff');
137
+ if (remotePublicHead(tag ? resolve(output, 'public-git') : source.repo, destination, source.tag) !== publicParent) throw Error('Public destination changed before handoff');
131
138
  if (tag && !dryRun) annotate(source.repo, source, source.commit);
132
139
  }
133
140
  return receipt;
@@ -135,5 +142,5 @@ export async function release({ repo, ref = 'HEAD', output, project, repository,
135
142
  // No incomplete directory should be mistaken for a completed export.
136
143
  rmSync(output, { recursive: true, force: true });
137
144
  throw error;
138
- } finally { rmSync(temporary, { recursive: true, force: true }); }
145
+ } finally { if (!cleanupAttempted) cleanupTemporary(temporary); }
139
146
  }
@@ -2,7 +2,7 @@ import { execFileSync } from 'node:child_process';
2
2
  import { resolve } from 'node:path';
3
3
  import { writeFileSync } from 'node:fs';
4
4
  import { git } from './source.mjs';
5
- import { annotate } from './export.mjs';
5
+ import { annotate, sourceTree } from './export.mjs';
6
6
  import { runningVersion } from '../../cli/version.mjs';
7
7
 
8
8
  export function validatePublicationDestination(source, project, repository) {
@@ -10,7 +10,11 @@ export function validatePublicationDestination(source, project, repository) {
10
10
  execFileSync('python3', ['-B', resolve(import.meta.dirname, '../../scripts/publish-check/destination.py'),
11
11
  '--repo', source.repo, '--ref', source.commit, '--project', project, '--repository', repository],
12
12
  { stdio: ['ignore', 'pipe', 'pipe'] });
13
- } catch { throw Error('Publication destination must agree with the committed edition and project identities'); }
13
+ } catch (error) {
14
+ if (error.status === 2) throw Error(error.stderr.toString().trim());
15
+ if (error.status === 1) throw Error('Publication destination must agree with the committed edition and project identities');
16
+ throw Error('Publication destination inspection could not run');
17
+ }
14
18
  }
15
19
 
16
20
  export function publicationScan(repo, ref, envFile, project, repository, publicBase, annotationFile) {
@@ -32,26 +36,32 @@ export function publicationScan(repo, ref, envFile, project, repository, publicB
32
36
  }
33
37
 
34
38
  // This is the ordinary release gate, also usable without installing/building a
35
- // seed. The scanner sees the original checkout and every source ref; only the
36
- // new annotation is read from the clean candidate, never verification output.
39
+ // seed. Materialize raw committed bytes, without checkout filters or export
40
+ // attributes, and mirror every source ref for the existing history policy.
41
+ // Neither the caller's local files nor verification output enters this tree.
37
42
  export function sourcePublicationScan(source, temporary, envFile, project, repository, publicBase, tagged, adopted) {
38
43
  const started = performance.now();
44
+ const candidate = sourceTree(source, resolve(temporary, 'publication'));
45
+ git(source.repo, 'clone', '--mirror', '--no-hardlinks', '--', source.repo, resolve(candidate, '.git'));
46
+ git(candidate, 'config', 'core.bare', 'false');
47
+ git(candidate, 'update-ref', '--no-deref', 'HEAD', source.commit);
48
+ git(candidate, 'read-tree', source.commit);
39
49
  let annotationFile;
40
50
  if (adopted) {
41
51
  annotationFile = resolve(temporary, 'annotation.tag');
42
52
  writeFileSync(annotationFile, git(adopted.repo, 'cat-file', 'tag', adopted.object), { mode: 0o600 });
43
53
  } else if (tagged) {
44
- const candidate = resolve(temporary, 'annotation');
45
- git(source.repo, 'clone', '--no-hardlinks', '--no-checkout', '--', source.repo, candidate);
46
- git(candidate, 'checkout', '--detach', source.commit);
47
54
  const staged = git(candidate, 'tag', '--list', source.tag).toString().trim();
48
55
  const name = staged ? `${source.tag}-publication-candidate` : source.tag;
49
56
  annotate(candidate, source, source.commit, name);
50
57
  annotationFile = resolve(temporary, 'annotation.tag');
51
58
  writeFileSync(annotationFile, git(candidate, 'cat-file', 'tag', `refs/tags/${name}`), { mode: 0o600 });
59
+ // Inspect the new tag once via annotationFile, leaving the mirrored source
60
+ // refs unchanged for the full-history scan.
61
+ git(candidate, 'tag', '-d', name);
52
62
  }
53
63
  const stagingSeconds = (performance.now() - started) / 1000;
54
- const report = publicationScan(source.repo, source.commit, envFile, project, repository, publicBase, annotationFile);
64
+ const report = publicationScan(candidate, source.commit, envFile, project, repository, publicBase, annotationFile);
55
65
  Object.assign(report.invocation, { stagingSeconds, totalSeconds: (performance.now() - started) / 1000 });
56
66
  return report;
57
67
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runeditrun/rer",
3
- "version": "0.2.66",
3
+ "version": "0.2.68",
4
4
  "description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -6,8 +6,11 @@ See [setup, command and scope](../publish-check-README.md).
6
6
  blob reads to system Git (`rev-parse`, `rev-list`, `ls-tree`, `cat-file --batch`).
7
7
  Before any tree inventory asks for blob sizes, `history_hydration.py` counts
8
8
  missing objects and hydrates a partial clone with one `fetch --refetch --no-filter`
9
- from its promisor remote. A failed fetch or remaining missing object makes the
10
- inspection incomplete. `history.hydration` reports the missing counts, fetch
9
+ from its promisor remote. The batch explicitly requests every missing object,
10
+ including selected history outside that remote's branch refspecs. A failed fetch
11
+ or remaining missing object makes the inspection incomplete and names a missing
12
+ blob and its committed path, with `fetch the full history` guidance.
13
+ `history.hydration` reports the missing counts, fetch
11
14
  count and elapsed time; subsequent history reads disable lazy fetching.
12
15
 
13
16
  Reports include implementation version, Python version and a digest of the
@@ -2,6 +2,7 @@
2
2
  import argparse
3
3
  import json
4
4
  import re
5
+ import sys
5
6
  from github_refs import FULL
6
7
  from project_identity import FIELDS, ordinary_document, tree_identity
7
8
 
@@ -115,7 +116,13 @@ if __name__ == '__main__':
115
116
  try:
116
117
  history = History(args.repo, args.ref)
117
118
  destination(history, args.project, args.repository)
118
- except (ValueError, TypeError, UnicodeError, OSError, InspectionError):
119
+ except InspectionError as error:
120
+ print(str(error), file=sys.stderr)
121
+ raise SystemExit(2) from None
122
+ except OSError:
123
+ print('Publication destination inspection could not run', file=sys.stderr)
124
+ raise SystemExit(2) from None
125
+ except (ValueError, TypeError, UnicodeError):
119
126
  raise SystemExit(1) from None
120
127
  finally:
121
128
  if history:
@@ -11,7 +11,7 @@ class InspectionError(Exception):
11
11
 
12
12
  def git_environment():
13
13
  env = {k: v for k, v in os.environ.items() if not k.startswith('GIT_')}
14
- env.update(GIT_TERMINAL_PROMPT='0', GIT_NO_REPLACE_OBJECTS='1')
14
+ env.update(GIT_TERMINAL_PROMPT='0', GIT_NO_REPLACE_OBJECTS='1', GIT_NO_LAZY_FETCH='1')
15
15
  return env
16
16
 
17
17
 
@@ -41,7 +41,6 @@ class History:
41
41
  self.revisions = [self.commit, *public_revisions] if public_base else ['--all', self.commit]
42
42
  from history_hydration import prepare
43
43
  self.hydration = prepare(self)
44
- self.env['GIT_NO_LAZY_FETCH'] = '1'
45
44
  self.commits = self.run('rev-list', *self.revisions).decode().splitlines()
46
45
  self.public_commits = set(self.run('rev-list', *public_revisions).decode().splitlines()) if public_base else set()
47
46
  self.trees = {}
@@ -49,10 +48,10 @@ class History:
49
48
  self.reader = subprocess.Popen(['git', '-C', self.repo, 'cat-file', '--batch'],
50
49
  env=self.env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
51
50
 
52
- def run(self, *args):
51
+ def run(self, *args, input=None):
53
52
  try:
54
53
  result = subprocess.run(['git', '-C', self.repo, '-c', 'core.quotePath=false', *args],
55
- env=self.env, capture_output=True, timeout=120)
54
+ env=self.env, input=input, capture_output=True, timeout=120)
56
55
  if result.returncode:
57
56
  raise InspectionError('history.git-failed')
58
57
  return result.stdout
@@ -89,11 +88,13 @@ class History:
89
88
  self.reader.stdin.flush()
90
89
  header = self.reader.stdout.readline().split()
91
90
  if len(header) != 3 or header[1] != b'blob':
92
- raise InspectionError('history.missing-blob')
91
+ from history_hydration import missing_blob
92
+ raise missing_blob(self, oid)
93
93
  size = int(header[2])
94
94
  data = self.reader.stdout.read(size)
95
95
  if len(data) != size or self.reader.stdout.read(1) != b'\n':
96
- raise InspectionError('history.missing-blob')
96
+ from history_hydration import missing_blob
97
+ raise missing_blob(self, oid)
97
98
  return data
98
99
 
99
100
  def selected(self, path):
@@ -3,26 +3,49 @@ from time import monotonic
3
3
  from git_history import InspectionError
4
4
 
5
5
 
6
+ def missing_blob(history, oid):
7
+ # Sizes need the blob itself. Locate a promised object using tree entries
8
+ # without -l, so a failed fetch cannot trigger another lazy fetch.
9
+ trees = set(history.run('log', '--format=%T', *history.revisions).splitlines())
10
+ for tree in sorted(trees):
11
+ for record in history.run('ls-tree', '-r', '-z', tree.decode()).split(b'\0'):
12
+ metadata, separator, path = record.partition(b'\t')
13
+ if separator and metadata.split()[2].decode() == oid:
14
+ return InspectionError(f'missing blob {oid} at {path.decode(errors="replace")}: fetch the full history')
15
+ return InspectionError(f'missing blob {oid} at <unknown path>: fetch the full history')
16
+
17
+
6
18
  def prepare(history):
7
19
  started = monotonic()
8
20
  def missing():
9
- return sum(line.startswith(b'?') for line in
10
- history.run('rev-list', '--objects', '--missing=print', *history.revisions).splitlines())
21
+ return sorted(line[1:] for line in
22
+ history.run('rev-list', '--objects', '--missing=print', *history.revisions).splitlines()
23
+ if line.startswith(b'?'))
11
24
  before = missing()
12
25
  fetches = 0
13
26
  if before:
14
27
  # A blob:none clone already owns the commits and trees. ls-tree -l and
15
28
  # cat-file would otherwise ask its promisor for one blob at a time.
16
- remotes = history.run('config', '--get-regexp', r'^remote\..*\.promisor$').decode().splitlines()
29
+ try:
30
+ remotes = history.run('config', '--get-regexp', r'^remote\..*\.promisor$').decode().splitlines()
31
+ except InspectionError:
32
+ remotes = []
17
33
  names = [line.rsplit(None, 1)[0][len('remote.'):-len('.promisor')]
18
34
  for line in remotes if line.rsplit(None, 1)[-1].lower() == 'true']
19
35
  if len(names) != 1:
20
- raise InspectionError('history.promisor-ambiguous')
21
- history.run('fetch', '--quiet', '--refetch', '--no-filter', '--no-tags',
22
- '--no-recurse-submodules', '--no-write-fetch-head', '--', names[0])
36
+ raise missing_blob(history, before[0].decode())
37
+ try:
38
+ # Explicit object wants cover selected commits fetched outside the
39
+ # remote's configured branch refspecs too. stdin keeps one batch
40
+ # even when a full history exceeds the command argument limit.
41
+ history.run('fetch', '--quiet', '--refetch', '--no-filter', '--no-tags',
42
+ '--no-recurse-submodules', '--no-write-fetch-head', '--stdin', '--', names[0],
43
+ input=b'\n'.join(before) + b'\n')
44
+ except InspectionError:
45
+ raise missing_blob(history, before[0].decode()) from None
23
46
  fetches = 1
24
- after = missing() if before else 0
47
+ after = missing() if before else []
25
48
  if after:
26
- raise InspectionError('history.missing-blob')
27
- return dict(missingBefore=before, missingAfter=after, fetches=fetches,
49
+ raise missing_blob(history, after[0].decode())
50
+ return dict(missingBefore=len(before), missingAfter=len(after), fetches=fetches,
28
51
  seconds=round(monotonic() - started, 6))