@runeditrun/rer 0.2.118 → 0.2.119

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -26,6 +26,10 @@ import { keepHookSecrets } from './deploy-secrets.mjs';
26
26
  import { withoutLocalSettings, refuseLocalEnvOutput } from './local-env.mjs';
27
27
  import { deployProductDirectory, deployProductArguments, DeployProductError } from './deploy-product.mjs';
28
28
 
29
+ // A hook may wrap this controller. Its needs return to the caller, which renders
30
+ // the combined dry-run summary once after every local plan step has succeeded.
31
+ const deferDryRunSummary = Symbol('defer dry-run summary');
32
+
29
33
  export function validateDeployOptions(options) {
30
34
  const changes = [options.remove && '--remove', options.setSecret && '--set-secret', options.unsetSecret && '--unset-secret', options.rotateSecret && '--rotate-secret', options.reopenInstallationKey && '--reopen-installation-key'].filter(Boolean);
31
35
  if (changes.length > 1 || changes.length && options.dryRun) throw invalid(`Use ${changes.join(', ')}${options.dryRun ? ' and --dry-run' : ''} on separate runs.`);
@@ -88,16 +92,18 @@ export async function deployCommand(context) {
88
92
  const plan = result?.plan ?? (!planned ? undefined : (await owner(['local-plan', '--declaration', path, '--environment', options.environment ?? 'production', ...await deployProductArguments(root, deployProductDirectory(declaration))], production.env, { cwd: root, localOnlyKeys })).plan);
89
93
  result = { ...result, dryRun: true, deployed: false, providerChecked: false, plan };
90
94
  const { signIn, mailOff } = await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed: context.setupSeed, adapter, declarationPath: path, options }) ?? {};
91
- if (mailOff) store.report(mailOffNotice(mailOff, options.environment ?? 'production'), { names: mailOff.credentialNames });
92
- // The one completion line: rer's own plan step (or its deliberate skip) succeeded. A hook's own line says only that the hook finished.
93
- store.report('deploy: local dry run complete; no credentials were needed, no Cloudflare requests were made and nothing was changed remotely. Target secrets and remote readiness were not checked.');
94
95
  const needed = [...new Set([...neededForRealDeploy({ declaration, env: production.env, signIn, secretsChecked: false }), ...(result.neededForRealDeploy ?? []), ...(extraNeeded ?? [])])];
95
- reportNeeded(store, needed);
96
- if (declaration.secrets?.some(item => item.binding === 'INSTALLATION_KEY')) store.report('deploy: INSTALLATION_KEY is a private generated binding; INSTALLATION_KEY_PREVIOUS is optional and managed only during rotation or an explicit retired-key reopen. Values are withheld.');
96
+ if (!context[deferDryRunSummary]) {
97
+ if (mailOff) store.report(mailOffNotice(mailOff, options.environment ?? 'production'), { names: mailOff.credentialNames });
98
+ // The one completion line: rer's own plan step (or its deliberate skip) succeeded. A hook's own line says only that the hook finished.
99
+ store.report('deploy: local dry run complete; no credentials were needed, no Cloudflare requests were made and nothing was changed remotely. Target secrets and remote readiness were not checked.');
100
+ reportNeeded(store, needed);
101
+ if (declaration.secrets?.some(item => item.binding === 'INSTALLATION_KEY')) store.report('deploy: INSTALLATION_KEY is a private generated binding; INSTALLATION_KEY_PREVIOUS is optional and managed only during rotation or an explicit retired-key reopen. Values are withheld.');
102
+ }
97
103
  return needed.length && result && typeof result === 'object' ? { ...result, neededForRealDeploy: needed } : result;
98
104
  };
99
105
  const runHook = hookContext => options.dryRun ? withLocalDeployPlan(async () => {
100
- try { return await adapter.deploy(hookContext); }
106
+ try { return await adapter.deploy({ ...hookContext, [deferDryRunSummary]: true }); }
101
107
  catch (error) {
102
108
  // Production-only owner/input preflights may refuse before the hook builds.
103
109
  // An offline plan defers those inputs, retaining the refusal as guidance.
@@ -123,7 +129,7 @@ export async function deployCommand(context) {
123
129
  const redaction = createRedaction();
124
130
  const kinds = await settingKinds({ declaration, setupManifest });
125
131
  const addSecret = value => { redaction.addSecrets([value]); store.addSecrets?.([value]); };
126
- for (const value of [...kinds.privateValues(env), env.CLOUDFLARE_ACCOUNT_ID, ...(declaration.secrets ?? []).map(item => env[item.environment])].filter(Boolean)) addSecret(value);
132
+ for (const value of kinds.privateValues(env)) addSecret(value);
127
133
  const environment = options.environment ?? 'production';
128
134
  // The hook uploads every value it is given: a generated secret it gets is the one the Worker keeps.
129
135
  if (!options.dryRun) await keepHookSecrets({ root, environment, declaration, setupManifest, env, generated, reused, rotated, again: deployAgain(options), report: (line, more) => store.report(line, more) });
@@ -77,11 +77,8 @@ export async function withDeployEnvironment(project, options, out, run) {
77
77
  const { declaredSettings, settingKinds, undeclaredNotice } = await import('./deploy-settings.mjs');
78
78
  const declared = await declaredSettings(project), kinds = await settingKinds(declared);
79
79
  const redaction = createRedaction();
80
- // Provider names are public enum values. Masking "resend" would also hide
81
- // RESEND_API_KEY in the preflight's instruction for the missing credential.
82
- const publicProvider = (key, value) => key === 'MAIL_PROVIDER' && /^(resend|sendgrid|cloudflare|rer|later|none)$/.test(value);
83
80
  const supplied = Object.entries(env).filter(([key, value]) => saved[key] !== value);
84
- redaction.addSecrets([...kinds.privateValues(env), ...supplied.filter(([key, value]) => !kinds.shown(key, value) && !publicProvider(key, value)).map(([, value]) => value)]);
81
+ redaction.addSecrets([...kinds.privateValues(env), ...supplied.filter(([key, value]) => !kinds.shown(key, value)).map(([, value]) => value)]);
85
82
  const undeclared = declared.declaration ? kinds.undeclared(supplied.map(([key]) => key)) : [];
86
83
  if (undeclared.length) {
87
84
  const message = undeclaredNotice(undeclared, declared.path);
@@ -3,24 +3,30 @@ import { shared } from './shared.mjs';
3
3
  // What a deployment's declaration says each value is. A value prints in full only
4
4
  // when the declaration names it a non-secret setting: a declared var, the origin
5
5
  // variable and its setting, a setup field that is not a secret, or a name in
6
- // "publicSettings". Anything the declaration calls a secret, any credential-named
7
- // key and any other --env-file value stays redacted.
6
+ // "publicSettings". Secret transport alone does not make an explicitly public
7
+ // setting private. Credential bindings, private setup fields and undeclared
8
+ // --env-file values stay redacted.
8
9
  const PRIVATE_KEYS = ['CLOUDFLARE_ACCOUNT_ID'];
9
10
 
10
11
  export async function settingKinds({ declaration = {}, setupManifest } = {}) {
11
- const { credentialName, credentialUrl, credentialValues } = await shared('redaction.mjs');
12
+ const { credentialName, credentialValue, credentialValues } = await shared('redaction.mjs');
12
13
  const secrets = Array.isArray(declaration.secrets) ? declaration.secrets : [];
13
14
  const names = list => (Array.isArray(list) ? list : []).filter(name => typeof name === 'string');
14
15
  const fields = Array.isArray(setupManifest?.fields) ? setupManifest.fields : [];
15
- const secret = new Set([...PRIVATE_KEYS, ...secrets.flatMap(item => [item?.binding, item?.environment]), ...fields.filter(field => field?.kind === 'secret').map(field => field.name)].filter(Boolean));
16
- const open = new Set([...Object.keys(declaration.vars ?? {}), declaration.originVar, declaration.publicOriginSetting, ...names(declaration.publicSettings), ...fields.filter(field => field?.kind !== 'secret').map(field => field?.name)].filter(name => name && !secret.has(name)));
17
- // A declared setting is exempt from the key-name heuristic (AUTH_ORIGIN is a
18
- // name, not a credential), never when its value carries credentials itself.
19
- const shown = (key, value) => open.has(key) && typeof value === 'string' && !credentialUrl(value);
16
+ const privateFields = new Set([...PRIVATE_KEYS, ...fields.filter(field => field?.kind === 'secret').map(field => field.name)]);
17
+ // Select fields describe public choices. Other transported inputs need an
18
+ // explicit publicSettings declaration; an owner email is still private.
19
+ const publicTransport = new Set([...names(declaration.publicSettings), ...fields.filter(field => field?.kind === 'select').map(field => field.name)]);
20
+ const publicBindings = secrets.filter(item => publicTransport.has(item?.binding) &&
21
+ [item.binding, item.environment].every(name => name && !privateFields.has(name) && !credentialName(name)));
22
+ const secret = new Set([...privateFields, ...secrets.filter(item => !publicBindings.includes(item)).flatMap(item => [item?.binding, item?.environment])].filter(Boolean));
23
+ const open = new Set([...Object.keys(declaration.vars ?? {}), declaration.originVar, declaration.publicOriginSetting, ...names(declaration.publicSettings), ...fields.filter(field => field?.kind !== 'secret').map(field => field?.name), ...publicBindings.flatMap(item => [item.binding, item.environment])].filter(name => name && !secret.has(name)));
24
+ // Public names never exempt a credential-shaped value from masking.
25
+ const shown = (key, value) => open.has(key) && typeof value === 'string' && !credentialValue(value);
20
26
  return {
21
27
  shown,
22
28
  // The values to keep out of output, from every key that is not shown.
23
- privateValues: env => Object.entries(env).flatMap(([key, value]) => typeof value !== 'string' || shown(key, value) ? [] : secret.has(key) ? (value ? [value] : []) : credentialValues({ [key]: value })),
29
+ privateValues: env => Object.entries(env).flatMap(([key, value]) => typeof value !== 'string' || shown(key, value) ? [] : (secret.has(key) || credentialValue(value)) ? (value ? [value] : []) : credentialValues({ [key]: value })),
24
30
  // Keys the deploy hides by default and the person may declare.
25
31
  undeclared: keys => keys.filter(key => !open.has(key) && !secret.has(key) && !credentialName(key)),
26
32
  };
package/cli/deploy.mjs CHANGED
@@ -61,7 +61,7 @@ export function ownerDeploy(declarationPath, lifecycle) {
61
61
  const { main } = await import('../modules/deploy-kit/owner-cli.mjs');
62
62
  const { createRedaction } = await shared('redaction.mjs');
63
63
  const redaction = createRedaction();
64
- redaction.addSecrets([...(await settingKinds({ declaration, setupManifest })).privateValues(env), env.CLOUDFLARE_ACCOUNT_ID, ...(declaration.secrets ?? []).map(item => env[item.environment]).filter(Boolean)]);
64
+ redaction.addSecrets((await settingKinds({ declaration, setupManifest })).privateValues(env));
65
65
  const claim = setupManifest?.steps?.find(item => item.type === 'owner-claim');
66
66
  const declared = binding => (Array.isArray(declaration.secrets) ? declaration.secrets : []).find(item => item?.binding === binding);
67
67
  const owner = async (command, extra = [], values = env, more = {}) => main([command, '--declaration', declarationPath, '--environment', environment, ...(['dry-run', 'local-plan', 'deploy', 'health'].includes(command) ? await deployProductArguments(root, productDirectory) : []), ...extra], values, { cwd: root, localOnlyKeys, ...more });
@@ -5,6 +5,7 @@ import { excludedTable } from '../modules/portability/catalogue.mjs';
5
5
  import { schemaSQL } from '../modules/portability/schema-identity.mjs';
6
6
  import { ensurePortabilitySchema, portabilityPlan } from '../modules/container/portability-schema.mjs';
7
7
  import { ensureImportLeases } from '../modules/portability/import-lease.mjs';
8
+ import { ensureObjectInventory } from '../modules/portability/object-inventory.mjs';
8
9
  import { splitStatements } from '../modules/portability/sql-text.mjs';
9
10
  import { createPortability } from '../modules/portability/index.mjs';
10
11
  import { migrationReplay } from '../modules/portability/migration-history.mjs';
@@ -55,6 +56,7 @@ export async function managedSchema(db, plan, policy, ledger) {
55
56
  const native = nativeDatabase(scratch);
56
57
  await ensurePortabilitySchema(native, await portabilityPlan(splitStatements));
57
58
  await ensureImportLeases(native);
59
+ await ensureObjectInventory(native);
58
60
  previous = new Map((await all(native, query)).map(object => [key(object), object]));
59
61
  const inventory = [], groups = new Map();
60
62
  const account = (object, disposition, migration) => {
@@ -1,3 +1,4 @@
1
+ import { failureStep } from '../modules/portability/archive-step.mjs';
1
2
  import { installD1Preflight } from '../modules/container/adapters/d1-workerd-setup.mjs';
2
3
  import { portabilityPlan, ensurePortabilitySchema } from '../modules/container/portability-schema.mjs';
3
4
  import { exportArchive, importArchive, fileRecords } from '../modules/portability/files.mjs';
@@ -94,7 +95,8 @@ catch (error) {
94
95
  const identityTable = error instanceof PortabilityError && error.code.startsWith('identity_table_undeclared:') ? error.code.slice('identity_table_undeclared:'.length) : undefined;
95
96
  const safety = error instanceof PortabilityError && error.code.startsWith('import_row_') ? error.code.slice(error.code.indexOf(':')+1) : undefined;
96
97
  // Anything that is not a portability refusal names its class and code, never its message: that can hold row data.
98
+ const step=failureStep(error) ?? (process.argv[2] === 'import' ? undefined : process.argv[2]);
97
99
  const native = error instanceof PortabilityError || error.message === 'archive_json' ? undefined : nativeFailure(error);
98
100
  process.exitCode = exitCode;
99
- await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, ...(reExport ? { reExport: true } : {}), ...(migration ? { migration } : {}), ...(unapplied ? { unapplied } : {}), ...(occupied ? { occupied } : {}), ...(identityTable ? { identityTable } : {}), ...(safety ? { safety } : {}), ...(native ? { native } : {}), exitCode });
101
+ await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, step, ...(reExport ? { reExport: true } : {}), ...(migration ? { migration } : {}), ...(unapplied ? { unapplied } : {}), ...(occupied ? { occupied } : {}), ...(identityTable ? { identityTable } : {}), ...(safety ? { safety } : {}), ...(native ? { native } : {}), exitCode });
100
102
  }
@@ -110,7 +110,7 @@ async function nativeWorker(store,direction,input,signal) {
110
110
  : receipt.reason?.startsWith('migration_ledger:') ? ledgerRefusal(receipt)
111
111
  : receipt.migration ? `Restore refused (schema_mismatch:migration_data): migration ${receipt.migration} changed data after this archive was made, so its rows would skip that change. Restore an archive made after it.`
112
112
  : receipt.occupied ? occupiedRefusal(input,receipt.reason)
113
- : `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'}${receipt.native ? `: ${receipt.native.class}${receipt.native.code && receipt.native.code !== receipt.reason ? `, code ${receipt.native.code}` : ''}` : ''}).${receipt.native && receipt.exitCode === 1 ? ' Run the same command again; it starts over.' : ''}`, receipt.exitCode ?? 1);
113
+ : `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'}${receipt.native ? `: ${receipt.native.class}${receipt.native.code && receipt.native.code !== receipt.reason ? `, code ${receipt.native.code}` : ''}` : ''})${receipt.step ? ` during ${receipt.step}` : ''}.${receipt.native && receipt.exitCode === 1 ? ' Run the same command again; it starts over.' : ''}`, receipt.exitCode ?? 1);
114
114
  delete receipt.ok;
115
115
  return receipt;
116
116
  }
@@ -1,5 +1,24 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.2.119
4
+
5
+ ### Large installation archive inventories
6
+
7
+ - Archive export and restore store large object inventories in bounded internal rows instead of one oversized job-metadata value, addressing the D1 cell limit reached by a company-copy export with 13,879 objects. Small jobs keep their existing metadata representation. Archive parts can group multiple small objects, and restore reuses a shared part while reading its objects. The archive format and version are unchanged.
8
+ - Managed intake accounts for the internal inventory table. Native archive failures name the transfer stage without printing SQL, object keys, row values or filesystem paths.
9
+
10
+ ### Publication scan copy pointers
11
+
12
+ - Publication scans locate Gitleaks full matches within their reported lines when directory-fragment columns are relative to a fragment beginning mid-line. Every possible occurrence must satisfy the existing copy-pointer or checksum content proof; unresolved matches and unproved occurrences retain their finding.
13
+
14
+ ### Deploy output and timing checks
15
+
16
+ - Declared public deployment settings, including `MAIL_PROVIDER`, remain readable even when carried through secret transport. Credential-named bindings and aliases, private setup fields and credential-shaped values stay masked.
17
+ - Nested deploy hooks defer their dry-run completion and needs summary to the outer command, so the combined summary prints once after local planning succeeds.
18
+ - The real 20-second listing-backoff test still uses the native timer and high-resolution timestamps, observes the configured delay and compares elapsed time at Node's millisecond timer precision to avoid a fractional early-wake failure. Runtime backoff behavior is unchanged.
19
+
20
+ These changes become available after adopting the released package or refreshed source. This preparation establishes no company-copy export result, seed adoption, deployment or production result. The actual diff changes portability inventory storage and archive-part grouping, adds no product migrations, changes no import-kit source and preserves the archive format identity.
21
+
3
22
  ## 0.2.118
4
23
 
5
24
  ### Local diagnostics and Move in guidance
@@ -1133,6 +1133,8 @@ modules/portability/sqlite-tokens.mjs
1133
1133
  modules/portability/bookkeeping.mjs
1134
1134
  modules/portability/snapshot-objects.mjs
1135
1135
  modules/portability/storage.mjs
1136
+ modules/portability/object-inventory.mjs
1137
+ modules/portability/archive-step.mjs
1136
1138
  modules/portability/copies.mjs
1137
1139
  modules/portability/derived.mjs
1138
1140
  modules/portability/dev-data.mjs
@@ -1268,3 +1270,9 @@ parameters of at most 2 MiB each. It refuses compressed or encrypted entries,
1268
1270
  unsafe paths, duplicate or unexpected members, missing ledgers, other versions,
1269
1271
  container transport, later-journal and legacy-recovery options. Refusals use
1270
1272
  constant `managed_data_*` reasons and do not print source values.
1273
+
1274
+ Large object inventories are stored in bounded internal rows so D1 cell and row
1275
+ limits do not prevent backups. Native CLI failures name the transfer stage (for
1276
+ example `export_snapshot`) without printing SQL, values or absolute paths. Keep
1277
+ the stopped source and retry after resolving the named stage; changing source
1278
+ objects or deleting data is not a remedy for an inventory storage failure.
@@ -0,0 +1,11 @@
1
+ // Stage names are fixed by the implementation. Never send native messages,
2
+ // SQL, object keys, row values or filesystem paths to the CLI.
3
+ const steps=new Set(['export_object_inventory','export_snapshot','export_part']);
4
+ export async function archiveStep(step,operation) {
5
+ try{return await operation();}
6
+ catch(error){
7
+ if(error && typeof error==='object' && !error.portabilityStep)error.portabilityStep=step;
8
+ throw error;
9
+ }
10
+ }
11
+ export const failureStep=error=>steps.has(error?.portabilityStep)?error.portabilityStep:undefined;
@@ -1,4 +1,6 @@
1
- import { statement, state, guard, job, save, partStatements, parts, readPart, cleanCopies, quote, stream, insist, run } from './storage.mjs';
1
+ import { prepareInventory, metadataJson, inventoryStatements } from './object-inventory.mjs';
2
+ import { archiveStep } from './archive-step.mjs';
3
+ import { statement, state, guard, job, save, partStatements, parts, readPart, cleanCopies, quote, stream, insist } from './storage.mjs';
2
4
  import { catalogue, encodedRow } from './catalogue.mjs';
3
5
  import { checkReferences } from './references.mjs';
4
6
  import { createCopy,readCopy,normalizeCopy } from './copies.mjs';
@@ -16,7 +18,7 @@ export async function createExport(context,actor='direct') {
16
18
  for(const [binding,spec] of Object.entries(config.buckets).sort()) {
17
19
  let cursor;
18
20
  do {
19
- const page=await buckets[binding].list({prefix:spec.prefix??'',cursor,include:['httpMetadata','customMetadata']});
21
+ const page=await archiveStep('export_object_inventory',()=>buckets[binding].list({prefix:spec.prefix??'',cursor,include:['httpMetadata','customMetadata']}));
20
22
  for(const object of page.objects) objects.push({bucket:binding,key:object.key,size:object.size,etag:object.etag,httpMetadata:object.httpMetadata??{},customMetadata:object.customMetadata??{}});
21
23
  cursor=page.truncated?page.cursor:undefined;
22
24
  }while(cursor);
@@ -27,9 +29,10 @@ export async function createExport(context,actor='direct') {
27
29
  const manifest={kind:'manifest',format:FORMAT,version:2,product:config.product,exportedAt:new Date().toISOString(),sourceIdentity:s.source_identity,schema:cat.identity,tables:cat.tables,schemaObjects:cat.schemaObjects,objects,payloads:[],excludedTables:cat.excludedTables,excludedColumns:config.secretColumns,requiredConfiguration:config.requiredConfiguration,partCount:0,epoch:s.epoch,journal:await journal(db,s.source_identity),...(migrations?{migrations}:{}),...(origin?{migrationLedger:origin}:{})};
28
30
  const row={id,kind:'export',actor,phase:'building',completed:0,metadata:{manifest,copies,cursor:{table:0,row:0,binary:0,offset:0},partBytes:config.partBytes}};
29
31
  const audit=context.onExport?context.onExport({id,actor:actor==='direct'?null:actor,scope:'installation'}):[];
30
- await db.batch([guard(db,'EXISTS(SELECT 1 FROM rer_portability_state WHERE id=1 AND applying=0 AND source_identity=? AND epoch=?)',s.source_identity,s.epoch),
31
- ...cat.tables.flatMap(t=>createCopy(db,t,copies[t.name],true)),statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'export',actor,row.phase,canonicalJson(row.metadata)),db.prepare('DELETE FROM rer_portability_guard'),
32
- ...audit.map(([sql,args])=>statement(db,sql,...args))]);
32
+ prepareInventory(row);
33
+ await archiveStep('export_snapshot',()=>db.batch([guard(db,'EXISTS(SELECT 1 FROM rer_portability_state WHERE id=1 AND applying=0 AND source_identity=? AND epoch=?)',s.source_identity,s.epoch),
34
+ ...cat.tables.flatMap(t=>createCopy(db,t,copies[t.name],true)),statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'export',actor,row.phase,metadataJson(row.metadata)),...inventoryStatements(db,row),db.prepare('DELETE FROM rer_portability_guard'),
35
+ ...audit.map(([sql,args])=>statement(db,sql,...args))]));
33
36
  try {
34
37
  await dropDevData(db,cat.tables,copies);
35
38
  await dropProviderData(db,cat.tables,copies);
@@ -38,7 +41,7 @@ export async function createExport(context,actor='direct') {
38
41
  for(const [name,producer] of Object.entries(config.extraPayloads).sort()) {
39
42
  const descriptor=await capturePayload(db,id,manifest.payloads.length,await producer.produce());manifest.payloads.push({name,...descriptor});
40
43
  }
41
- await run(db,'UPDATE rer_portability_jobs SET metadata=? WHERE id=?',canonicalJson(row.metadata),id);
44
+ await save(db,row).run();
42
45
  await checkReferences(context,copies,objects);
43
46
  return id;
44
47
  }catch(e){await abortExport(context,id);throw e;}
@@ -68,24 +71,30 @@ async function nextItems(context,row) {
68
71
  }
69
72
  const entries=[...manifest.objects,...manifest.payloads];
70
73
  if(cursor.binary===entries.length)return null;
71
- const entry=entries[cursor.binary],length=Math.min(entry.size-cursor.offset,Math.max(1,Math.floor((partBytes-200-utf8(canonicalJson({bucket:entry.bucket??null,key:entry.key??null,name:entry.name??null})).length)*3/4)));
72
- let bytes,item;
73
- if(cursor.binary<manifest.objects.length) {
74
- const object=await buckets[entry.bucket].get(entry.key,length?{range:{offset:cursor.offset,length}}:undefined);
75
- insist(object && object.etag===entry.etag,'object_changed:'+entry.bucket+':'+entry.key);
76
- bytes=new Uint8Array(await object.arrayBuffer());insist(bytes.length===length,'object_changed:'+entry.key);
77
- item={kind:'object',bucket:entry.bucket,key:entry.key,offset:cursor.offset,data:base64(bytes)};
78
- }else{
79
- bytes=await payloadRange(db,row.id,cursor.binary-manifest.objects.length,cursor.offset,length);
80
- item={kind:'payload',name:entry.name,offset:cursor.offset,data:base64(bytes)};
74
+ const items=[];let used=200;
75
+ while(cursor.binary<entries.length && items.length<1000 && partBytes-used>512) {
76
+ const entry=entries[cursor.binary],header=utf8(canonicalJson({bucket:entry.bucket??null,key:entry.key??null,name:entry.name??null})).length;
77
+ if(used+200+header>=partBytes){insist(items.length,'binary_part_too_large');break;}
78
+ const length=Math.min(entry.size-cursor.offset,Math.max(1,Math.floor((partBytes-used-200-header)*3/4)));
79
+ let bytes,item;
80
+ if(cursor.binary<manifest.objects.length) {
81
+ const object=await buckets[entry.bucket].get(entry.key,length?{range:{offset:cursor.offset,length}}:undefined);
82
+ insist(object && object.etag===entry.etag,'object_changed:'+entry.bucket+':'+entry.key);
83
+ bytes=new Uint8Array(await object.arrayBuffer());insist(bytes.length===length,'object_changed:'+entry.key);
84
+ item={kind:'object',bucket:entry.bucket,key:entry.key,offset:cursor.offset,data:base64(bytes)};
85
+ }else{
86
+ bytes=await payloadRange(db,row.id,cursor.binary-manifest.objects.length,cursor.offset,length);
87
+ item={kind:'payload',name:entry.name,offset:cursor.offset,data:base64(bytes)};
88
+ }
89
+ const hash=new StreamHash();if(cursor.hash){hash.h=Uint32Array.from(cursor.hash.h);hash.block=Uint8Array.from(cursor.hash.block);hash.size=cursor.hash.size;hash.used=cursor.hash.used;}
90
+ hash.update(bytes);cursor.offset+=length;
91
+ if(cursor.offset===entry.size) {
92
+ const digest=hash.digest();if(entry.sha256)insist(entry.sha256===digest,'payload_changed');entry.sha256=digest;delete entry.etag;
93
+ cursor.binary++;cursor.offset=0;delete cursor.hash;
94
+ }else cursor.hash={h:[...hash.h],block:[...hash.block],size:hash.size,used:hash.used};
95
+ items.push(item);used+=utf8(canonicalJson(item)).length+1;
81
96
  }
82
- const hash=new StreamHash();if(cursor.hash){hash.h=Uint32Array.from(cursor.hash.h);hash.block=Uint8Array.from(cursor.hash.block);hash.size=cursor.hash.size;hash.used=cursor.hash.used;}
83
- hash.update(bytes);cursor.offset+=length;
84
- if(cursor.offset===entry.size) {
85
- const digest=hash.digest();if(entry.sha256)insist(entry.sha256===digest,'payload_changed');entry.sha256=digest;delete entry.etag;
86
- cursor.binary++;cursor.offset=0;delete cursor.hash;
87
- }else cursor.hash={h:[...hash.h],block:[...hash.block],size:hash.size,used:hash.used};
88
- return [item];
97
+ return items;
89
98
  }
90
99
  export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
91
100
  insist(Number.isSafeInteger(maxParts) && maxParts>0 && maxParts<=100,'advance_limit');
@@ -94,10 +103,10 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
94
103
  if(row.phase==='complete')return exportStatus(context,id);
95
104
  insist(row.phase==='building','export_phase',409);
96
105
  if(expectedPart!==undefined && row.completed!==expectedPart)return exportStatus(context,id);
97
- const before=row.completed,items=await nextItems(context,row);
106
+ const before=row.completed,binary=row.metadata.cursor.binary,items=await archiveStep('export_part',()=>nextItems(context,row));
98
107
  if(items) {
99
108
  const part=await makePart(before,items),size=await checkPart(part,before,row.metadata.partBytes);row.completed++;row.metadata.bytes=(row.metadata.bytes??0)+size+1;
100
- try{await context.db.batch([guard(context.db,'(SELECT completed FROM rer_portability_jobs WHERE id=?)=?',id,before),...partStatements(context.db,id,part),save(context.db,row),context.db.prepare('DELETE FROM rer_portability_guard')]);}
109
+ try{await context.db.batch([guard(context.db,'(SELECT completed FROM rer_portability_jobs WHERE id=?)=?',id,before),...partStatements(context.db,id,part),...inventoryStatements(context.db,row,binary,row.metadata.cursor.binary),save(context.db,row),context.db.prepare('DELETE FROM rer_portability_guard')]);}
101
110
  catch(e){if((await job(context.db,id)).completed===before)throw e;}
102
111
  if(expectedPart!==undefined)expectedPart++;
103
112
  }else{
@@ -1,3 +1,4 @@
1
+ import { metadataJson } from './object-inventory.mjs';
1
2
  import { encodedRow, validateRow, bindCell } from './catalogue.mjs';
2
3
  import { readCopy, copyRowid } from './copies.mjs';
3
4
  import { canonicalJson } from './hash.mjs';
@@ -44,7 +45,7 @@ const guardSql=`INSERT INTO rer_portability_guard(value) SELECT CASE WHEN (
44
45
  ) THEN 1 ELSE 0 END`;
45
46
  const checkpoint=(row,before,metadata,queries)=>[
46
47
  {sql:guardSql,params:[row.id,before,row.id]},...queries,
47
- {sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,canonicalJson(metadata),row.id]},
48
+ {sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,metadataJson(metadata),row.id]},
48
49
  {sql:'DELETE FROM rer_portability_guard'},
49
50
  ];
50
51
  // Each page's staged updates, cursor and value-free change counts commit together.
@@ -55,10 +56,10 @@ export async function normalizeStaged(context,row) {
55
56
  const copy=row.metadata.copies[table.name],count=row.metadata.stagedCounts?.[table.name]??0;
56
57
  let offset=row.metadata.safetyProgress?.[table.name]??0;
57
58
  while(offset<count) {
58
- let before=canonicalJson(row.metadata),queries=[];
59
+ let before=metadataJson(row.metadata),queries=[];
59
60
  const commit=async()=>{
60
61
  await db.batch(checkpoint(row,before,row.metadata,queries).map(q=>statement(db,q.sql,...(q.params??[]))));
61
- before=canonicalJson(row.metadata);queries=[];
62
+ before=metadataJson(row.metadata);queries=[];
62
63
  };
63
64
  for(const {row:raw,meta} of await readCopy(db,table,copy,offset,100,true)) {
64
65
  const original=encodedRow(table,raw),after=mapImportRow(config,table,original);
@@ -1,5 +1,5 @@
1
+ import { metadataJson } from './object-inventory.mjs';
1
2
  import { RPC_LIMIT, rpcSize, rpcSizeBytes } from '../d1-transport/index.mjs';
2
- import { canonicalJson } from './hash.mjs';
3
3
  import { statement, save, readPart, insist } from './storage.mjs';
4
4
  import { checkPart } from './format.mjs';
5
5
 
@@ -18,11 +18,11 @@ export async function stagePart(context,row,part,queries,stagedCounts) {
18
18
  insist(!pending || (pending.index===part.index && pending.sha256===part.sha256),'part_conflict',409);
19
19
  let next=pending?.next??0;
20
20
  while(next<queries.length) {
21
- const before=canonicalJson(row.metadata),chunk=[];
21
+ const before=metadataJson(row.metadata),chunk=[];
22
22
  // Reserve the largest cursor for this part before admitting any data.
23
23
  row.metadata.pendingPart={index:part.index,sha256:part.sha256,next:queries.length};
24
24
  const guard={sql:guardSql,params:[row.id,row.completed,before,row.id]};
25
- const checkpoint={sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,canonicalJson(row.metadata),row.id]};
25
+ const checkpoint={sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,metadataJson(row.metadata),row.id]};
26
26
  let size=rpcSize([guard,checkpoint,{sql:'DELETE FROM rer_portability_guard'}]),end=next;
27
27
  for(;end<queries.length;end++) {
28
28
  const candidate=combine(size,rpcSize([queries[end]]));
@@ -37,7 +37,7 @@ export async function stagePart(context,row,part,queries,stagedCounts) {
37
37
  // Authenticate the reassembled stored bytes before acknowledging the part.
38
38
  const stored=await readPart(db,row.id,part.index);
39
39
  await checkPart(stored,part.index,config.partBytes);insist(stored.sha256===part.sha256,'part_checksum');
40
- const before=canonicalJson(row.metadata),completed=row.completed;
40
+ const before=metadataJson(row.metadata),completed=row.completed;
41
41
  delete row.metadata.pendingPart;row.metadata.stagedCounts=stagedCounts;row.completed++;
42
42
  await db.batch([statement(db,guardSql,row.id,completed,before,row.id),
43
43
  statement(db,'INSERT INTO rer_portability_parts VALUES(?,?,?)',row.id,part.index,part.sha256),
@@ -1,3 +1,4 @@
1
+ import { prepareInventory, metadataJson, inventoryStatements } from './object-inventory.mjs';
1
2
  import { libraryMigrations } from './library-history.mjs';
2
3
  import { all, statement, state, guard, job, save, partQueries, parts, readPart, dropCopyStatements, cleanCopyStatements, quote, stream, insist, run } from './storage.mjs';
3
4
  import { stagePart } from './import-staging.mjs';
@@ -78,11 +79,12 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
78
79
  }
79
80
  const copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
80
81
  const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(unverified?{schemaUnverified:{reason:'acknowledged recovery of a version 2 archive without schemaObjects',sourceIdentity:original.sourceIdentity,exportedAt:original.exportedAt,acknowledgedAt:new Date().toISOString()}}:{}),...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge.fromSchema?{bridge}:{}),...(replays?{replays}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
82
+ prepareInventory(row);
81
83
  try {
82
84
  await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
83
85
  statement(db,'UPDATE rer_portability_state SET import_job=?,applying=? WHERE id=1',id,unverified?1:0),
84
86
  ...targets.flatMap(t=>createCopy(db,t,copies[t.name])),
85
- statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'import',actor,row.phase,canonicalJson(row.metadata)),createImportLease(db,id),db.prepare('DELETE FROM rer_portability_guard')]);
87
+ statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'import',actor,row.phase,metadataJson(row.metadata)),...inventoryStatements(db,row),createImportLease(db,id),db.prepare('DELETE FROM rer_portability_guard')]);
86
88
  }catch(e){const running=(await state(db)).import_job;if(running)throw await importRunning(db,running);throw e;}
87
89
  return id;
88
90
  }
@@ -136,10 +138,10 @@ async function uploadLeasedPart(context,id,index,part) {
136
138
  return importStatus(context,id,{includeLease:false});
137
139
  }
138
140
  const binaryKey=entry=>canonicalJson(entry.bucket===undefined?['payload',entry.name]:['object',entry.bucket,entry.key]);
139
- function binaryStream(context,id,entry,indexes) {
141
+ function binaryStream(entry,indexes,read) {
140
142
  const selected=indexes.get(binaryKey(entry));insist(selected,'binary_inventory');
141
143
  return stream((async function*(){
142
- for(const index of selected)for(const item of (await readPart(context.db,id,index)).items) {
144
+ for(const index of selected)for(const item of (await read(index)).items) {
143
145
  if(entry.bucket===undefined?item.kind==='payload' && item.name===entry.name:item.kind==='object' && item.bucket===entry.bucket && item.key===entry.key)yield unbase64(item.data);
144
146
  }
145
147
  })());
@@ -218,17 +220,24 @@ async function finalizeLeasedImport(context,id,{contentHash,archiveHash,laterJou
218
220
  }
219
221
  row.phase='applying';
220
222
  await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1)=?',id),db.prepare('UPDATE rer_portability_state SET applying=1 WHERE id=1'),save(db,row),db.prepare('DELETE FROM rer_portability_guard')]);
223
+ // Grouped objects often share a part. Keep one immutable part during this
224
+ // leased finalize, rather than rereading its bytes for every small object.
225
+ let cached;
226
+ const readBinaryPart=async index=>{
227
+ if(cached?.index!==index)cached=await readPart(db,id,index);
228
+ return cached;
229
+ };
221
230
  try {
222
231
  for(const object of manifest.objects) {
223
232
  await run(db,'INSERT OR IGNORE INTO rer_portability_owned VALUES(?,?,?)',id,object.bucket,object.key);
224
- await writeObject(context.buckets[object.bucket],object,binaryStream(context,id,object,binaryIndexes));
233
+ await writeObject(context.buckets[object.bucket],object,binaryStream(object,binaryIndexes,readBinaryPart));
225
234
  const stored=await context.buckets[object.bucket].get(object.key);insist(stored,'object_write_failed');
226
235
  // Read back independently, including multipart results, before the commit.
227
236
  const hash=new (await import('./hash.mjs')).StreamHash();const reader=stored.body.getReader();
228
237
  for(;;){const {value,done}=await reader.read();if(done)break;hash.update(value);}reader.releaseLock();
229
238
  insist(hash.digest()===object.sha256,'object_write_checksum');
230
239
  }
231
- for(const payload of manifest.payloads)await config.extraPayloads[payload.name].consume(binaryStream(context,id,payload,binaryIndexes),payload);
240
+ for(const payload of manifest.payloads)await config.extraPayloads[payload.name].consume(binaryStream(payload,binaryIndexes,readBinaryPart),payload);
232
241
  const derived=await rebuildDerivedStatements(db,config);
233
242
  row.phase='complete';row.metadata.contentHash=digest;
234
243
  if(row.metadata.schemaUnverified)row.metadata.schemaUnverified.archiveHash=archiveHash;
@@ -1,3 +1,4 @@
1
+ import { ensureObjectInventory } from './object-inventory.mjs';
1
2
  import { ensureImportLeases } from './import-lease.mjs';
2
3
  import { policy, catalogue } from './catalogue.mjs';
3
4
  import { wireDatabase } from '../d1-transport/index.mjs';
@@ -14,7 +15,7 @@ export { createPortabilityHandler } from './http.mjs';
14
15
  export async function createPortability({db,buckets={},policy:input,migrations,onExport,migrationRoot}) {
15
16
  insist(onExport===undefined || typeof onExport==='function','invalid_on_export');
16
17
  db=wireDatabase(db);
17
- const config=policy(input);await catalogue(db,config);
18
+ const config=policy(input);await ensureObjectInventory(db);await catalogue(db,config);
18
19
  for(const name of Object.keys(config.buckets))insist(buckets[name] && typeof buckets[name].get==='function','missing_bucket:'+name);
19
20
  await ensureImportLeases(db);
20
21
  const context={db,buckets,config,migrations,onExport};
@@ -0,0 +1,45 @@
1
+ import { canonicalJson, utf8 } from './hash.mjs';
2
+
3
+ const table='rer_portability_object_inventory';
4
+ const chunkSize=50;
5
+ export async function ensureObjectInventory(db) {
6
+ await db.prepare(`CREATE TABLE IF NOT EXISTS ${table}(job TEXT NOT NULL REFERENCES rer_portability_jobs(id) ON DELETE CASCADE, chunk INTEGER NOT NULL, data TEXT NOT NULL, PRIMARY KEY(job,chunk))`).run();
7
+ }
8
+ // Keep large manifests out of D1's per-value and per-row limits. Only the
9
+ // storage representation changes; archive manifests still contain every object.
10
+ export function prepareInventory(row) {
11
+ if(utf8(canonicalJson(row.metadata.manifest.objects)).length>256*1024)
12
+ row.metadata.objectInventory=Math.ceil(row.metadata.manifest.objects.length/chunkSize);
13
+ }
14
+ export function metadataJson(metadata) {
15
+ if(!metadata.objectInventory)return canonicalJson(metadata);
16
+ const compact={...metadata,manifest:{...metadata.manifest,objects:[]}};
17
+ if(metadata.restoreManifest)compact.restoreManifest={...metadata.restoreManifest,objects:[]};
18
+ return canonicalJson(compact);
19
+ }
20
+ export function inventoryStatements(db,row,start=0,end=row.metadata.manifest.objects.length) {
21
+ if(!row.metadata.objectInventory)return [];
22
+ const objects=row.metadata.manifest.objects,statements=[];
23
+ for(let chunk=Math.floor(start/chunkSize);chunk<Math.ceil(end/chunkSize);chunk++)
24
+ statements.push(db.prepare(`INSERT OR REPLACE INTO ${table}(job,chunk,data) VALUES(?,?,?)`).bind(row.id,chunk,canonicalJson(objects.slice(chunk*chunkSize,(chunk+1)*chunkSize))));
25
+ return statements;
26
+ }
27
+ export async function loadInventory(db,row) {
28
+ if(!row.metadata.objectInventory)return row;
29
+ const objects=[];
30
+ // Read bounded pages: a complete inventory can exceed the RPC result limit.
31
+ for(let offset=0;offset<row.metadata.objectInventory;offset+=16) {
32
+ const {results}=await db.prepare(`SELECT data FROM ${table} WHERE job=? ORDER BY chunk LIMIT 16 OFFSET ?`).bind(row.id,offset).all();
33
+ for(const item of results)objects.push(...JSON.parse(item.data));
34
+ }
35
+ row.metadata.manifest.objects=objects;
36
+ if(row.metadata.restoreManifest)row.metadata.restoreManifest.objects=objects;
37
+ return row;
38
+ }
39
+ export function clearInventory(db,row) {
40
+ if(!row.metadata.objectInventory)return [];
41
+ delete row.metadata.objectInventory;
42
+ row.metadata.manifest.objects=[];
43
+ if(row.metadata.restoreManifest)row.metadata.restoreManifest.objects=[];
44
+ return [db.prepare(`DELETE FROM ${table} WHERE job=?`).bind(row.id)];
45
+ }
@@ -1,3 +1,4 @@
1
+ import { metadataJson, loadInventory, clearInventory } from './object-inventory.mjs';
1
2
  import { canonicalJson } from './hash.mjs';
2
3
  // Installed helpers and copied runtimes can coexist in one Worker. Share only
3
4
  // constructor-issued identities, so arbitrary objects with code/status fields
@@ -26,10 +27,10 @@ export async function state(db) {
26
27
  export async function job(db,id,kind) {
27
28
  const row=await statement(db,'SELECT * FROM rer_portability_jobs WHERE id=?',id).first();
28
29
  insist(row && (!kind || row.kind===kind),'job_not_found',404);
29
- return {...row, metadata:JSON.parse(row.metadata)};
30
+ return loadInventory(db,{...row, metadata:JSON.parse(row.metadata)});
30
31
  }
31
32
  export function save(db,row) {
32
- return statement(db,'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',row.phase,row.completed,canonicalJson(row.metadata),row.id);
33
+ return statement(db,'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',row.phase,row.completed,metadataJson(row.metadata),row.id);
33
34
  }
34
35
  export function partStatements(db,id,part) {
35
36
  return partQueries(id,part).map(({sql,params})=>statement(db,sql,...params));
@@ -64,7 +65,7 @@ export function dropCopyStatements(db,row) {
64
65
  return transferTables(row).map(name=>db.prepare(`DROP TABLE IF EXISTS ${quote(name)}`));
65
66
  }
66
67
  export function cleanCopyStatements(db,row) {
67
- return [...dropCopyStatements(db,row),statement(db,'DELETE FROM rer_portability_parts WHERE job=?',row.id),
68
+ return [...dropCopyStatements(db,row),...clearInventory(db,row),statement(db,'DELETE FROM rer_portability_parts WHERE job=?',row.id),
68
69
  statement(db,'DELETE FROM rer_portability_segments WHERE job=?',row.id)];
69
70
  }
70
71
  export async function cleanCopies(db,row) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@runeditrun/rer",
3
- "version": "0.2.118",
3
+ "version": "0.2.119",
4
4
  "description": "The rer command: install, set up, run and launch Run Edit Run checkouts.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -148,7 +148,11 @@ entry that does not prove, a 40-hex value, a key that names no tree file, or a h
148
148
  value with no file-name key keeps the native finding. Historical matches use only
149
149
  that commit's blobs and working-tree preimages reject linked paths. There is no
150
150
  path, filename, hash-value or repository exemption. `hit_location.py` resolves
151
- the native match's exact bytes for both proofs.
151
+ the native match's exact bytes for both proofs. Gitleaks directory fragments can
152
+ start mid-line without reporting their starting column. The resolver therefore
153
+ locates full matches within the reported LF-delimited lines. An unresolved match
154
+ retains its finding; repeated matches must all satisfy the structural and digest
155
+ proofs above, even if a reported column appears to identify just one of them.
152
156
 
153
157
  `trust_bundle.py` classifies an email match as public CA certificate metadata
154
158
  only when all of these hold: the file's path is an artifact listed by a committed
@@ -35,18 +35,22 @@ class ChecksumMaps:
35
35
  def matches(self, data, hit):
36
36
  if hit.get('RuleID') not in ('generic-api-key', 'hex-api-key'):
37
37
  return False
38
+ spans = match_spans(data, hit)
39
+ if not spans:
40
+ return False
38
41
  entries = list(ENTRY.finditer(data))
39
- for left, right in match_spans(data, hit):
42
+ for left, right in spans:
40
43
  keyed = [i for i, e in enumerate(entries)
41
44
  if left <= e.start('digest') and e.end('digest') <= right
42
45
  and e.group('digest').decode() == hit.get('Secret')]
43
46
  if len(keyed) != 1:
44
- continue
47
+ return False
45
48
  first = last = keyed[0]
46
49
  while first > 0 and SEPARATOR.fullmatch(data, entries[first - 1].end(), entries[first].start()):
47
50
  first -= 1
48
51
  while last + 1 < len(entries) and SEPARATOR.fullmatch(data, entries[last].end(), entries[last + 1].start()):
49
52
  last += 1
50
- return all(self.names_file((e.group('quoted') or e.group('bare')).decode('utf-8', 'replace'),
51
- e.group('digest').decode()) for e in entries[first:last + 1])
52
- return False
53
+ if not all(self.names_file((e.group('quoted') or e.group('bare')).decode('utf-8', 'replace'),
54
+ e.group('digest').decode()) for e in entries[first:last + 1]):
55
+ return False
56
+ return True
@@ -1,18 +1,29 @@
1
- """Resolve a native Gitleaks hit to the exact byte spans of its match."""
1
+ """Resolve a native Gitleaks hit to its possible full-match byte spans."""
2
2
 
3
3
 
4
4
  def match_spans(data, hit):
5
- lines = data.splitlines(keepends=True)
6
5
  try:
6
+ # Gitleaks counts LF bytes, not Python's broader set of line breaks.
7
+ lines = data.split(b'\n')
7
8
  first, last = hit['StartLine'] - 1, hit['EndLine'] - 1
8
9
  if not (0 <= first <= last < len(lines)):
9
10
  return []
10
- start = sum(map(len, lines[:first])) + hit['StartColumn'] - 1
11
- end = sum(map(len, lines[:last])) + hit['EndColumn']
12
- # Gitleaks 8.30.1 includes the preceding newline in columns
13
- # after fragment line one; Git fragments may begin mid-file.
14
- # Resolve that one-byte difference only against the exact match.
15
- return [(start - shift, end - shift) for shift in (0, 1)
16
- if data[start - shift:end - shift] == hit['Match'].encode('utf-8')]
11
+ match = hit['Match'].encode('utf-8')
12
+ if not match:
13
+ return []
14
+ left = sum(len(line) + 1 for line in lines[:first])
15
+ right = left + sum(len(line) + 1 for line in lines[first:last + 1])
16
+ # Directory fragments can start mid-line. Their columns are relative
17
+ # to that fragment, whose starting column is absent from the report.
18
+ # Later fragment lines also count the preceding LF in their columns.
19
+ # Bind the full match within the reported lines instead. Even an
20
+ # apparently exact column cannot disambiguate repeated occurrences.
21
+ # Callers must prove every candidate, never just the first one.
22
+ spans = []
23
+ start = data.find(match, left, right)
24
+ while start >= 0:
25
+ spans.append((start, start + len(match)))
26
+ start = data.find(match, start + 1, right)
27
+ return spans
17
28
  except (KeyError, TypeError, IndexError, UnicodeError):
18
29
  return []
@@ -80,19 +80,20 @@ class Document:
80
80
  secret = hit.get('Secret', '').encode('utf-8')
81
81
  if not secret:
82
82
  return False
83
- located = set()
84
- for left, right in match_spans(self.data, hit):
83
+ spans = match_spans(self.data, hit)
84
+ if not spans:
85
+ return False
86
+ allowed = [self.spans[pointer] for pointer in pointers if pointer in self.spans]
87
+ for left, right in spans:
85
88
  start = self.data.find(secret, left, right)
86
89
  if start < 0 or self.data.find(secret, start + 1, right) >= 0:
87
- continue
88
- for pointer in pointers:
89
- span = self.spans.get(pointer)
90
- if span:
91
- a, b, value = span
92
- # Native fragments can truncate a digest capture. Bind the
93
- # captured secret's bytes, including a partial capture, to
94
- # this exact unescaped JSON value.
95
- if (a + 1 <= start and start + len(secret) <= b - 1
96
- and self.data[a + 1:b - 1] == value.encode('ascii')):
97
- located.add((a, b))
98
- return len(located) == 1
90
+ return False
91
+ # Location alone is not proof. Every possible native occurrence
92
+ # must bind its captured bytes (even a truncated capture) to an
93
+ # exact unescaped JSON value at a proved pointer. An identical
94
+ # credential elsewhere on the line therefore retains the finding.
95
+ if not any(a + 1 <= start and start + len(secret) <= b - 1
96
+ and self.data[a + 1:b - 1] == value.encode('ascii')
97
+ for a, b, value in allowed):
98
+ return False
99
+ return True