@runeditrun/rer 0.2.118 → 0.2.119
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli/deploy-command.mjs +13 -7
- package/cli/deploy-inputs.mjs +1 -4
- package/cli/deploy-settings.mjs +15 -9
- package/cli/deploy.mjs +1 -1
- package/cli/managed-data-schema.mjs +2 -0
- package/cli/portability-worker.mjs +3 -1
- package/cli/portability.mjs +1 -1
- package/global/CHANGELOG.md +19 -0
- package/modules/portability/README.md +8 -0
- package/modules/portability/archive-step.mjs +11 -0
- package/modules/portability/export.mjs +34 -25
- package/modules/portability/import-safety.mjs +4 -3
- package/modules/portability/import-staging.mjs +4 -4
- package/modules/portability/import.mjs +14 -5
- package/modules/portability/index.mjs +2 -1
- package/modules/portability/object-inventory.mjs +45 -0
- package/modules/portability/storage.mjs +4 -3
- package/package.json +1 -1
- package/scripts/publish-check/README.md +5 -1
- package/scripts/publish-check/checksum_maps.py +9 -5
- package/scripts/publish-check/hit_location.py +20 -9
- package/scripts/publish-check/json_identity.py +15 -14
package/cli/deploy-command.mjs
CHANGED
|
@@ -26,6 +26,10 @@ import { keepHookSecrets } from './deploy-secrets.mjs';
|
|
|
26
26
|
import { withoutLocalSettings, refuseLocalEnvOutput } from './local-env.mjs';
|
|
27
27
|
import { deployProductDirectory, deployProductArguments, DeployProductError } from './deploy-product.mjs';
|
|
28
28
|
|
|
29
|
+
// A hook may wrap this controller. Its needs return to the caller, which renders
|
|
30
|
+
// the combined dry-run summary once after every local plan step has succeeded.
|
|
31
|
+
const deferDryRunSummary = Symbol('defer dry-run summary');
|
|
32
|
+
|
|
29
33
|
export function validateDeployOptions(options) {
|
|
30
34
|
const changes = [options.remove && '--remove', options.setSecret && '--set-secret', options.unsetSecret && '--unset-secret', options.rotateSecret && '--rotate-secret', options.reopenInstallationKey && '--reopen-installation-key'].filter(Boolean);
|
|
31
35
|
if (changes.length > 1 || changes.length && options.dryRun) throw invalid(`Use ${changes.join(', ')}${options.dryRun ? ' and --dry-run' : ''} on separate runs.`);
|
|
@@ -88,16 +92,18 @@ export async function deployCommand(context) {
|
|
|
88
92
|
const plan = result?.plan ?? (!planned ? undefined : (await owner(['local-plan', '--declaration', path, '--environment', options.environment ?? 'production', ...await deployProductArguments(root, deployProductDirectory(declaration))], production.env, { cwd: root, localOnlyKeys })).plan);
|
|
89
93
|
result = { ...result, dryRun: true, deployed: false, providerChecked: false, plan };
|
|
90
94
|
const { signIn, mailOff } = await requireProductionAuth({ root, declaration, env: production.env, setupManifest, setupSeed: context.setupSeed, adapter, declarationPath: path, options }) ?? {};
|
|
91
|
-
if (mailOff) store.report(mailOffNotice(mailOff, options.environment ?? 'production'), { names: mailOff.credentialNames });
|
|
92
|
-
// The one completion line: rer's own plan step (or its deliberate skip) succeeded. A hook's own line says only that the hook finished.
|
|
93
|
-
store.report('deploy: local dry run complete; no credentials were needed, no Cloudflare requests were made and nothing was changed remotely. Target secrets and remote readiness were not checked.');
|
|
94
95
|
const needed = [...new Set([...neededForRealDeploy({ declaration, env: production.env, signIn, secretsChecked: false }), ...(result.neededForRealDeploy ?? []), ...(extraNeeded ?? [])])];
|
|
95
|
-
|
|
96
|
-
|
|
96
|
+
if (!context[deferDryRunSummary]) {
|
|
97
|
+
if (mailOff) store.report(mailOffNotice(mailOff, options.environment ?? 'production'), { names: mailOff.credentialNames });
|
|
98
|
+
// The one completion line: rer's own plan step (or its deliberate skip) succeeded. A hook's own line says only that the hook finished.
|
|
99
|
+
store.report('deploy: local dry run complete; no credentials were needed, no Cloudflare requests were made and nothing was changed remotely. Target secrets and remote readiness were not checked.');
|
|
100
|
+
reportNeeded(store, needed);
|
|
101
|
+
if (declaration.secrets?.some(item => item.binding === 'INSTALLATION_KEY')) store.report('deploy: INSTALLATION_KEY is a private generated binding; INSTALLATION_KEY_PREVIOUS is optional and managed only during rotation or an explicit retired-key reopen. Values are withheld.');
|
|
102
|
+
}
|
|
97
103
|
return needed.length && result && typeof result === 'object' ? { ...result, neededForRealDeploy: needed } : result;
|
|
98
104
|
};
|
|
99
105
|
const runHook = hookContext => options.dryRun ? withLocalDeployPlan(async () => {
|
|
100
|
-
try { return await adapter.deploy(hookContext); }
|
|
106
|
+
try { return await adapter.deploy({ ...hookContext, [deferDryRunSummary]: true }); }
|
|
101
107
|
catch (error) {
|
|
102
108
|
// Production-only owner/input preflights may refuse before the hook builds.
|
|
103
109
|
// An offline plan defers those inputs, retaining the refusal as guidance.
|
|
@@ -123,7 +129,7 @@ export async function deployCommand(context) {
|
|
|
123
129
|
const redaction = createRedaction();
|
|
124
130
|
const kinds = await settingKinds({ declaration, setupManifest });
|
|
125
131
|
const addSecret = value => { redaction.addSecrets([value]); store.addSecrets?.([value]); };
|
|
126
|
-
for (const value of
|
|
132
|
+
for (const value of kinds.privateValues(env)) addSecret(value);
|
|
127
133
|
const environment = options.environment ?? 'production';
|
|
128
134
|
// The hook uploads every value it is given: a generated secret it gets is the one the Worker keeps.
|
|
129
135
|
if (!options.dryRun) await keepHookSecrets({ root, environment, declaration, setupManifest, env, generated, reused, rotated, again: deployAgain(options), report: (line, more) => store.report(line, more) });
|
package/cli/deploy-inputs.mjs
CHANGED
|
@@ -77,11 +77,8 @@ export async function withDeployEnvironment(project, options, out, run) {
|
|
|
77
77
|
const { declaredSettings, settingKinds, undeclaredNotice } = await import('./deploy-settings.mjs');
|
|
78
78
|
const declared = await declaredSettings(project), kinds = await settingKinds(declared);
|
|
79
79
|
const redaction = createRedaction();
|
|
80
|
-
// Provider names are public enum values. Masking "resend" would also hide
|
|
81
|
-
// RESEND_API_KEY in the preflight's instruction for the missing credential.
|
|
82
|
-
const publicProvider = (key, value) => key === 'MAIL_PROVIDER' && /^(resend|sendgrid|cloudflare|rer|later|none)$/.test(value);
|
|
83
80
|
const supplied = Object.entries(env).filter(([key, value]) => saved[key] !== value);
|
|
84
|
-
redaction.addSecrets([...kinds.privateValues(env), ...supplied.filter(([key, value]) => !kinds.shown(key, value)
|
|
81
|
+
redaction.addSecrets([...kinds.privateValues(env), ...supplied.filter(([key, value]) => !kinds.shown(key, value)).map(([, value]) => value)]);
|
|
85
82
|
const undeclared = declared.declaration ? kinds.undeclared(supplied.map(([key]) => key)) : [];
|
|
86
83
|
if (undeclared.length) {
|
|
87
84
|
const message = undeclaredNotice(undeclared, declared.path);
|
package/cli/deploy-settings.mjs
CHANGED
|
@@ -3,24 +3,30 @@ import { shared } from './shared.mjs';
|
|
|
3
3
|
// What a deployment's declaration says each value is. A value prints in full only
|
|
4
4
|
// when the declaration names it a non-secret setting: a declared var, the origin
|
|
5
5
|
// variable and its setting, a setup field that is not a secret, or a name in
|
|
6
|
-
// "publicSettings".
|
|
7
|
-
//
|
|
6
|
+
// "publicSettings". Secret transport alone does not make an explicitly public
|
|
7
|
+
// setting private. Credential bindings, private setup fields and undeclared
|
|
8
|
+
// --env-file values stay redacted.
|
|
8
9
|
const PRIVATE_KEYS = ['CLOUDFLARE_ACCOUNT_ID'];
|
|
9
10
|
|
|
10
11
|
export async function settingKinds({ declaration = {}, setupManifest } = {}) {
|
|
11
|
-
const { credentialName,
|
|
12
|
+
const { credentialName, credentialValue, credentialValues } = await shared('redaction.mjs');
|
|
12
13
|
const secrets = Array.isArray(declaration.secrets) ? declaration.secrets : [];
|
|
13
14
|
const names = list => (Array.isArray(list) ? list : []).filter(name => typeof name === 'string');
|
|
14
15
|
const fields = Array.isArray(setupManifest?.fields) ? setupManifest.fields : [];
|
|
15
|
-
const
|
|
16
|
-
|
|
17
|
-
//
|
|
18
|
-
|
|
19
|
-
const
|
|
16
|
+
const privateFields = new Set([...PRIVATE_KEYS, ...fields.filter(field => field?.kind === 'secret').map(field => field.name)]);
|
|
17
|
+
// Select fields describe public choices. Other transported inputs need an
|
|
18
|
+
// explicit publicSettings declaration; an owner email is still private.
|
|
19
|
+
const publicTransport = new Set([...names(declaration.publicSettings), ...fields.filter(field => field?.kind === 'select').map(field => field.name)]);
|
|
20
|
+
const publicBindings = secrets.filter(item => publicTransport.has(item?.binding) &&
|
|
21
|
+
[item.binding, item.environment].every(name => name && !privateFields.has(name) && !credentialName(name)));
|
|
22
|
+
const secret = new Set([...privateFields, ...secrets.filter(item => !publicBindings.includes(item)).flatMap(item => [item?.binding, item?.environment])].filter(Boolean));
|
|
23
|
+
const open = new Set([...Object.keys(declaration.vars ?? {}), declaration.originVar, declaration.publicOriginSetting, ...names(declaration.publicSettings), ...fields.filter(field => field?.kind !== 'secret').map(field => field?.name), ...publicBindings.flatMap(item => [item.binding, item.environment])].filter(name => name && !secret.has(name)));
|
|
24
|
+
// Public names never exempt a credential-shaped value from masking.
|
|
25
|
+
const shown = (key, value) => open.has(key) && typeof value === 'string' && !credentialValue(value);
|
|
20
26
|
return {
|
|
21
27
|
shown,
|
|
22
28
|
// The values to keep out of output, from every key that is not shown.
|
|
23
|
-
privateValues: env => Object.entries(env).flatMap(([key, value]) => typeof value !== 'string' || shown(key, value) ? [] : secret.has(key) ? (value ? [value] : []) : credentialValues({ [key]: value })),
|
|
29
|
+
privateValues: env => Object.entries(env).flatMap(([key, value]) => typeof value !== 'string' || shown(key, value) ? [] : (secret.has(key) || credentialValue(value)) ? (value ? [value] : []) : credentialValues({ [key]: value })),
|
|
24
30
|
// Keys the deploy hides by default and the person may declare.
|
|
25
31
|
undeclared: keys => keys.filter(key => !open.has(key) && !secret.has(key) && !credentialName(key)),
|
|
26
32
|
};
|
package/cli/deploy.mjs
CHANGED
|
@@ -61,7 +61,7 @@ export function ownerDeploy(declarationPath, lifecycle) {
|
|
|
61
61
|
const { main } = await import('../modules/deploy-kit/owner-cli.mjs');
|
|
62
62
|
const { createRedaction } = await shared('redaction.mjs');
|
|
63
63
|
const redaction = createRedaction();
|
|
64
|
-
redaction.addSecrets(
|
|
64
|
+
redaction.addSecrets((await settingKinds({ declaration, setupManifest })).privateValues(env));
|
|
65
65
|
const claim = setupManifest?.steps?.find(item => item.type === 'owner-claim');
|
|
66
66
|
const declared = binding => (Array.isArray(declaration.secrets) ? declaration.secrets : []).find(item => item?.binding === binding);
|
|
67
67
|
const owner = async (command, extra = [], values = env, more = {}) => main([command, '--declaration', declarationPath, '--environment', environment, ...(['dry-run', 'local-plan', 'deploy', 'health'].includes(command) ? await deployProductArguments(root, productDirectory) : []), ...extra], values, { cwd: root, localOnlyKeys, ...more });
|
|
@@ -5,6 +5,7 @@ import { excludedTable } from '../modules/portability/catalogue.mjs';
|
|
|
5
5
|
import { schemaSQL } from '../modules/portability/schema-identity.mjs';
|
|
6
6
|
import { ensurePortabilitySchema, portabilityPlan } from '../modules/container/portability-schema.mjs';
|
|
7
7
|
import { ensureImportLeases } from '../modules/portability/import-lease.mjs';
|
|
8
|
+
import { ensureObjectInventory } from '../modules/portability/object-inventory.mjs';
|
|
8
9
|
import { splitStatements } from '../modules/portability/sql-text.mjs';
|
|
9
10
|
import { createPortability } from '../modules/portability/index.mjs';
|
|
10
11
|
import { migrationReplay } from '../modules/portability/migration-history.mjs';
|
|
@@ -55,6 +56,7 @@ export async function managedSchema(db, plan, policy, ledger) {
|
|
|
55
56
|
const native = nativeDatabase(scratch);
|
|
56
57
|
await ensurePortabilitySchema(native, await portabilityPlan(splitStatements));
|
|
57
58
|
await ensureImportLeases(native);
|
|
59
|
+
await ensureObjectInventory(native);
|
|
58
60
|
previous = new Map((await all(native, query)).map(object => [key(object), object]));
|
|
59
61
|
const inventory = [], groups = new Map();
|
|
60
62
|
const account = (object, disposition, migration) => {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { failureStep } from '../modules/portability/archive-step.mjs';
|
|
1
2
|
import { installD1Preflight } from '../modules/container/adapters/d1-workerd-setup.mjs';
|
|
2
3
|
import { portabilityPlan, ensurePortabilitySchema } from '../modules/container/portability-schema.mjs';
|
|
3
4
|
import { exportArchive, importArchive, fileRecords } from '../modules/portability/files.mjs';
|
|
@@ -94,7 +95,8 @@ catch (error) {
|
|
|
94
95
|
const identityTable = error instanceof PortabilityError && error.code.startsWith('identity_table_undeclared:') ? error.code.slice('identity_table_undeclared:'.length) : undefined;
|
|
95
96
|
const safety = error instanceof PortabilityError && error.code.startsWith('import_row_') ? error.code.slice(error.code.indexOf(':')+1) : undefined;
|
|
96
97
|
// Anything that is not a portability refusal names its class and code, never its message: that can hold row data.
|
|
98
|
+
const step=failureStep(error) ?? (process.argv[2] === 'import' ? undefined : process.argv[2]);
|
|
97
99
|
const native = error instanceof PortabilityError || error.message === 'archive_json' ? undefined : nativeFailure(error);
|
|
98
100
|
process.exitCode = exitCode;
|
|
99
|
-
await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, ...(reExport ? { reExport: true } : {}), ...(migration ? { migration } : {}), ...(unapplied ? { unapplied } : {}), ...(occupied ? { occupied } : {}), ...(identityTable ? { identityTable } : {}), ...(safety ? { safety } : {}), ...(native ? { native } : {}), exitCode });
|
|
101
|
+
await reply({ ok: false, code: exitCode === 2 ? 'invalid_archive' : exitCode === 4 ? 'restore_refused' : 'transfer_failed', reason, step, ...(reExport ? { reExport: true } : {}), ...(migration ? { migration } : {}), ...(unapplied ? { unapplied } : {}), ...(occupied ? { occupied } : {}), ...(identityTable ? { identityTable } : {}), ...(safety ? { safety } : {}), ...(native ? { native } : {}), exitCode });
|
|
100
102
|
}
|
package/cli/portability.mjs
CHANGED
|
@@ -110,7 +110,7 @@ async function nativeWorker(store,direction,input,signal) {
|
|
|
110
110
|
: receipt.reason?.startsWith('migration_ledger:') ? ledgerRefusal(receipt)
|
|
111
111
|
: receipt.migration ? `Restore refused (schema_mismatch:migration_data): migration ${receipt.migration} changed data after this archive was made, so its rows would skip that change. Restore an archive made after it.`
|
|
112
112
|
: receipt.occupied ? occupiedRefusal(input,receipt.reason)
|
|
113
|
-
: `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'}${receipt.native ? `: ${receipt.native.class}${receipt.native.code && receipt.native.code !== receipt.reason ? `, code ${receipt.native.code}` : ''}` : ''}).${receipt.native && receipt.exitCode === 1 ? ' Run the same command again; it starts over.' : ''}`, receipt.exitCode ?? 1);
|
|
113
|
+
: `The portability module refused or failed the archive operation (${receipt.reason ?? 'incomplete'}${receipt.native ? `: ${receipt.native.class}${receipt.native.code && receipt.native.code !== receipt.reason ? `, code ${receipt.native.code}` : ''}` : ''})${receipt.step ? ` during ${receipt.step}` : ''}.${receipt.native && receipt.exitCode === 1 ? ' Run the same command again; it starts over.' : ''}`, receipt.exitCode ?? 1);
|
|
114
114
|
delete receipt.ok;
|
|
115
115
|
return receipt;
|
|
116
116
|
}
|
package/global/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.2.119
|
|
4
|
+
|
|
5
|
+
### Large installation archive inventories
|
|
6
|
+
|
|
7
|
+
- Archive export and restore store large object inventories in bounded internal rows instead of one oversized job-metadata value, addressing the D1 cell limit reached by a company-copy export with 13,879 objects. Small jobs keep their existing metadata representation. Archive parts can group multiple small objects, and restore reuses a shared part while reading its objects. The archive format and version are unchanged.
|
|
8
|
+
- Managed intake accounts for the internal inventory table. Native archive failures name the transfer stage without printing SQL, object keys, row values or filesystem paths.
|
|
9
|
+
|
|
10
|
+
### Publication scan copy pointers
|
|
11
|
+
|
|
12
|
+
- Publication scans locate Gitleaks full matches within their reported lines when directory-fragment columns are relative to a fragment beginning mid-line. Every possible occurrence must satisfy the existing copy-pointer or checksum content proof; unresolved matches and unproved occurrences retain their finding.
|
|
13
|
+
|
|
14
|
+
### Deploy output and timing checks
|
|
15
|
+
|
|
16
|
+
- Declared public deployment settings, including `MAIL_PROVIDER`, remain readable even when carried through secret transport. Credential-named bindings and aliases, private setup fields and credential-shaped values stay masked.
|
|
17
|
+
- Nested deploy hooks defer their dry-run completion and needs summary to the outer command, so the combined summary prints once after local planning succeeds.
|
|
18
|
+
- The real 20-second listing-backoff test still uses the native timer and high-resolution timestamps, observes the configured delay and compares elapsed time at Node's millisecond timer precision to avoid a fractional early-wake failure. Runtime backoff behavior is unchanged.
|
|
19
|
+
|
|
20
|
+
These changes become available after adopting the released package or refreshed source. This preparation establishes no company-copy export result, seed adoption, deployment or production result. The actual diff changes portability inventory storage and archive-part grouping, adds no product migrations, changes no import-kit source and preserves the archive format identity.
|
|
21
|
+
|
|
3
22
|
## 0.2.118
|
|
4
23
|
|
|
5
24
|
### Local diagnostics and Move in guidance
|
|
@@ -1133,6 +1133,8 @@ modules/portability/sqlite-tokens.mjs
|
|
|
1133
1133
|
modules/portability/bookkeeping.mjs
|
|
1134
1134
|
modules/portability/snapshot-objects.mjs
|
|
1135
1135
|
modules/portability/storage.mjs
|
|
1136
|
+
modules/portability/object-inventory.mjs
|
|
1137
|
+
modules/portability/archive-step.mjs
|
|
1136
1138
|
modules/portability/copies.mjs
|
|
1137
1139
|
modules/portability/derived.mjs
|
|
1138
1140
|
modules/portability/dev-data.mjs
|
|
@@ -1268,3 +1270,9 @@ parameters of at most 2 MiB each. It refuses compressed or encrypted entries,
|
|
|
1268
1270
|
unsafe paths, duplicate or unexpected members, missing ledgers, other versions,
|
|
1269
1271
|
container transport, later-journal and legacy-recovery options. Refusals use
|
|
1270
1272
|
constant `managed_data_*` reasons and do not print source values.
|
|
1273
|
+
|
|
1274
|
+
Large object inventories are stored in bounded internal rows so D1 cell and row
|
|
1275
|
+
limits do not prevent backups. Native CLI failures name the transfer stage (for
|
|
1276
|
+
example `export_snapshot`) without printing SQL, values or absolute paths. Keep
|
|
1277
|
+
the stopped source and retry after resolving the named stage; changing source
|
|
1278
|
+
objects or deleting data is not a remedy for an inventory storage failure.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// Stage names are fixed by the implementation. Never send native messages,
|
|
2
|
+
// SQL, object keys, row values or filesystem paths to the CLI.
|
|
3
|
+
const steps=new Set(['export_object_inventory','export_snapshot','export_part']);
|
|
4
|
+
export async function archiveStep(step,operation) {
|
|
5
|
+
try{return await operation();}
|
|
6
|
+
catch(error){
|
|
7
|
+
if(error && typeof error==='object' && !error.portabilityStep)error.portabilityStep=step;
|
|
8
|
+
throw error;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
export const failureStep=error=>steps.has(error?.portabilityStep)?error.portabilityStep:undefined;
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { prepareInventory, metadataJson, inventoryStatements } from './object-inventory.mjs';
|
|
2
|
+
import { archiveStep } from './archive-step.mjs';
|
|
3
|
+
import { statement, state, guard, job, save, partStatements, parts, readPart, cleanCopies, quote, stream, insist } from './storage.mjs';
|
|
2
4
|
import { catalogue, encodedRow } from './catalogue.mjs';
|
|
3
5
|
import { checkReferences } from './references.mjs';
|
|
4
6
|
import { createCopy,readCopy,normalizeCopy } from './copies.mjs';
|
|
@@ -16,7 +18,7 @@ export async function createExport(context,actor='direct') {
|
|
|
16
18
|
for(const [binding,spec] of Object.entries(config.buckets).sort()) {
|
|
17
19
|
let cursor;
|
|
18
20
|
do {
|
|
19
|
-
const page=await buckets[binding].list({prefix:spec.prefix??'',cursor,include:['httpMetadata','customMetadata']});
|
|
21
|
+
const page=await archiveStep('export_object_inventory',()=>buckets[binding].list({prefix:spec.prefix??'',cursor,include:['httpMetadata','customMetadata']}));
|
|
20
22
|
for(const object of page.objects) objects.push({bucket:binding,key:object.key,size:object.size,etag:object.etag,httpMetadata:object.httpMetadata??{},customMetadata:object.customMetadata??{}});
|
|
21
23
|
cursor=page.truncated?page.cursor:undefined;
|
|
22
24
|
}while(cursor);
|
|
@@ -27,9 +29,10 @@ export async function createExport(context,actor='direct') {
|
|
|
27
29
|
const manifest={kind:'manifest',format:FORMAT,version:2,product:config.product,exportedAt:new Date().toISOString(),sourceIdentity:s.source_identity,schema:cat.identity,tables:cat.tables,schemaObjects:cat.schemaObjects,objects,payloads:[],excludedTables:cat.excludedTables,excludedColumns:config.secretColumns,requiredConfiguration:config.requiredConfiguration,partCount:0,epoch:s.epoch,journal:await journal(db,s.source_identity),...(migrations?{migrations}:{}),...(origin?{migrationLedger:origin}:{})};
|
|
28
30
|
const row={id,kind:'export',actor,phase:'building',completed:0,metadata:{manifest,copies,cursor:{table:0,row:0,binary:0,offset:0},partBytes:config.partBytes}};
|
|
29
31
|
const audit=context.onExport?context.onExport({id,actor:actor==='direct'?null:actor,scope:'installation'}):[];
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
...
|
|
32
|
+
prepareInventory(row);
|
|
33
|
+
await archiveStep('export_snapshot',()=>db.batch([guard(db,'EXISTS(SELECT 1 FROM rer_portability_state WHERE id=1 AND applying=0 AND source_identity=? AND epoch=?)',s.source_identity,s.epoch),
|
|
34
|
+
...cat.tables.flatMap(t=>createCopy(db,t,copies[t.name],true)),statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'export',actor,row.phase,metadataJson(row.metadata)),...inventoryStatements(db,row),db.prepare('DELETE FROM rer_portability_guard'),
|
|
35
|
+
...audit.map(([sql,args])=>statement(db,sql,...args))]));
|
|
33
36
|
try {
|
|
34
37
|
await dropDevData(db,cat.tables,copies);
|
|
35
38
|
await dropProviderData(db,cat.tables,copies);
|
|
@@ -38,7 +41,7 @@ export async function createExport(context,actor='direct') {
|
|
|
38
41
|
for(const [name,producer] of Object.entries(config.extraPayloads).sort()) {
|
|
39
42
|
const descriptor=await capturePayload(db,id,manifest.payloads.length,await producer.produce());manifest.payloads.push({name,...descriptor});
|
|
40
43
|
}
|
|
41
|
-
await
|
|
44
|
+
await save(db,row).run();
|
|
42
45
|
await checkReferences(context,copies,objects);
|
|
43
46
|
return id;
|
|
44
47
|
}catch(e){await abortExport(context,id);throw e;}
|
|
@@ -68,24 +71,30 @@ async function nextItems(context,row) {
|
|
|
68
71
|
}
|
|
69
72
|
const entries=[...manifest.objects,...manifest.payloads];
|
|
70
73
|
if(cursor.binary===entries.length)return null;
|
|
71
|
-
const
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
74
|
+
const items=[];let used=200;
|
|
75
|
+
while(cursor.binary<entries.length && items.length<1000 && partBytes-used>512) {
|
|
76
|
+
const entry=entries[cursor.binary],header=utf8(canonicalJson({bucket:entry.bucket??null,key:entry.key??null,name:entry.name??null})).length;
|
|
77
|
+
if(used+200+header>=partBytes){insist(items.length,'binary_part_too_large');break;}
|
|
78
|
+
const length=Math.min(entry.size-cursor.offset,Math.max(1,Math.floor((partBytes-used-200-header)*3/4)));
|
|
79
|
+
let bytes,item;
|
|
80
|
+
if(cursor.binary<manifest.objects.length) {
|
|
81
|
+
const object=await buckets[entry.bucket].get(entry.key,length?{range:{offset:cursor.offset,length}}:undefined);
|
|
82
|
+
insist(object && object.etag===entry.etag,'object_changed:'+entry.bucket+':'+entry.key);
|
|
83
|
+
bytes=new Uint8Array(await object.arrayBuffer());insist(bytes.length===length,'object_changed:'+entry.key);
|
|
84
|
+
item={kind:'object',bucket:entry.bucket,key:entry.key,offset:cursor.offset,data:base64(bytes)};
|
|
85
|
+
}else{
|
|
86
|
+
bytes=await payloadRange(db,row.id,cursor.binary-manifest.objects.length,cursor.offset,length);
|
|
87
|
+
item={kind:'payload',name:entry.name,offset:cursor.offset,data:base64(bytes)};
|
|
88
|
+
}
|
|
89
|
+
const hash=new StreamHash();if(cursor.hash){hash.h=Uint32Array.from(cursor.hash.h);hash.block=Uint8Array.from(cursor.hash.block);hash.size=cursor.hash.size;hash.used=cursor.hash.used;}
|
|
90
|
+
hash.update(bytes);cursor.offset+=length;
|
|
91
|
+
if(cursor.offset===entry.size) {
|
|
92
|
+
const digest=hash.digest();if(entry.sha256)insist(entry.sha256===digest,'payload_changed');entry.sha256=digest;delete entry.etag;
|
|
93
|
+
cursor.binary++;cursor.offset=0;delete cursor.hash;
|
|
94
|
+
}else cursor.hash={h:[...hash.h],block:[...hash.block],size:hash.size,used:hash.used};
|
|
95
|
+
items.push(item);used+=utf8(canonicalJson(item)).length+1;
|
|
81
96
|
}
|
|
82
|
-
|
|
83
|
-
hash.update(bytes);cursor.offset+=length;
|
|
84
|
-
if(cursor.offset===entry.size) {
|
|
85
|
-
const digest=hash.digest();if(entry.sha256)insist(entry.sha256===digest,'payload_changed');entry.sha256=digest;delete entry.etag;
|
|
86
|
-
cursor.binary++;cursor.offset=0;delete cursor.hash;
|
|
87
|
-
}else cursor.hash={h:[...hash.h],block:[...hash.block],size:hash.size,used:hash.used};
|
|
88
|
-
return [item];
|
|
97
|
+
return items;
|
|
89
98
|
}
|
|
90
99
|
export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
|
|
91
100
|
insist(Number.isSafeInteger(maxParts) && maxParts>0 && maxParts<=100,'advance_limit');
|
|
@@ -94,10 +103,10 @@ export async function advanceExport(context,id,{expectedPart,maxParts=1}={}) {
|
|
|
94
103
|
if(row.phase==='complete')return exportStatus(context,id);
|
|
95
104
|
insist(row.phase==='building','export_phase',409);
|
|
96
105
|
if(expectedPart!==undefined && row.completed!==expectedPart)return exportStatus(context,id);
|
|
97
|
-
const before=row.completed,items=await nextItems(context,row);
|
|
106
|
+
const before=row.completed,binary=row.metadata.cursor.binary,items=await archiveStep('export_part',()=>nextItems(context,row));
|
|
98
107
|
if(items) {
|
|
99
108
|
const part=await makePart(before,items),size=await checkPart(part,before,row.metadata.partBytes);row.completed++;row.metadata.bytes=(row.metadata.bytes??0)+size+1;
|
|
100
|
-
try{await context.db.batch([guard(context.db,'(SELECT completed FROM rer_portability_jobs WHERE id=?)=?',id,before),...partStatements(context.db,id,part),save(context.db,row),context.db.prepare('DELETE FROM rer_portability_guard')]);}
|
|
109
|
+
try{await context.db.batch([guard(context.db,'(SELECT completed FROM rer_portability_jobs WHERE id=?)=?',id,before),...partStatements(context.db,id,part),...inventoryStatements(context.db,row,binary,row.metadata.cursor.binary),save(context.db,row),context.db.prepare('DELETE FROM rer_portability_guard')]);}
|
|
101
110
|
catch(e){if((await job(context.db,id)).completed===before)throw e;}
|
|
102
111
|
if(expectedPart!==undefined)expectedPart++;
|
|
103
112
|
}else{
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { metadataJson } from './object-inventory.mjs';
|
|
1
2
|
import { encodedRow, validateRow, bindCell } from './catalogue.mjs';
|
|
2
3
|
import { readCopy, copyRowid } from './copies.mjs';
|
|
3
4
|
import { canonicalJson } from './hash.mjs';
|
|
@@ -44,7 +45,7 @@ const guardSql=`INSERT INTO rer_portability_guard(value) SELECT CASE WHEN (
|
|
|
44
45
|
) THEN 1 ELSE 0 END`;
|
|
45
46
|
const checkpoint=(row,before,metadata,queries)=>[
|
|
46
47
|
{sql:guardSql,params:[row.id,before,row.id]},...queries,
|
|
47
|
-
{sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,
|
|
48
|
+
{sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,metadataJson(metadata),row.id]},
|
|
48
49
|
{sql:'DELETE FROM rer_portability_guard'},
|
|
49
50
|
];
|
|
50
51
|
// Each page's staged updates, cursor and value-free change counts commit together.
|
|
@@ -55,10 +56,10 @@ export async function normalizeStaged(context,row) {
|
|
|
55
56
|
const copy=row.metadata.copies[table.name],count=row.metadata.stagedCounts?.[table.name]??0;
|
|
56
57
|
let offset=row.metadata.safetyProgress?.[table.name]??0;
|
|
57
58
|
while(offset<count) {
|
|
58
|
-
let before=
|
|
59
|
+
let before=metadataJson(row.metadata),queries=[];
|
|
59
60
|
const commit=async()=>{
|
|
60
61
|
await db.batch(checkpoint(row,before,row.metadata,queries).map(q=>statement(db,q.sql,...(q.params??[]))));
|
|
61
|
-
before=
|
|
62
|
+
before=metadataJson(row.metadata);queries=[];
|
|
62
63
|
};
|
|
63
64
|
for(const {row:raw,meta} of await readCopy(db,table,copy,offset,100,true)) {
|
|
64
65
|
const original=encodedRow(table,raw),after=mapImportRow(config,table,original);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import { metadataJson } from './object-inventory.mjs';
|
|
1
2
|
import { RPC_LIMIT, rpcSize, rpcSizeBytes } from '../d1-transport/index.mjs';
|
|
2
|
-
import { canonicalJson } from './hash.mjs';
|
|
3
3
|
import { statement, save, readPart, insist } from './storage.mjs';
|
|
4
4
|
import { checkPart } from './format.mjs';
|
|
5
5
|
|
|
@@ -18,11 +18,11 @@ export async function stagePart(context,row,part,queries,stagedCounts) {
|
|
|
18
18
|
insist(!pending || (pending.index===part.index && pending.sha256===part.sha256),'part_conflict',409);
|
|
19
19
|
let next=pending?.next??0;
|
|
20
20
|
while(next<queries.length) {
|
|
21
|
-
const before=
|
|
21
|
+
const before=metadataJson(row.metadata),chunk=[];
|
|
22
22
|
// Reserve the largest cursor for this part before admitting any data.
|
|
23
23
|
row.metadata.pendingPart={index:part.index,sha256:part.sha256,next:queries.length};
|
|
24
24
|
const guard={sql:guardSql,params:[row.id,row.completed,before,row.id]};
|
|
25
|
-
const checkpoint={sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,
|
|
25
|
+
const checkpoint={sql:'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',params:[row.phase,row.completed,metadataJson(row.metadata),row.id]};
|
|
26
26
|
let size=rpcSize([guard,checkpoint,{sql:'DELETE FROM rer_portability_guard'}]),end=next;
|
|
27
27
|
for(;end<queries.length;end++) {
|
|
28
28
|
const candidate=combine(size,rpcSize([queries[end]]));
|
|
@@ -37,7 +37,7 @@ export async function stagePart(context,row,part,queries,stagedCounts) {
|
|
|
37
37
|
// Authenticate the reassembled stored bytes before acknowledging the part.
|
|
38
38
|
const stored=await readPart(db,row.id,part.index);
|
|
39
39
|
await checkPart(stored,part.index,config.partBytes);insist(stored.sha256===part.sha256,'part_checksum');
|
|
40
|
-
const before=
|
|
40
|
+
const before=metadataJson(row.metadata),completed=row.completed;
|
|
41
41
|
delete row.metadata.pendingPart;row.metadata.stagedCounts=stagedCounts;row.completed++;
|
|
42
42
|
await db.batch([statement(db,guardSql,row.id,completed,before,row.id),
|
|
43
43
|
statement(db,'INSERT INTO rer_portability_parts VALUES(?,?,?)',row.id,part.index,part.sha256),
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { prepareInventory, metadataJson, inventoryStatements } from './object-inventory.mjs';
|
|
1
2
|
import { libraryMigrations } from './library-history.mjs';
|
|
2
3
|
import { all, statement, state, guard, job, save, partQueries, parts, readPart, dropCopyStatements, cleanCopyStatements, quote, stream, insist, run } from './storage.mjs';
|
|
3
4
|
import { stagePart } from './import-staging.mjs';
|
|
@@ -78,11 +79,12 @@ export async function createImport(context,manifest,actor='direct',legacyRecover
|
|
|
78
79
|
}
|
|
79
80
|
const copies=Object.fromEntries(targets.map((t,i)=>[t.name,'rer_portability_stage_'+id.replaceAll('-','')+'_'+i]));
|
|
80
81
|
const row={id,kind:'import',actor,phase:'staging',completed:0,metadata:{manifest:original,...(unverified?{schemaUnverified:{reason:'acknowledged recovery of a version 2 archive without schemaObjects',sourceIdentity:original.sourceIdentity,exportedAt:original.exportedAt,acknowledgedAt:new Date().toISOString()}}:{}),...(skippedTables.length?{restoreManifest:manifest,skippedTables}:{}),copies,targets,triggers:cat.triggers,...(bridge.fromSchema?{bridge}:{}),...(replays?{replays}:{}),...(legacyRecovery?{legacyRecovery}: {}),...(ledgers?{ledgers}:{})}};
|
|
82
|
+
prepareInventory(row);
|
|
81
83
|
try {
|
|
82
84
|
await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1) IS NULL'),
|
|
83
85
|
statement(db,'UPDATE rer_portability_state SET import_job=?,applying=? WHERE id=1',id,unverified?1:0),
|
|
84
86
|
...targets.flatMap(t=>createCopy(db,t,copies[t.name])),
|
|
85
|
-
statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'import',actor,row.phase,
|
|
87
|
+
statement(db,'INSERT INTO rer_portability_jobs(id,kind,actor,phase,metadata) VALUES(?,?,?,?,?)',id,'import',actor,row.phase,metadataJson(row.metadata)),...inventoryStatements(db,row),createImportLease(db,id),db.prepare('DELETE FROM rer_portability_guard')]);
|
|
86
88
|
}catch(e){const running=(await state(db)).import_job;if(running)throw await importRunning(db,running);throw e;}
|
|
87
89
|
return id;
|
|
88
90
|
}
|
|
@@ -136,10 +138,10 @@ async function uploadLeasedPart(context,id,index,part) {
|
|
|
136
138
|
return importStatus(context,id,{includeLease:false});
|
|
137
139
|
}
|
|
138
140
|
const binaryKey=entry=>canonicalJson(entry.bucket===undefined?['payload',entry.name]:['object',entry.bucket,entry.key]);
|
|
139
|
-
function binaryStream(
|
|
141
|
+
function binaryStream(entry,indexes,read) {
|
|
140
142
|
const selected=indexes.get(binaryKey(entry));insist(selected,'binary_inventory');
|
|
141
143
|
return stream((async function*(){
|
|
142
|
-
for(const index of selected)for(const item of (await
|
|
144
|
+
for(const index of selected)for(const item of (await read(index)).items) {
|
|
143
145
|
if(entry.bucket===undefined?item.kind==='payload' && item.name===entry.name:item.kind==='object' && item.bucket===entry.bucket && item.key===entry.key)yield unbase64(item.data);
|
|
144
146
|
}
|
|
145
147
|
})());
|
|
@@ -218,17 +220,24 @@ async function finalizeLeasedImport(context,id,{contentHash,archiveHash,laterJou
|
|
|
218
220
|
}
|
|
219
221
|
row.phase='applying';
|
|
220
222
|
await db.batch([guard(db,'(SELECT import_job FROM rer_portability_state WHERE id=1)=?',id),db.prepare('UPDATE rer_portability_state SET applying=1 WHERE id=1'),save(db,row),db.prepare('DELETE FROM rer_portability_guard')]);
|
|
223
|
+
// Grouped objects often share a part. Keep one immutable part during this
|
|
224
|
+
// leased finalize, rather than rereading its bytes for every small object.
|
|
225
|
+
let cached;
|
|
226
|
+
const readBinaryPart=async index=>{
|
|
227
|
+
if(cached?.index!==index)cached=await readPart(db,id,index);
|
|
228
|
+
return cached;
|
|
229
|
+
};
|
|
221
230
|
try {
|
|
222
231
|
for(const object of manifest.objects) {
|
|
223
232
|
await run(db,'INSERT OR IGNORE INTO rer_portability_owned VALUES(?,?,?)',id,object.bucket,object.key);
|
|
224
|
-
await writeObject(context.buckets[object.bucket],object,binaryStream(
|
|
233
|
+
await writeObject(context.buckets[object.bucket],object,binaryStream(object,binaryIndexes,readBinaryPart));
|
|
225
234
|
const stored=await context.buckets[object.bucket].get(object.key);insist(stored,'object_write_failed');
|
|
226
235
|
// Read back independently, including multipart results, before the commit.
|
|
227
236
|
const hash=new (await import('./hash.mjs')).StreamHash();const reader=stored.body.getReader();
|
|
228
237
|
for(;;){const {value,done}=await reader.read();if(done)break;hash.update(value);}reader.releaseLock();
|
|
229
238
|
insist(hash.digest()===object.sha256,'object_write_checksum');
|
|
230
239
|
}
|
|
231
|
-
for(const payload of manifest.payloads)await config.extraPayloads[payload.name].consume(binaryStream(
|
|
240
|
+
for(const payload of manifest.payloads)await config.extraPayloads[payload.name].consume(binaryStream(payload,binaryIndexes,readBinaryPart),payload);
|
|
232
241
|
const derived=await rebuildDerivedStatements(db,config);
|
|
233
242
|
row.phase='complete';row.metadata.contentHash=digest;
|
|
234
243
|
if(row.metadata.schemaUnverified)row.metadata.schemaUnverified.archiveHash=archiveHash;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { ensureObjectInventory } from './object-inventory.mjs';
|
|
1
2
|
import { ensureImportLeases } from './import-lease.mjs';
|
|
2
3
|
import { policy, catalogue } from './catalogue.mjs';
|
|
3
4
|
import { wireDatabase } from '../d1-transport/index.mjs';
|
|
@@ -14,7 +15,7 @@ export { createPortabilityHandler } from './http.mjs';
|
|
|
14
15
|
export async function createPortability({db,buckets={},policy:input,migrations,onExport,migrationRoot}) {
|
|
15
16
|
insist(onExport===undefined || typeof onExport==='function','invalid_on_export');
|
|
16
17
|
db=wireDatabase(db);
|
|
17
|
-
const config=policy(input);await catalogue(db,config);
|
|
18
|
+
const config=policy(input);await ensureObjectInventory(db);await catalogue(db,config);
|
|
18
19
|
for(const name of Object.keys(config.buckets))insist(buckets[name] && typeof buckets[name].get==='function','missing_bucket:'+name);
|
|
19
20
|
await ensureImportLeases(db);
|
|
20
21
|
const context={db,buckets,config,migrations,onExport};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { canonicalJson, utf8 } from './hash.mjs';
|
|
2
|
+
|
|
3
|
+
const table='rer_portability_object_inventory';
|
|
4
|
+
const chunkSize=50;
|
|
5
|
+
export async function ensureObjectInventory(db) {
|
|
6
|
+
await db.prepare(`CREATE TABLE IF NOT EXISTS ${table}(job TEXT NOT NULL REFERENCES rer_portability_jobs(id) ON DELETE CASCADE, chunk INTEGER NOT NULL, data TEXT NOT NULL, PRIMARY KEY(job,chunk))`).run();
|
|
7
|
+
}
|
|
8
|
+
// Keep large manifests out of D1's per-value and per-row limits. Only the
|
|
9
|
+
// storage representation changes; archive manifests still contain every object.
|
|
10
|
+
export function prepareInventory(row) {
|
|
11
|
+
if(utf8(canonicalJson(row.metadata.manifest.objects)).length>256*1024)
|
|
12
|
+
row.metadata.objectInventory=Math.ceil(row.metadata.manifest.objects.length/chunkSize);
|
|
13
|
+
}
|
|
14
|
+
export function metadataJson(metadata) {
|
|
15
|
+
if(!metadata.objectInventory)return canonicalJson(metadata);
|
|
16
|
+
const compact={...metadata,manifest:{...metadata.manifest,objects:[]}};
|
|
17
|
+
if(metadata.restoreManifest)compact.restoreManifest={...metadata.restoreManifest,objects:[]};
|
|
18
|
+
return canonicalJson(compact);
|
|
19
|
+
}
|
|
20
|
+
export function inventoryStatements(db,row,start=0,end=row.metadata.manifest.objects.length) {
|
|
21
|
+
if(!row.metadata.objectInventory)return [];
|
|
22
|
+
const objects=row.metadata.manifest.objects,statements=[];
|
|
23
|
+
for(let chunk=Math.floor(start/chunkSize);chunk<Math.ceil(end/chunkSize);chunk++)
|
|
24
|
+
statements.push(db.prepare(`INSERT OR REPLACE INTO ${table}(job,chunk,data) VALUES(?,?,?)`).bind(row.id,chunk,canonicalJson(objects.slice(chunk*chunkSize,(chunk+1)*chunkSize))));
|
|
25
|
+
return statements;
|
|
26
|
+
}
|
|
27
|
+
export async function loadInventory(db,row) {
|
|
28
|
+
if(!row.metadata.objectInventory)return row;
|
|
29
|
+
const objects=[];
|
|
30
|
+
// Read bounded pages: a complete inventory can exceed the RPC result limit.
|
|
31
|
+
for(let offset=0;offset<row.metadata.objectInventory;offset+=16) {
|
|
32
|
+
const {results}=await db.prepare(`SELECT data FROM ${table} WHERE job=? ORDER BY chunk LIMIT 16 OFFSET ?`).bind(row.id,offset).all();
|
|
33
|
+
for(const item of results)objects.push(...JSON.parse(item.data));
|
|
34
|
+
}
|
|
35
|
+
row.metadata.manifest.objects=objects;
|
|
36
|
+
if(row.metadata.restoreManifest)row.metadata.restoreManifest.objects=objects;
|
|
37
|
+
return row;
|
|
38
|
+
}
|
|
39
|
+
export function clearInventory(db,row) {
|
|
40
|
+
if(!row.metadata.objectInventory)return [];
|
|
41
|
+
delete row.metadata.objectInventory;
|
|
42
|
+
row.metadata.manifest.objects=[];
|
|
43
|
+
if(row.metadata.restoreManifest)row.metadata.restoreManifest.objects=[];
|
|
44
|
+
return [db.prepare(`DELETE FROM ${table} WHERE job=?`).bind(row.id)];
|
|
45
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { metadataJson, loadInventory, clearInventory } from './object-inventory.mjs';
|
|
1
2
|
import { canonicalJson } from './hash.mjs';
|
|
2
3
|
// Installed helpers and copied runtimes can coexist in one Worker. Share only
|
|
3
4
|
// constructor-issued identities, so arbitrary objects with code/status fields
|
|
@@ -26,10 +27,10 @@ export async function state(db) {
|
|
|
26
27
|
export async function job(db,id,kind) {
|
|
27
28
|
const row=await statement(db,'SELECT * FROM rer_portability_jobs WHERE id=?',id).first();
|
|
28
29
|
insist(row && (!kind || row.kind===kind),'job_not_found',404);
|
|
29
|
-
return {...row, metadata:JSON.parse(row.metadata)};
|
|
30
|
+
return loadInventory(db,{...row, metadata:JSON.parse(row.metadata)});
|
|
30
31
|
}
|
|
31
32
|
export function save(db,row) {
|
|
32
|
-
return statement(db,'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',row.phase,row.completed,
|
|
33
|
+
return statement(db,'UPDATE rer_portability_jobs SET phase=?,completed=?,metadata=? WHERE id=?',row.phase,row.completed,metadataJson(row.metadata),row.id);
|
|
33
34
|
}
|
|
34
35
|
export function partStatements(db,id,part) {
|
|
35
36
|
return partQueries(id,part).map(({sql,params})=>statement(db,sql,...params));
|
|
@@ -64,7 +65,7 @@ export function dropCopyStatements(db,row) {
|
|
|
64
65
|
return transferTables(row).map(name=>db.prepare(`DROP TABLE IF EXISTS ${quote(name)}`));
|
|
65
66
|
}
|
|
66
67
|
export function cleanCopyStatements(db,row) {
|
|
67
|
-
return [...dropCopyStatements(db,row),statement(db,'DELETE FROM rer_portability_parts WHERE job=?',row.id),
|
|
68
|
+
return [...dropCopyStatements(db,row),...clearInventory(db,row),statement(db,'DELETE FROM rer_portability_parts WHERE job=?',row.id),
|
|
68
69
|
statement(db,'DELETE FROM rer_portability_segments WHERE job=?',row.id)];
|
|
69
70
|
}
|
|
70
71
|
export async function cleanCopies(db,row) {
|
package/package.json
CHANGED
|
@@ -148,7 +148,11 @@ entry that does not prove, a 40-hex value, a key that names no tree file, or a h
|
|
|
148
148
|
value with no file-name key keeps the native finding. Historical matches use only
|
|
149
149
|
that commit's blobs and working-tree preimages reject linked paths. There is no
|
|
150
150
|
path, filename, hash-value or repository exemption. `hit_location.py` resolves
|
|
151
|
-
the native match's exact bytes for both proofs.
|
|
151
|
+
the native match's exact bytes for both proofs. Gitleaks directory fragments can
|
|
152
|
+
start mid-line without reporting their starting column. The resolver therefore
|
|
153
|
+
locates full matches within the reported LF-delimited lines. An unresolved match
|
|
154
|
+
retains its finding; repeated matches must all satisfy the structural and digest
|
|
155
|
+
proofs above, even if a reported column appears to identify just one of them.
|
|
152
156
|
|
|
153
157
|
`trust_bundle.py` classifies an email match as public CA certificate metadata
|
|
154
158
|
only when all of these hold: the file's path is an artifact listed by a committed
|
|
@@ -35,18 +35,22 @@ class ChecksumMaps:
|
|
|
35
35
|
def matches(self, data, hit):
|
|
36
36
|
if hit.get('RuleID') not in ('generic-api-key', 'hex-api-key'):
|
|
37
37
|
return False
|
|
38
|
+
spans = match_spans(data, hit)
|
|
39
|
+
if not spans:
|
|
40
|
+
return False
|
|
38
41
|
entries = list(ENTRY.finditer(data))
|
|
39
|
-
for left, right in
|
|
42
|
+
for left, right in spans:
|
|
40
43
|
keyed = [i for i, e in enumerate(entries)
|
|
41
44
|
if left <= e.start('digest') and e.end('digest') <= right
|
|
42
45
|
and e.group('digest').decode() == hit.get('Secret')]
|
|
43
46
|
if len(keyed) != 1:
|
|
44
|
-
|
|
47
|
+
return False
|
|
45
48
|
first = last = keyed[0]
|
|
46
49
|
while first > 0 and SEPARATOR.fullmatch(data, entries[first - 1].end(), entries[first].start()):
|
|
47
50
|
first -= 1
|
|
48
51
|
while last + 1 < len(entries) and SEPARATOR.fullmatch(data, entries[last].end(), entries[last + 1].start()):
|
|
49
52
|
last += 1
|
|
50
|
-
|
|
51
|
-
e.group('digest').decode()) for e in entries[first:last + 1])
|
|
52
|
-
|
|
53
|
+
if not all(self.names_file((e.group('quoted') or e.group('bare')).decode('utf-8', 'replace'),
|
|
54
|
+
e.group('digest').decode()) for e in entries[first:last + 1]):
|
|
55
|
+
return False
|
|
56
|
+
return True
|
|
@@ -1,18 +1,29 @@
|
|
|
1
|
-
"""Resolve a native Gitleaks hit to
|
|
1
|
+
"""Resolve a native Gitleaks hit to its possible full-match byte spans."""
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
def match_spans(data, hit):
|
|
5
|
-
lines = data.splitlines(keepends=True)
|
|
6
5
|
try:
|
|
6
|
+
# Gitleaks counts LF bytes, not Python's broader set of line breaks.
|
|
7
|
+
lines = data.split(b'\n')
|
|
7
8
|
first, last = hit['StartLine'] - 1, hit['EndLine'] - 1
|
|
8
9
|
if not (0 <= first <= last < len(lines)):
|
|
9
10
|
return []
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
11
|
+
match = hit['Match'].encode('utf-8')
|
|
12
|
+
if not match:
|
|
13
|
+
return []
|
|
14
|
+
left = sum(len(line) + 1 for line in lines[:first])
|
|
15
|
+
right = left + sum(len(line) + 1 for line in lines[first:last + 1])
|
|
16
|
+
# Directory fragments can start mid-line. Their columns are relative
|
|
17
|
+
# to that fragment, whose starting column is absent from the report.
|
|
18
|
+
# Later fragment lines also count the preceding LF in their columns.
|
|
19
|
+
# Bind the full match within the reported lines instead. Even an
|
|
20
|
+
# apparently exact column cannot disambiguate repeated occurrences.
|
|
21
|
+
# Callers must prove every candidate, never just the first one.
|
|
22
|
+
spans = []
|
|
23
|
+
start = data.find(match, left, right)
|
|
24
|
+
while start >= 0:
|
|
25
|
+
spans.append((start, start + len(match)))
|
|
26
|
+
start = data.find(match, start + 1, right)
|
|
27
|
+
return spans
|
|
17
28
|
except (KeyError, TypeError, IndexError, UnicodeError):
|
|
18
29
|
return []
|
|
@@ -80,19 +80,20 @@ class Document:
|
|
|
80
80
|
secret = hit.get('Secret', '').encode('utf-8')
|
|
81
81
|
if not secret:
|
|
82
82
|
return False
|
|
83
|
-
|
|
84
|
-
|
|
83
|
+
spans = match_spans(self.data, hit)
|
|
84
|
+
if not spans:
|
|
85
|
+
return False
|
|
86
|
+
allowed = [self.spans[pointer] for pointer in pointers if pointer in self.spans]
|
|
87
|
+
for left, right in spans:
|
|
85
88
|
start = self.data.find(secret, left, right)
|
|
86
89
|
if start < 0 or self.data.find(secret, start + 1, right) >= 0:
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
located.add((a, b))
|
|
98
|
-
return len(located) == 1
|
|
90
|
+
return False
|
|
91
|
+
# Location alone is not proof. Every possible native occurrence
|
|
92
|
+
# must bind its captured bytes (even a truncated capture) to an
|
|
93
|
+
# exact unescaped JSON value at a proved pointer. An identical
|
|
94
|
+
# credential elsewhere on the line therefore retains the finding.
|
|
95
|
+
if not any(a + 1 <= start and start + len(secret) <= b - 1
|
|
96
|
+
and self.data[a + 1:b - 1] == value.encode('ascii')
|
|
97
|
+
for a, b, value in allowed):
|
|
98
|
+
return False
|
|
99
|
+
return True
|