@tiangong-lca/cli 0.0.23 → 0.0.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -8
- package/dist/src/cli.js +213 -12
- package/dist/src/cli.js.map +1 -1
- package/dist/src/lib/dataset-maintenance-alias-request.js +99 -0
- package/dist/src/lib/dataset-maintenance-alias-request.js.map +1 -0
- package/dist/src/lib/dataset-maintenance-apply.js +247 -337
- package/dist/src/lib/dataset-maintenance-apply.js.map +1 -1
- package/dist/src/lib/dataset-maintenance-contract.js +118 -16
- package/dist/src/lib/dataset-maintenance-contract.js.map +1 -1
- package/dist/src/lib/dataset-maintenance-derivatives.js +265 -0
- package/dist/src/lib/dataset-maintenance-derivatives.js.map +1 -0
- package/dist/src/lib/dataset-maintenance-plan.js +38 -6
- package/dist/src/lib/dataset-maintenance-plan.js.map +1 -1
- package/dist/src/lib/dataset-maintenance-protected-contract.js +918 -0
- package/dist/src/lib/dataset-maintenance-protected-contract.js.map +1 -0
- package/dist/src/lib/dataset-maintenance-protected-run.js +771 -0
- package/dist/src/lib/dataset-maintenance-protected-run.js.map +1 -0
- package/dist/src/lib/dataset-maintenance-protected-verify.js +435 -0
- package/dist/src/lib/dataset-maintenance-protected-verify.js.map +1 -0
- package/dist/src/lib/dataset-maintenance-remote.js +140 -1
- package/dist/src/lib/dataset-maintenance-remote.js.map +1 -1
- package/dist/src/lib/dataset-maintenance-verify.js +309 -3
- package/dist/src/lib/dataset-maintenance-verify.js.map +1 -1
- package/package.json +1 -1
|
@@ -3,10 +3,12 @@ import path from 'node:path';
|
|
|
3
3
|
import { writeJsonArtifact } from './artifacts.js';
|
|
4
4
|
import { CliError } from './errors.js';
|
|
5
5
|
import { withStateFileLock } from './state-lock.js';
|
|
6
|
-
import {
|
|
6
|
+
import { buildAliasBatchRequest, buildAliasPlanRequest, loadMaintenanceDesiredPayload, orderedAliasBatches, } from './dataset-maintenance-alias-request.js';
|
|
7
|
+
import { MAINTENANCE_SCAN_TABLES, appendStableJsonLine, isJsonObject, maintenanceRowKey, parseMaintenancePlan, readJsonFile, readJsonLinesIfPresent, sha256Json, snapshotRemoteRow, writeImmutableJson, } from './dataset-maintenance-contract.js';
|
|
8
|
+
import { buildDerivativePlanRequest, derivativePlanAction, parseDerivativeSnapshotResponse, parseDerivativeSubmitResponse, } from './dataset-maintenance-derivatives.js';
|
|
7
9
|
import { maintenanceProjectedReferenceFingerprint } from './dataset-maintenance-plan.js';
|
|
8
10
|
import { isSnapshotCompletenessCompatible } from './dataset-maintenance-pagination.js';
|
|
9
|
-
import { applyMaintenanceAliasPlan, deleteMaintenanceRow, fetchMaintenanceAccountRows, fetchMaintenanceExactRows, resolveMaintenanceRemoteContext, saveDraftMaintenanceRow, } from './dataset-maintenance-remote.js';
|
|
11
|
+
import { applyMaintenanceAliasPlan, applyMaintenanceDerivativeRebuild, deleteMaintenanceRow, fetchMaintenanceAccountRows, fetchMaintenanceDerivativeSnapshot, fetchMaintenanceExactRows, resolveMaintenanceRemoteContext, saveDraftMaintenanceRow, } from './dataset-maintenance-remote.js';
|
|
10
12
|
const POSITIVE_INTEGER_TEXT = /^[1-9]\d*$/u;
|
|
11
13
|
function clock(options) {
|
|
12
14
|
return (options.now ?? new Date()).toISOString();
|
|
@@ -15,21 +17,7 @@ function errorMessage(error) {
|
|
|
15
17
|
return error instanceof Error ? error.message : String(error);
|
|
16
18
|
}
|
|
17
19
|
function loadDesiredPayload(planDir, action) {
|
|
18
|
-
|
|
19
|
-
throw new CliError(`save_draft action lacks desired payload: ${action.action_id}`, {
|
|
20
|
-
code: 'DATASET_MAINTENANCE_PLAN_INVALID',
|
|
21
|
-
exitCode: 2,
|
|
22
|
-
});
|
|
23
|
-
}
|
|
24
|
-
const payloadPath = resolveMaintenancePlanArtifactPath(planDir, action.desired_payload.path, 'Maintenance desired payload path');
|
|
25
|
-
const payload = readJsonFile(payloadPath, 'Maintenance desired payload');
|
|
26
|
-
if (!isJsonObject(payload) || sha256Json(payload) !== action.desired_payload.sha256) {
|
|
27
|
-
throw new CliError(`Desired payload hash mismatch for action ${action.action_id}.`, {
|
|
28
|
-
code: 'DATASET_MAINTENANCE_DESIRED_PAYLOAD_HASH_MISMATCH',
|
|
29
|
-
exitCode: 1,
|
|
30
|
-
});
|
|
31
|
-
}
|
|
32
|
-
return payload;
|
|
20
|
+
return loadMaintenanceDesiredPayload(planDir, action);
|
|
33
21
|
}
|
|
34
22
|
function parseProgress(plan, progressPath) {
|
|
35
23
|
const rawEntries = readJsonLinesIfPresent(progressPath);
|
|
@@ -101,6 +89,98 @@ function parseProgress(plan, progressPath) {
|
|
|
101
89
|
}
|
|
102
90
|
return { entries, successes, latestFailures };
|
|
103
91
|
}
|
|
92
|
+
function derivativeProofIdentity(proof) {
|
|
93
|
+
return sha256Json({
|
|
94
|
+
schema_version: proof.schema_version,
|
|
95
|
+
plan_sha256: proof.plan_sha256,
|
|
96
|
+
operation_id: proof.operation_id,
|
|
97
|
+
target_visibility: proof.target_visibility,
|
|
98
|
+
plan_request_sha256: proof.plan_request_sha256,
|
|
99
|
+
action_count: proof.action_count,
|
|
100
|
+
accepted_count: proof.accepted_count,
|
|
101
|
+
summary_audit_id: proof.summary_audit_id,
|
|
102
|
+
request_id: proof.request_id,
|
|
103
|
+
action_request_sha256: proof.action_request_sha256,
|
|
104
|
+
database_audit_id: proof.database_audit_id,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
function parseDerivativeSubmitProgress(plan, progressPath) {
|
|
108
|
+
const action = derivativePlanAction(plan);
|
|
109
|
+
const entries = readJsonLinesIfPresent(progressPath).map((value) => {
|
|
110
|
+
const rawProof = isJsonObject(value) && isJsonObject(value.proof) ? value.proof : null;
|
|
111
|
+
let proof;
|
|
112
|
+
try {
|
|
113
|
+
proof = parseDerivativeSubmitResponse(rawProof
|
|
114
|
+
? {
|
|
115
|
+
ok: true,
|
|
116
|
+
command: 'cmd_dataset_derivative_rebuild_plan_guarded',
|
|
117
|
+
...rawProof,
|
|
118
|
+
}
|
|
119
|
+
: null, plan);
|
|
120
|
+
}
|
|
121
|
+
catch (error) {
|
|
122
|
+
throw new CliError('Derivative submit progress contains an invalid RPC proof.', {
|
|
123
|
+
code: 'DATASET_MAINTENANCE_DERIVATIVE_PROGRESS_INVALID',
|
|
124
|
+
exitCode: 1,
|
|
125
|
+
details: errorMessage(error),
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
if (!isJsonObject(value) ||
|
|
129
|
+
value.schema_version !== 1 ||
|
|
130
|
+
value.plan_sha256 !== plan.plan_sha256 ||
|
|
131
|
+
value.operation_id !== plan.operation_id ||
|
|
132
|
+
value.action_id !== action.action_id ||
|
|
133
|
+
value.target_mode !== 'owner_draft' ||
|
|
134
|
+
!isJsonObject(value.actor) ||
|
|
135
|
+
value.actor.user_id !== plan.account.user_id ||
|
|
136
|
+
value.actor.email !== plan.account.email ||
|
|
137
|
+
typeof value.started_at_utc !== 'string' ||
|
|
138
|
+
typeof value.ended_at_utc !== 'string' ||
|
|
139
|
+
value.result !== 'accepted') {
|
|
140
|
+
throw new CliError('Derivative submit progress contains an invalid or foreign entry.', {
|
|
141
|
+
code: 'DATASET_MAINTENANCE_DERIVATIVE_PROGRESS_INVALID',
|
|
142
|
+
exitCode: 1,
|
|
143
|
+
details: value,
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
return { ...value, proof };
|
|
147
|
+
});
|
|
148
|
+
const identities = new Set(entries.map((entry) => derivativeProofIdentity(entry.proof)));
|
|
149
|
+
if (identities.size > 1) {
|
|
150
|
+
throw new CliError('Derivative submit replays do not identify one durable request.', {
|
|
151
|
+
code: 'DATASET_MAINTENANCE_DERIVATIVE_PROGRESS_INVALID',
|
|
152
|
+
exitCode: 1,
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
return { entries, latest: entries.at(-1) ?? null };
|
|
156
|
+
}
|
|
157
|
+
function validateDerivativeAdmissionAttempt(options) {
|
|
158
|
+
if (!existsSync(options.path))
|
|
159
|
+
return null;
|
|
160
|
+
const value = readJsonFile(options.path, 'Derivative admission attempt');
|
|
161
|
+
const action = derivativePlanAction(options.plan);
|
|
162
|
+
if (!isJsonObject(value) ||
|
|
163
|
+
value.schema_version !== 1 ||
|
|
164
|
+
value.plan_sha256 !== options.plan.plan_sha256 ||
|
|
165
|
+
value.operation_id !== options.plan.operation_id ||
|
|
166
|
+
value.action_id !== action.action_id ||
|
|
167
|
+
value.table !== 'processes' ||
|
|
168
|
+
value.id !== action.id ||
|
|
169
|
+
value.version !== action.version ||
|
|
170
|
+
value.expected_snapshot_sha256 !== action.derivative_before?.snapshot_sha256 ||
|
|
171
|
+
!isJsonObject(value.actor) ||
|
|
172
|
+
value.actor.user_id !== options.context.account.user_id ||
|
|
173
|
+
value.actor.email !== options.context.account.email ||
|
|
174
|
+
typeof value.prepared_at_utc !== 'string' ||
|
|
175
|
+
!Number.isFinite(Date.parse(value.prepared_at_utc))) {
|
|
176
|
+
throw new CliError('Derivative admission attempt is invalid or belongs to another plan.', {
|
|
177
|
+
code: 'DATASET_MAINTENANCE_DERIVATIVE_ATTEMPT_INVALID',
|
|
178
|
+
exitCode: 1,
|
|
179
|
+
details: value,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
return value;
|
|
183
|
+
}
|
|
104
184
|
function parseAliasBatchProgress(plan, progressPath) {
|
|
105
185
|
const batches = new Map(plan.alias_batches.map((batch) => [batch.batch_id, batch]));
|
|
106
186
|
const entries = readJsonLinesIfPresent(progressPath).map((value) => {
|
|
@@ -433,81 +513,6 @@ async function assertAliasSupportSnapshots(options) {
|
|
|
433
513
|
}
|
|
434
514
|
}
|
|
435
515
|
}
|
|
436
|
-
function buildAliasBatchRequest(options) {
|
|
437
|
-
if (options.plan.target_mode !== 'owner_draft') {
|
|
438
|
-
throw new CliError('Alias batch request requires target_mode=owner_draft.', {
|
|
439
|
-
code: 'DATASET_MAINTENANCE_TARGET_MODE_INVALID',
|
|
440
|
-
exitCode: 2,
|
|
441
|
-
});
|
|
442
|
-
}
|
|
443
|
-
const targetSnapshot = (snapshot) => {
|
|
444
|
-
return {
|
|
445
|
-
id: snapshot.id,
|
|
446
|
-
version: snapshot.version,
|
|
447
|
-
expected_modified_at: snapshot.modified_at,
|
|
448
|
-
expected_json_ordered: snapshot.json_ordered,
|
|
449
|
-
};
|
|
450
|
-
};
|
|
451
|
-
const actions = options.batch.action_ids.map((actionId) => {
|
|
452
|
-
const action = options.plan.actions.find((entry) => entry.action_id === actionId);
|
|
453
|
-
return {
|
|
454
|
-
action_id: action.action_id,
|
|
455
|
-
action: 'update_json_ordered',
|
|
456
|
-
table: action.table,
|
|
457
|
-
id: action.id,
|
|
458
|
-
version: action.version,
|
|
459
|
-
expected_state_code: 0,
|
|
460
|
-
expected_modified_at: action.before.modified_at,
|
|
461
|
-
expected_json_ordered: action.before.json_ordered,
|
|
462
|
-
desired_json_ordered: loadDesiredPayload(options.planDir, action),
|
|
463
|
-
mutation: action.alias_mutation,
|
|
464
|
-
};
|
|
465
|
-
});
|
|
466
|
-
return {
|
|
467
|
-
schema_version: 'dataset-alias-batch.v1',
|
|
468
|
-
target_visibility: 'owner_draft',
|
|
469
|
-
plan_sha256: options.plan.plan_sha256,
|
|
470
|
-
operation_id: options.plan.operation_id,
|
|
471
|
-
batch_id: options.batch.batch_id,
|
|
472
|
-
dimension: options.batch.dimension,
|
|
473
|
-
factor: options.batch.factor,
|
|
474
|
-
target: {
|
|
475
|
-
flowproperty: targetSnapshot(options.batch.target_snapshots.flowproperty),
|
|
476
|
-
unitgroup: targetSnapshot(options.batch.target_snapshots.unitgroup),
|
|
477
|
-
source_unitgroup: targetSnapshot(options.batch.target_snapshots.source_unitgroup),
|
|
478
|
-
},
|
|
479
|
-
actions,
|
|
480
|
-
};
|
|
481
|
-
}
|
|
482
|
-
function orderedAliasBatches(plan) {
|
|
483
|
-
const time = plan.alias_batches.find((batch) => batch.dimension === 'time');
|
|
484
|
-
const lengthTime = plan.alias_batches.find((batch) => batch.dimension === 'length_time');
|
|
485
|
-
return [time, lengthTime].filter((batch) => batch !== undefined);
|
|
486
|
-
}
|
|
487
|
-
function buildAliasPlanRequest(options) {
|
|
488
|
-
if (options.plan.target_mode !== 'owner_draft') {
|
|
489
|
-
throw new CliError('Alias plan request requires target_mode=owner_draft.', {
|
|
490
|
-
code: 'DATASET_MAINTENANCE_TARGET_MODE_INVALID',
|
|
491
|
-
exitCode: 2,
|
|
492
|
-
});
|
|
493
|
-
}
|
|
494
|
-
const batches = orderedAliasBatches(options.plan);
|
|
495
|
-
if (batches.length !== 2 ||
|
|
496
|
-
batches[0]?.dimension !== 'time' ||
|
|
497
|
-
batches[1]?.dimension !== 'length_time') {
|
|
498
|
-
throw new CliError('Alias plan request requires time followed by length_time exactly once.', {
|
|
499
|
-
code: 'DATASET_MAINTENANCE_ALIAS_PLAN_INVALID',
|
|
500
|
-
exitCode: 2,
|
|
501
|
-
});
|
|
502
|
-
}
|
|
503
|
-
return {
|
|
504
|
-
schema_version: 'dataset-alias-plan.v1',
|
|
505
|
-
plan_sha256: options.plan.plan_sha256,
|
|
506
|
-
operation_id: options.plan.operation_id,
|
|
507
|
-
target_visibility: 'owner_draft',
|
|
508
|
-
batches: batches.map((batch) => buildAliasBatchRequest({ plan: options.plan, batch, planDir: options.planDir })),
|
|
509
|
-
};
|
|
510
|
-
}
|
|
511
516
|
function validateAliasRpcResult(value, batch, plan) {
|
|
512
517
|
const audit = Array.isArray(value.audit) ? value.audit : [];
|
|
513
518
|
const proofs = audit.filter(isJsonObject).map((entry) => ({
|
|
@@ -658,106 +663,6 @@ async function executeAliasPlan(options) {
|
|
|
658
663
|
function aliasExchangeProgressKey(value) {
|
|
659
664
|
return `${value.batch_id}\u0000${value.action_id}\u0000${value.exchange_index}\u0000${value.data_set_internal_id}`;
|
|
660
665
|
}
|
|
661
|
-
function aliasBatchDerivedLogsComplete(options) {
|
|
662
|
-
const planBatchProof = options.planSuccess.batches.find((proof) => proof.batch_id === options.batch.batch_id);
|
|
663
|
-
if (!planBatchProof ||
|
|
664
|
-
planBatchProof.dimension !== options.batch.dimension ||
|
|
665
|
-
planBatchProof.batch_request_sha256 !== options.batchSuccess.batch_request_sha256 ||
|
|
666
|
-
planBatchProof.summary_audit_id !== options.batchSuccess.summary_audit_id ||
|
|
667
|
-
options.batchSuccess.plan_request_sha256 !== options.planSuccess.plan_request_sha256 ||
|
|
668
|
-
options.batchSuccess.plan_summary_audit_id !== options.planSuccess.summary_audit_id ||
|
|
669
|
-
!options.batch.action_ids.every((actionId) => options.progress.successes.get(actionId)?.batch_request_sha256 ===
|
|
670
|
-
options.batchSuccess.batch_request_sha256 &&
|
|
671
|
-
options.progress.successes.get(actionId)?.summary_audit_id ===
|
|
672
|
-
options.batchSuccess.summary_audit_id &&
|
|
673
|
-
options.progress.successes.get(actionId)?.plan_request_sha256 ===
|
|
674
|
-
options.planSuccess.plan_request_sha256 &&
|
|
675
|
-
options.progress.successes.get(actionId)?.plan_summary_audit_id ===
|
|
676
|
-
options.planSuccess.summary_audit_id)) {
|
|
677
|
-
return false;
|
|
678
|
-
}
|
|
679
|
-
const expected = new Map(options.batch.exchange_rewrites.map((rewrite) => [
|
|
680
|
-
aliasExchangeProgressKey({ batch_id: options.batch.batch_id, ...rewrite }),
|
|
681
|
-
rewrite,
|
|
682
|
-
]));
|
|
683
|
-
const exchangeKeys = new Set();
|
|
684
|
-
for (const value of readJsonLinesIfPresent(options.exchangeProgressPath)) {
|
|
685
|
-
if (!isJsonObject(value) || value.batch_id !== options.batch.batch_id)
|
|
686
|
-
continue;
|
|
687
|
-
const key = typeof value.action_id === 'string' &&
|
|
688
|
-
typeof value.exchange_index === 'number' &&
|
|
689
|
-
typeof value.data_set_internal_id === 'string'
|
|
690
|
-
? aliasExchangeProgressKey({
|
|
691
|
-
batch_id: options.batch.batch_id,
|
|
692
|
-
action_id: value.action_id,
|
|
693
|
-
exchange_index: value.exchange_index,
|
|
694
|
-
data_set_internal_id: value.data_set_internal_id,
|
|
695
|
-
})
|
|
696
|
-
: '';
|
|
697
|
-
const rewrite = expected.get(key);
|
|
698
|
-
const rowProof = rewrite ? options.progress.successes.get(rewrite.action_id) : null;
|
|
699
|
-
if (!rewrite ||
|
|
700
|
-
!rowProof ||
|
|
701
|
-
value.schema_version !== 1 ||
|
|
702
|
-
value.plan_sha256 !== options.plan.plan_sha256 ||
|
|
703
|
-
value.operation_id !== options.plan.operation_id ||
|
|
704
|
-
value.target_mode !== 'owner_draft' ||
|
|
705
|
-
value.batch_request_sha256 !== options.batchSuccess.batch_request_sha256 ||
|
|
706
|
-
value.batch_request_sha256 !== rowProof.batch_request_sha256 ||
|
|
707
|
-
value.summary_audit_id !== options.batchSuccess.summary_audit_id ||
|
|
708
|
-
value.summary_audit_id !== rowProof.summary_audit_id ||
|
|
709
|
-
value.plan_request_sha256 !== options.planSuccess.plan_request_sha256 ||
|
|
710
|
-
value.plan_request_sha256 !== rowProof.plan_request_sha256 ||
|
|
711
|
-
value.plan_summary_audit_id !== options.planSuccess.summary_audit_id ||
|
|
712
|
-
value.plan_summary_audit_id !== rowProof.plan_summary_audit_id ||
|
|
713
|
-
value.factor !== options.batch.factor ||
|
|
714
|
-
value.result !== 'success' ||
|
|
715
|
-
!isJsonObject(value.actor) ||
|
|
716
|
-
value.actor.user_id !== options.plan.account.user_id ||
|
|
717
|
-
value.actor.email !== options.plan.account.email ||
|
|
718
|
-
typeof value.logged_at_utc !== 'string' ||
|
|
719
|
-
typeof value.database_audit_id !== 'string' ||
|
|
720
|
-
!POSITIVE_INTEGER_TEXT.test(value.database_audit_id) ||
|
|
721
|
-
value.database_audit_id !== rowProof.database_audit_id ||
|
|
722
|
-
typeof value.summary_audit_id !== 'string' ||
|
|
723
|
-
!POSITIVE_INTEGER_TEXT.test(value.summary_audit_id) ||
|
|
724
|
-
sha256Json({
|
|
725
|
-
action_id: value.action_id,
|
|
726
|
-
process_id: value.process_id,
|
|
727
|
-
process_version: value.process_version,
|
|
728
|
-
exchange_index: value.exchange_index,
|
|
729
|
-
data_set_internal_id: value.data_set_internal_id,
|
|
730
|
-
flow_id: value.flow_id,
|
|
731
|
-
flow_version: value.flow_version,
|
|
732
|
-
direction: value.direction,
|
|
733
|
-
before_exchange_sha256: value.before_exchange_sha256,
|
|
734
|
-
before_mean_amount: value.before_mean_amount,
|
|
735
|
-
before_resulting_amount: value.before_resulting_amount,
|
|
736
|
-
after_mean_amount: value.after_mean_amount,
|
|
737
|
-
after_resulting_amount: value.after_resulting_amount,
|
|
738
|
-
after_exchange_sha256: value.after_exchange_sha256,
|
|
739
|
-
}) !== sha256Json(rewrite) ||
|
|
740
|
-
exchangeKeys.has(key)) {
|
|
741
|
-
return false;
|
|
742
|
-
}
|
|
743
|
-
exchangeKeys.add(key);
|
|
744
|
-
}
|
|
745
|
-
return options.batch.exchange_rewrites.every((rewrite) => exchangeKeys.has(aliasExchangeProgressKey({ batch_id: options.batch.batch_id, ...rewrite })));
|
|
746
|
-
}
|
|
747
|
-
function aliasPlanDerivedLogsComplete(options) {
|
|
748
|
-
return orderedAliasBatches(options.plan).every((batch) => {
|
|
749
|
-
const batchSuccess = options.batchProgress.successes.get(batch.batch_id);
|
|
750
|
-
return Boolean(batchSuccess &&
|
|
751
|
-
aliasBatchDerivedLogsComplete({
|
|
752
|
-
plan: options.plan,
|
|
753
|
-
batch,
|
|
754
|
-
planSuccess: options.planSuccess,
|
|
755
|
-
batchSuccess,
|
|
756
|
-
progress: options.progress,
|
|
757
|
-
exchangeProgressPath: options.exchangeProgressPath,
|
|
758
|
-
}));
|
|
759
|
-
});
|
|
760
|
-
}
|
|
761
666
|
function appendAliasSuccessLogs(options) {
|
|
762
667
|
const batchRpc = options.execution.rpc.batches.get(options.batch.dimension);
|
|
763
668
|
for (const actionId of options.batch.action_ids) {
|
|
@@ -1012,33 +917,13 @@ function appendAliasProofProgress(options) {
|
|
|
1012
917
|
options.planProgress.latestFailure = null;
|
|
1013
918
|
return entry;
|
|
1014
919
|
}
|
|
1015
|
-
function appendAliasPlanFailure(options) {
|
|
1016
|
-
const entry = {
|
|
1017
|
-
schema_version: 1,
|
|
1018
|
-
plan_sha256: options.plan.plan_sha256,
|
|
1019
|
-
operation_id: options.plan.operation_id,
|
|
1020
|
-
target_mode: 'owner_draft',
|
|
1021
|
-
actor: { user_id: options.context.account.user_id, email: options.context.account.email },
|
|
1022
|
-
started_at_utc: options.startedAt,
|
|
1023
|
-
ended_at_utc: options.endedAt,
|
|
1024
|
-
plan_request_sha256: null,
|
|
1025
|
-
idempotent_replay: null,
|
|
1026
|
-
batch_count: 2,
|
|
1027
|
-
row_count: 52,
|
|
1028
|
-
exchange_count: 59,
|
|
1029
|
-
summary_audit_id: null,
|
|
1030
|
-
batches: [],
|
|
1031
|
-
result: 'failed',
|
|
1032
|
-
error: errorMessage(options.error),
|
|
1033
|
-
};
|
|
1034
|
-
appendStableJsonLine(options.progressPath, entry);
|
|
1035
|
-
options.planProgress.entries.push(entry);
|
|
1036
|
-
if (!options.planProgress.success) {
|
|
1037
|
-
options.planProgress.latestFailure = entry;
|
|
1038
|
-
}
|
|
1039
|
-
return entry;
|
|
1040
|
-
}
|
|
1041
920
|
async function executeAction(options) {
|
|
921
|
+
if (options.action.action === 'rebuild_derivatives') {
|
|
922
|
+
throw new CliError('Derivative rebuild actions may only execute through the guarded whole-plan RPC.', {
|
|
923
|
+
code: 'DATASET_MAINTENANCE_DERIVATIVE_SEQUENTIAL_WRITE_FORBIDDEN',
|
|
924
|
+
exitCode: 1,
|
|
925
|
+
});
|
|
926
|
+
}
|
|
1042
927
|
if (!options.action.before) {
|
|
1043
928
|
throw new CliError(`Action lacks a before snapshot: ${options.action.action_id}`, {
|
|
1044
929
|
code: 'DATASET_MAINTENANCE_PLAN_INVALID',
|
|
@@ -1114,6 +999,12 @@ async function executeAction(options) {
|
|
|
1114
999
|
exitCode: 1,
|
|
1115
1000
|
});
|
|
1116
1001
|
}
|
|
1002
|
+
if (options.action.action !== 'delete') {
|
|
1003
|
+
throw new CliError(`Unsupported maintenance action: ${options.action.action}`, {
|
|
1004
|
+
code: 'DATASET_MAINTENANCE_ACTION_UNSUPPORTED',
|
|
1005
|
+
exitCode: 2,
|
|
1006
|
+
});
|
|
1007
|
+
}
|
|
1117
1008
|
const remoteResult = await deleteMaintenanceRow({
|
|
1118
1009
|
context: options.context,
|
|
1119
1010
|
table: options.action.table,
|
|
@@ -1135,6 +1026,60 @@ async function executeAction(options) {
|
|
|
1135
1026
|
}
|
|
1136
1027
|
return { afterSha256: null, remoteResultSha256: sha256Json(remoteResult) };
|
|
1137
1028
|
}
|
|
1029
|
+
async function executeDerivativeAdmission(options) {
|
|
1030
|
+
const action = derivativePlanAction(options.plan);
|
|
1031
|
+
const plannedSnapshot = action.derivative_before;
|
|
1032
|
+
const preflight = parseDerivativeSnapshotResponse(await fetchMaintenanceDerivativeSnapshot({
|
|
1033
|
+
context: options.context,
|
|
1034
|
+
id: action.id,
|
|
1035
|
+
version: action.version,
|
|
1036
|
+
}), { id: action.id, version: action.version, userId: action.expected_user_id });
|
|
1037
|
+
if (preflight.modified_at !== plannedSnapshot.modified_at ||
|
|
1038
|
+
preflight.json_sha256 !== plannedSnapshot.json_sha256 ||
|
|
1039
|
+
preflight.json_ordered_sha256 !== plannedSnapshot.json_ordered_sha256 ||
|
|
1040
|
+
preflight.extracted_text_sha256 !== plannedSnapshot.extracted_text_sha256) {
|
|
1041
|
+
throw new CliError('Derivative action primary preconditions drifted after planning.', {
|
|
1042
|
+
code: 'DATASET_MAINTENANCE_DERIVATIVE_PRIMARY_DRIFT',
|
|
1043
|
+
exitCode: 1,
|
|
1044
|
+
details: {
|
|
1045
|
+
expected_snapshot_sha256: plannedSnapshot.snapshot_sha256,
|
|
1046
|
+
actual_snapshot_sha256: preflight.snapshot_sha256,
|
|
1047
|
+
},
|
|
1048
|
+
});
|
|
1049
|
+
}
|
|
1050
|
+
if (!options.replayPossible && preflight.snapshot_sha256 !== plannedSnapshot.snapshot_sha256) {
|
|
1051
|
+
throw new CliError('Derivative action-scoped snapshot drifted before first admission.', {
|
|
1052
|
+
code: 'DATASET_MAINTENANCE_DERIVATIVE_SNAPSHOT_DRIFT',
|
|
1053
|
+
exitCode: 1,
|
|
1054
|
+
details: {
|
|
1055
|
+
expected: plannedSnapshot.snapshot_sha256,
|
|
1056
|
+
actual: preflight.snapshot_sha256,
|
|
1057
|
+
},
|
|
1058
|
+
});
|
|
1059
|
+
}
|
|
1060
|
+
if (!options.replayPossible) {
|
|
1061
|
+
writeImmutableJson(options.attemptPath, {
|
|
1062
|
+
schema_version: 1,
|
|
1063
|
+
plan_sha256: options.plan.plan_sha256,
|
|
1064
|
+
operation_id: options.plan.operation_id,
|
|
1065
|
+
action_id: action.action_id,
|
|
1066
|
+
table: 'processes',
|
|
1067
|
+
id: action.id,
|
|
1068
|
+
version: action.version,
|
|
1069
|
+
expected_snapshot_sha256: plannedSnapshot.snapshot_sha256,
|
|
1070
|
+
actor: {
|
|
1071
|
+
user_id: options.context.account.user_id,
|
|
1072
|
+
email: options.context.account.email,
|
|
1073
|
+
},
|
|
1074
|
+
prepared_at_utc: options.preparedAtUtc,
|
|
1075
|
+
});
|
|
1076
|
+
}
|
|
1077
|
+
const result = await applyMaintenanceDerivativeRebuild({
|
|
1078
|
+
context: options.context,
|
|
1079
|
+
plan: buildDerivativePlanRequest(options.plan),
|
|
1080
|
+
});
|
|
1081
|
+
return parseDerivativeSubmitResponse(result, options.plan);
|
|
1082
|
+
}
|
|
1138
1083
|
function nextAttemptPath(planDir) {
|
|
1139
1084
|
let attempt = 1;
|
|
1140
1085
|
while (existsSync(path.join(planDir, `commit-report.attempt-${String(attempt).padStart(4, '0')}.json`))) {
|
|
@@ -1165,6 +1110,12 @@ export async function runDatasetMaintenanceApply(options) {
|
|
|
1165
1110
|
details: plan.blockers,
|
|
1166
1111
|
});
|
|
1167
1112
|
}
|
|
1113
|
+
if (plan.operation === 'merge-support-aliases') {
|
|
1114
|
+
throw new CliError('merge-support-aliases is sealed for dataset maintenance run-protected and cannot use ordinary apply.', {
|
|
1115
|
+
code: 'DATASET_MAINTENANCE_PROTECTED_RUN_REQUIRED',
|
|
1116
|
+
exitCode: 1,
|
|
1117
|
+
});
|
|
1118
|
+
}
|
|
1168
1119
|
if (plan.operation === 'redo-import' &&
|
|
1169
1120
|
!plan.source_import_run_id &&
|
|
1170
1121
|
plan.source_lineage === null) {
|
|
@@ -1173,10 +1124,9 @@ export async function runDatasetMaintenanceApply(options) {
|
|
|
1173
1124
|
exitCode: 1,
|
|
1174
1125
|
});
|
|
1175
1126
|
}
|
|
1176
|
-
const progressPath = path.join(planDir, '
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
const aliasExchangeProgressPath = path.join(planDir, 'alias-exchange-progress.jsonl');
|
|
1127
|
+
const progressPath = path.join(planDir, plan.operation === 'rebuild-derivatives'
|
|
1128
|
+
? 'derivative-submit-progress.jsonl'
|
|
1129
|
+
: 'apply-progress.jsonl');
|
|
1180
1130
|
return withStateFileLock(progressPath, { reason: `dataset_maintenance_apply_${plan.operation_id}` }, async () => {
|
|
1181
1131
|
const context = await resolveMaintenanceRemoteContext({
|
|
1182
1132
|
env: options.env,
|
|
@@ -1197,14 +1147,13 @@ export async function runDatasetMaintenanceApply(options) {
|
|
|
1197
1147
|
exitCode: 2,
|
|
1198
1148
|
});
|
|
1199
1149
|
}
|
|
1200
|
-
const
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
: { entries: [], successes: new Map() };
|
|
1150
|
+
const derivativeProgress = plan.operation === 'rebuild-derivatives'
|
|
1151
|
+
? parseDerivativeSubmitProgress(plan, progressPath)
|
|
1152
|
+
: { entries: [], latest: null };
|
|
1153
|
+
const progress = plan.operation === 'rebuild-derivatives'
|
|
1154
|
+
? { entries: [], successes: new Map(), latestFailures: new Map() }
|
|
1155
|
+
: parseProgress(plan, progressPath);
|
|
1156
|
+
const resumedSuccesses = progress.successes.size;
|
|
1208
1157
|
const current = await fetchMaintenanceAccountRows({
|
|
1209
1158
|
context,
|
|
1210
1159
|
userId: plan.account.user_id,
|
|
@@ -1214,14 +1163,12 @@ export async function runDatasetMaintenanceApply(options) {
|
|
|
1214
1163
|
planDir,
|
|
1215
1164
|
currentRows: current.rows,
|
|
1216
1165
|
progress,
|
|
1217
|
-
aliasPlanProgress,
|
|
1166
|
+
aliasPlanProgress: { entries: [], success: null, latestFailure: null },
|
|
1218
1167
|
});
|
|
1219
|
-
if (plan.operation === 'merge-support-aliases') {
|
|
1220
|
-
await assertAliasSupportSnapshots({ plan, context });
|
|
1221
|
-
}
|
|
1222
1168
|
const approvalPath = path.join(planDir, 'approval-record.json');
|
|
1169
|
+
const approvalAlreadyExisted = existsSync(approvalPath);
|
|
1223
1170
|
validateApprovalRecord({ path: approvalPath, plan, context });
|
|
1224
|
-
if (!
|
|
1171
|
+
if (!approvalAlreadyExisted) {
|
|
1225
1172
|
writeImmutableJson(approvalPath, {
|
|
1226
1173
|
schema_version: 1,
|
|
1227
1174
|
approved_at_utc: clock(options),
|
|
@@ -1243,87 +1190,48 @@ export async function runDatasetMaintenanceApply(options) {
|
|
|
1243
1190
|
: null,
|
|
1244
1191
|
});
|
|
1245
1192
|
}
|
|
1246
|
-
if (plan.operation === '
|
|
1247
|
-
const
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
try {
|
|
1261
|
-
const execution = await executeAliasPlan({ plan, planDir, context });
|
|
1262
|
-
const endedAt = clock(options);
|
|
1263
|
-
for (const batch of orderedAliasBatches(plan)) {
|
|
1264
|
-
appendAliasSuccessLogs({
|
|
1265
|
-
plan,
|
|
1266
|
-
batch,
|
|
1267
|
-
execution,
|
|
1268
|
-
progress,
|
|
1269
|
-
progressPath,
|
|
1270
|
-
exchangeProgressPath: aliasExchangeProgressPath,
|
|
1271
|
-
context,
|
|
1272
|
-
startedAt,
|
|
1273
|
-
endedAt,
|
|
1274
|
-
});
|
|
1275
|
-
}
|
|
1276
|
-
planSuccess = appendAliasProofProgress({
|
|
1277
|
-
plan,
|
|
1278
|
-
execution,
|
|
1279
|
-
planProgress: aliasPlanProgress,
|
|
1280
|
-
batchProgress: aliasBatchProgress,
|
|
1281
|
-
planProgressPath: aliasPlanProgressPath,
|
|
1282
|
-
batchProgressPath: aliasBatchProgressPath,
|
|
1283
|
-
context,
|
|
1284
|
-
startedAt,
|
|
1285
|
-
endedAt,
|
|
1286
|
-
});
|
|
1287
|
-
}
|
|
1288
|
-
catch (error) {
|
|
1289
|
-
planFailure = appendAliasPlanFailure({
|
|
1290
|
-
plan,
|
|
1291
|
-
planProgress: aliasPlanProgress,
|
|
1292
|
-
progressPath: aliasPlanProgressPath,
|
|
1293
|
-
context,
|
|
1294
|
-
startedAt,
|
|
1295
|
-
endedAt: clock(options),
|
|
1296
|
-
error,
|
|
1297
|
-
});
|
|
1298
|
-
}
|
|
1299
|
-
}
|
|
1300
|
-
const fullyProven = Boolean(planSuccess &&
|
|
1301
|
-
aliasPlanDerivedLogsComplete({
|
|
1302
|
-
plan,
|
|
1303
|
-
planSuccess,
|
|
1304
|
-
batchProgress: aliasBatchProgress,
|
|
1305
|
-
progress,
|
|
1306
|
-
exchangeProgressPath: aliasExchangeProgressPath,
|
|
1307
|
-
}));
|
|
1308
|
-
const failureError = planFailure?.error ?? 'Whole-plan proof is incomplete.';
|
|
1309
|
-
const actions = plan.actions.map((action) => {
|
|
1310
|
-
return {
|
|
1311
|
-
action_id: action.action_id,
|
|
1312
|
-
action: action.action,
|
|
1313
|
-
table: action.table,
|
|
1314
|
-
id: action.id,
|
|
1315
|
-
version: action.version,
|
|
1316
|
-
status: fullyProven ? 'success' : 'failed',
|
|
1317
|
-
error: fullyProven ? null : failureError,
|
|
1318
|
-
};
|
|
1193
|
+
if (plan.operation === 'rebuild-derivatives') {
|
|
1194
|
+
const startedAt = clock(options);
|
|
1195
|
+
const derivativeAttemptPath = path.join(planDir, 'derivative-admission-attempt.json');
|
|
1196
|
+
const derivativeAttempt = validateDerivativeAdmissionAttempt({
|
|
1197
|
+
path: derivativeAttemptPath,
|
|
1198
|
+
plan,
|
|
1199
|
+
context,
|
|
1200
|
+
});
|
|
1201
|
+
const proof = await executeDerivativeAdmission({
|
|
1202
|
+
plan,
|
|
1203
|
+
context,
|
|
1204
|
+
replayPossible: derivativeAttempt !== null,
|
|
1205
|
+
attemptPath: derivativeAttemptPath,
|
|
1206
|
+
preparedAtUtc: startedAt,
|
|
1319
1207
|
});
|
|
1320
|
-
|
|
1321
|
-
|
|
1208
|
+
if (derivativeProgress.latest &&
|
|
1209
|
+
derivativeProofIdentity(derivativeProgress.latest.proof) !==
|
|
1210
|
+
derivativeProofIdentity(proof)) {
|
|
1211
|
+
throw new CliError('Derivative guarded-RPC replay returned a different request proof.', {
|
|
1212
|
+
code: 'DATASET_MAINTENANCE_DERIVATIVE_REPLAY_MISMATCH',
|
|
1213
|
+
exitCode: 1,
|
|
1214
|
+
});
|
|
1215
|
+
}
|
|
1216
|
+
const action = derivativePlanAction(plan);
|
|
1217
|
+
const entry = {
|
|
1218
|
+
schema_version: 1,
|
|
1219
|
+
plan_sha256: plan.plan_sha256,
|
|
1220
|
+
operation_id: plan.operation_id,
|
|
1221
|
+
action_id: action.action_id,
|
|
1222
|
+
target_mode: 'owner_draft',
|
|
1223
|
+
actor: { user_id: context.account.user_id, email: context.account.email },
|
|
1224
|
+
started_at_utc: startedAt,
|
|
1225
|
+
ended_at_utc: clock(options),
|
|
1226
|
+
result: 'accepted',
|
|
1227
|
+
proof,
|
|
1228
|
+
};
|
|
1229
|
+
appendStableJsonLine(progressPath, entry);
|
|
1322
1230
|
const attemptPath = nextAttemptPath(planDir);
|
|
1323
1231
|
const report = {
|
|
1324
1232
|
schema_version: 1,
|
|
1325
1233
|
generated_at_utc: clock(options),
|
|
1326
|
-
status:
|
|
1234
|
+
status: 'accepted',
|
|
1327
1235
|
task_id: plan.task_id,
|
|
1328
1236
|
operation: plan.operation,
|
|
1329
1237
|
operation_id: plan.operation_id,
|
|
@@ -1331,21 +1239,31 @@ export async function runDatasetMaintenanceApply(options) {
|
|
|
1331
1239
|
plan_sha256: plan.plan_sha256,
|
|
1332
1240
|
actor: { user_id: context.account.user_id, email: context.account.email },
|
|
1333
1241
|
summary: {
|
|
1334
|
-
actions:
|
|
1335
|
-
success:
|
|
1336
|
-
failed:
|
|
1337
|
-
pending:
|
|
1338
|
-
resumed_successes:
|
|
1242
|
+
actions: 1,
|
|
1243
|
+
success: 0,
|
|
1244
|
+
failed: 0,
|
|
1245
|
+
pending: 1,
|
|
1246
|
+
resumed_successes: 0,
|
|
1247
|
+
accepted: 1,
|
|
1339
1248
|
},
|
|
1340
|
-
actions
|
|
1249
|
+
actions: [
|
|
1250
|
+
{
|
|
1251
|
+
action_id: action.action_id,
|
|
1252
|
+
action: action.action,
|
|
1253
|
+
table: action.table,
|
|
1254
|
+
id: action.id,
|
|
1255
|
+
version: action.version,
|
|
1256
|
+
status: 'accepted',
|
|
1257
|
+
error: null,
|
|
1258
|
+
},
|
|
1259
|
+
],
|
|
1341
1260
|
artifacts: {
|
|
1342
1261
|
approval_record: approvalPath,
|
|
1343
1262
|
apply_progress: progressPath,
|
|
1263
|
+
derivative_submit_progress: progressPath,
|
|
1264
|
+
derivative_admission_attempt: derivativeAttemptPath,
|
|
1344
1265
|
commit_report: path.join(planDir, 'commit-report.json'),
|
|
1345
1266
|
attempt_report: attemptPath,
|
|
1346
|
-
alias_plan_progress: aliasPlanProgressPath,
|
|
1347
|
-
alias_batch_progress: aliasBatchProgressPath,
|
|
1348
|
-
alias_exchange_progress: aliasExchangeProgressPath,
|
|
1349
1267
|
},
|
|
1350
1268
|
database_audit: {
|
|
1351
1269
|
rpc_transaction_log: 'public.command_audit_log',
|
|
@@ -1353,25 +1271,14 @@ export async function runDatasetMaintenanceApply(options) {
|
|
|
1353
1271
|
correlation_fields: [
|
|
1354
1272
|
'plan_sha256',
|
|
1355
1273
|
'operation_id',
|
|
1274
|
+
'action_id',
|
|
1356
1275
|
'target_visibility',
|
|
1357
1276
|
'plan_request_sha256',
|
|
1358
|
-
'
|
|
1359
|
-
'
|
|
1360
|
-
'batch_request_sha256',
|
|
1277
|
+
'action_request_sha256',
|
|
1278
|
+
'request_id',
|
|
1361
1279
|
],
|
|
1362
1280
|
},
|
|
1363
|
-
...
|
|
1364
|
-
? {
|
|
1365
|
-
alias_plan_proof: {
|
|
1366
|
-
plan_request_sha256: planSuccess.plan_request_sha256,
|
|
1367
|
-
summary_audit_id: planSuccess.summary_audit_id,
|
|
1368
|
-
batch_count: 2,
|
|
1369
|
-
row_count: 52,
|
|
1370
|
-
exchange_count: 59,
|
|
1371
|
-
idempotent_replay: planSuccess.idempotent_replay,
|
|
1372
|
-
},
|
|
1373
|
-
}
|
|
1374
|
-
: {}),
|
|
1281
|
+
derivative_admission: { ...proof, admission: 'accepted' },
|
|
1375
1282
|
};
|
|
1376
1283
|
writeImmutableJson(attemptPath, report);
|
|
1377
1284
|
writeJsonArtifact(report.artifacts.commit_report, report);
|
|
@@ -1381,6 +1288,7 @@ export async function runDatasetMaintenanceApply(options) {
|
|
|
1381
1288
|
const rank = {
|
|
1382
1289
|
save_draft: 0,
|
|
1383
1290
|
update_json_ordered: 0,
|
|
1291
|
+
rebuild_derivatives: 0,
|
|
1384
1292
|
delete: 1,
|
|
1385
1293
|
};
|
|
1386
1294
|
const actionOrder = rank[left.action] - rank[right.action];
|
|
@@ -1515,21 +1423,23 @@ export async function runDatasetMaintenanceApply(options) {
|
|
|
1515
1423
|
});
|
|
1516
1424
|
}
|
|
1517
1425
|
export const __testInternals = {
|
|
1518
|
-
aliasBatchDerivedLogsComplete,
|
|
1519
|
-
aliasPlanDerivedLogsComplete,
|
|
1520
1426
|
aliasExchangeProgressKey,
|
|
1521
1427
|
appendAliasSuccessLogs,
|
|
1428
|
+
appendAliasProofProgress,
|
|
1522
1429
|
assertApplyPreconditions,
|
|
1523
1430
|
assertAliasSupportSnapshots,
|
|
1524
1431
|
buildAliasBatchRequest,
|
|
1525
1432
|
buildAliasPlanRequest,
|
|
1526
1433
|
clock,
|
|
1527
1434
|
errorMessage,
|
|
1435
|
+
executeDerivativeAdmission,
|
|
1528
1436
|
executeAliasPlan,
|
|
1529
1437
|
executeAction,
|
|
1530
1438
|
finalProjectedRows,
|
|
1531
1439
|
loadDesiredPayload,
|
|
1532
1440
|
nextAttemptPath,
|
|
1441
|
+
parseDerivativeSubmitProgress,
|
|
1442
|
+
validateDerivativeAdmissionAttempt,
|
|
1533
1443
|
parseAliasBatchProgress,
|
|
1534
1444
|
parseAliasPlanProgress,
|
|
1535
1445
|
parseProgress,
|