@aopslabs/aops-server 0.2.24 → 0.2.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/THIRD_PARTY_NOTICES +2 -2
- package/THIRD_PARTY_NOTICES.inventory.json +5 -5
- package/build/client/_app/immutable/chunks/{B7Ksy9Vx.js → Cs-rWsD4.js} +1 -1
- package/build/client/_app/immutable/entry/{app.Bqm7erUX.js → app.Ck59hFAi.js} +2 -2
- package/build/client/_app/immutable/entry/start.mFYy5iA-.js +1 -0
- package/build/client/_app/immutable/nodes/{1.BwqSn0AV.js → 1.YS2rDA-X.js} +1 -1
- package/build/client/_app/version.json +1 -1
- package/build/handler.js +4 -4
- package/build/index.js +4 -4
- package/build/server/chunks/chunks/{internal.js-FlQEV1Hh.js → internal.js-v_HGj6nU.js} +1 -1
- package/build/server/chunks/{handler-efTCmtgM.js → handler-CZshoFnY.js} +2 -2
- package/build/server/chunks/{index.js-CGfay5SX.js → index.js-CySur4Kt.js} +1 -1
- package/build/server/chunks/{manifest.js-BTST1jcU.js → manifest.js-vXuDslAA.js} +2 -2
- package/build/server/chunks/nodes/{1.js-K2ekrPGa.js → 1.js-BbsuS5PD.js} +1 -1
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/runtime-closure.package.json +1 -1
- package/scripts/community-domain-ownership-v1.json +187 -0
- package/scripts/domain-backup-provider.mjs +246 -0
- package/scripts/owned-set-backup.mjs +437 -0
- package/scripts/owned-set-restore.mjs +655 -0
- package/scripts/owned-set-snapshot.mjs +182 -0
- package/build/client/_app/immutable/entry/start.2cgNGUw5.js +0 -1
|
@@ -0,0 +1,655 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Restores a first-party backup, in one of two explicit modes.
|
|
4
|
+
*
|
|
5
|
+
* existing-exact-predecessor -- the schema is already the state the bundle was
|
|
6
|
+
* taken from. Migrations are not re-applied and the ledgers are not touched;
|
|
7
|
+
* the schema is verified to match exactly, then the owned data is replaced in
|
|
8
|
+
* one transaction.
|
|
9
|
+
*
|
|
10
|
+
* empty-disaster-target -- there is no schema. The embedded closure builds the
|
|
11
|
+
* predecessor schema, the engine writes the ledgers as a side effect of
|
|
12
|
+
* applying it, and only then is the data loaded. The bundle's ledger evidence
|
|
13
|
+
* is compared against what the engine produced rather than replayed over it.
|
|
14
|
+
*
|
|
15
|
+
* Ledger rows are never inserted as data in either mode. A ledger row asserts
|
|
16
|
+
* that a migration ran; asserting that independently of the DDL produces a
|
|
17
|
+
* database that lies about its own history, and nothing downstream would catch
|
|
18
|
+
* it -- the schema would look right and the applied counts would look right.
|
|
19
|
+
*
|
|
20
|
+
* Expected integrity comes from the receipt OUTSIDE the bundle. A bundle that
|
|
21
|
+
* validates itself validates a coordinated edit too: an earlier version of this
|
|
22
|
+
* design accepted exactly that attack, with the chunk rewritten and the manifest
|
|
23
|
+
* rewritten to match.
|
|
24
|
+
*/
|
|
25
|
+
import { createHash } from 'node:crypto'
|
|
26
|
+
import { gunzipSync } from 'node:zlib'
|
|
27
|
+
import { readFileSync, readdirSync, existsSync } from 'node:fs'
|
|
28
|
+
import path from 'node:path'
|
|
29
|
+
import { fileURLToPath } from 'node:url'
|
|
30
|
+
|
|
31
|
+
const CONTRACT = 'aops.owned-set-backup/v2'
|
|
32
|
+
const RECEIPT_CONTRACT = 'aops.backup-receipt/v1'
|
|
33
|
+
const INSERT_BATCH = 500
|
|
34
|
+
|
|
35
|
+
const quote = (id) => `"${String(id).replace(/"/g, '""')}"`
|
|
36
|
+
const sha256 = (buffer) => createHash('sha256').update(buffer).digest('hex')
|
|
37
|
+
|
|
38
|
+
export const RESTORE_MODES = Object.freeze(['existing-exact-predecessor', 'empty-disaster-target'])
|
|
39
|
+
|
|
40
|
+
/** Every chunk verified before anything is written. All-or-nothing starts here. */
|
|
41
|
+
export function verifyBundle(bundleDirectory, anchorPath, { expectedProviderContractSha256 } = {}) {
|
|
42
|
+
const manifestPath = path.join(bundleDirectory, 'manifest.json')
|
|
43
|
+
let manifestBytes
|
|
44
|
+
try { manifestBytes = readFileSync(manifestPath) } catch { throw new Error('bundle_manifest_missing_or_unreadable') }
|
|
45
|
+
const manifest = JSON.parse(manifestBytes.toString('utf8'))
|
|
46
|
+
if (manifest.contract !== CONTRACT) throw new Error(`bundle_contract_mismatch:${manifest.contract}`)
|
|
47
|
+
if (manifest.identity?.identityMode !== 'semantic-owned-v1') {
|
|
48
|
+
throw new Error(`bundle_not_backup_eligible:identityMode=${String(manifest.identity?.identityMode)}`)
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
if (!anchorPath) throw new Error('bundle_unanchored:automatic_eligibility_denied')
|
|
52
|
+
let anchor
|
|
53
|
+
try { anchor = JSON.parse(readFileSync(anchorPath, 'utf8')) }
|
|
54
|
+
catch { throw new Error('bundle_anchor_missing_or_unreadable') }
|
|
55
|
+
if (anchor.contract !== RECEIPT_CONTRACT) throw new Error('anchor_contract_mismatch')
|
|
56
|
+
const measured = sha256(manifestBytes)
|
|
57
|
+
if (anchor.bundleSha256 !== measured) {
|
|
58
|
+
throw new Error(`bundle_sha_mismatch_vs_anchor:anchor=${String(anchor.bundleSha256).slice(0, 12)}:measured=${measured.slice(0, 12)}`)
|
|
59
|
+
}
|
|
60
|
+
if (anchor.backupSetId !== manifest.backupSetId) {
|
|
61
|
+
throw new Error(`bundle_set_id_mismatch_vs_anchor:${String(anchor.backupSetId)}:${String(manifest.backupSetId)}`)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Three-way, and each leg refuses something the others cannot.
|
|
65
|
+
//
|
|
66
|
+
// The manifest says which provider contract produced this bundle. The anchor
|
|
67
|
+
// says the same thing from outside the bundle, so a coordinated edit of the
|
|
68
|
+
// bundle cannot move it. And the caller's expectation -- the value the lineage
|
|
69
|
+
// declares -- says the contract is the one this installation was reviewed
|
|
70
|
+
// under, not merely one that was internally consistent.
|
|
71
|
+
//
|
|
72
|
+
// A bundle produced before this field existed carries neither half. That is
|
|
73
|
+
// refused rather than tolerated: an unversioned backup restored into a
|
|
74
|
+
// contract-checked world is exactly the state nobody can reason about.
|
|
75
|
+
if (typeof manifest.providerContractSha256 !== 'string' || manifest.providerContractSha256.length !== 64) {
|
|
76
|
+
throw new Error('bundle_provider_contract_absent')
|
|
77
|
+
}
|
|
78
|
+
if (anchor.providerContractSha256 !== manifest.providerContractSha256) {
|
|
79
|
+
throw new Error('bundle_provider_contract_mismatch_vs_anchor:' +
|
|
80
|
+
`${String(anchor.providerContractSha256).slice(0, 12)}:${manifest.providerContractSha256.slice(0, 12)}`)
|
|
81
|
+
}
|
|
82
|
+
if (expectedProviderContractSha256 && expectedProviderContractSha256 !== manifest.providerContractSha256) {
|
|
83
|
+
throw new Error('bundle_provider_contract_mismatch_vs_expected:' +
|
|
84
|
+
`${expectedProviderContractSha256.slice(0, 12)}:${manifest.providerContractSha256.slice(0, 12)}`)
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const rowsByRelation = new Map()
|
|
88
|
+
for (const chunk of manifest.chunks) {
|
|
89
|
+
const packed = readFileSync(path.join(bundleDirectory, chunk.file))
|
|
90
|
+
if (sha256(packed) !== chunk.compressedSha256) throw new Error(`chunk_compressed_sha_mismatch:${chunk.file}`)
|
|
91
|
+
if (packed.length !== chunk.compressedBytes) throw new Error(`chunk_compressed_size_mismatch:${chunk.file}`)
|
|
92
|
+
const plain = gunzipSync(packed)
|
|
93
|
+
if (sha256(plain) !== chunk.uncompressedSha256) throw new Error(`chunk_uncompressed_sha_mismatch:${chunk.file}`)
|
|
94
|
+
if (plain.length !== chunk.uncompressedBytes) throw new Error(`chunk_uncompressed_size_mismatch:${chunk.file}`)
|
|
95
|
+
const lines = plain.toString('utf8').split('\n').filter((line) => line.length > 0)
|
|
96
|
+
if (lines.length !== chunk.rowCount) throw new Error(`chunk_row_count_mismatch:${chunk.file}`)
|
|
97
|
+
if (!rowsByRelation.has(chunk.relation)) rowsByRelation.set(chunk.relation, [])
|
|
98
|
+
rowsByRelation.get(chunk.relation).push({ index: chunk.index, lines })
|
|
99
|
+
}
|
|
100
|
+
const onDisk = readdirSync(path.join(bundleDirectory, 'fragments')).filter((name) => name.endsWith('.gz'))
|
|
101
|
+
if (onDisk.length !== manifest.chunks.length) {
|
|
102
|
+
throw new Error(`bundle_chunk_count_mismatch:disk=${onDisk.length}:manifest=${manifest.chunks.length}`)
|
|
103
|
+
}
|
|
104
|
+
return { manifest, rowsByRelation, bundleSha256: measured }
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** The embedded closure, re-verified against the manifest before it is applied. */
|
|
108
|
+
export function verifyEmbeddedClosure(bundleDirectory, manifest) {
|
|
109
|
+
if (!manifest.migrationSetEmbedded || !manifest.closure) {
|
|
110
|
+
throw new Error('bundle_closure_absent:not_disaster_recovery_eligible')
|
|
111
|
+
}
|
|
112
|
+
for (const file of manifest.closure.files) {
|
|
113
|
+
const target = path.join(bundleDirectory, 'closure', file.root, file.relativePath)
|
|
114
|
+
if (!existsSync(target)) throw new Error(`closure_file_missing:${file.root}:${file.relativePath}`)
|
|
115
|
+
const bytes = readFileSync(target)
|
|
116
|
+
if (sha256(bytes) !== file.sha256) throw new Error(`closure_file_sha_mismatch:${file.root}:${file.relativePath}`)
|
|
117
|
+
if (bytes.length !== file.byteLength) throw new Error(`closure_file_size_mismatch:${file.root}:${file.relativePath}`)
|
|
118
|
+
}
|
|
119
|
+
return { rootCount: manifest.closure.rootCount, fileCount: manifest.closure.fileCount }
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
async function assertTargetSchemaExact(client, manifest) {
|
|
123
|
+
const relations = Object.keys(manifest.shape)
|
|
124
|
+
const { rows } = await client.query(`
|
|
125
|
+
SELECT c.relname AS relation, a.attname AS column, a.atttypid::int AS typeOid
|
|
126
|
+
FROM pg_attribute a
|
|
127
|
+
JOIN pg_class c ON c.oid = a.attrelid
|
|
128
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
129
|
+
WHERE n.nspname = 'public' AND c.relname = ANY($1)
|
|
130
|
+
AND a.attnum > 0 AND NOT a.attisdropped AND c.relkind IN ('r', 'p')
|
|
131
|
+
ORDER BY c.relname, a.attnum`, [relations])
|
|
132
|
+
const live = new Map()
|
|
133
|
+
for (const row of rows) {
|
|
134
|
+
if (!live.has(row.relation)) live.set(row.relation, [])
|
|
135
|
+
live.get(row.relation).push(`${row.column}:${row.typeoid}`)
|
|
136
|
+
}
|
|
137
|
+
for (const [relation, columns] of Object.entries(manifest.shape)) {
|
|
138
|
+
const actual = live.get(relation)
|
|
139
|
+
if (!actual) throw new Error(`target_relation_missing:${relation}`)
|
|
140
|
+
// Compared as a set of name:type pairs, not as an ordered list.
|
|
141
|
+
//
|
|
142
|
+
// Column order is not a property this restore depends on -- every INSERT names
|
|
143
|
+
// its columns explicitly -- and it legitimately differs between a database
|
|
144
|
+
// built by successive migrations and one rebuilt from a closure in one pass.
|
|
145
|
+
// Measured: `agent-sessions.missionId` is the last column in the production
|
|
146
|
+
// copy and the fourth in the closure-rebuilt schema, with identical names and
|
|
147
|
+
// type OIDs throughout. Requiring the order would reject a correct restore
|
|
148
|
+
// while catching nothing a set comparison misses.
|
|
149
|
+
const expected = [...columns.map((column) => `${column.column}:${column.typeOid}`)].sort()
|
|
150
|
+
const observed = [...actual].sort()
|
|
151
|
+
if (expected.join(',') !== observed.join(',')) {
|
|
152
|
+
const missing = expected.filter((entry) => !observed.includes(entry))
|
|
153
|
+
const extra = observed.filter((entry) => !expected.includes(entry))
|
|
154
|
+
throw new Error(`target_schema_mismatch:${relation}:missing=${missing.join('|')}:extra=${extra.join('|')}`)
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Ledgers are compared, never replayed -- and compared by CONTENT, not by count.
|
|
161
|
+
*
|
|
162
|
+
* A row count answers "how many migrations does this database think it ran", which
|
|
163
|
+
* is the question two different histories can answer identically: swap one tag for
|
|
164
|
+
* another and the count is unchanged while the claim is completely different. The
|
|
165
|
+
* digest covers the ordered tags and their recorded hashes, so a same-count
|
|
166
|
+
* substitution fails here rather than passing as agreement.
|
|
167
|
+
*/
|
|
168
|
+
export async function readLedgerDigest(client, relation) {
|
|
169
|
+
const { rows } = await client.query(`
|
|
170
|
+
SELECT a.attname AS column, format_type(a.atttypid, a.atttypmod) AS type
|
|
171
|
+
FROM pg_attribute a
|
|
172
|
+
JOIN pg_class c ON c.oid = a.attrelid
|
|
173
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
174
|
+
WHERE n.nspname = 'public' AND c.relname = $1 AND a.attnum > 0 AND NOT a.attisdropped
|
|
175
|
+
ORDER BY a.attnum`, [relation])
|
|
176
|
+
if (rows.length === 0) return { rowCount: 0, contentSha256: sha256(Buffer.alloc(0)) }
|
|
177
|
+
// `appliedAt` is excluded: it records when a migration ran on this machine, not
|
|
178
|
+
// which migration ran, and comparing it would make every legitimate restore
|
|
179
|
+
// look like drift.
|
|
180
|
+
// Timestamps are excluded BY TYPE, not by name. The column is `applied_at` in
|
|
181
|
+
// these ledgers and `appliedAt` elsewhere; excluding one spelling left the other
|
|
182
|
+
// in the digest, and every legitimately rebuilt ledger then differed from the
|
|
183
|
+
// one it was rebuilt from -- for no reason except that it was applied at a
|
|
184
|
+
// different moment, which is the one thing the digest must not care about.
|
|
185
|
+
const columns = rows
|
|
186
|
+
.filter((row) => !/^timestamp( |$)/.test(String(row.type)))
|
|
187
|
+
.map((row) => row.column)
|
|
188
|
+
const projection = columns.map((column) => `coalesce(${quote(column)}::text, '')`).join(` || '|' || `)
|
|
189
|
+
const digest = await client.query(`
|
|
190
|
+
SELECT count(*)::int AS rows,
|
|
191
|
+
coalesce(string_agg(line, E'\n' ORDER BY line), '') AS body
|
|
192
|
+
FROM (SELECT ${projection} AS line FROM public.${quote(relation)}) source`)
|
|
193
|
+
return {
|
|
194
|
+
rowCount: digest.rows[0].rows,
|
|
195
|
+
contentSha256: sha256(Buffer.from(digest.rows[0].body)),
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
async function assertLedgerEvidence(client, manifest) {
|
|
200
|
+
const mismatches = []
|
|
201
|
+
for (const [relation, evidence] of Object.entries(manifest.ledgerEvidence ?? {})) {
|
|
202
|
+
const observed = await readLedgerDigest(client, relation)
|
|
203
|
+
if (observed.rowCount !== evidence.rowCount) {
|
|
204
|
+
mismatches.push(`${relation}:count:expected=${evidence.rowCount}:actual=${observed.rowCount}`)
|
|
205
|
+
continue
|
|
206
|
+
}
|
|
207
|
+
if (!evidence.contentSha256) {
|
|
208
|
+
// An evidence record without a content digest cannot be checked for the
|
|
209
|
+
// substitution it exists to catch, so it is not weaker evidence -- it is
|
|
210
|
+
// absent evidence.
|
|
211
|
+
mismatches.push(`${relation}:content_digest_absent`)
|
|
212
|
+
continue
|
|
213
|
+
}
|
|
214
|
+
if (observed.contentSha256 !== evidence.contentSha256) {
|
|
215
|
+
mismatches.push(`${relation}:content:expected=${evidence.contentSha256.slice(0, 12)}:actual=${observed.contentSha256.slice(0, 12)}`)
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
if (mismatches.length > 0) throw new Error(`ledger_evidence_mismatch:${mismatches.join(',')}`)
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* The owned semantic identity of the live database, compared to the bundle's.
|
|
223
|
+
*
|
|
224
|
+
* Separate from the column check above on purpose. Columns and type OIDs can
|
|
225
|
+
* agree while the projection differs -- an index or a constraint moved -- and the
|
|
226
|
+
* projection can agree while a ledger disagrees. Collapsing them into one check
|
|
227
|
+
* would let either half mask the other's absence.
|
|
228
|
+
*/
|
|
229
|
+
async function assertOwnedSemanticIdentity(client, manifest, semanticIdentity) {
|
|
230
|
+
if (!semanticIdentity) {
|
|
231
|
+
// No way to measure it is not the same as measuring it and finding it equal.
|
|
232
|
+
throw new Error('restore_semantic_identity_unavailable:eligibility_denied')
|
|
233
|
+
}
|
|
234
|
+
const expected = manifest.identity?.ownedFingerprintSha256
|
|
235
|
+
if (!expected) throw new Error('bundle_owned_fingerprint_absent:eligibility_denied')
|
|
236
|
+
const projection = await semanticIdentity.readProjection(client)
|
|
237
|
+
const actual = semanticIdentity.fingerprint(projection, manifest.identity.ownedRelations)
|
|
238
|
+
if (actual !== expected) {
|
|
239
|
+
throw new Error(`predecessor_semantic_identity_mismatch:expected=${expected.slice(0, 12)}:actual=${actual.slice(0, 12)}`)
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* A policy whose migration roots point at the bundle's embedded closure instead of
|
|
245
|
+
* an installed package.
|
|
246
|
+
*
|
|
247
|
+
* This is what "bundle-only" means in practice: the engine resolves every root
|
|
248
|
+
* under the bundle directory, so applying it cannot reach a migration the bundle
|
|
249
|
+
* does not carry. Mixing the two would defeat the purpose -- a disaster restore
|
|
250
|
+
* that quietly used the locally installed migrations would rebuild whatever this
|
|
251
|
+
* machine happens to have rather than what the backup was taken from.
|
|
252
|
+
*
|
|
253
|
+
* The closure directory is named after the drizzle output subdirectory, which is
|
|
254
|
+
* not always the root id (`agentspace` lives in `agentspace-community`), so the
|
|
255
|
+
* mapping is taken from the policy's own path rather than assumed.
|
|
256
|
+
*/
|
|
257
|
+
export function bundleOnlyPolicy(basePolicy, manifest, bundleDirectory) {
|
|
258
|
+
if (!manifest.migrationSetEmbedded || !manifest.closure) {
|
|
259
|
+
throw new Error('bundle_closure_absent:not_disaster_recovery_eligible')
|
|
260
|
+
}
|
|
261
|
+
const embeddedRoots = new Set(manifest.closure.files.map((file) => file.root))
|
|
262
|
+
const policy = JSON.parse(JSON.stringify(basePolicy))
|
|
263
|
+
for (const root of policy.roots) {
|
|
264
|
+
const directoryName = root.migrationsDir.split('/').pop()
|
|
265
|
+
if (!embeddedRoots.has(directoryName)) {
|
|
266
|
+
throw new Error(`bundle_closure_root_missing:${root.id}:${directoryName}`)
|
|
267
|
+
}
|
|
268
|
+
root.migrationsDir = `closure/${directoryName}`
|
|
269
|
+
|
|
270
|
+
// The root's journal and migration list are rebuilt FROM THE EMBEDDED BYTES,
|
|
271
|
+
// not carried over from the current policy.
|
|
272
|
+
//
|
|
273
|
+
// Measured reason: the closure is the predecessor's, and the running policy
|
|
274
|
+
// describes the target. agentspace has one migration in the 0.3.24 closure and
|
|
275
|
+
// two in the current tree, so the current journal hash cannot match what the
|
|
276
|
+
// bundle carries -- the engine refuses with journal_hash_mismatch, correctly.
|
|
277
|
+
// Rewriting only the paths made the bundle "self-contained" in the sense that
|
|
278
|
+
// the SQL travelled with it, while the description that validates the SQL
|
|
279
|
+
// still came from this machine.
|
|
280
|
+
const journalPath = path.join(bundleDirectory, 'closure', directoryName, 'meta', '_journal.json')
|
|
281
|
+
const journalBytes = readFileSync(journalPath)
|
|
282
|
+
const journal = JSON.parse(journalBytes.toString('utf8'))
|
|
283
|
+
root.journalSha256 = sha256(journalBytes)
|
|
284
|
+
root.journalSha256History = [root.journalSha256]
|
|
285
|
+
root.migrations = journal.entries.map((entry) => {
|
|
286
|
+
const sqlPath = path.join(bundleDirectory, 'closure', directoryName, `${entry.tag}.sql`)
|
|
287
|
+
const declared = manifest.closure.files.find((file) =>
|
|
288
|
+
file.root === directoryName && file.relativePath === `${entry.tag}.sql`)
|
|
289
|
+
if (!declared) throw new Error(`bundle_closure_sql_undeclared:${directoryName}:${entry.tag}`)
|
|
290
|
+
const bytes = readFileSync(sqlPath)
|
|
291
|
+
const actual = sha256(bytes)
|
|
292
|
+
if (actual !== declared.sha256) {
|
|
293
|
+
throw new Error(`bundle_closure_sql_mismatch:${directoryName}:${entry.tag}`)
|
|
294
|
+
}
|
|
295
|
+
return { idx: entry.idx, tag: entry.tag, sha256: actual, risk: 'additive' }
|
|
296
|
+
})
|
|
297
|
+
}
|
|
298
|
+
// Only the bundle's own lineage survives. The others describe states this
|
|
299
|
+
// closure cannot produce -- `strict-v4` counts two agentspace migrations where
|
|
300
|
+
// the predecessor closure has one -- and a policy carrying them is invalid
|
|
301
|
+
// against its own roots. Keeping them would also mean a disaster restore could
|
|
302
|
+
// classify into a lineage whose migrations are not in the bundle.
|
|
303
|
+
const own = policy.lineages.find((lineage) => lineage.id === manifest.identity.lineageId)
|
|
304
|
+
if (!own) throw new Error(`bundle_lineage_not_in_policy:${manifest.identity.lineageId}`)
|
|
305
|
+
// Keep the lineages this closure can actually produce -- every record whose
|
|
306
|
+
// applied counts fit inside the rebuilt roots -- and drop the rest. The same
|
|
307
|
+
// release observed on another PostgreSQL major is such a record and belongs
|
|
308
|
+
// here; `strict-v4`, which counts migrations the predecessor closure does not
|
|
309
|
+
// contain, does not. The policy also requires more than one lineage, so keeping
|
|
310
|
+
// the producible siblings is both correct and what makes the policy valid.
|
|
311
|
+
const producible = policy.lineages.filter((lineage) =>
|
|
312
|
+
lineage.appliedCounts.every((count, index) => count <= (policy.roots[index]?.migrations.length ?? -1)))
|
|
313
|
+
if (!producible.some((lineage) => lineage.id === own.id)) {
|
|
314
|
+
throw new Error(`bundle_lineage_not_producible_by_closure:${own.id}`)
|
|
315
|
+
}
|
|
316
|
+
policy.lineages = producible.map((lineage) =>
|
|
317
|
+
(lineage.id === own.id ? { ...lineage, kind: 'strict' } : lineage))
|
|
318
|
+
policy.lineageReconciliations = []
|
|
319
|
+
policy.targetLineageId = manifest.identity.lineageId
|
|
320
|
+
// The contract is derived from the target state; a predecessor closure does not
|
|
321
|
+
// produce it, and leaving it in would compare the rebuilt schema to a shape it
|
|
322
|
+
// is not supposed to have yet.
|
|
323
|
+
delete policy.semanticContract
|
|
324
|
+
return policy
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
export async function restoreOwnedSet({
|
|
328
|
+
client, bundleDirectory, anchorPath, mode, confirmReplace = false, semanticIdentity,
|
|
329
|
+
expectedProviderContractSha256,
|
|
330
|
+
}) {
|
|
331
|
+
if (!RESTORE_MODES.includes(mode)) throw new Error(`restore_mode_invalid:${String(mode)}`)
|
|
332
|
+
// Taken from the caller's explicit argument first, and from the semantic
|
|
333
|
+
// identity only as a fallback, so a caller that holds a reviewed expectation
|
|
334
|
+
// cannot have it silently replaced by whatever the identity happens to carry.
|
|
335
|
+
const expectedContract = expectedProviderContractSha256 ?? semanticIdentity?.providerContractSha256
|
|
336
|
+
const { manifest, rowsByRelation } = verifyBundle(bundleDirectory, anchorPath,
|
|
337
|
+
{ expectedProviderContractSha256: expectedContract })
|
|
338
|
+
|
|
339
|
+
if (mode === 'empty-disaster-target') {
|
|
340
|
+
// The closure has to be present and intact before anything else is decided:
|
|
341
|
+
// this mode's whole premise is that the bundle can rebuild the schema.
|
|
342
|
+
verifyEmbeddedClosure(bundleDirectory, manifest)
|
|
343
|
+
const { rows } = await client.query(`
|
|
344
|
+
SELECT count(*)::int AS relations FROM pg_class c
|
|
345
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
346
|
+
WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p')`)
|
|
347
|
+
if (rows[0].relations !== 0) {
|
|
348
|
+
throw new Error(`empty_disaster_target_not_empty:${rows[0].relations}`)
|
|
349
|
+
}
|
|
350
|
+
// Applying the closure is the engine's job, not this module's; it is invoked
|
|
351
|
+
// by the caller between verification and load so the engine writes the
|
|
352
|
+
// ledgers itself. Reaching here with a non-empty target is the failure.
|
|
353
|
+
return { mode, verifiedOnly: true, closure: manifest.closure.fileCount, chunks: manifest.chunks.length }
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
if (!confirmReplace) throw new Error('restore_replace_owned_data_requires_confirmation')
|
|
357
|
+
// Three gates, deliberately separate. Each can fail while the others pass, and
|
|
358
|
+
// one must never stand in for another: a ledger that agrees about which
|
|
359
|
+
// migrations ran says nothing about whether the schema those migrations
|
|
360
|
+
// produce is actually there, and vice versa.
|
|
361
|
+
await assertTargetSchemaExact(client, manifest)
|
|
362
|
+
await assertOwnedSemanticIdentity(client, manifest, semanticIdentity)
|
|
363
|
+
await assertLedgerEvidence(client, manifest)
|
|
364
|
+
|
|
365
|
+
const payloadRelations = manifest.restoreOrder.filter((relation) =>
|
|
366
|
+
!Object.hasOwn(manifest.ledgerEvidence ?? {}, relation))
|
|
367
|
+
|
|
368
|
+
await client.query('BEGIN')
|
|
369
|
+
try {
|
|
370
|
+
// One TRUNCATE naming exactly the owned payload. Never CASCADE: PostgreSQL
|
|
371
|
+
// requires every referencing relation in the same statement, so this single
|
|
372
|
+
// statement is itself the proof the set is closed -- a reference from outside
|
|
373
|
+
// makes it fail rather than quietly reaching past the boundary.
|
|
374
|
+
await client.query(`TRUNCATE TABLE ${payloadRelations.map((relation) => `public.${quote(relation)}`).join(', ')}`)
|
|
375
|
+
let written = 0
|
|
376
|
+
for (const relation of payloadRelations) {
|
|
377
|
+
const columns = manifest.shape[relation] ?? []
|
|
378
|
+
const chunks = (rowsByRelation.get(relation) ?? []).sort((a, b) => a.index - b.index)
|
|
379
|
+
const selfColumns = manifest.selfReferencing?.[relation] ?? []
|
|
380
|
+
const columnList = columns.map((column) => quote(column.column)).join(', ')
|
|
381
|
+
const secondPass = []
|
|
382
|
+
for (const chunk of chunks) {
|
|
383
|
+
for (let offset = 0; offset < chunk.lines.length; offset += INSERT_BATCH) {
|
|
384
|
+
const batch = chunk.lines.slice(offset, offset + INSERT_BATCH).map((line) => JSON.parse(line))
|
|
385
|
+
const parameters = []
|
|
386
|
+
const values = batch.map((row) => `(${columns.map((column, index) => {
|
|
387
|
+
if (selfColumns.includes(column.column)) return 'NULL'
|
|
388
|
+
parameters.push(row[index])
|
|
389
|
+
return `$${parameters.length}::${column.type}`
|
|
390
|
+
}).join(', ')})`)
|
|
391
|
+
await client.query(
|
|
392
|
+
`INSERT INTO public.${quote(relation)} (${columnList}) VALUES ${values.join(', ')}`, parameters)
|
|
393
|
+
if (selfColumns.length > 0) secondPass.push(...batch)
|
|
394
|
+
written += batch.length
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
// A self-reference cannot be satisfied on first insert, so it is loaded as
|
|
398
|
+
// NULL and linked once every row of that relation exists.
|
|
399
|
+
if (selfColumns.length > 0 && secondPass.length > 0) {
|
|
400
|
+
const key = columns[0]
|
|
401
|
+
for (const row of secondPass) {
|
|
402
|
+
const indexes = selfColumns.map((name) => columns.findIndex((column) => column.column === name))
|
|
403
|
+
if (indexes.every((index) => row[index] === null)) continue
|
|
404
|
+
const assignments = selfColumns.map((name, position) =>
|
|
405
|
+
`${quote(name)} = $${position + 2}::${columns[indexes[position]].type}`)
|
|
406
|
+
await client.query(
|
|
407
|
+
`UPDATE public.${quote(relation)} SET ${assignments.join(', ')} WHERE ${quote(key.column)} = $1::${key.type}`,
|
|
408
|
+
[row[0], ...indexes.map((index) => row[index])])
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
for (const sequence of manifest.sequences) {
|
|
413
|
+
if (sequence.lastValue === null) continue
|
|
414
|
+
// The name is quoted inside the literal rather than passed as a parameter.
|
|
415
|
+
// A parameter reaching `setval` is cast through regclass, which lowercases
|
|
416
|
+
// an unquoted identifier -- so any sequence with a capital in its name
|
|
417
|
+
// resolves to a relation that does not exist. Measured, not guessed: a
|
|
418
|
+
// bigserial column named "seqTest" produced
|
|
419
|
+
// relation "public.agent-sessions_seqtest_seq" does not exist.
|
|
420
|
+
await client.query(
|
|
421
|
+
`SELECT setval('public.${quote(sequence.name).replace(/'/g, "''")}',` +
|
|
422
|
+
` GREATEST($1::bigint, (SELECT last_value FROM public.${quote(sequence.name)})), true)`,
|
|
423
|
+
[sequence.lastValue])
|
|
424
|
+
}
|
|
425
|
+
// Read back BEFORE the commit, not after. Writing setval and moving on is not
|
|
426
|
+
// the same as knowing it took -- a sequence left below its restored value
|
|
427
|
+
// collides on the next insert. But a check that runs after COMMIT can only
|
|
428
|
+
// report a database it can no longer repair: the data is already visible and
|
|
429
|
+
// the transaction that could have withdrawn it is gone. Inside the
|
|
430
|
+
// transaction the same failure rolls everything back.
|
|
431
|
+
const behind = []
|
|
432
|
+
for (const sequence of manifest.sequences) {
|
|
433
|
+
if (sequence.lastValue === null) continue
|
|
434
|
+
const { rows } = await client.query(`SELECT last_value FROM public.${quote(sequence.name)}`)
|
|
435
|
+
if (BigInt(rows[0].last_value) < BigInt(sequence.lastValue)) {
|
|
436
|
+
behind.push(`${sequence.name}:read=${rows[0].last_value}:expected=${sequence.lastValue}`)
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
if (behind.length > 0) throw new Error(`sequence_readback_mismatch:${behind.slice(0, 3).join(',')}`)
|
|
440
|
+
|
|
441
|
+
await client.query('COMMIT')
|
|
442
|
+
|
|
443
|
+
return {
|
|
444
|
+
mode,
|
|
445
|
+
relations: payloadRelations.length,
|
|
446
|
+
rows: written,
|
|
447
|
+
chunks: manifest.chunks.length,
|
|
448
|
+
sequencesVerified: manifest.sequences.filter((sequence) => sequence.lastValue !== null).length,
|
|
449
|
+
}
|
|
450
|
+
} catch (error) {
|
|
451
|
+
await client.query('ROLLBACK').catch(() => undefined)
|
|
452
|
+
throw error
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
/**
|
|
457
|
+
* The whole disaster-recovery path, as one product operation.
|
|
458
|
+
*
|
|
459
|
+
* It used to stop after verification and hand the engine and the load back to
|
|
460
|
+
* whoever called it. That left the interesting half -- does the embedded closure
|
|
461
|
+
* actually rebuild this schema, and does the data then fit it -- outside the
|
|
462
|
+
* product, provable only by a script somebody ran once. The steps are ordered so
|
|
463
|
+
* that each one's failure is cheap: verify bytes, refuse a non-empty target,
|
|
464
|
+
* build the schema, check the schema is the one the bundle claims, and only then
|
|
465
|
+
* write data into it.
|
|
466
|
+
*
|
|
467
|
+
* The engine is injected rather than imported so this module does not depend on
|
|
468
|
+
* the bootstrap package, and so a caller can see exactly which engine ran.
|
|
469
|
+
*/
|
|
470
|
+
export async function restoreDisasterRecovery({
|
|
471
|
+
connectionString, bundleDirectory, anchorPath, basePolicy, applySchema, semanticIdentity, clientFactory,
|
|
472
|
+
expectedProviderContractSha256,
|
|
473
|
+
}) {
|
|
474
|
+
if (typeof applySchema !== 'function') throw new Error('disaster_recovery_engine_required')
|
|
475
|
+
if (typeof clientFactory !== 'function') throw new Error('disaster_recovery_client_factory_required')
|
|
476
|
+
if (!semanticIdentity) throw new Error('disaster_recovery_semantic_identity_required')
|
|
477
|
+
|
|
478
|
+
const expectedContract = expectedProviderContractSha256 ?? semanticIdentity?.providerContractSha256
|
|
479
|
+
const gate = await withClient(clientFactory, connectionString, (client) =>
|
|
480
|
+
restoreOwnedSet({
|
|
481
|
+
client, bundleDirectory, anchorPath, mode: 'empty-disaster-target',
|
|
482
|
+
expectedProviderContractSha256: expectedContract,
|
|
483
|
+
}))
|
|
484
|
+
const { manifest } = verifyBundle(bundleDirectory, anchorPath,
|
|
485
|
+
{ expectedProviderContractSha256: expectedContract })
|
|
486
|
+
const policy = bundleOnlyPolicy(basePolicy, manifest, bundleDirectory)
|
|
487
|
+
|
|
488
|
+
// The engine writes the ledgers as a side effect of applying the closure. That
|
|
489
|
+
// is the whole reason the bundle carries them as evidence instead of as rows.
|
|
490
|
+
const applied = await applySchema({ repoUrl: connectionString, workspaceRoot: bundleDirectory, policy })
|
|
491
|
+
if (applied?.lineageId !== manifest.identity.lineageId) {
|
|
492
|
+
throw new Error(`disaster_recovery_lineage_mismatch:expected=${manifest.identity.lineageId}:actual=${String(applied?.lineageId)}`)
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
const schemaFingerprint = await withClient(clientFactory, connectionString, async (client) => {
|
|
496
|
+
const projection = await semanticIdentity.readProjection(client)
|
|
497
|
+
return semanticIdentity.fingerprint(projection, manifest.identity.ownedRelations)
|
|
498
|
+
})
|
|
499
|
+
if (schemaFingerprint !== manifest.identity.ownedFingerprintSha256) {
|
|
500
|
+
throw new Error(`disaster_recovery_schema_mismatch:expected=${manifest.identity.ownedFingerprintSha256.slice(0, 12)}:actual=${schemaFingerprint.slice(0, 12)}`)
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
const loaded = await withClient(clientFactory, connectionString, (client) =>
|
|
504
|
+
restoreOwnedSet({
|
|
505
|
+
client, bundleDirectory, anchorPath, mode: 'existing-exact-predecessor',
|
|
506
|
+
confirmReplace: true, semanticIdentity,
|
|
507
|
+
}))
|
|
508
|
+
|
|
509
|
+
return {
|
|
510
|
+
mode: 'empty-disaster-target',
|
|
511
|
+
lineageId: applied.lineageId,
|
|
512
|
+
schemaAction: applied.migrationPlan?.action ?? null,
|
|
513
|
+
closureFiles: gate.closure,
|
|
514
|
+
relations: loaded.relations,
|
|
515
|
+
rows: loaded.rows,
|
|
516
|
+
sequencesVerified: loaded.sequencesVerified,
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
async function withClient(clientFactory, connectionString, body) {
|
|
521
|
+
const client = clientFactory(connectionString)
|
|
522
|
+
await client.connect()
|
|
523
|
+
try { return await body(client) } finally { await client.end() }
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
/**
|
|
527
|
+
* A callable entry, because the snapshot's attestation has to name a restore
|
|
528
|
+
* that works.
|
|
529
|
+
*
|
|
530
|
+
* The update seam writes "here is the backup, here is the command that puts it
|
|
531
|
+
* back". While the only restore path was pg_dump-shaped, switching the backup to
|
|
532
|
+
* a first-party bundle would have made that sentence false — the same class of
|
|
533
|
+
* untruth as a receipt that says "verified" without saying how. So the bundle
|
|
534
|
+
* gets a real way back before anything starts producing bundles.
|
|
535
|
+
*/
|
|
536
|
+
async function restoreFromCli(options) {
|
|
537
|
+
const repoUrl = process.env.AOPS_PG_URL
|
|
538
|
+
if (!repoUrl) throw new Error('owned_set_restore_pg_url_absent')
|
|
539
|
+
for (const required of ['bundleDirectory', 'receiptPath', 'mode']) {
|
|
540
|
+
if (!options[required]) throw new Error(`owned_set_restore_${required}_required`)
|
|
541
|
+
}
|
|
542
|
+
// Replacing live data is never inferred from the mode alone.
|
|
543
|
+
if (options.mode === 'existing-exact-predecessor' && options.confirm !== 'confirm-data-rewind') {
|
|
544
|
+
throw new Error('owned_set_restore_confirmation_required:confirm-data-rewind')
|
|
545
|
+
}
|
|
546
|
+
const { Client } = await import('pg')
|
|
547
|
+
const bootstrap = await import('@aopslabs/aops-pg-bootstrap')
|
|
548
|
+
const policy = JSON.parse(readFileSync(
|
|
549
|
+
options.policyPath ?? path.join(path.dirname(fileURLToPath(import.meta.url)),
|
|
550
|
+
'community-migration-policy-package-v1.json'), 'utf8'))
|
|
551
|
+
// The provider contract binding is resolved from the bundle, not from a flag.
|
|
552
|
+
//
|
|
553
|
+
// It used to come from `policy.lineages.find(l => l.id === options.lineageId)`,
|
|
554
|
+
// so omitting `--lineage-id` produced `undefined`, the `??` fallback found no
|
|
555
|
+
// `providerContractSha256` on the identity object either, and `verifyBundle`
|
|
556
|
+
// guards the comparison with `if (expectedProviderContractSha256 && ...)` --
|
|
557
|
+
// three steps, each individually reasonable, whose combined effect was that
|
|
558
|
+
// the default invocation skipped the check entirely. A bundle taken under a
|
|
559
|
+
// different provider contract restored without a word. The end-to-end run that
|
|
560
|
+
// "proved" restore never passed the flag, so it exercised the path with its own
|
|
561
|
+
// gate switched off.
|
|
562
|
+
//
|
|
563
|
+
// So the lineage comes from the manifest the bundle declares, the policy must
|
|
564
|
+
// carry a record for it, and that record must carry a contract. A caller that
|
|
565
|
+
// holds a reviewed expectation may still pass `--lineage-id`, but it has to
|
|
566
|
+
// agree rather than select.
|
|
567
|
+
const manifestLineageId = (() => {
|
|
568
|
+
let parsed
|
|
569
|
+
try {
|
|
570
|
+
parsed = JSON.parse(readFileSync(path.join(options.bundleDirectory, 'manifest.json'), 'utf8'))
|
|
571
|
+
} catch { throw new Error('owned_set_restore_manifest_unreadable') }
|
|
572
|
+
const id = parsed?.identity?.lineageId
|
|
573
|
+
if (typeof id !== 'string' || !id) throw new Error('owned_set_restore_manifest_lineage_absent')
|
|
574
|
+
return id
|
|
575
|
+
})()
|
|
576
|
+
if (options.lineageId && options.lineageId !== manifestLineageId) {
|
|
577
|
+
throw new Error(`owned_set_restore_lineage_id_disagrees:${options.lineageId}:${manifestLineageId}`)
|
|
578
|
+
}
|
|
579
|
+
const boundLineage = policy.lineages.find((lineage) => lineage.id === manifestLineageId)
|
|
580
|
+
if (!boundLineage) throw new Error(`owned_set_restore_lineage_not_in_policy:${manifestLineageId}`)
|
|
581
|
+
const expectedProviderContractSha256 = boundLineage.providerContractSha256
|
|
582
|
+
if (typeof expectedProviderContractSha256 !== 'string' || expectedProviderContractSha256.length !== 64) {
|
|
583
|
+
throw new Error(`owned_set_restore_policy_contract_absent:${manifestLineageId}`)
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
// Both call sites hand this the owned relation list, and both compare the
|
|
587
|
+
// result against the bundle's `ownedFingerprintSha256` -- so the projection is
|
|
588
|
+
// part of the identity, not an optimisation. Fingerprinting the raw projection
|
|
589
|
+
// instead measures every ambient object the database happens to carry, which
|
|
590
|
+
// made `existing-exact-predecessor` unable to match any real installation: the
|
|
591
|
+
// operator's own database has five relations outside the owned set.
|
|
592
|
+
const semanticIdentity = {
|
|
593
|
+
readProjection: (pgClient) => bootstrap.readCommunityStrictCatalogProjection(pgClient),
|
|
594
|
+
fingerprint: (projection, ownedRelations) => bootstrap.fingerprintCommunityStrictSemanticCatalog(
|
|
595
|
+
bootstrap.projectCommunityStrictSemanticCatalog(projection, ownedRelations)),
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
// Disaster recovery is a different operation, not a mode flag on this one.
|
|
599
|
+
// `restoreOwnedSet` deliberately stops after refusing a non-empty target and
|
|
600
|
+
// returns `verifiedOnly`, because applying the closure is the engine's job --
|
|
601
|
+
// so calling it alone for this mode verified the bundle, rebuilt nothing,
|
|
602
|
+
// loaded no rows, and exited zero with a result that reads like success. The
|
|
603
|
+
// engine has to be handed in here, which is also the only place that may
|
|
604
|
+
// import the bootstrap package.
|
|
605
|
+
if (options.mode === 'empty-disaster-target') {
|
|
606
|
+
return await restoreDisasterRecovery({
|
|
607
|
+
connectionString: repoUrl,
|
|
608
|
+
bundleDirectory: options.bundleDirectory,
|
|
609
|
+
anchorPath: options.receiptPath,
|
|
610
|
+
basePolicy: policy,
|
|
611
|
+
applySchema: (params) => bootstrap.applyCommunityStrictPgSchema(params),
|
|
612
|
+
clientFactory: (connectionString) => new Client({ connectionString }),
|
|
613
|
+
semanticIdentity,
|
|
614
|
+
expectedProviderContractSha256,
|
|
615
|
+
})
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
const client = new Client({ connectionString: repoUrl })
|
|
619
|
+
await client.connect()
|
|
620
|
+
try {
|
|
621
|
+
return await restoreOwnedSet({
|
|
622
|
+
client,
|
|
623
|
+
bundleDirectory: options.bundleDirectory,
|
|
624
|
+
anchorPath: options.receiptPath,
|
|
625
|
+
mode: options.mode,
|
|
626
|
+
confirmReplace: options.confirm === 'confirm-data-rewind',
|
|
627
|
+
semanticIdentity,
|
|
628
|
+
expectedProviderContractSha256,
|
|
629
|
+
})
|
|
630
|
+
} finally {
|
|
631
|
+
await client.end()
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
636
|
+
const value = (name) => {
|
|
637
|
+
const index = process.argv.indexOf(name)
|
|
638
|
+
return index >= 0 ? process.argv[index + 1] : undefined
|
|
639
|
+
}
|
|
640
|
+
restoreFromCli({
|
|
641
|
+
bundleDirectory: value('--bundle-dir'),
|
|
642
|
+
receiptPath: value('--receipt'),
|
|
643
|
+
mode: value('--mode'),
|
|
644
|
+
confirm: value('--confirm'),
|
|
645
|
+
lineageId: value('--lineage-id'),
|
|
646
|
+
policyPath: value('--policy'),
|
|
647
|
+
})
|
|
648
|
+
.then((result) => process.stdout.write(`${JSON.stringify(result, null, 2)}\n`))
|
|
649
|
+
.catch((error) => {
|
|
650
|
+
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
|
651
|
+
process.exit(1)
|
|
652
|
+
})
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
export { CONTRACT, RECEIPT_CONTRACT, restoreFromCli }
|