@forgezero/runtime 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -3
- package/dist/audit.js +35 -2
- package/dist/backup.d.ts +19 -4
- package/dist/backup.js +252 -45
- package/dist/finance/chain-deposits.js +5 -1
- package/dist/finance/chain-withdrawals.js +5 -1
- package/dist/finance/commission.js +5 -1
- package/dist/finance/ledger.d.ts +8 -2
- package/dist/finance/ledger.js +6 -1
- package/dist/jobs.d.ts +31 -1
- package/dist/jobs.js +145 -14
- package/dist/query.d.ts +47 -0
- package/dist/query.js +51 -0
- package/dist/queue.d.ts +26 -0
- package/dist/queue.js +36 -2
- package/dist/schema-typebox.d.ts +3 -0
- package/dist/schema-typebox.js +32 -2
- package/dist/schema.d.ts +2 -0
- package/dist/schema.js +15 -2
- package/package.json +10 -6
package/README.md
CHANGED
|
@@ -5,7 +5,7 @@ themselves, keyed queues, a transactional outbox, a hash-chained audit trail,
|
|
|
5
5
|
templated mail, encrypted backups, schema validation, and money that is never a
|
|
6
6
|
floating-point number.
|
|
7
7
|
|
|
8
|
-
Thirty-
|
|
8
|
+
Thirty-four public modules. Each is its own entry point, so you install one package and
|
|
9
9
|
your bundler includes only what you imported.
|
|
10
10
|
|
|
11
11
|
```bash
|
|
@@ -24,7 +24,7 @@ import { createScheduler } from '@forgezero/runtime/jobs';
|
|
|
24
24
|
|
|
25
25
|
| | |
|
|
26
26
|
|---|---|
|
|
27
|
-
| **Spine** | `jobs` `queue` `outbox` |
|
|
27
|
+
| **Spine** | `query` `jobs` `queue` `outbox` |
|
|
28
28
|
| **Record** | `audit` `backup` `compliance` `calendar` |
|
|
29
29
|
| **Identity** | `identity` `totp` `notify` `notify/templates` `schema` `schema/typebox` |
|
|
30
30
|
| **Finance** | `finance/money` `finance/storage` `finance/ledger` `finance/commission` `finance/rates` `finance/transfers` `finance/tax` `finance/chain` `finance/custody` `finance/derive` `finance/venues` `finance/binance` |
|
|
@@ -32,6 +32,28 @@ import { createScheduler } from '@forgezero/runtime/jobs';
|
|
|
32
32
|
None of it knows what your product does. You bind your own rules to these; you
|
|
33
33
|
do not fork them.
|
|
34
34
|
|
|
35
|
+
## Typed function queries
|
|
36
|
+
|
|
37
|
+
`query` binds typed input/output codecs to an ordinary function and any context
|
|
38
|
+
you choose. It has no database or ForgeZero realm dependency. Inputs may decode
|
|
39
|
+
wire values; outputs are checked strictly so a wrong handler result cannot be
|
|
40
|
+
coerced into looking correct.
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
import { defineQuery, implementQuery } from '@forgezero/runtime/query';
|
|
44
|
+
import { typeboxQueryCodec, T } from '@forgezero/runtime/schema/typebox';
|
|
45
|
+
|
|
46
|
+
const contract = defineQuery({
|
|
47
|
+
name: 'invoice.by-reference',
|
|
48
|
+
input: typeboxQueryCodec(T.Object({ reference: T.String() })),
|
|
49
|
+
output: typeboxQueryCodec(T.Object({ total: T.String() }))
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
const findInvoice = implementQuery(contract, async (services: MyServices, input, { signal }) =>
|
|
53
|
+
services.invoices.find(input.reference, { signal })
|
|
54
|
+
);
|
|
55
|
+
```
|
|
56
|
+
|
|
35
57
|
## One reusable async queue
|
|
36
58
|
|
|
37
59
|
`queue` is deliberately memory-only: it accepts a function, returns that
|
|
@@ -43,7 +65,9 @@ ArangoDB unique claim, while another caller may use any database or no database.
|
|
|
43
65
|
```ts
|
|
44
66
|
import { createQueue } from '@forgezero/runtime/queue';
|
|
45
67
|
|
|
46
|
-
|
|
68
|
+
// Default admission is 60% of reported logical CPUs (with one reserved).
|
|
69
|
+
// Use `width` for an exact ceiling, or an explicit dynamic resource policy:
|
|
70
|
+
const queue = createQueue({ resources: { percent: 60, reserve: 1, max: 32 } });
|
|
47
71
|
const task = queue.run('tenant-a:wallet-7', transfer, amount, destination);
|
|
48
72
|
const receipt = await task.result;
|
|
49
73
|
|
|
@@ -55,6 +79,25 @@ queue.cancel(task.id); // pending task only
|
|
|
55
79
|
await queue.stop(30_000); // close intake and drain all work
|
|
56
80
|
```
|
|
57
81
|
|
|
82
|
+
Different async keys overlap immediately. CPU-heavy JavaScript does not become
|
|
83
|
+
multi-core merely by entering a queue: put that handler in Bun/standard Workers
|
|
84
|
+
and await the Worker result from the queue.
|
|
85
|
+
|
|
86
|
+
Jobs accept intervals down to seconds or a local wall-clock schedule with an
|
|
87
|
+
IANA timezone and weekday filter. `overlap: 'wait'` (the default) schedules the
|
|
88
|
+
next run after completion; `overlap: 'skip'` keeps clock cadence and drops a tick
|
|
89
|
+
when the same key is still busy. Same-key overlap is never allowed.
|
|
90
|
+
|
|
91
|
+
```ts
|
|
92
|
+
defineJob({
|
|
93
|
+
key: 'tenant:acme:invoice',
|
|
94
|
+
label: 'Monthly invoice preparation',
|
|
95
|
+
schedule: { timezone: 'Asia/Kolkata', time: '00:00:15', weekdays: [1] },
|
|
96
|
+
overlap: 'skip',
|
|
97
|
+
run: async ({ signal }) => generateInvoices({ signal })
|
|
98
|
+
});
|
|
99
|
+
```
|
|
100
|
+
|
|
58
101
|
## Three things worth knowing before you use it
|
|
59
102
|
|
|
60
103
|
**Money is never a number.** An amount is minor units as a `bigint` with its
|
package/dist/audit.js
CHANGED
|
@@ -33,10 +33,41 @@ var DEFAULT_RETRY = {
|
|
|
33
33
|
attempts: 1,
|
|
34
34
|
backoffMs: (attempt) => Math.min(30000, 2 ** attempt * 100)
|
|
35
35
|
};
|
|
36
|
+
var reportedParallelism = () => {
|
|
37
|
+
const reported = globalThis.navigator?.hardwareConcurrency;
|
|
38
|
+
return Number.isSafeInteger(reported) && reported > 0 ? reported : 1;
|
|
39
|
+
};
|
|
40
|
+
function queueWidthFor(policy = {}) {
|
|
41
|
+
const percent = policy.percent ?? 60;
|
|
42
|
+
const reserve = policy.reserve ?? 1;
|
|
43
|
+
const min = policy.min ?? 1;
|
|
44
|
+
const max = policy.max ?? Number.MAX_SAFE_INTEGER;
|
|
45
|
+
const available = (policy.available ?? reportedParallelism)();
|
|
46
|
+
if (!Number.isFinite(percent) || percent <= 0 || percent > 100) {
|
|
47
|
+
throw new RangeError("queue: resource percent must be greater than 0 and at most 100");
|
|
48
|
+
}
|
|
49
|
+
for (const [name, value] of [["reserve", reserve], ["min", min], ["max", max]]) {
|
|
50
|
+
if (!Number.isSafeInteger(value) || value < (name === "reserve" ? 0 : 1)) {
|
|
51
|
+
throw new RangeError(`queue: resource ${name} must be ${name === "reserve" ? "a non-negative" : "a positive"} integer`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
if (!Number.isSafeInteger(available) || available < 1) {
|
|
55
|
+
throw new RangeError("queue: available parallelism must be a positive integer");
|
|
56
|
+
}
|
|
57
|
+
if (min > max)
|
|
58
|
+
throw new RangeError("queue: resource min cannot exceed max");
|
|
59
|
+
const usable = Math.max(1, available - reserve);
|
|
60
|
+
return Math.min(max, Math.max(min, Math.floor(usable * percent / 100)));
|
|
61
|
+
}
|
|
36
62
|
function createQueue(options = {}) {
|
|
37
|
-
|
|
63
|
+
if (options.width !== undefined && options.resources !== undefined) {
|
|
64
|
+
throw new Error("queue: choose either width or resources, not both");
|
|
65
|
+
}
|
|
66
|
+
const configuredWidth = options.width;
|
|
67
|
+
const widthNow = () => configuredWidth ?? queueWidthFor(options.resources);
|
|
68
|
+
const initialWidth = widthNow();
|
|
38
69
|
const retry = { ...DEFAULT_RETRY, ...options.retry };
|
|
39
|
-
if (!Number.isSafeInteger(
|
|
70
|
+
if (!Number.isSafeInteger(initialWidth) || initialWidth < 1) {
|
|
40
71
|
throw new RangeError("queue: width must be a positive integer");
|
|
41
72
|
}
|
|
42
73
|
if (!Number.isSafeInteger(retry.attempts) || retry.attempts < 1) {
|
|
@@ -68,6 +99,7 @@ function createQueue(options = {}) {
|
|
|
68
99
|
announceIdle();
|
|
69
100
|
return;
|
|
70
101
|
}
|
|
102
|
+
const width = widthNow();
|
|
71
103
|
for (const [key, lane] of lanes) {
|
|
72
104
|
if (running.size >= width)
|
|
73
105
|
break;
|
|
@@ -215,6 +247,7 @@ function createQueue(options = {}) {
|
|
|
215
247
|
for (const lane of lanes.values())
|
|
216
248
|
queued += lane.length;
|
|
217
249
|
return {
|
|
250
|
+
width: widthNow(),
|
|
218
251
|
running: running.size,
|
|
219
252
|
queued,
|
|
220
253
|
keys: lanes.size,
|
package/dist/backup.d.ts
CHANGED
|
@@ -32,11 +32,11 @@
|
|
|
32
32
|
* refuse rather than doing their best.
|
|
33
33
|
*/
|
|
34
34
|
export declare class BackupError extends Error {
|
|
35
|
-
readonly code: 'CHUNK_MISSING' | 'DIGEST_MISMATCH' | 'REALM_MISMATCH' | 'BAD_MANIFEST' | 'UNSUPPORTED_FORMAT' | 'DECRYPT_FAILED' | 'EMPTY_SNAPSHOT' | 'RETENTION_WOULD_EMPTY';
|
|
36
|
-
constructor(code: 'CHUNK_MISSING' | 'DIGEST_MISMATCH' | 'REALM_MISMATCH' | 'BAD_MANIFEST' | 'UNSUPPORTED_FORMAT' | 'DECRYPT_FAILED' | 'EMPTY_SNAPSHOT' | 'RETENTION_WOULD_EMPTY', message: string);
|
|
35
|
+
readonly code: 'CHUNK_MISSING' | 'DIGEST_MISMATCH' | 'REALM_MISMATCH' | 'BAD_MANIFEST' | 'UNSUPPORTED_FORMAT' | 'DECRYPT_FAILED' | 'MAINTENANCE_REQUIRED' | 'EMPTY_SNAPSHOT' | 'RETENTION_WOULD_EMPTY';
|
|
36
|
+
constructor(code: 'CHUNK_MISSING' | 'DIGEST_MISMATCH' | 'REALM_MISMATCH' | 'BAD_MANIFEST' | 'UNSUPPORTED_FORMAT' | 'DECRYPT_FAILED' | 'MAINTENANCE_REQUIRED' | 'EMPTY_SNAPSHOT' | 'RETENTION_WOULD_EMPTY', message: string);
|
|
37
37
|
}
|
|
38
38
|
/** Bumped only for a change that an older reader cannot handle. */
|
|
39
|
-
export declare const SNAPSHOT_FORMAT =
|
|
39
|
+
export declare const SNAPSHOT_FORMAT = 2;
|
|
40
40
|
export interface ChunkRecord {
|
|
41
41
|
index: number;
|
|
42
42
|
/** Object key in the store. */
|
|
@@ -55,7 +55,11 @@ export interface SnapshotManifest {
|
|
|
55
55
|
collections: Record<string, number>;
|
|
56
56
|
chunks: readonly ChunkRecord[];
|
|
57
57
|
totalRows: number;
|
|
58
|
-
/**
|
|
58
|
+
/**
|
|
59
|
+
* HMAC-SHA-256 over the canonical manifest body. Despite the legacy field
|
|
60
|
+
* name this is a keyed authenticator, not a plain digest. Format 1 used an
|
|
61
|
+
* unkeyed digest and is deliberately not accepted by this reader.
|
|
62
|
+
*/
|
|
59
63
|
sealDigest: string;
|
|
60
64
|
/** Free-form: schema version, application version, who triggered it. */
|
|
61
65
|
labels?: Record<string, string>;
|
|
@@ -76,8 +80,14 @@ export interface ObjectStore {
|
|
|
76
80
|
}
|
|
77
81
|
/** Where the rows come from. Async-iterable so a large collection never lands in memory whole. */
|
|
78
82
|
export interface RowSource {
|
|
83
|
+
/** Acquire a point-in-time source transaction or equivalent read fence. */
|
|
84
|
+
begin?(): Promise<void>;
|
|
79
85
|
collections(): Promise<string[]>;
|
|
80
86
|
rows(collection: string): AsyncIterable<Record<string, unknown>>;
|
|
87
|
+
/** Release the read fence after every row has been captured. */
|
|
88
|
+
commit?(): Promise<void>;
|
|
89
|
+
/** Release it after an interrupted or failed capture. */
|
|
90
|
+
abort?(): Promise<void>;
|
|
81
91
|
}
|
|
82
92
|
/** Where they go on the way back. */
|
|
83
93
|
export interface RowSink {
|
|
@@ -90,7 +100,10 @@ export interface RowSink {
|
|
|
90
100
|
write(collection: string, rows: Record<string, unknown>[]): Promise<void>;
|
|
91
101
|
/** Called before anything is written. The sink's chance to refuse or truncate. */
|
|
92
102
|
begin?(manifest: SnapshotManifest): Promise<void>;
|
|
103
|
+
/** Commit all staged rows atomically. */
|
|
93
104
|
finish?(manifest: SnapshotManifest): Promise<void>;
|
|
105
|
+
/** Roll back all staged rows after any write or commit failure. */
|
|
106
|
+
abort?(manifest: SnapshotManifest): Promise<void>;
|
|
94
107
|
}
|
|
95
108
|
export interface SnapshotOptions {
|
|
96
109
|
store: ObjectStore;
|
|
@@ -174,6 +187,7 @@ export declare function restore(options: RestoreOptions): Promise<RestoreReport>
|
|
|
174
187
|
export declare function listSnapshots(args: {
|
|
175
188
|
store: ObjectStore;
|
|
176
189
|
realm: string;
|
|
190
|
+
masterSeed: Uint8Array;
|
|
177
191
|
prefix?: string;
|
|
178
192
|
}): Promise<SnapshotManifest[]>;
|
|
179
193
|
export interface RetentionPolicy {
|
|
@@ -200,6 +214,7 @@ export declare function selectForDeletion(manifests: readonly SnapshotManifest[]
|
|
|
200
214
|
export declare function prune(args: {
|
|
201
215
|
store: ObjectStore;
|
|
202
216
|
realm: string;
|
|
217
|
+
masterSeed: Uint8Array;
|
|
203
218
|
policy?: RetentionPolicy;
|
|
204
219
|
prefix?: string;
|
|
205
220
|
/** Report what would go without removing it. */
|
package/dist/backup.js
CHANGED
|
@@ -7,7 +7,14 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
|
|
|
7
7
|
});
|
|
8
8
|
|
|
9
9
|
// src/backup.ts
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
hkdf,
|
|
12
|
+
hmacSha256,
|
|
13
|
+
seal,
|
|
14
|
+
open,
|
|
15
|
+
sha256,
|
|
16
|
+
timingSafeEqual
|
|
17
|
+
} from "@forgezero/access/security";
|
|
11
18
|
|
|
12
19
|
class BackupError extends Error {
|
|
13
20
|
code;
|
|
@@ -17,15 +24,145 @@ class BackupError extends Error {
|
|
|
17
24
|
this.name = "BackupError";
|
|
18
25
|
}
|
|
19
26
|
}
|
|
20
|
-
var SNAPSHOT_FORMAT =
|
|
27
|
+
var SNAPSHOT_FORMAT = 2;
|
|
21
28
|
var keyFor = (masterSeed, realm, snapshotId) => hkdf(masterSeed, `forgezero:backup:v1:${realm}:${snapshotId}`, 32);
|
|
29
|
+
var manifestKeyFor = (masterSeed, realm, snapshotId) => hkdf(masterSeed, `forgezero:backup:manifest:v2:${realm}:${snapshotId}`, 32);
|
|
22
30
|
var aadFor = (realm, snapshotId, index) => `backup:${realm}:${snapshotId}:${index}`;
|
|
23
31
|
var objectKey = (prefix, realm, id, part) => `${prefix}${realm}/${id}/${part}`;
|
|
32
|
+
function canonicalManifest(manifest) {
|
|
33
|
+
return JSON.stringify([
|
|
34
|
+
manifest.format,
|
|
35
|
+
manifest.id,
|
|
36
|
+
manifest.realm,
|
|
37
|
+
manifest.createdAtMs,
|
|
38
|
+
Object.entries(manifest.collections).sort(([left], [right]) => left.localeCompare(right)),
|
|
39
|
+
manifest.chunks.map((chunk) => [
|
|
40
|
+
chunk.index,
|
|
41
|
+
chunk.object,
|
|
42
|
+
chunk.digest,
|
|
43
|
+
chunk.rows,
|
|
44
|
+
chunk.bytes
|
|
45
|
+
]),
|
|
46
|
+
manifest.totalRows,
|
|
47
|
+
manifest.labels ? Object.entries(manifest.labels).sort(([left], [right]) => left.localeCompare(right)) : null
|
|
48
|
+
]);
|
|
49
|
+
}
|
|
50
|
+
async function sealManifest(manifest, masterSeed) {
|
|
51
|
+
const key = await manifestKeyFor(masterSeed, manifest.realm, manifest.id);
|
|
52
|
+
return hmacSha256(key, canonicalManifest(manifest));
|
|
53
|
+
}
|
|
54
|
+
var MANIFEST_KEYS = new Set([
|
|
55
|
+
"format",
|
|
56
|
+
"id",
|
|
57
|
+
"realm",
|
|
58
|
+
"createdAtMs",
|
|
59
|
+
"collections",
|
|
60
|
+
"chunks",
|
|
61
|
+
"totalRows",
|
|
62
|
+
"sealDigest",
|
|
63
|
+
"labels"
|
|
64
|
+
]);
|
|
65
|
+
var isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
66
|
+
function manifestProblem(value) {
|
|
67
|
+
if (!isRecord(value))
|
|
68
|
+
return "The manifest is not an object.";
|
|
69
|
+
if (Object.keys(value).some((key) => !MANIFEST_KEYS.has(key)))
|
|
70
|
+
return "The manifest has unknown fields.";
|
|
71
|
+
if (value.format !== SNAPSHOT_FORMAT) {
|
|
72
|
+
return `Snapshot format ${String(value.format)} is not supported by this reader (${SNAPSHOT_FORMAT}).`;
|
|
73
|
+
}
|
|
74
|
+
if (typeof value.id !== "string" || !/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(value.id)) {
|
|
75
|
+
return "The snapshot id is invalid.";
|
|
76
|
+
}
|
|
77
|
+
if (typeof value.realm !== "string" || !/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(value.realm)) {
|
|
78
|
+
return "The snapshot realm is invalid.";
|
|
79
|
+
}
|
|
80
|
+
if (!Number.isSafeInteger(value.createdAtMs) || value.createdAtMs <= 0) {
|
|
81
|
+
return "The snapshot creation time is invalid.";
|
|
82
|
+
}
|
|
83
|
+
if (!isRecord(value.collections) || Object.keys(value.collections).length === 0) {
|
|
84
|
+
return "The manifest has no collection inventory.";
|
|
85
|
+
}
|
|
86
|
+
for (const [collection, count] of Object.entries(value.collections)) {
|
|
87
|
+
if (!/^[a-z][a-z0-9_]*$/.test(collection) || !Number.isSafeInteger(count) || count < 0) {
|
|
88
|
+
return `The collection inventory entry "${collection}" is invalid.`;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
if (!Array.isArray(value.chunks) || value.chunks.length === 0)
|
|
92
|
+
return "The manifest has no chunks.";
|
|
93
|
+
if (!Number.isSafeInteger(value.totalRows) || value.totalRows <= 0) {
|
|
94
|
+
return "The manifest row total is invalid.";
|
|
95
|
+
}
|
|
96
|
+
if (typeof value.sealDigest !== "string" || !/^[a-f0-9]{64}$/.test(value.sealDigest)) {
|
|
97
|
+
return "The manifest authenticator is invalid.";
|
|
98
|
+
}
|
|
99
|
+
if (value.labels !== undefined) {
|
|
100
|
+
if (!isRecord(value.labels))
|
|
101
|
+
return "The manifest labels are invalid.";
|
|
102
|
+
for (const [label, content] of Object.entries(value.labels)) {
|
|
103
|
+
if (label.length === 0 || label.length > 128 || typeof content !== "string" || content.length > 1024) {
|
|
104
|
+
return `The manifest label "${label}" is invalid.`;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
let chunkRows = 0;
|
|
109
|
+
let root;
|
|
110
|
+
for (const [position, candidate] of value.chunks.entries()) {
|
|
111
|
+
if (!isRecord(candidate))
|
|
112
|
+
return `Chunk ${position} is not an object.`;
|
|
113
|
+
const keys = Object.keys(candidate).sort().join(",");
|
|
114
|
+
if (keys !== "bytes,digest,index,object,rows")
|
|
115
|
+
return `Chunk ${position} has an invalid shape.`;
|
|
116
|
+
if (candidate.index !== position)
|
|
117
|
+
return `Chunk indexes are not contiguous at position ${position}.`;
|
|
118
|
+
if (typeof candidate.object !== "string" || candidate.object.length > 1024) {
|
|
119
|
+
return `Chunk ${position} has an invalid object key.`;
|
|
120
|
+
}
|
|
121
|
+
const filename = `chunk-${String(position).padStart(5, "0")}.json`;
|
|
122
|
+
const suffix = `${value.realm}/${value.id}/${filename}`;
|
|
123
|
+
if (!candidate.object.endsWith(suffix))
|
|
124
|
+
return `Chunk ${position} is outside the snapshot prefix.`;
|
|
125
|
+
const candidateRoot = candidate.object.slice(0, -filename.length);
|
|
126
|
+
root ??= candidateRoot;
|
|
127
|
+
if (candidateRoot !== root)
|
|
128
|
+
return `Chunk ${position} does not share the snapshot prefix.`;
|
|
129
|
+
if (typeof candidate.digest !== "string" || !/^[a-f0-9]{64}$/.test(candidate.digest)) {
|
|
130
|
+
return `Chunk ${position} has an invalid digest.`;
|
|
131
|
+
}
|
|
132
|
+
if (!Number.isSafeInteger(candidate.rows) || candidate.rows <= 0) {
|
|
133
|
+
return `Chunk ${position} has an invalid row count.`;
|
|
134
|
+
}
|
|
135
|
+
if (!Number.isSafeInteger(candidate.bytes) || candidate.bytes <= 0) {
|
|
136
|
+
return `Chunk ${position} has an invalid byte count.`;
|
|
137
|
+
}
|
|
138
|
+
chunkRows += candidate.rows;
|
|
139
|
+
}
|
|
140
|
+
if (chunkRows !== value.totalRows)
|
|
141
|
+
return "Chunk row counts do not equal the manifest total.";
|
|
142
|
+
const collectionRows = Object.values(value.collections).reduce((sum, count) => sum + count, 0);
|
|
143
|
+
if (collectionRows !== value.totalRows)
|
|
144
|
+
return "Collection row counts do not equal the manifest total.";
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
24
147
|
async function snapshot(options) {
|
|
25
148
|
const now = options.now ?? Date.now;
|
|
26
149
|
const chunkRows = options.chunkRows ?? 1000;
|
|
27
150
|
const prefix = options.prefix ?? "snapshots/";
|
|
28
|
-
const
|
|
151
|
+
const nonce = crypto.getRandomValues(new Uint8Array(8));
|
|
152
|
+
const randomSuffix = Array.from(nonce, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
153
|
+
const id = options.id ?? `snap_${new Date(now()).toISOString().replace(/[-:.TZ]/g, "")}_${randomSuffix}`;
|
|
154
|
+
if (!Number.isSafeInteger(chunkRows) || chunkRows <= 0) {
|
|
155
|
+
throw new BackupError("BAD_MANIFEST", "chunkRows must be a positive safe integer.");
|
|
156
|
+
}
|
|
157
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(options.realm)) {
|
|
158
|
+
throw new BackupError("BAD_MANIFEST", "The snapshot realm is not safe for an object key.");
|
|
159
|
+
}
|
|
160
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(id)) {
|
|
161
|
+
throw new BackupError("BAD_MANIFEST", "The snapshot id is not safe for an object key.");
|
|
162
|
+
}
|
|
163
|
+
if (!prefix || prefix.includes("..") || prefix.startsWith("/") || !prefix.endsWith("/")) {
|
|
164
|
+
throw new BackupError("BAD_MANIFEST", "The snapshot prefix must be a relative object prefix ending in /.");
|
|
165
|
+
}
|
|
29
166
|
const key = await keyFor(options.masterSeed, options.realm, id);
|
|
30
167
|
const chunks = [];
|
|
31
168
|
const collections = {};
|
|
@@ -47,26 +184,38 @@ async function snapshot(options) {
|
|
|
47
184
|
bytes: body.length
|
|
48
185
|
});
|
|
49
186
|
};
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
for
|
|
54
|
-
batch
|
|
55
|
-
count
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
187
|
+
let sourceFinalized = false;
|
|
188
|
+
try {
|
|
189
|
+
await options.source.begin?.();
|
|
190
|
+
for (const collection of await options.source.collections()) {
|
|
191
|
+
let batch = [];
|
|
192
|
+
let count = 0;
|
|
193
|
+
for await (const row of options.source.rows(collection)) {
|
|
194
|
+
batch.push(row);
|
|
195
|
+
count += 1;
|
|
196
|
+
if (batch.length >= chunkRows) {
|
|
197
|
+
await flush(collection, batch);
|
|
198
|
+
batch = [];
|
|
199
|
+
}
|
|
59
200
|
}
|
|
201
|
+
await flush(collection, batch);
|
|
202
|
+
collections[collection] = count;
|
|
203
|
+
totalRows += count;
|
|
204
|
+
options.onProgress?.({ collection, rows: count, chunks: chunks.length });
|
|
60
205
|
}
|
|
61
|
-
await
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
206
|
+
await options.source.commit?.();
|
|
207
|
+
sourceFinalized = true;
|
|
208
|
+
} catch (cause) {
|
|
209
|
+
if (!sourceFinalized)
|
|
210
|
+
await options.source.abort?.().catch(() => {
|
|
211
|
+
return;
|
|
212
|
+
});
|
|
213
|
+
throw cause;
|
|
65
214
|
}
|
|
66
215
|
if (totalRows === 0) {
|
|
67
216
|
throw new BackupError("EMPTY_SNAPSHOT", "The source produced no rows. Refusing to write an empty snapshot.");
|
|
68
217
|
}
|
|
69
|
-
const
|
|
218
|
+
const unsigned = {
|
|
70
219
|
format: SNAPSHOT_FORMAT,
|
|
71
220
|
id,
|
|
72
221
|
realm: options.realm,
|
|
@@ -74,9 +223,12 @@ async function snapshot(options) {
|
|
|
74
223
|
collections,
|
|
75
224
|
chunks,
|
|
76
225
|
totalRows,
|
|
77
|
-
sealDigest: await sha256(chunks.map((chunk) => chunk.digest).join("")),
|
|
78
226
|
...options.labels ? { labels: options.labels } : {}
|
|
79
227
|
};
|
|
228
|
+
const manifest = {
|
|
229
|
+
...unsigned,
|
|
230
|
+
sealDigest: await sealManifest(unsigned, options.masterSeed)
|
|
231
|
+
};
|
|
80
232
|
await options.store.putObject({
|
|
81
233
|
key: objectKey(prefix, options.realm, id, "manifest.json"),
|
|
82
234
|
body: JSON.stringify(manifest, null, 2),
|
|
@@ -86,9 +238,9 @@ async function snapshot(options) {
|
|
|
86
238
|
}
|
|
87
239
|
async function verifySnapshot(args) {
|
|
88
240
|
const problems = [];
|
|
89
|
-
const key = await keyFor(args.masterSeed, args.manifest.realm, args.manifest.id);
|
|
90
241
|
let rowsChecked = 0;
|
|
91
|
-
|
|
242
|
+
const invalid = manifestProblem(args.manifest);
|
|
243
|
+
if (invalid) {
|
|
92
244
|
return {
|
|
93
245
|
ok: false,
|
|
94
246
|
id: args.manifest.id,
|
|
@@ -97,12 +249,25 @@ async function verifySnapshot(args) {
|
|
|
97
249
|
problems: [
|
|
98
250
|
{
|
|
99
251
|
chunk: -1,
|
|
100
|
-
code: "UNSUPPORTED_FORMAT",
|
|
101
|
-
message:
|
|
252
|
+
code: args.manifest.format !== SNAPSHOT_FORMAT ? "UNSUPPORTED_FORMAT" : "BAD_MANIFEST",
|
|
253
|
+
message: invalid
|
|
102
254
|
}
|
|
103
255
|
]
|
|
104
256
|
};
|
|
105
257
|
}
|
|
258
|
+
const { sealDigest, ...unsigned } = args.manifest;
|
|
259
|
+
const expectedSeal = await sealManifest(unsigned, args.masterSeed);
|
|
260
|
+
if (!timingSafeEqual(expectedSeal, sealDigest)) {
|
|
261
|
+
return {
|
|
262
|
+
ok: false,
|
|
263
|
+
id: args.manifest.id,
|
|
264
|
+
chunksChecked: 0,
|
|
265
|
+
rowsChecked: 0,
|
|
266
|
+
problems: [{ chunk: -1, code: "BAD_MANIFEST", message: "The manifest did not authenticate." }]
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
const key = await keyFor(args.masterSeed, args.manifest.realm, args.manifest.id);
|
|
270
|
+
const collectionRows = {};
|
|
106
271
|
for (const chunk of args.manifest.chunks) {
|
|
107
272
|
let raw;
|
|
108
273
|
try {
|
|
@@ -135,15 +300,34 @@ async function verifySnapshot(args) {
|
|
|
135
300
|
});
|
|
136
301
|
continue;
|
|
137
302
|
}
|
|
138
|
-
|
|
303
|
+
let parsed;
|
|
304
|
+
try {
|
|
305
|
+
parsed = JSON.parse(plaintext);
|
|
306
|
+
} catch {
|
|
307
|
+
problems.push({ chunk: chunk.index, code: "BAD_MANIFEST", message: `${chunk.object} is not JSON.` });
|
|
308
|
+
continue;
|
|
309
|
+
}
|
|
310
|
+
if (!isRecord(parsed) || typeof parsed.collection !== "string" || !Object.hasOwn(args.manifest.collections, parsed.collection) || !Array.isArray(parsed.rows) || parsed.rows.length !== chunk.rows || parsed.rows.some((row) => !isRecord(row))) {
|
|
311
|
+
problems.push({
|
|
312
|
+
chunk: chunk.index,
|
|
313
|
+
code: "BAD_MANIFEST",
|
|
314
|
+
message: `${chunk.object} does not match its declared collection and row count.`
|
|
315
|
+
});
|
|
316
|
+
continue;
|
|
317
|
+
}
|
|
318
|
+
collectionRows[parsed.collection] = (collectionRows[parsed.collection] ?? 0) + parsed.rows.length;
|
|
319
|
+
rowsChecked += parsed.rows.length;
|
|
139
320
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
321
|
+
if (problems.length === 0) {
|
|
322
|
+
for (const [collection, expectedRows] of Object.entries(args.manifest.collections)) {
|
|
323
|
+
if ((collectionRows[collection] ?? 0) === expectedRows)
|
|
324
|
+
continue;
|
|
325
|
+
problems.push({
|
|
326
|
+
chunk: -1,
|
|
327
|
+
code: "BAD_MANIFEST",
|
|
328
|
+
message: `Collection ${collection} restored ${collectionRows[collection] ?? 0} rows, expected ${expectedRows}.`
|
|
329
|
+
});
|
|
330
|
+
}
|
|
147
331
|
}
|
|
148
332
|
return {
|
|
149
333
|
ok: problems.length === 0,
|
|
@@ -158,8 +342,8 @@ async function restore(options) {
|
|
|
158
342
|
if (manifest.realm !== options.realm) {
|
|
159
343
|
throw new BackupError("REALM_MISMATCH", `This snapshot belongs to realm "${manifest.realm}" and cannot be restored into "${options.realm}".`);
|
|
160
344
|
}
|
|
161
|
-
if (manifest.format
|
|
162
|
-
throw new BackupError("UNSUPPORTED_FORMAT", `Snapshot format ${manifest.format} is
|
|
345
|
+
if (manifest.format !== SNAPSHOT_FORMAT) {
|
|
346
|
+
throw new BackupError("UNSUPPORTED_FORMAT", `Snapshot format ${manifest.format} is not supported by this reader (${SNAPSHOT_FORMAT}).`);
|
|
163
347
|
}
|
|
164
348
|
if (!options.skipVerify) {
|
|
165
349
|
const report = await verifySnapshot({
|
|
@@ -172,21 +356,31 @@ async function restore(options) {
|
|
|
172
356
|
throw new BackupError(first.code, `Refusing to restore: ${report.problems.length} problem(s), first at chunk ${first.chunk} — ${first.message}`);
|
|
173
357
|
}
|
|
174
358
|
}
|
|
175
|
-
await options.sink.begin?.(manifest);
|
|
176
359
|
const key = await keyFor(options.masterSeed, manifest.realm, manifest.id);
|
|
177
360
|
const collections = {};
|
|
178
361
|
let rowsRestored = 0;
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
362
|
+
let sinkFinalized = false;
|
|
363
|
+
try {
|
|
364
|
+
await options.sink.begin?.(manifest);
|
|
365
|
+
for (const chunk of manifest.chunks) {
|
|
366
|
+
const raw = await options.store.getObject(chunk.object);
|
|
367
|
+
const sealed = JSON.parse(new TextDecoder().decode(raw));
|
|
368
|
+
const plaintext = await open(key, sealed, aadFor(manifest.realm, manifest.id, chunk.index));
|
|
369
|
+
const { collection, rows } = JSON.parse(plaintext);
|
|
370
|
+
await options.sink.write(collection, rows);
|
|
371
|
+
collections[collection] = (collections[collection] ?? 0) + rows.length;
|
|
372
|
+
rowsRestored += rows.length;
|
|
373
|
+
options.onProgress?.({ chunk: chunk.index + 1, of: manifest.chunks.length, rows: rowsRestored });
|
|
374
|
+
}
|
|
375
|
+
await options.sink.finish?.(manifest);
|
|
376
|
+
sinkFinalized = true;
|
|
377
|
+
} catch (cause) {
|
|
378
|
+
if (!sinkFinalized)
|
|
379
|
+
await options.sink.abort?.(manifest).catch(() => {
|
|
380
|
+
return;
|
|
381
|
+
});
|
|
382
|
+
throw cause;
|
|
188
383
|
}
|
|
189
|
-
await options.sink.finish?.(manifest);
|
|
190
384
|
return { id: manifest.id, rowsRestored, collections };
|
|
191
385
|
}
|
|
192
386
|
async function listSnapshots(args) {
|
|
@@ -197,7 +391,14 @@ async function listSnapshots(args) {
|
|
|
197
391
|
if (!object.key.endsWith("/manifest.json"))
|
|
198
392
|
continue;
|
|
199
393
|
try {
|
|
200
|
-
|
|
394
|
+
const manifest = JSON.parse(new TextDecoder().decode(await args.store.getObject(object.key)));
|
|
395
|
+
const invalid = manifestProblem(manifest);
|
|
396
|
+
if (invalid || manifest.realm !== args.realm)
|
|
397
|
+
continue;
|
|
398
|
+
const { sealDigest, ...unsigned } = manifest;
|
|
399
|
+
if (!timingSafeEqual(await sealManifest(unsigned, args.masterSeed), sealDigest))
|
|
400
|
+
continue;
|
|
401
|
+
manifests.push(manifest);
|
|
201
402
|
} catch {}
|
|
202
403
|
}
|
|
203
404
|
return manifests.sort((a, b) => b.createdAtMs - a.createdAtMs);
|
|
@@ -245,7 +446,12 @@ function selectForDeletion(manifests, policy = DEFAULT_RETENTION) {
|
|
|
245
446
|
return newestFirst.filter((manifest) => !keep.has(manifest.id));
|
|
246
447
|
}
|
|
247
448
|
async function prune(args) {
|
|
248
|
-
const manifests = await listSnapshots({
|
|
449
|
+
const manifests = await listSnapshots({
|
|
450
|
+
store: args.store,
|
|
451
|
+
realm: args.realm,
|
|
452
|
+
masterSeed: args.masterSeed,
|
|
453
|
+
prefix: args.prefix
|
|
454
|
+
});
|
|
249
455
|
const doomed = selectForDeletion(manifests, args.policy);
|
|
250
456
|
if (doomed.length >= manifests.length) {
|
|
251
457
|
throw new BackupError("RETENTION_WOULD_EMPTY", "This policy would delete every snapshot. Refusing.");
|
|
@@ -276,6 +482,7 @@ function backupJob(options) {
|
|
|
276
482
|
const pruned = await prune({
|
|
277
483
|
store: options.store,
|
|
278
484
|
realm: options.realm,
|
|
485
|
+
masterSeed: options.masterSeed,
|
|
279
486
|
policy: options.policy,
|
|
280
487
|
prefix: options.prefix
|
|
281
488
|
});
|
|
@@ -168,10 +168,14 @@ function parseAccount(id) {
|
|
|
168
168
|
return { kind, owner, bucket };
|
|
169
169
|
}
|
|
170
170
|
var queueKeyFor = (account) => `${account.kind}:${account.owner}`;
|
|
171
|
+
var MAX_LEDGER_ENTRIES = 64;
|
|
171
172
|
function assertBalanced(transaction) {
|
|
172
|
-
if (transaction.entries.length
|
|
173
|
+
if (transaction.entries.length < 2) {
|
|
173
174
|
throw new LedgerError("EMPTY_TRANSACTION", "A transaction needs at least two entries.");
|
|
174
175
|
}
|
|
176
|
+
if (transaction.entries.length > MAX_LEDGER_ENTRIES) {
|
|
177
|
+
throw new LedgerError("TOO_MANY_ENTRIES", `A transaction may contain at most ${MAX_LEDGER_ENTRIES} entries; split this batch into separate transactions.`);
|
|
178
|
+
}
|
|
175
179
|
const totals = new Map;
|
|
176
180
|
for (const entry of transaction.entries) {
|
|
177
181
|
if (entry.amount.units === 0n) {
|
|
@@ -168,10 +168,14 @@ function parseAccount(id) {
|
|
|
168
168
|
return { kind, owner, bucket };
|
|
169
169
|
}
|
|
170
170
|
var queueKeyFor = (account) => `${account.kind}:${account.owner}`;
|
|
171
|
+
var MAX_LEDGER_ENTRIES = 64;
|
|
171
172
|
function assertBalanced(transaction) {
|
|
172
|
-
if (transaction.entries.length
|
|
173
|
+
if (transaction.entries.length < 2) {
|
|
173
174
|
throw new LedgerError("EMPTY_TRANSACTION", "A transaction needs at least two entries.");
|
|
174
175
|
}
|
|
176
|
+
if (transaction.entries.length > MAX_LEDGER_ENTRIES) {
|
|
177
|
+
throw new LedgerError("TOO_MANY_ENTRIES", `A transaction may contain at most ${MAX_LEDGER_ENTRIES} entries; split this batch into separate transactions.`);
|
|
178
|
+
}
|
|
175
179
|
const totals = new Map;
|
|
176
180
|
for (const entry of transaction.entries) {
|
|
177
181
|
if (entry.amount.units === 0n) {
|
|
@@ -168,10 +168,14 @@ function parseAccount(id) {
|
|
|
168
168
|
return { kind, owner, bucket };
|
|
169
169
|
}
|
|
170
170
|
var queueKeyFor = (account) => `${account.kind}:${account.owner}`;
|
|
171
|
+
var MAX_LEDGER_ENTRIES = 64;
|
|
171
172
|
function assertBalanced(transaction) {
|
|
172
|
-
if (transaction.entries.length
|
|
173
|
+
if (transaction.entries.length < 2) {
|
|
173
174
|
throw new LedgerError("EMPTY_TRANSACTION", "A transaction needs at least two entries.");
|
|
174
175
|
}
|
|
176
|
+
if (transaction.entries.length > MAX_LEDGER_ENTRIES) {
|
|
177
|
+
throw new LedgerError("TOO_MANY_ENTRIES", `A transaction may contain at most ${MAX_LEDGER_ENTRIES} entries; split this batch into separate transactions.`);
|
|
178
|
+
}
|
|
175
179
|
const totals = new Map;
|
|
176
180
|
for (const entry of transaction.entries) {
|
|
177
181
|
if (entry.amount.units === 0n) {
|