@ocap/statedb-sqlite 1.30.26 → 1.31.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/esm/db.d.mts +2 -0
- package/esm/db.mjs +52 -36
- package/esm/index.d.mts +3 -1
- package/esm/index.mjs +3 -1
- package/esm/migrations/003-token-charter-assets.d.mts +14 -0
- package/esm/migrations/003-token-charter-assets.mjs +116 -0
- package/esm/migrations/index.d.mts +2 -1
- package/esm/migrations/index.mjs +6 -3
- package/esm/package.mjs +2 -0
- package/esm/reconcile.d.mts +82 -0
- package/esm/reconcile.mjs +833 -0
- package/esm/table/balance.mjs +35 -16
- package/esm/table/base.mjs +24 -17
- package/esm/tombstone-cleanup.d.mts +59 -0
- package/esm/tombstone-cleanup.mjs +207 -0
- package/lib/db.cjs +52 -36
- package/lib/db.d.cts +2 -0
- package/lib/index.cjs +12 -1
- package/lib/index.d.cts +3 -1
- package/lib/migrations/003-token-charter-assets.cjs +123 -0
- package/lib/migrations/003-token-charter-assets.d.cts +14 -0
- package/lib/migrations/index.cjs +12 -3
- package/lib/migrations/index.d.cts +2 -1
- package/lib/package.cjs +2 -0
- package/lib/reconcile.cjs +836 -0
- package/lib/reconcile.d.cts +82 -0
- package/lib/table/balance.cjs +35 -16
- package/lib/table/base.cjs +24 -17
- package/lib/tombstone-cleanup.cjs +213 -0
- package/lib/tombstone-cleanup.d.cts +59 -0
- package/package.json +8 -6
package/esm/table/balance.mjs
CHANGED
|
@@ -78,12 +78,13 @@ var BalanceTable = class extends base_default {
|
|
|
78
78
|
* @returns Updated token balances
|
|
79
79
|
*/
|
|
80
80
|
async updateBalance({ address, tokens, context = {} }, ctx) {
|
|
81
|
-
if (!Object.keys(tokens).length) return {};
|
|
82
81
|
const normalizedAddress = ensureChecksumAddress(address);
|
|
83
82
|
const tokenBalances = await this.getBalance(normalizedAddress, ctx);
|
|
83
|
+
const removedTokens = Object.keys(tokenBalances).filter((token) => !(token in tokens));
|
|
84
84
|
const updatedTokens = Object.keys(tokens).filter((token) => tokenBalances[token] && tokenBalances[token] !== "0" || tokens[token] && tokens[token] !== "0").filter((token) => tokenBalances[token] !== tokens[token]);
|
|
85
|
-
if (updatedTokens.length === 0) return tokens;
|
|
85
|
+
if (updatedTokens.length === 0 && removedTokens.length === 0) return { ...tokens };
|
|
86
86
|
const batchRows = [];
|
|
87
|
+
const pendingOps = [];
|
|
87
88
|
for (const tokenAddress of updatedTokens) {
|
|
88
89
|
const key = {
|
|
89
90
|
address: normalizedAddress,
|
|
@@ -99,22 +100,40 @@ var BalanceTable = class extends base_default {
|
|
|
99
100
|
enumerable: false,
|
|
100
101
|
configurable: true
|
|
101
102
|
});
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
103
|
+
const insertAttrs = {
|
|
104
|
+
...payload,
|
|
105
|
+
...key
|
|
106
|
+
};
|
|
107
|
+
batchRows.push(this.serializeForDb(insertAttrs));
|
|
108
|
+
pendingOps.push({
|
|
109
|
+
isUpdate: Boolean(tokenBalances[tokenAddress]),
|
|
110
|
+
key,
|
|
111
|
+
payload,
|
|
112
|
+
tokenAddress
|
|
113
|
+
});
|
|
105
114
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
115
|
+
const sqlContext = ctx;
|
|
116
|
+
const executor = this.getExecutor(sqlContext);
|
|
117
|
+
if (removedTokens.length > 0) await executor.deleteFrom("balance").where("address", "=", normalizedAddress).where("tokenAddress", "in", removedTokens).execute();
|
|
118
|
+
if (batchRows.length > 0) for (let i = 0; i < batchRows.length; i += UPSERT_BATCH_SIZE) {
|
|
119
|
+
const batch = batchRows.slice(i, i + UPSERT_BATCH_SIZE);
|
|
120
|
+
await executor.insertInto("balance").values(batch).onConflict((oc) => oc.columns(["address", "tokenAddress"]).doUpdateSet((eb) => ({
|
|
121
|
+
balance: eb.ref("excluded.balance"),
|
|
122
|
+
context: eb.ref("excluded.context")
|
|
123
|
+
}))).execute();
|
|
124
|
+
}
|
|
125
|
+
const noopCollector = [];
|
|
126
|
+
for (const op of pendingOps) {
|
|
127
|
+
Object.defineProperty(op.payload, BATCH_COLLECTOR, {
|
|
128
|
+
value: noopCollector,
|
|
129
|
+
enumerable: false,
|
|
130
|
+
configurable: true
|
|
131
|
+
});
|
|
132
|
+
if (op.isUpdate) await this.update(op.key, op.payload, ctx);
|
|
133
|
+
else await this.create(op.key, op.payload, ctx);
|
|
134
|
+
tokenBalances[op.tokenAddress] = tokens[op.tokenAddress];
|
|
116
135
|
}
|
|
117
|
-
return tokens;
|
|
136
|
+
return { ...tokens };
|
|
118
137
|
}
|
|
119
138
|
};
|
|
120
139
|
var balance_default = BalanceTable;
|
package/esm/table/base.mjs
CHANGED
|
@@ -124,11 +124,14 @@ var SqliteTable = class SqliteTable extends StateDBTable {
|
|
|
124
124
|
if (this.syncBalance) delete insertAttrs.tokens;
|
|
125
125
|
const serialized = this.serializeForDb(insertAttrs);
|
|
126
126
|
await executor.insertInto(this.name).values(serialized).execute();
|
|
127
|
-
if (this.syncBalance && attrs.tokens
|
|
128
|
-
|
|
129
|
-
tokens
|
|
130
|
-
|
|
131
|
-
|
|
127
|
+
if (this.syncBalance && attrs.tokens !== void 0) {
|
|
128
|
+
const rawTokens = attrs.tokens || {};
|
|
129
|
+
insertAttrs.tokens = Object.keys(rawTokens).length ? await this.balanceTable.updateBalance({
|
|
130
|
+
address: insertAttrs.address || (typeof key === "string" ? key : ""),
|
|
131
|
+
tokens: rawTokens,
|
|
132
|
+
context: attrs.context
|
|
133
|
+
}, context) : {};
|
|
134
|
+
}
|
|
132
135
|
return insertAttrs;
|
|
133
136
|
}
|
|
134
137
|
async _get(key, context) {
|
|
@@ -147,7 +150,8 @@ var SqliteTable = class SqliteTable extends StateDBTable {
|
|
|
147
150
|
const result = await qb.executeTakeFirst();
|
|
148
151
|
const deserialized = this.deserializeFromDb(result);
|
|
149
152
|
if (deserialized && this.syncBalance) {
|
|
150
|
-
const
|
|
153
|
+
const balanceAddr = deserialized.address || (typeof key === "string" ? key : "");
|
|
154
|
+
const balance = await this.balanceTable.getBalance(balanceAddr, context);
|
|
151
155
|
deserialized.tokens = {
|
|
152
156
|
...deserialized.tokens || {},
|
|
153
157
|
...balance
|
|
@@ -197,17 +201,20 @@ var SqliteTable = class SqliteTable extends StateDBTable {
|
|
|
197
201
|
if (!existingRow) throw new Error(`${this.name} does not exist: ${JSON.stringify(key)}`);
|
|
198
202
|
merged = this.deserializeFromDb(existingRow);
|
|
199
203
|
}
|
|
200
|
-
if (this.syncBalance)
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
204
|
+
if (this.syncBalance) {
|
|
205
|
+
const balanceAddr = merged.address || (typeof key === "string" ? key : "");
|
|
206
|
+
if (updates.tokens !== void 0) merged.tokens = await this.balanceTable.updateBalance({
|
|
207
|
+
address: balanceAddr,
|
|
208
|
+
tokens: updates.tokens || {},
|
|
209
|
+
context: updates.context
|
|
210
|
+
}, context);
|
|
211
|
+
else {
|
|
212
|
+
const balance = await this.balanceTable.getBalance(balanceAddr, context);
|
|
213
|
+
merged.tokens = {
|
|
214
|
+
...merged.tokens || {},
|
|
215
|
+
...balance
|
|
216
|
+
};
|
|
217
|
+
}
|
|
211
218
|
}
|
|
212
219
|
return merged;
|
|
213
220
|
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { Database } from "./interfaces.mjs";
|
|
2
|
+
import SqliteStateDB from "./db.mjs";
|
|
3
|
+
import { Kysely, Transaction } from "kysely";
|
|
4
|
+
|
|
5
|
+
//#region src/tombstone-cleanup.d.ts
|
|
6
|
+
interface TombstoneBalanceRow {
|
|
7
|
+
accountAddress: string;
|
|
8
|
+
tokenAddress: string;
|
|
9
|
+
balance: string;
|
|
10
|
+
balanceUpdatedAt: string;
|
|
11
|
+
accountUpdatedAt: string;
|
|
12
|
+
migratedTo: string[];
|
|
13
|
+
}
|
|
14
|
+
interface TombstoneTokenSummary {
|
|
15
|
+
tokenAddress: string;
|
|
16
|
+
staleBalanceSum: string;
|
|
17
|
+
rowCount: number;
|
|
18
|
+
accountCount: number;
|
|
19
|
+
}
|
|
20
|
+
interface TombstoneCleanupPlan {
|
|
21
|
+
mode: 'plan';
|
|
22
|
+
totalRows: number;
|
|
23
|
+
totalAccounts: number;
|
|
24
|
+
totalTokens: number;
|
|
25
|
+
perTokenTotals: Record<string, TombstoneTokenSummary>;
|
|
26
|
+
rows: TombstoneBalanceRow[];
|
|
27
|
+
skippedPostMigrationRows: TombstoneBalanceRow[];
|
|
28
|
+
planDigest: string;
|
|
29
|
+
}
|
|
30
|
+
interface ApplyTombstoneCleanupOptions {
|
|
31
|
+
apply: boolean;
|
|
32
|
+
expectedPlanDigest: string;
|
|
33
|
+
}
|
|
34
|
+
interface TombstoneCleanupApplyResult {
|
|
35
|
+
mode: 'apply';
|
|
36
|
+
applied: boolean;
|
|
37
|
+
deletedRows: number;
|
|
38
|
+
deletedAccounts: number;
|
|
39
|
+
perTokenDeletedTotals: Record<string, TombstoneTokenSummary>;
|
|
40
|
+
rows: TombstoneBalanceRow[];
|
|
41
|
+
planDigest: string;
|
|
42
|
+
}
|
|
43
|
+
declare function extractContextUpdatedAt(rawContext: unknown): string;
|
|
44
|
+
declare function compareUpdatedAtTimestamps(a: string, b: string): number;
|
|
45
|
+
declare function isBalanceTimestampNotAfterAccount(balanceUpdatedAt: string, accountUpdatedAt: string): boolean;
|
|
46
|
+
declare function computeTombstoneCleanupPlanDigest(rows: ReadonlyArray<Pick<TombstoneBalanceRow, 'accountAddress' | 'tokenAddress' | 'balance' | 'balanceUpdatedAt'>>): string;
|
|
47
|
+
type KyselyExecutor = Kysely<Database> | Transaction<Database>;
|
|
48
|
+
/**
|
|
49
|
+
* Produce a read-only, deterministic cleanup plan for stale `balance` rows attached to
|
|
50
|
+
* migrated tombstone accounts (`json_array_length(account.migratedTo) > 0` and `balance.updatedAt <= account.updatedAt`).
|
|
51
|
+
*/
|
|
52
|
+
declare function planTombstoneBalanceCleanup(dbOrKysely: SqliteStateDB | KyselyExecutor): Promise<TombstoneCleanupPlan>;
|
|
53
|
+
/**
|
|
54
|
+
* Execute digest-guarded deletion of stale tombstone `balance` rows inside an atomic transaction.
|
|
55
|
+
* Recomputes the plan inside the transaction and aborts if `planDigest !== expectedPlanDigest`.
|
|
56
|
+
*/
|
|
57
|
+
declare function applyTombstoneBalanceCleanup(dbOrKysely: SqliteStateDB | KyselyExecutor, options: ApplyTombstoneCleanupOptions): Promise<TombstoneCleanupApplyResult>;
|
|
58
|
+
//#endregion
|
|
59
|
+
export { ApplyTombstoneCleanupOptions, TombstoneBalanceRow, TombstoneCleanupApplyResult, TombstoneCleanupPlan, TombstoneTokenSummary, applyTombstoneBalanceCleanup, compareUpdatedAtTimestamps, computeTombstoneCleanupPlanDigest, extractContextUpdatedAt, isBalanceTimestampNotAfterAccount, planTombstoneBalanceCleanup };
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { sql } from "kysely";
|
|
3
|
+
import { BN } from "@ocap/util";
|
|
4
|
+
|
|
5
|
+
//#region src/tombstone-cleanup.ts
|
|
6
|
+
const CANONICAL_UINT_REGEX = /^(?:0|[1-9]\d*)$/;
|
|
7
|
+
const SHA256_HEX_REGEX = /^[a-f0-9]{64}$/i;
|
|
8
|
+
function parseJsonValue(raw, fallback) {
|
|
9
|
+
if (raw === null || raw === void 0 || raw === "") return fallback;
|
|
10
|
+
if (typeof raw === "object") return raw;
|
|
11
|
+
if (typeof raw !== "string") return fallback;
|
|
12
|
+
try {
|
|
13
|
+
return JSON.parse(raw) ?? fallback;
|
|
14
|
+
} catch {
|
|
15
|
+
return fallback;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
function extractContextUpdatedAt(rawContext) {
|
|
19
|
+
const ctx = parseJsonValue(rawContext, {});
|
|
20
|
+
if (!ctx || typeof ctx !== "object") return "";
|
|
21
|
+
const candidate = ctx.renaissanceTime || ctx.updatedAt || ctx.genesisTime || "";
|
|
22
|
+
return typeof candidate === "string" ? candidate.trim() : String(candidate ?? "").trim();
|
|
23
|
+
}
|
|
24
|
+
function compareUpdatedAtTimestamps(a, b) {
|
|
25
|
+
const normA = a.trim();
|
|
26
|
+
const normB = b.trim();
|
|
27
|
+
if (normA === normB) return 0;
|
|
28
|
+
if (!normA) return -1;
|
|
29
|
+
if (!normB) return 1;
|
|
30
|
+
if (/^-?\d+(?:\.\d+)?$/.test(normA) && /^-?\d+(?:\.\d+)?$/.test(normB)) {
|
|
31
|
+
const numA = Number(normA);
|
|
32
|
+
const numB = Number(normB);
|
|
33
|
+
if (Number.isFinite(numA) && Number.isFinite(numB) && numA !== numB) return numA < numB ? -1 : 1;
|
|
34
|
+
}
|
|
35
|
+
const msA = Date.parse(normA);
|
|
36
|
+
const msB = Date.parse(normB);
|
|
37
|
+
if (Number.isFinite(msA) && Number.isFinite(msB)) {
|
|
38
|
+
if (msA !== msB) return msA < msB ? -1 : 1;
|
|
39
|
+
return 0;
|
|
40
|
+
}
|
|
41
|
+
return normA < normB ? -1 : 1;
|
|
42
|
+
}
|
|
43
|
+
function isBalanceTimestampNotAfterAccount(balanceUpdatedAt, accountUpdatedAt) {
|
|
44
|
+
const balTs = balanceUpdatedAt.trim();
|
|
45
|
+
const accTs = accountUpdatedAt.trim();
|
|
46
|
+
if (!balTs) return true;
|
|
47
|
+
if (!accTs) return false;
|
|
48
|
+
return compareUpdatedAtTimestamps(balTs, accTs) <= 0;
|
|
49
|
+
}
|
|
50
|
+
function compareTombstoneRows(a, b) {
|
|
51
|
+
if (a.accountAddress !== b.accountAddress) return a.accountAddress < b.accountAddress ? -1 : 1;
|
|
52
|
+
if (a.tokenAddress !== b.tokenAddress) return a.tokenAddress < b.tokenAddress ? -1 : 1;
|
|
53
|
+
return 0;
|
|
54
|
+
}
|
|
55
|
+
function computeTombstoneCleanupPlanDigest(rows) {
|
|
56
|
+
const canonical = [...rows].sort(compareTombstoneRows).map((r) => [
|
|
57
|
+
r.accountAddress,
|
|
58
|
+
r.tokenAddress,
|
|
59
|
+
r.balance,
|
|
60
|
+
r.balanceUpdatedAt
|
|
61
|
+
]);
|
|
62
|
+
return createHash("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
63
|
+
}
|
|
64
|
+
function buildPerTokenSummaries(rows) {
|
|
65
|
+
const byToken = /* @__PURE__ */ new Map();
|
|
66
|
+
for (const row of rows) {
|
|
67
|
+
const balBn = CANONICAL_UINT_REGEX.test(row.balance) ? new BN(row.balance, 10) : new BN(0);
|
|
68
|
+
const existing = byToken.get(row.tokenAddress);
|
|
69
|
+
if (existing) {
|
|
70
|
+
existing.sum = existing.sum.add(balBn);
|
|
71
|
+
existing.rowCount += 1;
|
|
72
|
+
existing.accounts.add(row.accountAddress);
|
|
73
|
+
} else byToken.set(row.tokenAddress, {
|
|
74
|
+
sum: balBn,
|
|
75
|
+
rowCount: 1,
|
|
76
|
+
accounts: new Set([row.accountAddress])
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
const sortedTokenAddresses = Array.from(byToken.keys()).sort();
|
|
80
|
+
const result = {};
|
|
81
|
+
for (const tokenAddress of sortedTokenAddresses) {
|
|
82
|
+
const entry = byToken.get(tokenAddress);
|
|
83
|
+
result[tokenAddress] = {
|
|
84
|
+
tokenAddress,
|
|
85
|
+
staleBalanceSum: entry.sum.toString(10),
|
|
86
|
+
rowCount: entry.rowCount,
|
|
87
|
+
accountCount: entry.accounts.size
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
return result;
|
|
91
|
+
}
|
|
92
|
+
function isSqliteStateDB(dbOrKysely) {
|
|
93
|
+
const candidate = dbOrKysely;
|
|
94
|
+
return Boolean(candidate && typeof candidate === "object" && typeof candidate.getKysely === "function" && typeof candidate.runAsLambda === "function");
|
|
95
|
+
}
|
|
96
|
+
function resolveKyselyExecutor(dbOrKysely) {
|
|
97
|
+
if (isSqliteStateDB(dbOrKysely)) return dbOrKysely.getKysely();
|
|
98
|
+
return dbOrKysely;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Produce a read-only, deterministic cleanup plan for stale `balance` rows attached to
|
|
102
|
+
* migrated tombstone accounts (`json_array_length(account.migratedTo) > 0` and `balance.updatedAt <= account.updatedAt`).
|
|
103
|
+
*/
|
|
104
|
+
async function planTombstoneBalanceCleanup(dbOrKysely) {
|
|
105
|
+
const rawRows = await resolveKyselyExecutor(dbOrKysely).selectFrom("balance as b").innerJoin("account as a", "b.address", "a.address").where("a.migratedTo", "is not", null).where(sql`CASE WHEN json_valid(a.migratedTo) THEN json_array_length(a.migratedTo) > 0 ELSE 0 END`).select([
|
|
106
|
+
"b.address as accountAddress",
|
|
107
|
+
"b.tokenAddress as tokenAddress",
|
|
108
|
+
"b.balance as balance",
|
|
109
|
+
"b.context as balanceContext",
|
|
110
|
+
"a.migratedTo as migratedTo",
|
|
111
|
+
"a.context as accountContext"
|
|
112
|
+
]).execute();
|
|
113
|
+
const eligibleRows = [];
|
|
114
|
+
const skippedPostMigrationRows = [];
|
|
115
|
+
for (const raw of rawRows) {
|
|
116
|
+
const migratedToRaw = parseJsonValue(raw.migratedTo, []);
|
|
117
|
+
if (!Array.isArray(migratedToRaw)) continue;
|
|
118
|
+
const migratedTo = migratedToRaw.map((x) => String(x ?? "").trim()).filter((x) => x.length > 0);
|
|
119
|
+
if (migratedTo.length === 0) continue;
|
|
120
|
+
const accountAddress = String(raw.accountAddress ?? "");
|
|
121
|
+
const tokenAddress = String(raw.tokenAddress ?? "");
|
|
122
|
+
const balance = String(raw.balance ?? "0");
|
|
123
|
+
const balanceUpdatedAt = extractContextUpdatedAt(raw.balanceContext);
|
|
124
|
+
const accountUpdatedAt = extractContextUpdatedAt(raw.accountContext);
|
|
125
|
+
const row = {
|
|
126
|
+
accountAddress,
|
|
127
|
+
tokenAddress,
|
|
128
|
+
balance,
|
|
129
|
+
balanceUpdatedAt,
|
|
130
|
+
accountUpdatedAt,
|
|
131
|
+
migratedTo
|
|
132
|
+
};
|
|
133
|
+
if (isBalanceTimestampNotAfterAccount(balanceUpdatedAt, accountUpdatedAt)) eligibleRows.push(row);
|
|
134
|
+
else skippedPostMigrationRows.push(row);
|
|
135
|
+
}
|
|
136
|
+
eligibleRows.sort(compareTombstoneRows);
|
|
137
|
+
skippedPostMigrationRows.sort(compareTombstoneRows);
|
|
138
|
+
const perTokenTotals = buildPerTokenSummaries(eligibleRows);
|
|
139
|
+
const uniqueAccounts = new Set(eligibleRows.map((r) => r.accountAddress));
|
|
140
|
+
const planDigest = computeTombstoneCleanupPlanDigest(eligibleRows);
|
|
141
|
+
return {
|
|
142
|
+
mode: "plan",
|
|
143
|
+
totalRows: eligibleRows.length,
|
|
144
|
+
totalAccounts: uniqueAccounts.size,
|
|
145
|
+
totalTokens: Object.keys(perTokenTotals).length,
|
|
146
|
+
perTokenTotals,
|
|
147
|
+
rows: eligibleRows,
|
|
148
|
+
skippedPostMigrationRows,
|
|
149
|
+
planDigest
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Execute digest-guarded deletion of stale tombstone `balance` rows inside an atomic transaction.
|
|
154
|
+
* Recomputes the plan inside the transaction and aborts if `planDigest !== expectedPlanDigest`.
|
|
155
|
+
*/
|
|
156
|
+
async function applyTombstoneBalanceCleanup(dbOrKysely, options) {
|
|
157
|
+
if (!options || options.apply !== true) throw new Error("applyTombstoneBalanceCleanup requires explicit { apply: true }");
|
|
158
|
+
const expectedDigest = String(options.expectedPlanDigest ?? "").trim().toLowerCase();
|
|
159
|
+
if (!SHA256_HEX_REGEX.test(expectedDigest)) throw new Error("applyTombstoneBalanceCleanup requires a valid 64-character SHA-256 expectedPlanDigest");
|
|
160
|
+
const executeInTxn = async (txn) => {
|
|
161
|
+
const freshPlan = await planTombstoneBalanceCleanup(txn);
|
|
162
|
+
if (freshPlan.planDigest.toLowerCase() !== expectedDigest) throw new Error(`PLAN_DIGEST_MISMATCH: expected planDigest "${options.expectedPlanDigest}", but current state produced "${freshPlan.planDigest}"`);
|
|
163
|
+
if (freshPlan.rows.length === 0) return {
|
|
164
|
+
mode: "apply",
|
|
165
|
+
applied: false,
|
|
166
|
+
deletedRows: 0,
|
|
167
|
+
deletedAccounts: 0,
|
|
168
|
+
perTokenDeletedTotals: {},
|
|
169
|
+
rows: [],
|
|
170
|
+
planDigest: freshPlan.planDigest
|
|
171
|
+
};
|
|
172
|
+
let deletedRows = 0;
|
|
173
|
+
const deletedAccountSet = /* @__PURE__ */ new Set();
|
|
174
|
+
for (const row of freshPlan.rows) {
|
|
175
|
+
const accountRow = await txn.selectFrom("account").where("address", "=", row.accountAddress).where("migratedTo", "is not", null).where(sql`CASE WHEN json_valid(migratedTo) THEN json_array_length(migratedTo) > 0 ELSE 0 END`).select([
|
|
176
|
+
"address",
|
|
177
|
+
"context",
|
|
178
|
+
"tokens"
|
|
179
|
+
]).executeTakeFirst();
|
|
180
|
+
if (!accountRow) throw new Error(`TOMBSTONE_STATE_CHANGED: account "${row.accountAddress}" is no longer a migrated tombstone during apply`);
|
|
181
|
+
const currentAccountUpdatedAt = extractContextUpdatedAt(accountRow.context);
|
|
182
|
+
if (!isBalanceTimestampNotAfterAccount(row.balanceUpdatedAt, currentAccountUpdatedAt)) throw new Error(`TOMBSTONE_TIMESTAMP_CHANGED: balance row (${row.accountAddress}, ${row.tokenAddress}) post-dates account timestamp during apply`);
|
|
183
|
+
const delResult = await txn.deleteFrom("balance").where("address", "=", row.accountAddress).where("tokenAddress", "=", row.tokenAddress).where("balance", "=", row.balance).executeTakeFirst();
|
|
184
|
+
const count = Number(delResult.numDeletedRows ?? 0n);
|
|
185
|
+
if (count !== 1) throw new Error(`TOMBSTONE_ROW_CONCURRENT_MODIFICATION: expected to delete 1 balance row for (${row.accountAddress}, ${row.tokenAddress}), deleted ${count}`);
|
|
186
|
+
deletedRows += count;
|
|
187
|
+
deletedAccountSet.add(row.accountAddress);
|
|
188
|
+
if (accountRow.tokens !== null && accountRow.tokens !== void 0 && accountRow.tokens !== "{}") await txn.updateTable("account").set({ tokens: "{}" }).where("address", "=", row.accountAddress).execute();
|
|
189
|
+
}
|
|
190
|
+
return {
|
|
191
|
+
mode: "apply",
|
|
192
|
+
applied: deletedRows > 0,
|
|
193
|
+
deletedRows,
|
|
194
|
+
deletedAccounts: deletedAccountSet.size,
|
|
195
|
+
perTokenDeletedTotals: freshPlan.perTokenTotals,
|
|
196
|
+
rows: freshPlan.rows,
|
|
197
|
+
planDigest: freshPlan.planDigest
|
|
198
|
+
};
|
|
199
|
+
};
|
|
200
|
+
if (isSqliteStateDB(dbOrKysely)) return dbOrKysely.runAsLambda((txn) => executeInTxn(txn));
|
|
201
|
+
const kysely = resolveKyselyExecutor(dbOrKysely);
|
|
202
|
+
if ("transaction" in kysely && typeof kysely.transaction === "function") return kysely.transaction().execute((txn) => executeInTxn(txn));
|
|
203
|
+
return executeInTxn(kysely);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
//#endregion
|
|
207
|
+
export { applyTombstoneBalanceCleanup, compareUpdatedAtTimestamps, computeTombstoneCleanupPlanDigest, extractContextUpdatedAt, isBalanceTimestampNotAfterAccount, planTombstoneBalanceCleanup };
|
package/lib/db.cjs
CHANGED
|
@@ -213,13 +213,13 @@ var SqliteStateDB = class extends _ocap_statedb.StateDB {
|
|
|
213
213
|
async initialize() {
|
|
214
214
|
this.closed = false;
|
|
215
215
|
if (this.initialized) {
|
|
216
|
-
if (this.trustedStorage && !this.trustedInitialized) await this.ensureTrustedStorageInitialized();
|
|
216
|
+
if (this.trustedStorage && !this.trustedInitialized) await this.ensureTrustedStorageInitialized(true);
|
|
217
217
|
this.startWalCheckpointWorker();
|
|
218
218
|
return;
|
|
219
219
|
}
|
|
220
220
|
await require_migrations_index.runMigrations(this.db);
|
|
221
221
|
this.initialized = true;
|
|
222
|
-
if (this.trustedStorage) await this.ensureTrustedStorageInitialized();
|
|
222
|
+
if (this.trustedStorage) await this.ensureTrustedStorageInitialized(true);
|
|
223
223
|
this.startWalCheckpointWorker();
|
|
224
224
|
}
|
|
225
225
|
/**
|
|
@@ -286,7 +286,7 @@ var SqliteStateDB = class extends _ocap_statedb.StateDB {
|
|
|
286
286
|
`.execute(this.db);
|
|
287
287
|
this.stateCommitTableReady = true;
|
|
288
288
|
}
|
|
289
|
-
async ensureTrustedStorageInitialized() {
|
|
289
|
+
async ensureTrustedStorageInitialized(allowForwardRecovery = false) {
|
|
290
290
|
if (!this.trustedStorage || this.trustedInitialized) return;
|
|
291
291
|
await this.ensureStateCommitTable();
|
|
292
292
|
if (typeof this.trustedStorage.initStorage === "function") await this.trustedStorage.initStorage();
|
|
@@ -294,7 +294,7 @@ var SqliteStateDB = class extends _ocap_statedb.StateDB {
|
|
|
294
294
|
const { currentHash } = await this.getCommitHash();
|
|
295
295
|
const newTrustedHash = await this.trustedStorage.createHash(currentHash);
|
|
296
296
|
await this.trustedStorage.writeHash(newTrustedHash);
|
|
297
|
-
} else await this.verifyTrustedHashInternal();
|
|
297
|
+
} else await this.verifyTrustedHashInternal(allowForwardRecovery);
|
|
298
298
|
this.trustedInitialized = true;
|
|
299
299
|
}
|
|
300
300
|
/**
|
|
@@ -311,17 +311,24 @@ var SqliteStateDB = class extends _ocap_statedb.StateDB {
|
|
|
311
311
|
async getCommitHash(executor = this.db) {
|
|
312
312
|
if (!this.stateCommitTableReady) await this.ensureStateCommitTable();
|
|
313
313
|
const row = (await kysely.sql`
|
|
314
|
-
SELECT "current_hash", "prev_hash" FROM "_state_commit" WHERE "id" = 1
|
|
314
|
+
SELECT "current_hash", "prev_hash", "seq", "msg" FROM "_state_commit" WHERE "id" = 1
|
|
315
315
|
`.execute(executor)).rows[0];
|
|
316
316
|
return {
|
|
317
317
|
currentHash: row?.current_hash || "GENESIS",
|
|
318
|
-
prevHash: row?.prev_hash || ""
|
|
318
|
+
prevHash: row?.prev_hash || "",
|
|
319
|
+
seq: Number(row?.seq || 0),
|
|
320
|
+
msg: row?.msg || ""
|
|
319
321
|
};
|
|
320
322
|
}
|
|
321
|
-
async verifyTrustedHashInternal() {
|
|
323
|
+
async verifyTrustedHashInternal(allowForwardRecovery = false) {
|
|
322
324
|
if (!this.trustedStorage) return;
|
|
323
|
-
const { currentHash } = await this.getCommitHash();
|
|
325
|
+
const { currentHash, prevHash, seq = 0 } = await this.getCommitHash();
|
|
324
326
|
if (!await this.trustedStorage.compare(currentHash)) {
|
|
327
|
+
const isValidSha256Hex = /^[0-9a-f]{64}$/i.test(currentHash);
|
|
328
|
+
if (allowForwardRecovery && prevHash && seq > 0 && currentHash !== prevHash && isValidSha256Hex && await this.trustedStorage.compare(prevHash)) {
|
|
329
|
+
await this.trustedStorage.updateHash(currentHash, prevHash);
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
325
332
|
const err = /* @__PURE__ */ new Error("Trusted hash verification failed");
|
|
326
333
|
err.code = "INVALID_HASH";
|
|
327
334
|
throw err;
|
|
@@ -333,8 +340,8 @@ var SqliteStateDB = class extends _ocap_statedb.StateDB {
|
|
|
333
340
|
async verifyTrustedHash() {
|
|
334
341
|
if (!this.trustedStorage) return;
|
|
335
342
|
await this.enqueueCommit(async () => {
|
|
336
|
-
await this.ensureTrustedStorageInitialized();
|
|
337
|
-
await this.verifyTrustedHashInternal();
|
|
343
|
+
await this.ensureTrustedStorageInitialized(true);
|
|
344
|
+
await this.verifyTrustedHashInternal(true);
|
|
338
345
|
});
|
|
339
346
|
}
|
|
340
347
|
enqueueCommit(task) {
|
|
@@ -356,50 +363,59 @@ var SqliteStateDB = class extends _ocap_statedb.StateDB {
|
|
|
356
363
|
}
|
|
357
364
|
async executeRunAsLambda(fn, options = {}) {
|
|
358
365
|
const { retryLimit = 2, verifyHash = false, commitMessage } = options;
|
|
366
|
+
if (!this.stateCommitTableReady) await this.ensureStateCommitTable();
|
|
359
367
|
if (this.trustedStorage) {
|
|
360
368
|
await this.ensureTrustedStorageInitialized();
|
|
361
|
-
if (verifyHash) await this.verifyTrustedHashInternal();
|
|
369
|
+
if (verifyHash) await this.verifyTrustedHashInternal(false);
|
|
362
370
|
}
|
|
363
371
|
let result;
|
|
364
372
|
let committedHash;
|
|
365
373
|
let previousHash;
|
|
366
374
|
let lastError;
|
|
367
375
|
let committed = false;
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
376
|
+
const opCtx = options.context;
|
|
377
|
+
for (let attempt = 0; attempt <= retryLimit; attempt++) {
|
|
378
|
+
const savedEvents = Array.isArray(opCtx?.events) ? [...opCtx.events] : void 0;
|
|
379
|
+
const savedTimerCounters = opCtx?.timer?.stateDbCounters && typeof opCtx.timer.stateDbCounters === "object" ? { ...opCtx.timer.stateDbCounters } : void 0;
|
|
372
380
|
try {
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
381
|
+
committedHash = void 0;
|
|
382
|
+
previousHash = void 0;
|
|
383
|
+
this._activeLambdaCount++;
|
|
384
|
+
try {
|
|
385
|
+
result = await this.db.transaction().execute(async (txn) => {
|
|
386
|
+
const txResult = await fn(txn);
|
|
376
387
|
const row = (await kysely.sql`
|
|
377
|
-
|
|
378
|
-
|
|
388
|
+
SELECT "current_hash", "seq" FROM "_state_commit" WHERE "id" = 1
|
|
389
|
+
`.execute(txn)).rows[0];
|
|
379
390
|
const prevHash = row?.current_hash || "GENESIS";
|
|
380
391
|
const nextSeq = Number(row?.seq || 0) + 1;
|
|
381
392
|
const msg = commitMessage || txResult?.hash || "commit";
|
|
382
393
|
const nextHash = node_crypto.default.createHash("sha256").update(`${prevHash}:${nextSeq}:${msg}`).digest("hex");
|
|
383
394
|
await kysely.sql`
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
395
|
+
UPDATE "_state_commit"
|
|
396
|
+
SET "current_hash" = ${nextHash}, "prev_hash" = ${prevHash}, "seq" = ${nextSeq}, "msg" = ${msg}
|
|
397
|
+
WHERE "id" = 1
|
|
398
|
+
`.execute(txn);
|
|
388
399
|
previousHash = prevHash;
|
|
389
400
|
committedHash = nextHash;
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
}
|
|
393
|
-
|
|
394
|
-
|
|
401
|
+
return txResult;
|
|
402
|
+
});
|
|
403
|
+
} finally {
|
|
404
|
+
this._activeLambdaCount--;
|
|
405
|
+
}
|
|
406
|
+
committed = true;
|
|
407
|
+
break;
|
|
408
|
+
} catch (err) {
|
|
409
|
+
if (opCtx && savedEvents !== void 0 && Array.isArray(opCtx.events)) {
|
|
410
|
+
opCtx.events.length = 0;
|
|
411
|
+
opCtx.events.push(...savedEvents);
|
|
412
|
+
}
|
|
413
|
+
if (opCtx?.timer?.stateDbCounters && savedTimerCounters !== void 0) Object.assign(opCtx.timer.stateDbCounters, savedTimerCounters);
|
|
414
|
+
lastError = err;
|
|
415
|
+
const code = err?.code;
|
|
416
|
+
if (!SQLITE_BUSY_CODES.includes(code)) throw err;
|
|
417
|
+
if (attempt < retryLimit) await new Promise((r) => setTimeout(r, 10 * 2 ** attempt));
|
|
395
418
|
}
|
|
396
|
-
committed = true;
|
|
397
|
-
break;
|
|
398
|
-
} catch (err) {
|
|
399
|
-
lastError = err;
|
|
400
|
-
const code = err?.code;
|
|
401
|
-
if (!SQLITE_BUSY_CODES.includes(code)) throw err;
|
|
402
|
-
if (attempt < retryLimit) await new Promise((r) => setTimeout(r, 10 * 2 ** attempt));
|
|
403
419
|
}
|
|
404
420
|
if (!committed) throw lastError;
|
|
405
421
|
if (this.trustedStorage && committedHash) await this.trustedStorage.updateHash(committedHash, previousHash);
|
package/lib/db.d.cts
CHANGED
|
@@ -103,6 +103,8 @@ declare class SqliteStateDB extends StateDB implements IStateDB {
|
|
|
103
103
|
getCommitHash(executor?: Kysely<Database> | Transaction<Database>): Promise<{
|
|
104
104
|
currentHash: string;
|
|
105
105
|
prevHash: string;
|
|
106
|
+
seq?: number;
|
|
107
|
+
msg?: string;
|
|
106
108
|
}>;
|
|
107
109
|
private verifyTrustedHashInternal;
|
|
108
110
|
/**
|
package/lib/index.cjs
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
Object.defineProperty(exports, '__esModule', { value: true });
|
|
2
2
|
const require_kysely = require('./kysely.cjs');
|
|
3
3
|
const require_db = require('./db.cjs');
|
|
4
|
+
const require_reconcile = require('./reconcile.cjs');
|
|
5
|
+
const require_tombstone_cleanup = require('./tombstone-cleanup.cjs');
|
|
4
6
|
|
|
5
7
|
//#region src/index.ts
|
|
6
8
|
var src_default = require_db.default;
|
|
@@ -8,6 +10,15 @@ var src_default = require_db.default;
|
|
|
8
10
|
//#endregion
|
|
9
11
|
exports.DurableObjectSqliteDialect = require_kysely.DurableObjectSqliteDialect;
|
|
10
12
|
exports.SqliteStateDB = require_db.default;
|
|
13
|
+
exports.applyTombstoneBalanceCleanup = require_tombstone_cleanup.applyTombstoneBalanceCleanup;
|
|
14
|
+
exports.assertSqliteStateDb = require_reconcile.assertSqliteStateDb;
|
|
15
|
+
exports.compareUpdatedAtTimestamps = require_tombstone_cleanup.compareUpdatedAtTimestamps;
|
|
16
|
+
exports.computeTombstoneCleanupPlanDigest = require_tombstone_cleanup.computeTombstoneCleanupPlanDigest;
|
|
11
17
|
exports.createKysely = require_kysely.createKysely;
|
|
12
18
|
exports.default = src_default;
|
|
13
|
-
exports.
|
|
19
|
+
exports.extractContextUpdatedAt = require_tombstone_cleanup.extractContextUpdatedAt;
|
|
20
|
+
exports.getRawDatabase = require_kysely.getRawDatabase;
|
|
21
|
+
exports.isBalanceTimestampNotAfterAccount = require_tombstone_cleanup.isBalanceTimestampNotAfterAccount;
|
|
22
|
+
exports.planTombstoneBalanceCleanup = require_tombstone_cleanup.planTombstoneBalanceCleanup;
|
|
23
|
+
exports.reconcileChainState = require_reconcile.reconcileChainState;
|
|
24
|
+
exports.verifyChainStateInvariants = require_reconcile.verifyChainStateInvariants;
|
package/lib/index.d.cts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { Database, SqliteOperationContext } from "./interfaces.cjs";
|
|
2
2
|
import { DurableObjectSqliteDialect, DurableObjectSqliteDialectConfig, IDurableObjectSqlStorage, IDurableObjectSqlStorageCursor, SqliteConfig, createKysely, getRawDatabase } from "./kysely.cjs";
|
|
3
3
|
import SqliteStateDB from "./db.cjs";
|
|
4
|
-
|
|
4
|
+
import { ChainStateAuditReport, InvariantSeverity, InvariantViolation, ReconcileAction, ReconcileOptions, ReconcileResult, assertSqliteStateDb, reconcileChainState, verifyChainStateInvariants } from "./reconcile.cjs";
|
|
5
|
+
import { ApplyTombstoneCleanupOptions, TombstoneBalanceRow, TombstoneCleanupApplyResult, TombstoneCleanupPlan, TombstoneTokenSummary, applyTombstoneBalanceCleanup, compareUpdatedAtTimestamps, computeTombstoneCleanupPlanDigest, extractContextUpdatedAt, isBalanceTimestampNotAfterAccount, planTombstoneBalanceCleanup } from "./tombstone-cleanup.cjs";
|
|
6
|
+
export { type ApplyTombstoneCleanupOptions, type ChainStateAuditReport, type Database, DurableObjectSqliteDialect, type DurableObjectSqliteDialectConfig, type IDurableObjectSqlStorage, type IDurableObjectSqlStorageCursor, type InvariantSeverity, type InvariantViolation, type ReconcileAction, type ReconcileOptions, type ReconcileResult, type SqliteConfig, type SqliteOperationContext, SqliteStateDB, SqliteStateDB as default, type TombstoneBalanceRow, type TombstoneCleanupApplyResult, type TombstoneCleanupPlan, type TombstoneTokenSummary, applyTombstoneBalanceCleanup, assertSqliteStateDb, compareUpdatedAtTimestamps, computeTombstoneCleanupPlanDigest, createKysely, extractContextUpdatedAt, getRawDatabase, isBalanceTimestampNotAfterAccount, planTombstoneBalanceCleanup, reconcileChainState, verifyChainStateInvariants };
|