@hot-updater/server 1.0.0-rc.16 → 1.0.0-rc.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/drizzle.cjs +5 -5
- package/dist/adapters/drizzle.mjs +4 -4
- package/dist/adapters/kysely.cjs +3 -3
- package/dist/adapters/kysely.mjs +2 -2
- package/dist/adapters/mongodb.cjs +3 -3
- package/dist/adapters/mongodb.mjs +2 -2
- package/dist/adapters/mongodbAdapter.cjs +19 -1
- package/dist/adapters/mongodbAdapter.mjs +19 -1
- package/dist/adapters/prisma.cjs +5 -5
- package/dist/adapters/prisma.mjs +4 -4
- package/dist/assembly/assemblePlugins.cjs +47 -28
- package/dist/assembly/assemblePlugins.d.cts +3 -7
- package/dist/assembly/assemblePlugins.d.mts +3 -7
- package/dist/assembly/assemblePlugins.mjs +46 -27
- package/dist/assembly/clientPlugins.cjs +86 -0
- package/dist/assembly/clientPlugins.d.cts +16 -0
- package/dist/assembly/clientPlugins.d.mts +16 -0
- package/dist/assembly/clientPlugins.mjs +86 -0
- package/dist/assembly/configError.cjs +7 -0
- package/dist/assembly/configError.d.cts +7 -0
- package/dist/assembly/configError.d.mts +7 -0
- package/dist/assembly/configError.mjs +7 -0
- package/dist/assembly/databasePlugins.cjs +5 -1
- package/dist/assembly/databasePlugins.d.cts +5 -5
- package/dist/assembly/databasePlugins.d.mts +5 -5
- package/dist/assembly/databasePlugins.mjs +5 -1
- package/dist/assembly/pluginCli.cjs +125 -0
- package/dist/assembly/pluginCli.d.cts +59 -0
- package/dist/assembly/pluginCli.d.mts +59 -0
- package/dist/assembly/pluginCli.mjs +121 -0
- package/dist/createHotUpdaterCore.cjs +14 -16
- package/dist/createHotUpdaterCore.d.cts +10 -3
- package/dist/createHotUpdaterCore.d.mts +10 -3
- package/dist/createHotUpdaterCore.mjs +11 -13
- package/dist/database/aggregateBatching.cjs +626 -0
- package/dist/database/aggregateBatching.d.cts +10 -0
- package/dist/database/aggregateBatching.d.mts +10 -0
- package/dist/database/aggregateBatching.mjs +625 -0
- package/dist/{insights/errors.cjs → database/busy.cjs} +2 -15
- package/dist/database/busy.d.cts +12 -0
- package/dist/database/busy.d.mts +12 -0
- package/dist/{insights/errors.mjs → database/busy.mjs} +3 -14
- package/dist/database/database.cjs +22 -3
- package/dist/database/database.d.cts +15 -2
- package/dist/database/database.d.mts +15 -2
- package/dist/database/database.mjs +22 -3
- package/dist/database/definitions.d.cts +10 -0
- package/dist/database/definitions.d.mts +10 -0
- package/dist/database/engine.d.cts +2 -1
- package/dist/database/engine.d.mts +2 -1
- package/dist/database/engineAggregates.cjs +7 -4
- package/dist/database/engineAggregates.mjs +8 -5
- package/dist/database/engineReads.cjs +5 -3
- package/dist/database/engineReads.mjs +6 -4
- package/dist/database/engineTransaction.cjs +2 -6
- package/dist/database/engineTransaction.mjs +2 -6
- package/dist/database/fence.cjs +2 -1
- package/dist/database/fence.mjs +2 -1
- package/dist/database/index.cjs +17 -8
- package/dist/database/index.d.cts +5 -4
- package/dist/database/index.d.mts +5 -4
- package/dist/database/index.mjs +6 -5
- package/dist/database/kv/kvAdapter.cjs +15 -3
- package/dist/database/kv/kvAdapter.d.cts +8 -0
- package/dist/database/kv/kvAdapter.d.mts +8 -0
- package/dist/database/kv/kvAdapter.mjs +16 -4
- package/dist/database/resolveSchema.cjs +4 -1
- package/dist/database/resolveSchema.d.cts +1 -1
- package/dist/database/resolveSchema.d.mts +1 -1
- package/dist/database/resolveSchema.mjs +4 -1
- package/dist/database/retention.cjs +92 -0
- package/dist/database/retention.mjs +90 -0
- package/dist/database/schema.cjs +5 -2
- package/dist/database/schema.d.cts +30 -1
- package/dist/database/schema.d.mts +30 -1
- package/dist/database/schema.mjs +5 -2
- package/dist/database/sql/sqlAdapter.cjs +8 -0
- package/dist/database/sql/sqlAdapter.mjs +9 -1
- package/dist/database/sql/sqlSchema.cjs +56 -0
- package/dist/database/sql/sqlSchema.mjs +55 -1
- package/dist/db/coreDatabase.cjs +85 -0
- package/dist/db/coreDatabase.d.cts +57 -0
- package/dist/db/coreDatabase.d.mts +57 -0
- package/dist/db/coreDatabase.mjs +78 -0
- package/dist/db/index.cjs +11 -3
- package/dist/db/index.d.cts +7 -3
- package/dist/db/index.d.mts +7 -3
- package/dist/db/index.mjs +6 -3
- package/dist/db/types.d.cts +3 -3
- package/dist/db/types.d.mts +3 -3
- package/dist/diff.cjs +3 -0
- package/dist/diff.d.cts +2 -0
- package/dist/diff.d.mts +2 -0
- package/dist/diff.mjs +2 -0
- package/dist/handler.cjs +2 -14
- package/dist/handler.mjs +1 -13
- package/dist/handlerReleaseCatalogRoutes.cjs +21 -2
- package/dist/handlerReleaseCatalogRoutes.mjs +21 -2
- package/dist/index.cjs +2 -6
- package/dist/index.d.cts +4 -7
- package/dist/index.d.mts +4 -7
- package/dist/index.mjs +2 -4
- package/dist/package.cjs +1 -1
- package/dist/package.mjs +1 -1
- package/dist/{apiKeys.cjs → plugins/api-keys/apiKeys.cjs} +1 -1
- package/dist/{apiKeys.d.cts → plugins/api-keys/apiKeys.d.cts} +2 -2
- package/dist/{apiKeys.d.mts → plugins/api-keys/apiKeys.d.mts} +2 -2
- package/dist/{apiKeys.mjs → plugins/api-keys/apiKeys.mjs} +1 -1
- package/dist/plugins/api-keys/cli.cjs +115 -0
- package/dist/plugins/api-keys/cli.mjs +114 -0
- package/dist/plugins/api-keys/index.cjs +9 -6
- package/dist/plugins/api-keys/index.d.cts +4 -5
- package/dist/plugins/api-keys/index.d.mts +4 -5
- package/dist/plugins/api-keys/index.mjs +9 -7
- package/dist/plugins/api-keys/model.d.cts +20 -2
- package/dist/plugins/api-keys/model.d.mts +20 -2
- package/dist/plugins/definePlugin.cjs +1 -1
- package/dist/plugins/definePlugin.d.cts +130 -3
- package/dist/plugins/definePlugin.d.mts +130 -3
- package/dist/plugins/definePlugin.mjs +1 -1
- package/dist/plugins/index.cjs +25 -4
- package/dist/plugins/index.d.cts +7 -4
- package/dist/plugins/index.d.mts +7 -4
- package/dist/plugins/index.mjs +6 -4
- package/dist/plugins/insights/contract.cjs +193 -0
- package/dist/plugins/insights/contract.mjs +189 -0
- package/dist/{insights → plugins/insights}/domain.d.cts +60 -15
- package/dist/{insights → plugins/insights}/domain.d.mts +60 -15
- package/dist/plugins/insights/errors.cjs +15 -0
- package/dist/plugins/insights/errors.mjs +14 -0
- package/dist/{insights → plugins/insights}/eventInput.cjs +155 -22
- package/dist/{insights → plugins/insights}/eventInput.mjs +155 -22
- package/dist/plugins/insights/eventRow.cjs +49 -0
- package/dist/plugins/insights/eventRow.d.cts +64 -0
- package/dist/plugins/insights/eventRow.d.mts +64 -0
- package/dist/plugins/insights/eventRow.mjs +47 -0
- package/dist/plugins/insights/index.cjs +106 -26
- package/dist/plugins/insights/index.d.cts +130 -27
- package/dist/plugins/insights/index.d.mts +130 -27
- package/dist/plugins/insights/index.mjs +106 -29
- package/dist/plugins/insights/model.cjs +2 -2
- package/dist/plugins/insights/model.d.cts +1 -1
- package/dist/plugins/insights/model.d.mts +1 -1
- package/dist/plugins/insights/model.mjs +1 -1
- package/dist/plugins/insights/modelTypes.d.cts +176 -0
- package/dist/plugins/insights/modelTypes.d.mts +176 -0
- package/dist/plugins/insights/overview.cjs +109 -0
- package/dist/plugins/insights/overview.d.cts +15 -0
- package/dist/plugins/insights/overview.d.mts +15 -0
- package/dist/plugins/insights/overview.mjs +107 -0
- package/dist/{insights → plugins/insights}/provider.cjs +33 -14
- package/dist/{insights → plugins/insights}/provider.d.cts +2 -2
- package/dist/{insights → plugins/insights}/provider.d.mts +2 -2
- package/dist/{insights → plugins/insights}/provider.mjs +30 -11
- package/dist/{insights → plugins/insights}/queryInput.cjs +2 -2
- package/dist/{insights → plugins/insights}/queryInput.mjs +2 -2
- package/dist/plugins/insights/reads.cjs +298 -88
- package/dist/plugins/insights/reads.d.cts +61 -0
- package/dist/plugins/insights/reads.d.mts +61 -0
- package/dist/plugins/insights/reads.mjs +298 -90
- package/dist/plugins/insights/recordEvent.cjs +223 -39
- package/dist/plugins/insights/recordEvent.d.cts +9 -4
- package/dist/plugins/insights/recordEvent.d.mts +9 -4
- package/dist/plugins/insights/recordEvent.mjs +221 -37
- package/dist/{insights → plugins/insights}/routes.cjs +3 -30
- package/dist/{insights → plugins/insights}/routes.mjs +4 -28
- package/dist/plugins/insights/schema.cjs +182 -40
- package/dist/plugins/insights/schema.d.cts +446 -6
- package/dist/plugins/insights/schema.d.mts +446 -6
- package/dist/plugins/insights/schema.mjs +180 -41
- package/dist/plugins/insights/testing/eventRow.d.mts +64 -0
- package/dist/plugins/insights/testing/expectInsightsIndex.mjs +18 -0
- package/dist/plugins/insights/testing/fixtures.d.mts +8 -0
- package/dist/plugins/insights/testing/fixtures.mjs +25 -0
- package/dist/plugins/insights/testing/index.d.mts +24 -0
- package/dist/plugins/insights/testing/index.mjs +25 -0
- package/dist/plugins/insights/testing/insightsModelTests.mjs +714 -0
- package/dist/plugins/insights/testing/modelTypes.d.mts +176 -0
- package/dist/plugins/insights/testing/setupInsightsHttpTestSuite.d.mts +11 -0
- package/dist/plugins/insights/testing/setupInsightsHttpTestSuite.mjs +107 -0
- package/dist/plugins/insights/testing/setupInsightsModelTestSuite.d.mts +11 -0
- package/dist/plugins/insights/testing/setupInsightsModelTestSuite.mjs +31 -0
- package/dist/plugins/insights/testing/types.d.mts +26 -0
- package/dist/{insights → plugins/insights}/types.d.cts +3 -2
- package/dist/{insights → plugins/insights}/types.d.mts +3 -2
- package/package.json +18 -7
- package/src/adapters/drizzle.conformance.integration.spec.ts +2 -2
- package/src/adapters/drizzle.readBudgets.integration.spec.ts +6 -4
- package/src/adapters/drizzle.spec.ts +15 -7
- package/src/adapters/drizzle.ts +6 -6
- package/src/adapters/kysely.conformance.integration.spec.ts +2 -2
- package/src/adapters/kysely.readBudgets.integration.spec.ts +4 -2
- package/src/adapters/kysely.spec.ts +13 -9
- package/src/adapters/kysely.ts +2 -2
- package/src/adapters/mongodb.integration.spec.ts +98 -7
- package/src/adapters/mongodb.ts +2 -2
- package/src/adapters/mongodbAdapter.ts +14 -0
- package/src/adapters/prisma.conformance.integration.spec.ts +2 -2
- package/src/adapters/prisma.integration.spec.ts +16 -8
- package/src/adapters/prisma.readBudgets.integration.spec.ts +5 -2
- package/src/adapters/prisma.spec.ts +17 -9
- package/src/adapters/prisma.ts +6 -6
- package/src/adapters/prismaTestClients.ts +8 -1
- package/src/assembly/assemblePlugins.ts +73 -40
- package/src/assembly/assembly.spec.ts +63 -11
- package/src/assembly/assembly.types.spec.ts +50 -2
- package/src/assembly/clientPlugins.ts +113 -0
- package/src/assembly/configError.ts +4 -0
- package/src/assembly/databasePlugins.ts +11 -5
- package/src/assembly/pluginCli.spec.ts +254 -0
- package/src/assembly/pluginCli.ts +248 -0
- package/src/assembly/{builtInRoutes.spec.ts → pluginRoutes.spec.ts} +10 -37
- package/src/createHotUpdaterCore.ts +21 -17
- package/src/database/aggregateBatching.spec.ts +577 -0
- package/src/database/aggregateBatching.ts +803 -0
- package/src/{insights/errors.spec.ts → database/busy.spec.ts} +2 -5
- package/src/{insights/errors.ts → database/busy.ts} +1 -13
- package/src/database/database.ts +39 -8
- package/src/database/definitions.ts +4 -0
- package/src/database/engineAggregates.spec.ts +4 -7
- package/src/database/engineAggregates.ts +5 -9
- package/src/database/engineReads.ts +7 -5
- package/src/database/engineTransaction.ts +1 -7
- package/src/database/fence.spec.ts +5 -5
- package/src/database/fence.ts +4 -0
- package/src/database/index.ts +13 -7
- package/src/database/kv/kvAdapter.batching.integration.spec.ts +15 -0
- package/src/database/kv/kvAdapter.conformance.integration.spec.ts +2 -0
- package/src/database/kv/kvAdapter.spec.ts +128 -1
- package/src/database/kv/kvAdapter.ts +30 -1
- package/src/database/kv/kvTestStore.ts +28 -1
- package/src/database/resolveSchema.ts +4 -1
- package/src/database/retention.spec.ts +314 -0
- package/src/database/retention.ts +129 -0
- package/src/database/schema.ts +44 -0
- package/src/database/sql/sqlAdapter.integration.spec.ts +4 -7
- package/src/database/sql/sqlAdapter.readBudgets.integration.spec.ts +18 -4
- package/src/database/sql/sqlAdapter.spec.ts +91 -3
- package/src/database/sql/sqlAdapter.ts +20 -0
- package/src/database/sql/sqlSchema.ts +81 -0
- package/src/databaseTestSuite.spec.ts +9 -4
- package/src/db/coreDatabase.ts +153 -0
- package/src/db/engineSql.spec.ts +8 -7
- package/src/db/index.ts +19 -2
- package/src/db/pluginTables.spec.ts +57 -27
- package/src/db/types.ts +3 -3
- package/src/diff.ts +6 -0
- package/src/handler-standalone.integration.spec.ts +2 -3
- package/src/handler.ts +1 -32
- package/src/handlerReleaseCatalogRoutes.ts +26 -2
- package/src/index.ts +0 -10
- package/src/{apiKeys.integration.spec.ts → plugins/api-keys/apiKeys.integration.spec.ts} +4 -4
- package/src/{apiKeys.spec.ts → plugins/api-keys/apiKeys.spec.ts} +3 -3
- package/src/{apiKeys.ts → plugins/api-keys/apiKeys.ts} +1 -1
- package/src/plugins/api-keys/apiKeysPlugin.spec.ts +11 -15
- package/src/plugins/api-keys/cli.spec.ts +187 -0
- package/src/plugins/api-keys/cli.ts +140 -0
- package/src/plugins/api-keys/index.ts +48 -41
- package/src/plugins/api-keys/model.ts +21 -2
- package/src/plugins/definePlugin.ts +144 -2
- package/src/plugins/index.ts +20 -2
- package/src/plugins/insights/contract.spec.ts +285 -0
- package/src/plugins/insights/contract.ts +372 -0
- package/src/{insights → plugins/insights}/domain.ts +73 -15
- package/src/plugins/insights/errors.ts +11 -0
- package/src/{insights → plugins/insights}/eventInput.ts +189 -29
- package/src/plugins/insights/eventRow.ts +209 -0
- package/src/plugins/insights/index.ts +181 -45
- package/src/{insights → plugins/insights}/insights.integration.spec.ts +330 -34
- package/src/plugins/insights/insightsRollout.integration.spec.ts +1 -2
- package/src/plugins/insights/model.spec.ts +3 -5
- package/src/plugins/insights/model.ts +2 -3
- package/src/plugins/insights/modelTypes.ts +197 -0
- package/src/plugins/insights/overview.spec.ts +64 -0
- package/src/plugins/insights/overview.ts +214 -0
- package/src/{insights → plugins/insights}/provider.spec.ts +21 -10
- package/src/{insights → plugins/insights}/provider.ts +58 -21
- package/src/{insights → plugins/insights}/queryInput.ts +2 -2
- package/src/plugins/insights/reads.spec.ts +340 -52
- package/src/plugins/insights/reads.ts +453 -123
- package/src/plugins/insights/recordEvent.spec.ts +474 -20
- package/src/plugins/insights/recordEvent.ts +301 -63
- package/src/plugins/insights/retention.spec.ts +115 -0
- package/src/{insights → plugins/insights}/routes.ts +1 -31
- package/src/plugins/insights/schema.ts +356 -176
- package/src/plugins/insights/testing/expectInsightsIndex.spec.ts +42 -0
- package/src/plugins/insights/testing/expectInsightsIndex.ts +20 -0
- package/src/plugins/insights/testing/fixtures.ts +30 -0
- package/src/plugins/insights/testing/index.ts +45 -0
- package/src/plugins/insights/testing/insightsModelTests.ts +860 -0
- package/src/plugins/insights/testing/setupInsightsHttpTestSuite.ts +152 -0
- package/src/plugins/insights/testing/setupInsightsModelTestSuite.ts +36 -0
- package/src/plugins/insights/testing/types.ts +30 -0
- package/src/{insights → plugins/insights}/types.ts +1 -1
- package/src/readBudgets.testFixtures.ts +2 -2
- package/src/releaseCatalogRoutes.integration.spec.ts +7 -0
- package/src/retention.spec.ts +73 -0
- package/src/runtime.spec.ts +11 -12
- package/src/runtime.testFixtures.ts +7 -2
- package/src/testUtilsMutation.integration.spec.ts +2 -1
- package/src/testUtilsPackage.integration.spec.ts +7 -3
- package/dist/db/builtInDatabase.cjs +0 -104
- package/dist/db/builtInDatabase.d.cts +0 -38
- package/dist/db/builtInDatabase.d.mts +0 -38
- package/dist/db/builtInDatabase.mjs +0 -97
- package/dist/plugins/builtIn.cjs +0 -7
- package/dist/plugins/builtIn.mjs +0 -6
- package/src/db/builtInDatabase.ts +0 -149
- package/src/plugins/builtIn.ts +0 -5
|
@@ -19,8 +19,13 @@ export interface MemoryKeyValueStoreOptions {
|
|
|
19
19
|
*/
|
|
20
20
|
export const createMemoryKeyValueStore = (
|
|
21
21
|
options: MemoryKeyValueStoreOptions = {},
|
|
22
|
-
): KeyValueStore & {
|
|
22
|
+
): KeyValueStore & {
|
|
23
|
+
readonly items: () => number;
|
|
24
|
+
/** Each item's expiry, as a store's native TTL would hold it, by `[pk, sk]` JSON. */
|
|
25
|
+
readonly expiries: () => ReadonlyMap<string, number>;
|
|
26
|
+
} => {
|
|
23
27
|
let partitions = new Map<string, Map<string, StoredRow>>();
|
|
28
|
+
let expiries = new Map<string, number>();
|
|
24
29
|
const read = (key: KvKey) => partitions.get(key.pk)?.get(key.sk) ?? null;
|
|
25
30
|
const holds = (key: KvKey, condition: KvCondition | undefined) => {
|
|
26
31
|
if (condition === undefined) return true;
|
|
@@ -34,6 +39,7 @@ export const createMemoryKeyValueStore = (
|
|
|
34
39
|
limits: options.limits ?? { items: 100, bytes: 4_000_000 },
|
|
35
40
|
items: () =>
|
|
36
41
|
[...partitions.values()].reduce((sum, items) => sum + items.size, 0),
|
|
42
|
+
expiries: () => expiries,
|
|
37
43
|
async get(keys) {
|
|
38
44
|
return keys.map((key) => structuredClone(read(key)));
|
|
39
45
|
},
|
|
@@ -63,15 +69,24 @@ export const createMemoryKeyValueStore = (
|
|
|
63
69
|
const failed = ops.findIndex((op) => !holds(op.key, op.condition));
|
|
64
70
|
if (failed !== -1) return { ok: false, failedOp: failed };
|
|
65
71
|
const next = new Map(partitions);
|
|
72
|
+
const stamped = new Map(expiries);
|
|
66
73
|
for (const op of ops) {
|
|
67
74
|
if (op.type === "check") continue;
|
|
68
75
|
const items = new Map(next.get(op.key.pk));
|
|
69
76
|
next.set(op.key.pk, items);
|
|
77
|
+
const id = JSON.stringify([op.key.pk, op.key.sk]);
|
|
70
78
|
if (op.type === "delete") {
|
|
71
79
|
items.delete(op.key.sk);
|
|
80
|
+
stamped.delete(id);
|
|
72
81
|
} else if (op.type === "put") {
|
|
73
82
|
items.set(op.key.sk, structuredClone(op.value));
|
|
83
|
+
// A put replaces the whole item, and any expiry it had.
|
|
84
|
+
if (op.expiresAt === undefined) stamped.delete(id);
|
|
85
|
+
else stamped.set(id, op.expiresAt);
|
|
74
86
|
} else {
|
|
87
|
+
if (!items.has(op.key.sk) && op.expiresAt !== undefined) {
|
|
88
|
+
stamped.set(id, op.expiresAt);
|
|
89
|
+
}
|
|
75
90
|
const item: Record<string, unknown> = structuredClone(
|
|
76
91
|
items.get(op.key.sk) ?? op.init ?? {},
|
|
77
92
|
);
|
|
@@ -82,10 +97,22 @@ export const createMemoryKeyValueStore = (
|
|
|
82
97
|
}
|
|
83
98
|
}
|
|
84
99
|
partitions = next;
|
|
100
|
+
expiries = stamped;
|
|
85
101
|
return { ok: true };
|
|
86
102
|
},
|
|
103
|
+
async deleteConsumed(keys) {
|
|
104
|
+
const next = new Map(partitions);
|
|
105
|
+
for (const { pk, sk } of keys) {
|
|
106
|
+
const items = new Map(next.get(pk));
|
|
107
|
+
items.delete(sk);
|
|
108
|
+
next.set(pk, items);
|
|
109
|
+
expiries.delete(JSON.stringify([pk, sk]));
|
|
110
|
+
}
|
|
111
|
+
partitions = next;
|
|
112
|
+
},
|
|
87
113
|
async dispose() {
|
|
88
114
|
partitions = new Map();
|
|
115
|
+
expiries = new Map();
|
|
89
116
|
},
|
|
90
117
|
};
|
|
91
118
|
};
|
|
@@ -13,6 +13,7 @@ import type {
|
|
|
13
13
|
SchemaShape,
|
|
14
14
|
TableShape,
|
|
15
15
|
} from "./definitions";
|
|
16
|
+
import { physicalRetention, retentionProblems } from "./retention";
|
|
16
17
|
|
|
17
18
|
export const SHARD_COLUMN = "_shard";
|
|
18
19
|
export const MAX_SHARDS = 64;
|
|
@@ -20,7 +21,7 @@ export const MAX_SHARDS = 64;
|
|
|
20
21
|
export interface SchemaModule {
|
|
21
22
|
readonly id: string;
|
|
22
23
|
readonly schema: SchemaShape;
|
|
23
|
-
/** Prefixes this module's table names;
|
|
24
|
+
/** Prefixes this module's table names; a plugin uses its id unless it sets `namespace: false`. */
|
|
24
25
|
readonly namespace?: string;
|
|
25
26
|
}
|
|
26
27
|
|
|
@@ -240,6 +241,7 @@ const problemsOf = (
|
|
|
240
241
|
}
|
|
241
242
|
if (definition.kind === "table") tableProblems(definition, add, table);
|
|
242
243
|
else aggregateProblems(definition, add);
|
|
244
|
+
for (const problem of retentionProblems(module.schema, model)) add(problem);
|
|
243
245
|
return problems;
|
|
244
246
|
};
|
|
245
247
|
|
|
@@ -355,6 +357,7 @@ export const resolveSchema = (
|
|
|
355
357
|
? [...definition.key, SHARD_COLUMN]
|
|
356
358
|
: [...definition.key],
|
|
357
359
|
indexes: indexesOf(definition),
|
|
360
|
+
...physicalRetention(definition),
|
|
358
361
|
},
|
|
359
362
|
references: references.filter(({ table }) => table === name),
|
|
360
363
|
referencedBy,
|
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
import {
|
|
2
|
+
createMemoryAdapter,
|
|
3
|
+
DatabaseSchemaError,
|
|
4
|
+
expiresAt,
|
|
5
|
+
type PhysicalTable,
|
|
6
|
+
} from "@hot-updater/plugin-core/internal";
|
|
7
|
+
import { describe, expect, it, vi } from "vitest";
|
|
8
|
+
|
|
9
|
+
import { createDatabaseEngine } from "./database";
|
|
10
|
+
import { NegativeGaugeError } from "./engineAggregates";
|
|
11
|
+
import { resolveSchema, type SchemaModule } from "./resolveSchema";
|
|
12
|
+
import { pruneDuringWrites } from "./retention";
|
|
13
|
+
import { defineAggregate, defineTable, type ModuleSchema } from "./schema";
|
|
14
|
+
|
|
15
|
+
const DAY = 86_400_000;
|
|
16
|
+
|
|
17
|
+
const events = defineTable(
|
|
18
|
+
{
|
|
19
|
+
id: { type: "string", maxLength: 36 },
|
|
20
|
+
received_at_ms: { type: "integer" },
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
key: ["id"],
|
|
24
|
+
derived: {
|
|
25
|
+
day: {
|
|
26
|
+
type: "integer",
|
|
27
|
+
compute: (row) => row.received_at_ms - (row.received_at_ms % DAY),
|
|
28
|
+
},
|
|
29
|
+
},
|
|
30
|
+
indexes: { byDay: { eq: ["day"], sort: ["received_at_ms"] } },
|
|
31
|
+
retention: { field: "day", days: 90 },
|
|
32
|
+
},
|
|
33
|
+
);
|
|
34
|
+
|
|
35
|
+
/** Installations counted in the day of their latest event. */
|
|
36
|
+
const latest = defineAggregate(
|
|
37
|
+
{ release: { type: "string", maxLength: 16 }, bucket: { type: "integer" } },
|
|
38
|
+
{
|
|
39
|
+
key: ["release", "bucket"],
|
|
40
|
+
gauges: ["installations"],
|
|
41
|
+
shards: 4,
|
|
42
|
+
indexes: { byRelease: { eq: ["release"], sort: ["bucket"] } },
|
|
43
|
+
retention: { field: "bucket", days: 400 },
|
|
44
|
+
},
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
const kept = defineTable(
|
|
48
|
+
{ id: { type: "string", maxLength: 36 } },
|
|
49
|
+
{ key: ["id"] },
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
const module = <S extends ModuleSchema>(schema: S) => ({
|
|
53
|
+
id: "retention",
|
|
54
|
+
schema,
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
const invalid = (schema: SchemaModule["schema"]) => () =>
|
|
58
|
+
resolveSchema([{ id: "retention", schema }]);
|
|
59
|
+
|
|
60
|
+
describe("retention", () => {
|
|
61
|
+
it("maps a model's retention onto its physical table", () => {
|
|
62
|
+
const { models } = resolveSchema([module({ events, latest, kept })]);
|
|
63
|
+
|
|
64
|
+
expect(models.get("events")!.table.retention).toEqual({
|
|
65
|
+
column: "day",
|
|
66
|
+
ms: 90 * DAY,
|
|
67
|
+
});
|
|
68
|
+
expect(models.get("latest")!.table.retention).toEqual({
|
|
69
|
+
column: "bucket",
|
|
70
|
+
ms: 400 * DAY,
|
|
71
|
+
});
|
|
72
|
+
expect(models.get("kept")!.table).not.toHaveProperty("retention");
|
|
73
|
+
expect(expiresAt(models.get("events")!.table, { day: 10 * DAY })).toBe(
|
|
74
|
+
100 * DAY,
|
|
75
|
+
);
|
|
76
|
+
expect(expiresAt(models.get("events")!.table, { day: null })).toBe(
|
|
77
|
+
undefined,
|
|
78
|
+
);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it("rejects retention on a field that holds no integer, or over no whole days", () => {
|
|
82
|
+
const table = (retention: { field: string; days: number }) =>
|
|
83
|
+
({ ...kept, retention }) as unknown as typeof kept;
|
|
84
|
+
|
|
85
|
+
expect(invalid({ kept: table({ field: "id", days: 1 }) })).toThrow(
|
|
86
|
+
'retention.kept: retention field "id" is not an integer field',
|
|
87
|
+
);
|
|
88
|
+
expect(invalid({ kept: table({ field: "missing", days: 1 }) })).toThrow(
|
|
89
|
+
'retention field "missing" is not an integer field',
|
|
90
|
+
);
|
|
91
|
+
expect(
|
|
92
|
+
invalid({
|
|
93
|
+
events: { ...events, retention: { field: "day", days: 0.5 } },
|
|
94
|
+
}),
|
|
95
|
+
).toThrow("retention days must be a whole number of at least 1");
|
|
96
|
+
expect(
|
|
97
|
+
invalid({ events: { ...events, retention: { field: "day", days: 0 } } }),
|
|
98
|
+
).toThrow(DatabaseSchemaError);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it("rejects retention on a model in a reference or a rooted index", () => {
|
|
102
|
+
const parent = defineTable(
|
|
103
|
+
{
|
|
104
|
+
id: { type: "string", maxLength: 36 },
|
|
105
|
+
at: { type: "integer" },
|
|
106
|
+
},
|
|
107
|
+
{ key: ["id"], retention: { field: "at", days: 1 } },
|
|
108
|
+
);
|
|
109
|
+
const child = defineTable(
|
|
110
|
+
{
|
|
111
|
+
id: { type: "string", maxLength: 36 },
|
|
112
|
+
parent: {
|
|
113
|
+
type: "string",
|
|
114
|
+
maxLength: 36,
|
|
115
|
+
references: { model: "parent", onDelete: "cascade" },
|
|
116
|
+
},
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
key: ["id"],
|
|
120
|
+
indexes: {
|
|
121
|
+
byParent: {
|
|
122
|
+
eq: ["parent"],
|
|
123
|
+
sort: [],
|
|
124
|
+
root: { model: "parent" },
|
|
125
|
+
},
|
|
126
|
+
},
|
|
127
|
+
},
|
|
128
|
+
);
|
|
129
|
+
|
|
130
|
+
expect(invalid({ parent, child })).toThrow(
|
|
131
|
+
"retention.parent: a model with retention is in no reference or rooted index",
|
|
132
|
+
);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
const lease: PhysicalTable = {
|
|
136
|
+
name: "settings",
|
|
137
|
+
columns: [
|
|
138
|
+
{ name: "key", type: "string", nullable: false, maxLength: 255 },
|
|
139
|
+
{ name: "value", type: "string", nullable: false, maxLength: 255 },
|
|
140
|
+
{ name: "_v", type: "integer", nullable: false, default: 0 },
|
|
141
|
+
],
|
|
142
|
+
key: ["key"],
|
|
143
|
+
indexes: [],
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
it("prunes after writes only where rows expire and the backend keeps them", () => {
|
|
147
|
+
const adapter = createMemoryAdapter();
|
|
148
|
+
const { prune: _prune, ...native } = adapter;
|
|
149
|
+
const options = { leaseTable: lease, now: () => 0 };
|
|
150
|
+
|
|
151
|
+
const expiring = resolveSchema([module({ events })]).tables;
|
|
152
|
+
expect(pruneDuringWrites(native, expiring, options)).toBe(native);
|
|
153
|
+
const kepts = resolveSchema([module({ kept })]).tables;
|
|
154
|
+
expect(pruneDuringWrites(adapter, kepts, options)).toBe(adapter);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
it("prunes again in a minute while a pass finds a full batch, else in an hour", async () => {
|
|
158
|
+
const { tables } = resolveSchema([module({ events })]);
|
|
159
|
+
const memory = createMemoryAdapter();
|
|
160
|
+
const prune = vi.fn(async () => 500);
|
|
161
|
+
let now = 100 * DAY;
|
|
162
|
+
const adapter = pruneDuringWrites({ ...memory, prune }, tables, {
|
|
163
|
+
leaseTable: lease,
|
|
164
|
+
now: () => now,
|
|
165
|
+
});
|
|
166
|
+
const writeAt = async (ms: number) => {
|
|
167
|
+
now += ms;
|
|
168
|
+
expect(await adapter.write([])).toEqual({ ok: true });
|
|
169
|
+
};
|
|
170
|
+
|
|
171
|
+
await writeAt(0);
|
|
172
|
+
await writeAt(30_000);
|
|
173
|
+
expect(prune).toHaveBeenCalledTimes(1);
|
|
174
|
+
expect(prune.mock.calls[0]).toEqual([tables[0], 10 * DAY, 500]);
|
|
175
|
+
await writeAt(31_000);
|
|
176
|
+
expect(prune).toHaveBeenCalledTimes(2);
|
|
177
|
+
|
|
178
|
+
prune.mockResolvedValue(3);
|
|
179
|
+
await writeAt(61_000);
|
|
180
|
+
await writeAt(30 * 60_000);
|
|
181
|
+
expect(prune).toHaveBeenCalledTimes(3);
|
|
182
|
+
await writeAt(31 * 60_000);
|
|
183
|
+
expect(prune).toHaveBeenCalledTimes(4);
|
|
184
|
+
const [row] = await memory.get(lease, [["retention.nextPassAt"]]);
|
|
185
|
+
expect(row).toMatchObject({ value: String(now + 3_600_000) });
|
|
186
|
+
});
|
|
187
|
+
|
|
188
|
+
it("writes even when its pass fails", async () => {
|
|
189
|
+
const { tables } = resolveSchema([module({ events })]);
|
|
190
|
+
const warn = vi.spyOn(console, "warn").mockImplementation(() => {});
|
|
191
|
+
const adapter = pruneDuringWrites(
|
|
192
|
+
{
|
|
193
|
+
...createMemoryAdapter(),
|
|
194
|
+
prune: async () => Promise.reject(new Error("down")),
|
|
195
|
+
},
|
|
196
|
+
tables,
|
|
197
|
+
{ leaseTable: lease, now: () => 100 * DAY },
|
|
198
|
+
);
|
|
199
|
+
|
|
200
|
+
await expect(adapter.write([])).resolves.toEqual({ ok: true });
|
|
201
|
+
expect(warn).toHaveBeenCalledWith(
|
|
202
|
+
"[hot-updater] Pruning expired rows failed.",
|
|
203
|
+
expect.objectContaining({ message: "down" }),
|
|
204
|
+
);
|
|
205
|
+
warn.mockRestore();
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
it("prunes each table with retention as of the write, and nothing else", async () => {
|
|
209
|
+
const { tables } = resolveSchema([module({ events, latest, kept })]);
|
|
210
|
+
const prune = vi.fn(
|
|
211
|
+
async (_table: PhysicalTable, _before: number, _limit: number) => 2,
|
|
212
|
+
);
|
|
213
|
+
const adapter = pruneDuringWrites(
|
|
214
|
+
{ ...createMemoryAdapter(), prune },
|
|
215
|
+
tables,
|
|
216
|
+
{ leaseTable: lease, now: () => 1_000 * DAY },
|
|
217
|
+
);
|
|
218
|
+
|
|
219
|
+
await adapter.write([]);
|
|
220
|
+
|
|
221
|
+
expect(
|
|
222
|
+
prune.mock.calls.map(([table, before, limit]) => [
|
|
223
|
+
table.name,
|
|
224
|
+
before,
|
|
225
|
+
limit,
|
|
226
|
+
]),
|
|
227
|
+
).toEqual([
|
|
228
|
+
["events", 910 * DAY, 500],
|
|
229
|
+
["latest", 600 * DAY, 500],
|
|
230
|
+
]);
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
it("deletes expired rows on the reference adapter, and keeps the rest", async () => {
|
|
234
|
+
const schema = resolveSchema([module({ events, kept })]);
|
|
235
|
+
let now = 100 * DAY;
|
|
236
|
+
const adapter = pruneDuringWrites(createMemoryAdapter(), schema.tables, {
|
|
237
|
+
leaseTable: lease,
|
|
238
|
+
now: () => now,
|
|
239
|
+
});
|
|
240
|
+
const db = createDatabaseEngine({ adapter, schema }).database(
|
|
241
|
+
module({ events, kept }),
|
|
242
|
+
);
|
|
243
|
+
|
|
244
|
+
// The write that takes the lease prunes before it stores its rows.
|
|
245
|
+
await db.transaction(async (tx) => {
|
|
246
|
+
tx.create("events", { id: "old", received_at_ms: 1 * DAY + 5 });
|
|
247
|
+
tx.create("events", { id: "new", received_at_ms: 95 * DAY });
|
|
248
|
+
tx.create("kept", { id: "kept" });
|
|
249
|
+
});
|
|
250
|
+
expect(await db.findOne("events", { id: "old" })).not.toBeNull();
|
|
251
|
+
|
|
252
|
+
// Day 1 expires at day 91; day 95 at day 185. The next pass is due in an hour.
|
|
253
|
+
now += 3_600_000;
|
|
254
|
+
await db.transaction(async (tx) => {
|
|
255
|
+
tx.create("kept", { id: "later" });
|
|
256
|
+
});
|
|
257
|
+
expect(await db.findOne("events", { id: "old" })).toBeNull();
|
|
258
|
+
expect(await db.findOne("events", { id: "new" })).not.toBeNull();
|
|
259
|
+
expect(await db.findOne("kept", { id: "kept" })).not.toBeNull();
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
it("drops a gauge decrement of a row retention already deleted", async () => {
|
|
263
|
+
const schema = resolveSchema([module({ latest })]);
|
|
264
|
+
const adapter = createMemoryAdapter();
|
|
265
|
+
const db = createDatabaseEngine({ adapter, schema }).database(
|
|
266
|
+
module({ latest }),
|
|
267
|
+
);
|
|
268
|
+
const move = (bucket: number, installations: number) =>
|
|
269
|
+
db.transaction(async (tx) => {
|
|
270
|
+
tx.aggregate(
|
|
271
|
+
"latest",
|
|
272
|
+
{ release: "r1", bucket },
|
|
273
|
+
{ installations },
|
|
274
|
+
{ shardBy: "install-1" },
|
|
275
|
+
);
|
|
276
|
+
});
|
|
277
|
+
await move(1, 1);
|
|
278
|
+
await adapter.prune!(schema.tables[0]!, 600 * DAY, 10);
|
|
279
|
+
|
|
280
|
+
// The installation's day was pruned; its move out of it changes nothing.
|
|
281
|
+
await move(1, -1);
|
|
282
|
+
await move(900 * DAY, 1);
|
|
283
|
+
|
|
284
|
+
const rows = await db.findAggregates("latest", {
|
|
285
|
+
index: "byRelease",
|
|
286
|
+
where: { release: "r1" },
|
|
287
|
+
limit: 10,
|
|
288
|
+
});
|
|
289
|
+
expect(rows.rows).toEqual([
|
|
290
|
+
{ release: "r1", bucket: 900 * DAY, installations: 1 },
|
|
291
|
+
]);
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
it("still refuses a gauge below zero on a model without retention", async () => {
|
|
295
|
+
const { retention: _retention, ...forever } = latest;
|
|
296
|
+
const schema = resolveSchema([module({ latest: forever })]);
|
|
297
|
+
const db = createDatabaseEngine({
|
|
298
|
+
adapter: createMemoryAdapter(),
|
|
299
|
+
schema,
|
|
300
|
+
retry: { attempts: 2, baseDelayMs: 0 },
|
|
301
|
+
}).database(module({ latest: forever }));
|
|
302
|
+
|
|
303
|
+
await expect(
|
|
304
|
+
db.transaction(async (tx) => {
|
|
305
|
+
tx.aggregate(
|
|
306
|
+
"latest",
|
|
307
|
+
{ release: "r1", bucket: 1 },
|
|
308
|
+
{ installations: -1 },
|
|
309
|
+
{ shardBy: "install-1" },
|
|
310
|
+
);
|
|
311
|
+
}),
|
|
312
|
+
).rejects.toThrow(NegativeGaugeError);
|
|
313
|
+
});
|
|
314
|
+
});
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
DatabaseAdapter,
|
|
3
|
+
PhysicalRetention,
|
|
4
|
+
PhysicalTable,
|
|
5
|
+
StoredRow,
|
|
6
|
+
WriteOp,
|
|
7
|
+
} from "@hot-updater/plugin-core/internal";
|
|
8
|
+
|
|
9
|
+
import type { ModelShape, SchemaShape } from "./definitions";
|
|
10
|
+
|
|
11
|
+
const MINUTE_MS = 60_000;
|
|
12
|
+
const HOUR_MS = 60 * MINUTE_MS;
|
|
13
|
+
const DAY_MS = 24 * HOUR_MS;
|
|
14
|
+
|
|
15
|
+
/** A model's retention as its physical table carries it, which each adapter maps natively. */
|
|
16
|
+
export const physicalRetention = ({
|
|
17
|
+
retention,
|
|
18
|
+
}: ModelShape): { readonly retention?: PhysicalRetention } =>
|
|
19
|
+
retention === undefined
|
|
20
|
+
? {}
|
|
21
|
+
: { retention: { column: retention.field, ms: retention.days * DAY_MS } };
|
|
22
|
+
|
|
23
|
+
/** Models in a reference or a rooted index, which an expired row's delete would break. */
|
|
24
|
+
const linkedModels = (schema: SchemaShape) =>
|
|
25
|
+
Object.entries(schema).flatMap(([name, { fields, indexes }]) =>
|
|
26
|
+
[
|
|
27
|
+
...Object.values(fields).map(({ references }) => references),
|
|
28
|
+
...Object.values(indexes).map(({ root }) => root),
|
|
29
|
+
].flatMap((link) => (link ? [name, link.model] : [])),
|
|
30
|
+
);
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* What is wrong with a model's retention: it counts from an integer field,
|
|
34
|
+
* over whole days, and expired rows are deleted without cascades, counter
|
|
35
|
+
* moves, or root bumps, so the model is in no reference or rooted index.
|
|
36
|
+
*/
|
|
37
|
+
export const retentionProblems = (
|
|
38
|
+
schema: SchemaShape,
|
|
39
|
+
model: string,
|
|
40
|
+
): string[] => {
|
|
41
|
+
const definition = schema[model]!;
|
|
42
|
+
if (definition.retention === undefined) return [];
|
|
43
|
+
const { field, days } = definition.retention;
|
|
44
|
+
const declared =
|
|
45
|
+
definition.fields[field] ??
|
|
46
|
+
(definition.kind === "table" ? definition.derived[field] : undefined);
|
|
47
|
+
return [
|
|
48
|
+
...(declared?.type !== "integer" ||
|
|
49
|
+
("multi" in declared && declared.multi === true)
|
|
50
|
+
? [`retention field "${field}" is not an integer field`]
|
|
51
|
+
: []),
|
|
52
|
+
...(Number.isSafeInteger(days) && days >= 1
|
|
53
|
+
? []
|
|
54
|
+
: ["retention days must be a whole number of at least 1"]),
|
|
55
|
+
...(linkedModels(schema).includes(model)
|
|
56
|
+
? ["a model with retention is in no reference or rooted index"]
|
|
57
|
+
: []),
|
|
58
|
+
];
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Rows a pass deletes from each table: one statement or two a table, so a
|
|
63
|
+
* pass stays well inside D1's 50 queries an invocation.
|
|
64
|
+
*/
|
|
65
|
+
const PASS_LIMIT = 500;
|
|
66
|
+
/** The lease row: when the next pass is due, in epoch ms. */
|
|
67
|
+
const LEASE_KEY = "retention.nextPassAt";
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* The adapter, pruning during its writes with no scheduler: once the lease
|
|
71
|
+
* row in `leaseTable` is due, the write that takes it first runs one bounded
|
|
72
|
+
* pass over the tables with retention, and other writers skip. The next pass is due in an hour, or in a minute while a pass still
|
|
73
|
+
* found a full batch of expired rows in some table. A backend that expires
|
|
74
|
+
* rows natively, or a schema without retention, gets its adapter back.
|
|
75
|
+
*/
|
|
76
|
+
export const pruneDuringWrites = (
|
|
77
|
+
adapter: DatabaseAdapter,
|
|
78
|
+
tables: readonly PhysicalTable[],
|
|
79
|
+
{ leaseTable: table, now }: { leaseTable: PhysicalTable; now: () => number },
|
|
80
|
+
): DatabaseAdapter => {
|
|
81
|
+
const expiring = tables.filter(({ retention }) => retention !== undefined);
|
|
82
|
+
if (adapter.prune === undefined || expiring.length === 0) return adapter;
|
|
83
|
+
const key = [LEASE_KEY];
|
|
84
|
+
let checkAt = 0;
|
|
85
|
+
// Moves the lease to `due` from `row`: the row it wrote, or null when
|
|
86
|
+
// another writer moved the lease first.
|
|
87
|
+
const lease = async (row: StoredRow | null, due: number) => {
|
|
88
|
+
const value = String(due);
|
|
89
|
+
const guard = { v: row === null ? -1 : Number(row._v) };
|
|
90
|
+
const next = { key: LEASE_KEY, value, _v: guard.v + 1 };
|
|
91
|
+
const op: WriteOp =
|
|
92
|
+
row === null
|
|
93
|
+
? { type: "insert", table, row: next }
|
|
94
|
+
: { type: "patch", table, key, set: { value }, guard, previous: row };
|
|
95
|
+
return (await adapter.write([op])).ok ? next : null;
|
|
96
|
+
};
|
|
97
|
+
const pass = async () => {
|
|
98
|
+
const at = now();
|
|
99
|
+
if (at < checkAt) return;
|
|
100
|
+
checkAt = at + MINUTE_MS;
|
|
101
|
+
const [row = null] = await adapter.get(table, [key]);
|
|
102
|
+
// Due, or missing: this process reads it again if another takes it first.
|
|
103
|
+
checkAt = row === null ? at : Number(row.value);
|
|
104
|
+
if (checkAt > at) return;
|
|
105
|
+
const held = await lease(row, at + HOUR_MS);
|
|
106
|
+
if (held === null) return;
|
|
107
|
+
checkAt = at + HOUR_MS;
|
|
108
|
+
const deleted: number[] = [];
|
|
109
|
+
for (const t of expiring) {
|
|
110
|
+
deleted.push(await adapter.prune!(t, at - t.retention!.ms, PASS_LIMIT));
|
|
111
|
+
}
|
|
112
|
+
const backlog = deleted.includes(PASS_LIMIT);
|
|
113
|
+
if (backlog && (await lease(held, at + MINUTE_MS)) !== null) {
|
|
114
|
+
checkAt = at + MINUTE_MS;
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
const warn = (error: unknown) =>
|
|
118
|
+
console.warn("[hot-updater] Pruning expired rows failed.", error);
|
|
119
|
+
// Inherits every other member, so it reads the adapter as it is at each
|
|
120
|
+
// call, a proxy's included.
|
|
121
|
+
return Object.assign(Object.create(adapter) as DatabaseAdapter, {
|
|
122
|
+
// Before the write, so a pass never deletes what that write stores:
|
|
123
|
+
// rows written already expired go at the next pass.
|
|
124
|
+
async write(ops: readonly WriteOp[]) {
|
|
125
|
+
await pass().catch(warn);
|
|
126
|
+
return adapter.write(ops);
|
|
127
|
+
},
|
|
128
|
+
});
|
|
129
|
+
};
|
package/src/database/schema.ts
CHANGED
|
@@ -16,6 +16,17 @@ export type {
|
|
|
16
16
|
ReferenceAction,
|
|
17
17
|
} from "./definitions";
|
|
18
18
|
|
|
19
|
+
/**
|
|
20
|
+
* How long a model keeps its rows: `days` past the epoch milliseconds in
|
|
21
|
+
* `field`, an integer field (a table's derived one too, where null keeps
|
|
22
|
+
* the row). Expired rows are deleted without cascades, so a model with
|
|
23
|
+
* retention is in no reference and no rooted index.
|
|
24
|
+
*/
|
|
25
|
+
export interface RetentionDefinition<TField extends string = string> {
|
|
26
|
+
readonly field: TField;
|
|
27
|
+
readonly days: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
19
30
|
export type FieldValue<TType extends FieldType> = TType extends "string"
|
|
20
31
|
? string
|
|
21
32
|
: TType extends "integer" | "number"
|
|
@@ -63,6 +74,17 @@ export type DerivedFields<
|
|
|
63
74
|
|
|
64
75
|
type Indexes = Readonly<Record<string, IndexDefinition>>;
|
|
65
76
|
|
|
77
|
+
/** The integer fields, declared or derived, that retention can count from. */
|
|
78
|
+
type IntegerFields<TFields extends Fields, TDerived extends DerivedTypes> =
|
|
79
|
+
| {
|
|
80
|
+
[K in keyof TFields]: TFields[K] extends { readonly type: "integer" }
|
|
81
|
+
? K
|
|
82
|
+
: never;
|
|
83
|
+
}[keyof TFields]
|
|
84
|
+
| {
|
|
85
|
+
[K in keyof TDerived]: TDerived[K] extends "integer" ? K : never;
|
|
86
|
+
}[keyof TDerived];
|
|
87
|
+
|
|
66
88
|
type MissingFields<TIndex, TNames extends string> = TIndex extends {
|
|
67
89
|
readonly eq: readonly (infer TEq)[];
|
|
68
90
|
readonly sort: readonly (infer TSort)[];
|
|
@@ -109,6 +131,10 @@ export const defineTable = <
|
|
|
109
131
|
readonly derived?: DerivedFields<TFields, TDerived>;
|
|
110
132
|
readonly indexes?: TIndexes &
|
|
111
133
|
CheckIndexes<TIndexes, (keyof TFields | keyof TDerived) & string>;
|
|
134
|
+
/** Rows expire `days` after the epoch milliseconds in `field`. */
|
|
135
|
+
readonly retention?: RetentionDefinition<
|
|
136
|
+
IntegerFields<TFields, TDerived> & string
|
|
137
|
+
>;
|
|
112
138
|
},
|
|
113
139
|
): TableDefinition<TFields, TDerived, TIndexes, TKey> => ({
|
|
114
140
|
kind: "table",
|
|
@@ -116,6 +142,7 @@ export const defineTable = <
|
|
|
116
142
|
key: options.key,
|
|
117
143
|
derived: options.derived ?? ({} as DerivedFields<TFields, TDerived>),
|
|
118
144
|
indexes: options.indexes ?? ({} as TIndexes),
|
|
145
|
+
...(options.retention === undefined ? {} : { retention: options.retention }),
|
|
119
146
|
});
|
|
120
147
|
|
|
121
148
|
export interface AggregateDefinition<
|
|
@@ -133,6 +160,12 @@ export interface AggregateDefinition<
|
|
|
133
160
|
readonly gauges: readonly TMetric[];
|
|
134
161
|
readonly distinct: readonly TSketch[];
|
|
135
162
|
readonly indexes: TIndexes;
|
|
163
|
+
/**
|
|
164
|
+
* On a database that batches aggregates, changes apply after their
|
|
165
|
+
* transaction commits, in one write per row with other transactions'
|
|
166
|
+
* changes, so reads may lag them. Elsewhere they commit with it.
|
|
167
|
+
*/
|
|
168
|
+
readonly batched?: true;
|
|
136
169
|
}
|
|
137
170
|
|
|
138
171
|
export const defineAggregate = <
|
|
@@ -151,6 +184,15 @@ export const defineAggregate = <
|
|
|
151
184
|
readonly shards?: number;
|
|
152
185
|
readonly indexes?: TIndexes &
|
|
153
186
|
CheckIndexes<TIndexes, keyof TFields & string>;
|
|
187
|
+
readonly batched?: true;
|
|
188
|
+
/**
|
|
189
|
+
* Rows expire `days` after the epoch milliseconds in `field`, an
|
|
190
|
+
* identity field such as a bucket's start; a change to a row already
|
|
191
|
+
* past it is dropped, so a late gauge decrement never goes below zero.
|
|
192
|
+
*/
|
|
193
|
+
readonly retention?: RetentionDefinition<
|
|
194
|
+
IntegerFields<TFields, {}> & string
|
|
195
|
+
>;
|
|
154
196
|
},
|
|
155
197
|
): AggregateDefinition<TFields, TMetric, TIndexes, TSketch, TKey> => ({
|
|
156
198
|
kind: "aggregate",
|
|
@@ -161,6 +203,8 @@ export const defineAggregate = <
|
|
|
161
203
|
distinct: options.distinct ?? [],
|
|
162
204
|
shards: options.shards ?? 1,
|
|
163
205
|
indexes: options.indexes ?? ({} as TIndexes),
|
|
206
|
+
...(options.batched ? { batched: true } : {}),
|
|
207
|
+
...(options.retention === undefined ? {} : { retention: options.retention }),
|
|
164
208
|
});
|
|
165
209
|
|
|
166
210
|
export type ModelDefinition = TableDefinition | AggregateDefinition;
|
|
@@ -10,10 +10,7 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
|
10
10
|
|
|
11
11
|
import { createBundleFixture } from "../../../../test-utils/src/databaseTestFixtures";
|
|
12
12
|
import { createInProcessCoreApi } from "../../core/api";
|
|
13
|
-
import {
|
|
14
|
-
createEngineDatabase,
|
|
15
|
-
migrateBuiltInSchema,
|
|
16
|
-
} from "../../db/builtInDatabase";
|
|
13
|
+
import { createEngineDatabase, migrateCoreSchema } from "../../db/coreDatabase";
|
|
17
14
|
import {
|
|
18
15
|
createSqlAdapter,
|
|
19
16
|
type SqlExecutor,
|
|
@@ -110,15 +107,15 @@ setupDatabaseAdapterConformanceSuite({
|
|
|
110
107
|
describe.each([
|
|
111
108
|
["PostgreSQL", () => pgExecutor(postgres)],
|
|
112
109
|
["MySQL", () => mysqlExecutor(mariadb)],
|
|
113
|
-
] as const)("
|
|
110
|
+
] as const)("core's schema on pooled %s", (name, executor) => {
|
|
114
111
|
it("migrates, passes the fence, and deploys into the longest scope key", async () => {
|
|
115
112
|
tests += 1;
|
|
116
113
|
const adapter = createSqlAdapter({
|
|
117
114
|
executor: executor(),
|
|
118
115
|
tablePrefix: `f${tests}_`,
|
|
119
116
|
});
|
|
120
|
-
await
|
|
121
|
-
await
|
|
117
|
+
await migrateCoreSchema(adapter, name);
|
|
118
|
+
await migrateCoreSchema(adapter, name);
|
|
122
119
|
const core = createInProcessCoreApi(
|
|
123
120
|
createEngineDatabase({ name, adapter }).adapter,
|
|
124
121
|
);
|