@hot-updater/azure 0.0.0-stage → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +33 -0
- package/dist/functions/host.json +8 -0
- package/dist/functions/index.mjs +70656 -0
- package/dist/index.cjs +737 -0
- package/dist/index.d.cts +71 -0
- package/dist/index.d.mts +71 -0
- package/dist/index.mjs +734 -0
- package/dist/init/index.cjs +7829 -0
- package/dist/init/index.d.cts +99 -0
- package/dist/init/index.d.mts +99 -0
- package/dist/init/index.mjs +7821 -0
- package/package.json +59 -3
- package/README.md +0 -3
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,734 @@
|
|
|
1
|
+
import { isTokenCredential } from "@azure/core-auth";
|
|
2
|
+
import { BlobSASPermissions, BlobServiceClient, RestError, StorageSharedKeyCredential, generateBlobSASQueryParameters } from "@azure/storage-blob";
|
|
3
|
+
import { SETTINGS_TABLE, createEngineDatabase, createKvAdapter, createStorageAdapter, createStorageKeyBuilder, createStorageUri, encodeKvKey, migrateCoreSchema, parseStorageUri, toolingTargetOf } from "@hot-updater/plugin-core";
|
|
4
|
+
import { CosmosClient } from "@azure/cosmos";
|
|
5
|
+
import { createHash } from "node:crypto";
|
|
6
|
+
//#region src/azureBlobStorage.ts
|
|
7
|
+
const PROTOCOL = "azblob";
|
|
8
|
+
/** Devices download right after the update check that returns the URL. */
|
|
9
|
+
const SAS_EXPIRES_IN_MS = 36e5;
|
|
10
|
+
/** A user delegation key signs SAS URLs for up to seven days; it is renewed well before. */
|
|
11
|
+
const DELEGATION_KEY_LIFETIME_MS = 864e5;
|
|
12
|
+
const isNotFound = (error) => error instanceof RestError && error.statusCode === 404;
|
|
13
|
+
const serviceOf = (config) => "connectionString" in config ? BlobServiceClient.fromConnectionString(config.connectionString) : new BlobServiceClient(config.accountUrl, config.credential);
|
|
14
|
+
/**
|
|
15
|
+
* Bundles in an Azure Blob Storage container. Without `getDownloadUrl`,
|
|
16
|
+
* devices download through SAS URLs: signed with the account key when the
|
|
17
|
+
* connection has one, otherwise with a user delegation key, which needs the
|
|
18
|
+
* identity's Storage Blob Delegator role.
|
|
19
|
+
*/
|
|
20
|
+
function azureBlobStorage(config) {
|
|
21
|
+
const { containerName, basePath, getDownloadUrl } = config;
|
|
22
|
+
const service = serviceOf(config);
|
|
23
|
+
const container = service.getContainerClient(containerName);
|
|
24
|
+
const normalizedBasePath = basePath?.replace(/^\/+|\/+$/g, "") ?? "";
|
|
25
|
+
const getStorageKey = createStorageKeyBuilder(normalizedBasePath);
|
|
26
|
+
const getListPrefix = (prefix = "") => {
|
|
27
|
+
const value = [normalizedBasePath, prefix.replace(/^\/+|\/+$/g, "")].filter(Boolean).join("/");
|
|
28
|
+
return value ? `${value}/` : "";
|
|
29
|
+
};
|
|
30
|
+
const getRelativeKey = (key) => {
|
|
31
|
+
const basePrefix = `${normalizedBasePath}/`;
|
|
32
|
+
return normalizedBasePath && key.startsWith(basePrefix) ? key.slice(basePrefix.length) : key;
|
|
33
|
+
};
|
|
34
|
+
/** The URI `put` returns for the key, or null for a key no `put` writes. */
|
|
35
|
+
const getListedStorageUri = (key) => {
|
|
36
|
+
try {
|
|
37
|
+
return createStorageUri({
|
|
38
|
+
bucket: containerName,
|
|
39
|
+
key,
|
|
40
|
+
protocol: PROTOCOL
|
|
41
|
+
});
|
|
42
|
+
} catch {
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
const parseAndValidate = (storageUri) => {
|
|
47
|
+
const parsed = parseStorageUri(storageUri, PROTOCOL);
|
|
48
|
+
if (parsed.bucket !== containerName) throw new Error(`Container name mismatch: expected "${containerName}", but found "${parsed.bucket}".`);
|
|
49
|
+
return parsed;
|
|
50
|
+
};
|
|
51
|
+
let delegation;
|
|
52
|
+
const delegationKey = () => {
|
|
53
|
+
const now = Date.now();
|
|
54
|
+
if (delegation === void 0 || delegation.renewAt <= now) {
|
|
55
|
+
const key = service.getUserDelegationKey(/* @__PURE__ */ new Date(now - 3e5), new Date(now + DELEGATION_KEY_LIFETIME_MS));
|
|
56
|
+
key.catch(() => {
|
|
57
|
+
delegation = void 0;
|
|
58
|
+
});
|
|
59
|
+
delegation = {
|
|
60
|
+
key,
|
|
61
|
+
renewAt: now + DELEGATION_KEY_LIFETIME_MS - SAS_EXPIRES_IN_MS
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
return delegation.key;
|
|
65
|
+
};
|
|
66
|
+
const signedUrl = async (key) => {
|
|
67
|
+
const blob = container.getBlobClient(key);
|
|
68
|
+
const expiresOn = new Date(Date.now() + SAS_EXPIRES_IN_MS);
|
|
69
|
+
const permissions = BlobSASPermissions.parse("r");
|
|
70
|
+
if (blob.credential instanceof StorageSharedKeyCredential) return blob.generateSasUrl({
|
|
71
|
+
permissions,
|
|
72
|
+
expiresOn
|
|
73
|
+
});
|
|
74
|
+
if (!isTokenCredential(blob.credential)) throw new Error("azureBlobStorage signs download URLs with an account key or a Microsoft Entra credential; a SAS connection string has neither. Pass getDownloadUrl, or connect with the account key.");
|
|
75
|
+
const sas = generateBlobSASQueryParameters({
|
|
76
|
+
containerName,
|
|
77
|
+
blobName: key,
|
|
78
|
+
permissions,
|
|
79
|
+
expiresOn
|
|
80
|
+
}, await delegationKey(), blob.accountName);
|
|
81
|
+
return `${blob.url}?${sas.toString()}`;
|
|
82
|
+
};
|
|
83
|
+
return createStorageAdapter({
|
|
84
|
+
name: "azureBlobStorage",
|
|
85
|
+
protocol: PROTOCOL,
|
|
86
|
+
async listObjects(prefix) {
|
|
87
|
+
const objects = [];
|
|
88
|
+
for await (const blob of container.listBlobsFlat({ prefix: getListPrefix(prefix) })) {
|
|
89
|
+
const storageUri = getListedStorageUri(blob.name);
|
|
90
|
+
if (storageUri === null) continue;
|
|
91
|
+
objects.push({
|
|
92
|
+
key: getRelativeKey(blob.name),
|
|
93
|
+
lastModifiedAt: blob.properties.lastModified,
|
|
94
|
+
size: blob.properties.contentLength ?? 0,
|
|
95
|
+
storageUri
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
return objects;
|
|
99
|
+
},
|
|
100
|
+
async deleteObjects(relativeKeys) {
|
|
101
|
+
const keys = relativeKeys.map((key) => getStorageKey(key));
|
|
102
|
+
for (let offset = 0; offset < keys.length; offset += 256) {
|
|
103
|
+
const batch = keys.slice(offset, offset + 256);
|
|
104
|
+
if (batch.length === 0) continue;
|
|
105
|
+
const failed = (await container.getBlobBatchClient().deleteBlobs(batch.map((key) => container.getBlobClient(key)))).subResponses.filter(({ status }) => status >= 300 && status !== 404);
|
|
106
|
+
if (failed.length > 0) throw new Error(`Failed to delete ${failed.length} Azure blob(s).`);
|
|
107
|
+
}
|
|
108
|
+
},
|
|
109
|
+
async put({ key, body, contentType }) {
|
|
110
|
+
const storageKey = getStorageKey(key);
|
|
111
|
+
const bytes = new Uint8Array(await new Response(body).arrayBuffer());
|
|
112
|
+
await container.getBlockBlobClient(storageKey).uploadData(bytes, { blobHTTPHeaders: {
|
|
113
|
+
blobContentType: contentType,
|
|
114
|
+
blobCacheControl: "max-age=31536000"
|
|
115
|
+
} });
|
|
116
|
+
return { storageUri: createStorageUri({
|
|
117
|
+
bucket: containerName,
|
|
118
|
+
key: storageKey,
|
|
119
|
+
protocol: PROTOCOL
|
|
120
|
+
}) };
|
|
121
|
+
},
|
|
122
|
+
async get({ storageUri }) {
|
|
123
|
+
const { key } = parseAndValidate(storageUri);
|
|
124
|
+
try {
|
|
125
|
+
const download = await container.getBlobClient(key).download();
|
|
126
|
+
const body = download.readableStreamBody;
|
|
127
|
+
if (body === void 0) return { response: null };
|
|
128
|
+
const headers = new Headers();
|
|
129
|
+
if (download.contentType) headers.set("content-type", download.contentType);
|
|
130
|
+
if (download.contentLength !== void 0) headers.set("content-length", String(download.contentLength));
|
|
131
|
+
const { Readable } = await import("node:stream");
|
|
132
|
+
return { response: new Response(Readable.toWeb(body), { headers }) };
|
|
133
|
+
} catch (error) {
|
|
134
|
+
if (isNotFound(error)) return { response: null };
|
|
135
|
+
throw error;
|
|
136
|
+
}
|
|
137
|
+
},
|
|
138
|
+
async getDownloadUrl({ storageUri }) {
|
|
139
|
+
const { key } = parseAndValidate(storageUri);
|
|
140
|
+
if (getDownloadUrl) return getDownloadUrl({ storageUri });
|
|
141
|
+
return { url: await signedUrl(key) };
|
|
142
|
+
},
|
|
143
|
+
async exists({ storageUri }) {
|
|
144
|
+
const { key } = parseAndValidate(storageUri);
|
|
145
|
+
return { exists: await container.getBlobClient(key).exists() };
|
|
146
|
+
},
|
|
147
|
+
async delete({ storageUri }) {
|
|
148
|
+
const { key } = parseAndValidate(storageUri);
|
|
149
|
+
await container.getBlobClient(key).deleteIfExists();
|
|
150
|
+
return { deleted: true };
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
//#endregion
|
|
155
|
+
//#region src/azureInfrastructureNames.ts
|
|
156
|
+
/** The Cosmos DB database `hot-updater init` creates. */
|
|
157
|
+
const AZURE_COSMOS_V1_DATABASE = "hot-updater";
|
|
158
|
+
/**
|
|
159
|
+
* The one container the storage engine keeps every item in. It namespaces
|
|
160
|
+
* the data, so item partitions are table names with no prefix.
|
|
161
|
+
*/
|
|
162
|
+
const AZURE_COSMOS_V1_CONTAINER = "hot_updater_v1";
|
|
163
|
+
//#endregion
|
|
164
|
+
//#region src/cosmosStore.ts
|
|
165
|
+
/**
|
|
166
|
+
* Cosmos DB commits a transactional batch only inside one logical
|
|
167
|
+
* partition, and the engine's writes span tables and their index items, so
|
|
168
|
+
* every item shares this partition key value. One logical partition holds
|
|
169
|
+
* at most 20 GB and serves at most 10,000 RU/s.
|
|
170
|
+
*/
|
|
171
|
+
const COSMOS_PARTITION = "hot-updater";
|
|
172
|
+
/** The container's partition key path, whose value is always `COSMOS_PARTITION`. */
|
|
173
|
+
const COSMOS_PARTITION_KEY_PATH = "/partition";
|
|
174
|
+
/**
|
|
175
|
+
* A transactional batch takes 100 operations and a 2 MB request, an item
|
|
176
|
+
* 2 MB. The kit measures a row's JSON, but a map or array value is stored
|
|
177
|
+
* as JSON text whose quotes and backslashes escape again, so an item takes
|
|
178
|
+
* up to twice the measured bytes; the rest leaves room for each item's
|
|
179
|
+
* envelope. Every op takes one operation but an `exists: false` check or
|
|
180
|
+
* delete, which takes two; the engine writes neither.
|
|
181
|
+
*/
|
|
182
|
+
const COSMOS_LIMITS = {
|
|
183
|
+
items: 100,
|
|
184
|
+
bytes: 1e6,
|
|
185
|
+
itemBytes: 1e6
|
|
186
|
+
};
|
|
187
|
+
/** The most operations one transactional batch takes. */
|
|
188
|
+
const BATCH_OPERATIONS = 100;
|
|
189
|
+
/** The most bytes of operations a transactional batch request takes, below its 2 MB. */
|
|
190
|
+
const BATCH_BYTES = 2e6;
|
|
191
|
+
const encoder = new TextEncoder();
|
|
192
|
+
/** A query reads at most this many ids a call. */
|
|
193
|
+
const GET_BATCH = 100;
|
|
194
|
+
/** A patch takes at most 10 operations. */
|
|
195
|
+
const PATCH_OPERATIONS = 10;
|
|
196
|
+
/**
|
|
197
|
+
* The container's indexing: only the id `get` looks up, and the partition
|
|
198
|
+
* and sort keys a range read filters and orders by, through one composite
|
|
199
|
+
* index that serves either order. Rows are never filtered on, so they are not indexed.
|
|
200
|
+
*/
|
|
201
|
+
const COSMOS_INDEXING_POLICY = {
|
|
202
|
+
indexingMode: "consistent",
|
|
203
|
+
automatic: true,
|
|
204
|
+
includedPaths: [{ path: "/pk/?" }, { path: "/sk/?" }],
|
|
205
|
+
excludedPaths: [{ path: "/*" }],
|
|
206
|
+
compositeIndexes: [[{
|
|
207
|
+
path: "/pk",
|
|
208
|
+
order: "ascending"
|
|
209
|
+
}, {
|
|
210
|
+
path: "/sk",
|
|
211
|
+
order: "ascending"
|
|
212
|
+
}]]
|
|
213
|
+
};
|
|
214
|
+
/** An item's id: a hash of its key, since keys may hold `/` and Cosmos DB ids take at most 1,023 bytes. */
|
|
215
|
+
const cosmosItemId = ({ pk, sk }) => createHash("sha256").update(JSON.stringify([pk, sk])).digest("hex");
|
|
216
|
+
/**
|
|
217
|
+
* An item is `{ id, partition, pk, sk, row, ttl, expireAt }`. Row values
|
|
218
|
+
* that are maps or arrays become JSON text inside a map, so a read returns
|
|
219
|
+
* them exactly as written, map key order included.
|
|
220
|
+
*/
|
|
221
|
+
const encode = (value) => value !== null && typeof value === "object" ? { json: JSON.stringify(value) } : value;
|
|
222
|
+
const decode = (value) => value !== null && typeof value === "object" && "json" in value ? JSON.parse(String(value.json)) : value;
|
|
223
|
+
const encodeRow = (row) => Object.fromEntries(Object.entries(row).map(([name, value]) => [name, encode(value)]));
|
|
224
|
+
const toRow = (row) => Object.fromEntries(Object.entries(row ?? {}).map(([name, value]) => [name, decode(value)]));
|
|
225
|
+
/**
|
|
226
|
+
* The least TTL an item gets. Cosmos DB hides an item the moment its TTL
|
|
227
|
+
* passes, where DynamoDB and Firestore delete expired items within a day or
|
|
228
|
+
* two and read them until then; an item written already past its retention,
|
|
229
|
+
* such as a late Insights event, lives this long rather than vanishing
|
|
230
|
+
* under the write that made it.
|
|
231
|
+
*/
|
|
232
|
+
const MIN_TTL_SECONDS = 86400;
|
|
233
|
+
/**
|
|
234
|
+
* Cosmos DB's TTL counts seconds from an item's last write, so every write
|
|
235
|
+
* sets it again from the absolute `expireAt`; `-1` never expires.
|
|
236
|
+
*/
|
|
237
|
+
const ttlOf = (expireAt) => expireAt === null || expireAt === void 0 ? -1 : Math.max(MIN_TTL_SECONDS, Math.ceil((expireAt - Date.now()) / 1e3));
|
|
238
|
+
/** The item Cosmos DB stores for a row. */
|
|
239
|
+
const toItem = (key, row, expiresAt) => ({
|
|
240
|
+
id: cosmosItemId(key),
|
|
241
|
+
partition: COSMOS_PARTITION,
|
|
242
|
+
pk: key.pk,
|
|
243
|
+
sk: key.sk,
|
|
244
|
+
row: encodeRow(row),
|
|
245
|
+
expireAt: expiresAt ?? null,
|
|
246
|
+
ttl: ttlOf(expiresAt)
|
|
247
|
+
});
|
|
248
|
+
/** An item that exists only between a Create and a Delete of one batch, to guard or to delete a key whether or not it exists. */
|
|
249
|
+
const placeholderOf = (key) => ({
|
|
250
|
+
id: cosmosItemId(key),
|
|
251
|
+
partition: COSMOS_PARTITION
|
|
252
|
+
});
|
|
253
|
+
/** Ids as an `IN` list: they are hex digests, so they need no parameters. */
|
|
254
|
+
const idList = (ids) => ids.map((id) => JSON.stringify(id)).join(", ");
|
|
255
|
+
/** A JSON Pointer to a row column. */
|
|
256
|
+
const columnPath = (name) => `/row/${name.replaceAll("~", "~0").replaceAll("/", "~1")}`;
|
|
257
|
+
const guardOf = (v) => `FROM c WHERE c.row._v = ${JSON.stringify(v)}`;
|
|
258
|
+
/** Statuses a rerun of the whole write can clear: timeout, throttling, retry-with, unavailable. */
|
|
259
|
+
const TRANSIENT = /* @__PURE__ */ new Set([
|
|
260
|
+
408,
|
|
261
|
+
429,
|
|
262
|
+
449,
|
|
263
|
+
503
|
|
264
|
+
]);
|
|
265
|
+
/** An error's HTTP status, or its cause's: the SDK wraps some answers, such as a failed batch's. */
|
|
266
|
+
const statusOf = (error) => {
|
|
267
|
+
for (let at = error; at !== null && typeof at === "object"; at = at.cause) {
|
|
268
|
+
const { statusCode, code } = at;
|
|
269
|
+
const status = statusCode ?? code;
|
|
270
|
+
if (typeof status === "number") return status;
|
|
271
|
+
}
|
|
272
|
+
};
|
|
273
|
+
/** A missing database or container reads as a missing schema, like SQL's 42P01. */
|
|
274
|
+
const missingContainer = (error) => {
|
|
275
|
+
throw statusOf(error) === 404 ? Object.assign(new Error(error.message, { cause: error }), { code: "42P01" }) : error;
|
|
276
|
+
};
|
|
277
|
+
/** The header of a container read that reports its index transformation, in percent. */
|
|
278
|
+
const INDEX_TRANSFORMATION_PROGRESS = "x-ms-documentdb-collection-index-transformation-progress";
|
|
279
|
+
/** How long a migration waits for a changed indexing policy to apply. */
|
|
280
|
+
const INDEX_TRANSFORMATION_TIMEOUT_MS = 18e5;
|
|
281
|
+
/**
|
|
282
|
+
* Waits until Cosmos DB has applied a changed indexing policy: until then,
|
|
283
|
+
* a query that orders by the new composite index fails.
|
|
284
|
+
*/
|
|
285
|
+
const waitForIndexTransformation = async (container) => {
|
|
286
|
+
const deadline = Date.now() + INDEX_TRANSFORMATION_TIMEOUT_MS;
|
|
287
|
+
for (;;) {
|
|
288
|
+
const { headers } = await container.read({ populateQuotaInfo: true });
|
|
289
|
+
const progress = Number(headers[INDEX_TRANSFORMATION_PROGRESS] ?? 100);
|
|
290
|
+
if (!(progress < 100)) return;
|
|
291
|
+
if (Date.now() > deadline) throw new Error(`Cosmos DB container ${container.id} is still indexing (${progress}%). Run the migration again once it finishes.`);
|
|
292
|
+
await new Promise((resolve) => setTimeout(resolve, 2e3));
|
|
293
|
+
}
|
|
294
|
+
};
|
|
295
|
+
/** The settings row that names the storage layout a container holds. */
|
|
296
|
+
const ENGINE_KEY = "schema.engine";
|
|
297
|
+
/** The `schema.engine` setting this version's storage layout writes. */
|
|
298
|
+
const currentCosmosEngine = () => toolingTargetOf([]).settings[ENGINE_KEY];
|
|
299
|
+
/**
|
|
300
|
+
* Refuses a container whose fixed settings Hot Updater's items cannot live
|
|
301
|
+
* under: unique keys, or another partition key.
|
|
302
|
+
*/
|
|
303
|
+
const assertContainerShape = (resource) => {
|
|
304
|
+
if ((resource.uniqueKeyPolicy?.uniqueKeys?.length ?? 0) > 0) throw new Error(`Cosmos DB container ${resource.id} has unique keys, which Hot Updater's items cannot satisfy. Use another container.`);
|
|
305
|
+
const paths = resource.partitionKey?.paths ?? [];
|
|
306
|
+
if (paths.length !== 1 || paths[0] !== "/partition") throw new Error(`Cosmos DB container ${resource.id} is partitioned by ${paths.join(", ")}; Hot Updater needs ${COSMOS_PARTITION_KEY_PATH}. Use another container.`);
|
|
307
|
+
};
|
|
308
|
+
/**
|
|
309
|
+
* Refuses a container a migration must not touch: one with unique keys or
|
|
310
|
+
* another partition key, one whose `schema.engine` is another storage
|
|
311
|
+
* layout's, or a populated one without the setting. Writing the settings
|
|
312
|
+
* would make the server read it as this layout. Read only, so init checks
|
|
313
|
+
* a reused container before it creates anything. Point reads and a scan: a
|
|
314
|
+
* reused container may index nothing yet, and the migration only fixes its
|
|
315
|
+
* indexing once this check passes.
|
|
316
|
+
*/
|
|
317
|
+
const assertCosmosContainerReusable = async (container, expected) => {
|
|
318
|
+
const missing = (error) => error.code === 404;
|
|
319
|
+
let existing;
|
|
320
|
+
try {
|
|
321
|
+
existing = (await container.read()).resource;
|
|
322
|
+
} catch (error) {
|
|
323
|
+
if (!missing(error)) throw error;
|
|
324
|
+
}
|
|
325
|
+
if (existing === void 0) return;
|
|
326
|
+
assertContainerShape(existing);
|
|
327
|
+
const { resource } = await container.item(cosmosItemId({
|
|
328
|
+
pk: SETTINGS_TABLE.name,
|
|
329
|
+
sk: encodeKvKey([ENGINE_KEY])
|
|
330
|
+
}), COSMOS_PARTITION).read().catch((error) => {
|
|
331
|
+
if (missing(error)) return { resource: void 0 };
|
|
332
|
+
throw error;
|
|
333
|
+
});
|
|
334
|
+
const found = resource?.row?.value;
|
|
335
|
+
if (found !== void 0) {
|
|
336
|
+
if (found !== expected) throw new Error(`Cosmos DB container ${container.id} holds storage engine "${String(found)}"; this version writes "${expected}". Use the Hot Updater version that wrote it, or another container.`);
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
const { resources } = await container.items.query("SELECT TOP 1 VALUE c.id FROM c", { enableScanInQuery: true }).fetchAll();
|
|
340
|
+
if (resources.length > 0) throw new Error(`Cosmos DB container ${container.id} holds items but no ${ENGINE_KEY} setting. Use an empty container.`);
|
|
341
|
+
};
|
|
342
|
+
/**
|
|
343
|
+
* Whether an add takes more operations than one patch does. When its
|
|
344
|
+
* patches would take a batch past its operations, it replaces the row it
|
|
345
|
+
* read instead, one operation, which is what `COSMOS_LIMITS.items` counts.
|
|
346
|
+
*/
|
|
347
|
+
const isWideAdd = (op) => op.type === "add" && Object.keys(op.by).length + (op.expiresAt === void 0 ? 0 : 2) > PATCH_OPERATIONS;
|
|
348
|
+
/**
|
|
349
|
+
* The PostgreSQL behind the vNext emulator aborts a batch that races
|
|
350
|
+
* another one with these errors, having written nothing. Cosmos DB itself
|
|
351
|
+
* runs the batches of a partition one at a time and never reports them.
|
|
352
|
+
*/
|
|
353
|
+
const emulatorAborted = (error) => statusOf(error) === 500 && /PostgresError\(SqlState\(E(25P02|40001|40P01)\)/.test(String(error instanceof Error ? error.message : error));
|
|
354
|
+
/**
|
|
355
|
+
* The key-value store over one Cosmos DB container of `{ pk, sk, row }`
|
|
356
|
+
* items in one logical partition, read through the composite index on
|
|
357
|
+
* (pk, sk). A write is one transactional batch whose operations carry most
|
|
358
|
+
* ops' conditions themselves: a Create fails on an item that exists, a
|
|
359
|
+
* Replace, Read or patch on one that is missing, and a patch whose
|
|
360
|
+
* predicate finds another `_v`. Deletes and `_v` checks read their items
|
|
361
|
+
* first, since a Delete of a missing item fails the batch and a patch
|
|
362
|
+
* restarts the item's TTL, and so does an add wider than one patch, which
|
|
363
|
+
* replaces the row it read.
|
|
364
|
+
*/
|
|
365
|
+
const createCosmosStore = ({ client, databaseId, containerId, engine, nativePageSize }) => {
|
|
366
|
+
const container = client.database(databaseId).container(containerId);
|
|
367
|
+
/**
|
|
368
|
+
* The state of the items `ids` names, by id, with the rows of `rowIds`;
|
|
369
|
+
* a missing item has none.
|
|
370
|
+
*/
|
|
371
|
+
const readStates = async (ids, rowIds) => {
|
|
372
|
+
const states = /* @__PURE__ */ new Map();
|
|
373
|
+
for (const [group, projection] of [[ids.filter((id) => !rowIds.has(id)), ""], [ids.filter((id) => rowIds.has(id)), ", c.row"]]) for (let at = 0; at < group.length; at += GET_BATCH) {
|
|
374
|
+
const { resources } = await container.items.query(`SELECT c.id, c._etag AS etag, c.row._v AS v, c.expireAt${projection} FROM c WHERE c.id IN (${idList(group.slice(at, at + GET_BATCH))})`, { partitionKey: COSMOS_PARTITION }).fetchAll().catch(missingContainer);
|
|
375
|
+
for (const { id, etag, v, expireAt, row } of resources) states.set(id, {
|
|
376
|
+
etag,
|
|
377
|
+
v,
|
|
378
|
+
expireAt: expireAt ?? null,
|
|
379
|
+
row
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
return states;
|
|
383
|
+
};
|
|
384
|
+
/** Whether `op` reads its item before the write plans it. */
|
|
385
|
+
const readsFirst = (op) => isWideAdd(op) || (op.condition === void 0 ? op.type === "delete" || op.type === "add" : "v" in op.condition && (op.type === "delete" || op.type === "check"));
|
|
386
|
+
/**
|
|
387
|
+
* The batch operations of `op`, given the state its item was read in;
|
|
388
|
+
* `failed` when that state already fails its condition.
|
|
389
|
+
*/
|
|
390
|
+
const operationsOf = (op, state, replaceWide) => {
|
|
391
|
+
const id = cosmosItemId(op.key);
|
|
392
|
+
const condition = op.condition;
|
|
393
|
+
const guard = condition !== void 0 && "v" in condition ? guardOf(condition.v) : void 0;
|
|
394
|
+
const patches = (operations, onFailure) => {
|
|
395
|
+
const chunks = [];
|
|
396
|
+
for (let at = 0; at < operations.length; at += PATCH_OPERATIONS) {
|
|
397
|
+
const operation = {
|
|
398
|
+
operationType: "Patch",
|
|
399
|
+
id,
|
|
400
|
+
resourceBody: {
|
|
401
|
+
...at === 0 && guard !== void 0 ? { condition: guard } : {},
|
|
402
|
+
operations: operations.slice(at, at + PATCH_OPERATIONS)
|
|
403
|
+
}
|
|
404
|
+
};
|
|
405
|
+
chunks.push([operation, onFailure]);
|
|
406
|
+
}
|
|
407
|
+
return chunks;
|
|
408
|
+
};
|
|
409
|
+
const expiry = (expiresAt) => [{
|
|
410
|
+
op: "set",
|
|
411
|
+
path: "/expireAt",
|
|
412
|
+
value: expiresAt ?? null
|
|
413
|
+
}, {
|
|
414
|
+
op: "set",
|
|
415
|
+
path: "/ttl",
|
|
416
|
+
value: ttlOf(expiresAt)
|
|
417
|
+
}];
|
|
418
|
+
/** A Create that fails on an item that exists, undone in the same batch. */
|
|
419
|
+
const absent = () => [[{
|
|
420
|
+
operationType: "Create",
|
|
421
|
+
resourceBody: placeholderOf(op.key)
|
|
422
|
+
}, "fail"], [{
|
|
423
|
+
operationType: "Delete",
|
|
424
|
+
id
|
|
425
|
+
}, "replan"]];
|
|
426
|
+
switch (op.type) {
|
|
427
|
+
case "put": {
|
|
428
|
+
const item = toItem(op.key, op.value, op.expiresAt);
|
|
429
|
+
if (condition === void 0) return [[{
|
|
430
|
+
operationType: "Upsert",
|
|
431
|
+
resourceBody: item
|
|
432
|
+
}, "replan"]];
|
|
433
|
+
if ("v" in condition) return patches([{
|
|
434
|
+
op: "set",
|
|
435
|
+
path: "/row",
|
|
436
|
+
value: item.row
|
|
437
|
+
}, ...expiry(op.expiresAt)], "fail");
|
|
438
|
+
return [[condition.exists ? {
|
|
439
|
+
operationType: "Replace",
|
|
440
|
+
id,
|
|
441
|
+
resourceBody: item
|
|
442
|
+
} : {
|
|
443
|
+
operationType: "Create",
|
|
444
|
+
resourceBody: item
|
|
445
|
+
}, "fail"]];
|
|
446
|
+
}
|
|
447
|
+
case "delete":
|
|
448
|
+
if (condition === void 0) {
|
|
449
|
+
if (state === void 0) return [];
|
|
450
|
+
return [[{
|
|
451
|
+
operationType: "Delete",
|
|
452
|
+
id
|
|
453
|
+
}, "replan"]];
|
|
454
|
+
}
|
|
455
|
+
if ("v" in condition) {
|
|
456
|
+
if (state === void 0 || state.v !== condition.v) return { failed: true };
|
|
457
|
+
return [[{
|
|
458
|
+
operationType: "Delete",
|
|
459
|
+
id,
|
|
460
|
+
ifMatch: state.etag
|
|
461
|
+
}, "replan"]];
|
|
462
|
+
}
|
|
463
|
+
return condition.exists ? [[{
|
|
464
|
+
operationType: "Delete",
|
|
465
|
+
id
|
|
466
|
+
}, "fail"]] : absent();
|
|
467
|
+
case "check":
|
|
468
|
+
if (condition === void 0) return [];
|
|
469
|
+
if ("v" in condition) {
|
|
470
|
+
if (state === void 0 || state.v !== condition.v) return { failed: true };
|
|
471
|
+
return patches(expiry(state.expireAt), "fail");
|
|
472
|
+
}
|
|
473
|
+
return condition.exists ? [[{
|
|
474
|
+
operationType: "Read",
|
|
475
|
+
id
|
|
476
|
+
}, "fail"]] : absent();
|
|
477
|
+
case "add":
|
|
478
|
+
if (condition === void 0 ? state === void 0 : "exists" in condition && !condition.exists) {
|
|
479
|
+
const created = { ...op.init };
|
|
480
|
+
for (const [name, delta] of Object.entries(op.by)) created[name] = Number(created[name] ?? 0) + delta;
|
|
481
|
+
return [[{
|
|
482
|
+
operationType: "Create",
|
|
483
|
+
resourceBody: toItem(op.key, created, op.expiresAt)
|
|
484
|
+
}, condition === void 0 ? "replan" : "fail"]];
|
|
485
|
+
}
|
|
486
|
+
if (replaceWide && isWideAdd(op)) {
|
|
487
|
+
if (state === void 0) return { failed: true };
|
|
488
|
+
if (guard !== void 0 && state.v !== condition.v) return { failed: true };
|
|
489
|
+
const row = { ...toRow(state.row) };
|
|
490
|
+
for (const [name, delta] of Object.entries(op.by)) row[name] = Number(row[name] ?? 0) + delta;
|
|
491
|
+
return [[{
|
|
492
|
+
operationType: "Replace",
|
|
493
|
+
id,
|
|
494
|
+
ifMatch: state.etag,
|
|
495
|
+
resourceBody: toItem(op.key, row, op.expiresAt ?? state.expireAt ?? void 0)
|
|
496
|
+
}, "replan"]];
|
|
497
|
+
}
|
|
498
|
+
return patches([...Object.entries(op.by).map(([name, delta]) => ({
|
|
499
|
+
op: "incr",
|
|
500
|
+
path: columnPath(name),
|
|
501
|
+
value: delta
|
|
502
|
+
})), ...op.expiresAt === void 0 ? [] : expiry(op.expiresAt)], condition === void 0 ? "replan" : "fail");
|
|
503
|
+
}
|
|
504
|
+
};
|
|
505
|
+
return {
|
|
506
|
+
id: "cosmos",
|
|
507
|
+
limits: COSMOS_LIMITS,
|
|
508
|
+
async get(keys) {
|
|
509
|
+
const found = /* @__PURE__ */ new Map();
|
|
510
|
+
const ids = [...new Set(keys.map(cosmosItemId))];
|
|
511
|
+
for (let at = 0; at < ids.length; at += GET_BATCH) {
|
|
512
|
+
const { resources } = await container.items.query(`SELECT c.id, c.row FROM c WHERE c.id IN (${idList(ids.slice(at, at + GET_BATCH))})`, { partitionKey: COSMOS_PARTITION }).fetchAll().catch(missingContainer);
|
|
513
|
+
for (const resource of resources) found.set(resource.id, toRow(resource.row));
|
|
514
|
+
}
|
|
515
|
+
return keys.map((key) => found.get(cosmosItemId(key)) ?? null);
|
|
516
|
+
},
|
|
517
|
+
async query({ pk, gte, lt, order, limit, after }) {
|
|
518
|
+
const size = Math.min(limit, nativePageSize ?? limit);
|
|
519
|
+
const filters = ["c.pk = @pk"];
|
|
520
|
+
const parameters = [{
|
|
521
|
+
name: "@pk",
|
|
522
|
+
value: pk
|
|
523
|
+
}, {
|
|
524
|
+
name: "@size",
|
|
525
|
+
value: size
|
|
526
|
+
}];
|
|
527
|
+
if (gte !== void 0) {
|
|
528
|
+
filters.push("c.sk >= @gte");
|
|
529
|
+
parameters.push({
|
|
530
|
+
name: "@gte",
|
|
531
|
+
value: gte
|
|
532
|
+
});
|
|
533
|
+
}
|
|
534
|
+
if (lt !== void 0) {
|
|
535
|
+
filters.push("c.sk < @lt");
|
|
536
|
+
parameters.push({
|
|
537
|
+
name: "@lt",
|
|
538
|
+
value: lt
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
if (after !== void 0) {
|
|
542
|
+
filters.push(order === "asc" ? "c.sk > @after" : "c.sk < @after");
|
|
543
|
+
parameters.push({
|
|
544
|
+
name: "@after",
|
|
545
|
+
value: after
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
const { resources } = await container.items.query({
|
|
549
|
+
query: `SELECT TOP @size c.sk, c.row FROM c WHERE ${filters.join(" AND ")} ORDER BY c.pk ${order === "asc" ? "ASC" : "DESC"}, c.sk ${order === "asc" ? "ASC" : "DESC"}`,
|
|
550
|
+
parameters
|
|
551
|
+
}, {
|
|
552
|
+
partitionKey: COSMOS_PARTITION,
|
|
553
|
+
maxItemCount: size
|
|
554
|
+
}).fetchAll().catch(missingContainer);
|
|
555
|
+
return {
|
|
556
|
+
items: resources.map((resource) => ({
|
|
557
|
+
sk: resource.sk,
|
|
558
|
+
value: toRow(resource.row)
|
|
559
|
+
})),
|
|
560
|
+
more: resources.length === size
|
|
561
|
+
};
|
|
562
|
+
},
|
|
563
|
+
async write(ops) {
|
|
564
|
+
for (let attempt = 0; attempt < 5; attempt += 1) {
|
|
565
|
+
const states = await readStates([...new Set(ops.filter(readsFirst).map((op) => cosmosItemId(op.key)))], new Set(ops.filter(isWideAdd).map((op) => cosmosItemId(op.key))));
|
|
566
|
+
const plan = (replaceWide) => {
|
|
567
|
+
const planned = [];
|
|
568
|
+
for (const [position, op] of ops.entries()) {
|
|
569
|
+
const operations = operationsOf(op, states.get(cosmosItemId(op.key)), replaceWide);
|
|
570
|
+
if ("failed" in operations) return { failedOp: position };
|
|
571
|
+
for (const [operation, onFailure] of operations) planned.push({
|
|
572
|
+
position,
|
|
573
|
+
operation,
|
|
574
|
+
onFailure
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
return planned;
|
|
578
|
+
};
|
|
579
|
+
let planned = plan(false);
|
|
580
|
+
if ("failedOp" in planned) return {
|
|
581
|
+
ok: false,
|
|
582
|
+
failedOp: planned.failedOp
|
|
583
|
+
};
|
|
584
|
+
if (planned.length > BATCH_OPERATIONS) {
|
|
585
|
+
const replaced = plan(true);
|
|
586
|
+
if ("failedOp" in replaced) return {
|
|
587
|
+
ok: false,
|
|
588
|
+
failedOp: replaced.failedOp
|
|
589
|
+
};
|
|
590
|
+
planned = replaced;
|
|
591
|
+
const bytes = encoder.encode(JSON.stringify(planned.map(({ operation }) => operation))).length;
|
|
592
|
+
if (bytes > BATCH_BYTES) throw new Error(`A Cosmos DB batch takes ${BATCH_BYTES} bytes; this write's rows of more counters than one patch takes need ${bytes}. Write fewer of them at once.`);
|
|
593
|
+
}
|
|
594
|
+
if (planned.length === 0) return { ok: true };
|
|
595
|
+
if (planned.length > BATCH_OPERATIONS) throw new Error(`A Cosmos DB batch takes ${BATCH_OPERATIONS} operations; this write needs ${planned.length}.`);
|
|
596
|
+
let results;
|
|
597
|
+
try {
|
|
598
|
+
const response = await container.items.batch(planned.map(({ operation }) => operation), COSMOS_PARTITION);
|
|
599
|
+
if (response.result === void 0) throw Object.assign(/* @__PURE__ */ new Error(`Cosmos DB refused the batch with status ${response.code}`), { code: response.code });
|
|
600
|
+
results = response.result;
|
|
601
|
+
} catch (error) {
|
|
602
|
+
if (emulatorAborted(error)) continue;
|
|
603
|
+
if (TRANSIENT.has(statusOf(error))) return {
|
|
604
|
+
ok: false,
|
|
605
|
+
retry: true
|
|
606
|
+
};
|
|
607
|
+
return missingContainer(error);
|
|
608
|
+
}
|
|
609
|
+
const failed = results.findIndex(({ statusCode }) => statusCode >= 400 && statusCode !== 424);
|
|
610
|
+
if (failed === -1) return { ok: true };
|
|
611
|
+
const status = results[failed].statusCode;
|
|
612
|
+
const { position, onFailure } = planned[failed];
|
|
613
|
+
if (TRANSIENT.has(status)) return {
|
|
614
|
+
ok: false,
|
|
615
|
+
retry: true
|
|
616
|
+
};
|
|
617
|
+
if (status !== 404 && status !== 409 && status !== 412) throw new Error(`Cosmos DB refused a batch operation with status ${status}.`);
|
|
618
|
+
if (onFailure === "fail") return {
|
|
619
|
+
ok: false,
|
|
620
|
+
failedOp: position
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
return {
|
|
624
|
+
ok: false,
|
|
625
|
+
retry: true
|
|
626
|
+
};
|
|
627
|
+
},
|
|
628
|
+
/** Plain deletes through the bulk API, where a missing item is no failure. */
|
|
629
|
+
async deleteConsumed(keys) {
|
|
630
|
+
let remaining = [...new Set(keys.map(cosmosItemId))];
|
|
631
|
+
for (let attempt = 0; remaining.length > 0; attempt += 1) {
|
|
632
|
+
const failed = (await container.items.executeBulkOperations(remaining.map((id) => ({
|
|
633
|
+
operationType: "Delete",
|
|
634
|
+
id,
|
|
635
|
+
partitionKey: COSMOS_PARTITION
|
|
636
|
+
}))).catch(missingContainer)).flatMap((result, at) => {
|
|
637
|
+
const status = result.response?.statusCode ?? statusOf(result.error);
|
|
638
|
+
return typeof status === "number" && status < 400 || status === 404 ? [] : [{
|
|
639
|
+
id: remaining[at],
|
|
640
|
+
error: result.error,
|
|
641
|
+
status
|
|
642
|
+
}];
|
|
643
|
+
});
|
|
644
|
+
if (failed.length === 0) return;
|
|
645
|
+
const retried = failed.filter(({ error, status }) => TRANSIENT.has(status) || emulatorAborted(error));
|
|
646
|
+
if (retried.length < failed.length || attempt >= 4) throw new Error(`Cosmos DB refused to delete ${failed.length} consumed item(s) with status ${failed[0].status}.`);
|
|
647
|
+
remaining = retried.map(({ id }) => id);
|
|
648
|
+
}
|
|
649
|
+
},
|
|
650
|
+
migrations: {
|
|
651
|
+
/** Creates the database and the container when they are missing. */
|
|
652
|
+
async apply() {
|
|
653
|
+
if (engine !== void 0) await assertCosmosContainerReusable(container, engine);
|
|
654
|
+
const { database } = await client.databases.createIfNotExists({ id: databaseId });
|
|
655
|
+
const { resource } = await database.containers.createIfNotExists({
|
|
656
|
+
id: containerId,
|
|
657
|
+
partitionKey: { paths: [COSMOS_PARTITION_KEY_PATH] },
|
|
658
|
+
defaultTtl: -1,
|
|
659
|
+
indexingPolicy: COSMOS_INDEXING_POLICY
|
|
660
|
+
});
|
|
661
|
+
if (resource === void 0) throw new Error(`Cosmos DB returned no container ${containerId}.`);
|
|
662
|
+
assertContainerShape(resource);
|
|
663
|
+
const policy = resource.indexingPolicy;
|
|
664
|
+
const indexed = (policy?.indexingMode ?? "consistent") === "consistent" && policy?.automatic !== false;
|
|
665
|
+
const composite = COSMOS_INDEXING_POLICY.compositeIndexes[0];
|
|
666
|
+
const ordered = policy?.compositeIndexes?.some((index) => index.length === 2 && index[0].path === "/pk" && index[1].path === "/sk" && (index[0].order ?? "ascending") === (index[1].order ?? "ascending"));
|
|
667
|
+
const expires = resource.defaultTtl !== void 0 && resource.defaultTtl !== null;
|
|
668
|
+
if (!indexed || !ordered || !expires) {
|
|
669
|
+
await database.container(containerId).replace({
|
|
670
|
+
...resource,
|
|
671
|
+
defaultTtl: expires ? resource.defaultTtl : -1,
|
|
672
|
+
indexingPolicy: {
|
|
673
|
+
...policy,
|
|
674
|
+
indexingMode: "consistent",
|
|
675
|
+
automatic: true,
|
|
676
|
+
compositeIndexes: ordered ? policy?.compositeIndexes : [...policy?.compositeIndexes ?? [], composite]
|
|
677
|
+
}
|
|
678
|
+
});
|
|
679
|
+
await waitForIndexTransformation(database.container(containerId));
|
|
680
|
+
}
|
|
681
|
+
} },
|
|
682
|
+
async dispose() {}
|
|
683
|
+
};
|
|
684
|
+
};
|
|
685
|
+
//#endregion
|
|
686
|
+
//#region src/cosmosDatabase.ts
|
|
687
|
+
const adapterOf = (config) => {
|
|
688
|
+
const { databaseId = AZURE_COSMOS_V1_DATABASE, containerId = AZURE_COSMOS_V1_CONTAINER, aggregateBatching: _, ...connection } = config;
|
|
689
|
+
if (connection.consistencyLevel !== void 0 && connection.consistencyLevel !== "Strong") throw new Error(`cosmosDatabase reads with strong consistency; consistencyLevel "${String(connection.consistencyLevel)}" is weaker.`);
|
|
690
|
+
const client = new CosmosClient(connection);
|
|
691
|
+
const store = createCosmosStore({
|
|
692
|
+
client,
|
|
693
|
+
databaseId,
|
|
694
|
+
containerId,
|
|
695
|
+
engine: currentCosmosEngine()
|
|
696
|
+
});
|
|
697
|
+
return {
|
|
698
|
+
client,
|
|
699
|
+
adapter: createKvAdapter({ store })
|
|
700
|
+
};
|
|
701
|
+
};
|
|
702
|
+
/**
|
|
703
|
+
* Creates the database and the container when they are missing, refusing a
|
|
704
|
+
* container of another storage layout or another app's items, and writes
|
|
705
|
+
* the schema settings of core and `plugins`, the plugins the server runs,
|
|
706
|
+
* which the database checks before its first read. `hot-updater init` runs
|
|
707
|
+
* it with the managed server's plugins.
|
|
708
|
+
*/
|
|
709
|
+
const migrateCosmosDatabase = (config, plugins = []) => {
|
|
710
|
+
const { client, adapter } = adapterOf(config);
|
|
711
|
+
return migrateCoreSchema(adapter, "cosmosDatabase", plugins).finally(() => client.dispose());
|
|
712
|
+
};
|
|
713
|
+
/**
|
|
714
|
+
* Hot Updater's database in one Azure Cosmos DB for NoSQL container,
|
|
715
|
+
* through the storage engine. Every item shares one logical partition, so a
|
|
716
|
+
* write commits as one transactional batch; that partition holds at most
|
|
717
|
+
* 20 GB.
|
|
718
|
+
*/
|
|
719
|
+
const cosmosDatabase = (config) => {
|
|
720
|
+
const { client, adapter } = adapterOf(config);
|
|
721
|
+
return {
|
|
722
|
+
...createEngineDatabase({
|
|
723
|
+
name: "cosmosDatabase",
|
|
724
|
+
adapter,
|
|
725
|
+
aggregateBatching: config.aggregateBatching ?? {}
|
|
726
|
+
}),
|
|
727
|
+
async dispose() {
|
|
728
|
+
await adapter.dispose?.();
|
|
729
|
+
client.dispose();
|
|
730
|
+
}
|
|
731
|
+
};
|
|
732
|
+
};
|
|
733
|
+
//#endregion
|
|
734
|
+
export { azureBlobStorage, cosmosDatabase, migrateCosmosDatabase };
|