@platforma-sdk/block-tools 2.6.44 → 2.6.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -9
- package/bin/dev.js +4 -4
- package/bin/run.js +3 -3
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +16 -6
- package/dist/cli.mjs.map +1 -1
- package/dist/cmd/index.d.ts +8 -8
- package/dist/cmd/mark-stable.d.ts +1 -1
- package/dist/cmd/publish.d.ts +1 -1
- package/dist/cmd/restore-overview-from-snapshot.d.ts +1 -1
- package/dist/{config-DKBY0B2u.mjs → config-Cc8_zV3b.mjs} +48 -17
- package/dist/config-Cc8_zV3b.mjs.map +1 -0
- package/dist/config-Ycas5fbX.js.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +6 -4
- package/dist/index.mjs.map +1 -1
- package/dist/registry_v1/config_schema.d.ts +7 -7
- package/dist/registry_v1/v1_repo_schema.d.ts +1 -1
- package/dist/v2/model/block_components.d.ts +19 -19
- package/dist/v2/model/block_description.d.ts +102 -102
- package/dist/v2/model/block_meta.d.ts +20 -20
- package/dist/v2/model/content_conversion.d.ts +2 -2
- package/dist/v2/registry/registry.d.ts +1 -1
- package/dist/v2/source_package.d.ts +1 -1
- package/package.json +31 -29
- package/src/cmd/build-meta.ts +15 -15
- package/src/cmd/build-model.ts +23 -26
- package/src/cmd/index.ts +20 -20
- package/src/cmd/list-overview-snapshots.ts +12 -12
- package/src/cmd/mark-stable.ts +30 -33
- package/src/cmd/pack.ts +15 -15
- package/src/cmd/publish.ts +46 -34
- package/src/cmd/refresh-registry.ts +15 -15
- package/src/cmd/restore-overview-from-snapshot.ts +27 -25
- package/src/cmd/update-deps.ts +8 -8
- package/src/cmd/upload-package-v1.ts +33 -33
- package/src/common_types.ts +1 -1
- package/src/io/folder_reader.test.ts +13 -13
- package/src/io/folder_reader.ts +18 -20
- package/src/io/index.ts +2 -2
- package/src/io/storage.test.ts +48 -48
- package/src/io/storage.ts +20 -15
- package/src/lib.ts +3 -3
- package/src/registry_v1/config.ts +11 -11
- package/src/registry_v1/config_schema.ts +5 -5
- package/src/registry_v1/flags.ts +4 -4
- package/src/registry_v1/index.ts +3 -3
- package/src/registry_v1/registry.test.ts +54 -54
- package/src/registry_v1/registry.ts +29 -25
- package/src/registry_v1/v1_repo_schema.ts +3 -3
- package/src/util.ts +6 -9
- package/src/v2/build_dist.test.ts +8 -9
- package/src/v2/build_dist.ts +10 -13
- package/src/v2/index.ts +4 -4
- package/src/v2/model/block_components.ts +5 -5
- package/src/v2/model/block_description.ts +12 -8
- package/src/v2/model/block_meta.ts +4 -5
- package/src/v2/model/content_conversion.ts +44 -44
- package/src/v2/model/index.ts +4 -4
- package/src/v2/registry/index.ts +3 -3
- package/src/v2/registry/registry.test.ts +223 -197
- package/src/v2/registry/registry.ts +93 -66
- package/src/v2/registry/registry_reader.test.ts +15 -15
- package/src/v2/registry/registry_reader.ts +29 -27
- package/src/v2/registry/schema_internal.ts +11 -10
- package/src/v2/registry/schema_public.ts +56 -47
- package/src/v2/source_package.test.ts +15 -15
- package/src/v2/source_package.ts +33 -26
- package/dist/config-DKBY0B2u.mjs.map +0 -1
|
@@ -1,17 +1,15 @@
|
|
|
1
|
-
import type { MiLogger } from
|
|
2
|
-
import { ConsoleLoggerAdapter } from
|
|
3
|
-
import { compare as compareSemver } from
|
|
4
|
-
import { gzip, gunzip } from
|
|
5
|
-
import { promisify } from
|
|
6
|
-
import type { RegistryStorage } from
|
|
7
|
-
import type {
|
|
8
|
-
BlockPackId,
|
|
9
|
-
BlockPackIdNoVersion } from '@milaboratories/pl-model-middle-layer';
|
|
1
|
+
import type { MiLogger } from "@milaboratories/ts-helpers";
|
|
2
|
+
import { ConsoleLoggerAdapter } from "@milaboratories/ts-helpers";
|
|
3
|
+
import { compare as compareSemver } from "semver";
|
|
4
|
+
import { gzip, gunzip } from "node:zlib";
|
|
5
|
+
import { promisify } from "node:util";
|
|
6
|
+
import type { RegistryStorage } from "../../io/storage";
|
|
7
|
+
import type { BlockPackId, BlockPackIdNoVersion } from "@milaboratories/pl-model-middle-layer";
|
|
10
8
|
import {
|
|
11
9
|
AnyChannel,
|
|
12
10
|
blockPackIdToString,
|
|
13
11
|
BlockPackManifest,
|
|
14
|
-
} from
|
|
12
|
+
} from "@milaboratories/pl-model-middle-layer";
|
|
15
13
|
import {
|
|
16
14
|
GlobalUpdateSeedInFile,
|
|
17
15
|
GlobalUpdateSeedOutFile,
|
|
@@ -22,7 +20,7 @@ import {
|
|
|
22
20
|
GlobalSnapshotsPrefix,
|
|
23
21
|
globalOverviewSnapshotPath,
|
|
24
22
|
packageOverviewSnapshotPath,
|
|
25
|
-
} from
|
|
23
|
+
} from "./schema_internal";
|
|
26
24
|
import {
|
|
27
25
|
GlobalOverviewReg,
|
|
28
26
|
GlobalOverviewPath,
|
|
@@ -37,11 +35,11 @@ import {
|
|
|
37
35
|
ChannelNameRegexp,
|
|
38
36
|
MainPrefix,
|
|
39
37
|
PackageManifestPattern,
|
|
40
|
-
} from
|
|
41
|
-
import type { RelativeContentReader } from
|
|
42
|
-
import { BlockPackDescriptionManifestAddRelativePathPrefix } from
|
|
43
|
-
import { randomUUID } from
|
|
44
|
-
import { calculateSha256 } from
|
|
38
|
+
} from "./schema_public";
|
|
39
|
+
import type { RelativeContentReader } from "../model";
|
|
40
|
+
import { BlockPackDescriptionManifestAddRelativePathPrefix } from "../model";
|
|
41
|
+
import { randomUUID } from "node:crypto";
|
|
42
|
+
import { calculateSha256 } from "../../util";
|
|
45
43
|
|
|
46
44
|
export interface BlockRegistrySettings {
|
|
47
45
|
skipSnapshotCreation?: boolean;
|
|
@@ -68,18 +66,27 @@ export class BlockRegistryV2 {
|
|
|
68
66
|
) {}
|
|
69
67
|
|
|
70
68
|
private generateTimestamp(): string {
|
|
71
|
-
const timestamp = new Date()
|
|
69
|
+
const timestamp = new Date()
|
|
70
|
+
.toISOString()
|
|
71
|
+
.replace(/:/g, "-")
|
|
72
|
+
.replace(/\.(\d{3})Z$/, ".$1Z");
|
|
72
73
|
const randomSuffix = Math.random().toString(36).substring(2, 6);
|
|
73
74
|
return `${timestamp}-${randomSuffix}`;
|
|
74
75
|
}
|
|
75
76
|
|
|
76
77
|
private generatePreWriteTimestamp(): string {
|
|
77
|
-
const timestamp = new Date()
|
|
78
|
+
const timestamp = new Date()
|
|
79
|
+
.toISOString()
|
|
80
|
+
.replace(/:/g, "-")
|
|
81
|
+
.replace(/\.(\d{3})Z$/, ".$1Z");
|
|
78
82
|
const randomSuffix = Math.random().toString(36).substring(2, 6);
|
|
79
83
|
return `${timestamp}-prewrite-${randomSuffix}`;
|
|
80
84
|
}
|
|
81
85
|
|
|
82
|
-
private async createGlobalOverviewSnapshot(
|
|
86
|
+
private async createGlobalOverviewSnapshot(
|
|
87
|
+
overviewData: string,
|
|
88
|
+
timestamp: string,
|
|
89
|
+
): Promise<void> {
|
|
83
90
|
if (this.settings.skipSnapshotCreation) return;
|
|
84
91
|
|
|
85
92
|
try {
|
|
@@ -92,7 +99,11 @@ export class BlockRegistryV2 {
|
|
|
92
99
|
}
|
|
93
100
|
}
|
|
94
101
|
|
|
95
|
-
private async createPackageOverviewSnapshot(
|
|
102
|
+
private async createPackageOverviewSnapshot(
|
|
103
|
+
pkg: BlockPackIdNoVersion,
|
|
104
|
+
overview: PackageOverview,
|
|
105
|
+
timestamp: string,
|
|
106
|
+
): Promise<void> {
|
|
96
107
|
if (this.settings.skipSnapshotCreation) return;
|
|
97
108
|
|
|
98
109
|
try {
|
|
@@ -100,14 +111,18 @@ export class BlockRegistryV2 {
|
|
|
100
111
|
const gzippedData = await this.gzipAsync(overviewData);
|
|
101
112
|
const snapshotPath = packageOverviewSnapshotPath(pkg, timestamp);
|
|
102
113
|
await this.storage.putFile(snapshotPath, Buffer.from(gzippedData));
|
|
103
|
-
this.logger.info(
|
|
114
|
+
this.logger.info(
|
|
115
|
+
`Package overview snapshot created at ${snapshotPath} for ${pkg.organization}:${pkg.name}`,
|
|
116
|
+
);
|
|
104
117
|
} catch (error) {
|
|
105
|
-
this.logger.warn(
|
|
118
|
+
this.logger.warn(
|
|
119
|
+
`Failed to create package overview snapshot for ${pkg.organization}:${pkg.name}: ${String(error)}`,
|
|
120
|
+
);
|
|
106
121
|
}
|
|
107
122
|
}
|
|
108
123
|
|
|
109
|
-
private async updateRegistry(mode:
|
|
110
|
-
this.logger.info(
|
|
124
|
+
private async updateRegistry(mode: "force" | "normal" | "dry-run" = "normal") {
|
|
125
|
+
this.logger.info("Initiating registry refresh...");
|
|
111
126
|
|
|
112
127
|
// Generate timestamp for all snapshots in this run
|
|
113
128
|
const snapshotTimestamp = this.generateTimestamp();
|
|
@@ -133,7 +148,7 @@ export class BlockRegistryV2 {
|
|
|
133
148
|
return false;
|
|
134
149
|
};
|
|
135
150
|
|
|
136
|
-
this.logger.info(
|
|
151
|
+
this.logger.info("Packages to be updated:");
|
|
137
152
|
for (const seedPath of rawSeedPaths) {
|
|
138
153
|
const match = seedPath.match(PackageUpdatePattern);
|
|
139
154
|
if (!match) continue;
|
|
@@ -143,7 +158,7 @@ export class BlockRegistryV2 {
|
|
|
143
158
|
this.logger.info(` - ${organization}:${name}:${version} added:${added}`);
|
|
144
159
|
}
|
|
145
160
|
|
|
146
|
-
if (mode ===
|
|
161
|
+
if (mode === "force") {
|
|
147
162
|
// Listing all the packages with all the versions and adding them to the list of packages to be updated
|
|
148
163
|
const allPaths = await this.storage.listFiles(MainPrefix);
|
|
149
164
|
for (const path of allPaths) {
|
|
@@ -159,18 +174,21 @@ export class BlockRegistryV2 {
|
|
|
159
174
|
const overviewContent = await this.storage.getFile(GlobalOverviewPath);
|
|
160
175
|
|
|
161
176
|
// Create pre-write snapshot in force mode if overview exists
|
|
162
|
-
if (mode ===
|
|
177
|
+
if (mode === "force" && overviewContent !== undefined) {
|
|
163
178
|
const preWriteTimestamp = this.generatePreWriteTimestamp();
|
|
164
179
|
await this.createGlobalOverviewSnapshot(overviewContent.toString(), preWriteTimestamp);
|
|
165
180
|
}
|
|
166
181
|
|
|
167
|
-
const overview: GlobalOverviewReg =
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
182
|
+
const overview: GlobalOverviewReg =
|
|
183
|
+
mode === "force"
|
|
184
|
+
? { schema: "v2", packages: [] }
|
|
185
|
+
: overviewContent === undefined
|
|
186
|
+
? { schema: "v2", packages: [] }
|
|
187
|
+
: GlobalOverviewReg.parse(JSON.parse(overviewContent.toString()));
|
|
172
188
|
let overviewPackages = overview.packages;
|
|
173
|
-
this.logger.info(
|
|
189
|
+
this.logger.info(
|
|
190
|
+
`Global overview ${mode === "force" ? "starting empty (force mode)" : "loaded"}, ${overviewPackages.length} records`,
|
|
191
|
+
);
|
|
174
192
|
|
|
175
193
|
// updating packages
|
|
176
194
|
for (const [, packageInfo] of packagesToUpdate.entries()) {
|
|
@@ -179,19 +197,24 @@ export class BlockRegistryV2 {
|
|
|
179
197
|
const pOverviewContent = await this.storage.getFile(overviewFile);
|
|
180
198
|
|
|
181
199
|
// Create pre-write snapshot in force mode if package overview exists
|
|
182
|
-
if (mode ===
|
|
200
|
+
if (mode === "force" && pOverviewContent !== undefined) {
|
|
183
201
|
const preWriteTimestamp = this.generatePreWriteTimestamp();
|
|
184
202
|
const existingOverview = PackageOverview.parse(JSON.parse(pOverviewContent.toString()));
|
|
185
|
-
await this.createPackageOverviewSnapshot(
|
|
203
|
+
await this.createPackageOverviewSnapshot(
|
|
204
|
+
packageInfo.package,
|
|
205
|
+
existingOverview,
|
|
206
|
+
preWriteTimestamp,
|
|
207
|
+
);
|
|
186
208
|
}
|
|
187
209
|
|
|
188
|
-
const packageOverview: PackageOverview =
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
210
|
+
const packageOverview: PackageOverview =
|
|
211
|
+
mode === "force"
|
|
212
|
+
? { schema: "v2", versions: [] }
|
|
213
|
+
: pOverviewContent === undefined
|
|
214
|
+
? { schema: "v2", versions: [] }
|
|
215
|
+
: PackageOverview.parse(JSON.parse(pOverviewContent.toString()));
|
|
193
216
|
this.logger.info(
|
|
194
|
-
`Updating ${packageInfo.package.organization}:${packageInfo.package.name} overview${mode ===
|
|
217
|
+
`Updating ${packageInfo.package.organization}:${packageInfo.package.name} overview${mode === "force" ? " (starting empty in force mode)" : ""}, ${packageOverview.versions.length} records`,
|
|
195
218
|
);
|
|
196
219
|
|
|
197
220
|
// removing versions that we will update
|
|
@@ -222,7 +245,7 @@ export class BlockRegistryV2 {
|
|
|
222
245
|
// pushing the overview
|
|
223
246
|
newVersions.push({
|
|
224
247
|
description: BlockPackDescriptionManifestAddRelativePathPrefix(version).parse(
|
|
225
|
-
BlockPackManifest.parse(JSON.parse(manifestContent.toString(
|
|
248
|
+
BlockPackManifest.parse(JSON.parse(manifestContent.toString("utf8"))).description,
|
|
226
249
|
),
|
|
227
250
|
manifestSha256: sha256,
|
|
228
251
|
channels,
|
|
@@ -235,15 +258,16 @@ export class BlockRegistryV2 {
|
|
|
235
258
|
);
|
|
236
259
|
|
|
237
260
|
// write package overview back
|
|
238
|
-
const packageOverviewData = { schema:
|
|
239
|
-
if (mode !==
|
|
240
|
-
await this.storage.putFile(
|
|
241
|
-
overviewFile,
|
|
242
|
-
Buffer.from(JSON.stringify(packageOverviewData)),
|
|
243
|
-
);
|
|
261
|
+
const packageOverviewData = { schema: "v2", versions: newVersions } satisfies PackageOverview;
|
|
262
|
+
if (mode !== "dry-run") {
|
|
263
|
+
await this.storage.putFile(overviewFile, Buffer.from(JSON.stringify(packageOverviewData)));
|
|
244
264
|
|
|
245
265
|
// Create snapshot after successful write
|
|
246
|
-
await this.createPackageOverviewSnapshot(
|
|
266
|
+
await this.createPackageOverviewSnapshot(
|
|
267
|
+
packageInfo.package,
|
|
268
|
+
packageOverviewData,
|
|
269
|
+
snapshotTimestamp,
|
|
270
|
+
);
|
|
247
271
|
}
|
|
248
272
|
this.logger.info(`Done (${newVersions.length} records)`);
|
|
249
273
|
|
|
@@ -254,8 +278,8 @@ export class BlockRegistryV2 {
|
|
|
254
278
|
// patching corresponding entry in overview
|
|
255
279
|
overviewPackages = overviewPackages.filter(
|
|
256
280
|
(e) =>
|
|
257
|
-
e.id.organization !== packageInfo.package.organization
|
|
258
|
-
|
|
281
|
+
e.id.organization !== packageInfo.package.organization ||
|
|
282
|
+
e.id.name !== packageInfo.package.name,
|
|
259
283
|
);
|
|
260
284
|
const relativeDescriptionSchema = BlockPackDescriptionManifestAddRelativePathPrefix(
|
|
261
285
|
`${packageInfo.package.organization}/${packageInfo.package.name}`,
|
|
@@ -278,7 +302,7 @@ export class BlockRegistryV2 {
|
|
|
278
302
|
[...allChannels, AnyChannel].map((c) => {
|
|
279
303
|
// if c === 'any' the first element will be "found"
|
|
280
304
|
const v = newVersions.find((v) => c === AnyChannel || v.channels.indexOf(c) !== -1);
|
|
281
|
-
if (!v) throw new Error(
|
|
305
|
+
if (!v) throw new Error("Assertion error");
|
|
282
306
|
return [
|
|
283
307
|
c,
|
|
284
308
|
{
|
|
@@ -292,8 +316,11 @@ export class BlockRegistryV2 {
|
|
|
292
316
|
}
|
|
293
317
|
|
|
294
318
|
// writing global overview
|
|
295
|
-
if (mode !==
|
|
296
|
-
const overviewData = JSON.stringify({
|
|
319
|
+
if (mode !== "dry-run") {
|
|
320
|
+
const overviewData = JSON.stringify({
|
|
321
|
+
schema: "v2",
|
|
322
|
+
packages: overviewPackages,
|
|
323
|
+
} satisfies GlobalOverviewReg);
|
|
297
324
|
const overviewBuffer = Buffer.from(overviewData);
|
|
298
325
|
|
|
299
326
|
// Write regular overview file
|
|
@@ -309,26 +336,26 @@ export class BlockRegistryV2 {
|
|
|
309
336
|
this.logger.info(`Global overview updated (${overviewPackages.length} records)`);
|
|
310
337
|
|
|
311
338
|
// deleting seeds
|
|
312
|
-
if (mode !==
|
|
339
|
+
if (mode !== "dry-run")
|
|
313
340
|
await this.storage.deleteFiles(...seedPaths.map((sp) => `${VersionUpdatesPrefix}${sp}`));
|
|
314
341
|
this.logger.info(`Version update requests cleared`);
|
|
315
342
|
}
|
|
316
343
|
|
|
317
|
-
public async updateIfNeeded(mode:
|
|
344
|
+
public async updateIfNeeded(mode: "force" | "normal" | "dry-run" = "normal"): Promise<void> {
|
|
318
345
|
// implementation of main convergence algorithm
|
|
319
346
|
|
|
320
347
|
this.logger.info(`Checking if registry requires refresh...`);
|
|
321
348
|
const updateRequestSeed = await this.storage.getFile(GlobalUpdateSeedInFile);
|
|
322
349
|
const currentUpdatedSeed = await this.storage.getFile(GlobalUpdateSeedOutFile);
|
|
323
|
-
if (mode !==
|
|
350
|
+
if (mode !== "force" && updateRequestSeed === undefined && currentUpdatedSeed === undefined) {
|
|
324
351
|
this.logger.info(`No global seed files found, update not needed.`);
|
|
325
352
|
return;
|
|
326
353
|
}
|
|
327
354
|
if (
|
|
328
|
-
mode !==
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
355
|
+
mode !== "force" &&
|
|
356
|
+
updateRequestSeed !== undefined &&
|
|
357
|
+
currentUpdatedSeed !== undefined &&
|
|
358
|
+
updateRequestSeed.equals(currentUpdatedSeed)
|
|
332
359
|
) {
|
|
333
360
|
this.logger.info(`Registry is up to date.`);
|
|
334
361
|
return;
|
|
@@ -337,7 +364,7 @@ export class BlockRegistryV2 {
|
|
|
337
364
|
await this.updateRegistry(mode);
|
|
338
365
|
|
|
339
366
|
if (updateRequestSeed) {
|
|
340
|
-
if (mode !==
|
|
367
|
+
if (mode !== "dry-run")
|
|
341
368
|
await this.storage.putFile(GlobalUpdateSeedOutFile, updateRequestSeed);
|
|
342
369
|
this.logger.info(`Refresh finished.`);
|
|
343
370
|
}
|
|
@@ -399,7 +426,7 @@ export class BlockRegistryV2 {
|
|
|
399
426
|
|
|
400
427
|
for (const path of snapshotPaths) {
|
|
401
428
|
// Extract filename from path
|
|
402
|
-
const filename = path.indexOf(
|
|
429
|
+
const filename = path.indexOf("/") === -1 ? path : path.substring(path.lastIndexOf("/") + 1);
|
|
403
430
|
|
|
404
431
|
const match = filename.match(GlobalOverviewSnapshotPattern);
|
|
405
432
|
if (match) {
|
|
@@ -425,7 +452,7 @@ export class BlockRegistryV2 {
|
|
|
425
452
|
}
|
|
426
453
|
|
|
427
454
|
const decompressedData = await this.gunzipAsync(snapshotData);
|
|
428
|
-
const overviewData = decompressedData.toString(
|
|
455
|
+
const overviewData = decompressedData.toString("utf8");
|
|
429
456
|
|
|
430
457
|
// Validate the data
|
|
431
458
|
try {
|
|
@@ -462,13 +489,13 @@ export class BlockRegistryV2 {
|
|
|
462
489
|
`Actual file SHA-256 don't match the checksum from the manifest file for ${f.name} (actual = ${sha256}; manifest = ${f.sha256.toUpperCase()})`,
|
|
463
490
|
);
|
|
464
491
|
|
|
465
|
-
const dst = prefix +
|
|
492
|
+
const dst = prefix + "/" + f.name;
|
|
466
493
|
this.logger.info(`Uploading ${f.name} -> ${dst} ...`);
|
|
467
494
|
await this.storage.putFile(dst, bytes);
|
|
468
495
|
}
|
|
469
496
|
|
|
470
497
|
// uploading manifest as the last upload action
|
|
471
|
-
const manifestDst = prefix +
|
|
498
|
+
const manifestDst = prefix + "/" + ManifestFileName;
|
|
472
499
|
this.logger.info(`Uploading manifest to ${manifestDst} ...`);
|
|
473
500
|
await this.storage.putFile(manifestDst, Buffer.from(JSON.stringify(manifest)));
|
|
474
501
|
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { expect, test } from
|
|
2
|
-
import { RegistryV2Reader } from
|
|
3
|
-
import { folderReaderByUrl } from
|
|
4
|
-
import { request } from
|
|
5
|
-
import { AnyChannel } from
|
|
1
|
+
import { expect, test } from "vitest";
|
|
2
|
+
import { RegistryV2Reader } from "./registry_reader";
|
|
3
|
+
import { folderReaderByUrl } from "../../io";
|
|
4
|
+
import { request } from "undici";
|
|
5
|
+
import { AnyChannel } from "@milaboratories/pl-model-middle-layer";
|
|
6
6
|
|
|
7
|
-
test(
|
|
8
|
-
const registryReader = new RegistryV2Reader(folderReaderByUrl(
|
|
7
|
+
test("test listing packets from global registry", async () => {
|
|
8
|
+
const registryReader = new RegistryV2Reader(folderReaderByUrl("https://blocks.pl-open.science"));
|
|
9
9
|
const listing = await registryReader.listBlockPacks();
|
|
10
10
|
expect(listing.length).toBeGreaterThanOrEqual(1);
|
|
11
11
|
// console.dir(listing, { depth: 5 });
|
|
@@ -13,21 +13,21 @@ test('test listing packets from global registry', async () => {
|
|
|
13
13
|
expect(listing[0].latestByChannel[AnyChannel]).toBeDefined();
|
|
14
14
|
}, 20000);
|
|
15
15
|
|
|
16
|
-
test(
|
|
17
|
-
const registryReader = new RegistryV2Reader(folderReaderByUrl(
|
|
16
|
+
test("test getting components from global registry", async () => {
|
|
17
|
+
const registryReader = new RegistryV2Reader(folderReaderByUrl("https://blocks.pl-open.science"));
|
|
18
18
|
const components = await registryReader.getComponents({
|
|
19
|
-
organization:
|
|
20
|
-
name:
|
|
21
|
-
version:
|
|
19
|
+
organization: "milaboratories",
|
|
20
|
+
name: "samples-and-data",
|
|
21
|
+
version: "1.3.0",
|
|
22
22
|
});
|
|
23
23
|
// console.dir(components, { depth: 5 });
|
|
24
24
|
expect((await (await request(components.ui.url)).body.arrayBuffer()).byteLength).toBeGreaterThan(
|
|
25
|
-
100
|
|
25
|
+
100,
|
|
26
26
|
);
|
|
27
27
|
expect(
|
|
28
|
-
(await (await request(components.model.url)).body.arrayBuffer()).byteLength
|
|
28
|
+
(await (await request(components.model.url)).body.arrayBuffer()).byteLength,
|
|
29
29
|
).toBeGreaterThan(100);
|
|
30
30
|
expect(
|
|
31
|
-
(await (await request(components.workflow.main.url)).body.arrayBuffer()).byteLength
|
|
31
|
+
(await (await request(components.workflow.main.url)).body.arrayBuffer()).byteLength,
|
|
32
32
|
).toBeGreaterThan(100);
|
|
33
33
|
}, 20000);
|
|
@@ -6,14 +6,14 @@ import type {
|
|
|
6
6
|
UpdateSuggestions,
|
|
7
7
|
SingleBlockPackOverview,
|
|
8
8
|
BlockPackOverviewNoRegistryId,
|
|
9
|
-
} from
|
|
9
|
+
} from "@milaboratories/pl-model-middle-layer";
|
|
10
10
|
import {
|
|
11
11
|
blockPackIdNoVersionEquals,
|
|
12
12
|
BlockPackManifest,
|
|
13
13
|
AnyChannel,
|
|
14
|
-
} from
|
|
15
|
-
import type { FolderReader } from
|
|
16
|
-
import canonicalize from
|
|
14
|
+
} from "@milaboratories/pl-model-middle-layer";
|
|
15
|
+
import type { FolderReader } from "../../io";
|
|
16
|
+
import canonicalize from "canonicalize";
|
|
17
17
|
import {
|
|
18
18
|
GlobalOverviewFileName,
|
|
19
19
|
GlobalOverviewReg,
|
|
@@ -21,12 +21,12 @@ import {
|
|
|
21
21
|
ManifestFileName,
|
|
22
22
|
ManifestSuffix,
|
|
23
23
|
packageContentPrefixInsideV2,
|
|
24
|
-
} from
|
|
25
|
-
import { BlockComponentsAbsoluteUrl, BlockPackMetaEmbedBytes } from
|
|
26
|
-
import { LRUCache } from
|
|
27
|
-
import * as semver from
|
|
28
|
-
import { calculateSha256 } from
|
|
29
|
-
import { retry, Retry2TimesWithDelay } from
|
|
24
|
+
} from "./schema_public";
|
|
25
|
+
import { BlockComponentsAbsoluteUrl, BlockPackMetaEmbedBytes } from "../model";
|
|
26
|
+
import { LRUCache } from "lru-cache";
|
|
27
|
+
import * as semver from "semver";
|
|
28
|
+
import { calculateSha256 } from "../../util";
|
|
29
|
+
import { retry, Retry2TimesWithDelay } from "@milaboratories/ts-helpers";
|
|
30
30
|
|
|
31
31
|
export type RegistryV2ReaderOps = {
|
|
32
32
|
/** Number of milliseconds to cache retrieved block list for */
|
|
@@ -42,8 +42,8 @@ const DefaultRegistryV2ReaderOps: RegistryV2ReaderOps = {
|
|
|
42
42
|
|
|
43
43
|
/** @param availableVersions must be reverse sorted (from highest version to lowest) */
|
|
44
44
|
export function inferUpdateSuggestions(currentVersion: string, availableVersions: string[]) {
|
|
45
|
-
const nextMinor = semver.inc(currentVersion,
|
|
46
|
-
const nextMajor = semver.inc(currentVersion,
|
|
45
|
+
const nextMinor = semver.inc(currentVersion, "minor")!;
|
|
46
|
+
const nextMajor = semver.inc(currentVersion, "major")!;
|
|
47
47
|
|
|
48
48
|
// first found = the highest (given the search criteria)
|
|
49
49
|
|
|
@@ -55,9 +55,9 @@ export function inferUpdateSuggestions(currentVersion: string, availableVersions
|
|
|
55
55
|
const minor = availableVersions.find((v) => semver.gte(v, nextMinor) && semver.lt(v, nextMajor));
|
|
56
56
|
const major = availableVersions.find((v) => semver.gte(v, nextMajor));
|
|
57
57
|
|
|
58
|
-
if (patch) suggestions.push({ type:
|
|
59
|
-
if (minor) suggestions.push({ type:
|
|
60
|
-
if (major) suggestions.push({ type:
|
|
58
|
+
if (patch) suggestions.push({ type: "patch", update: patch });
|
|
59
|
+
if (minor) suggestions.push({ type: "minor", update: minor });
|
|
60
|
+
if (major) suggestions.push({ type: "major", update: major });
|
|
61
61
|
|
|
62
62
|
return suggestions;
|
|
63
63
|
}
|
|
@@ -71,7 +71,7 @@ export class RegistryV2Reader {
|
|
|
71
71
|
ops?: Partial<RegistryV2ReaderOps>,
|
|
72
72
|
) {
|
|
73
73
|
this.v2RootFolderReader = registryReader.relativeReader(MainPrefix);
|
|
74
|
-
this.ops = { ...DefaultRegistryV2ReaderOps, ...
|
|
74
|
+
this.ops = { ...DefaultRegistryV2ReaderOps, ...ops };
|
|
75
75
|
}
|
|
76
76
|
|
|
77
77
|
/**
|
|
@@ -87,11 +87,11 @@ export class RegistryV2Reader {
|
|
|
87
87
|
max: 500,
|
|
88
88
|
fetchMethod: async (_key, _staleValue, options) =>
|
|
89
89
|
await retry(async () => {
|
|
90
|
-
const contentReader
|
|
91
|
-
|
|
90
|
+
const contentReader =
|
|
91
|
+
options.context.relativeTo !== undefined
|
|
92
92
|
? this.v2RootFolderReader
|
|
93
|
-
|
|
94
|
-
|
|
93
|
+
.relativeReader(packageContentPrefixInsideV2(options.context.relativeTo))
|
|
94
|
+
.getContentReader()
|
|
95
95
|
: this.v2RootFolderReader.getContentReader();
|
|
96
96
|
return await BlockPackMetaEmbedBytes(contentReader).parseAsync(options.context.meta);
|
|
97
97
|
}, Retry2TimesWithDelay),
|
|
@@ -114,8 +114,8 @@ export class RegistryV2Reader {
|
|
|
114
114
|
|
|
115
115
|
public async listBlockPacks(): Promise<BlockPackOverviewNoRegistryId[]> {
|
|
116
116
|
if (
|
|
117
|
-
this.listCache !== undefined
|
|
118
|
-
|
|
117
|
+
this.listCache !== undefined &&
|
|
118
|
+
Date.now() - this.listCacheTimestamp <= this.ops.cacheBlockListFor
|
|
119
119
|
)
|
|
120
120
|
return this.listCache;
|
|
121
121
|
try {
|
|
@@ -142,7 +142,7 @@ export class RegistryV2Reader {
|
|
|
142
142
|
),
|
|
143
143
|
featureFlags: data.description.featureFlags,
|
|
144
144
|
spec: {
|
|
145
|
-
type:
|
|
145
|
+
type: "from-registry-v2",
|
|
146
146
|
id: data.description.id,
|
|
147
147
|
registryUrl: this.registryReader.rootUrl.toString(),
|
|
148
148
|
channel,
|
|
@@ -152,7 +152,9 @@ export class RegistryV2Reader {
|
|
|
152
152
|
);
|
|
153
153
|
return {
|
|
154
154
|
id: p.id,
|
|
155
|
-
latestByChannel: Object.fromEntries(
|
|
155
|
+
latestByChannel: Object.fromEntries(
|
|
156
|
+
byChannelEntries,
|
|
157
|
+
) as BlockPackOverviewNoRegistryId["latestByChannel"],
|
|
156
158
|
allVersions: p.allVersionsWithChannels,
|
|
157
159
|
} satisfies BlockPackOverviewNoRegistryId;
|
|
158
160
|
}),
|
|
@@ -165,8 +167,8 @@ export class RegistryV2Reader {
|
|
|
165
167
|
}, Retry2TimesWithDelay);
|
|
166
168
|
} catch (e: unknown) {
|
|
167
169
|
if (
|
|
168
|
-
this.listCache !== undefined
|
|
169
|
-
|
|
170
|
+
this.listCache !== undefined &&
|
|
171
|
+
Date.now() - this.listCacheTimestamp <= this.ops.keepStaleBlockListFor
|
|
170
172
|
)
|
|
171
173
|
return this.listCache;
|
|
172
174
|
throw e;
|
|
@@ -221,7 +223,7 @@ export class RegistryV2Reader {
|
|
|
221
223
|
overview.description.meta,
|
|
222
224
|
),
|
|
223
225
|
spec: {
|
|
224
|
-
type:
|
|
226
|
+
type: "from-registry-v2",
|
|
225
227
|
id,
|
|
226
228
|
registryUrl: this.registryReader.rootUrl.toString(),
|
|
227
229
|
channel,
|
|
@@ -1,23 +1,24 @@
|
|
|
1
|
-
import type { BlockPackId, BlockPackIdNoVersion } from
|
|
1
|
+
import type { BlockPackId, BlockPackIdNoVersion } from "@milaboratories/pl-model-middle-layer";
|
|
2
2
|
|
|
3
|
-
export const VersionUpdatesPrefix =
|
|
3
|
+
export const VersionUpdatesPrefix = "_updates_v2/per_package_version/";
|
|
4
4
|
|
|
5
5
|
export function packageUpdateSeedPath(bp: BlockPackId, seed: string): string {
|
|
6
6
|
return `${VersionUpdatesPrefix}${bp.organization}/${bp.name}/${bp.version}/${seed}`;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
-
export const PackageUpdatePattern
|
|
10
|
-
|
|
9
|
+
export const PackageUpdatePattern =
|
|
10
|
+
/(?<packageKeyWithoutVersion>(?<organization>[^/]+)\/(?<name>[^/]+))\/(?<version>[^/]+)\/(?<seed>[^/]+)$/;
|
|
11
11
|
|
|
12
|
-
export const GlobalUpdateSeedInFile =
|
|
13
|
-
export const GlobalUpdateSeedOutFile =
|
|
12
|
+
export const GlobalUpdateSeedInFile = "_updates_v2/_global_update_in";
|
|
13
|
+
export const GlobalUpdateSeedOutFile = "_updates_v2/_global_update_out";
|
|
14
14
|
|
|
15
15
|
// Snapshot storage structure
|
|
16
|
-
export const OverviewSnapshotsPrefix =
|
|
17
|
-
export const GlobalSnapshotsPrefix =
|
|
18
|
-
export const PackageSnapshotsPrefix =
|
|
16
|
+
export const OverviewSnapshotsPrefix = "_overview_snapshots_v2/";
|
|
17
|
+
export const GlobalSnapshotsPrefix = "_overview_snapshots_v2/global/";
|
|
18
|
+
export const PackageSnapshotsPrefix = "_overview_snapshots_v2/per_package/";
|
|
19
19
|
|
|
20
|
-
export const GlobalOverviewSnapshotPattern =
|
|
20
|
+
export const GlobalOverviewSnapshotPattern =
|
|
21
|
+
/^(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.\d{3}Z-[a-z0-9]+)\.json\.gz$/;
|
|
21
22
|
|
|
22
23
|
export function globalOverviewSnapshotPath(timestamp: string): string {
|
|
23
24
|
return `${GlobalSnapshotsPrefix}${timestamp}.json.gz`;
|