@platforma-sdk/block-tools 2.6.27 → 2.6.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +3 -3
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +15 -13
- package/dist/cli.mjs.map +1 -1
- package/dist/{config-XBQ2O39y.mjs → config-DKBY0B2u.mjs} +272 -273
- package/dist/config-DKBY0B2u.mjs.map +1 -0
- package/dist/config-Ycas5fbX.js +3 -0
- package/dist/config-Ycas5fbX.js.map +1 -0
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +18 -17
- package/dist/index.mjs.map +1 -1
- package/dist/registry_v1/v1_repo_schema.d.ts +2 -2
- package/package.json +16 -13
- package/src/cmd/build-meta.ts +9 -9
- package/src/cmd/build-model.ts +20 -16
- package/src/cmd/index.ts +0 -1
- package/src/cmd/list-overview-snapshots.ts +5 -5
- package/src/cmd/mark-stable.ts +14 -15
- package/src/cmd/pack.ts +7 -7
- package/src/cmd/publish.ts +16 -16
- package/src/cmd/refresh-registry.ts +4 -10
- package/src/cmd/restore-overview-from-snapshot.ts +16 -16
- package/src/cmd/update-deps.ts +3 -2
- package/src/cmd/upload-package-v1.ts +12 -12
- package/src/io/folder_reader.ts +15 -11
- package/src/io/storage.ts +25 -23
- package/src/registry_v1/config.ts +11 -10
- package/src/registry_v1/config_schema.ts +5 -5
- package/src/registry_v1/flags.ts +3 -2
- package/src/registry_v1/registry.ts +34 -35
- package/src/registry_v1/v1_repo_schema.ts +2 -2
- package/src/util.ts +15 -11
- package/src/v2/build_dist.ts +12 -9
- package/src/v2/model/block_components.ts +6 -6
- package/src/v2/model/block_description.ts +13 -13
- package/src/v2/model/block_meta.ts +10 -9
- package/src/v2/model/content_conversion.ts +28 -26
- package/src/v2/registry/index.ts +3 -3
- package/src/v2/registry/registry.test.ts +1 -1
- package/src/v2/registry/registry.ts +64 -63
- package/src/v2/registry/registry_reader.ts +47 -46
- package/src/v2/registry/schema_internal.ts +3 -3
- package/src/v2/registry/schema_public.ts +29 -26
- package/src/v2/source_package.ts +26 -23
- package/dist/config-DjpRXRy9.js +0 -3
- package/dist/config-DjpRXRy9.js.map +0 -1
- package/dist/config-XBQ2O39y.mjs.map +0 -1
|
@@ -4,7 +4,7 @@ import { randomUUID } from 'crypto';
|
|
|
4
4
|
import path from 'path';
|
|
5
5
|
import fsp from 'fs/promises';
|
|
6
6
|
import { BlockRegistryV2 } from './registry';
|
|
7
|
-
import semver from 'semver';
|
|
7
|
+
import * as semver from 'semver';
|
|
8
8
|
import { UpdateSuggestions, BlockPackManifest } from '@milaboratories/pl-model-middle-layer';
|
|
9
9
|
import { inferUpdateSuggestions } from './registry_reader';
|
|
10
10
|
import { OverviewSnapshotsPrefix } from './schema_internal';
|
|
@@ -1,14 +1,16 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import type { MiLogger } from '@milaboratories/ts-helpers';
|
|
2
|
+
import { ConsoleLoggerAdapter } from '@milaboratories/ts-helpers';
|
|
3
|
+
import { compare as compareSemver } from 'semver';
|
|
3
4
|
import { gzip, gunzip } from 'node:zlib';
|
|
4
5
|
import { promisify } from 'node:util';
|
|
5
|
-
import { RegistryStorage } from '../../io/storage';
|
|
6
|
+
import type { RegistryStorage } from '../../io/storage';
|
|
7
|
+
import type {
|
|
8
|
+
BlockPackId,
|
|
9
|
+
BlockPackIdNoVersion } from '@milaboratories/pl-model-middle-layer';
|
|
6
10
|
import {
|
|
7
11
|
AnyChannel,
|
|
8
|
-
BlockPackId,
|
|
9
|
-
BlockPackIdNoVersion,
|
|
10
12
|
blockPackIdToString,
|
|
11
|
-
BlockPackManifest
|
|
13
|
+
BlockPackManifest,
|
|
12
14
|
} from '@milaboratories/pl-model-middle-layer';
|
|
13
15
|
import {
|
|
14
16
|
GlobalUpdateSeedInFile,
|
|
@@ -17,11 +19,9 @@ import {
|
|
|
17
19
|
packageUpdateSeedPath,
|
|
18
20
|
VersionUpdatesPrefix,
|
|
19
21
|
GlobalOverviewSnapshotPattern,
|
|
20
|
-
OverviewSnapshotsPrefix,
|
|
21
22
|
GlobalSnapshotsPrefix,
|
|
22
|
-
PackageSnapshotsPrefix,
|
|
23
23
|
globalOverviewSnapshotPath,
|
|
24
|
-
packageOverviewSnapshotPath
|
|
24
|
+
packageOverviewSnapshotPath,
|
|
25
25
|
} from './schema_internal';
|
|
26
26
|
import {
|
|
27
27
|
GlobalOverviewReg,
|
|
@@ -36,9 +36,10 @@ import {
|
|
|
36
36
|
packageChannelPrefix,
|
|
37
37
|
ChannelNameRegexp,
|
|
38
38
|
MainPrefix,
|
|
39
|
-
PackageManifestPattern
|
|
39
|
+
PackageManifestPattern,
|
|
40
40
|
} from './schema_public';
|
|
41
|
-
import {
|
|
41
|
+
import type { RelativeContentReader } from '../model';
|
|
42
|
+
import { BlockPackDescriptionManifestAddRelativePathPrefix } from '../model';
|
|
42
43
|
import { randomUUID } from 'node:crypto';
|
|
43
44
|
import { calculateSha256 } from '../../util';
|
|
44
45
|
|
|
@@ -53,7 +54,7 @@ export interface GlobalOverviewBackupDescription {
|
|
|
53
54
|
|
|
54
55
|
type PackageUpdateInfo = {
|
|
55
56
|
package: BlockPackIdNoVersion;
|
|
56
|
-
versions: Set<
|
|
57
|
+
versions: Set<string>;
|
|
57
58
|
};
|
|
58
59
|
|
|
59
60
|
export class BlockRegistryV2 {
|
|
@@ -63,7 +64,7 @@ export class BlockRegistryV2 {
|
|
|
63
64
|
constructor(
|
|
64
65
|
private readonly storage: RegistryStorage,
|
|
65
66
|
private readonly logger: MiLogger = new ConsoleLoggerAdapter(),
|
|
66
|
-
private readonly settings: BlockRegistrySettings = {}
|
|
67
|
+
private readonly settings: BlockRegistrySettings = {},
|
|
67
68
|
) {}
|
|
68
69
|
|
|
69
70
|
private generateTimestamp(): string {
|
|
@@ -80,20 +81,20 @@ export class BlockRegistryV2 {
|
|
|
80
81
|
|
|
81
82
|
private async createGlobalOverviewSnapshot(overviewData: string, timestamp: string): Promise<void> {
|
|
82
83
|
if (this.settings.skipSnapshotCreation) return;
|
|
83
|
-
|
|
84
|
+
|
|
84
85
|
try {
|
|
85
86
|
const gzippedData = await this.gzipAsync(overviewData);
|
|
86
87
|
const snapshotPath = globalOverviewSnapshotPath(timestamp);
|
|
87
88
|
await this.storage.putFile(snapshotPath, Buffer.from(gzippedData));
|
|
88
89
|
this.logger.info(`Global overview snapshot created at ${snapshotPath}`);
|
|
89
90
|
} catch (error) {
|
|
90
|
-
this.logger.warn(`Failed to create global overview snapshot: ${error}`);
|
|
91
|
+
this.logger.warn(`Failed to create global overview snapshot: ${String(error)}`);
|
|
91
92
|
}
|
|
92
93
|
}
|
|
93
94
|
|
|
94
95
|
private async createPackageOverviewSnapshot(pkg: BlockPackIdNoVersion, overview: PackageOverview, timestamp: string): Promise<void> {
|
|
95
96
|
if (this.settings.skipSnapshotCreation) return;
|
|
96
|
-
|
|
97
|
+
|
|
97
98
|
try {
|
|
98
99
|
const overviewData = JSON.stringify(overview);
|
|
99
100
|
const gzippedData = await this.gzipAsync(overviewData);
|
|
@@ -101,7 +102,7 @@ export class BlockRegistryV2 {
|
|
|
101
102
|
await this.storage.putFile(snapshotPath, Buffer.from(gzippedData));
|
|
102
103
|
this.logger.info(`Package overview snapshot created at ${snapshotPath} for ${pkg.organization}:${pkg.name}`);
|
|
103
104
|
} catch (error) {
|
|
104
|
-
this.logger.warn(`Failed to create package overview snapshot for ${pkg.organization}:${pkg.name}: ${error}`);
|
|
105
|
+
this.logger.warn(`Failed to create package overview snapshot for ${pkg.organization}:${pkg.name}: ${String(error)}`);
|
|
105
106
|
}
|
|
106
107
|
}
|
|
107
108
|
|
|
@@ -118,11 +119,11 @@ export class BlockRegistryV2 {
|
|
|
118
119
|
|
|
119
120
|
const addVersionToBeUpdated = ({ organization, name, version }: BlockPackId) => {
|
|
120
121
|
const keyNoVersion = `${organization}:${name}`;
|
|
121
|
-
|
|
122
|
+
const update = packagesToUpdate.get(keyNoVersion);
|
|
122
123
|
if (!update) {
|
|
123
124
|
packagesToUpdate.set(keyNoVersion, {
|
|
124
125
|
package: { organization, name },
|
|
125
|
-
versions: new Set([version])
|
|
126
|
+
versions: new Set([version]),
|
|
126
127
|
});
|
|
127
128
|
return true;
|
|
128
129
|
} else if (!update.versions.has(version)) {
|
|
@@ -137,7 +138,7 @@ export class BlockRegistryV2 {
|
|
|
137
138
|
const match = seedPath.match(PackageUpdatePattern);
|
|
138
139
|
if (!match) continue;
|
|
139
140
|
seedPaths.push(seedPath);
|
|
140
|
-
const { organization, name, version, seed } = match.groups!;
|
|
141
|
+
const { organization, name, version, seed: _seed } = match.groups!;
|
|
141
142
|
const added = addVersionToBeUpdated({ organization, name, version });
|
|
142
143
|
this.logger.info(` - ${organization}:${name}:${version} added:${added}`);
|
|
143
144
|
}
|
|
@@ -156,13 +157,13 @@ export class BlockRegistryV2 {
|
|
|
156
157
|
|
|
157
158
|
// loading global overview
|
|
158
159
|
const overviewContent = await this.storage.getFile(GlobalOverviewPath);
|
|
159
|
-
|
|
160
|
+
|
|
160
161
|
// Create pre-write snapshot in force mode if overview exists
|
|
161
162
|
if (mode === 'force' && overviewContent !== undefined) {
|
|
162
163
|
const preWriteTimestamp = this.generatePreWriteTimestamp();
|
|
163
164
|
await this.createGlobalOverviewSnapshot(overviewContent.toString(), preWriteTimestamp);
|
|
164
165
|
}
|
|
165
|
-
|
|
166
|
+
|
|
166
167
|
const overview: GlobalOverviewReg = mode === 'force'
|
|
167
168
|
? { schema: 'v2', packages: [] }
|
|
168
169
|
: overviewContent === undefined
|
|
@@ -176,26 +177,26 @@ export class BlockRegistryV2 {
|
|
|
176
177
|
// reading existing overview
|
|
177
178
|
const overviewFile = packageOverviewPath(packageInfo.package);
|
|
178
179
|
const pOverviewContent = await this.storage.getFile(overviewFile);
|
|
179
|
-
|
|
180
|
+
|
|
180
181
|
// Create pre-write snapshot in force mode if package overview exists
|
|
181
182
|
if (mode === 'force' && pOverviewContent !== undefined) {
|
|
182
183
|
const preWriteTimestamp = this.generatePreWriteTimestamp();
|
|
183
184
|
const existingOverview = PackageOverview.parse(JSON.parse(pOverviewContent.toString()));
|
|
184
185
|
await this.createPackageOverviewSnapshot(packageInfo.package, existingOverview, preWriteTimestamp);
|
|
185
186
|
}
|
|
186
|
-
|
|
187
|
+
|
|
187
188
|
const packageOverview: PackageOverview = mode === 'force'
|
|
188
189
|
? { schema: 'v2', versions: [] }
|
|
189
190
|
: pOverviewContent === undefined
|
|
190
191
|
? { schema: 'v2', versions: [] }
|
|
191
192
|
: PackageOverview.parse(JSON.parse(pOverviewContent.toString()));
|
|
192
193
|
this.logger.info(
|
|
193
|
-
`Updating ${packageInfo.package.organization}:${packageInfo.package.name} overview${mode === 'force' ? ' (starting empty in force mode)' : ''}, ${packageOverview.versions.length} records
|
|
194
|
+
`Updating ${packageInfo.package.organization}:${packageInfo.package.name} overview${mode === 'force' ? ' (starting empty in force mode)' : ''}, ${packageOverview.versions.length} records`,
|
|
194
195
|
);
|
|
195
196
|
|
|
196
197
|
// removing versions that we will update
|
|
197
198
|
const newVersions = packageOverview.versions.filter(
|
|
198
|
-
(e) => !packageInfo.versions.has(e.description.id.version)
|
|
199
|
+
(e) => !packageInfo.versions.has(e.description.id.version),
|
|
199
200
|
);
|
|
200
201
|
|
|
201
202
|
// reading new entries
|
|
@@ -203,10 +204,10 @@ export class BlockRegistryV2 {
|
|
|
203
204
|
const version = v.toString();
|
|
204
205
|
const id: BlockPackId = {
|
|
205
206
|
...packageInfo.package,
|
|
206
|
-
version
|
|
207
|
+
version,
|
|
207
208
|
};
|
|
208
209
|
const manifestContent = await this.storage.getFile(
|
|
209
|
-
packageContentPrefix(id) + ManifestSuffix
|
|
210
|
+
packageContentPrefix(id) + ManifestSuffix,
|
|
210
211
|
);
|
|
211
212
|
if (!manifestContent) continue; // absent package
|
|
212
213
|
const sha256 = await calculateSha256(manifestContent);
|
|
@@ -221,16 +222,16 @@ export class BlockRegistryV2 {
|
|
|
221
222
|
// pushing the overview
|
|
222
223
|
newVersions.push({
|
|
223
224
|
description: BlockPackDescriptionManifestAddRelativePathPrefix(version).parse(
|
|
224
|
-
BlockPackManifest.parse(JSON.parse(manifestContent.toString('utf8'))).description
|
|
225
|
+
BlockPackManifest.parse(JSON.parse(manifestContent.toString('utf8'))).description,
|
|
225
226
|
),
|
|
226
227
|
manifestSha256: sha256,
|
|
227
|
-
channels
|
|
228
|
+
channels,
|
|
228
229
|
});
|
|
229
230
|
}
|
|
230
231
|
|
|
231
232
|
// sorting entries according to version
|
|
232
233
|
newVersions.sort((e1, e2) =>
|
|
233
|
-
compareSemver(e2.description.id.version, e1.description.id.version)
|
|
234
|
+
compareSemver(e2.description.id.version, e1.description.id.version),
|
|
234
235
|
);
|
|
235
236
|
|
|
236
237
|
// write package overview back
|
|
@@ -238,9 +239,9 @@ export class BlockRegistryV2 {
|
|
|
238
239
|
if (mode !== 'dry-run') {
|
|
239
240
|
await this.storage.putFile(
|
|
240
241
|
overviewFile,
|
|
241
|
-
Buffer.from(JSON.stringify(packageOverviewData))
|
|
242
|
+
Buffer.from(JSON.stringify(packageOverviewData)),
|
|
242
243
|
);
|
|
243
|
-
|
|
244
|
+
|
|
244
245
|
// Create snapshot after successful write
|
|
245
246
|
await this.createPackageOverviewSnapshot(packageInfo.package, packageOverviewData, snapshotTimestamp);
|
|
246
247
|
}
|
|
@@ -253,16 +254,16 @@ export class BlockRegistryV2 {
|
|
|
253
254
|
// patching corresponding entry in overview
|
|
254
255
|
overviewPackages = overviewPackages.filter(
|
|
255
256
|
(e) =>
|
|
256
|
-
e.id.organization !== packageInfo.package.organization
|
|
257
|
-
e.id.name !== packageInfo.package.name
|
|
257
|
+
e.id.organization !== packageInfo.package.organization
|
|
258
|
+
|| e.id.name !== packageInfo.package.name,
|
|
258
259
|
);
|
|
259
260
|
const relativeDescriptionSchema = BlockPackDescriptionManifestAddRelativePathPrefix(
|
|
260
|
-
`${packageInfo.package.organization}/${packageInfo.package.name}
|
|
261
|
+
`${packageInfo.package.organization}/${packageInfo.package.name}`,
|
|
261
262
|
);
|
|
262
263
|
overviewPackages.push({
|
|
263
264
|
id: {
|
|
264
265
|
organization: packageInfo.package.organization,
|
|
265
|
-
name: packageInfo.package.name
|
|
266
|
+
name: packageInfo.package.name,
|
|
266
267
|
},
|
|
267
268
|
// left for backward compatibility
|
|
268
269
|
allVersions: newVersions.map((e) => e.description.id.version).reverse(),
|
|
@@ -282,11 +283,11 @@ export class BlockRegistryV2 {
|
|
|
282
283
|
c,
|
|
283
284
|
{
|
|
284
285
|
description: relativeDescriptionSchema.parse(v.description),
|
|
285
|
-
manifestSha256: v?.manifestSha256
|
|
286
|
-
}
|
|
286
|
+
manifestSha256: v?.manifestSha256,
|
|
287
|
+
},
|
|
287
288
|
];
|
|
288
|
-
})
|
|
289
|
-
)
|
|
289
|
+
}),
|
|
290
|
+
),
|
|
290
291
|
});
|
|
291
292
|
}
|
|
292
293
|
|
|
@@ -294,14 +295,14 @@ export class BlockRegistryV2 {
|
|
|
294
295
|
if (mode !== 'dry-run') {
|
|
295
296
|
const overviewData = JSON.stringify({ schema: 'v2', packages: overviewPackages } satisfies GlobalOverviewReg);
|
|
296
297
|
const overviewBuffer = Buffer.from(overviewData);
|
|
297
|
-
|
|
298
|
+
|
|
298
299
|
// Write regular overview file
|
|
299
300
|
await this.storage.putFile(GlobalOverviewPath, overviewBuffer);
|
|
300
|
-
|
|
301
|
+
|
|
301
302
|
// Write gzipped overview file
|
|
302
303
|
const gzippedBuffer = await this.gzipAsync(overviewData);
|
|
303
304
|
await this.storage.putFile(GlobalOverviewGzPath, Buffer.from(gzippedBuffer));
|
|
304
|
-
|
|
305
|
+
|
|
305
306
|
// Create snapshot after successful writes
|
|
306
307
|
await this.createGlobalOverviewSnapshot(overviewData, snapshotTimestamp);
|
|
307
308
|
}
|
|
@@ -324,10 +325,10 @@ export class BlockRegistryV2 {
|
|
|
324
325
|
return;
|
|
325
326
|
}
|
|
326
327
|
if (
|
|
327
|
-
mode !== 'force'
|
|
328
|
-
updateRequestSeed !== undefined
|
|
329
|
-
currentUpdatedSeed !== undefined
|
|
330
|
-
updateRequestSeed.equals(currentUpdatedSeed)
|
|
328
|
+
mode !== 'force'
|
|
329
|
+
&& updateRequestSeed !== undefined
|
|
330
|
+
&& currentUpdatedSeed !== undefined
|
|
331
|
+
&& updateRequestSeed.equals(currentUpdatedSeed)
|
|
331
332
|
) {
|
|
332
333
|
this.logger.info(`Registry is up to date.`);
|
|
333
334
|
return;
|
|
@@ -343,7 +344,7 @@ export class BlockRegistryV2 {
|
|
|
343
344
|
}
|
|
344
345
|
|
|
345
346
|
public async getPackageOverview(
|
|
346
|
-
name: BlockPackIdNoVersion
|
|
347
|
+
name: BlockPackIdNoVersion,
|
|
347
348
|
): Promise<undefined | PackageOverview> {
|
|
348
349
|
const content = await this.storage.getFile(packageOverviewPath(name));
|
|
349
350
|
if (content === undefined) return undefined;
|
|
@@ -395,20 +396,20 @@ export class BlockRegistryV2 {
|
|
|
395
396
|
public async listGlobalOverviewSnapshots(): Promise<GlobalOverviewBackupDescription[]> {
|
|
396
397
|
const snapshotPaths = await this.storage.listFiles(GlobalSnapshotsPrefix);
|
|
397
398
|
const snapshots: GlobalOverviewBackupDescription[] = [];
|
|
398
|
-
|
|
399
|
+
|
|
399
400
|
for (const path of snapshotPaths) {
|
|
400
401
|
// Extract filename from path
|
|
401
402
|
const filename = path.indexOf('/') === -1 ? path : path.substring(path.lastIndexOf('/') + 1);
|
|
402
|
-
|
|
403
|
+
|
|
403
404
|
const match = filename.match(GlobalOverviewSnapshotPattern);
|
|
404
405
|
if (match) {
|
|
405
406
|
snapshots.push({
|
|
406
407
|
timestamp: match.groups!.timestamp,
|
|
407
|
-
path: GlobalSnapshotsPrefix + filename
|
|
408
|
+
path: GlobalSnapshotsPrefix + filename,
|
|
408
409
|
});
|
|
409
410
|
}
|
|
410
411
|
}
|
|
411
|
-
|
|
412
|
+
|
|
412
413
|
// Sort by timestamp descending (newest first)
|
|
413
414
|
snapshots.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
|
|
414
415
|
return snapshots;
|
|
@@ -416,36 +417,36 @@ export class BlockRegistryV2 {
|
|
|
416
417
|
|
|
417
418
|
public async restoreGlobalOverviewFromSnapshot(backupId: string): Promise<void> {
|
|
418
419
|
const snapshotPath = globalOverviewSnapshotPath(backupId);
|
|
419
|
-
|
|
420
|
+
|
|
420
421
|
// Read and decompress the snapshot
|
|
421
422
|
const snapshotData = await this.storage.getFile(snapshotPath);
|
|
422
423
|
if (!snapshotData) {
|
|
423
424
|
throw new Error(`Snapshot ${backupId} not found at ${snapshotPath}`);
|
|
424
425
|
}
|
|
425
|
-
|
|
426
|
+
|
|
426
427
|
const decompressedData = await this.gunzipAsync(snapshotData);
|
|
427
428
|
const overviewData = decompressedData.toString('utf8');
|
|
428
|
-
|
|
429
|
+
|
|
429
430
|
// Validate the data
|
|
430
431
|
try {
|
|
431
432
|
GlobalOverviewReg.parse(JSON.parse(overviewData));
|
|
432
433
|
} catch (error) {
|
|
433
|
-
throw new Error(`Invalid snapshot data in ${backupId}: ${error}`);
|
|
434
|
+
throw new Error(`Invalid snapshot data in ${backupId}: ${String(error)}`);
|
|
434
435
|
}
|
|
435
|
-
|
|
436
|
+
|
|
436
437
|
// Write both regular and gzipped versions
|
|
437
438
|
const overviewBuffer = Buffer.from(overviewData);
|
|
438
439
|
await this.storage.putFile(GlobalOverviewPath, overviewBuffer);
|
|
439
|
-
|
|
440
|
+
|
|
440
441
|
const gzippedBuffer = await this.gzipAsync(overviewData);
|
|
441
442
|
await this.storage.putFile(GlobalOverviewGzPath, Buffer.from(gzippedBuffer));
|
|
442
|
-
|
|
443
|
+
|
|
443
444
|
this.logger.info(`Global overview restored from snapshot ${backupId}`);
|
|
444
445
|
}
|
|
445
446
|
|
|
446
447
|
public async publishPackage(
|
|
447
448
|
manifest: BlockPackManifest,
|
|
448
|
-
fileReader: RelativeContentReader
|
|
449
|
+
fileReader: RelativeContentReader,
|
|
449
450
|
): Promise<void> {
|
|
450
451
|
const prefix = packageContentPrefix(manifest.description.id);
|
|
451
452
|
// uploading content files
|
|
@@ -453,12 +454,12 @@ export class BlockRegistryV2 {
|
|
|
453
454
|
const bytes = await fileReader(f.name);
|
|
454
455
|
if (bytes.length !== f.size)
|
|
455
456
|
throw new Error(
|
|
456
|
-
`Actual file size don't match file size from the manifest file for ${f.name} (actual = ${bytes.length}; manifest = ${f.size})
|
|
457
|
+
`Actual file size don't match file size from the manifest file for ${f.name} (actual = ${bytes.length}; manifest = ${f.size})`,
|
|
457
458
|
);
|
|
458
459
|
const sha256 = await calculateSha256(bytes);
|
|
459
460
|
if (sha256 !== f.sha256.toUpperCase())
|
|
460
461
|
throw new Error(
|
|
461
|
-
`Actual file SHA-256 don't match the checksum from the manifest file for ${f.name} (actual = ${sha256}; manifest = ${f.sha256.toUpperCase()})
|
|
462
|
+
`Actual file SHA-256 don't match the checksum from the manifest file for ${f.name} (actual = ${sha256}; manifest = ${f.sha256.toUpperCase()})`,
|
|
462
463
|
);
|
|
463
464
|
|
|
464
465
|
const dst = prefix + '/' + f.name;
|
|
@@ -1,17 +1,18 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type {
|
|
2
2
|
BlockPackId,
|
|
3
3
|
BlockPackIdNoVersion,
|
|
4
|
-
blockPackIdNoVersionEquals,
|
|
5
|
-
BlockPackManifest,
|
|
6
4
|
BlockPackMetaEmbeddedBytes,
|
|
7
5
|
BlockPackMetaManifest,
|
|
8
|
-
BlockPackOverview,
|
|
9
6
|
UpdateSuggestions,
|
|
10
7
|
SingleBlockPackOverview,
|
|
8
|
+
BlockPackOverviewNoRegistryId,
|
|
9
|
+
} from '@milaboratories/pl-model-middle-layer';
|
|
10
|
+
import {
|
|
11
|
+
blockPackIdNoVersionEquals,
|
|
12
|
+
BlockPackManifest,
|
|
11
13
|
AnyChannel,
|
|
12
|
-
BlockPackOverviewNoRegistryId
|
|
13
14
|
} from '@milaboratories/pl-model-middle-layer';
|
|
14
|
-
import { FolderReader } from '../../io';
|
|
15
|
+
import type { FolderReader } from '../../io';
|
|
15
16
|
import canonicalize from 'canonicalize';
|
|
16
17
|
import {
|
|
17
18
|
GlobalOverviewFileName,
|
|
@@ -19,11 +20,11 @@ import {
|
|
|
19
20
|
MainPrefix,
|
|
20
21
|
ManifestFileName,
|
|
21
22
|
ManifestSuffix,
|
|
22
|
-
packageContentPrefixInsideV2
|
|
23
|
+
packageContentPrefixInsideV2,
|
|
23
24
|
} from './schema_public';
|
|
24
25
|
import { BlockComponentsAbsoluteUrl, BlockPackMetaEmbedBytes } from '../model';
|
|
25
26
|
import { LRUCache } from 'lru-cache';
|
|
26
|
-
import semver from 'semver';
|
|
27
|
+
import * as semver from 'semver';
|
|
27
28
|
import { calculateSha256 } from '../../util';
|
|
28
29
|
import { retry, Retry2TimesWithDelay } from '@milaboratories/ts-helpers';
|
|
29
30
|
|
|
@@ -36,7 +37,7 @@ export type RegistryV2ReaderOps = {
|
|
|
36
37
|
|
|
37
38
|
const DefaultRegistryV2ReaderOps: RegistryV2ReaderOps = {
|
|
38
39
|
cacheBlockListFor: 45e3, // 45 seconds
|
|
39
|
-
keepStaleBlockListFor: 300e3 // 5 minutes
|
|
40
|
+
keepStaleBlockListFor: 300e3, // 5 minutes
|
|
40
41
|
};
|
|
41
42
|
|
|
42
43
|
/** @param availableVersions must be reverse sorted (from highest version to lowest) */
|
|
@@ -49,7 +50,7 @@ export function inferUpdateSuggestions(currentVersion: string, availableVersions
|
|
|
49
50
|
const suggestions: UpdateSuggestions<string> = [];
|
|
50
51
|
|
|
51
52
|
const patch = availableVersions.find(
|
|
52
|
-
(v) => semver.gt(v, currentVersion) && semver.lt(v, nextMinor)
|
|
53
|
+
(v) => semver.gt(v, currentVersion) && semver.lt(v, nextMinor),
|
|
53
54
|
);
|
|
54
55
|
const minor = availableVersions.find((v) => semver.gte(v, nextMinor) && semver.lt(v, nextMajor));
|
|
55
56
|
const major = availableVersions.find((v) => semver.gte(v, nextMajor));
|
|
@@ -67,7 +68,7 @@ export class RegistryV2Reader {
|
|
|
67
68
|
|
|
68
69
|
constructor(
|
|
69
70
|
private readonly registryReader: FolderReader,
|
|
70
|
-
ops?: Partial<RegistryV2ReaderOps
|
|
71
|
+
ops?: Partial<RegistryV2ReaderOps>,
|
|
71
72
|
) {
|
|
72
73
|
this.v2RootFolderReader = registryReader.relativeReader(MainPrefix);
|
|
73
74
|
this.ops = { ...DefaultRegistryV2ReaderOps, ...(ops ?? {}) };
|
|
@@ -86,25 +87,25 @@ export class RegistryV2Reader {
|
|
|
86
87
|
max: 500,
|
|
87
88
|
fetchMethod: async (_key, _staleValue, options) =>
|
|
88
89
|
await retry(async () => {
|
|
89
|
-
const contentReader
|
|
90
|
-
options.context.relativeTo !== undefined
|
|
90
|
+
const contentReader
|
|
91
|
+
= options.context.relativeTo !== undefined
|
|
91
92
|
? this.v2RootFolderReader
|
|
92
|
-
|
|
93
|
-
|
|
93
|
+
.relativeReader(packageContentPrefixInsideV2(options.context.relativeTo))
|
|
94
|
+
.getContentReader()
|
|
94
95
|
: this.v2RootFolderReader.getContentReader();
|
|
95
96
|
return await BlockPackMetaEmbedBytes(contentReader).parseAsync(options.context.meta);
|
|
96
|
-
}, Retry2TimesWithDelay)
|
|
97
|
+
}, Retry2TimesWithDelay),
|
|
97
98
|
});
|
|
98
99
|
|
|
99
100
|
private async embedMetaContent(
|
|
100
101
|
id: BlockPackId,
|
|
101
102
|
sha256: string,
|
|
102
103
|
absolutePath: boolean,
|
|
103
|
-
meta: BlockPackMetaManifest
|
|
104
|
+
meta: BlockPackMetaManifest,
|
|
104
105
|
): Promise<BlockPackMetaEmbeddedBytes> {
|
|
105
106
|
return await this.embeddedGlobalMetaCache.forceFetch(
|
|
106
107
|
canonicalize({ id, sha256, absolutePath })!,
|
|
107
|
-
{ context: { meta, relativeTo: absolutePath ? undefined : id } }
|
|
108
|
+
{ context: { meta, relativeTo: absolutePath ? undefined : id } },
|
|
108
109
|
);
|
|
109
110
|
}
|
|
110
111
|
|
|
@@ -113,8 +114,8 @@ export class RegistryV2Reader {
|
|
|
113
114
|
|
|
114
115
|
public async listBlockPacks(): Promise<BlockPackOverviewNoRegistryId[]> {
|
|
115
116
|
if (
|
|
116
|
-
this.listCache !== undefined
|
|
117
|
-
Date.now() - this.listCacheTimestamp <= this.ops.cacheBlockListFor
|
|
117
|
+
this.listCache !== undefined
|
|
118
|
+
&& Date.now() - this.listCacheTimestamp <= this.ops.cacheBlockListFor
|
|
118
119
|
)
|
|
119
120
|
return this.listCache;
|
|
120
121
|
try {
|
|
@@ -122,8 +123,8 @@ export class RegistryV2Reader {
|
|
|
122
123
|
// const rootContentReader = this.v2RootFolderReader.getContentReader();
|
|
123
124
|
const globalOverview = GlobalOverviewReg.parse(
|
|
124
125
|
JSON.parse(
|
|
125
|
-
Buffer.from(await this.v2RootFolderReader.readFile(GlobalOverviewFileName)).toString()
|
|
126
|
-
)
|
|
126
|
+
Buffer.from(await this.v2RootFolderReader.readFile(GlobalOverviewFileName)).toString(),
|
|
127
|
+
),
|
|
127
128
|
);
|
|
128
129
|
|
|
129
130
|
const result = await Promise.all(
|
|
@@ -137,24 +138,24 @@ export class RegistryV2Reader {
|
|
|
137
138
|
data.description.id,
|
|
138
139
|
data.manifestSha256,
|
|
139
140
|
true,
|
|
140
|
-
data.description.meta
|
|
141
|
+
data.description.meta,
|
|
141
142
|
),
|
|
142
143
|
featureFlags: data.description.featureFlags,
|
|
143
144
|
spec: {
|
|
144
145
|
type: 'from-registry-v2',
|
|
145
146
|
id: data.description.id,
|
|
146
147
|
registryUrl: this.registryReader.rootUrl.toString(),
|
|
147
|
-
channel
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
])
|
|
148
|
+
channel,
|
|
149
|
+
},
|
|
150
|
+
},
|
|
151
|
+
]),
|
|
151
152
|
);
|
|
152
153
|
return {
|
|
153
154
|
id: p.id,
|
|
154
|
-
latestByChannel: Object.fromEntries(byChannelEntries),
|
|
155
|
-
allVersions: p.allVersionsWithChannels
|
|
155
|
+
latestByChannel: Object.fromEntries(byChannelEntries) as BlockPackOverviewNoRegistryId['latestByChannel'],
|
|
156
|
+
allVersions: p.allVersionsWithChannels,
|
|
156
157
|
} satisfies BlockPackOverviewNoRegistryId;
|
|
157
|
-
})
|
|
158
|
+
}),
|
|
158
159
|
);
|
|
159
160
|
|
|
160
161
|
this.listCache = result;
|
|
@@ -164,8 +165,8 @@ export class RegistryV2Reader {
|
|
|
164
165
|
}, Retry2TimesWithDelay);
|
|
165
166
|
} catch (e: unknown) {
|
|
166
167
|
if (
|
|
167
|
-
this.listCache !== undefined
|
|
168
|
-
Date.now() - this.listCacheTimestamp <= this.ops.keepStaleBlockListFor
|
|
168
|
+
this.listCache !== undefined
|
|
169
|
+
&& Date.now() - this.listCacheTimestamp <= this.ops.keepStaleBlockListFor
|
|
169
170
|
)
|
|
170
171
|
return this.listCache;
|
|
171
172
|
throw e;
|
|
@@ -174,10 +175,10 @@ export class RegistryV2Reader {
|
|
|
174
175
|
|
|
175
176
|
public async getLatestOverview(
|
|
176
177
|
id: BlockPackIdNoVersion,
|
|
177
|
-
channel: string
|
|
178
|
+
channel: string,
|
|
178
179
|
): Promise<SingleBlockPackOverview | undefined> {
|
|
179
180
|
const overview = (await this.listBlockPacks()).find((e) =>
|
|
180
|
-
blockPackIdNoVersionEquals(id, e.id)
|
|
181
|
+
blockPackIdNoVersionEquals(id, e.id),
|
|
181
182
|
);
|
|
182
183
|
if (overview === undefined) return undefined;
|
|
183
184
|
return overview.latestByChannel[channel];
|
|
@@ -185,10 +186,10 @@ export class RegistryV2Reader {
|
|
|
185
186
|
|
|
186
187
|
public async getUpdateSuggestions(
|
|
187
188
|
id: BlockPackId,
|
|
188
|
-
channel: string
|
|
189
|
+
channel: string,
|
|
189
190
|
): Promise<UpdateSuggestions<string> | undefined> {
|
|
190
191
|
const overview = (await this.listBlockPacks()).find((e) =>
|
|
191
|
-
blockPackIdNoVersionEquals(id, e.id)
|
|
192
|
+
blockPackIdNoVersionEquals(id, e.id),
|
|
192
193
|
);
|
|
193
194
|
if (overview === undefined) return undefined;
|
|
194
195
|
|
|
@@ -204,11 +205,11 @@ export class RegistryV2Reader {
|
|
|
204
205
|
|
|
205
206
|
public async getSpecificOverview(
|
|
206
207
|
id: BlockPackId,
|
|
207
|
-
channel: string
|
|
208
|
+
channel: string,
|
|
208
209
|
): Promise<SingleBlockPackOverview> {
|
|
209
210
|
return await retry(async () => {
|
|
210
211
|
const manifestContent = await this.v2RootFolderReader.readFile(
|
|
211
|
-
packageContentPrefixInsideV2(id) + ManifestSuffix
|
|
212
|
+
packageContentPrefixInsideV2(id) + ManifestSuffix,
|
|
212
213
|
);
|
|
213
214
|
const overview = BlockPackManifest.parse(JSON.parse(Buffer.from(manifestContent).toString()));
|
|
214
215
|
return {
|
|
@@ -217,32 +218,32 @@ export class RegistryV2Reader {
|
|
|
217
218
|
id,
|
|
218
219
|
await calculateSha256(manifestContent),
|
|
219
220
|
false,
|
|
220
|
-
overview.description.meta
|
|
221
|
+
overview.description.meta,
|
|
221
222
|
),
|
|
222
223
|
spec: {
|
|
223
224
|
type: 'from-registry-v2',
|
|
224
225
|
id,
|
|
225
226
|
registryUrl: this.registryReader.rootUrl.toString(),
|
|
226
|
-
channel
|
|
227
|
-
}
|
|
227
|
+
channel,
|
|
228
|
+
},
|
|
228
229
|
};
|
|
229
230
|
}, Retry2TimesWithDelay);
|
|
230
231
|
}
|
|
231
232
|
|
|
232
233
|
private readonly componentsCache = new LRUCache<string, BlockComponentsAbsoluteUrl, BlockPackId>({
|
|
233
234
|
max: 500,
|
|
234
|
-
fetchMethod: async (
|
|
235
|
+
fetchMethod: async (_key, _staleValue, { context: id }) =>
|
|
235
236
|
await retry(async () => {
|
|
236
237
|
const packageFolderReader = this.v2RootFolderReader.relativeReader(
|
|
237
|
-
packageContentPrefixInsideV2(id)
|
|
238
|
+
packageContentPrefixInsideV2(id),
|
|
238
239
|
);
|
|
239
240
|
const manifest = BlockPackManifest.parse(
|
|
240
|
-
JSON.parse(Buffer.from(await packageFolderReader.readFile(ManifestFileName)).toString())
|
|
241
|
+
JSON.parse(Buffer.from(await packageFolderReader.readFile(ManifestFileName)).toString()),
|
|
241
242
|
);
|
|
242
243
|
return BlockComponentsAbsoluteUrl(packageFolderReader.rootUrl.toString()).parse(
|
|
243
|
-
manifest.description.components
|
|
244
|
+
manifest.description.components,
|
|
244
245
|
);
|
|
245
|
-
}, Retry2TimesWithDelay)
|
|
246
|
+
}, Retry2TimesWithDelay),
|
|
246
247
|
});
|
|
247
248
|
|
|
248
249
|
public async getComponents(id: BlockPackId): Promise<BlockComponentsAbsoluteUrl> {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { BlockPackId, BlockPackIdNoVersion } from '@milaboratories/pl-model-middle-layer';
|
|
1
|
+
import type { BlockPackId, BlockPackIdNoVersion } from '@milaboratories/pl-model-middle-layer';
|
|
2
2
|
|
|
3
3
|
export const VersionUpdatesPrefix = '_updates_v2/per_package_version/';
|
|
4
4
|
|
|
@@ -6,8 +6,8 @@ export function packageUpdateSeedPath(bp: BlockPackId, seed: string): string {
|
|
|
6
6
|
return `${VersionUpdatesPrefix}${bp.organization}/${bp.name}/${bp.version}/${seed}`;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
-
export const PackageUpdatePattern
|
|
10
|
-
/(?<packageKeyWithoutVersion>(?<organization>[
|
|
9
|
+
export const PackageUpdatePattern
|
|
10
|
+
= /(?<packageKeyWithoutVersion>(?<organization>[^/]+)\/(?<name>[^/]+))\/(?<version>[^/]+)\/(?<seed>[^/]+)$/;
|
|
11
11
|
|
|
12
12
|
export const GlobalUpdateSeedInFile = '_updates_v2/_global_update_in';
|
|
13
13
|
export const GlobalUpdateSeedOutFile = '_updates_v2/_global_update_out';
|