@platforma-sdk/block-tools 2.4.2 → 2.4.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/block-tools",
-  "version": "2.4.2",
+  "version": "2.4.3",
   "description": "Utility to manipulate Platforma Blocks and Block Registry",
   "types": "./dist/lib.d.ts",
   "main": "./dist/index.js",
@@ -33,9 +33,9 @@
     "tar": "^7.4.3",
     "yaml": "^2.6.1",
     "zod": "^3.23.8",
+    "@milaboratories/ts-helpers-oclif": "^1.1.11",
     "@milaboratories/pl-model-middle-layer": "^1.6.1",
     "@milaboratories/resolve-helper": "^1.0.2",
-    "@milaboratories/ts-helpers-oclif": "^1.1.11",
     "@milaboratories/ts-helpers": "^1.1.2"
   },
   "devDependencies": {
@@ -69,8 +69,8 @@
   "scripts": {
     "type-check": "tsc --noEmit --composite false",
     "oclif:index": "oclif-index --commands-root=./src/cmd/ --index-file=./src/cmd/index.ts",
-    "build": "pnpm run oclif:index && vite build",
-    "readme": "exec oclif readme",
+    "build": "vite build",
+    "readme": "oclif readme",
     "test": "jest"
   }
 }

package/src/cmd/index.ts

@@ -5,7 +5,8 @@ import Cmd1 from './build-model';
 import Cmd3 from './mark-stable';
 import Cmd4 from './pack';
 import Cmd5 from './publish';
-import Cmd6 from './upload-package-v1';
+import Cmd6 from './refresh-registry';
+import Cmd7 from './upload-package-v1';
 
 // prettier-ignore
 export const COMMANDS = {
@@ -14,5 +15,6 @@ export const COMMANDS = {
   'mark-stable': Cmd3,
   'pack': Cmd4,
   'publish': Cmd5,
-  'upload-package-v1': Cmd6
+  'refresh-registry': Cmd6,
+  'upload-package-v1': Cmd7
 };

package/src/cmd/refresh-registry.ts

@@ -0,0 +1,40 @@
+import { Command, Flags } from '@oclif/core';
+import { BlockRegistryV2, loadPackDescriptionRaw } from '../v2';
+import path from 'path';
+import {
+  overrideDescriptionVersion,
+  overrideManifestVersion,
+  StableChannel
+} from '@milaboratories/pl-model-middle-layer';
+import { storageByUrl } from '../io';
+import { OclifLoggerAdapter } from '@milaboratories/ts-helpers-oclif';
+
+export default class RefreshRegistry extends Command {
+  static description = 'Refresh overview files based on published but not proecessed artefacts';
+
+  static flags = {
+    registry: Flags.string({
+      char: 'r',
+      summary: 'full address of the registry',
+      helpValue: '<address>',
+      env: 'PL_REGISTRY',
+      required: true
+    }),
+
+    mode: Flags.string({
+      char: 'm',
+      summary: 'refresh mode (allowed valiues: "force", "normal", "dry-run")',
+      helpValue: '<mode>',
+      options: ['force', 'normal', 'dry-run'],
+      env: 'PL_REGISTRY_REFRESH_DRY_RUN',
+      default: 'normal'
+    })
+  };
+
+  public async run(): Promise<void> {
+    const { flags } = await this.parse(RefreshRegistry);
+    const storage = storageByUrl(flags.registry);
+    const registry = new BlockRegistryV2(storage, new OclifLoggerAdapter(this));
+    await registry.updateIfNeeded(flags.mode as 'force' | 'normal' | 'dry-run');
+  }
+}

package/src/io/storage.test.ts

@@ -65,6 +65,10 @@ if (testS3Address) {
     const testS3AddressURL = new URL(testS3Address);
     testS3AddressURL.pathname = `${testS3AddressURL.pathname.replace(/\/$/, '')}/${uuid}`;
     const storage = storageByUrl(testS3AddressURL.toString()) as S3Storage;
+
+    const nonExistingFolderListing = await storage.listFiles('not_existing_folder');
+    expect(nonExistingFolderListing).toEqual([]);
+
     await storage.putFile('some/deep1/file_1.txt', Buffer.from('test1'));
     await storage.putFile('some/deep1/file_2.txt', Buffer.from('test1'));
     await storage.putFile('some/deep2/file_1.txt', Buffer.from('test1'));

package/src/io/storage.ts

@@ -47,7 +47,7 @@ export class S3Storage implements RegistryStorage {
     );
     const result: string[] = [];
     for await (const page of paginator)
-      result.push(...page.Contents!.map((e) => pathPosix.relative(listRoot, e.Key!)));
+      result.push(...(page.Contents ?? []).map((e) => pathPosix.relative(listRoot, e.Key!)));
     return result;
   }
 

package/src/v2/registry/registry.ts

@@ -45,14 +45,14 @@ export class BlockRegistryV2 {
     private readonly logger: MiLogger = new ConsoleLoggerAdapter()
   ) {}
 
-  private async updateRegistry() {
-    this.logger?.info('Initiating registry refresh...');
+  private async updateRegistry(dryRun: boolean = false) {
+    this.logger.info('Initiating registry refresh...');
 
     // reading update requests
     const packagesToUpdate = new Map<string, PackageUpdateInfo>();
     const seedPaths: string[] = [];
     const rawSeedPaths = await this.storage.listFiles(VersionUpdatesPrefix);
-    this.logger?.info('Packages to be updated:');
+    this.logger.info('Packages to be updated:');
     for (const seedPath of rawSeedPaths) {
       const match = seedPath.match(PackageUpdatePattern);
       if (!match) continue;
@@ -71,7 +71,7 @@ export class BlockRegistryV2 {
         update.versions.add(version);
         added = true;
       }
-      this.logger?.info(`  - ${organization}:${name}:${version} added:${added}`);
+      this.logger.info(`  - ${organization}:${name}:${version} added:${added}`);
     }
 
     // loading global overview
@@ -81,7 +81,7 @@ export class BlockRegistryV2 {
         ? { schema: 'v2', packages: [] }
         : GlobalOverviewReg.parse(JSON.parse(overviewContent.toString()));
     let overviewPackages = overview.packages;
-    this.logger?.info(`Global overview loaded, ${overviewPackages.length} records`);
+    this.logger.info(`Global overview loaded, ${overviewPackages.length} records`);
 
     // updating packages
     for (const [, packageInfo] of packagesToUpdate.entries()) {
@@ -92,7 +92,7 @@ export class BlockRegistryV2 {
         pOverviewContent === undefined
           ? { schema: 'v2', versions: [] }
           : PackageOverview.parse(JSON.parse(pOverviewContent.toString()));
-      this.logger?.info(
+      this.logger.info(
         `Updating ${packageInfo.package.organization}:${packageInfo.package.name} overview, ${packageOverview.versions.length} records`
       );
 
@@ -137,13 +137,14 @@ export class BlockRegistryV2 {
       );
 
       // write package overview back
-      await this.storage.putFile(
-        overviewFile,
-        Buffer.from(
-          JSON.stringify({ schema: 'v2', versions: newVersions } satisfies PackageOverview)
-        )
-      );
-      this.logger?.info(`Done (${newVersions.length} records)`);
+      if (!dryRun)
+        await this.storage.putFile(
+          overviewFile,
+          Buffer.from(
+            JSON.stringify({ schema: 'v2', versions: newVersions } satisfies PackageOverview)
+          )
+        );
+      this.logger.info(`Done (${newVersions.length} records)`);
 
       // calculating all channels
       const allChannels = new Set<string>();
@@ -183,39 +184,47 @@ export class BlockRegistryV2 {
     }
 
     // writing global overview
-    await this.storage.putFile(
-      GlobalOverviewPath,
-      Buffer.from(
-        JSON.stringify({ schema: 'v2', packages: overviewPackages } satisfies GlobalOverviewReg)
-      )
-    );
-    this.logger?.info(`Global overview updated (${overviewPackages.length} records)`);
+    if (!dryRun)
+      await this.storage.putFile(
+        GlobalOverviewPath,
+        Buffer.from(
+          JSON.stringify({ schema: 'v2', packages: overviewPackages } satisfies GlobalOverviewReg)
+        )
+      );
+    this.logger.info(`Global overview updated (${overviewPackages.length} records)`);
 
     // deleting seeds
-    await this.storage.deleteFiles(...seedPaths.map((sp) => `${VersionUpdatesPrefix}${sp}`));
-    this.logger?.info(`Version update requests cleared`);
+    if (!dryRun)
+      await this.storage.deleteFiles(...seedPaths.map((sp) => `${VersionUpdatesPrefix}${sp}`));
+    this.logger.info(`Version update requests cleared`);
   }
 
-  public async updateIfNeeded(force: boolean = false): Promise<void> {
+  public async updateIfNeeded(mode: 'force' | 'normal' | 'dry-run' = 'normal'): Promise<void> {
     // implementation of main convergence algorithm
 
-    this.logger?.info(`Checking if registry requires refresh...`);
+    this.logger.info(`Checking if registry requires refresh...`);
     const updateRequestSeed = await this.storage.getFile(GlobalUpdateSeedInFile);
     const currentUpdatedSeed = await this.storage.getFile(GlobalUpdateSeedOutFile);
-    if (!force && updateRequestSeed === undefined && currentUpdatedSeed === undefined) return;
+    if (mode !== 'force' && updateRequestSeed === undefined && currentUpdatedSeed === undefined) {
+      this.logger.info(`No global seed files found, update not needed.`);
+      return;
+    }
     if (
-      !force &&
+      mode !== 'force' &&
       updateRequestSeed !== undefined &&
       currentUpdatedSeed !== undefined &&
       updateRequestSeed.equals(currentUpdatedSeed)
-    )
+    ) {
+      this.logger.info(`Registry is up to date.`);
       return;
+    }
 
-    await this.updateRegistry();
+    await this.updateRegistry(mode === 'dry-run');
 
     if (updateRequestSeed) {
-      await this.storage.putFile(GlobalUpdateSeedOutFile, updateRequestSeed);
-      this.logger?.info(`Refresh finished`);
+      if (mode !== 'dry-run')
+        await this.storage.putFile(GlobalUpdateSeedOutFile, updateRequestSeed);
+      this.logger.info(`Refresh finished.`);
     }
   }
 
@@ -237,9 +246,9 @@ export class BlockRegistryV2 {
     // adding update seed
     const seed = randomUUID();
     const seedPath = packageUpdateSeedPath(id, seed);
-    this.logger?.info(`Creating update seed at ${seedPath} ...`);
+    this.logger.info(`Creating update seed at ${seedPath} ...`);
     await this.storage.putFile(seedPath, Buffer.from(seed));
-    this.logger?.info(`Touching global update seed ${GlobalUpdateSeedInFile} ...`);
+    this.logger.info(`Touching global update seed ${GlobalUpdateSeedInFile} ...`);
     await this.storage.putFile(GlobalUpdateSeedInFile, Buffer.from(seed));
   }
 
@@ -288,13 +297,13 @@ export class BlockRegistryV2 {
         );
 
       const dst = prefix + '/' + f.name;
-      this.logger?.info(`Uploading ${f.name} -> ${dst} ...`);
+      this.logger.info(`Uploading ${f.name} -> ${dst} ...`);
       await this.storage.putFile(dst, bytes);
     }
 
     // uploading manifest as the last upload action
     const manifestDst = prefix + '/' + ManifestFileName;
-    this.logger?.info(`Uploading manifest to ${manifestDst} ...`);
+    this.logger.info(`Uploading manifest to ${manifestDst} ...`);
     await this.storage.putFile(manifestDst, Buffer.from(JSON.stringify(manifest)));
 
     await this.marchChanged(manifest.description.id);