@platforma-sdk/block-tools 2.5.92 → 2.6.0

package/dist/v2/registry/registry.d.ts

@@ -3,11 +3,24 @@ import { RegistryStorage } from '../../io/storage';
 import { BlockPackId, BlockPackIdNoVersion, BlockPackManifest } from '@milaboratories/pl-model-middle-layer';
 import { GlobalOverviewReg, PackageOverview } from './schema_public';
 import { RelativeContentReader } from '../model';
+export interface BlockRegistrySettings {
+    skipSnapshotCreation?: boolean;
+}
+export interface GlobalOverviewBackupDescription {
+    timestamp: string;
+    path: string;
+}
 export declare class BlockRegistryV2 {
     private readonly storage;
     private readonly logger;
+    private readonly settings;
     private readonly gzipAsync;
-    constructor(storage: RegistryStorage, logger?: MiLogger);
+    private readonly gunzipAsync;
+    constructor(storage: RegistryStorage, logger?: MiLogger, settings?: BlockRegistrySettings);
+    private generateTimestamp;
+    private generatePreWriteTimestamp;
+    private createGlobalOverviewSnapshot;
+    private createPackageOverviewSnapshot;
     private updateRegistry;
     updateIfNeeded(mode?: 'force' | 'normal' | 'dry-run'): Promise<void>;
     getPackageOverview(name: BlockPackIdNoVersion): Promise<undefined | PackageOverview>;
@@ -15,5 +28,7 @@ export declare class BlockRegistryV2 {
     private marchChanged;
     addPackageToChannel(id: BlockPackId, channel: string): Promise<void>;
     removePackageFromChannel(id: BlockPackId, channel: string): Promise<void>;
+    listGlobalOverviewSnapshots(): Promise<GlobalOverviewBackupDescription[]>;
+    restoreGlobalOverviewFromSnapshot(backupId: string): Promise<void>;
     publishPackage(manifest: BlockPackManifest, fileReader: RelativeContentReader): Promise<void>;
 }

package/dist/v2/registry/registry_reader.d.ts

@@ -1,7 +1,6 @@
-import { BlockPackId, BlockPackIdNoVersion, BlockPackOverview, UpdateSuggestions, SingleBlockPackOverview } from '@milaboratories/pl-model-middle-layer';
+import { BlockPackId, BlockPackIdNoVersion, UpdateSuggestions, SingleBlockPackOverview, BlockPackOverviewNoRegistryId } from '@milaboratories/pl-model-middle-layer';
 import { FolderReader } from '../../io';
 import { BlockComponentsAbsoluteUrl } from '../model';
-export type BlockPackOverviewNoRegLabel = Omit<BlockPackOverview, 'registryId'>;
 export type RegistryV2ReaderOps = {
     /** Number of milliseconds to cache retrieved block list for */
     cacheBlockListFor: number;
@@ -24,7 +23,7 @@ export declare class RegistryV2Reader {
     private embedMetaContent;
     private listCacheTimestamp;
     private listCache;
-    listBlockPacks(): Promise<BlockPackOverviewNoRegLabel[]>;
+    listBlockPacks(): Promise<BlockPackOverviewNoRegistryId[]>;
     getLatestOverview(id: BlockPackIdNoVersion, channel: string): Promise<SingleBlockPackOverview | undefined>;
     getUpdateSuggestions(id: BlockPackId, channel: string): Promise<UpdateSuggestions<string> | undefined>;
     getSpecificOverview(id: BlockPackId, channel: string): Promise<SingleBlockPackOverview>;

package/dist/v2/registry/schema_internal.d.ts

@@ -1,6 +1,12 @@
-import { BlockPackId } from '@milaboratories/pl-model-middle-layer';
+import { BlockPackId, BlockPackIdNoVersion } from '@milaboratories/pl-model-middle-layer';
 export declare const VersionUpdatesPrefix = "_updates_v2/per_package_version/";
 export declare function packageUpdateSeedPath(bp: BlockPackId, seed: string): string;
 export declare const PackageUpdatePattern: RegExp;
 export declare const GlobalUpdateSeedInFile = "_updates_v2/_global_update_in";
 export declare const GlobalUpdateSeedOutFile = "_updates_v2/_global_update_out";
+export declare const OverviewSnapshotsPrefix = "_overview_snapshots_v2/";
+export declare const GlobalSnapshotsPrefix = "_overview_snapshots_v2/global/";
+export declare const PackageSnapshotsPrefix = "_overview_snapshots_v2/per_package/";
+export declare const GlobalOverviewSnapshotPattern: RegExp;
+export declare function globalOverviewSnapshotPath(timestamp: string): string;
+export declare function packageOverviewSnapshotPath(bp: BlockPackIdNoVersion, timestamp: string): string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/block-tools",
-  "version": "2.5.92",
+  "version": "2.6.0",
   "description": "Utility to manipulate Platforma Blocks and Block Registry",
   "types": "./dist/lib.d.ts",
   "main": "./dist/index.js",
@@ -33,12 +33,12 @@
     "tar": "^7.4.3",
     "yaml": "^2.8.0",
     "zod": "~3.23.8",
-    "@milaboratories/pl-model-common": "1.19.14",
-    "@milaboratories/pl-model-middle-layer": "1.8.21",
-    "@milaboratories/resolve-helper": "1.1.1",
     "@milaboratories/pl-http": "1.1.7",
+    "@milaboratories/pl-model-common": "1.19.15",
+    "@milaboratories/resolve-helper": "1.1.1",
     "@milaboratories/ts-helpers-oclif": "1.1.27",
-    "@milaboratories/ts-helpers": "1.4.6"
+    "@milaboratories/ts-helpers": "1.4.6",
+    "@milaboratories/pl-model-middle-layer": "1.8.22"
   },
   "devDependencies": {
     "typescript": "~5.6.3",
@@ -55,9 +55,9 @@
     "@jest/globals": "^29.7.0",
     "oclif": "^4.16.2",
     "ts-jest": "^29.2.6",
-    "@milaboratories/build-configs": "1.0.8",
+    "@milaboratories/ts-configs": "1.0.6",
     "@milaboratories/oclif-index": "1.1.1",
-    "@milaboratories/ts-configs": "1.0.6"
+    "@milaboratories/build-configs": "1.0.8"
   },
   "oclif": {
     "bin": "block-tools",

package/src/cmd/index.ts

@@ -2,19 +2,23 @@
 
 import Cmd0 from './build-meta';
 import Cmd1 from './build-model';
-import Cmd3 from './mark-stable';
-import Cmd4 from './pack';
-import Cmd5 from './publish';
-import Cmd6 from './refresh-registry';
-import Cmd7 from './upload-package-v1';
+import Cmd3 from './list-overview-snapshots';
+import Cmd4 from './mark-stable';
+import Cmd5 from './pack';
+import Cmd6 from './publish';
+import Cmd7 from './refresh-registry';
+import Cmd8 from './restore-overview-from-snapshot';
+import Cmd9 from './upload-package-v1';
 
 // prettier-ignore
 export const COMMANDS = {
   'build-meta': Cmd0,
   'build-model': Cmd1,
-  'mark-stable': Cmd3,
-  'pack': Cmd4,
-  'publish': Cmd5,
-  'refresh-registry': Cmd6,
-  'upload-package-v1': Cmd7
+  'list-overview-snapshots': Cmd3,
+  'mark-stable': Cmd4,
+  'pack': Cmd5,
+  'publish': Cmd6,
+  'refresh-registry': Cmd7,
+  'restore-overview-from-snapshot': Cmd8,
+  'upload-package-v1': Cmd9,
 };

package/src/cmd/list-overview-snapshots.ts

@@ -0,0 +1,46 @@
+import { Command, Flags } from '@oclif/core';
+import { BlockRegistryV2 } from '../v2/registry/registry';
+import { storageByUrl } from '../io/storage';
+import { OclifLoggerAdapter } from '@milaboratories/ts-helpers-oclif';
+
+export default class ListOverviewSnapshots extends Command {
+  static description = 'List all available global overview snapshots in the registry';
+
+  static flags = {
+    registry: Flags.string({
+      char: 'r',
+      summary: 'full address of the registry',
+      helpValue: '<address>',
+      env: 'PL_REGISTRY',
+      required: true
+    }),
+
+    json: Flags.boolean({
+      summary: 'output in JSON format',
+      default: false
+    })
+  };
+
+  public async run(): Promise<void> {
+    const { flags } = await this.parse(ListOverviewSnapshots);
+    const storage = storageByUrl(flags.registry);
+    const registry = new BlockRegistryV2(storage, new OclifLoggerAdapter(this));
+    
+    const snapshots = await registry.listGlobalOverviewSnapshots();
+    
+    if (flags.json) {
+      this.log(JSON.stringify(snapshots, null, 2));
+    } else {
+      if (snapshots.length === 0) {
+        this.log('No snapshots found.');
+      } else {
+        this.log(`Found ${snapshots.length} snapshot(s):\n`);
+        for (const snapshot of snapshots) {
+          this.log(`  ${snapshot.timestamp}`);
+          this.log(`    Path: ${snapshot.path}`);
+          this.log('');
+        }
+      }
+    }
+  }
+}

package/src/cmd/restore-overview-from-snapshot.ts

@@ -0,0 +1,78 @@
+import { Command, Flags } from '@oclif/core';
+import { BlockRegistryV2 } from '../v2/registry/registry';
+import { storageByUrl } from '../io/storage';
+import { OclifLoggerAdapter } from '@milaboratories/ts-helpers-oclif';
+
+export default class RestoreOverviewFromSnapshot extends Command {
+  static description = 'Restore global overview from a snapshot';
+
+  static flags = {
+    registry: Flags.string({
+      char: 'r',
+      summary: 'full address of the registry',
+      helpValue: '<address>',
+      env: 'PL_REGISTRY',
+      required: true
+    }),
+
+    snapshot: Flags.string({
+      char: 's',
+      summary: 'snapshot timestamp ID to restore from',
+      helpValue: '<timestamp>',
+      required: true
+    }),
+
+    'skip-confirmation': Flags.boolean({
+      summary: 'skip confirmation prompt (use with caution)',
+      default: false
+    })
+  };
+
+  public async run(): Promise<void> {
+    const { flags } = await this.parse(RestoreOverviewFromSnapshot);
+    const storage = storageByUrl(flags.registry);
+    const registry = new BlockRegistryV2(storage, new OclifLoggerAdapter(this));
+    
+    // Check if snapshot exists
+    const snapshots = await registry.listGlobalOverviewSnapshots();
+    const targetSnapshot = snapshots.find(s => s.timestamp === flags.snapshot);
+    
+    if (!targetSnapshot) {
+      this.error(`Snapshot '${flags.snapshot}' not found. Available snapshots:\n${
+        snapshots.map(s => `  - ${s.timestamp}`).join('\n') || '  (none)'
+      }`);
+    }
+
+    // Confirmation prompt (unless skipped)
+    if (!flags['skip-confirmation']) {
+      const readline = await import('node:readline');
+      const rl = readline.createInterface({
+        input: process.stdin,
+        output: process.stdout
+      });
+
+      const answer = await new Promise<string>((resolve) => {
+        rl.question(
+          `⚠️  This will overwrite the current global overview with snapshot '${flags.snapshot}'.\n` +
+          `Are you sure you want to continue? (y/N): `,
+          resolve
+        );
+      });
+      
+      rl.close();
+
+      if (answer.toLowerCase() !== 'y' && answer.toLowerCase() !== 'yes') {
+        this.log('Restore cancelled.');
+        return;
+      }
+    }
+
+    // Perform restore
+    try {
+      await registry.restoreGlobalOverviewFromSnapshot(flags.snapshot);
+      this.log(`✅ Successfully restored global overview from snapshot '${flags.snapshot}'`);
+    } catch (error) {
+      this.error(`Failed to restore from snapshot: ${error}`);
+    }
+  }
+}

package/src/v2/registry/registry.test.ts

@@ -5,8 +5,9 @@ import path from 'path';
 import fsp from 'fs/promises';
 import { BlockRegistryV2 } from './registry';
 import semver from 'semver';
-import { UpdateSuggestions } from '@milaboratories/pl-model-middle-layer';
+import { UpdateSuggestions, BlockPackManifest } from '@milaboratories/pl-model-middle-layer';
 import { inferUpdateSuggestions } from './registry_reader';
+import { OverviewSnapshotsPrefix } from './schema_internal';
 
 type TestStorageInstance = {
   storage: RegistryStorage;
@@ -55,11 +56,182 @@ if (testS3Address !== undefined) {
   });
 }
 
-test.each(testStorages)('full registry test with $name', async ({ storageProvider }) => {
+test.each(testStorages)('registry snapshots test with $name', async ({ storageProvider }) => {
   const { storage, teardown } = storageProvider();
   const registry = new BlockRegistryV2(storage);
-  registry.updateIfNeeded;
-  await teardown();
+  
+  try {
+    // Force an update to trigger snapshot creation (even with empty registry)
+    await registry.updateIfNeeded('force');
+    
+    // Check that snapshot files actually exist in storage
+    const snapshotFiles = await storage.listFiles(OverviewSnapshotsPrefix);
+    expect(snapshotFiles.length).toBeGreaterThan(0);
+    
+    // Check that snapshots were created
+    const snapshots = await registry.listGlobalOverviewSnapshots();
+    expect(snapshots.length).toBeGreaterThan(0);
+    expect(snapshots[0]).toHaveProperty('timestamp');
+    expect(snapshots[0]).toHaveProperty('path');
+    expect(snapshots[0].path).toMatch(/^_overview_snapshots_v2\/global\/.*\.json\.gz$/);
+    
+    // Test that global overview files exist (should be empty initially)
+    const globalOverview = await registry.getGlobalOverview();
+    expect(globalOverview).toBeDefined();
+    expect(globalOverview?.packages).toHaveLength(0);
+    
+    // Test restore functionality
+    const snapshotId = snapshots[0].timestamp;
+    await registry.restoreGlobalOverviewFromSnapshot(snapshotId);
+    
+    // Verify restored overview is still valid
+    const restoredOverview = await registry.getGlobalOverview();
+    expect(restoredOverview).toBeDefined();
+    expect(restoredOverview?.packages).toHaveLength(0);
+    
+  } finally {
+    await teardown();
+  }
+});
+
+test.each(testStorages)('registry snapshots disabled test with $name', async ({ storageProvider }) => {
+  const { storage, teardown } = storageProvider();
+  const registry = new BlockRegistryV2(storage, undefined, { skipSnapshotCreation: true });
+  
+  try {
+    // Force an update which would normally create snapshots
+    await registry.updateIfNeeded('force');
+    
+    // Check that no snapshots were created
+    const snapshots = await registry.listGlobalOverviewSnapshots();
+    expect(snapshots).toHaveLength(0);
+    
+    const snapshotFiles = await storage.listFiles(OverviewSnapshotsPrefix);
+    expect(snapshotFiles).toHaveLength(0);
+    
+  } finally {
+    await teardown();
+  }
+});
+
+test.each(testStorages)('force mode removes deleted packages and versions with $name', async ({ storageProvider }) => {
+  const { storage, teardown } = storageProvider();
+  const registry = new BlockRegistryV2(storage);
+  
+  try {
+    // Create mock manifests for testing
+    const createMockManifest = (org: string, name: string, version: string): BlockPackManifest => ({
+      schema: 'v2',
+      description: {
+        id: { organization: org, name: name, version: version },
+        title: `Test ${name}`,
+        summary: 'Test package',
+        components: {
+          workflow: { type: 'workflow-v1', main: { type: 'relative', path: 'workflow.json' } },
+          model: { type: 'relative', path: 'model.json' },
+          ui: { type: 'relative', path: 'ui.json' }
+        },
+        meta: {
+          title: `Test ${name}`,
+          description: 'Test package description',
+          organization: {
+            name: 'Test Organization',
+            url: 'https://test.com'
+          },
+          tags: []
+        }
+      },
+      files: [
+        {
+          name: 'model.json',
+          size: 13,
+          sha256: '6FD977DB9B2AFE87A9CEEE48432881299A6AAF83D935FBBE83007660287F9C2E'
+        }
+      ]
+    });
+    
+    const mockFileReader = async (fileName: string) => {
+      if (fileName === 'model.json') {
+        return Buffer.from('{"test":true}');
+      }
+      throw new Error(`Unknown file: ${fileName}`);
+    };
+    
+    // 1. Publish multiple packages and versions
+    const pkg1v1 = createMockManifest('testorg', 'pkg1', '1.0.0');
+    const pkg1v2 = createMockManifest('testorg', 'pkg1', '2.0.0'); 
+    const pkg2v1 = createMockManifest('testorg', 'pkg2', '1.0.0');
+    const pkg3v1 = createMockManifest('anotherorg', 'pkg3', '1.0.0');
+    
+    await registry.publishPackage(pkg1v1, mockFileReader);
+    await registry.publishPackage(pkg1v2, mockFileReader);
+    await registry.publishPackage(pkg2v1, mockFileReader);
+    await registry.publishPackage(pkg3v1, mockFileReader);
+    
+    // Update registry to create overviews
+    await registry.updateIfNeeded('normal');
+    
+    // Verify initial state
+    let globalOverview = await registry.getGlobalOverview();
+    expect(globalOverview?.packages).toHaveLength(3); // testorg:pkg1, testorg:pkg2, anotherorg:pkg3
+    
+    let pkg1Overview = await registry.getPackageOverview({ organization: 'testorg', name: 'pkg1' });
+    expect(pkg1Overview?.versions).toHaveLength(2); // v1.0.0 and v2.0.0
+    
+    let pkg2Overview = await registry.getPackageOverview({ organization: 'testorg', name: 'pkg2' });
+    expect(pkg2Overview?.versions).toHaveLength(1); // v1.0.0
+    
+    let pkg3Overview = await registry.getPackageOverview({ organization: 'anotherorg', name: 'pkg3' });
+    expect(pkg3Overview?.versions).toHaveLength(1); // v1.0.0
+    
+    // 2. Manually delete some packages/versions from storage (simulating external deletion)
+    // Delete pkg1 v1.0.0 
+    await storage.deleteFiles('v2/testorg/pkg1/1.0.0/manifest.json', 'v2/testorg/pkg1/1.0.0/model.json');
+    
+    // Delete entire pkg2
+    await storage.deleteFiles('v2/testorg/pkg2/1.0.0/manifest.json', 'v2/testorg/pkg2/1.0.0/model.json');
+    
+    // Leave pkg1 v2.0.0 and pkg3 v1.0.0 intact
+    
+    // 3. Count snapshots before force mode
+    const initialSnapshots = await storage.listFiles('_overview_snapshots_v2/');
+    
+    // 4. Run force mode - should create pre-write snapshots and rebuild from scratch
+    await registry.updateIfNeeded('force');
+    
+    // 5. Verify pre-write snapshots were created
+    const finalSnapshots = await storage.listFiles('_overview_snapshots_v2/');
+    expect(finalSnapshots.length).toBeGreaterThan(initialSnapshots.length);
+    
+    // Check for pre-write snapshots (should contain "-prewrite-" in filename)
+    const preWriteSnapshots = finalSnapshots.filter(s => s.includes('-prewrite-'));
+    expect(preWriteSnapshots.length).toBeGreaterThan(0);
+    
+    // 6. Verify overviews now only reflect what exists in storage
+    globalOverview = await registry.getGlobalOverview();
+    expect(globalOverview?.packages).toHaveLength(2); // Only testorg:pkg1 and anotherorg:pkg3 should remain
+    
+    const remainingPackageNames = globalOverview?.packages.map(p => `${p.id.organization}:${p.id.name}`).sort();
+    expect(remainingPackageNames).toEqual(['anotherorg:pkg3', 'testorg:pkg1']);
+    
+    // 7. Verify pkg1 now only has v2.0.0
+    pkg1Overview = await registry.getPackageOverview({ organization: 'testorg', name: 'pkg1' });
+    expect(pkg1Overview?.versions).toHaveLength(1);
+    expect(pkg1Overview?.versions[0].description.id.version).toBe('2.0.0');
+    
+    // 8. Verify pkg2 overview is unchanged (since pkg2 was completely deleted, 
+    // force mode doesn't process it, so the old overview file remains)
+    pkg2Overview = await registry.getPackageOverview({ organization: 'testorg', name: 'pkg2' });
+    expect(pkg2Overview?.versions).toHaveLength(1); // Old overview remains
+    
+    // 9. Verify pkg3 is unchanged
+    pkg3Overview = await registry.getPackageOverview({ organization: 'anotherorg', name: 'pkg3' });
+    expect(pkg3Overview?.versions).toHaveLength(1);
+    expect(pkg3Overview?.versions[0].description.id.version).toBe('1.0.0');
+    
+  } finally {
+    await teardown();
+  }
 });
 
 test.each([