@platforma-sdk/block-tools 2.5.91 → 2.6.0

package/src/v2/registry/registry.ts

@@ -1,6 +1,6 @@
 import { ConsoleLoggerAdapter, MiLogger } from '@milaboratories/ts-helpers';
 import { compare as compareSemver, satisfies } from 'semver';
-import { gzip } from 'node:zlib';
+import { gzip, gunzip } from 'node:zlib';
 import { promisify } from 'node:util';
 import { RegistryStorage } from '../../io/storage';
 import {
@@ -15,7 +15,13 @@ import {
   GlobalUpdateSeedOutFile,
   PackageUpdatePattern,
   packageUpdateSeedPath,
-  VersionUpdatesPrefix
+  VersionUpdatesPrefix,
+  GlobalOverviewSnapshotPattern,
+  OverviewSnapshotsPrefix,
+  GlobalSnapshotsPrefix,
+  PackageSnapshotsPrefix,
+  globalOverviewSnapshotPath,
+  packageOverviewSnapshotPath
 } from './schema_internal';
 import {
   GlobalOverviewReg,
@@ -36,6 +42,15 @@ import { BlockPackDescriptionManifestAddRelativePathPrefix, RelativeContentReade
 import { randomUUID } from 'node:crypto';
 import { calculateSha256 } from '../../util';
 
+export interface BlockRegistrySettings {
+  skipSnapshotCreation?: boolean;
+}
+
+export interface GlobalOverviewBackupDescription {
+  timestamp: string;
+  path: string;
+}
+
 type PackageUpdateInfo = {
   package: BlockPackIdNoVersion;
   versions: Set<String>;
@@ -43,15 +58,59 @@ type PackageUpdateInfo = {
 
 export class BlockRegistryV2 {
   private readonly gzipAsync = promisify(gzip);
+  private readonly gunzipAsync = promisify(gunzip);
 
   constructor(
     private readonly storage: RegistryStorage,
-    private readonly logger: MiLogger = new ConsoleLoggerAdapter()
+    private readonly logger: MiLogger = new ConsoleLoggerAdapter(),
+    private readonly settings: BlockRegistrySettings = {}
   ) {}
 
+  private generateTimestamp(): string {
+    const timestamp = new Date().toISOString().replace(/:/g, '-').replace(/\.(\d{3})Z$/, '.$1Z');
+    const randomSuffix = Math.random().toString(36).substring(2, 6);
+    return `${timestamp}-${randomSuffix}`;
+  }
+
+  private generatePreWriteTimestamp(): string {
+    const timestamp = new Date().toISOString().replace(/:/g, '-').replace(/\.(\d{3})Z$/, '.$1Z');
+    const randomSuffix = Math.random().toString(36).substring(2, 6);
+    return `${timestamp}-prewrite-${randomSuffix}`;
+  }
+
+  private async createGlobalOverviewSnapshot(overviewData: string, timestamp: string): Promise<void> {
+    if (this.settings.skipSnapshotCreation) return;
+    
+    try {
+      const gzippedData = await this.gzipAsync(overviewData);
+      const snapshotPath = globalOverviewSnapshotPath(timestamp);
+      await this.storage.putFile(snapshotPath, Buffer.from(gzippedData));
+      this.logger.info(`Global overview snapshot created at ${snapshotPath}`);
+    } catch (error) {
+      this.logger.warn(`Failed to create global overview snapshot: ${error}`);
+    }
+  }
+
+  private async createPackageOverviewSnapshot(pkg: BlockPackIdNoVersion, overview: PackageOverview, timestamp: string): Promise<void> {
+    if (this.settings.skipSnapshotCreation) return;
+    
+    try {
+      const overviewData = JSON.stringify(overview);
+      const gzippedData = await this.gzipAsync(overviewData);
+      const snapshotPath = packageOverviewSnapshotPath(pkg, timestamp);
+      await this.storage.putFile(snapshotPath, Buffer.from(gzippedData));
+      this.logger.info(`Package overview snapshot created at ${snapshotPath} for ${pkg.organization}:${pkg.name}`);
+    } catch (error) {
+      this.logger.warn(`Failed to create package overview snapshot for ${pkg.organization}:${pkg.name}: ${error}`);
+    }
+  }
+
   private async updateRegistry(mode: 'force' | 'normal' | 'dry-run' = 'normal') {
     this.logger.info('Initiating registry refresh...');
 
+    // Generate timestamp for all snapshots in this run
+    const snapshotTimestamp = this.generateTimestamp();
+
     // reading update requests
     const packagesToUpdate = new Map<string, PackageUpdateInfo>();
     const seedPaths: string[] = [];
@@ -97,24 +156,41 @@ export class BlockRegistryV2 {
 
     // loading global overview
     const overviewContent = await this.storage.getFile(GlobalOverviewPath);
-    const overview: GlobalOverviewReg =
-      overviewContent === undefined
+    
+    // Create pre-write snapshot in force mode if overview exists
+    if (mode === 'force' && overviewContent !== undefined) {
+      const preWriteTimestamp = this.generatePreWriteTimestamp();
+      await this.createGlobalOverviewSnapshot(overviewContent.toString(), preWriteTimestamp);
+    }
+    
+    const overview: GlobalOverviewReg = mode === 'force'
+      ? { schema: 'v2', packages: [] }
+      : overviewContent === undefined
         ? { schema: 'v2', packages: [] }
         : GlobalOverviewReg.parse(JSON.parse(overviewContent.toString()));
     let overviewPackages = overview.packages;
-    this.logger.info(`Global overview loaded, ${overviewPackages.length} records`);
+    this.logger.info(`Global overview ${mode === 'force' ? 'starting empty (force mode)' : 'loaded'}, ${overviewPackages.length} records`);
 
     // updating packages
     for (const [, packageInfo] of packagesToUpdate.entries()) {
       // reading existing overview
       const overviewFile = packageOverviewPath(packageInfo.package);
       const pOverviewContent = await this.storage.getFile(overviewFile);
-      const packageOverview: PackageOverview =
-        pOverviewContent === undefined
+      
+      // Create pre-write snapshot in force mode if package overview exists
+      if (mode === 'force' && pOverviewContent !== undefined) {
+        const preWriteTimestamp = this.generatePreWriteTimestamp();
+        const existingOverview = PackageOverview.parse(JSON.parse(pOverviewContent.toString()));
+        await this.createPackageOverviewSnapshot(packageInfo.package, existingOverview, preWriteTimestamp);
+      }
+      
+      const packageOverview: PackageOverview = mode === 'force'
+        ? { schema: 'v2', versions: [] }
+        : pOverviewContent === undefined
           ? { schema: 'v2', versions: [] }
           : PackageOverview.parse(JSON.parse(pOverviewContent.toString()));
       this.logger.info(
-        `Updating ${packageInfo.package.organization}:${packageInfo.package.name} overview, ${packageOverview.versions.length} records`
+        `Updating ${packageInfo.package.organization}:${packageInfo.package.name} overview${mode === 'force' ? ' (starting empty in force mode)' : ''}, ${packageOverview.versions.length} records`
       );
 
       // removing versions that we will update
@@ -158,13 +234,16 @@ export class BlockRegistryV2 {
       );
 
       // write package overview back
-      if (mode !== 'dry-run')
+      const packageOverviewData = { schema: 'v2', versions: newVersions } satisfies PackageOverview;
+      if (mode !== 'dry-run') {
         await this.storage.putFile(
           overviewFile,
-          Buffer.from(
-            JSON.stringify({ schema: 'v2', versions: newVersions } satisfies PackageOverview)
-          )
+          Buffer.from(JSON.stringify(packageOverviewData))
         );
+        
+        // Create snapshot after successful write
+        await this.createPackageOverviewSnapshot(packageInfo.package, packageOverviewData, snapshotTimestamp);
+      }
       this.logger.info(`Done (${newVersions.length} records)`);
 
       // calculating all channels
@@ -222,6 +301,9 @@ export class BlockRegistryV2 {
       // Write gzipped overview file
       const gzippedBuffer = await this.gzipAsync(overviewData);
       await this.storage.putFile(GlobalOverviewGzPath, Buffer.from(gzippedBuffer));
+      
+      // Create snapshot after successful writes
+      await this.createGlobalOverviewSnapshot(overviewData, snapshotTimestamp);
     }
     this.logger.info(`Global overview updated (${overviewPackages.length} records)`);
 
@@ -310,6 +392,57 @@ export class BlockRegistryV2 {
     await this.marchChanged(id);
   }
 
+  public async listGlobalOverviewSnapshots(): Promise<GlobalOverviewBackupDescription[]> {
+    const snapshotPaths = await this.storage.listFiles(GlobalSnapshotsPrefix);
+    const snapshots: GlobalOverviewBackupDescription[] = [];
+    
+    for (const path of snapshotPaths) {
+      // Extract filename from path
+      const filename = path.indexOf('/') === -1 ? path : path.substring(path.lastIndexOf('/') + 1);
+      
+      const match = filename.match(GlobalOverviewSnapshotPattern);
+      if (match) {
+        snapshots.push({
+          timestamp: match.groups!.timestamp,
+          path: GlobalSnapshotsPrefix + filename
+        });
+      }
+    }
+    
+    // Sort by timestamp descending (newest first)
+    snapshots.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
+    return snapshots;
+  }
+
+  public async restoreGlobalOverviewFromSnapshot(backupId: string): Promise<void> {
+    const snapshotPath = globalOverviewSnapshotPath(backupId);
+    
+    // Read and decompress the snapshot
+    const snapshotData = await this.storage.getFile(snapshotPath);
+    if (!snapshotData) {
+      throw new Error(`Snapshot ${backupId} not found at ${snapshotPath}`);
+    }
+    
+    const decompressedData = await this.gunzipAsync(snapshotData);
+    const overviewData = decompressedData.toString('utf8');
+    
+    // Validate the data
+    try {
+      GlobalOverviewReg.parse(JSON.parse(overviewData));
+    } catch (error) {
+      throw new Error(`Invalid snapshot data in ${backupId}: ${error}`);
+    }
+    
+    // Write both regular and gzipped versions
+    const overviewBuffer = Buffer.from(overviewData);
+    await this.storage.putFile(GlobalOverviewPath, overviewBuffer);
+    
+    const gzippedBuffer = await this.gzipAsync(overviewData);
+    await this.storage.putFile(GlobalOverviewGzPath, Buffer.from(gzippedBuffer));
+    
+    this.logger.info(`Global overview restored from snapshot ${backupId}`);
+  }
+
   public async publishPackage(
     manifest: BlockPackManifest,
     fileReader: RelativeContentReader

package/src/v2/registry/registry_reader.ts

@@ -8,7 +8,8 @@ import {
   BlockPackOverview,
   UpdateSuggestions,
   SingleBlockPackOverview,
-  AnyChannel
+  AnyChannel,
+  BlockPackOverviewNoRegistryId
 } from '@milaboratories/pl-model-middle-layer';
 import { FolderReader } from '../../io';
 import canonicalize from 'canonicalize';
@@ -26,8 +27,6 @@ import semver from 'semver';
 import { calculateSha256 } from '../../util';
 import { retry, Retry2TimesWithDelay } from '@milaboratories/ts-helpers';
 
-export type BlockPackOverviewNoRegLabel = Omit<BlockPackOverview, 'registryId'>;
-
 export type RegistryV2ReaderOps = {
   /** Number of milliseconds to cache retrieved block list for */
   cacheBlockListFor: number;
@@ -110,9 +109,9 @@ export class RegistryV2Reader {
   }
 
   private listCacheTimestamp: number = 0;
-  private listCache: BlockPackOverviewNoRegLabel[] | undefined = undefined;
+  private listCache: BlockPackOverviewNoRegistryId[] | undefined = undefined;
 
-  public async listBlockPacks(): Promise<BlockPackOverviewNoRegLabel[]> {
+  public async listBlockPacks(): Promise<BlockPackOverviewNoRegistryId[]> {
     if (
       this.listCache !== undefined &&
       Date.now() - this.listCacheTimestamp <= this.ops.cacheBlockListFor
@@ -154,7 +153,7 @@ export class RegistryV2Reader {
               id: p.id,
               latestByChannel: Object.fromEntries(byChannelEntries),
               allVersions: p.allVersionsWithChannels
-            } satisfies BlockPackOverviewNoRegLabel;
+            } satisfies BlockPackOverviewNoRegistryId;
           })
         );
 

package/src/v2/registry/schema_internal.ts

@@ -1,4 +1,4 @@
-import { BlockPackId } from '@milaboratories/pl-model-middle-layer';
+import { BlockPackId, BlockPackIdNoVersion } from '@milaboratories/pl-model-middle-layer';
 
 export const VersionUpdatesPrefix = '_updates_v2/per_package_version/';
 
@@ -11,3 +11,18 @@ export const PackageUpdatePattern =
 
 export const GlobalUpdateSeedInFile = '_updates_v2/_global_update_in';
 export const GlobalUpdateSeedOutFile = '_updates_v2/_global_update_out';
+
+// Snapshot storage structure
+export const OverviewSnapshotsPrefix = '_overview_snapshots_v2/';
+export const GlobalSnapshotsPrefix = '_overview_snapshots_v2/global/';
+export const PackageSnapshotsPrefix = '_overview_snapshots_v2/per_package/';
+
+export const GlobalOverviewSnapshotPattern = /^(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.\d{3}Z-[a-z0-9]+)\.json\.gz$/;
+
+export function globalOverviewSnapshotPath(timestamp: string): string {
+  return `${GlobalSnapshotsPrefix}${timestamp}.json.gz`;
+}
+
+export function packageOverviewSnapshotPath(bp: BlockPackIdNoVersion, timestamp: string): string {
+  return `${PackageSnapshotsPrefix}${bp.organization}/${bp.name}/${timestamp}.json.gz`;
+}

package/src/v2/registry/schema_public.ts

@@ -44,13 +44,13 @@ export const PackageOverviewVersionEntry = z.object({
   description: BlockPackDescriptionManifest,
   channels: z.array(z.string()).default(() => []),
   manifestSha256: Sha256Schema
-});
+}).passthrough();
 export type PackageOverviewVersionEntry = z.infer<typeof PackageOverviewVersionEntry>;
 
 export const PackageOverview = z.object({
   schema: z.literal('v2'),
   versions: z.array(PackageOverviewVersionEntry)
-});
+}).passthrough();
 export type PackageOverview = z.infer<typeof PackageOverview>;
 
 export function packageOverviewPathInsideV2(bp: BlockPackIdNoVersion): string {
@@ -93,10 +93,10 @@ export function GlobalOverviewEntry<const Description extends z.ZodTypeAny>(
         z.object({
           description: descriptionType,
           manifestSha256: Sha256Schema
-        })
+        }).passthrough()
       )
       .default({})
-  });
+  }).passthrough();
   return (
     universalSchema
       .transform((o) => {
@@ -131,7 +131,7 @@ export function GlobalOverview<const Description extends z.ZodTypeAny>(
   return z.object({
     schema: z.literal('v2'),
     packages: z.array(GlobalOverviewEntry(descriptionType))
-  });
+  }).passthrough();
 }
 
 export const GlobalOverviewReg = GlobalOverview(BlockPackDescriptionManifest);