@platforma-sdk/block-tools 2.1.11 → 2.3.0

package/src/v2/registry/registry.ts

@@ -0,0 +1,240 @@
+import { MiLogger } from '@milaboratories/ts-helpers';
+import { compare as compareSemver, satisfies } from 'semver';
+import { RegistryStorage } from '../../io/storage';
+import { BlockPackIdNoVersion, BlockPackManifest } from '@milaboratories/pl-model-middle-layer';
+import {
+  GlobalUpdateSeedInFile,
+  GlobalUpdateSeedOutFile,
+  PackageUpdatePattern,
+  packageUpdateSeedPath,
+  VersionUpdatesPrefix
+} from './schema_internal';
+import {
+  GlobalOverviewReg,
+  GlobalOverviewPath,
+  ManifestSuffix,
+  packageContentPrefix,
+  PackageOverview,
+  packageOverviewPath,
+  ManifestFileName
+} from './schema_public';
+import { BlockPackDescriptionManifestAddRelativePathPrefix, RelativeContentReader } from '../model';
+import { randomUUID } from 'node:crypto';
+
+type PackageUpdateInfo = {
+  package: BlockPackIdNoVersion;
+  versions: Set<String>;
+};
+
+export class BlockRegistryV2 {
+  constructor(
+    private readonly storage: RegistryStorage,
+    private readonly logger?: MiLogger
+  ) {}
+
+  private async updateRegistry() {
+    this.logger?.info('Initiating registry refresh...');
+
+    // reading update requests
+    const packagesToUpdate = new Map<string, PackageUpdateInfo>();
+    const seedPaths: string[] = [];
+    const rawSeedPaths = await this.storage.listFiles(VersionUpdatesPrefix);
+    this.logger?.info('Packages to be updated:');
+    for (const seedPath of rawSeedPaths) {
+      const match = seedPath.match(PackageUpdatePattern);
+      if (!match) continue;
+      seedPaths.push(seedPath);
+      const { packageKeyWithoutVersion, organization, name, version, seed } = match.groups!;
+
+      let update = packagesToUpdate.get(packageKeyWithoutVersion);
+      let added = false;
+      if (!update) {
+        packagesToUpdate.set(packageKeyWithoutVersion, {
+          package: { organization, name },
+          versions: new Set([version])
+        });
+        added = true;
+      } else if (!update.versions.has(version)) {
+        update.versions.add(version);
+        added = true;
+      }
+      this.logger?.info(`  - ${organization}:${name}:${version} added:${added}`);
+    }
+
+    // loading global overview
+    const overviewContent = await this.storage.getFile(GlobalOverviewPath);
+    const overview: GlobalOverviewReg =
+      overviewContent === undefined
+        ? { schema: 'v2', packages: [] }
+        : GlobalOverviewReg.parse(JSON.parse(overviewContent.toString()));
+    let overviewPackages = overview.packages;
+    this.logger?.info(`Global overview loaded, ${overviewPackages.length} records`);
+
+    // updating packages
+    for (const [, packageInfo] of packagesToUpdate.entries()) {
+      // reading existing overview
+      const overviewFile = packageOverviewPath(packageInfo.package);
+      const pOverviewContent = await this.storage.getFile(overviewFile);
+      const packageOverview: PackageOverview =
+        pOverviewContent === undefined
+          ? { schema: 'v2', versions: [] }
+          : PackageOverview.parse(JSON.parse(pOverviewContent.toString()));
+      this.logger?.info(
+        `Updating ${packageInfo.package.organization}:${packageInfo.package.name} overview, ${packageOverview.versions.length} records`
+      );
+
+      // removing versions that we will update
+      const newVersions = packageOverview.versions.filter(
+        (e) => !packageInfo.versions.has(e.description.id.version)
+      );
+
+      // reading new entries
+      for (const [v] of packageInfo.versions.entries()) {
+        const version = v.toString();
+        const manifestContent = await this.storage.getFile(
+          packageContentPrefix({
+            ...packageInfo.package,
+            version
+          }) + ManifestSuffix
+        );
+        if (!manifestContent) continue; // absent package
+        const sha256 = Buffer.from(await crypto.subtle.digest('sha-256', manifestContent))
+          .toString('hex')
+          .toUpperCase();
+        newVersions.push({
+          description: BlockPackDescriptionManifestAddRelativePathPrefix(version).parse(
+            BlockPackManifest.parse(JSON.parse(manifestContent.toString('utf8'))).description
+          ),
+          manifestSha256: sha256
+        });
+      }
+
+      // sorting entries according to version
+      newVersions.sort((e1, e2) =>
+        compareSemver(e2.description.id.version, e1.description.id.version)
+      );
+
+      // write package overview back
+      await this.storage.putFile(
+        overviewFile,
+        Buffer.from(
+          JSON.stringify({ schema: 'v2', versions: newVersions } satisfies PackageOverview)
+        )
+      );
+      this.logger?.info(`Done (${newVersions.length} records)`);
+
+      // patching corresponding entry in overview
+      overviewPackages = overviewPackages.filter(
+        (e) =>
+          e.id.organization !== packageInfo.package.organization ||
+          e.id.name !== packageInfo.package.name
+      );
+      overviewPackages.push({
+        id: {
+          organization: packageInfo.package.organization,
+          name: packageInfo.package.name
+        },
+        allVersions: newVersions.map((e) => e.description.id.version).reverse(),
+        latest: BlockPackDescriptionManifestAddRelativePathPrefix(
+          `${packageInfo.package.organization}/${packageInfo.package.name}`
+        ).parse(newVersions[0].description),
+        latestManifestSha256: newVersions[0].manifestSha256
+      });
+    }
+
+    // writing global overview
+    await this.storage.putFile(
+      GlobalOverviewPath,
+      Buffer.from(
+        JSON.stringify({ schema: 'v2', packages: overviewPackages } satisfies GlobalOverviewReg)
+      )
+    );
+    this.logger?.info(`Global overview updated (${overviewPackages.length} records)`);
+
+    // deleting seeds
+    await this.storage.deleteFiles(...seedPaths.map((sp) => `${VersionUpdatesPrefix}${sp}`));
+    this.logger?.info(`Version update requests cleared`);
+  }
+
+  public async updateIfNeeded(force: boolean = false): Promise<void> {
+    // implementation of main convergence algorithm
+
+    this.logger?.info(`Checking if registry requires refresh...`);
+    const updateRequestSeed = await this.storage.getFile(GlobalUpdateSeedInFile);
+    const currentUpdatedSeed = await this.storage.getFile(GlobalUpdateSeedOutFile);
+    if (!force && updateRequestSeed === undefined && currentUpdatedSeed === undefined) return;
+    if (
+      !force &&
+      updateRequestSeed !== undefined &&
+      currentUpdatedSeed !== undefined &&
+      updateRequestSeed.equals(currentUpdatedSeed)
+    )
+      return;
+
+    await this.updateRegistry();
+
+    if (updateRequestSeed) {
+      await this.storage.putFile(GlobalUpdateSeedOutFile, updateRequestSeed);
+      this.logger?.info(`Refresh finished`);
+    }
+  }
+
+  public async getPackageOverview(
+    name: BlockPackIdNoVersion
+  ): Promise<undefined | PackageOverview> {
+    const content = await this.storage.getFile(packageOverviewPath(name));
+    if (content === undefined) return undefined;
+    return PackageOverview.parse(JSON.parse(content.toString()));
+  }
+
+  public async getGlobalOverview(): Promise<undefined | GlobalOverviewReg> {
+    const content = await this.storage.getFile(GlobalOverviewPath);
+    if (content === undefined) return undefined;
+    return GlobalOverviewReg.parse(JSON.parse(content.toString()));
+  }
+
+  // public async getGlobalOverviewExplicitBytes(): Promise<undefined | GlobalOverviewReg> {
+  //   const content = await this.storage.getFile(GlobalOverviewPath);
+  //   if (content === undefined) return undefined;
+  //   return GlobalOverviewReg.parse(JSON.parse(content.toString()));
+  // }
+
+  public async publishPackage(
+    manifest: BlockPackManifest,
+    fileReader: RelativeContentReader
+  ): Promise<void> {
+    const prefix = packageContentPrefix(manifest.description.id);
+    // uploading content files
+    for (const f of manifest.files) {
+      const bytes = await fileReader(f.name);
+      if (bytes.length !== f.size)
+        throw new Error(
+          `Actual file size don't match file size from the manifest file for ${f.name} (actual = ${bytes.length}; manifest = ${f.size})`
+        );
+      const sha256 = Buffer.from(await crypto.subtle.digest('sha-256', bytes))
+        .toString('hex')
+        .toUpperCase();
+      if (sha256 !== f.sha256.toUpperCase())
+        throw new Error(
+          `Actual file SHA-256 don't match the checksum from the manifest file for ${f.name} (actual = ${sha256}; manifest = ${f.sha256.toUpperCase()})`
+        );
+
+      const dst = prefix + '/' + f.name;
+      this.logger?.info(`Uploading ${f.name} -> ${dst} ...`);
+      await this.storage.putFile(dst, bytes);
+    }
+
+    // uploading manifest as the last upload action
+    const manifestDst = prefix + '/' + ManifestFileName;
+    this.logger?.info(`Uploading manifest to ${manifestDst} ...`);
+    await this.storage.putFile(manifestDst, Buffer.from(JSON.stringify(manifest)));
+
+    // adding update seed
+    const seed = randomUUID();
+    const seedPath = packageUpdateSeedPath(manifest.description.id, seed);
+    this.logger?.info(`Creating update seed at ${seedPath} ...`);
+    await this.storage.putFile(seedPath, Buffer.from(seed));
+    this.logger?.info(`Touching global update seed ${GlobalUpdateSeedInFile} ...`);
+    await this.storage.putFile(GlobalUpdateSeedInFile, Buffer.from(seed));
+  }
+}

package/src/v2/registry/registry_reader.ts

@@ -0,0 +1,71 @@
+import {
+  BlockPackDescriptionManifest,
+  BlockPackMetaEmbeddedBytes,
+  BlockPackMetaManifest,
+  BlockPackOverview
+} from '@milaboratories/pl-model-middle-layer';
+import { FolderReader } from '../../io';
+import canonicalize from 'canonicalize';
+import {
+  GlobalOverview,
+  GlobalOverviewEntryReg,
+  GlobalOverviewFileName,
+  GlobalOverviewReg,
+  MainPrefix,
+  ManifestFileName
+} from './schema_public';
+import { BlockPackMetaEmbedBytes } from '../model';
+
+export type BlockPackOverviewNoRegLabel = Omit<BlockPackOverview, 'registryId'>;
+
+export class RegistryV2Reader {
+  private readonly metaCache = new Map<
+    string,
+    { sha256: string; meta: BlockPackMetaEmbeddedBytes }
+  >();
+
+  private readonly rootFolderReader: FolderReader;
+
+  constructor(
+    private readonly url: string,
+    reader: FolderReader
+  ) {
+    this.rootFolderReader = reader.relativeReader(MainPrefix);
+  }
+
+  private async embedMetaContent(
+    entry: GlobalOverviewEntryReg
+  ): Promise<BlockPackMetaEmbeddedBytes> {
+    const id = canonicalize(entry.id)!;
+    const fromCache = this.metaCache.get(id);
+    if (fromCache && fromCache.sha256 === entry.latestManifestSha256) return fromCache.meta;
+    const rootContentReader = this.rootFolderReader.getContentReader();
+    const meta = await BlockPackMetaEmbedBytes(rootContentReader).parseAsync(entry.latest.meta);
+    this.metaCache.set(id, { sha256: entry.latestManifestSha256, meta });
+    return meta;
+  }
+
+  public async listBlockPacks(): Promise<BlockPackOverviewNoRegLabel[]> {
+    const rootContentReader = this.rootFolderReader.getContentReader();
+    const globalOverview = GlobalOverviewReg.parse(
+      JSON.parse(
+        Buffer.from(await this.rootFolderReader.readFile(GlobalOverviewFileName)).toString()
+      )
+    );
+    return await Promise.all(
+      globalOverview.packages.map(
+        async (p) =>
+          ({
+            id: p.latest.id,
+            meta: await this.embedMetaContent(p),
+            spec: {
+              type: 'from-registry-v2',
+              id: p.latest.id,
+              registryUrl: this.url
+            },
+            otherVersions: p.allVersions
+          }) satisfies BlockPackOverviewNoRegLabel
+      )
+    );
+  }
+}

package/src/v2/registry/schema_internal.ts

@@ -0,0 +1,13 @@
+import { BlockPackId } from '@milaboratories/pl-model-middle-layer';
+
+export const VersionUpdatesPrefix = '_updates_v2/per_package_version/';
+
+export function packageUpdateSeedPath(bp: BlockPackId, seed: string): string {
+  return `${VersionUpdatesPrefix}${bp.organization}/${bp.name}/${bp.version}/${seed}`;
+}
+
+export const PackageUpdatePattern =
+  /(?<packageKeyWithoutVersion>(?<organization>[^\/]+)\/(?<name>[^\/]+))\/(?<version>[^\/]+)\/(?<seed>[^\/]+)$/;
+
+export const GlobalUpdateSeedInFile = '_updates_v2/_global_update_in';
+export const GlobalUpdateSeedOutFile = '_updates_v2/_global_update_out';

package/src/v2/registry/schema_public.ts

@@ -0,0 +1,112 @@
+import {
+  BlockComponentsManifest,
+  BlockPackDescriptionManifest,
+  BlockPackId,
+  BlockPackIdNoVersion,
+  BlockPackMeta,
+  ContentRelativeBinary,
+  ContentRelativeText,
+  CreateBlockPackDescriptionSchema,
+  SemVer,
+  Sha256Schema
+} from '@milaboratories/pl-model-middle-layer';
+import { z } from 'zod';
+import { RelativeContentReader, relativeToExplicitBytes, relativeToExplicitString } from '../model';
+
+export const MainPrefix = 'v2/';
+
+export const GlobalOverviewFileName = 'overview.json';
+export const PackageOverviewFileName = 'overview.json';
+export const ManifestFileName = 'manifest.json';
+
+export function packageContentPrefix(bp: BlockPackId): string {
+  return `${MainPrefix}${bp.organization}/${bp.name}/${bp.version}`;
+}
+
+export const ManifestSuffix = '/' + ManifestFileName;
+
+// export function payloadFilePath(bp: BlockPackId, file: string): string {
+//   return `${MainPrefix}${bp.organization}/${bp.name}/${bp.version}/${file}`;
+// }
+
+export const PackageOverviewVersionEntry = z.object({
+  description: BlockPackDescriptionManifest,
+  manifestSha256: Sha256Schema
+});
+export type PackageOverviewVersionEntry = z.infer<typeof PackageOverviewVersionEntry>;
+
+export const PackageOverview = z.object({
+  schema: z.literal('v2'),
+  versions: z.array(PackageOverviewVersionEntry)
+});
+export type PackageOverview = z.infer<typeof PackageOverview>;
+
+export function packageOverviewPath(bp: BlockPackIdNoVersion): string {
+  return `${MainPrefix}${bp.organization}/${bp.name}/${PackageOverviewFileName}`;
+}
+
+export const GlobalOverviewPath = `${MainPrefix}${GlobalOverviewFileName}`;
+
+export function GlobalOverviewEntry<const Description extends z.ZodTypeAny>(
+  descriptionType: Description
+) {
+  return z.object({
+    id: BlockPackIdNoVersion,
+    allVersions: z.array(z.string()),
+    latest: descriptionType,
+    latestManifestSha256: Sha256Schema
+  });
+}
+export const GlobalOverviewEntryReg = GlobalOverviewEntry(BlockPackDescriptionManifest);
+export type GlobalOverviewEntryReg = z.infer<typeof GlobalOverviewEntryReg>;
+
+export function GlobalOverview<const Description extends z.ZodTypeAny>(
+  descriptionType: Description
+) {
+  return z.object({
+    schema: z.literal('v2'),
+    packages: z.array(GlobalOverviewEntry(descriptionType))
+  });
+}
+
+export const GlobalOverviewReg = GlobalOverview(BlockPackDescriptionManifest);
+export type GlobalOverviewReg = z.infer<typeof GlobalOverviewReg>;
+
+export function BlockDescriptionToExplicitBinaryBytes(reader: RelativeContentReader) {
+  return CreateBlockPackDescriptionSchema(
+    BlockComponentsManifest,
+    BlockPackMeta(
+      ContentRelativeText.transform(relativeToExplicitString(reader)),
+      ContentRelativeBinary.transform(relativeToExplicitBytes(reader))
+    )
+  );
+}
+export function GlobalOverviewToExplicitBinaryBytes(reader: RelativeContentReader) {
+  return GlobalOverview(
+    CreateBlockPackDescriptionSchema(
+      BlockComponentsManifest,
+      BlockPackMeta(
+        ContentRelativeText.transform(relativeToExplicitString(reader)),
+        ContentRelativeBinary.transform(relativeToExplicitBytes(reader))
+      )
+    )
+  );
+}
+export type GlobalOverviewExplicitBinaryBytes = z.infer<
+  ReturnType<typeof GlobalOverviewToExplicitBinaryBytes>
+>;
+
+export function GlobalOverviewToExplicitBinaryBase64(reader: RelativeContentReader) {
+  return GlobalOverview(
+    CreateBlockPackDescriptionSchema(
+      BlockComponentsManifest,
+      BlockPackMeta(
+        ContentRelativeText.transform(relativeToExplicitString(reader)),
+        ContentRelativeBinary.transform(relativeToExplicitBytes(reader))
+      )
+    )
+  );
+}
+export type GlobalOverviewExplicitBinaryBase64 = z.infer<
+  ReturnType<typeof GlobalOverviewToExplicitBinaryBase64>
+>;