@platforma-sdk/block-tools 2.1.5 → 2.1.6

package/src/v2/model/content_conversion.ts

@@ -0,0 +1,178 @@
+import { z } from 'zod';
+import path from 'path';
+import fsp from 'node:fs/promises';
+import * as mime from 'mime-types';
+import * as tar from 'tar';
+import {
+  ContentAbsoluteBinaryLocal,
+  ContentAbsoluteFile,
+  ContentAbsoluteFolder,
+  ContentAbsoluteTextLocal,
+  ContentAbsoluteUrl,
+  ContentAnyLocal,
+  ContentExplicitBase64,
+  ContentRelative
+} from '@milaboratories/pl-model-middle-layer';
+
+type ContentCtxFs = {
+  type: 'local';
+  /** Folder relative to which content should be resolved */
+  path: string;
+};
+
+type ContentCtxUrl = {
+  type: 'remote';
+  /** URL prefix from which content should be resolved */
+  url: string;
+};
+
+/** Describes a place relative to which any content references should be interpreted */
+export type ContentCtx = ContentCtxFs | ContentCtxUrl;
+
+function tryResolve(root: string, request: string): string | undefined {
+  try {
+    return require.resolve(request, {
+      paths: [root]
+    });
+  } catch (err: any) {
+    if (err.code !== 'MODULE_NOT_FOUND') throw err;
+  }
+  return undefined;
+}
+
+function mustResolve(root: string, request: string): string {
+  const res = tryResolve(root, request);
+  if (res === undefined) throw new Error(`Can't resolve ${request} against ${root}`);
+  return res;
+}
+
+/** Zod type that resolves node module request into absolute content */
+export function ResolvedModuleFile(moduleRoot: string) {
+  return z.string().transform<ContentAbsoluteFile>((request, ctx) => {
+    const result = tryResolve(moduleRoot, request);
+    if (result === undefined) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `Can't resolve ${request} against ${moduleRoot}`
+      });
+      return z.NEVER;
+    }
+    return {
+      type: 'absolute-file',
+      file: result
+    };
+  });
+}
+
+/**
+ * Zod type that resolves node module request for folder into absolute folder,
+ * given a list of expected index files in that folder
+ * */
+export function ResolvedModuleFolder(
+  moduleRoot: string,
+  ...indexFilesToLookFor: [string, ...string[]]
+) {
+  return z.string().transform<ContentAbsoluteFolder>((request, ctx) => {
+    const requestWithSlash = request.endsWith('/') ? request : `${request}/`;
+
+    for (const idxFile of indexFilesToLookFor) {
+      const result = tryResolve(moduleRoot, requestWithSlash + idxFile);
+      if (result !== undefined) {
+        if (!result.endsWith(idxFile))
+          throw new Error(`Unexpected resolve result ${result} with index file ${idxFile}`);
+        return {
+          type: 'absolute-folder',
+          folder: result.slice(0, result.length - idxFile.length)
+        };
+      }
+    }
+
+    ctx.addIssue({
+      code: z.ZodIssueCode.custom,
+      message: `Can't resolve ${request} folder against ${moduleRoot}, no index file found (${indexFilesToLookFor.join(', ')})`
+    });
+    return z.NEVER;
+  });
+}
+
+export function mapLocalToAbsolute(
+  root: string
+): <T extends ContentAnyLocal>(value: T) => Exclude<T, ContentRelative> | ContentAbsoluteFile {
+  return <T extends ContentAnyLocal>(value: T) =>
+    value.type === 'relative'
+      ? { type: 'absolute-file', file: path.resolve(root, value.path) }
+      : (value as Exclude<T, ContentRelative>);
+}
+
+export function absoluteToString(): (value: ContentAbsoluteTextLocal) => Promise<string> {
+  return async (value) => {
+    if (value.type === 'absolute-file')
+      return await fsp.readFile(value.file, { encoding: 'utf-8' });
+    else return value.content;
+  };
+}
+
+// TODO add type and size limitations
+export function absoluteToBase64(): (
+  value: ContentAbsoluteBinaryLocal
+) => Promise<ContentExplicitBase64> {
+  return async (value) => {
+    if (value.type === 'absolute-file') {
+      const mimeType = mime.lookup(value.file);
+      if (!mimeType) throw new Error(`Can't recognize mime type of the file: ${value.file}.`);
+      return {
+        type: 'explicit-base64',
+        mimeType,
+        content: await fsp.readFile(value.file, { encoding: 'base64' })
+      };
+    } else return value;
+  };
+}
+
+export function cpAbsoluteToRelative(
+  dstFolder: string,
+  fileAccumulator?: string[]
+): <T extends Exclude<ContentAnyLocal, ContentRelative>>(
+  value: T
+) => Promise<Exclude<T, ContentAbsoluteFile> | ContentRelative> {
+  return async <T extends Exclude<ContentAnyLocal, ContentRelative>>(value: T) => {
+    if (value.type === 'absolute-file') {
+      const fileName = path.basename(value.file);
+      const dst = path.resolve(dstFolder, fileName);
+      fileAccumulator?.push(fileName);
+      await fsp.cp(value.file, dst);
+      return { type: 'relative', path: fileName };
+    } else return value as Exclude<T, ContentAbsoluteFile>;
+  };
+}
+
+export function packFolderToRelativeTgz(
+  dstFolder: string,
+  tgzName: string,
+  fileAccumulator?: string[]
+): (value: ContentAbsoluteFolder) => Promise<ContentRelative> {
+  if (!tgzName.endsWith('.tgz')) throw new Error(`Unexpected tgz file name: ${tgzName}`);
+  return async (value: ContentAbsoluteFolder) => {
+    const dst = path.resolve(dstFolder, tgzName);
+    await tar.create(
+      {
+        gzip: true,
+        file: dst,
+        cwd: value.folder
+      },
+      [value.folder]
+    );
+    fileAccumulator?.push(tgzName);
+    return { type: 'relative', path: tgzName };
+  };
+}
+
+export function mapRemoteToAbsolute(
+  rootUrl: string
+): <T extends ContentAnyLocal>(value: T) => Exclude<T, ContentRelative> | ContentAbsoluteUrl {
+  const rootWithSlash = rootUrl.endsWith('/') ? rootUrl : `${rootUrl}/`;
+  return <T extends ContentAnyLocal>(value: T) =>
+    value.type === 'relative'
+      ? { type: 'absolute-url', url: rootWithSlash + value.path }
+      : (value as Exclude<T, ContentRelative>);
+}

package/src/v2/model/content_types.ts

@@ -0,0 +1,233 @@
+import { z } from 'zod';
+
+//
+// Base content types
+//
+
+export const ContentExplicitString = z
+  .object({
+    type: z.literal('explicit-string'),
+    content: z.string().describe('Actual string value')
+  })
+  .strict();
+export type ContentExplicitString = z.infer<typeof ContentExplicitString>;
+
+export const ContentExplicitBase64 = z
+  .object({
+    type: z.literal('explicit-base64'),
+    mimeType: z
+      .string()
+      .regex(/\w+\/[-+.\w]+/)
+      .describe('MIME type to interpret content'),
+    content: z.string().base64().describe('Base64 encoded binary value')
+  })
+  .strict();
+export type ContentExplicitBase64 = z.infer<typeof ContentExplicitBase64>;
+
+export const ContentRelative = z
+  .object({
+    type: z.literal('relative'),
+    path: z
+      .string()
+      .describe(
+        'Address of the file, in most cases relative to the file which this structure is a part of'
+      )
+  })
+  .strict();
+export type ContentRelative = z.infer<typeof ContentRelative>;
+
+export const ContentAbsoluteFile = z
+  .object({
+    type: z.literal('absolute-file'),
+    file: z.string().startsWith('/').describe('Absolute address of the file in local file system')
+  })
+  .strict();
+export type ContentAbsoluteFile = z.infer<typeof ContentAbsoluteFile>;
+
+export const ContentAbsoluteUrl = z
+  .object({
+    type: z.literal('absolute-url'),
+    url: z.string().url().describe('Global URL to reach the requested file')
+  })
+  .strict();
+export type ContentAbsoluteUrl = z.infer<typeof ContentAbsoluteUrl>;
+
+//
+// Special content types
+//
+
+export const ContentExplicit = z
+  .object({
+    type: z.literal('explicit'),
+    content: z.instanceof(Uint8Array).describe('Raw content')
+  })
+  .strict();
+export type ContentExplicit = z.infer<typeof ContentExplicit>;
+
+export const ContentAbsoluteFolder = z
+  .object({
+    type: z.literal('absolute-folder'),
+    folder: z
+      .string()
+      .startsWith('/')
+      .describe('Absolute address of the folder in local file system')
+  })
+  .strict();
+export type ContentAbsoluteFolder = z.infer<typeof ContentAbsoluteFolder>;
+
+//
+// Unions
+//
+
+export const ContentAny = z.discriminatedUnion('type', [
+  ContentExplicitString,
+  ContentExplicitBase64,
+  ContentRelative,
+  ContentAbsoluteFile,
+  ContentAbsoluteUrl
+]);
+export type ContentAny = z.infer<typeof ContentAny>;
+
+export const ContentAnyLocal = z.discriminatedUnion('type', [
+  ContentExplicitString,
+  ContentExplicitBase64,
+  ContentRelative,
+  ContentAbsoluteFile
+]);
+export type ContentAnyLocal = z.infer<typeof ContentAnyLocal>;
+
+export const ContentAnyRemote = z.discriminatedUnion('type', [
+  ContentExplicitString,
+  ContentExplicitBase64,
+  ContentRelative,
+  ContentAbsoluteUrl
+]);
+export type ContentAnyRemote = z.infer<typeof ContentAnyRemote>;
+
+//
+// Narrow types with relative option
+//
+
+// export const ContentAnyBinaryRemote = z.discriminatedUnion('type', [
+//   ContentExplicitBase64,
+//   ContentRelative,
+//   ContentAbsoluteUrl
+// ]);
+// export type ContentAnyBinaryRemote = z.infer<typeof ContentAnyBinaryRemote>;
+
+export const ContentAnyBinaryLocal = z.discriminatedUnion('type', [
+  ContentExplicitBase64,
+  ContentRelative,
+  ContentAbsoluteFile
+]);
+export type ContentAnyBinaryLocal = z.infer<typeof ContentAnyBinaryLocal>;
+
+// export const ContentAnyTextRemote = z.discriminatedUnion('type', [
+//   ContentExplicitString,
+//   ContentRelative,
+//   ContentAbsoluteUrl
+// ]);
+// export type ContentAnyTextRemote = z.infer<typeof ContentAnyTextRemote>;
+
+export const ContentAnyTextLocal = z.discriminatedUnion('type', [
+  ContentExplicitString,
+  ContentRelative,
+  ContentAbsoluteFile
+]);
+export type ContentAnyTextLocal = z.infer<typeof ContentAnyTextLocal>;
+
+//
+// Narrow absolute types
+//
+
+export const ContentAbsoluteBinaryRemote = z.discriminatedUnion('type', [
+  ContentExplicitBase64,
+  ContentAbsoluteUrl
+]);
+export type ContentAbsoluteBinaryRemote = z.infer<typeof ContentAbsoluteBinaryRemote>;
+
+export const ContentAbsoluteBinaryLocal = z.discriminatedUnion('type', [
+  ContentExplicitBase64,
+  ContentAbsoluteFile
+]);
+export type ContentAbsoluteBinaryLocal = z.infer<typeof ContentAbsoluteBinaryLocal>;
+
+export const ContentAbsoluteTextRemote = z.discriminatedUnion('type', [
+  ContentExplicitString,
+  ContentAbsoluteUrl
+]);
+export type ContentAbsoluteTextRemote = z.infer<typeof ContentAbsoluteTextRemote>;
+
+export const ContentAbsoluteTextLocal = z.discriminatedUnion('type', [
+  ContentExplicitString,
+  ContentAbsoluteFile
+]);
+export type ContentAbsoluteTextLocal = z.infer<typeof ContentAbsoluteTextLocal>;
+
+//
+// Narrow relative types
+//
+
+export const ContentRelativeBinary = z.discriminatedUnion('type', [
+  ContentExplicitBase64,
+  ContentRelative
+]);
+export type ContentRelativeBinary = z.infer<typeof ContentRelativeBinary>;
+
+export const ContentRelativeText = z.discriminatedUnion('type', [
+  ContentExplicitString,
+  ContentRelative
+]);
+export type ContentRelativeText = z.infer<typeof ContentRelativeText>;
+
+// export function ConstructContent(
+//   contentType: 'text',
+//   contextType: 'local'
+// ): typeof ContentAnyTextLocal;
+// export function ConstructContent(
+//   contentType: 'text',
+//   contextType: 'remote'
+// ): typeof ContentAnyTextRemote;
+// export function ConstructContent(
+//   contentType: 'binary',
+//   contextType: 'local'
+// ): typeof ContentAnyBinaryLocal;
+// export function ConstructContent(
+//   contentType: 'binary',
+//   contextType: 'remote'
+// ): typeof ContentAnyBinaryRemote;
+// export function ConstructContent(
+//   contentType: ContentType,
+//   contextType: ContextType
+// ):
+//   | typeof ContentAnyTextLocal
+//   | typeof ContentAnyTextRemote
+//   | typeof ContentAnyBinaryLocal
+//   | typeof ContentAnyBinaryRemote;
+// export function ConstructContent(contentType: ContentType, contextType: ContextType) {
+//   return contentType === 'text'
+//     ? contextType === 'local'
+//       ? ContentAnyTextLocal
+//       : ContentAnyTextRemote
+//     : contextType === 'local'
+//       ? ContentAnyBinaryLocal
+//       : ContentAnyBinaryRemote;
+// }
+
+export const DescriptionContentBinary = z.union([
+  z
+    .string()
+    .startsWith('file:')
+    .transform<ContentRelativeBinary>((value, ctx) => ({ type: 'relative', path: value.slice(5) })),
+  ContentAnyBinaryLocal
+]);
+export type DescriptionContentBinary = z.infer<typeof DescriptionContentBinary>;
+
+export const DescriptionContentText = z.union([
+  z.string().transform<ContentRelativeText>((value, ctx) => {
+    if (value.startsWith('file:')) return { type: 'relative', path: value.slice(5) };
+    else return { type: 'explicit-string', content: value };
+  }),
+  ContentAnyTextLocal
+]);
+export type DescriptionContentText = z.infer<typeof DescriptionContentText>;

package/src/v2/model/index.ts

@@ -0,0 +1,46 @@
+import { z } from 'zod';
+import { BlockComponentsConsolidate, BlockComponentsDescription } from './block_components';
+import {
+  BlockComponentsManifest,
+  BlockPackMetaManifest,
+  CreateBlockPackDescriptionSchema
+} from '@milaboratories/pl-model-middle-layer';
+import { BlockPackMetaConsolidate, BlockPackMetaDescription } from './meta';
+
+export * from './block_components';
+export * from './content_conversion';
+
+export function ResolvedBlockPackDescriptionFromPackageJson(root: string) {
+  return CreateBlockPackDescriptionSchema(
+    BlockComponentsDescription(root),
+    BlockPackMetaDescription(root)
+  );
+}
+export type BlockPackDescriptionAbsolute = z.infer<
+  ReturnType<typeof ResolvedBlockPackDescriptionFromPackageJson>
+>;
+
+export function BlockPackDescriptionConsolidateToFolder(
+  dstFolder: string,
+  fileAccumulator?: string[]
+) {
+  return CreateBlockPackDescriptionSchema(
+    BlockComponentsConsolidate(dstFolder, fileAccumulator),
+    //BlockPackMetaToExplicit
+    BlockPackMetaConsolidate(dstFolder, fileAccumulator)
+  ).pipe(BlockPackDescriptionManifest);
+}
+
+export const BlockPackDescriptionManifest = CreateBlockPackDescriptionSchema(
+  BlockComponentsManifest,
+  BlockPackMetaManifest
+);
+export type BlockPackDescriptionManifest = z.infer<typeof BlockPackDescriptionManifest>;
+
+export const BlockPackManifest = BlockPackDescriptionManifest.extend({
+  schema: z.literal('v1'),
+  files: z.array(z.string())
+});
+export type BlockPackManifest = z.infer<typeof BlockPackManifest>;
+
+export const BlockPackManifestFile = 'manifest.json';

package/src/v2/model/meta.ts

@@ -0,0 +1,36 @@
+import {
+  BlockPackMeta,
+  ContentAbsoluteBinaryLocal,
+  ContentAbsoluteTextLocal,
+  DescriptionContentBinary,
+  DescriptionContentText
+} from '@milaboratories/pl-model-middle-layer';
+import {
+  absoluteToBase64,
+  absoluteToString,
+  cpAbsoluteToRelative,
+  mapLocalToAbsolute
+} from './content_conversion';
+import { z } from 'zod';
+import { BlockPackMetaEmbeddedContent } from '@milaboratories/pl-model-middle-layer';
+
+export function BlockPackMetaDescription(root: string) {
+  return BlockPackMeta(
+    DescriptionContentText.transform(mapLocalToAbsolute(root)),
+    DescriptionContentBinary.transform(mapLocalToAbsolute(root))
+  );
+}
+export type BlockPackMetaDescription = z.infer<ReturnType<typeof BlockPackMetaDescription>>;
+
+export function BlockPackMetaConsolidate(dstFolder: string, fileAccumulator?: string[]) {
+  return BlockPackMeta(
+    ContentAbsoluteTextLocal.transform(cpAbsoluteToRelative(dstFolder, fileAccumulator)),
+    ContentAbsoluteBinaryLocal.transform(cpAbsoluteToRelative(dstFolder, fileAccumulator))
+  );
+}
+
+export const BlockPackMetaEmbed = BlockPackMeta(
+  ContentAbsoluteTextLocal.transform(absoluteToString()),
+  ContentAbsoluteBinaryLocal.transform(absoluteToBase64())
+).pipe(BlockPackMetaEmbeddedContent);
+export type BlockPackMetaEmbed = z.infer<typeof BlockPackMetaEmbed>;

package/src/v2/registry/schema.ts

@@ -0,0 +1,29 @@
+import { BlockPackId, BlockPackIdNoVersion } from '@milaboratories/pl-model-middle-layer';
+
+const MainPrefix = 'v2/';
+
+export function packageContentPrefix(bp: BlockPackId): string {
+  return `${MainPrefix}${bp.organization}/${bp.name}/${bp.version}`;
+}
+
+export function payloadFilePath(bp: BlockPackId, file: string): string {
+  return `${MainPrefix}${bp.organization}/${bp.name}/${bp.version}/${file}`;
+}
+
+export function packageOverviewPath(bp: BlockPackIdNoVersion): string {
+  return `${MainPrefix}${bp.organization}/${bp.name}/overview.json`;
+}
+
+export const GlobalOverviewPath = `${MainPrefix}overview.json`;
+
+export const ManifestFile = 'manifest.json';
+
+export interface GlobalOverviewEntry {
+  organization: string;
+  package: string;
+  allVersions: string[];
+  latestVersion: string;
+  latestMeta: object;
+}
+
+export type GlobalOverview = GlobalOverviewEntry[];

package/src/v2/source_package.test.ts

@@ -0,0 +1,27 @@
+import { loadPackDescription, parsePackageName } from './source_package';
+
+test('test parsing of convention package names', () => {
+  expect(parsePackageName('@milaboratory/milaboratories.block-template')).toStrictEqual({
+    organization: 'milaboratories',
+    name: 'block-template'
+  });
+
+  expect(parsePackageName('milaboratories.block-template')).toStrictEqual({
+    organization: 'milaboratories',
+    name: 'block-template'
+  });
+
+  expect(parsePackageName('mi-laboratories.block-template')).toStrictEqual({
+    organization: 'mi-laboratories',
+    name: 'block-template'
+  });
+
+  expect(() => parsePackageName('block-template')).toThrow(/Malformed/);
+});
+
+test.skip('full description parsing test', async () => {
+  const description = await loadPackDescription(
+    '/Volumes/Data/Projects/MiLaboratory/blocks-beta/block-template'
+  );
+  console.dir(description, { depth: 5 });
+});

package/src/v2/source_package.ts

@@ -0,0 +1,82 @@
+import path from 'path';
+import { tryLoadFile } from '../util';
+import { ResolvedBlockPackDescriptionFromPackageJson, BlockPackDescriptionAbsolute } from './model';
+import { notEmpty } from '@milaboratories/ts-helpers';
+import fsp from 'node:fs/promises';
+import {
+  BlockPackDescriptionFromPackageJsonRaw,
+  BlockPackDescriptionRaw,
+  BlockPackId,
+  BlockPackMetaDescriptionRaw,
+  SemVer
+} from '@milaboratories/pl-model-middle-layer';
+
+export const BlockDescriptionPackageJsonField = 'block';
+
+const ConventionPackageNamePattern =
+  /(?:@[a-zA-Z0-9-.]+\/)?(?<organization>[a-zA-Z0-9-]+)\.(?<name>[a-zA-Z0-9-]+)/;
+
+export function parsePackageName(packageName: string): Pick<BlockPackId, 'organization' | 'name'> {
+  const match = packageName.match(ConventionPackageNamePattern);
+  if (!match)
+    throw new Error(
+      `Malformed package name (${packageName}), can't infer organization and block pack name.`
+    );
+  const { name, organization } = match.groups!;
+  return { name, organization };
+}
+
+export async function tryLoadPackDescription(
+  moduleRoot: string
+): Promise<BlockPackDescriptionAbsolute | undefined> {
+  const fullPackageJsonPath = path.resolve(moduleRoot, 'package.json');
+  try {
+    const packageJson = await tryLoadFile(fullPackageJsonPath, (buf) =>
+      JSON.parse(buf.toString('utf-8'))
+    );
+    if (packageJson === undefined) return undefined;
+    const descriptionNotParsed = packageJson[BlockDescriptionPackageJsonField];
+    if (descriptionNotParsed === undefined) return undefined;
+    const descriptionRaw = {
+      ...BlockPackDescriptionFromPackageJsonRaw.parse(descriptionNotParsed),
+      id: {
+        ...parsePackageName(
+          notEmpty(packageJson['name'], `"name" not found in ${fullPackageJsonPath}`)
+        ),
+        version: SemVer.parse(packageJson['version'])
+      }
+    };
+    const descriptionParsingResult =
+      await ResolvedBlockPackDescriptionFromPackageJson(moduleRoot).safeParseAsync(descriptionRaw);
+    if (descriptionParsingResult.success) return descriptionParsingResult.data;
+    return undefined;
+  } catch (e: any) {
+    return undefined;
+  }
+}
+
+export async function loadPackDescriptionRaw(moduleRoot: string): Promise<BlockPackDescriptionRaw> {
+  const fullPackageJsonPath = path.resolve(moduleRoot, 'package.json');
+  const packageJson = JSON.parse(await fsp.readFile(fullPackageJsonPath, { encoding: 'utf-8' }));
+  const descriptionNotParsed = packageJson[BlockDescriptionPackageJsonField];
+  if (descriptionNotParsed === undefined)
+    throw new Error(
+      `Block description (field ${BlockDescriptionPackageJsonField}) not found in ${fullPackageJsonPath}.`
+    );
+  return {
+    ...BlockPackDescriptionFromPackageJsonRaw.parse(descriptionNotParsed),
+    id: {
+      ...parsePackageName(
+        notEmpty(packageJson['name'], `"name" not found in ${fullPackageJsonPath}`)
+      ),
+      version: SemVer.parse(packageJson['version'])
+    }
+  };
+}
+
+export async function loadPackDescription(
+  moduleRoot: string
+): Promise<BlockPackDescriptionAbsolute> {
+  const descriptionRaw = await loadPackDescriptionRaw(moduleRoot);
+  return await ResolvedBlockPackDescriptionFromPackageJson(moduleRoot).parseAsync(descriptionRaw);
+}