@superblocksteam/util 2.0.162 → 2.0.163-next.0

package/src/api-files/api.mts

@@ -1,506 +0,0 @@
-import fs from "fs-extra";
-import { cloneDeep } from "lodash-es";
-import slugify from "slugify";
-import { parse, stringify as ymlstringify } from "yaml";
-
-import {
-  DEFAULT_LINES_FOR_LARGE_STEPS,
-  LANGUAGE_STEP_EXTENSIONS,
-} from "./constants.js";
-import type {
-  AdditionalStepFiles,
-  Api,
-  ApiRepresentation,
-  ApiWrapper,
-  Block,
-  LanguageContent,
-  Step,
-} from "./types.js";
-
-function slugifyName(originalName: any): string {
-  return (slugify as any)(originalName, {
-    replacement: "_",
-    lower: true,
-  });
-}
-
-function findUnusedFilePath(
-  filename: string,
-  extension: string,
-  existingFilePaths: Set<string>,
-): string {
-  let proposedFilename = `${filename}.${extension}`;
-  let i = 1;
-  while (existingFilePaths.has(proposedFilename)) {
-    proposedFilename = `${filename}_${i}.${extension}`;
-    i++;
-  }
-  existingFilePaths.add(proposedFilename);
-  return proposedFilename;
-}
-
-function extractAdditionalStepFiles(
-  api: Api,
-  additionalStepFiles: AdditionalStepFiles[],
-  stepPathMap: Record<string, string>,
-  currentStepNames: string[],
-  apiRepresentation?: ApiRepresentation,
-): Api {
-  if (!api.blocks || !apiRepresentation?.extractLargeSourceFiles) {
-    return api;
-  }
-  // TODO: lodash
-  const apiClone = cloneDeep(api);
-  const minLinesForExtraction =
-    apiRepresentation?.minLinesForExtraction ?? DEFAULT_LINES_FOR_LARGE_STEPS;
-  const existingFilePaths = new Set<string>();
-  extractAdditionalStepFilesFromBlocks(
-    apiClone.blocks ?? [],
-    additionalStepFiles,
-    existingFilePaths,
-    minLinesForExtraction,
-    stepPathMap,
-    currentStepNames,
-  );
-  return apiClone;
-}
-
-function extractAdditionalStepFilesFromStep(
-  step: Step,
-  name: string,
-  additionalStepFiles: AdditionalStepFiles[],
-  existingFilePaths: Set<string>,
-  minLinesForExtraction: number,
-  stepPathMap: Record<string, string>,
-  currentStepNames: string[],
-) {
-  // Handle language step files
-  if (step && "integration" in step) {
-    const integration = step.integration as keyof Step;
-    const extension =
-      LANGUAGE_STEP_EXTENSIONS[
-        integration as keyof typeof LANGUAGE_STEP_EXTENSIONS
-      ];
-    if (extension) {
-      const languageContent = step[integration] as LanguageContent;
-      if (languageContent) {
-        const blockBody = languageContent.body;
-        if (blockBody && typeof blockBody === "string") {
-          const lines = blockBody?.split("\n");
-          const linesNum = lines?.length;
-          if (linesNum && linesNum > minLinesForExtraction) {
-            // Record the separate file is needed, with the language contents
-            let relativePath;
-            if (stepPathMap?.[name]) {
-              relativePath = stepPathMap[name].startsWith("./")
-                ? stepPathMap[name].split("./")?.[1]
-                : stepPathMap[name];
-            } else {
-              relativePath = findUnusedFilePath(
-                slugifyName(name),
-                extension,
-                existingFilePaths,
-              );
-              stepPathMap[name] = `./${relativePath}`;
-            }
-            additionalStepFiles.push({
-              relativePath: relativePath,
-              contents: blockBody,
-              language: integration,
-            });
-            currentStepNames.push(name);
-            // Replace the body with a reference to the separate file
-            languageContent.body = {
-              path: `./${relativePath}`,
-            };
-          }
-        }
-      }
-    }
-  }
-}
-
-function extractAdditionalStepFilesFromBlocks(
-  blocks: Block[],
-  additionalStepFiles: AdditionalStepFiles[],
-  existingFilePaths: Set<string>,
-  minLinesForExtraction: number,
-  stepPathMap: Record<string, string>,
-  currentStepNames: string[],
-) {
-  for (const block of blocks) {
-    // Handle language step files
-    if (block.step) {
-      extractAdditionalStepFilesFromStep(
-        block.step,
-        block.name,
-        additionalStepFiles,
-        existingFilePaths,
-        minLinesForExtraction,
-        stepPathMap,
-        currentStepNames,
-      );
-    }
-
-    // Handle loops
-    extractAdditionalStepFilesFromBlocks(
-      block.loop?.blocks ?? [],
-      additionalStepFiles,
-      existingFilePaths,
-      minLinesForExtraction,
-      stepPathMap,
-      currentStepNames,
-    );
-    // Handle try/catch
-    if (block.tryCatch) {
-      extractAdditionalStepFilesFromBlocks(
-        block.tryCatch.try?.blocks ?? [],
-        additionalStepFiles,
-        existingFilePaths,
-        minLinesForExtraction,
-        stepPathMap,
-        currentStepNames,
-      );
-      extractAdditionalStepFilesFromBlocks(
-        block.tryCatch.catch?.blocks ?? [],
-        additionalStepFiles,
-        existingFilePaths,
-        minLinesForExtraction,
-        stepPathMap,
-        currentStepNames,
-      );
-      extractAdditionalStepFilesFromBlocks(
-        block.tryCatch.finally?.blocks ?? [],
-        additionalStepFiles,
-        existingFilePaths,
-        minLinesForExtraction,
-        stepPathMap,
-        currentStepNames,
-      );
-    }
-    // Handle conditional
-    if (block.conditional) {
-      extractAdditionalStepFilesFromBlocks(
-        block.conditional.if?.blocks ?? [],
-        additionalStepFiles,
-        existingFilePaths,
-        minLinesForExtraction,
-        stepPathMap,
-        currentStepNames,
-      );
-      for (const elseIfBlock of block.conditional.elseIf ?? []) {
-        extractAdditionalStepFilesFromBlocks(
-          elseIfBlock.blocks,
-          additionalStepFiles,
-          existingFilePaths,
-          minLinesForExtraction,
-          stepPathMap,
-          currentStepNames,
-        );
-      }
-      extractAdditionalStepFilesFromBlocks(
-        block.conditional.else?.blocks ?? [],
-        additionalStepFiles,
-        existingFilePaths,
-        minLinesForExtraction,
-        stepPathMap,
-        currentStepNames,
-      );
-    }
-    // Handle Parallel API
-    if (block.parallel) {
-      for (const path of Object.values(block.parallel?.static?.paths ?? {})) {
-        extractAdditionalStepFilesFromBlocks(
-          path.blocks,
-          additionalStepFiles,
-          existingFilePaths,
-          minLinesForExtraction,
-          stepPathMap,
-          currentStepNames,
-        );
-      }
-      extractAdditionalStepFilesFromBlocks(
-        block.parallel?.dynamic?.blocks ?? [],
-        additionalStepFiles,
-        existingFilePaths,
-        minLinesForExtraction,
-        stepPathMap,
-        currentStepNames,
-      );
-    }
-    // Handle stream
-    if (block.stream) {
-      const trigger = block.stream.trigger;
-      if (trigger && trigger.step) {
-        extractAdditionalStepFilesFromStep(
-          trigger.step,
-          trigger.name,
-          additionalStepFiles,
-          existingFilePaths,
-          minLinesForExtraction,
-          stepPathMap,
-          currentStepNames,
-        );
-      }
-      extractAdditionalStepFilesFromBlocks(
-        block.stream.process?.blocks ?? [],
-        additionalStepFiles,
-        existingFilePaths,
-        minLinesForExtraction,
-        stepPathMap,
-        currentStepNames,
-      );
-    }
-  }
-}
-
-async function readYamlFile(path: string): Promise<any> {
-  return parse(await fs.readFile(path, "utf8"));
-}
-
-export async function resolveLanguageSpecificStepContentFromBlocks(
-  parentPath: string,
-  blocks: Block[],
-  stepPathMap: Record<string, string>,
-) {
-  for (const block of blocks) {
-    // Handle language-specific step content
-    if (block.step) {
-      const step = block.step;
-      await resolveLanguageSpecificStepContentFromStep(
-        parentPath,
-        step,
-        block.name,
-        stepPathMap,
-      );
-    }
-    // Handle conditional blocks
-    if (block.conditional) {
-      await resolveLanguageSpecificStepContentFromBlocks(
-        parentPath,
-        block.conditional.if?.blocks ?? [],
-        stepPathMap,
-      );
-      for (const elseIfBlock of block.conditional.elseIf) {
-        await resolveLanguageSpecificStepContentFromBlocks(
-          parentPath,
-          elseIfBlock.blocks,
-          stepPathMap,
-        );
-      }
-      await resolveLanguageSpecificStepContentFromBlocks(
-        parentPath,
-        block.conditional.else?.blocks ?? [],
-        stepPathMap,
-      );
-    }
-    // Handle loop blocks
-    await resolveLanguageSpecificStepContentFromBlocks(
-      parentPath,
-      block.loop?.blocks ?? [],
-      stepPathMap,
-    );
-    // Handle try-catch blocks
-    if (block.tryCatch) {
-      await resolveLanguageSpecificStepContentFromBlocks(
-        parentPath,
-        block.tryCatch.try?.blocks ?? [],
-        stepPathMap,
-      );
-      await resolveLanguageSpecificStepContentFromBlocks(
-        parentPath,
-        block.tryCatch.catch?.blocks ?? [],
-        stepPathMap,
-      );
-      await resolveLanguageSpecificStepContentFromBlocks(
-        parentPath,
-        block.tryCatch.finally?.blocks ?? [],
-        stepPathMap,
-      );
-    }
-    // Handle parallel blocks
-    if (block.parallel) {
-      for (const path of Object.values(block.parallel?.static?.paths ?? {})) {
-        await resolveLanguageSpecificStepContentFromBlocks(
-          parentPath,
-          path.blocks,
-          stepPathMap,
-        );
-      }
-      await resolveLanguageSpecificStepContentFromBlocks(
-        parentPath,
-        block.parallel?.dynamic?.blocks ?? [],
-        stepPathMap,
-      );
-    }
-    // Handle stream blocks
-    if (block.stream) {
-      await resolveLanguageSpecificStepContentFromBlocks(
-        parentPath,
-        block.stream.process?.blocks ?? [],
-        stepPathMap,
-      );
-      await resolveLanguageSpecificStepContentFromStep(
-        parentPath,
-        block.stream.trigger?.step ?? ({} as Step),
-        block.stream.trigger?.name,
-        stepPathMap,
-      );
-    }
-  }
-}
-
-async function resolveLanguageSpecificStepContentFromStep(
-  parentPath: string,
-  step: Step,
-  stepName?: string,
-  stepPathMap?: Record<string, string>,
-) {
-  // Handle language-specific step content
-  if (
-    !step?.integration ||
-    !LANGUAGE_STEP_EXTENSIONS[
-      step.integration as keyof typeof LANGUAGE_STEP_EXTENSIONS
-    ]
-  ) {
-    return;
-  }
-  const languageContent = step[
-    step.integration as keyof Step
-  ] as LanguageContent;
-  if (
-    languageContent.body &&
-    typeof languageContent.body === "object" &&
-    "path" in languageContent.body
-  ) {
-    const languageFilePath = `${parentPath}/${languageContent.body.path}`;
-    if (await fs.pathExists(languageFilePath)) {
-      // Read the file content
-      const stepBodyContent = await fs.readFile(languageFilePath, "utf8");
-      if (stepBodyContent && languageContent) {
-        if (stepPathMap && stepName) {
-          stepPathMap[stepName] = languageContent.body.path;
-        }
-        languageContent.body = stepBodyContent;
-      }
-    }
-  }
-}
-
-export async function writeApiFiles(
-  api: ApiWrapper,
-  apiNameSlug: string,
-  directoryPath: string,
-  nestedApiFiles: boolean,
-  apiPromises: Array<Promise<void>>,
-  additionalStepFiles: AdditionalStepFiles[],
-  apiRepresentation?: ApiRepresentation,
-  existingFilePaths?: Set<string>,
-  stepPathMap?: Record<string, string>,
-) {
-  let pathForApiFile = directoryPath;
-  let yamlFileName = `${apiNameSlug}.yaml`;
-
-  const currentStepNames: string[] = [];
-  // Determine whether to extract into separate files  the source code from the steps
-  const updatedApi = extractAdditionalStepFiles(
-    api.apiPb,
-    additionalStepFiles,
-    stepPathMap ?? {},
-    currentStepNames,
-    apiRepresentation,
-  );
-
-  // Stop tracking unused step paths from removed steps (e.g. when a step is renamed or removed)
-  if (stepPathMap) {
-    Object.keys(stepPathMap).forEach((stepName) => {
-      if (!currentStepNames.includes(stepName)) {
-        const stepFileName = stepPathMap[stepName].startsWith("./")
-          ? stepPathMap[stepName].split("./")[1]
-          : stepPathMap[stepName];
-        fs.remove(`${pathForApiFile}/${stepFileName}`);
-        delete stepPathMap[stepName];
-      }
-    });
-  }
-
-  if (apiRepresentation?.extractLargeSourceFiles) {
-    if (nestedApiFiles) {
-      // This API has at least one large step code file, so we need a nested directory
-      pathForApiFile = `${pathForApiFile}/${apiNameSlug}`;
-      yamlFileName = "api.yaml";
-    }
-    // Make sure the directory exists
-    await fs.ensureDir(pathForApiFile);
-    // Write any additional step source code files to the disk
-    for (const stepFile of additionalStepFiles) {
-      const sourceFilePath = `${pathForApiFile}/${stepFile.relativePath}`;
-      const handleApi = async () => {
-        await fs.outputFile(sourceFilePath, stepFile.contents);
-      };
-      apiPromises.push(handleApi());
-      // Mark the existing source file as being reused
-      existingFilePaths?.delete(sourceFilePath);
-    }
-  } else {
-    // No additional step files to write, so mark the API directory as existing so that it will be removed later.
-    // Do this even if the current representation does not extract large source files, in case they were previously
-    existingFilePaths?.add(`${directoryPath}/${apiNameSlug}`);
-  }
-
-  // Convert the updated API to YAML and write it to disk
-  const apiContent = ymlstringify(updatedApi, {
-    sortMapEntries: true,
-    blockQuote: "literal",
-  });
-  const yamlFilePath = `${pathForApiFile}/${yamlFileName}`;
-  const handleApi = async () => {
-    await fs.outputFile(yamlFilePath, apiContent);
-  };
-  apiPromises.push(handleApi());
-  // Mark the existing source file as being reused
-  existingFilePaths?.delete(yamlFilePath);
-}
-
-export async function readAppApiYamlFile(
-  parentDirectory: string,
-  apiName?: string,
-  slugifyDirectoryName: boolean = true,
-): Promise<{ api: Api; stepPathMap: Record<string, string> }> {
-  // The API is either stored in its entirety in a single YAML file, or
-  // or in a subdirectory containing the YAML file and zero or more language-specific files.
-  const apiNameSlug = slugifyName(apiName ?? "api");
-  const singularApiYamlPath = `${parentDirectory}/${apiNameSlug}.yaml`;
-  const apiDirPath = `${parentDirectory}/${slugifyDirectoryName ? apiNameSlug : apiName}`;
-  const nestedApiYamlPath = `${apiDirPath}/api.yaml`;
-
-  // Read the YAML file
-  let yamlPath: string;
-  let yamlParentPath: string;
-  if (await fs.pathExists(singularApiYamlPath)) {
-    // The API YAML file is in the parent directory
-    yamlPath = singularApiYamlPath;
-    yamlParentPath = parentDirectory;
-  } else if (await fs.pathExists(nestedApiYamlPath)) {
-    // The API YAML file is nested in an API-specific directory
-    yamlPath = nestedApiYamlPath;
-    yamlParentPath = apiDirPath;
-  } else {
-    throw new Error(`API ${apiName ?? ""} not found at ${parentDirectory}`);
-  }
-
-  // That is nested in an API-specific directory
-  const apiDefn = (await readYamlFile(yamlPath)) as Api;
-
-  // Map step name -> step path
-  const stepPathMap: Record<string, string> = {};
-
-  // The API YAML file may or may not have language-specific content in separate files.
-  // Replace any file references with the actual content
-  await resolveLanguageSpecificStepContentFromBlocks(
-    yamlParentPath,
-    apiDefn.blocks ?? [],
-    stepPathMap,
-  );
-  return { api: apiDefn, stepPathMap };
-}

package/src/api-files/constants.ts

@@ -1,6 +0,0 @@
-export const LANGUAGE_STEP_EXTENSIONS = {
-  javascript: "js",
-  python: "py",
-} as const;
-
-export const DEFAULT_LINES_FOR_LARGE_STEPS = 1;

package/src/api-files/index.ts

@@ -1,22 +0,0 @@
-import type * as Api from "./api.mjs" with { "resolution-mode": "import" };
-
-export async function writeApiFiles(
-  ...args: Parameters<typeof Api.writeApiFiles>
-) {
-  const esmFunction = await import("./api.mjs");
-  return esmFunction.writeApiFiles(...args);
-}
-
-export async function readAppApiYamlFile(
-  ...args: Parameters<typeof Api.readAppApiYamlFile>
-) {
-  const esmFunction = await import("./api.mjs");
-  return esmFunction.readAppApiYamlFile(...args);
-}
-
-export async function resolveLanguageSpecificStepContentFromBlocks(
-  ...args: Parameters<typeof Api.resolveLanguageSpecificStepContentFromBlocks>
-) {
-  const esmFunction = await import("./api.mjs");
-  return esmFunction.resolveLanguageSpecificStepContentFromBlocks(...args);
-}

package/src/api-files/types.ts

@@ -1,131 +0,0 @@
-export type AdditionalStepFiles = {
-  relativePath: string;
-  contents: string;
-  language: keyof Step;
-};
-
-export type ApiRepresentation = {
-  extractLargeSourceFiles: boolean;
-  minLinesForExtraction: number;
-};
-
-export interface ApiWrapper {
-  apiPb: Api;
-  name?: string; //@deprecated this attribute is used for getting a name for backends(jobs and workflows) that were never migrated to the new API version.
-}
-
-/** A signature, as produced by the agent. */
-export interface Signature {
-  /** The id of the key used to sign the data. */
-  keyId: string;
-  /** The actual signature, in base64. */
-  data: string;
-}
-
-export type Api = {
-  metadata: {
-    id: string;
-    name: string;
-    organization: string;
-    timestamps?: {
-      created: string;
-      updated: string;
-      deactivated: boolean;
-    };
-    // These properties are merged in from the v3 api entity
-    creator?: {
-      id: string;
-      name: string;
-    };
-    folder?: string;
-  };
-  blocks?: Block[];
-  trigger: any;
-  signature?: Signature;
-};
-
-export type Block = {
-  name: string;
-  step?: Step;
-  loop?: Loop;
-  tryCatch?: TryCatch;
-  conditional?: Conditional;
-  parallel?: Parallel;
-  stream?: Stream;
-};
-
-export type Step = {
-  integration: string;
-  javascript?: LanguageContent;
-  python?: LanguageContent;
-};
-
-type Loop = {
-  blocks: Block[];
-  range: string;
-  type: LoopType;
-  variables?: Variables;
-};
-
-export enum LoopType {
-  FOR_EACH = "TYPE_FOREACH",
-}
-
-type TryCatch = {
-  try?: Blocks;
-  catch?: Blocks;
-  finally?: Blocks;
-  variables?: Variables;
-};
-
-type Conditional = {
-  if?: ConditionalBlockWithCondition;
-  elseIf: ConditionalBlockWithCondition[];
-  else?: Blocks;
-};
-
-type Blocks = {
-  blocks: Block[];
-};
-
-type ConditionalBlockWithCondition = Blocks & {
-  condition?: string;
-};
-
-type Parallel = {
-  static?: {
-    paths: Record<string, Blocks>;
-  };
-  dynamic?: {
-    blocks: Block[];
-    variables?: Variables;
-    paths?: string;
-  };
-  poolSize?: number;
-  wait?: WAIT_TYPE;
-};
-
-type Stream = {
-  trigger?: StreamTrigger;
-  process?: Blocks;
-  variables?: Variables;
-};
-
-type StreamTrigger = {
-  name: string;
-  step?: Step;
-};
-
-enum WAIT_TYPE {
-  ALL = "WAIT_ALL",
-  ANY = "WAIT_NONE",
-  UNSPECIFIED = "WAIT_UNSPECIFIED",
-}
-
-type Variables = Record<string, string>;
-
-export type LanguageContent = {
-  body: Body;
-};
-
-type Body = string | { path: string };