@fsai-flow/core 0.0.1

package/src/lib/NodesLoader/constants.ts

@@ -0,0 +1,112 @@
+import type { INodeProperties } from '@fsai-flow/workflow';
+import { cronNodeOptions } from '@fsai-flow/workflow';
+
+export const CUSTOM_NODES_CATEGORY = 'Custom Nodes';
+
+export const commonPollingParameters: INodeProperties[] = [
+	{
+		displayName: 'Poll Times',
+		name: 'pollTimes',
+		type: 'fixedCollection',
+		typeOptions: {
+			multipleValues: true,
+			multipleValueButtonText: 'Add Poll Time',
+		},
+		default: { item: [{ mode: 'everyMinute' }] },
+		description: 'Time at which polling should occur',
+		placeholder: 'Add Poll Time',
+		options: cronNodeOptions,
+	},
+];
+
+export const commonCORSParameters: INodeProperties[] = [
+	{
+		displayName: 'Allowed Origins (CORS)',
+		name: 'allowedOrigins',
+		type: 'string',
+		default: '*',
+		description:
+			'Comma-separated list of URLs allowed for cross-origin non-preflight requests. Use * (default) to allow all origins.',
+	},
+];
+
+export const commonDeclarativeNodeOptionParameters: INodeProperties = {
+	displayName: 'Request Options',
+	name: 'requestOptions',
+	type: 'collection',
+	isNodeSetting: true,
+	placeholder: 'Add Option',
+	default: {},
+	options: [
+		{
+			displayName: 'Batching',
+			name: 'batching',
+			placeholder: 'Add Batching',
+			type: 'fixedCollection',
+			typeOptions: {
+				multipleValues: false,
+			},
+			default: {
+				batch: {},
+			},
+			options: [
+				{
+					displayName: 'Batching',
+					name: 'batch',
+					values: [
+						{
+							displayName: 'Items per Batch',
+							name: 'batchSize',
+							type: 'number',
+							typeOptions: {
+								minValue: -1,
+							},
+							default: 50,
+							description:
+								'Input will be split in batches to throttle requests. -1 for disabled. 0 will be treated as 1.',
+						},
+						{
+							displayName: 'Batch Interval (ms)',
+							name: 'batchInterval',
+							type: 'number',
+							typeOptions: {
+								minValue: 0,
+							},
+							default: 1000,
+							description: 'Time (in milliseconds) between each batch of requests. 0 for disabled.',
+						},
+					],
+				},
+			],
+		},
+		{
+			displayName: 'Ignore SSL Issues (Insecure)',
+			name: 'allowUnauthorizedCerts',
+			type: 'boolean',
+			noDataExpression: true,
+			default: false,
+			description:
+				'Whether to accept the response even if SSL certificate validation is not possible',
+		},
+		{
+			displayName: 'Proxy',
+			name: 'proxy',
+			type: 'string',
+			default: '',
+			placeholder: 'e.g. http://myproxy:3128',
+			description:
+				'HTTP proxy to use. If authentication is required it can be defined as follow: http://username:password@myproxy:3128',
+		},
+		{
+			displayName: 'Timeout',
+			name: 'timeout',
+			type: 'number',
+			typeOptions: {
+				minValue: 1,
+			},
+			default: 10000,
+			description:
+				'Time in ms to wait for the server to send response headers (and start the response body) before aborting the request',
+		},
+	],
+};

package/src/lib/NodesLoader/custom-directory-loader.ts

@@ -0,0 +1,31 @@
+import glob from 'fast-glob';
+
+import { DirectoryLoader } from './directory-loader';
+
+/**
+ * Loader for source files of nodes and credentials located in a custom dir,
+ * e.g. `~/.n8n/custom`
+ */
+export class CustomDirectoryLoader extends DirectoryLoader {
+	packageName = 'CUSTOM';
+
+	override async loadAll() {
+		const nodes = await glob('**/*.node.js', {
+			cwd: this.directory,
+			absolute: true,
+		});
+
+		for (const nodePath of nodes) {
+			this.loadNodeFromFile(nodePath);
+		}
+
+		const credentials = await glob('**/*.credentials.js', {
+			cwd: this.directory,
+			absolute: true,
+		});
+
+		for (const credentialPath of credentials) {
+			this.loadCredentialFromFile(credentialPath);
+		}
+	}
+}

package/src/lib/NodesLoader/directory-loader.ts

@@ -0,0 +1,458 @@
+import uniqBy from 'lodash/uniqBy';
+import type {
+  CodexData,
+  DocumentationLink,
+  ICredentialType,
+  ICredentialTypeData,
+  INodeCredentialDescription,
+  INodePropertyOptions,
+  INodeType,
+  INodeTypeBaseDescription,
+  INodeTypeData,
+  INodeTypeDescription,
+  INodeTypeNameVersion,
+  IVersionedNodeType,
+  KnownNodesAndCredentials,
+} from '@fsai-flow/workflow';
+import { isSubNodeType } from '@fsai-flow/workflow';
+import * as path from 'path';
+
+import {
+  commonCORSParameters,
+  commonDeclarativeNodeOptionParameters,
+  commonPollingParameters,
+  CUSTOM_NODES_CATEGORY,
+} from './constants';
+import { loadClassInIsolation } from './load-class-in-isolation';
+
+function toJSON(this: ICredentialType) {
+  return {
+    ...this,
+    authenticate:
+      typeof this.authenticate === 'function' ? {} : this.authenticate,
+  };
+}
+
+type Codex = {
+  categories: string[];
+  subcategories: { [subcategory: string]: string[] };
+  resources: {
+    primaryDocumentation: DocumentationLink[];
+    credentialDocumentation: DocumentationLink[];
+  };
+  alias: string[];
+};
+
+export type Types = {
+  nodes: INodeTypeBaseDescription[];
+  credentials: ICredentialType[];
+};
+
+/**
+ * Base class for loading n8n nodes and credentials from a directory.
+ * Handles the common functionality for resolving paths, loading classes, and managing node and credential types.
+ */
+export abstract class DirectoryLoader {
+  isLazyLoaded = false;
+
+  // Another way of keeping track of the names and versions of a node. This
+  // seems to only be used by the installedPackages repository
+  loadedNodes: INodeTypeNameVersion[] = [];
+
+  // Stores the loaded descriptions and sourcepaths
+  nodeTypes: INodeTypeData = {};
+
+  credentialTypes: ICredentialTypeData = {};
+
+  // Stores the location and classnames of the nodes and credentials that are
+  // loaded; used to actually load the files in lazy-loading scenario.
+  known: KnownNodesAndCredentials = { nodes: {}, credentials: {} };
+
+  // Stores the different versions with their individual descriptions
+  types: Types = { nodes: [], credentials: [] };
+
+  readonly nodesByCredential: Record<string, string[]> = {};
+
+  constructor(
+    readonly directory: string,
+    protected excludeNodes: string[] = [],
+    protected includeNodes: string[] = [],
+  ) {}
+
+  abstract packageName: string;
+
+  abstract loadAll(): Promise<void>;
+
+  reset() {
+    this.loadedNodes = [];
+    this.nodeTypes = {};
+    this.credentialTypes = {};
+    this.known = { nodes: {}, credentials: {} };
+    this.types = { nodes: [], credentials: [] };
+  }
+
+  protected resolvePath(file: string) {
+    return path.resolve(this.directory, file);
+  }
+
+  private loadClass<T>(sourcePath: string) {
+    const filePath = this.resolvePath(sourcePath);
+    const [className] = path.parse(sourcePath).name.split('.');
+    try {
+      return loadClassInIsolation<T>(filePath, className);
+    } catch (error) {
+      throw error instanceof TypeError
+        ? new Error(
+            'Class could not be found. Please check if the class is named correctly.',
+          )
+        : error;
+    }
+  }
+
+  /** Loads a nodes class from a file, fixes icons, and augments the codex */
+  loadNodeFromFile(filePath: string) {
+    const tempNode = this.loadClass<INodeType | IVersionedNodeType>(filePath);
+    this.addCodex(tempNode, filePath);
+
+    const nodeType = tempNode.description.name;
+
+    if (this.includeNodes.length && !this.includeNodes.includes(nodeType)) {
+      return;
+    }
+
+    if (this.excludeNodes.includes(nodeType)) {
+      return;
+    }
+
+    this.fixIconPaths(tempNode.description, filePath);
+
+    let nodeVersion = 1;
+    if ('nodeVersions' in tempNode) {
+      for (const versionNode of Object.values(tempNode.nodeVersions)) {
+        this.fixIconPaths(versionNode.description, filePath);
+      }
+
+      for (const version of Object.values(tempNode.nodeVersions)) {
+        this.addLoadOptionsMethods(version);
+        this.applySpecialNodeParameters(version);
+      }
+
+      const currentVersionNode = tempNode.nodeVersions[tempNode.currentVersion];
+      this.addCodex(currentVersionNode, filePath);
+      nodeVersion = tempNode.currentVersion;
+
+      if (currentVersionNode.hasOwnProperty('executeSingle')) {
+        throw new Error(
+          '"executeSingle" has been removed. Please update the code of this node to use "execute" instead.',
+        );
+      }
+    } else {
+      this.addLoadOptionsMethods(tempNode);
+      this.applySpecialNodeParameters(tempNode);
+
+      // Short renaming to avoid type issues
+      nodeVersion = Array.isArray(tempNode.description.version)
+        ? tempNode.description.version.slice(-1)[0]
+        : tempNode.description.version;
+    }
+
+    this.known.nodes[nodeType] = {
+      className: tempNode.constructor.name,
+      sourcePath: filePath,
+    };
+
+    this.nodeTypes[nodeType] = {
+      type: tempNode,
+      sourcePath: filePath,
+    };
+
+    this.loadedNodes.push({
+      name: nodeType,
+      version: nodeVersion,
+    });
+
+    this.getVersionedNodeTypeAll(tempNode).forEach(({ description }) => {
+      this.types.nodes.push(description);
+    });
+
+    for (const credential of this.getCredentialsForNode(tempNode)) {
+      if (!this.nodesByCredential[credential.name]) {
+        this.nodesByCredential[credential.name] = [];
+      }
+      this.nodesByCredential[credential.name].push(nodeType);
+    }
+  }
+
+  getNode(nodeType: string) {
+    const {
+      nodeTypes,
+      known: { nodes: knownNodes },
+    } = this;
+    if (!(nodeType in nodeTypes) && nodeType in knownNodes) {
+      const { sourcePath } = knownNodes[nodeType];
+      this.loadNodeFromFile(sourcePath);
+    }
+
+    if (nodeType in nodeTypes) {
+      return nodeTypes[nodeType];
+    }
+
+    throw new Error(`Unrecognized node type: ${nodeType}`);
+  }
+
+  /** Loads a credential class from a file, and fixes icons */
+  loadCredentialFromFile(filePath: string): void {
+    const tempCredential = this.loadClass<ICredentialType>(filePath);
+    // Add serializer method "toJSON" to the class so that authenticate method (if defined)
+    // gets mapped to the authenticate attribute before it is sent to the client.
+    // The authenticate property is used by the client to decide whether or not to
+    // include the credential type in the predefined credentials (HTTP node)
+    Object.assign(tempCredential, { toJSON });
+
+    this.fixIconPaths(tempCredential, filePath);
+
+    const credentialType = tempCredential.name;
+    this.known.credentials[credentialType] = {
+      className: tempCredential.constructor.name,
+      sourcePath: filePath,
+      extends: tempCredential.extends,
+      supportedNodes: this.nodesByCredential[credentialType],
+    };
+
+    this.credentialTypes[credentialType] = {
+      type: tempCredential,
+      sourcePath: filePath,
+    };
+
+    this.types.credentials.push(tempCredential);
+  }
+
+  getCredential(credentialType: string) {
+    const {
+      credentialTypes,
+      known: { credentials: knownCredentials },
+    } = this;
+    if (
+      !(credentialType in credentialTypes) &&
+      credentialType in knownCredentials
+    ) {
+      const { sourcePath } = knownCredentials[credentialType];
+      this.loadCredentialFromFile(sourcePath);
+    }
+
+    if (credentialType in credentialTypes) {
+      return credentialTypes[credentialType];
+    }
+
+    throw new Error(`Unrecognized credential type: ${credentialType}`);
+  }
+
+  /**
+   * Returns an array of credential descriptions that are supported by a node.
+   * For versioned nodes, combines and deduplicates credentials from all versions.
+   */
+  getCredentialsForNode(
+    object: IVersionedNodeType | INodeType,
+  ): INodeCredentialDescription[] {
+    if ('nodeVersions' in object) {
+      const credentials = Object.values(object.nodeVersions).flatMap(
+        ({ description }) => description.credentials ?? [],
+      );
+      return uniqBy(credentials, 'name');
+    }
+    return object.description.credentials ?? [];
+  }
+
+  /**
+   * Returns an array of all versions of a node type.
+   * For non-versioned nodes, returns an array with just that node.
+   * For versioned nodes, returns all available versions.
+   */
+  getVersionedNodeTypeAll(object: IVersionedNodeType | INodeType): INodeType[] {
+    if ('nodeVersions' in object) {
+      const nodeVersions = Object.values(object.nodeVersions).map((element) => {
+        element.description.name = object.description.name;
+        element.description.codex = object.description.codex;
+        return element;
+      });
+      return uniqBy(nodeVersions.reverse(), (node) => {
+        const { version } = node.description;
+        return Array.isArray(version) ? version.join(',') : version.toString();
+      });
+    }
+    return [object];
+  }
+
+  /**
+   * Retrieves `categories`, `subcategories` and alias (if defined)
+   * from the codex data for the node at the given file path.
+   */
+  private getCodex(filePath: string): CodexData {
+    const codexFilePath = this.resolvePath(`${filePath}on`); // .js to .json
+
+    const {
+      categories,
+      subcategories,
+      resources: { primaryDocumentation, credentialDocumentation },
+      alias,
+    } = module.require(codexFilePath) as Codex;
+
+    return {
+      ...(categories && { categories }),
+      ...(subcategories && { subcategories }),
+      ...(alias && { alias }),
+      resources: {
+        primaryDocumentation,
+        credentialDocumentation,
+      },
+    };
+  }
+
+  /**
+   * Adds a node codex `categories` and `subcategories` (if defined)
+   * to a node description `codex` property.
+   */
+  private addCodex(node: INodeType | IVersionedNodeType, filePath: string) {
+    const isCustom = this.packageName === 'CUSTOM';
+    try {
+      let codex;
+
+      if (!isCustom) {
+        codex = node.description.codex;
+      }
+
+      if (codex === undefined) {
+        codex = this.getCodex(filePath);
+      }
+
+      if (isCustom) {
+        codex.categories = codex.categories
+          ? codex.categories.concat(CUSTOM_NODES_CATEGORY)
+          : [CUSTOM_NODES_CATEGORY];
+      }
+
+      node.description.codex = codex;
+    } catch {
+      console.debug(`No codex available for: ${node.description.name}`);
+
+      if (isCustom) {
+        node.description.codex = {
+          categories: [CUSTOM_NODES_CATEGORY],
+        };
+      }
+    }
+  }
+
+  private addLoadOptionsMethods(node: INodeType) {
+    if (node?.methods?.loadOptions) {
+      node.description.__loadOptionsMethods = Object.keys(
+        node.methods.loadOptions,
+      );
+    }
+  }
+
+  private applySpecialNodeParameters(nodeType: INodeType): void {
+    const { properties, polling, supportsCORS } = nodeType.description;
+    if (polling) {
+      properties.unshift(...commonPollingParameters);
+    }
+    if (nodeType.webhook && supportsCORS) {
+      const optionsProperty = properties.find(({ name }) => name === 'options');
+      if (optionsProperty)
+        optionsProperty.options = [
+          ...commonCORSParameters,
+          ...(optionsProperty.options as INodePropertyOptions[]),
+        ];
+      else properties.push(...commonCORSParameters);
+    }
+
+    DirectoryLoader.applyDeclarativeNodeOptionParameters(nodeType);
+  }
+
+  private getIconPath(icon: string, filePath: string) {
+    const iconPath = path.join(
+      path.dirname(filePath),
+      icon.replace('file:', ''),
+    );
+    return `icons/${this.packageName}/${iconPath}`;
+  }
+
+  private fixIconPaths(
+    obj: INodeTypeDescription | INodeTypeBaseDescription | ICredentialType,
+    filePath: string,
+  ) {
+    const { icon } = obj;
+    if (!icon) return;
+
+    if (typeof icon === 'string') {
+      if (icon.startsWith('file:')) {
+        obj.iconUrl = this.getIconPath(icon, filePath);
+        obj.icon = undefined;
+      }
+    } else if (
+      icon.light.startsWith('file:') &&
+      icon.dark.startsWith('file:')
+    ) {
+      obj.iconUrl = {
+        light: this.getIconPath(icon.light, filePath),
+        dark: this.getIconPath(icon.dark, filePath),
+      };
+      obj.icon = undefined;
+    }
+  }
+
+  /** Augments additional `Request Options` property on declarative node-type */
+  static applyDeclarativeNodeOptionParameters(nodeType: INodeType): void {
+    if (
+      !!nodeType.execute ||
+      !!nodeType.trigger ||
+      !!nodeType.webhook ||
+      !!nodeType.description.polling ||
+      isSubNodeType(nodeType.description)
+    ) {
+      return;
+    }
+
+    const parameters = nodeType.description.properties;
+    if (!parameters) {
+      return;
+    }
+
+    // Was originally under "options" instead of "requestOptions" so the chance
+    // that that existed was quite high. With this name the chance is actually
+    // very low that it already exists but lets leave it in anyway to be sure.
+    const existingRequestOptionsIndex = parameters.findIndex(
+      (parameter) => parameter.name === 'requestOptions',
+    );
+    if (existingRequestOptionsIndex !== -1) {
+      parameters[existingRequestOptionsIndex] = {
+        ...commonDeclarativeNodeOptionParameters,
+        options: [
+          ...(commonDeclarativeNodeOptionParameters.options ?? []),
+          ...(parameters[existingRequestOptionsIndex]?.options ?? []),
+        ],
+      };
+
+      const options = parameters[existingRequestOptionsIndex]?.options;
+
+      if (options) {
+        options.sort((a, b) => {
+          if ('displayName' in a && 'displayName' in b) {
+            if (a.displayName < b.displayName) {
+              return -1;
+            }
+            if (a.displayName > b.displayName) {
+              return 1;
+            }
+          }
+
+          return 0;
+        });
+      }
+    } else {
+      parameters.push(commonDeclarativeNodeOptionParameters);
+    }
+
+    return;
+  }
+}

package/src/lib/NodesLoader/index.ts

@@ -0,0 +1,5 @@
+export { DirectoryLoader, type Types } from './directory-loader';
+export { CustomDirectoryLoader } from './custom-directory-loader';
+export { PackageDirectoryLoader } from './package-directory-loader';
+export { LazyPackageDirectoryLoader } from './lazy-package-directory-loader';
+export type { n8n } from './types';

package/src/lib/NodesLoader/lazy-package-directory-loader.ts

@@ -0,0 +1,55 @@
+import { PackageDirectoryLoader } from './package-directory-loader';
+
+/**
+ * This loader extends PackageDirectoryLoader to load node and credentials lazily, if possible
+ */
+export class LazyPackageDirectoryLoader extends PackageDirectoryLoader {
+	override async loadAll() {
+		try {
+			this.known.nodes = await this.readJSON('dist/known/nodes.json');
+			this.known.credentials = await this.readJSON('dist/known/credentials.json');
+
+			this.types.nodes = await this.readJSON('dist/types/nodes.json');
+			this.types.credentials = await this.readJSON('dist/types/credentials.json');
+
+			if (this.includeNodes.length) {
+				const allowedNodes: typeof this.known.nodes = {};
+				for (const nodeType of this.includeNodes) {
+					if (nodeType in this.known.nodes) {
+						allowedNodes[nodeType] = this.known.nodes[nodeType];
+					}
+				}
+				this.known.nodes = allowedNodes;
+
+				this.types.nodes = this.types.nodes.filter((nodeType) =>
+					this.includeNodes.includes(nodeType.name),
+				);
+			}
+
+			if (this.excludeNodes.length) {
+				for (const nodeType of this.excludeNodes) {
+					delete this.known.nodes[nodeType];
+				}
+
+				this.types.nodes = this.types.nodes.filter(
+					(nodeType) => !this.excludeNodes.includes(nodeType.name),
+				);
+			}
+
+			console.debug(
+				`Lazy-loading nodes and credentials from ${this.packageJson.name}`,
+				{
+					nodes: this.types.nodes?.length ?? 0,
+					credentials: this.types.credentials?.length ?? 0,
+				},
+			);
+
+			this.isLazyLoaded = true;
+
+			return; // We can load nodes and credentials lazily now
+		} catch {
+			console.debug("Can't enable lazy-loading");
+			await super.loadAll();
+		}
+	}
+}

package/src/lib/NodesLoader/load-class-in-isolation.ts

@@ -0,0 +1,19 @@
+import { createContext, Script } from 'vm';
+
+import { inTest } from '../Constants';
+
+const context = createContext({ require });
+export const loadClassInIsolation = <T>(filePath: string, className: string) => {
+	if (process.platform === 'win32') {
+		filePath = filePath.replace(/\\/g, '/');
+	}
+
+	// Note: Skip the isolation because it breaks nock mocks in tests
+	if (inTest) {
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-var-requires, @typescript-eslint/no-unsafe-member-access
+		return new (require(filePath)[className])() as T;
+	} else {
+		const script = new Script(`new (require('${filePath}').${className})()`);
+		return script.runInContext(context) as T;
+	}
+};