@syncbridge/common 0.4.0

package/registry.js

@@ -0,0 +1,205 @@
+import * as fs from 'node:fs';
+import path from 'node:path';
+import process from 'node:process';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { deepClone, isPlainObject } from '@jsopen/objects';
+import { COMPONENT_OPTIONS, PROCESSOR_OPTIONS } from './constants.js';
+import { isExtensionPackage } from './utils/make-extension-package.js';
+export var ExtensionRegistry;
+(function (ExtensionRegistry) {
+    let _resolver = import.meta.resolve;
+    ExtensionRegistry.packages = new Map();
+    ExtensionRegistry.components = new Map();
+    ExtensionRegistry.processors = new Map();
+    function setResolver(resolver) {
+        _resolver = resolver;
+    }
+    ExtensionRegistry.setResolver = setResolver;
+    function getProcessor(className) {
+        const rec = ExtensionRegistry.processors.get(className);
+        if (!rec) {
+            throw new Error(`Processor "${className}" not found in registry`);
+        }
+        return rec;
+    }
+    ExtensionRegistry.getProcessor = getProcessor;
+    function getComponent(className) {
+        const rec = ExtensionRegistry.components.get(className);
+        if (!rec) {
+            throw new Error(`Component "${className}" not found in registry`);
+        }
+        return rec;
+    }
+    ExtensionRegistry.getComponent = getComponent;
+    async function registerPackage(...specifiers) {
+        const errors = [];
+        await Promise.all(specifiers.map(specifier => _registerPackage(specifier).catch(err => {
+            errors.push(`\n    ${specifier}: ${err.message}`);
+        })));
+        if (errors.length) {
+            console.error(`Failed to register extensions:${errors}`);
+        }
+    }
+    ExtensionRegistry.registerPackage = registerPackage;
+    async function _registerPackage(specifier, options) {
+        let packagePath = fileURLToPath(_resolver(specifier));
+        const json = locatePkgJson(path.dirname(packagePath));
+        if (!json) {
+            if (options?.ignoreInvalid)
+                return;
+            throw new TypeError(`Can't locate package.json file for "${specifier}"`);
+        }
+        if (path.isAbsolute(packagePath) && process.platform === 'win32') {
+            packagePath = pathToFileURL(packagePath).href;
+        }
+        const pkg = (await import(packagePath)).default;
+        if (!isExtensionPackage(pkg)) {
+            if (options?.ignoreInvalid)
+                return;
+            throw new TypeError(`"${specifier}" do not export ExtensionPackage interface`);
+        }
+        // ignore if already registered
+        if (ExtensionRegistry.packages.get(json.name))
+            return;
+        // Register dependencies
+        if (pkg.dependencies) {
+            for (const dep of pkg.dependencies)
+                await _registerPackage(dep, { ignoreInvalid: true });
+        }
+        const packageRec = Object.freeze({
+            name: json.name,
+            version: json.version,
+            description: json.description,
+        });
+        ExtensionRegistry.packages.set(packageRec.name, packageRec);
+        if (Array.isArray(pkg.components)) {
+            for (const ctor of pkg.components) {
+                await registerComponent(ctor, packageRec.name);
+            }
+        }
+        if (Array.isArray(pkg.processors)) {
+            for (const ctor of pkg.processors) {
+                await registerProcessor(ctor, packageRec.name);
+            }
+        }
+    }
+    async function registerComponent(ctor, packageName) {
+        const _metadata = Reflect.getMetadata(COMPONENT_OPTIONS, ctor);
+        if (!_metadata)
+            throw new TypeError(`Class "${ctor.name}" has no component metadata.`);
+        if (_metadata.abstract)
+            return;
+        let componentRec = ExtensionRegistry.components.get(_metadata.className);
+        if (componentRec)
+            throw new TypeError(`Component "${_metadata.className}" already registered by "${componentRec.package}" package`);
+        let metadata = await resolvePromisesDeep(_metadata);
+        metadata = deepClone({
+            className: metadata.className,
+            displayName: metadata.displayName,
+            description: metadata.description,
+            iconUrl: metadata.iconUrl,
+            author: metadata.author,
+            interfaces: metadata.interfaces,
+            tags: metadata.tags,
+            abstract: metadata.abstract,
+            variables: metadata.variables,
+            components: metadata.components,
+            ...metadata,
+        });
+        if (metadata.variables) {
+            /** Convert object enumValues to arrays */
+            for (const v of Object.values(metadata.variables)) {
+                if (v.enumValues &&
+                    typeof v.enumValues === 'object' &&
+                    !Array.isArray(v.enumValues)) {
+                    let values = v.enumValues;
+                    const keys = Object.keys(values).filter(k => !/^\d+$/.test(k));
+                    values = keys.reduce((a, k) => {
+                        if (values[k] != null)
+                            a.push(values[k]);
+                        return a;
+                        // v => !(typeof v === 'number' && !keys.includes(String(v))),
+                    }, []);
+                    v.enumValues = values;
+                }
+            }
+        }
+        componentRec = {
+            package: packageName,
+            ctor,
+            metadata,
+        };
+        ExtensionRegistry.components.set(metadata.className, componentRec);
+    }
+    ExtensionRegistry.registerComponent = registerComponent;
+    async function registerProcessor(ctor, packageName) {
+        const _metadata = Reflect.getMetadata(PROCESSOR_OPTIONS, ctor);
+        if (!_metadata)
+            throw new TypeError(`Class "${ctor.name}" has no processor metadata.`);
+        if (_metadata.abstract)
+            return;
+        let processorRec = ExtensionRegistry.processors.get(_metadata.className);
+        if (processorRec)
+            throw new TypeError(`Processor "${_metadata.className}" already registered by "${processorRec.package}" package`);
+        let metadata = await resolvePromisesDeep(_metadata);
+        metadata = deepClone({
+            className: metadata.className,
+            displayName: metadata.displayName,
+            description: metadata.description,
+            iconUrl: metadata.iconUrl,
+            author: metadata.author,
+            tags: metadata.tags,
+            abstract: metadata.abstract,
+            variables: metadata.variables,
+            components: metadata.components,
+            ...metadata,
+        });
+        if (metadata.variables) {
+            /** Convert object enumValues to arrays */
+            for (const v of Object.values(metadata.variables)) {
+                if (v.enumValues &&
+                    typeof v.enumValues === 'object' &&
+                    !Array.isArray(v.enumValues)) {
+                    let values = v.enumValues;
+                    const keys = Object.keys(values).filter(k => !/^\d+$/.test(k));
+                    values = keys.reduce((a, k) => {
+                        if (values[k] != null)
+                            a.push(values[k]);
+                        return a;
+                        // v => !(typeof v === 'number' && !keys.includes(String(v))),
+                    }, []);
+                    v.enumValues = values;
+                }
+            }
+        }
+        processorRec = {
+            package: packageName,
+            ctor,
+            metadata,
+        };
+        ExtensionRegistry.processors.set(metadata.className, processorRec);
+    }
+    ExtensionRegistry.registerProcessor = registerProcessor;
+})(ExtensionRegistry || (ExtensionRegistry = {}));
+function locatePkgJson(directory) {
+    if (directory.startsWith('file:'))
+        directory = fileURLToPath(directory);
+    for (let i = 0; i < 3; i++) {
+        const f = path.resolve(directory, 'package.json');
+        if (fs.existsSync(f)) {
+            const json = JSON.parse(fs.readFileSync(f, 'utf8'));
+            if (json.name && json.version)
+                return json;
+        }
+        directory = path.dirname(directory);
+    }
+}
+async function resolvePromisesDeep(obj) {
+    obj = await obj;
+    if (obj && isPlainObject(obj)) {
+        for (const k of Object.keys(obj)) {
+            obj[k] = await resolvePromisesDeep(obj[k]);
+        }
+    }
+    return obj;
+}

package/utils/encrypt-helpers.d.ts

@@ -0,0 +1,2 @@
+export declare const secretEncryptor: (secretKey: string) => import("valgen").Validator<string, string, import("valgen").ExecutionOptions>;
+export declare const secretDecryptor: (secretKey: string) => import("valgen").Validator<string, string, import("valgen").ExecutionOptions>;

package/utils/encrypt-helpers.js

@@ -0,0 +1,43 @@
+import crypto from 'node:crypto';
+import { validator } from 'valgen';
+export const secretEncryptor = (secretKey) => {
+    return validator((input) => {
+        if (input.startsWith('sbscrt::')) {
+            input = decryptText(input, secretKey);
+        }
+        return 'sbscrt::' + encryptText(input, secretKey);
+    });
+};
+export const secretDecryptor = (secretKey) => {
+    return validator((input) => {
+        if (input.startsWith('sbscrt::')) {
+            return decryptText(input.substring(6), secretKey);
+        }
+        return input;
+    });
+};
+function encryptText(input, secretKey) {
+    const salt = crypto.randomBytes(16);
+    const key = crypto.scryptSync(secretKey, salt, 32);
+    const iv = crypto.randomBytes(16);
+    const cipher = crypto.createCipheriv('aes-256-cbc', key, iv);
+    const encrypted = Buffer.concat([
+        cipher.update(input, 'utf8'),
+        cipher.final(),
+    ]);
+    // Store salt + IV + EncryptedData
+    return Buffer.concat([salt, iv, encrypted]).toString('base64');
+}
+function decryptText(input, secretKey) {
+    const data = Buffer.from(input, 'base64');
+    const salt = data.subarray(0, 16);
+    const iv = data.subarray(16, 32);
+    const encrypted = data.subarray(32);
+    const key = crypto.scryptSync(secretKey, salt, 32);
+    const decipher = crypto.createDecipheriv('aes-256-cbc', key, iv);
+    const decrypted = Buffer.concat([
+        decipher.update(encrypted),
+        decipher.final(),
+    ]);
+    return decrypted.toString('utf8');
+}

package/utils/make-extension-package.d.ts

@@ -0,0 +1,3 @@
+import { IExtensionPackage } from '../interfaces/index.js';
+export declare function makeExtensionPackage(pkg: IExtensionPackage): IExtensionPackage;
+export declare function isExtensionPackage(pkg: IExtensionPackage): pkg is IExtensionPackage;

package/utils/make-extension-package.js

@@ -0,0 +1,13 @@
+const marker = Symbol('marker');
+export function makeExtensionPackage(pkg) {
+    Object.defineProperty(pkg, marker, {
+        configurable: false,
+        writable: false,
+        enumerable: false,
+        value: 1,
+    });
+    return pkg;
+}
+export function isExtensionPackage(pkg) {
+    return pkg && typeof pkg === 'object' && pkg[marker] === 1;
+}

package/utils/metadata-utils.d.ts

@@ -0,0 +1,7 @@
+import { StackExecutor } from '../classes/stack-executor.js';
+import { ProcessorMetadata, Profile } from '../models/index.js';
+export declare function materializeMetadata(profile: Profile): ProcessorMetadata;
+export declare function materializeMetadataSilent(profile: Profile): {
+    metadata: ProcessorMetadata;
+    issues?: StackExecutor.Issue[];
+};

package/utils/metadata-utils.js

@@ -0,0 +1,99 @@
+import { clone, merge, omitUndefined } from '@jsopen/objects';
+import { SbError } from '../classes/sb-error.js';
+import { StackExecutor } from '../classes/stack-executor.js';
+import { ExtensionRegistry } from '../registry.js';
+export function materializeMetadata(profile) {
+    const { metadata, issues } = materializeMetadataSilent(profile);
+    if (issues?.length) {
+        const msg = 'Worker profile validation error.\n  ' +
+            issues
+                .map(issue => `- ${issue.message}. Location: /${issue.location}`)
+                .join('\n  ');
+        throw new SbError(msg, {
+            workerId: profile.id,
+            issues,
+        });
+    }
+    return metadata;
+}
+export function materializeMetadataSilent(profile) {
+    const stackExecutor = new StackExecutor();
+    const processorRec = ExtensionRegistry.getProcessor(profile.processorClass);
+    const metadata = omitUndefined(clone({
+        className: processorRec.metadata.className,
+        displayName: processorRec.metadata.displayName,
+        description: processorRec.metadata.description,
+        iconUrl: processorRec.metadata.iconUrl,
+        author: processorRec.metadata.author,
+        tags: processorRec.metadata.tags,
+        variables: processorRec.metadata.variables,
+        components: processorRec.metadata.components,
+        ...processorRec.metadata,
+    }, { deep: 'full' }));
+    delete metadata.components;
+    if (processorRec.metadata.components) {
+        stackExecutor.execute('components', () => {
+            metadata.components = _materializeComponents(stackExecutor, processorRec.metadata, profile);
+        });
+    }
+    if (stackExecutor.issues?.length)
+        return { metadata, issues: stackExecutor.issues };
+    return { metadata };
+}
+/**
+ *
+ * @private
+ */
+function _materializeComponents(stackExecutor, metadata, profile) {
+    const out = {};
+    for (const [componentName, metadataComponent,] of Object.entries(metadata.components)) {
+        stackExecutor.execute(componentName, () => {
+            const materializedComponent = {
+                interfaces: metadataComponent.interfaces
+                    ? [...metadataComponent.interfaces]
+                    : undefined,
+                required: metadataComponent.required,
+                ...metadataComponent,
+            };
+            out[componentName] = materializedComponent;
+            /** Check if component defined in profile */
+            const profileComponent = profile?.components?.[componentName];
+            if (!profileComponent) {
+                if (metadataComponent.required)
+                    throw new Error(`Component "${componentName}" should be configured`);
+                return;
+            }
+            /** istanbul ignore next */
+            if (!profileComponent.className)
+                throw new Error(`"className" property required`);
+            const reg = ExtensionRegistry.getComponent(profileComponent.className);
+            materializedComponent.className = profileComponent.className;
+            materializedComponent.variables = merge({}, [reg.metadata.variables || {}, metadataComponent.variables || {}], { deep: 'full' });
+            if (metadataComponent.className) {
+                if (materializedComponent.className &&
+                    materializedComponent.className !== metadataComponent.className)
+                    throw new Error(`Selected component class "${profileComponent.className}" do not match with definition component class (${metadataComponent.className}).`);
+            }
+            else {
+                const requiredInterfaces = metadataComponent.interfaces || [];
+                const implementations = reg.metadata.interfaces || [];
+                const filtered = requiredInterfaces.filter(x => !implementations.includes(x));
+                if (filtered.length) {
+                    throw new Error(`Selected component "${profileComponent.className}" do not implements some of required interfaces (${filtered.join(',')}).`);
+                }
+            }
+            if (materializedComponent.variables) {
+                for (const [k, variable] of Object.entries(materializedComponent.variables)) {
+                    if (variable.ignored)
+                        delete materializedComponent.variables[k];
+                }
+            }
+            if (reg.metadata.components) {
+                stackExecutor.execute('components', () => {
+                    materializedComponent.components = _materializeComponents(stackExecutor, reg.metadata, profileComponent);
+                });
+            }
+        });
+    }
+    return out;
+}

package/utils/profile-utils.d.ts

@@ -0,0 +1,21 @@
+import { StackExecutor } from '../classes/stack-executor.js';
+import { ProcessorMetadata, Profile } from '../models/index.js';
+export interface CodecOptions {
+    setDefaultValues?: boolean;
+    cryptoSecretKey?: string;
+}
+/**
+ * Decodes profile
+ */
+export declare function decodeProfile(materializedMetadata: ProcessorMetadata, rawProfile: Profile, options?: CodecOptions): Profile;
+export declare function decodeProfileSilent(materializedMetadata: ProcessorMetadata, rawProfile: Profile, options?: CodecOptions): {
+    profile: Profile;
+    issues?: StackExecutor.Issue[];
+};
+/**
+ * Encodes profile
+ */
+export declare function encodeProfileSilent(metadata: ProcessorMetadata, rawProfile: Profile, options?: CodecOptions): {
+    profile: Profile;
+    issues?: StackExecutor.Issue[];
+};

package/utils/profile-utils.js

@@ -0,0 +1,222 @@
+import { isAny, isBase64, isBoolean, isDateString, isEmail, isHex, isNumber, isString, isTime, isURL, validator, vg, } from 'valgen';
+import { SbError } from '../classes/sb-error.js';
+import { StackExecutor } from '../classes/stack-executor.js';
+import { Profile, VariableType, } from '../models/index.js';
+import { getModelsDocument } from '../models-document.js';
+import { secretDecryptor, secretEncryptor } from './encrypt-helpers.js';
+/**
+ * Decodes profile
+ */
+export function decodeProfile(materializedMetadata, rawProfile, options) {
+    const { profile: out, issues } = decodeProfileSilent(materializedMetadata, rawProfile, options);
+    if (issues?.length) {
+        const msg = 'Worker profile validation error.\n  ' +
+            issues
+                .map(issue => `- ${issue.message}. Location: /${issue.location}`)
+                .join('\n  ');
+        throw new SbError(msg, {
+            workerId: rawProfile.id,
+            issues,
+        });
+    }
+    return out;
+}
+export function decodeProfileSilent(materializedMetadata, rawProfile, options) {
+    const modelsDocument = getModelsDocument();
+    const t = modelsDocument.node.getComplexType(Profile);
+    const fn = t.generateCodec('decode');
+    const out = fn(rawProfile);
+    const stackExecutor = new StackExecutor();
+    _codecProcessElement(stackExecutor, 'decode', materializedMetadata, out, options);
+    return { profile: out, issues: stackExecutor.issues };
+}
+/**
+ * Encodes profile
+ */
+export function encodeProfileSilent(metadata, rawProfile, options) {
+    const modelsDocument = getModelsDocument();
+    const t = modelsDocument.node.getComplexType(Profile);
+    const fn = t.generateCodec('encode', { coerce: true });
+    const out = fn({
+        id: rawProfile.id,
+        processorClass: rawProfile.processorClass,
+        displayName: rawProfile.displayName,
+        group: rawProfile.group,
+        description: rawProfile.description,
+        iconUrl: rawProfile.iconUrl,
+        settings: {
+            autostart: rawProfile.settings?.autostart ?? false,
+            logs: rawProfile.settings?.logs ?? {},
+            ...rawProfile.settings,
+        },
+        values: rawProfile.values,
+        ...rawProfile,
+    });
+    const stackExecutor = new StackExecutor();
+    _codecProcessElement(stackExecutor, 'encode', metadata, out, options);
+    return { profile: out, issues: stackExecutor.issues };
+}
+/**
+ *
+ * @private
+ */
+function _codecProcessElement(stackExecutor, codec, metadata, profile, options) {
+    profile.values = profile.values || {};
+    if (metadata.variables) {
+        stackExecutor.execute('values', () => {
+            _decodeVariables(stackExecutor, codec, metadata, profile, options);
+        });
+    }
+    else
+        delete profile.values;
+    if (metadata.components) {
+        stackExecutor.execute('components', () => {
+            profile.components = profile.components || {};
+            for (const [k, defComponent] of Object.entries(metadata.components)) {
+                if (!profile.components[k]) {
+                    if (defComponent.required) {
+                        stackExecutor.issues.push({
+                            message: `Component "${k}" is required`,
+                            severity: 'error',
+                            location: `/components/${k}`,
+                        });
+                    }
+                    continue;
+                }
+                stackExecutor.execute(k, () => {
+                    _codecProcessElement(stackExecutor, codec, defComponent, profile.components[k], options);
+                });
+            }
+        });
+    }
+    else
+        delete profile?.components;
+}
+/**
+ *
+ * @protected
+ */
+function _decodeVariables(stackExecutor, codec, metadata, profile, options) {
+    if (!metadata.variables) {
+        delete profile.values;
+        return;
+    }
+    const oldValues = profile.values;
+    profile.values = {};
+    const process = (variables, target, oldVals) => {
+        for (const [k, defVariable] of Object.entries(variables)) {
+            stackExecutor.execute(k, () => {
+                if (defVariable == null)
+                    return;
+                const fns = [];
+                switch (defVariable.type) {
+                    case VariableType.String: {
+                        switch (defVariable.format) {
+                            case 'hex':
+                                fns.push(isHex);
+                                break;
+                            case 'base64':
+                                fns.push(isBase64);
+                                break;
+                            case 'url':
+                                fns.push(isURL);
+                                break;
+                            case 'email':
+                                fns.push(isEmail);
+                                break;
+                            case 'date':
+                                fns.push(vg.isDateString({
+                                    precisionMin: 'day',
+                                    precisionMax: 'day',
+                                    trim: true,
+                                }));
+                                break;
+                            case 'date-time':
+                                fns.push(isDateString);
+                                break;
+                            case 'time':
+                                fns.push(isTime);
+                                break;
+                            default:
+                                fns.push(isString);
+                        }
+                        if (defVariable.pattern)
+                            fns.push(vg.matches(defVariable.pattern));
+                        break;
+                    }
+                    case VariableType.Secret: {
+                        fns.push(isString);
+                        if (options?.cryptoSecretKey) {
+                            if (codec === 'decode')
+                                fns.push(secretDecryptor(options.cryptoSecretKey));
+                            else
+                                fns.push(secretEncryptor(options.cryptoSecretKey));
+                        }
+                        break;
+                    }
+                    case VariableType.Number: {
+                        fns.push(isNumber);
+                        break;
+                    }
+                    case VariableType.Boolean: {
+                        fns.push(isBoolean);
+                        break;
+                    }
+                    case VariableType.Enum: {
+                        fns.push(vg.isEnum(defVariable.enumValues));
+                        break;
+                    }
+                    case VariableType.Nested: {
+                        if (!defVariable.variables) {
+                            delete target[k];
+                        }
+                        else {
+                            fns.push(validator(old => {
+                                if (!old)
+                                    return;
+                                const v = {};
+                                process(defVariable.variables, v, old);
+                                return v;
+                            }));
+                            // fns.push(isAny);
+                        }
+                        break;
+                    }
+                    default:
+                        fns.push(isAny);
+                }
+                if (codec === 'decode') {
+                    if (defVariable.minValue)
+                        fns.push(vg.isGte(defVariable.minValue));
+                    if (defVariable.maxValue)
+                        fns.push(vg.isLt(defVariable.maxValue));
+                }
+                let fn = fns.length > 1 ? vg.pipe(fns) : fns[0];
+                if (defVariable.isArray) {
+                    fn = vg.isArray(fn);
+                    if (defVariable.minOccurs !== null || defVariable.maxOccurs) {
+                        const p = [fn];
+                        if (defVariable.minOccurs != null)
+                            p.push(vg.lengthMin(1));
+                        if (defVariable.maxOccurs != null)
+                            p.push(vg.lengthMax(defVariable.maxOccurs));
+                        fn = vg.pipe(p, { returnIndex: 0 });
+                    }
+                }
+                if (options?.setDefaultValues ?? true) {
+                    fn =
+                        defVariable.required || defVariable.default != null
+                            ? vg.required(fn, { default: defVariable.default })
+                            : vg.optional(fn);
+                }
+                else
+                    fn =
+                        !defVariable.required || defVariable.default != null
+                            ? vg.optional(fn)
+                            : vg.required(fn);
+                target[k] = fn(oldVals?.[k], { coerce: true });
+            });
+        }
+    };
+    process(metadata.variables, profile.values, oldValues);
+}