@kapeta/local-cluster-service 0.0.0-96f91ef

package/src/RepositoryWatcher.ts

@@ -0,0 +1,363 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import chokidar, { FSWatcher } from 'chokidar';
+import ClusterConfiguration, { Definition, DefinitionInfo } from '@kapeta/local-cluster-config';
+import FS from 'fs-extra';
+import Path from 'node:path';
+import YAML from 'yaml';
+import { parseKapetaUri } from '@kapeta/nodejs-utils';
+import _ from 'lodash';
+import { socketManager } from './socketManager';
+import { SourceOfChange, WatchEventName } from './types';
+import { cacheManager } from './cacheManager';
+import { EventEmitter } from 'node:events';
+import { assetManager } from './assetManager';
+import { definitionsManager } from './definitionsManager';
+
+interface AssetIdentity {
+    handle: string;
+    name: string;
+    version: string;
+}
+const KAPETA_YML_RX = /^kapeta.ya?ml$/;
+
+let definitions: DefinitionInfo[] | undefined;
+let definitionTimeout: NodeJS.Timeout | undefined;
+
+function getDefinitionsDebounced() {
+    if (definitionTimeout) {
+        clearTimeout(definitionTimeout);
+        definitionTimeout = undefined;
+    }
+    if (!definitions) {
+        definitions = ClusterConfiguration.getDefinitions();
+    } else {
+        definitionTimeout = setTimeout(() => {
+            definitionTimeout = undefined;
+            definitions = undefined;
+        }, 500);
+    }
+
+    return definitions;
+}
+
+export class RepositoryWatcher extends EventEmitter {
+    private watcher?: FSWatcher;
+    private disabled: boolean = false;
+    private readonly baseDir: string;
+    private allDefinitions: DefinitionInfo[] = [];
+    private symbolicLinks: { [link: string]: string } = {};
+    private sourceOfChange: Map<string, SourceOfChange> = new Map();
+    constructor() {
+        super();
+        this.baseDir = ClusterConfiguration.getRepositoryBasedir();
+    }
+
+    setDisabled(disabled: boolean) {
+        this.disabled = disabled;
+    }
+    public watch() {
+        if (!FS.existsSync(this.baseDir)) {
+            FS.mkdirpSync(this.baseDir);
+        }
+
+        this.allDefinitions = ClusterConfiguration.getDefinitions();
+
+        try {
+            this.watcher = chokidar.watch(this.baseDir, {
+                followSymlinks: false,
+                ignorePermissionErrors: true,
+                disableGlobbing: true,
+                persistent: true,
+                depth: 2,
+                ignored: (path) => this.ignoreFile(path),
+            });
+            this.watcher.on('all', this.handleFileChange.bind(this));
+            this.watcher.on('error', (error) => {
+                console.log('Error watching repository', error);
+            });
+            this.watcher.on('ready', () => {
+                console.log('Watching local repository for provider changes: %s', this.baseDir);
+            });
+        } catch (e) {
+            // Fallback to run without watch mode due to potential platform issues.
+            // https://nodejs.org/docs/latest/api/fs.html#caveats
+            console.log('Unable to watch for changes. Changes to assets will not update automatically.', e);
+            return;
+        }
+    }
+
+    async setSourceOfChangeFor(file: string, source: SourceOfChange) {
+        this.sourceOfChange.set(file, source);
+        try {
+            const realPath = await FS.realpath(file);
+            if (realPath !== file) {
+                this.sourceOfChange.set(realPath, source);
+            }
+        } catch (e) {
+            // Ignore
+        }
+    }
+
+    async clearSourceOfChangeFor(file: string) {
+        this.sourceOfChange.delete(file);
+        try {
+            const realPath = await FS.realpath(file);
+            if (realPath !== file) {
+                this.sourceOfChange.delete(realPath);
+            }
+        } catch (e) {
+            // Ignore
+        }
+    }
+
+    public async unwatch() {
+        if (!this.watcher) {
+            return;
+        }
+        this.symbolicLinks = {};
+        await this.watcher.close();
+        this.watcher = undefined;
+    }
+
+    private async getAssetIdentity(path: string): Promise<AssetIdentity | undefined> {
+        const baseName = Path.basename(path);
+        let handle, name, version;
+        if (path.startsWith(this.baseDir)) {
+            const relativePath = Path.relative(this.baseDir, path);
+            // Inside the repo we can use the path to determine the handle, name and version
+            [handle, name, version] = relativePath.split(Path.sep);
+            if (!handle || !name || !version) {
+                // Do nothing with this
+                return;
+            }
+
+            return {
+                handle,
+                name,
+                version,
+            };
+        }
+
+        if (!KAPETA_YML_RX.test(baseName)) {
+            // Do nothing with this
+            return;
+        }
+        // Outside the repo we need to use the file content to determine the handle, name
+        // Version is always 'local'
+        version = 'local';
+
+        try {
+            const definition: Definition = YAML.parse((await FS.readFile(path)).toString());
+            const uri = parseKapetaUri(definition.metadata.name);
+            handle = uri.handle;
+            name = uri.name;
+            return {
+                handle,
+                name,
+                version,
+            };
+        } catch (e) {
+            // Ignore issues in the YML file
+            return;
+        }
+    }
+
+    private async handleFileChange(eventName: WatchEventName, path: string) {
+        if (!path) {
+            return;
+        }
+
+        if (this.disabled) {
+            return;
+        }
+
+        const assetIdentity = await this.getAssetIdentity(path);
+        if (!assetIdentity) {
+            return;
+        }
+
+        // If this is false it's because we're watching a symlink target
+        const withinRepo = path.startsWith(this.baseDir);
+        if (withinRepo && assetIdentity.version === 'local' && path.endsWith(Path.sep + 'local')) {
+            // This is likely a symlink target
+            if (eventName === 'add') {
+                //console.log('Checking if we should add symlink target', handle, name, version, path);
+                await this.addSymlinkTarget(path);
+            }
+
+            if (eventName === 'unlink') {
+                await this.removeSymlinkTarget(path);
+            }
+
+            if (eventName === 'change') {
+                await this.updateSymlinkTarget(path);
+            }
+        }
+
+        const repoPath = this.getRepositoryPath(assetIdentity);
+
+        if (
+            eventName === 'change' &&
+            !withinRepo &&
+            assetIdentity.version === 'local' &&
+            Path.basename(path) === 'kapeta.yml' &&
+            (await this.exists(path)) &&
+            !(await this.exists(repoPath))
+        ) {
+            // This happens when a local asset is renamed
+            const oldPath = _.findKey(this.symbolicLinks, (link) => link === path);
+            if (oldPath) {
+                await FS.unlink(oldPath);
+                await assetManager.importFile(path);
+                return;
+            }
+        }
+
+        const sourceOfChange = this.sourceOfChange.get(path) ?? 'filesystem';
+        await this.checkForChange(assetIdentity, sourceOfChange);
+
+        // We consume the sourceOfChange when the file is changed
+        this.sourceOfChange.delete(path);
+    }
+
+    private getRepositoryPath(assetIdentity: AssetIdentity) {
+        return Path.join(this.baseDir, assetIdentity.handle, assetIdentity.name, assetIdentity.version);
+    }
+
+    private async checkForChange(assetIdentity: AssetIdentity, sourceOfChange: SourceOfChange) {
+        const ymlPath = Path.join(this.getRepositoryPath(assetIdentity), 'kapeta.yml');
+        const newDefinitions = getDefinitionsDebounced();
+
+        const newDefinition = newDefinitions.find((d) => d.ymlPath === ymlPath);
+        let currentDefinition = this.allDefinitions.find((d) => d.ymlPath === ymlPath);
+        const ymlExists = await this.exists(ymlPath);
+
+        let type;
+        if (ymlExists) {
+            if (currentDefinition) {
+                if (newDefinition && _.isEqual(currentDefinition, newDefinition)) {
+                    //Definition was not changed
+                    return;
+                }
+                type = 'updated';
+            } else if (newDefinition) {
+                type = 'added';
+                currentDefinition = newDefinition;
+            } else {
+                //Other definition was added / updated - ignore
+                return;
+            }
+        } else {
+            if (currentDefinition) {
+                //Something was removed
+                type = 'removed';
+            } else {
+                //Other definition was removed - ignore
+                return;
+            }
+        }
+
+        const payload = {
+            type,
+            definition: newDefinition?.definition ?? currentDefinition?.definition,
+            asset: assetIdentity,
+            sourceOfChange,
+        };
+
+        this.allDefinitions = newDefinitions;
+
+        //console.log('Asset changed', payload);
+        socketManager.emitGlobal('asset-change', payload);
+        this.emit('change', payload);
+
+        cacheManager.flush();
+    }
+
+    private async exists(path: string): Promise<boolean> {
+        try {
+            await FS.access(path);
+            return true;
+        } catch (e) {
+            return false;
+        }
+    }
+    private async removeSymlinkTarget(path: string) {
+        if (this.symbolicLinks[path]) {
+            //console.log('Unwatching symlink target %s => %s', path, this.symbolicLinks[path]);
+            this.watcher?.unwatch(this.symbolicLinks[path]);
+            delete this.symbolicLinks[path];
+        }
+    }
+
+    private async updateSymlinkTarget(path: string) {
+        if (this.symbolicLinks[path]) {
+            //console.log('Updating symlink target %s => %s', path, this.symbolicLinks[path]);
+            this.watcher?.unwatch(this.symbolicLinks[path]);
+            delete this.symbolicLinks[path];
+            await this.addSymlinkTarget(path);
+        }
+    }
+
+    private async addSymlinkTarget(path: string) {
+        try {
+            // Make sure we're not watching the symlink target
+            await this.removeSymlinkTarget(path);
+            let symbolicLink = false;
+            try {
+                const stat = await FS.lstat(path);
+                symbolicLink = stat.isSymbolicLink();
+            } catch (e) {}
+
+            if (symbolicLink) {
+                try {
+                    const realPath = Path.join(await FS.realpath(path), 'kapeta.yml');
+                    if (await this.exists(realPath)) {
+                        //console.log('Watching symlink target %s => %s', path, realPath);
+                        this.watcher?.add(realPath);
+                        this.symbolicLinks[path] = realPath;
+                    }
+                } catch (e) {
+                    // Remove the symlink - it's broken
+                    await FS.remove(path);
+                }
+            }
+        } catch (e) {
+            // Ignore
+            console.warn('Failed to check local symlink target', e);
+        }
+    }
+
+    private ignoreFile(path: string) {
+        if (!path.startsWith(this.baseDir)) {
+            return false;
+        }
+        if (path.includes(Path.sep + 'node_modules' + Path.sep)) {
+            return true;
+        }
+
+        const filename = Path.basename(path);
+        if (filename.startsWith('.')) {
+            return true;
+        }
+
+        const relativePath = Path.relative(this.baseDir, path).split(Path.sep);
+
+        try {
+            if (FS.statSync(path).isDirectory()) {
+                if (relativePath.length > 3) {
+                    return true;
+                }
+                return false;
+            }
+        } catch (e) {
+            // Didn't exist - dont ignore
+            return false;
+        }
+
+        return !/^kapeta\.ya?ml$/.test(filename);
+    }
+}

package/src/ai/aiClient.ts

@@ -0,0 +1,93 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+import request from 'request';
+import { PlanContext, transformToPlan } from './transform';
+import { Application } from './types';
+import { KapetaAPI } from '@kapeta/nodejs-api-client';
+import ClusterConfiguration from '@kapeta/local-cluster-config';
+import { getRemoteUrl } from '../utils/utils';
+
+export type PromptResult = {
+    explanation: string;
+    response: string;
+    context?: PlanContext;
+};
+
+export interface AIMessage {
+    content: string;
+    role: 'user' | 'assistant';
+}
+
+export interface AIRequest {
+    messages: AIMessage[];
+}
+
+class AIClient {
+    private readonly _baseUrl: string;
+
+    constructor() {
+        this._baseUrl = getRemoteUrl('ai-service', 'https://ai.kapeta.com');
+    }
+
+    public async sendPrompt(handle: string, body: AIRequest): Promise<PromptResult> {
+        const url = `${this._baseUrl}/v1/plan?type=chat`;
+
+        const headers: { [k: string]: string } = {};
+        const api = new KapetaAPI();
+        if (api.hasToken()) {
+            headers['Authorization'] = `Bearer ${await api.getAccessToken()}`;
+        }
+
+        const options = {
+            url,
+            method: 'POST',
+            json: true,
+            body,
+            headers,
+        };
+
+        return new Promise((resolve, reject) => {
+            request(options, async (error, response, application: Application) => {
+                if (error) {
+                    console.error(error);
+                    reject(error);
+                }
+
+                if (response.statusCode !== 200) {
+                    console.log('Prompt failed', response.statusCode, response.body);
+                    reject(new Error(`Invalid response code: ${response.statusCode}`));
+                    return;
+                }
+
+                try {
+                    if (application?.name) {
+                        const planContext = await transformToPlan(handle, application);
+                        resolve({
+                            explanation: application.explanation,
+                            response: application.response ?? application.explanation ?? 'Plan was generated',
+                            context: planContext,
+                        });
+                    } else {
+                        resolve({
+                            explanation: application.explanation,
+                            response:
+                                application.response ??
+                                application.explanation ??
+                                'I did not understand your request. Please rephrase.',
+                        });
+                    }
+                } catch (err: any) {
+                    console.error(err);
+                    resolve({
+                        explanation: '',
+                        response: 'I did not understand your request. Please rephrase.',
+                    });
+                }
+            });
+        });
+    }
+}
+
+export const aiClient = new AIClient();

package/src/ai/routes.ts

@@ -0,0 +1,39 @@
+/**
+ * Copyright 2023 Kapeta Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import Router from 'express-promise-router';
+import { Response } from 'express';
+
+import { corsHandler } from '../middleware/cors';
+
+import { stringBody, StringBodyRequest } from '../middleware/stringBody';
+import { aiClient, AIRequest } from './aiClient';
+import { KapetaBodyRequest } from '../types';
+import YAML from 'yaml';
+
+const router = Router();
+
+router.use('/', corsHandler);
+router.use('/', stringBody);
+
+router.post('/prompt/:handle', async (req: KapetaBodyRequest, res: Response) => {
+    const handle = req.params.handle;
+    try {
+        const aiRequest: AIRequest = JSON.parse(req.stringBody ?? '{}');
+        const result = await aiClient.sendPrompt(handle, aiRequest);
+        if (req.accepts('application/yaml')) {
+            res.set('Content-Type', 'application/yaml');
+            res.send(YAML.stringify(result));
+        } else {
+            res.json(result);
+        }
+    } catch (err: any) {
+        console.error('Failed to send prompt', err);
+        res.status(400).send({ error: err.message });
+        return;
+    }
+});
+
+export default router;