@kapeta/local-cluster-service 0.72.0 → 0.73.0

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [0.73.0](https://github.com/kapetacom/local-cluster-service/compare/v0.72.0...v0.73.0) (2024-09-25)
+
+
+### Features
+
+* add conversation CRUD endpoints and zip upload [CORE-3499] ([#257](https://github.com/kapetacom/local-cluster-service/issues/257)) ([81c1c31](https://github.com/kapetacom/local-cluster-service/commit/81c1c3193e676f1c9e4a802cbf060ca63c8f909d))
+
 # [0.72.0](https://github.com/kapetacom/local-cluster-service/compare/v0.71.6...v0.72.0) (2024-09-24)
 
 

package/dist/cjs/src/storm/page-utils.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.writeConversationToFile = exports.readConversationFromFile = exports.readPageFromDisk = exports.readPageFromDiskAsString = exports.resolveReadPath = exports.getSystemBaseImplDir = exports.getSystemBaseDir = exports.hasPageOnDisk = exports.writeImageToDisk = exports.writeAssetToDisk = exports.writePageToDisk = exports.normalizePath = exports.SystemIdHeader = void 0;
+const filesystemManager_1 = require("../filesystemManager");
 const node_os_1 = __importDefault(require("node:os"));
 const path_1 = __importDefault(require("path"));
 const fs_extra_1 = __importDefault(require("fs-extra"));
@@ -61,7 +62,7 @@ function hasPageOnDisk(systemId, path, method) {
 }
 exports.hasPageOnDisk = hasPageOnDisk;
 function getSystemBaseDir(systemId) {
-    return path_1.default.join(node_os_1.default.tmpdir(), 'ai-systems', systemId);
+    return path_1.default.join(filesystemManager_1.filesystemManager.getProjectRootFolder() || node_os_1.default.tmpdir(), 'ai-systems', systemId);
 }
 exports.getSystemBaseDir = getSystemBaseDir;
 function getSystemBaseImplDir(systemId) {

package/dist/cjs/src/storm/routes.js

@@ -23,9 +23,10 @@ const node_uuid_1 = __importDefault(require("node-uuid"));
 const page_utils_1 = require("./page-utils");
 const UIServer_1 = require("./UIServer");
 const crypto_1 = require("crypto");
-const PageGenerator_1 = require("./PageGenerator");
 const utils_1 = require("./utils");
 const utils_2 = require("../utils/utils");
+const stormService_1 = __importDefault(require("../stormService"));
+const PageGenerator_1 = require("./PageGenerator");
 const UI_SERVERS = {};
 const router = (0, express_promise_router_1.default)();
 router.use('/', cors_1.corsHandler);
@@ -70,6 +71,43 @@ router.post('/ui/serve/:systemId', async (req, res) => {
     }
     res.status(200).send({ status: 'running', url: svr.getUrl(), resetUrl: svr.resolveUrlFromPath('/_reset') });
 });
+router.get('/ui/conversations', async (req, res) => {
+    const local = await stormService_1.default.listLocalConversations();
+    const remote = await stormService_1.default.listRemoteConversations();
+    res.send({
+        local,
+        remote,
+    });
+});
+router.get('/ui/conversations/:systemId', async (req, res) => {
+    const systemId = req.params.systemId;
+    const eventsFile = (0, page_utils_1.getSystemBaseDir)(systemId) + '/events.ndjson';
+    res.set('Content-Type', 'application/x-ndjson');
+    res.set('Access-Control-Expose-Headers', stormClient_1.ConversationIdHeader);
+    res.set(stormClient_1.ConversationIdHeader, systemId);
+    if (!fs_extra_1.default.existsSync(eventsFile)) {
+        res.status(404).send({ error: 'No events found' });
+        return;
+    }
+    res.send(fs_extra_1.default.readFileSync(eventsFile));
+});
+router.put('/ui/conversations/:systemId', async (req, res) => {
+    const systemId = req.params.systemId;
+    const events = req.stringBody ? JSON.parse(req.stringBody) : [];
+    await stormService_1.default.saveConversation(systemId, events);
+    res.send({ ok: true });
+});
+router.delete('/ui/conversations/:systemId', async (req, res) => {
+    const systemId = req.params.systemId;
+    await stormService_1.default.deleteConversation(systemId);
+    res.send({ ok: true });
+});
+router.post('/ui/conversations/:systemId/append', async (req, res) => {
+    const systemId = req.params.systemId;
+    const events = req.stringBody ? JSON.parse(req.stringBody) : [];
+    await stormService_1.default.appendConversation(systemId, events);
+    res.send({ ok: true });
+});
 router.post('/ui/create-system/:handle/:systemId', async (req, res) => {
     const systemId = req.params.systemId;
     const srcDir = (0, page_utils_1.getSystemBaseDir)(systemId);
@@ -142,6 +180,18 @@ router.post('/ui/create-system-simple/:handle/:systemId', async (req, res) => {
         }
     }
 });
+router.post('/ui/systems/:handle/:systemId/download', async (req, res) => {
+    const systemId = req.params.systemId;
+    const handle = req.params.handle;
+    await stormService_1.default.installProjectById(handle, systemId);
+    res.send({ ok: true });
+});
+router.post('/ui/systems/:handle/:systemId/upload', async (req, res) => {
+    const systemId = req.params.systemId;
+    const handle = req.params.handle;
+    await stormService_1.default.uploadConversation(handle, systemId);
+    res.send({ ok: true });
+});
 router.delete('/ui/serve/:systemId', async (req, res) => {
     const systemId = req.params.systemId;
     if (!systemId) {

package/dist/cjs/src/storm/stormClient.d.ts

@@ -1,4 +1,5 @@
 /// <reference types="node" />
+/// <reference types="node" />
 import { ConversationItem, CreateSimpleBackendRequest, HTMLPage, ImplementAPIClients, StormFileImplementationPrompt, StormStream, StormUIImplementationPrompt, StormUIListPrompt } from './stream';
 import { Page, StormEventPageUrl } from './events';
 export declare const STORM_ID = "storm";
@@ -84,6 +85,8 @@ declare class StormClient {
     createErrorDetails(prompt: string, history?: ConversationItem[], conversationId?: string): Promise<StormStream>;
     generateCode(prompt: StormFileImplementationPrompt, conversationId?: string): Promise<StormStream>;
     deleteUIPageConversation(conversationId: string): Promise<string>;
+    downloadSystem(handle: string, conversationId: string): Promise<Buffer>;
+    uploadSystem(handle: string, conversationId: string, buffer: Buffer): Promise<Response>;
 }
 export declare const stormClient: StormClient;
 export {};

package/dist/cjs/src/storm/stormClient.js

@@ -139,7 +139,7 @@ class StormClient {
             method: 'POST',
             body: JSON.stringify(prompt.pages),
         });
-        return await response.json();
+        return (await response.json());
     }
     async generatePrompt(pages) {
         const u = `${this._baseUrl}/v2/ui/prompt`;
@@ -232,5 +232,24 @@ class StormClient {
         const response = await fetch(options.url, options);
         return response.text();
     }
+    async downloadSystem(handle, conversationId) {
+        const u = `${this._baseUrl}/v2/systems/${handle}/${conversationId}/download`;
+        const response = await fetch(u);
+        if (!response.ok) {
+            throw new Error(`Failed to download system: ${response.status}`);
+        }
+        return Buffer.from(await response.arrayBuffer());
+    }
+    async uploadSystem(handle, conversationId, buffer) {
+        const u = `${this._baseUrl}/v2/systems/${handle}/${conversationId}`;
+        const response = await fetch(u, {
+            method: 'PUT',
+            body: buffer,
+            headers: {
+                'content-type': 'application/zip',
+            },
+        });
+        return response;
+    }
 }
 exports.stormClient = new StormClient();

package/dist/cjs/src/stormService.d.ts

@@ -0,0 +1,19 @@
+import { StormEvent } from './storm/events';
+export declare class StormService {
+    private getConversationFile;
+    private getConversationTarball;
+    listRemoteConversations(): Promise<never[]>;
+    listLocalConversations(): Promise<{
+        id: string;
+        description: string;
+        title: string;
+    }[]>;
+    getConversation(conversationId: string): Promise<string>;
+    saveConversation(conversationId: string, events: StormEvent[]): Promise<void>;
+    appendConversation(conversationId: string, events: StormEvent[]): Promise<void>;
+    deleteConversation(conversationId: string): Promise<void>;
+    uploadConversation(handle: string, conversationId: string): Promise<void>;
+    installProjectById(handle: string, conversationId: string): Promise<void>;
+}
+declare const _default: StormService;
+export default _default;

package/dist/cjs/src/stormService.js

@@ -0,0 +1,127 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StormService = void 0;
+const promises_1 = __importDefault(require("fs/promises"));
+const glob_1 = require("glob");
+const filesystemManager_1 = require("./filesystemManager");
+const path_1 = __importDefault(require("path"));
+const fs_1 = require("fs");
+const tar = __importStar(require("tar"));
+const stormClient_1 = require("./storm/stormClient");
+class StormService {
+    getConversationFile(conversationId) {
+        return path_1.default.join(filesystemManager_1.filesystemManager.getProjectRootFolder(), 'ai-systems', conversationId, 'events.ndjson');
+    }
+    getConversationTarball(conversationId) {
+        return path_1.default.join(filesystemManager_1.filesystemManager.getProjectRootFolder(), 'ai-systems', conversationId, 'system.tar.gz');
+    }
+    async listRemoteConversations() {
+        // i.e. conversations from org / user on registry
+        return [];
+    }
+    async listLocalConversations() {
+        const systemsFolder = path_1.default.join(filesystemManager_1.filesystemManager.getProjectRootFolder(), 'ai-systems');
+        const eventFiles = await (0, glob_1.glob)('*/events.ndjson', {
+            cwd: systemsFolder,
+            absolute: true,
+        });
+        // Returns list of UUIDs - probably want to make it more useful than that
+        const conversations = [];
+        for (const file of eventFiles) {
+            const nldContents = await promises_1.default.readFile(file, 'utf8');
+            const events = nldContents.split('\n').map((e) => JSON.parse(e));
+            // find the shell and get the title tag
+            const shellEvent = events.find((e) => e.type === 'AI' && e.event.type === 'UI_SHELL')?.event;
+            const html = shellEvent?.payload.content;
+            const title = html?.match(/<title>(.*?)<\/title>/)?.[1];
+            const id = events.find((e) => e.type === 'AI')?.systemId;
+            const initialPrompt = events.find((e) => e.type === 'AI' && e.event.type === 'PROMPT_IMPROVE')?.event?.payload?.prompt || events[0].text;
+            if (!id) {
+                continue;
+            }
+            conversations.push({
+                id,
+                description: initialPrompt,
+                title: title || 'New system',
+            });
+        }
+        return conversations;
+    }
+    async getConversation(conversationId) {
+        const conversationFile = this.getConversationFile(conversationId);
+        if (!(0, fs_1.existsSync)(conversationFile)) {
+            throw new Error('Conversation not found');
+        }
+        return promises_1.default.readFile(conversationFile, 'utf8');
+    }
+    async saveConversation(conversationId, events) {
+        const conversationFile = this.getConversationFile(conversationId);
+        await promises_1.default.mkdir(path_1.default.dirname(conversationFile), { recursive: true });
+        await promises_1.default.writeFile(conversationFile, events.map((e) => JSON.stringify(e)).join('\n'));
+    }
+    async appendConversation(conversationId, events) {
+        const conversationFile = this.getConversationFile(conversationId);
+        await promises_1.default.appendFile(conversationFile, events.map((e) => JSON.stringify(e)).join('\n'));
+    }
+    async deleteConversation(conversationId) {
+        const conversationFile = this.getConversationFile(conversationId);
+        await promises_1.default.unlink(conversationFile);
+        // if it matches a UUID, it's safe to delete the directory too
+        const conversationDir = path_1.default.dirname(conversationFile);
+        if (conversationDir.match(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/)) {
+            await promises_1.default.rm(conversationDir, { recursive: true, force: true });
+        }
+    }
+    async uploadConversation(handle, conversationId) {
+        const tarballFile = this.getConversationTarball(conversationId);
+        const destDir = path_1.default.dirname(tarballFile);
+        const tarballName = path_1.default.basename(tarballFile);
+        await tar.create({
+            file: tarballFile,
+            cwd: destDir,
+            gzip: true,
+            filter: (entry) => !entry.includes(tarballName),
+        }, ['.']);
+        await stormClient_1.stormClient.uploadSystem(handle, conversationId, await promises_1.default.readFile(tarballFile));
+    }
+    async installProjectById(handle, conversationId) {
+        const tarballFile = this.getConversationTarball(conversationId);
+        const destDir = path_1.default.dirname(tarballFile);
+        const buffer = await stormClient_1.stormClient.downloadSystem(handle, conversationId);
+        await promises_1.default.mkdir(destDir, { recursive: true });
+        await promises_1.default.writeFile(tarballFile, buffer);
+        await tar.extract({
+            file: tarballFile,
+            cwd: destDir,
+        });
+        await promises_1.default.unlink(tarballFile);
+    }
+}
+exports.StormService = StormService;
+exports.default = new StormService();

package/dist/esm/src/storm/page-utils.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.writeConversationToFile = exports.readConversationFromFile = exports.readPageFromDisk = exports.readPageFromDiskAsString = exports.resolveReadPath = exports.getSystemBaseImplDir = exports.getSystemBaseDir = exports.hasPageOnDisk = exports.writeImageToDisk = exports.writeAssetToDisk = exports.writePageToDisk = exports.normalizePath = exports.SystemIdHeader = void 0;
+const filesystemManager_1 = require("../filesystemManager");
 const node_os_1 = __importDefault(require("node:os"));
 const path_1 = __importDefault(require("path"));
 const fs_extra_1 = __importDefault(require("fs-extra"));
@@ -61,7 +62,7 @@ function hasPageOnDisk(systemId, path, method) {
 }
 exports.hasPageOnDisk = hasPageOnDisk;
 function getSystemBaseDir(systemId) {
-    return path_1.default.join(node_os_1.default.tmpdir(), 'ai-systems', systemId);
+    return path_1.default.join(filesystemManager_1.filesystemManager.getProjectRootFolder() || node_os_1.default.tmpdir(), 'ai-systems', systemId);
 }
 exports.getSystemBaseDir = getSystemBaseDir;
 function getSystemBaseImplDir(systemId) {

package/dist/esm/src/storm/routes.js

@@ -23,9 +23,10 @@ const node_uuid_1 = __importDefault(require("node-uuid"));
 const page_utils_1 = require("./page-utils");
 const UIServer_1 = require("./UIServer");
 const crypto_1 = require("crypto");
-const PageGenerator_1 = require("./PageGenerator");
 const utils_1 = require("./utils");
 const utils_2 = require("../utils/utils");
+const stormService_1 = __importDefault(require("../stormService"));
+const PageGenerator_1 = require("./PageGenerator");
 const UI_SERVERS = {};
 const router = (0, express_promise_router_1.default)();
 router.use('/', cors_1.corsHandler);
@@ -70,6 +71,43 @@ router.post('/ui/serve/:systemId', async (req, res) => {
     }
     res.status(200).send({ status: 'running', url: svr.getUrl(), resetUrl: svr.resolveUrlFromPath('/_reset') });
 });
+router.get('/ui/conversations', async (req, res) => {
+    const local = await stormService_1.default.listLocalConversations();
+    const remote = await stormService_1.default.listRemoteConversations();
+    res.send({
+        local,
+        remote,
+    });
+});
+router.get('/ui/conversations/:systemId', async (req, res) => {
+    const systemId = req.params.systemId;
+    const eventsFile = (0, page_utils_1.getSystemBaseDir)(systemId) + '/events.ndjson';
+    res.set('Content-Type', 'application/x-ndjson');
+    res.set('Access-Control-Expose-Headers', stormClient_1.ConversationIdHeader);
+    res.set(stormClient_1.ConversationIdHeader, systemId);
+    if (!fs_extra_1.default.existsSync(eventsFile)) {
+        res.status(404).send({ error: 'No events found' });
+        return;
+    }
+    res.send(fs_extra_1.default.readFileSync(eventsFile));
+});
+router.put('/ui/conversations/:systemId', async (req, res) => {
+    const systemId = req.params.systemId;
+    const events = req.stringBody ? JSON.parse(req.stringBody) : [];
+    await stormService_1.default.saveConversation(systemId, events);
+    res.send({ ok: true });
+});
+router.delete('/ui/conversations/:systemId', async (req, res) => {
+    const systemId = req.params.systemId;
+    await stormService_1.default.deleteConversation(systemId);
+    res.send({ ok: true });
+});
+router.post('/ui/conversations/:systemId/append', async (req, res) => {
+    const systemId = req.params.systemId;
+    const events = req.stringBody ? JSON.parse(req.stringBody) : [];
+    await stormService_1.default.appendConversation(systemId, events);
+    res.send({ ok: true });
+});
 router.post('/ui/create-system/:handle/:systemId', async (req, res) => {
     const systemId = req.params.systemId;
     const srcDir = (0, page_utils_1.getSystemBaseDir)(systemId);
@@ -142,6 +180,18 @@ router.post('/ui/create-system-simple/:handle/:systemId', async (req, res) => {
         }
     }
 });
+router.post('/ui/systems/:handle/:systemId/download', async (req, res) => {
+    const systemId = req.params.systemId;
+    const handle = req.params.handle;
+    await stormService_1.default.installProjectById(handle, systemId);
+    res.send({ ok: true });
+});
+router.post('/ui/systems/:handle/:systemId/upload', async (req, res) => {
+    const systemId = req.params.systemId;
+    const handle = req.params.handle;
+    await stormService_1.default.uploadConversation(handle, systemId);
+    res.send({ ok: true });
+});
 router.delete('/ui/serve/:systemId', async (req, res) => {
     const systemId = req.params.systemId;
     if (!systemId) {

package/dist/esm/src/storm/stormClient.d.ts

@@ -1,4 +1,5 @@
 /// <reference types="node" />
+/// <reference types="node" />
 import { ConversationItem, CreateSimpleBackendRequest, HTMLPage, ImplementAPIClients, StormFileImplementationPrompt, StormStream, StormUIImplementationPrompt, StormUIListPrompt } from './stream';
 import { Page, StormEventPageUrl } from './events';
 export declare const STORM_ID = "storm";
@@ -84,6 +85,8 @@ declare class StormClient {
     createErrorDetails(prompt: string, history?: ConversationItem[], conversationId?: string): Promise<StormStream>;
     generateCode(prompt: StormFileImplementationPrompt, conversationId?: string): Promise<StormStream>;
     deleteUIPageConversation(conversationId: string): Promise<string>;
+    downloadSystem(handle: string, conversationId: string): Promise<Buffer>;
+    uploadSystem(handle: string, conversationId: string, buffer: Buffer): Promise<Response>;
 }
 export declare const stormClient: StormClient;
 export {};

package/dist/esm/src/storm/stormClient.js

@@ -139,7 +139,7 @@ class StormClient {
             method: 'POST',
             body: JSON.stringify(prompt.pages),
         });
-        return await response.json();
+        return (await response.json());
     }
     async generatePrompt(pages) {
         const u = `${this._baseUrl}/v2/ui/prompt`;
@@ -232,5 +232,24 @@ class StormClient {
         const response = await fetch(options.url, options);
         return response.text();
     }
+    async downloadSystem(handle, conversationId) {
+        const u = `${this._baseUrl}/v2/systems/${handle}/${conversationId}/download`;
+        const response = await fetch(u);
+        if (!response.ok) {
+            throw new Error(`Failed to download system: ${response.status}`);
+        }
+        return Buffer.from(await response.arrayBuffer());
+    }
+    async uploadSystem(handle, conversationId, buffer) {
+        const u = `${this._baseUrl}/v2/systems/${handle}/${conversationId}`;
+        const response = await fetch(u, {
+            method: 'PUT',
+            body: buffer,
+            headers: {
+                'content-type': 'application/zip',
+            },
+        });
+        return response;
+    }
 }
 exports.stormClient = new StormClient();

package/dist/esm/src/stormService.d.ts

@@ -0,0 +1,19 @@
+import { StormEvent } from './storm/events';
+export declare class StormService {
+    private getConversationFile;
+    private getConversationTarball;
+    listRemoteConversations(): Promise<never[]>;
+    listLocalConversations(): Promise<{
+        id: string;
+        description: string;
+        title: string;
+    }[]>;
+    getConversation(conversationId: string): Promise<string>;
+    saveConversation(conversationId: string, events: StormEvent[]): Promise<void>;
+    appendConversation(conversationId: string, events: StormEvent[]): Promise<void>;
+    deleteConversation(conversationId: string): Promise<void>;
+    uploadConversation(handle: string, conversationId: string): Promise<void>;
+    installProjectById(handle: string, conversationId: string): Promise<void>;
+}
+declare const _default: StormService;
+export default _default;

package/dist/esm/src/stormService.js

@@ -0,0 +1,127 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StormService = void 0;
+const promises_1 = __importDefault(require("fs/promises"));
+const glob_1 = require("glob");
+const filesystemManager_1 = require("./filesystemManager");
+const path_1 = __importDefault(require("path"));
+const fs_1 = require("fs");
+const tar = __importStar(require("tar"));
+const stormClient_1 = require("./storm/stormClient");
+class StormService {
+    getConversationFile(conversationId) {
+        return path_1.default.join(filesystemManager_1.filesystemManager.getProjectRootFolder(), 'ai-systems', conversationId, 'events.ndjson');
+    }
+    getConversationTarball(conversationId) {
+        return path_1.default.join(filesystemManager_1.filesystemManager.getProjectRootFolder(), 'ai-systems', conversationId, 'system.tar.gz');
+    }
+    async listRemoteConversations() {
+        // i.e. conversations from org / user on registry
+        return [];
+    }
+    async listLocalConversations() {
+        const systemsFolder = path_1.default.join(filesystemManager_1.filesystemManager.getProjectRootFolder(), 'ai-systems');
+        const eventFiles = await (0, glob_1.glob)('*/events.ndjson', {
+            cwd: systemsFolder,
+            absolute: true,
+        });
+        // Returns list of UUIDs - probably want to make it more useful than that
+        const conversations = [];
+        for (const file of eventFiles) {
+            const nldContents = await promises_1.default.readFile(file, 'utf8');
+            const events = nldContents.split('\n').map((e) => JSON.parse(e));
+            // find the shell and get the title tag
+            const shellEvent = events.find((e) => e.type === 'AI' && e.event.type === 'UI_SHELL')?.event;
+            const html = shellEvent?.payload.content;
+            const title = html?.match(/<title>(.*?)<\/title>/)?.[1];
+            const id = events.find((e) => e.type === 'AI')?.systemId;
+            const initialPrompt = events.find((e) => e.type === 'AI' && e.event.type === 'PROMPT_IMPROVE')?.event?.payload?.prompt || events[0].text;
+            if (!id) {
+                continue;
+            }
+            conversations.push({
+                id,
+                description: initialPrompt,
+                title: title || 'New system',
+            });
+        }
+        return conversations;
+    }
+    async getConversation(conversationId) {
+        const conversationFile = this.getConversationFile(conversationId);
+        if (!(0, fs_1.existsSync)(conversationFile)) {
+            throw new Error('Conversation not found');
+        }
+        return promises_1.default.readFile(conversationFile, 'utf8');
+    }
+    async saveConversation(conversationId, events) {
+        const conversationFile = this.getConversationFile(conversationId);
+        await promises_1.default.mkdir(path_1.default.dirname(conversationFile), { recursive: true });
+        await promises_1.default.writeFile(conversationFile, events.map((e) => JSON.stringify(e)).join('\n'));
+    }
+    async appendConversation(conversationId, events) {
+        const conversationFile = this.getConversationFile(conversationId);
+        await promises_1.default.appendFile(conversationFile, events.map((e) => JSON.stringify(e)).join('\n'));
+    }
+    async deleteConversation(conversationId) {
+        const conversationFile = this.getConversationFile(conversationId);
+        await promises_1.default.unlink(conversationFile);
+        // if it matches a UUID, it's safe to delete the directory too
+        const conversationDir = path_1.default.dirname(conversationFile);
+        if (conversationDir.match(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/)) {
+            await promises_1.default.rm(conversationDir, { recursive: true, force: true });
+        }
+    }
+    async uploadConversation(handle, conversationId) {
+        const tarballFile = this.getConversationTarball(conversationId);
+        const destDir = path_1.default.dirname(tarballFile);
+        const tarballName = path_1.default.basename(tarballFile);
+        await tar.create({
+            file: tarballFile,
+            cwd: destDir,
+            gzip: true,
+            filter: (entry) => !entry.includes(tarballName),
+        }, ['.']);
+        await stormClient_1.stormClient.uploadSystem(handle, conversationId, await promises_1.default.readFile(tarballFile));
+    }
+    async installProjectById(handle, conversationId) {
+        const tarballFile = this.getConversationTarball(conversationId);
+        const destDir = path_1.default.dirname(tarballFile);
+        const buffer = await stormClient_1.stormClient.downloadSystem(handle, conversationId);
+        await promises_1.default.mkdir(destDir, { recursive: true });
+        await promises_1.default.writeFile(tarballFile, buffer);
+        await tar.extract({
+            file: tarballFile,
+            cwd: destDir,
+        });
+        await promises_1.default.unlink(tarballFile);
+    }
+}
+exports.StormService = StormService;
+exports.default = new StormService();

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@kapeta/local-cluster-service",
-    "version": "0.72.0",
+    "version": "0.73.0",
     "description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
     "type": "commonjs",
     "exports": {
@@ -72,7 +72,7 @@
         "express-promise-router": "^4.1.1",
         "fetch-retry": "^6.0.0",
         "fs-extra": "^11.1.0",
-        "glob": "^7.1.6",
+        "glob": "^11.0.0",
         "gunzip-maybe": "^1.4.2",
         "js-yaml": "^4.1.0",
         "lodash": "^4.17.15",
@@ -87,6 +87,7 @@
         "request": "2.88.2",
         "socket.io": "^4.5.2",
         "stream-json": "^1.8.0",
+        "tar": "^7.4.3",
         "tar-stream": "^3.1.6",
         "typescript": "^5.1.6",
         "uuid": "^9.0.1",
@@ -99,7 +100,6 @@
         "@types/async-lock": "^1.4.0",
         "@types/express": "^4.17.17",
         "@types/fs-extra": "^11.0.1",
-        "@types/glob": "^8.1.0",
         "@types/gunzip-maybe": "^1.4.0",
         "@types/jest": "^29.5.4",
         "@types/js-yaml": "^4.0.9",

package/src/storm/page-utils.ts

@@ -3,6 +3,7 @@
  * SPDX-License-Identifier: BUSL-1.1
  */
 import { StormEventFileDone, StormEventPage, StormImage } from './events';
+import { filesystemManager } from '../filesystemManager';
 
 import { Response } from 'express';
 import os from 'node:os';
@@ -12,6 +13,7 @@ import FSExtra from 'fs-extra';
 import { ConversationItem } from './stream';
 import exp from 'node:constants';
 import { ImagePrompt } from './PageGenerator';
+import clusterConfiguration from '@kapeta/local-cluster-config';
 
 export const SystemIdHeader = 'System-Id';
 
@@ -76,7 +78,7 @@ export function hasPageOnDisk(systemId: string, path: string, method: string) {
 }
 
 export function getSystemBaseDir(systemId: string) {
-    return Path.join(os.tmpdir(), 'ai-systems', systemId);
+    return Path.join(filesystemManager.getProjectRootFolder() || os.tmpdir(), 'ai-systems', systemId);
 }
 
 export function getSystemBaseImplDir(systemId: string) {

package/src/storm/routes.ts

@@ -49,9 +49,10 @@ import {
 } from './page-utils';
 import { UIServer } from './UIServer';
 import { randomUUID } from 'crypto';
-import { PageQueue } from './PageGenerator';
 import { copyDirectory, createFuture, readFilesAndContent } from './utils';
 import { getRemoteUrl } from '../utils/utils';
+import stormService from '../stormService';
+import { PageQueue } from './PageGenerator';
 
 const UI_SERVERS: { [key: string]: UIServer } = {};
 const router = Router();
@@ -105,6 +106,50 @@ router.post('/ui/serve/:systemId', async (req: KapetaBodyRequest, res: Response)
     res.status(200).send({ status: 'running', url: svr.getUrl(), resetUrl: svr.resolveUrlFromPath('/_reset') });
 });
 
+router.get('/ui/conversations', async (req: KapetaBodyRequest, res: Response) => {
+    const local = await stormService.listLocalConversations();
+    const remote = await stormService.listRemoteConversations();
+    res.send({
+        local,
+        remote,
+    });
+});
+
+router.get('/ui/conversations/:systemId', async (req: KapetaBodyRequest, res: Response) => {
+    const systemId = req.params.systemId as string;
+    const eventsFile = getSystemBaseDir(systemId) + '/events.ndjson';
+
+    res.set('Content-Type', 'application/x-ndjson');
+    res.set('Access-Control-Expose-Headers', ConversationIdHeader);
+    res.set(ConversationIdHeader, systemId);
+
+    if (!FS.existsSync(eventsFile)) {
+        res.status(404).send({ error: 'No events found' });
+        return;
+    }
+    res.send(FS.readFileSync(eventsFile));
+});
+
+router.put('/ui/conversations/:systemId', async (req: KapetaBodyRequest, res: Response) => {
+    const systemId = req.params.systemId as string;
+    const events: StormEvent[] = req.stringBody ? JSON.parse(req.stringBody) : [];
+    await stormService.saveConversation(systemId, events);
+    res.send({ ok: true });
+});
+
+router.delete('/ui/conversations/:systemId', async (req: KapetaBodyRequest, res: Response) => {
+    const systemId = req.params.systemId as string;
+    await stormService.deleteConversation(systemId);
+    res.send({ ok: true });
+});
+
+router.post('/ui/conversations/:systemId/append', async (req: KapetaBodyRequest, res: Response) => {
+    const systemId = req.params.systemId as string;
+    const events: StormEvent[] = req.stringBody ? JSON.parse(req.stringBody) : [];
+    await stormService.appendConversation(systemId, events);
+    res.send({ ok: true });
+});
+
 router.post('/ui/create-system/:handle/:systemId', async (req: KapetaBodyRequest, res: Response) => {
     const systemId = req.params.systemId as string;
     const srcDir = getSystemBaseDir(systemId);
@@ -197,6 +242,20 @@ router.post('/ui/create-system-simple/:handle/:systemId', async (req: KapetaBody
     }
 });
 
+router.post('/ui/systems/:handle/:systemId/download', async (req: KapetaBodyRequest, res: Response) => {
+    const systemId = req.params.systemId as string;
+    const handle = req.params.handle as string;
+    await stormService.installProjectById(handle, systemId);
+    res.send({ ok: true });
+});
+
+router.post('/ui/systems/:handle/:systemId/upload', async (req: KapetaBodyRequest, res: Response) => {
+    const systemId = req.params.systemId as string;
+    const handle = req.params.handle as string;
+    await stormService.uploadConversation(handle, systemId);
+    res.send({ ok: true });
+});
+
 router.delete('/ui/serve/:systemId', async (req: KapetaBodyRequest, res: Response) => {
     const systemId = req.params.systemId as string | undefined;
     if (!systemId) {
@@ -371,6 +430,7 @@ router.post('/:handle/ui/iterative', async (req: KapetaBodyRequest, res: Respons
 
 router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
     const handle = req.params.handle as string;
+
     try {
         const outerConversationId =
             (req.headers[ConversationIdHeader.toLowerCase()] as string | undefined) || randomUUID();
@@ -929,6 +989,7 @@ function sendEvent(res: Response, evt: StormEvent) {
     if (res.closed) {
         return;
     }
+
     res.write(JSON.stringify(evt) + '\n');
 }
 

package/src/storm/stormClient.ts

@@ -8,7 +8,8 @@ import readLine from 'node:readline/promises';
 import { Readable } from 'node:stream';
 import {
     ConversationItem,
-    CreateSimpleBackendRequest, HTMLPage,
+    CreateSimpleBackendRequest,
+    HTMLPage,
     ImplementAPIClients,
     StormContextRequest,
     StormFileImplementationPrompt,
@@ -245,7 +246,7 @@ class StormClient {
             method: 'POST',
             body: JSON.stringify(prompt.pages),
         });
-        return await response.json() as HTMLPage[];
+        return (await response.json()) as HTMLPage[];
     }
 
     public async generatePrompt(pages: string[]): Promise<string> {
@@ -353,6 +354,28 @@ class StormClient {
 
         return response.text();
     }
+
+    async downloadSystem(handle: string, conversationId: string) {
+        const u = `${this._baseUrl}/v2/systems/${handle}/${conversationId}/download`;
+        const response = await fetch(u);
+        if (!response.ok) {
+            throw new Error(`Failed to download system: ${response.status}`);
+        }
+        return Buffer.from(await response.arrayBuffer());
+    }
+
+    async uploadSystem(handle: string, conversationId: string, buffer: Buffer) {
+        const u = `${this._baseUrl}/v2/systems/${handle}/${conversationId}`;
+        const response = await fetch(u, {
+            method: 'PUT',
+            body: buffer,
+            headers: {
+                'content-type': 'application/zip',
+            },
+        });
+
+        return response;
+    }
 }
 
 export const stormClient = new StormClient();

package/src/stormService.ts

@@ -0,0 +1,129 @@
+import fs from 'fs/promises';
+import { glob } from 'glob';
+import { filesystemManager } from './filesystemManager';
+import path from 'path';
+import { existsSync } from 'fs';
+import { StormEvent, StormEventPromptImprove, StormEventUIShell } from './storm/events';
+import * as tar from 'tar';
+import { stormClient } from './storm/stormClient';
+
+export class StormService {
+    private getConversationFile(conversationId: string) {
+        return path.join(filesystemManager.getProjectRootFolder()!, 'ai-systems', conversationId, 'events.ndjson');
+    }
+
+    private getConversationTarball(conversationId: string) {
+        return path.join(filesystemManager.getProjectRootFolder()!, 'ai-systems', conversationId, 'system.tar.gz');
+    }
+
+    async listRemoteConversations() {
+        // i.e. conversations from org / user on registry
+        return [];
+    }
+
+    async listLocalConversations() {
+        const systemsFolder = path.join(filesystemManager.getProjectRootFolder()!, 'ai-systems');
+        const eventFiles = await glob('*/events.ndjson', {
+            cwd: systemsFolder,
+            absolute: true,
+        });
+        // Returns list of UUIDs - probably want to make it more useful than that
+        const conversations: { id: string; description: string; title: string }[] = [];
+        for (const file of eventFiles) {
+            const nldContents = await fs.readFile(file as string, 'utf8');
+            const events = nldContents.split('\n').map((e) => JSON.parse(e)) as {
+                // | { type: 'USER'; event: any } // IS stupid!
+                type: 'AI';
+                systemId: string;
+                event: StormEvent;
+            }[];
+            // find the shell and get the title tag
+            const shellEvent = events.find((e) => e.type === 'AI' && e.event.type === 'UI_SHELL')?.event as
+                | StormEventUIShell
+                | undefined;
+            const html = shellEvent?.payload.content;
+            const title = html?.match(/<title>(.*?)<\/title>/)?.[1];
+            const id = events.find((e) => e.type === 'AI')?.systemId;
+
+            const initialPrompt =
+                (
+                    events.find((e) => e.type === 'AI' && e.event.type === 'PROMPT_IMPROVE')?.event as
+                        | StormEventPromptImprove
+                        | undefined
+                )?.payload?.prompt || (events[0] as any).text;
+
+            if (!id) {
+                continue;
+            }
+
+            conversations.push({
+                id,
+                description: initialPrompt,
+                title: title || 'New system',
+            });
+        }
+
+        return conversations;
+    }
+
+    async getConversation(conversationId: string) {
+        const conversationFile = this.getConversationFile(conversationId);
+        if (!existsSync(conversationFile)) {
+            throw new Error('Conversation not found');
+        }
+        return fs.readFile(conversationFile, 'utf8');
+    }
+
+    async saveConversation(conversationId: string, events: StormEvent[]) {
+        const conversationFile = this.getConversationFile(conversationId);
+        await fs.mkdir(path.dirname(conversationFile), { recursive: true });
+        await fs.writeFile(conversationFile, events.map((e) => JSON.stringify(e)).join('\n'));
+    }
+
+    async appendConversation(conversationId: string, events: StormEvent[]) {
+        const conversationFile = this.getConversationFile(conversationId);
+        await fs.appendFile(conversationFile, events.map((e) => JSON.stringify(e)).join('\n'));
+    }
+
+    async deleteConversation(conversationId: string) {
+        const conversationFile = this.getConversationFile(conversationId);
+        await fs.unlink(conversationFile);
+
+        // if it matches a UUID, it's safe to delete the directory too
+        const conversationDir = path.dirname(conversationFile);
+        if (conversationDir.match(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/)) {
+            await fs.rm(conversationDir, { recursive: true, force: true });
+        }
+    }
+
+    async uploadConversation(handle: string, conversationId: string) {
+        const tarballFile = this.getConversationTarball(conversationId);
+        const destDir = path.dirname(tarballFile);
+        const tarballName = path.basename(tarballFile);
+        await tar.create(
+            {
+                file: tarballFile,
+                cwd: destDir,
+                gzip: true,
+                filter: (entry) => !entry.includes(tarballName),
+            },
+            ['.']
+        );
+        await stormClient.uploadSystem(handle, conversationId, await fs.readFile(tarballFile));
+    }
+
+    async installProjectById(handle: string, conversationId: string) {
+        const tarballFile = this.getConversationTarball(conversationId);
+        const destDir = path.dirname(tarballFile);
+        const buffer = await stormClient.downloadSystem(handle, conversationId);
+        await fs.mkdir(destDir, { recursive: true });
+        await fs.writeFile(tarballFile, buffer);
+        await tar.extract({
+            file: tarballFile,
+            cwd: destDir,
+        });
+        await fs.unlink(tarballFile);
+    }
+}
+
+export default new StormService();