@kapeta/local-cluster-service 0.66.0 → 0.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/cjs/src/storm/PageGenerator.d.ts +14 -1
- package/dist/cjs/src/storm/PageGenerator.js +30 -7
- package/dist/cjs/src/storm/PromiseQueue.d.ts +7 -0
- package/dist/cjs/src/storm/PromiseQueue.js +27 -1
- package/dist/cjs/src/storm/events.d.ts +9 -1
- package/dist/cjs/src/storm/page-utils.d.ts +5 -1
- package/dist/cjs/src/storm/page-utils.js +30 -3
- package/dist/cjs/src/storm/routes.js +49 -2
- package/dist/cjs/src/storm/stormClient.d.ts +1 -0
- package/dist/cjs/src/storm/stormClient.js +6 -0
- package/dist/esm/src/storm/PageGenerator.d.ts +14 -1
- package/dist/esm/src/storm/PageGenerator.js +30 -7
- package/dist/esm/src/storm/PromiseQueue.d.ts +7 -0
- package/dist/esm/src/storm/PromiseQueue.js +27 -1
- package/dist/esm/src/storm/events.d.ts +9 -1
- package/dist/esm/src/storm/page-utils.d.ts +5 -1
- package/dist/esm/src/storm/page-utils.js +30 -3
- package/dist/esm/src/storm/routes.js +49 -2
- package/dist/esm/src/storm/stormClient.d.ts +1 -0
- package/dist/esm/src/storm/stormClient.js +6 -0
- package/package.json +1 -1
- package/src/storm/PageGenerator.ts +51 -15
- package/src/storm/PromiseQueue.ts +34 -0
- package/src/storm/events.ts +11 -1
- package/src/storm/page-utils.ts +38 -4
- package/src/storm/routes.ts +51 -3
- package/src/storm/stormClient.ts +7 -0
package/CHANGELOG.md
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
# [0.67.0](https://github.com/kapetacom/local-cluster-service/compare/v0.66.0...v0.67.0) (2024-08-27)
|
2
|
+
|
3
|
+
|
4
|
+
### Features
|
5
|
+
|
6
|
+
* Resolve images from references ([#226](https://github.com/kapetacom/local-cluster-service/issues/226)) ([c8b47b3](https://github.com/kapetacom/local-cluster-service/commit/c8b47b3dea5293471f66e5e7a41ea496db8cf2c6))
|
7
|
+
|
1
8
|
# [0.66.0](https://github.com/kapetacom/local-cluster-service/compare/v0.65.0...v0.66.0) (2024-08-27)
|
2
9
|
|
3
10
|
|
@@ -4,8 +4,18 @@
|
|
4
4
|
*/
|
5
5
|
/// <reference types="node" />
|
6
6
|
import { UIPagePrompt } from './stormClient';
|
7
|
-
import { ReferenceClassification, StormEvent, StormEventPage, UIShell } from './events';
|
7
|
+
import { ReferenceClassification, StormEvent, StormEventPage, StormImage, UIShell } from './events';
|
8
8
|
import { EventEmitter } from 'node:events';
|
9
|
+
import { FuturePromise } from './PromiseQueue';
|
10
|
+
export interface ImagePrompt {
|
11
|
+
name: string;
|
12
|
+
description: string;
|
13
|
+
source: 'local' | 'cdn' | 'example';
|
14
|
+
title: string;
|
15
|
+
type: 'image' | 'css' | 'javascript' | 'html';
|
16
|
+
url: string;
|
17
|
+
content: string;
|
18
|
+
}
|
9
19
|
export declare class PageQueue extends EventEmitter {
|
10
20
|
private readonly queue;
|
11
21
|
private readonly systemId;
|
@@ -15,14 +25,17 @@ export declare class PageQueue extends EventEmitter {
|
|
15
25
|
constructor(systemId: string, concurrency?: number);
|
16
26
|
on(event: 'event', listener: (data: StormEvent) => void): this;
|
17
27
|
on(event: 'page', listener: (data: StormEventPage) => void): this;
|
28
|
+
on(event: 'image', listener: (data: StormImage, source: ImagePrompt, future: FuturePromise<void>) => void): this;
|
18
29
|
emit(type: 'event', event: StormEvent): boolean;
|
19
30
|
emit(type: 'page', event: StormEventPage): boolean;
|
31
|
+
emit(type: 'image', event: StormImage, source: ImagePrompt, future: FuturePromise<void>): boolean;
|
20
32
|
addUiShell(uiShell: UIShell): void;
|
21
33
|
setUiTheme(theme: string): void;
|
22
34
|
addPrompt(initialPrompt: Omit<UIPagePrompt, 'shell_page'>, conversationId?: string, overwrite?: boolean): Promise<void>;
|
23
35
|
private addPageGenerator;
|
24
36
|
cancel(): void;
|
25
37
|
wait(): Promise<void>;
|
38
|
+
private addImagePrompt;
|
26
39
|
}
|
27
40
|
export declare class PageGenerator extends EventEmitter {
|
28
41
|
private readonly conversationId;
|
@@ -38,11 +38,11 @@ class PageQueue extends node_events_1.EventEmitter {
|
|
38
38
|
}
|
39
39
|
addPrompt(initialPrompt, conversationId = node_uuid_1.default.v4(), overwrite = false) {
|
40
40
|
if (!overwrite && this.references.has(initialPrompt.path)) {
|
41
|
-
console.log('Ignoring duplicate prompt', initialPrompt.path);
|
41
|
+
//console.log('Ignoring duplicate prompt', initialPrompt.path);
|
42
42
|
return Promise.resolve();
|
43
43
|
}
|
44
44
|
if (!overwrite && (0, page_utils_1.hasPageOnDisk)(this.systemId, initialPrompt.method, initialPrompt.path)) {
|
45
|
-
console.log('Ignoring prompt with existing page', initialPrompt.path);
|
45
|
+
//console.log('Ignoring prompt with existing page', initialPrompt.path);
|
46
46
|
return Promise.resolve();
|
47
47
|
}
|
48
48
|
const prompt = {
|
@@ -56,9 +56,8 @@ class PageQueue extends node_events_1.EventEmitter {
|
|
56
56
|
}
|
57
57
|
async addPageGenerator(generator) {
|
58
58
|
generator.on('event', (event) => this.emit('event', event));
|
59
|
-
generator.on('page_refs', ({ event, references }) => {
|
60
|
-
|
61
|
-
references.forEach((reference) => {
|
59
|
+
generator.on('page_refs', async ({ event, references }) => {
|
60
|
+
const promises = references.map(async (reference) => {
|
62
61
|
if (reference.url.startsWith('#') ||
|
63
62
|
reference.url.startsWith('javascript:') ||
|
64
63
|
reference.url.startsWith('http://') ||
|
@@ -67,14 +66,17 @@ class PageQueue extends node_events_1.EventEmitter {
|
|
67
66
|
}
|
68
67
|
switch (reference.type) {
|
69
68
|
case 'image':
|
70
|
-
|
69
|
+
await this.addImagePrompt({
|
70
|
+
...reference,
|
71
|
+
content: event.payload.content,
|
72
|
+
});
|
71
73
|
break;
|
72
74
|
case 'css':
|
73
75
|
case 'javascript':
|
74
76
|
//console.log('Ignoring reference', reference);
|
75
77
|
break;
|
76
78
|
case 'html':
|
77
|
-
console.log('Adding page generator for', reference);
|
79
|
+
//console.log('Adding page generator for', reference);
|
78
80
|
const paths = Array.from(this.references.keys());
|
79
81
|
this.addPrompt({
|
80
82
|
name: reference.name,
|
@@ -94,6 +96,8 @@ class PageQueue extends node_events_1.EventEmitter {
|
|
94
96
|
break;
|
95
97
|
}
|
96
98
|
});
|
99
|
+
await Promise.allSettled(promises);
|
100
|
+
this.emit('page', event);
|
97
101
|
});
|
98
102
|
return this.queue.add(() => generator.generate());
|
99
103
|
}
|
@@ -103,6 +107,25 @@ class PageQueue extends node_events_1.EventEmitter {
|
|
103
107
|
wait() {
|
104
108
|
return this.queue.wait();
|
105
109
|
}
|
110
|
+
async addImagePrompt(prompt) {
|
111
|
+
const result = await stormClient_1.stormClient.createImage(`Create an image for the url "${prompt.url}" with this description: ${prompt.description}`.trim());
|
112
|
+
const futures = [];
|
113
|
+
result.on('data', async (event) => {
|
114
|
+
if (event.type === 'IMAGE') {
|
115
|
+
const future = (0, PromiseQueue_1.createFuture)();
|
116
|
+
futures.push(future);
|
117
|
+
this.emit('image', event, prompt, future);
|
118
|
+
setTimeout(() => {
|
119
|
+
if (!future.isResolved()) {
|
120
|
+
console.log('Image prompt timed out', prompt);
|
121
|
+
future.reject(new Error('Image prompt timed out'));
|
122
|
+
}
|
123
|
+
}, 30000);
|
124
|
+
}
|
125
|
+
});
|
126
|
+
await result.waitForDone();
|
127
|
+
await Promise.allSettled(futures.map((f) => f.promise));
|
128
|
+
}
|
106
129
|
}
|
107
130
|
exports.PageQueue = PageQueue;
|
108
131
|
class PageGenerator extends node_events_1.EventEmitter {
|
@@ -3,6 +3,13 @@
|
|
3
3
|
* SPDX-License-Identifier: BUSL-1.1
|
4
4
|
*/
|
5
5
|
export type Future<T> = () => Promise<T>;
|
6
|
+
export type FuturePromise<T> = {
|
7
|
+
promise: Promise<T>;
|
8
|
+
resolve: (value: T) => void;
|
9
|
+
reject: (reason: any) => void;
|
10
|
+
isResolved: () => boolean;
|
11
|
+
};
|
12
|
+
export declare function createFuture<T = void>(): FuturePromise<T>;
|
6
13
|
export declare class PromiseQueue {
|
7
14
|
private readonly queue;
|
8
15
|
private readonly active;
|
@@ -1,6 +1,32 @@
|
|
1
1
|
"use strict";
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
-
exports.PromiseQueue = void 0;
|
3
|
+
exports.PromiseQueue = exports.createFuture = void 0;
|
4
|
+
function createFuture() {
|
5
|
+
let resolved = false;
|
6
|
+
let resolve = () => {
|
7
|
+
resolved = true;
|
8
|
+
};
|
9
|
+
let reject = () => {
|
10
|
+
resolved = true;
|
11
|
+
};
|
12
|
+
const promise = new Promise((res, rej) => {
|
13
|
+
resolve = (value) => {
|
14
|
+
resolved = true;
|
15
|
+
res(value);
|
16
|
+
};
|
17
|
+
reject = (reason) => {
|
18
|
+
resolved = true;
|
19
|
+
rej(reason);
|
20
|
+
};
|
21
|
+
});
|
22
|
+
return {
|
23
|
+
promise,
|
24
|
+
resolve,
|
25
|
+
reject,
|
26
|
+
isResolved: () => resolved,
|
27
|
+
};
|
28
|
+
}
|
29
|
+
exports.createFuture = createFuture;
|
4
30
|
class PromiseQueue {
|
5
31
|
queue = [];
|
6
32
|
active = [];
|
@@ -255,6 +255,14 @@ export interface StormEventDone {
|
|
255
255
|
type: 'DONE';
|
256
256
|
created: number;
|
257
257
|
}
|
258
|
+
export interface StormImage {
|
259
|
+
type: 'IMAGE';
|
260
|
+
reason: string;
|
261
|
+
created: number;
|
262
|
+
payload: {
|
263
|
+
href: string;
|
264
|
+
};
|
265
|
+
}
|
258
266
|
export interface StormEventDefinitionChange {
|
259
267
|
type: 'DEFINITION_CHANGE';
|
260
268
|
reason: string;
|
@@ -384,5 +392,5 @@ export interface StormEventUIStarted {
|
|
384
392
|
resetUrl: string;
|
385
393
|
};
|
386
394
|
}
|
387
|
-
export type StormEvent = StormEventCreateBlock | StormEventCreateConnection | StormEventCreatePlanProperties | StormEventInvalidResponse | StormEventPlanRetry | StormEventCreateDSL | StormEventCreateDSLResource | StormEventError | StormEventScreen | StormEventScreenCandidate | StormEventFileLogical | StormEventFileState | StormEventFileDone | StormEventFileFailed | StormEventFileChunk | StormEventDone | StormEventDefinitionChange | StormEventErrorClassifier | StormEventCodeFix | StormEventErrorDetails | StormEventBlockReady | StormEventPhases | StormEventBlockStatus | StormEventCreateDSLRetry | StormEventUserJourney | StormEventUIShell | StormEventPage | StormEventPageUrl | StormEventPromptImprove | StormEventLandingPage | StormEventReferenceClassification | StormEventApiBase | StormEventUIStarted;
|
395
|
+
export type StormEvent = StormEventCreateBlock | StormEventCreateConnection | StormEventCreatePlanProperties | StormEventInvalidResponse | StormEventPlanRetry | StormEventCreateDSL | StormEventCreateDSLResource | StormEventError | StormEventScreen | StormEventScreenCandidate | StormEventFileLogical | StormEventFileState | StormEventFileDone | StormEventFileFailed | StormEventFileChunk | StormEventDone | StormEventDefinitionChange | StormEventErrorClassifier | StormEventCodeFix | StormEventErrorDetails | StormEventBlockReady | StormEventPhases | StormEventBlockStatus | StormEventCreateDSLRetry | StormEventUserJourney | StormEventUIShell | StormEventPage | StormEventPageUrl | StormEventPromptImprove | StormEventLandingPage | StormEventReferenceClassification | StormEventApiBase | StormEventUIStarted | StormImage;
|
388
396
|
export {};
|
@@ -2,9 +2,10 @@
|
|
2
2
|
* Copyright 2023 Kapeta Inc.
|
3
3
|
* SPDX-License-Identifier: BUSL-1.1
|
4
4
|
*/
|
5
|
-
import { StormEventFileDone, StormEventPage } from './events';
|
5
|
+
import { StormEventFileDone, StormEventPage, StormImage } from './events';
|
6
6
|
import { Response } from 'express';
|
7
7
|
import { ConversationItem } from './stream';
|
8
|
+
import { ImagePrompt } from './PageGenerator';
|
8
9
|
export declare const SystemIdHeader = "System-Id";
|
9
10
|
export declare function writePageToDisk(systemId: string, event: StormEventPage): Promise<{
|
10
11
|
path: string;
|
@@ -12,6 +13,9 @@ export declare function writePageToDisk(systemId: string, event: StormEventPage)
|
|
12
13
|
export declare function writeAssetToDisk(systemId: string, event: StormEventFileDone): Promise<{
|
13
14
|
path: string;
|
14
15
|
}>;
|
16
|
+
export declare function writeImageToDisk(systemId: string, event: StormImage, prompt: ImagePrompt): Promise<{
|
17
|
+
path: string;
|
18
|
+
}>;
|
15
19
|
export declare function hasPageOnDisk(systemId: string, method: string, path: string): boolean;
|
16
20
|
export declare function getSystemBaseDir(systemId: string): string;
|
17
21
|
export declare function resolveReadPath(systemId: string, path: string, method: string): string | null;
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
4
4
|
};
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
6
|
-
exports.writeConversationToFile = exports.readConversationFromFile = exports.readPageFromDisk = exports.readPageFromDiskAsString = exports.resolveReadPath = exports.getSystemBaseDir = exports.hasPageOnDisk = exports.writeAssetToDisk = exports.writePageToDisk = exports.SystemIdHeader = void 0;
|
6
|
+
exports.writeConversationToFile = exports.readConversationFromFile = exports.readPageFromDisk = exports.readPageFromDiskAsString = exports.resolveReadPath = exports.getSystemBaseDir = exports.hasPageOnDisk = exports.writeImageToDisk = exports.writeAssetToDisk = exports.writePageToDisk = exports.SystemIdHeader = void 0;
|
7
7
|
const node_os_1 = __importDefault(require("node:os"));
|
8
8
|
const path_1 = __importDefault(require("path"));
|
9
9
|
const fs_extra_1 = __importDefault(require("fs-extra"));
|
@@ -36,6 +36,22 @@ async function writeAssetToDisk(systemId, event) {
|
|
36
36
|
};
|
37
37
|
}
|
38
38
|
exports.writeAssetToDisk = writeAssetToDisk;
|
39
|
+
async function writeImageToDisk(systemId, event, prompt) {
|
40
|
+
const baseDir = getSystemBaseDir(systemId);
|
41
|
+
const path = path_1.default.join(baseDir, normalizePath(prompt.url));
|
42
|
+
const response = await fetch(event.payload.href);
|
43
|
+
if (!response.ok || !response.body) {
|
44
|
+
throw new Error(`Failed to fetch image: ${event.payload.href}`);
|
45
|
+
}
|
46
|
+
await fs_extra_1.default.ensureDir(path_1.default.dirname(path));
|
47
|
+
const buffer = await response.arrayBuffer();
|
48
|
+
await fs_extra_1.default.writeFile(path, Buffer.from(buffer));
|
49
|
+
console.log(`Image written to disk: ${event.payload.href} > ${path}`);
|
50
|
+
return {
|
51
|
+
path,
|
52
|
+
};
|
53
|
+
}
|
54
|
+
exports.writeImageToDisk = writeImageToDisk;
|
39
55
|
function hasPageOnDisk(systemId, method, path) {
|
40
56
|
const baseDir = getSystemBaseDir(systemId);
|
41
57
|
const filePath = getFilePath(method);
|
@@ -53,8 +69,19 @@ function getFilePath(method) {
|
|
53
69
|
function resolveReadPath(systemId, path, method) {
|
54
70
|
const baseDir = getSystemBaseDir(systemId);
|
55
71
|
path = normalizePath(path);
|
72
|
+
let fullPath = path_1.default.join(baseDir, path);
|
73
|
+
//First check if there is a file at the exact path
|
74
|
+
try {
|
75
|
+
const stat = fs_extra_1.default.statSync(fullPath);
|
76
|
+
if (stat && stat.isFile()) {
|
77
|
+
return fullPath;
|
78
|
+
}
|
79
|
+
}
|
80
|
+
catch (e) {
|
81
|
+
// Ignore
|
82
|
+
}
|
56
83
|
const filePath = getFilePath(method);
|
57
|
-
|
84
|
+
fullPath = path_1.default.join(baseDir, path, filePath);
|
58
85
|
if (fs_extra_1.default.existsSync(fullPath)) {
|
59
86
|
return fullPath;
|
60
87
|
}
|
@@ -95,7 +122,7 @@ function readPageFromDisk(systemId, path, method, res) {
|
|
95
122
|
return;
|
96
123
|
}
|
97
124
|
res.type(filePath.split('.').pop());
|
98
|
-
const content = fs_extra_1.default.readFileSync(filePath
|
125
|
+
const content = fs_extra_1.default.readFileSync(filePath);
|
99
126
|
res.write(content);
|
100
127
|
res.end();
|
101
128
|
}
|
@@ -21,8 +21,8 @@ const assetManager_1 = require("../assetManager");
|
|
21
21
|
const node_uuid_1 = __importDefault(require("node-uuid"));
|
22
22
|
const page_utils_1 = require("./page-utils");
|
23
23
|
const UIServer_1 = require("./UIServer");
|
24
|
-
const PageGenerator_1 = require("./PageGenerator");
|
25
24
|
const crypto_1 = require("crypto");
|
25
|
+
const PageGenerator_1 = require("./PageGenerator");
|
26
26
|
const UI_SERVERS = {};
|
27
27
|
const router = (0, express_promise_router_1.default)();
|
28
28
|
router.use('/', cors_1.corsHandler);
|
@@ -78,6 +78,21 @@ router.post('/ui/screen', async (req, res) => {
|
|
78
78
|
promises.push(sendPageEvent(systemId, data, res));
|
79
79
|
}
|
80
80
|
});
|
81
|
+
queue.on('image', async (screenData, prompt, future) => {
|
82
|
+
if (!systemId) {
|
83
|
+
return;
|
84
|
+
}
|
85
|
+
try {
|
86
|
+
const promise = handleImageEvent(systemId, screenData, prompt);
|
87
|
+
promises.push(promise);
|
88
|
+
await promise;
|
89
|
+
future.resolve();
|
90
|
+
}
|
91
|
+
catch (e) {
|
92
|
+
console.error('Failed to handle image event', e);
|
93
|
+
future.reject(e);
|
94
|
+
}
|
95
|
+
});
|
81
96
|
await queue.addPrompt(aiRequest, conversationId, true);
|
82
97
|
await queue.wait();
|
83
98
|
await Promise.allSettled(promises);
|
@@ -159,6 +174,18 @@ router.post('/:handle/ui/iterative', async (req, res) => {
|
|
159
174
|
pageQueue.on('page', (screenData) => {
|
160
175
|
pageEventPromises.push(sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
|
161
176
|
});
|
177
|
+
pageQueue.on('image', async (screenData, prompt, future) => {
|
178
|
+
try {
|
179
|
+
const promise = handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
|
180
|
+
pageEventPromises.push(promise);
|
181
|
+
await promise;
|
182
|
+
future.resolve();
|
183
|
+
}
|
184
|
+
catch (e) {
|
185
|
+
console.error('Failed to handle image event', e);
|
186
|
+
future.reject(e);
|
187
|
+
}
|
188
|
+
});
|
162
189
|
pageQueue.on('event', (screenData) => {
|
163
190
|
sendEvent(res, screenData);
|
164
191
|
});
|
@@ -274,7 +301,7 @@ router.post('/:handle/ui', async (req, res) => {
|
|
274
301
|
const queue = new PageGenerator_1.PageQueue(outerConversationId, 5);
|
275
302
|
queue.setUiTheme(theme);
|
276
303
|
shellsStream.on('data', (data) => {
|
277
|
-
console.log('Processing shell event', data);
|
304
|
+
//console.log('Processing shell event', data);
|
278
305
|
sendEvent(res, data);
|
279
306
|
if (data.type !== 'UI_SHELL') {
|
280
307
|
return;
|
@@ -310,6 +337,18 @@ router.post('/:handle/ui', async (req, res) => {
|
|
310
337
|
queue.on('page', (screenData) => {
|
311
338
|
pageEventPromises.push(sendPageEvent(outerConversationId, screenData, res));
|
312
339
|
});
|
340
|
+
queue.on('image', async (screenData, prompt, future) => {
|
341
|
+
try {
|
342
|
+
const promise = handleImageEvent(outerConversationId, screenData, prompt);
|
343
|
+
pageEventPromises.push(promise);
|
344
|
+
await promise;
|
345
|
+
future.resolve();
|
346
|
+
}
|
347
|
+
catch (e) {
|
348
|
+
console.error('Failed to handle image event', e);
|
349
|
+
future.reject(e);
|
350
|
+
}
|
351
|
+
});
|
313
352
|
queue.on('event', (screenData) => {
|
314
353
|
sendEvent(res, screenData);
|
315
354
|
});
|
@@ -625,4 +664,12 @@ async function sendPageEvent(mainConversationId, data, res) {
|
|
625
664
|
}
|
626
665
|
sendEvent(res, convertPageEvent(data, data.payload.conversationId, mainConversationId));
|
627
666
|
}
|
667
|
+
async function handleImageEvent(mainConversationId, data, prompt) {
|
668
|
+
try {
|
669
|
+
await (0, page_utils_1.writeImageToDisk)(mainConversationId, data, prompt);
|
670
|
+
}
|
671
|
+
catch (err) {
|
672
|
+
console.error('Failed to write image to disk', err);
|
673
|
+
}
|
674
|
+
}
|
628
675
|
exports.default = router;
|
@@ -73,6 +73,7 @@ declare class StormClient {
|
|
73
73
|
classifyUIReferences(prompt: string, conversationId?: string): Promise<StormStream>;
|
74
74
|
editPages(prompt: UIPageEditPrompt, conversationId?: string): Promise<StormStream>;
|
75
75
|
listScreens(prompt: StormUIListPrompt, conversationId?: string): Promise<StormStream>;
|
76
|
+
createImage(prompt: string, conversationId?: string): Promise<StormStream>;
|
76
77
|
createUIImplementation(prompt: StormUIImplementationPrompt, conversationId?: string): Promise<StormStream>;
|
77
78
|
createServiceImplementation(prompt: StormFileImplementationPrompt, conversationId?: string): Promise<StormStream>;
|
78
79
|
createErrorClassification(prompt: string, history?: ConversationItem[], conversationId?: string): Promise<StormStream>;
|
@@ -155,6 +155,12 @@ class StormClient {
|
|
155
155
|
conversationId,
|
156
156
|
});
|
157
157
|
}
|
158
|
+
createImage(prompt, conversationId) {
|
159
|
+
return this.send('/v2/ui/image', {
|
160
|
+
prompt,
|
161
|
+
conversationId,
|
162
|
+
});
|
163
|
+
}
|
158
164
|
createUIImplementation(prompt, conversationId) {
|
159
165
|
return this.send('/v2/ui/merge', {
|
160
166
|
prompt,
|
@@ -4,8 +4,18 @@
|
|
4
4
|
*/
|
5
5
|
/// <reference types="node" />
|
6
6
|
import { UIPagePrompt } from './stormClient';
|
7
|
-
import { ReferenceClassification, StormEvent, StormEventPage, UIShell } from './events';
|
7
|
+
import { ReferenceClassification, StormEvent, StormEventPage, StormImage, UIShell } from './events';
|
8
8
|
import { EventEmitter } from 'node:events';
|
9
|
+
import { FuturePromise } from './PromiseQueue';
|
10
|
+
export interface ImagePrompt {
|
11
|
+
name: string;
|
12
|
+
description: string;
|
13
|
+
source: 'local' | 'cdn' | 'example';
|
14
|
+
title: string;
|
15
|
+
type: 'image' | 'css' | 'javascript' | 'html';
|
16
|
+
url: string;
|
17
|
+
content: string;
|
18
|
+
}
|
9
19
|
export declare class PageQueue extends EventEmitter {
|
10
20
|
private readonly queue;
|
11
21
|
private readonly systemId;
|
@@ -15,14 +25,17 @@ export declare class PageQueue extends EventEmitter {
|
|
15
25
|
constructor(systemId: string, concurrency?: number);
|
16
26
|
on(event: 'event', listener: (data: StormEvent) => void): this;
|
17
27
|
on(event: 'page', listener: (data: StormEventPage) => void): this;
|
28
|
+
on(event: 'image', listener: (data: StormImage, source: ImagePrompt, future: FuturePromise<void>) => void): this;
|
18
29
|
emit(type: 'event', event: StormEvent): boolean;
|
19
30
|
emit(type: 'page', event: StormEventPage): boolean;
|
31
|
+
emit(type: 'image', event: StormImage, source: ImagePrompt, future: FuturePromise<void>): boolean;
|
20
32
|
addUiShell(uiShell: UIShell): void;
|
21
33
|
setUiTheme(theme: string): void;
|
22
34
|
addPrompt(initialPrompt: Omit<UIPagePrompt, 'shell_page'>, conversationId?: string, overwrite?: boolean): Promise<void>;
|
23
35
|
private addPageGenerator;
|
24
36
|
cancel(): void;
|
25
37
|
wait(): Promise<void>;
|
38
|
+
private addImagePrompt;
|
26
39
|
}
|
27
40
|
export declare class PageGenerator extends EventEmitter {
|
28
41
|
private readonly conversationId;
|
@@ -38,11 +38,11 @@ class PageQueue extends node_events_1.EventEmitter {
|
|
38
38
|
}
|
39
39
|
addPrompt(initialPrompt, conversationId = node_uuid_1.default.v4(), overwrite = false) {
|
40
40
|
if (!overwrite && this.references.has(initialPrompt.path)) {
|
41
|
-
console.log('Ignoring duplicate prompt', initialPrompt.path);
|
41
|
+
//console.log('Ignoring duplicate prompt', initialPrompt.path);
|
42
42
|
return Promise.resolve();
|
43
43
|
}
|
44
44
|
if (!overwrite && (0, page_utils_1.hasPageOnDisk)(this.systemId, initialPrompt.method, initialPrompt.path)) {
|
45
|
-
console.log('Ignoring prompt with existing page', initialPrompt.path);
|
45
|
+
//console.log('Ignoring prompt with existing page', initialPrompt.path);
|
46
46
|
return Promise.resolve();
|
47
47
|
}
|
48
48
|
const prompt = {
|
@@ -56,9 +56,8 @@ class PageQueue extends node_events_1.EventEmitter {
|
|
56
56
|
}
|
57
57
|
async addPageGenerator(generator) {
|
58
58
|
generator.on('event', (event) => this.emit('event', event));
|
59
|
-
generator.on('page_refs', ({ event, references }) => {
|
60
|
-
|
61
|
-
references.forEach((reference) => {
|
59
|
+
generator.on('page_refs', async ({ event, references }) => {
|
60
|
+
const promises = references.map(async (reference) => {
|
62
61
|
if (reference.url.startsWith('#') ||
|
63
62
|
reference.url.startsWith('javascript:') ||
|
64
63
|
reference.url.startsWith('http://') ||
|
@@ -67,14 +66,17 @@ class PageQueue extends node_events_1.EventEmitter {
|
|
67
66
|
}
|
68
67
|
switch (reference.type) {
|
69
68
|
case 'image':
|
70
|
-
|
69
|
+
await this.addImagePrompt({
|
70
|
+
...reference,
|
71
|
+
content: event.payload.content,
|
72
|
+
});
|
71
73
|
break;
|
72
74
|
case 'css':
|
73
75
|
case 'javascript':
|
74
76
|
//console.log('Ignoring reference', reference);
|
75
77
|
break;
|
76
78
|
case 'html':
|
77
|
-
console.log('Adding page generator for', reference);
|
79
|
+
//console.log('Adding page generator for', reference);
|
78
80
|
const paths = Array.from(this.references.keys());
|
79
81
|
this.addPrompt({
|
80
82
|
name: reference.name,
|
@@ -94,6 +96,8 @@ class PageQueue extends node_events_1.EventEmitter {
|
|
94
96
|
break;
|
95
97
|
}
|
96
98
|
});
|
99
|
+
await Promise.allSettled(promises);
|
100
|
+
this.emit('page', event);
|
97
101
|
});
|
98
102
|
return this.queue.add(() => generator.generate());
|
99
103
|
}
|
@@ -103,6 +107,25 @@ class PageQueue extends node_events_1.EventEmitter {
|
|
103
107
|
wait() {
|
104
108
|
return this.queue.wait();
|
105
109
|
}
|
110
|
+
async addImagePrompt(prompt) {
|
111
|
+
const result = await stormClient_1.stormClient.createImage(`Create an image for the url "${prompt.url}" with this description: ${prompt.description}`.trim());
|
112
|
+
const futures = [];
|
113
|
+
result.on('data', async (event) => {
|
114
|
+
if (event.type === 'IMAGE') {
|
115
|
+
const future = (0, PromiseQueue_1.createFuture)();
|
116
|
+
futures.push(future);
|
117
|
+
this.emit('image', event, prompt, future);
|
118
|
+
setTimeout(() => {
|
119
|
+
if (!future.isResolved()) {
|
120
|
+
console.log('Image prompt timed out', prompt);
|
121
|
+
future.reject(new Error('Image prompt timed out'));
|
122
|
+
}
|
123
|
+
}, 30000);
|
124
|
+
}
|
125
|
+
});
|
126
|
+
await result.waitForDone();
|
127
|
+
await Promise.allSettled(futures.map((f) => f.promise));
|
128
|
+
}
|
106
129
|
}
|
107
130
|
exports.PageQueue = PageQueue;
|
108
131
|
class PageGenerator extends node_events_1.EventEmitter {
|
@@ -3,6 +3,13 @@
|
|
3
3
|
* SPDX-License-Identifier: BUSL-1.1
|
4
4
|
*/
|
5
5
|
export type Future<T> = () => Promise<T>;
|
6
|
+
export type FuturePromise<T> = {
|
7
|
+
promise: Promise<T>;
|
8
|
+
resolve: (value: T) => void;
|
9
|
+
reject: (reason: any) => void;
|
10
|
+
isResolved: () => boolean;
|
11
|
+
};
|
12
|
+
export declare function createFuture<T = void>(): FuturePromise<T>;
|
6
13
|
export declare class PromiseQueue {
|
7
14
|
private readonly queue;
|
8
15
|
private readonly active;
|
@@ -1,6 +1,32 @@
|
|
1
1
|
"use strict";
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
-
exports.PromiseQueue = void 0;
|
3
|
+
exports.PromiseQueue = exports.createFuture = void 0;
|
4
|
+
function createFuture() {
|
5
|
+
let resolved = false;
|
6
|
+
let resolve = () => {
|
7
|
+
resolved = true;
|
8
|
+
};
|
9
|
+
let reject = () => {
|
10
|
+
resolved = true;
|
11
|
+
};
|
12
|
+
const promise = new Promise((res, rej) => {
|
13
|
+
resolve = (value) => {
|
14
|
+
resolved = true;
|
15
|
+
res(value);
|
16
|
+
};
|
17
|
+
reject = (reason) => {
|
18
|
+
resolved = true;
|
19
|
+
rej(reason);
|
20
|
+
};
|
21
|
+
});
|
22
|
+
return {
|
23
|
+
promise,
|
24
|
+
resolve,
|
25
|
+
reject,
|
26
|
+
isResolved: () => resolved,
|
27
|
+
};
|
28
|
+
}
|
29
|
+
exports.createFuture = createFuture;
|
4
30
|
class PromiseQueue {
|
5
31
|
queue = [];
|
6
32
|
active = [];
|
@@ -255,6 +255,14 @@ export interface StormEventDone {
|
|
255
255
|
type: 'DONE';
|
256
256
|
created: number;
|
257
257
|
}
|
258
|
+
export interface StormImage {
|
259
|
+
type: 'IMAGE';
|
260
|
+
reason: string;
|
261
|
+
created: number;
|
262
|
+
payload: {
|
263
|
+
href: string;
|
264
|
+
};
|
265
|
+
}
|
258
266
|
export interface StormEventDefinitionChange {
|
259
267
|
type: 'DEFINITION_CHANGE';
|
260
268
|
reason: string;
|
@@ -384,5 +392,5 @@ export interface StormEventUIStarted {
|
|
384
392
|
resetUrl: string;
|
385
393
|
};
|
386
394
|
}
|
387
|
-
export type StormEvent = StormEventCreateBlock | StormEventCreateConnection | StormEventCreatePlanProperties | StormEventInvalidResponse | StormEventPlanRetry | StormEventCreateDSL | StormEventCreateDSLResource | StormEventError | StormEventScreen | StormEventScreenCandidate | StormEventFileLogical | StormEventFileState | StormEventFileDone | StormEventFileFailed | StormEventFileChunk | StormEventDone | StormEventDefinitionChange | StormEventErrorClassifier | StormEventCodeFix | StormEventErrorDetails | StormEventBlockReady | StormEventPhases | StormEventBlockStatus | StormEventCreateDSLRetry | StormEventUserJourney | StormEventUIShell | StormEventPage | StormEventPageUrl | StormEventPromptImprove | StormEventLandingPage | StormEventReferenceClassification | StormEventApiBase | StormEventUIStarted;
|
395
|
+
export type StormEvent = StormEventCreateBlock | StormEventCreateConnection | StormEventCreatePlanProperties | StormEventInvalidResponse | StormEventPlanRetry | StormEventCreateDSL | StormEventCreateDSLResource | StormEventError | StormEventScreen | StormEventScreenCandidate | StormEventFileLogical | StormEventFileState | StormEventFileDone | StormEventFileFailed | StormEventFileChunk | StormEventDone | StormEventDefinitionChange | StormEventErrorClassifier | StormEventCodeFix | StormEventErrorDetails | StormEventBlockReady | StormEventPhases | StormEventBlockStatus | StormEventCreateDSLRetry | StormEventUserJourney | StormEventUIShell | StormEventPage | StormEventPageUrl | StormEventPromptImprove | StormEventLandingPage | StormEventReferenceClassification | StormEventApiBase | StormEventUIStarted | StormImage;
|
388
396
|
export {};
|
@@ -2,9 +2,10 @@
|
|
2
2
|
* Copyright 2023 Kapeta Inc.
|
3
3
|
* SPDX-License-Identifier: BUSL-1.1
|
4
4
|
*/
|
5
|
-
import { StormEventFileDone, StormEventPage } from './events';
|
5
|
+
import { StormEventFileDone, StormEventPage, StormImage } from './events';
|
6
6
|
import { Response } from 'express';
|
7
7
|
import { ConversationItem } from './stream';
|
8
|
+
import { ImagePrompt } from './PageGenerator';
|
8
9
|
export declare const SystemIdHeader = "System-Id";
|
9
10
|
export declare function writePageToDisk(systemId: string, event: StormEventPage): Promise<{
|
10
11
|
path: string;
|
@@ -12,6 +13,9 @@ export declare function writePageToDisk(systemId: string, event: StormEventPage)
|
|
12
13
|
export declare function writeAssetToDisk(systemId: string, event: StormEventFileDone): Promise<{
|
13
14
|
path: string;
|
14
15
|
}>;
|
16
|
+
export declare function writeImageToDisk(systemId: string, event: StormImage, prompt: ImagePrompt): Promise<{
|
17
|
+
path: string;
|
18
|
+
}>;
|
15
19
|
export declare function hasPageOnDisk(systemId: string, method: string, path: string): boolean;
|
16
20
|
export declare function getSystemBaseDir(systemId: string): string;
|
17
21
|
export declare function resolveReadPath(systemId: string, path: string, method: string): string | null;
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
4
4
|
};
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
6
|
-
exports.writeConversationToFile = exports.readConversationFromFile = exports.readPageFromDisk = exports.readPageFromDiskAsString = exports.resolveReadPath = exports.getSystemBaseDir = exports.hasPageOnDisk = exports.writeAssetToDisk = exports.writePageToDisk = exports.SystemIdHeader = void 0;
|
6
|
+
exports.writeConversationToFile = exports.readConversationFromFile = exports.readPageFromDisk = exports.readPageFromDiskAsString = exports.resolveReadPath = exports.getSystemBaseDir = exports.hasPageOnDisk = exports.writeImageToDisk = exports.writeAssetToDisk = exports.writePageToDisk = exports.SystemIdHeader = void 0;
|
7
7
|
const node_os_1 = __importDefault(require("node:os"));
|
8
8
|
const path_1 = __importDefault(require("path"));
|
9
9
|
const fs_extra_1 = __importDefault(require("fs-extra"));
|
@@ -36,6 +36,22 @@ async function writeAssetToDisk(systemId, event) {
|
|
36
36
|
};
|
37
37
|
}
|
38
38
|
exports.writeAssetToDisk = writeAssetToDisk;
|
39
|
+
async function writeImageToDisk(systemId, event, prompt) {
|
40
|
+
const baseDir = getSystemBaseDir(systemId);
|
41
|
+
const path = path_1.default.join(baseDir, normalizePath(prompt.url));
|
42
|
+
const response = await fetch(event.payload.href);
|
43
|
+
if (!response.ok || !response.body) {
|
44
|
+
throw new Error(`Failed to fetch image: ${event.payload.href}`);
|
45
|
+
}
|
46
|
+
await fs_extra_1.default.ensureDir(path_1.default.dirname(path));
|
47
|
+
const buffer = await response.arrayBuffer();
|
48
|
+
await fs_extra_1.default.writeFile(path, Buffer.from(buffer));
|
49
|
+
console.log(`Image written to disk: ${event.payload.href} > ${path}`);
|
50
|
+
return {
|
51
|
+
path,
|
52
|
+
};
|
53
|
+
}
|
54
|
+
exports.writeImageToDisk = writeImageToDisk;
|
39
55
|
function hasPageOnDisk(systemId, method, path) {
|
40
56
|
const baseDir = getSystemBaseDir(systemId);
|
41
57
|
const filePath = getFilePath(method);
|
@@ -53,8 +69,19 @@ function getFilePath(method) {
|
|
53
69
|
function resolveReadPath(systemId, path, method) {
|
54
70
|
const baseDir = getSystemBaseDir(systemId);
|
55
71
|
path = normalizePath(path);
|
72
|
+
let fullPath = path_1.default.join(baseDir, path);
|
73
|
+
//First check if there is a file at the exact path
|
74
|
+
try {
|
75
|
+
const stat = fs_extra_1.default.statSync(fullPath);
|
76
|
+
if (stat && stat.isFile()) {
|
77
|
+
return fullPath;
|
78
|
+
}
|
79
|
+
}
|
80
|
+
catch (e) {
|
81
|
+
// Ignore
|
82
|
+
}
|
56
83
|
const filePath = getFilePath(method);
|
57
|
-
|
84
|
+
fullPath = path_1.default.join(baseDir, path, filePath);
|
58
85
|
if (fs_extra_1.default.existsSync(fullPath)) {
|
59
86
|
return fullPath;
|
60
87
|
}
|
@@ -95,7 +122,7 @@ function readPageFromDisk(systemId, path, method, res) {
|
|
95
122
|
return;
|
96
123
|
}
|
97
124
|
res.type(filePath.split('.').pop());
|
98
|
-
const content = fs_extra_1.default.readFileSync(filePath
|
125
|
+
const content = fs_extra_1.default.readFileSync(filePath);
|
99
126
|
res.write(content);
|
100
127
|
res.end();
|
101
128
|
}
|
@@ -21,8 +21,8 @@ const assetManager_1 = require("../assetManager");
|
|
21
21
|
const node_uuid_1 = __importDefault(require("node-uuid"));
|
22
22
|
const page_utils_1 = require("./page-utils");
|
23
23
|
const UIServer_1 = require("./UIServer");
|
24
|
-
const PageGenerator_1 = require("./PageGenerator");
|
25
24
|
const crypto_1 = require("crypto");
|
25
|
+
const PageGenerator_1 = require("./PageGenerator");
|
26
26
|
const UI_SERVERS = {};
|
27
27
|
const router = (0, express_promise_router_1.default)();
|
28
28
|
router.use('/', cors_1.corsHandler);
|
@@ -78,6 +78,21 @@ router.post('/ui/screen', async (req, res) => {
|
|
78
78
|
promises.push(sendPageEvent(systemId, data, res));
|
79
79
|
}
|
80
80
|
});
|
81
|
+
queue.on('image', async (screenData, prompt, future) => {
|
82
|
+
if (!systemId) {
|
83
|
+
return;
|
84
|
+
}
|
85
|
+
try {
|
86
|
+
const promise = handleImageEvent(systemId, screenData, prompt);
|
87
|
+
promises.push(promise);
|
88
|
+
await promise;
|
89
|
+
future.resolve();
|
90
|
+
}
|
91
|
+
catch (e) {
|
92
|
+
console.error('Failed to handle image event', e);
|
93
|
+
future.reject(e);
|
94
|
+
}
|
95
|
+
});
|
81
96
|
await queue.addPrompt(aiRequest, conversationId, true);
|
82
97
|
await queue.wait();
|
83
98
|
await Promise.allSettled(promises);
|
@@ -159,6 +174,18 @@ router.post('/:handle/ui/iterative', async (req, res) => {
|
|
159
174
|
pageQueue.on('page', (screenData) => {
|
160
175
|
pageEventPromises.push(sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
|
161
176
|
});
|
177
|
+
pageQueue.on('image', async (screenData, prompt, future) => {
|
178
|
+
try {
|
179
|
+
const promise = handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
|
180
|
+
pageEventPromises.push(promise);
|
181
|
+
await promise;
|
182
|
+
future.resolve();
|
183
|
+
}
|
184
|
+
catch (e) {
|
185
|
+
console.error('Failed to handle image event', e);
|
186
|
+
future.reject(e);
|
187
|
+
}
|
188
|
+
});
|
162
189
|
pageQueue.on('event', (screenData) => {
|
163
190
|
sendEvent(res, screenData);
|
164
191
|
});
|
@@ -274,7 +301,7 @@ router.post('/:handle/ui', async (req, res) => {
|
|
274
301
|
const queue = new PageGenerator_1.PageQueue(outerConversationId, 5);
|
275
302
|
queue.setUiTheme(theme);
|
276
303
|
shellsStream.on('data', (data) => {
|
277
|
-
console.log('Processing shell event', data);
|
304
|
+
//console.log('Processing shell event', data);
|
278
305
|
sendEvent(res, data);
|
279
306
|
if (data.type !== 'UI_SHELL') {
|
280
307
|
return;
|
@@ -310,6 +337,18 @@ router.post('/:handle/ui', async (req, res) => {
|
|
310
337
|
queue.on('page', (screenData) => {
|
311
338
|
pageEventPromises.push(sendPageEvent(outerConversationId, screenData, res));
|
312
339
|
});
|
340
|
+
queue.on('image', async (screenData, prompt, future) => {
|
341
|
+
try {
|
342
|
+
const promise = handleImageEvent(outerConversationId, screenData, prompt);
|
343
|
+
pageEventPromises.push(promise);
|
344
|
+
await promise;
|
345
|
+
future.resolve();
|
346
|
+
}
|
347
|
+
catch (e) {
|
348
|
+
console.error('Failed to handle image event', e);
|
349
|
+
future.reject(e);
|
350
|
+
}
|
351
|
+
});
|
313
352
|
queue.on('event', (screenData) => {
|
314
353
|
sendEvent(res, screenData);
|
315
354
|
});
|
@@ -625,4 +664,12 @@ async function sendPageEvent(mainConversationId, data, res) {
|
|
625
664
|
}
|
626
665
|
sendEvent(res, convertPageEvent(data, data.payload.conversationId, mainConversationId));
|
627
666
|
}
|
667
|
+
async function handleImageEvent(mainConversationId, data, prompt) {
|
668
|
+
try {
|
669
|
+
await (0, page_utils_1.writeImageToDisk)(mainConversationId, data, prompt);
|
670
|
+
}
|
671
|
+
catch (err) {
|
672
|
+
console.error('Failed to write image to disk', err);
|
673
|
+
}
|
674
|
+
}
|
628
675
|
exports.default = router;
|
@@ -73,6 +73,7 @@ declare class StormClient {
|
|
73
73
|
classifyUIReferences(prompt: string, conversationId?: string): Promise<StormStream>;
|
74
74
|
editPages(prompt: UIPageEditPrompt, conversationId?: string): Promise<StormStream>;
|
75
75
|
listScreens(prompt: StormUIListPrompt, conversationId?: string): Promise<StormStream>;
|
76
|
+
createImage(prompt: string, conversationId?: string): Promise<StormStream>;
|
76
77
|
createUIImplementation(prompt: StormUIImplementationPrompt, conversationId?: string): Promise<StormStream>;
|
77
78
|
createServiceImplementation(prompt: StormFileImplementationPrompt, conversationId?: string): Promise<StormStream>;
|
78
79
|
createErrorClassification(prompt: string, history?: ConversationItem[], conversationId?: string): Promise<StormStream>;
|
@@ -155,6 +155,12 @@ class StormClient {
|
|
155
155
|
conversationId,
|
156
156
|
});
|
157
157
|
}
|
158
|
+
createImage(prompt, conversationId) {
|
159
|
+
return this.send('/v2/ui/image', {
|
160
|
+
prompt,
|
161
|
+
conversationId,
|
162
|
+
});
|
163
|
+
}
|
158
164
|
createUIImplementation(prompt, conversationId) {
|
159
165
|
return this.send('/v2/ui/merge', {
|
160
166
|
prompt,
|
package/package.json
CHANGED
@@ -5,17 +5,21 @@
|
|
5
5
|
|
6
6
|
import uuid from 'node-uuid';
|
7
7
|
import { stormClient, UIPagePrompt } from './stormClient';
|
8
|
-
import {
|
9
|
-
ReferenceClassification,
|
10
|
-
StormEvent,
|
11
|
-
StormEventPage,
|
12
|
-
StormEventReferenceClassification,
|
13
|
-
UIShell,
|
14
|
-
} from './events';
|
8
|
+
import { ReferenceClassification, StormEvent, StormEventPage, StormImage, UIShell } from './events';
|
15
9
|
import { EventEmitter } from 'node:events';
|
16
|
-
import { PromiseQueue } from './PromiseQueue';
|
10
|
+
import { createFuture, Future, FuturePromise, PromiseQueue } from './PromiseQueue';
|
17
11
|
import { hasPageOnDisk } from './page-utils';
|
18
12
|
|
13
|
+
export interface ImagePrompt {
|
14
|
+
name: string;
|
15
|
+
description: string;
|
16
|
+
source: 'local' | 'cdn' | 'example';
|
17
|
+
title: string;
|
18
|
+
type: 'image' | 'css' | 'javascript' | 'html';
|
19
|
+
url: string;
|
20
|
+
content: string;
|
21
|
+
}
|
22
|
+
|
19
23
|
export class PageQueue extends EventEmitter {
|
20
24
|
private readonly queue: PromiseQueue;
|
21
25
|
private readonly systemId: string;
|
@@ -31,6 +35,7 @@ export class PageQueue extends EventEmitter {
|
|
31
35
|
|
32
36
|
on(event: 'event', listener: (data: StormEvent) => void): this;
|
33
37
|
on(event: 'page', listener: (data: StormEventPage) => void): this;
|
38
|
+
on(event: 'image', listener: (data: StormImage, source: ImagePrompt, future: FuturePromise<void>) => void): this;
|
34
39
|
|
35
40
|
on(event: string, listener: (...args: any[]) => void): this {
|
36
41
|
return super.on(event, listener);
|
@@ -38,6 +43,7 @@ export class PageQueue extends EventEmitter {
|
|
38
43
|
|
39
44
|
emit(type: 'event', event: StormEvent): boolean;
|
40
45
|
emit(type: 'page', event: StormEventPage): boolean;
|
46
|
+
emit(type: 'image', event: StormImage, source: ImagePrompt, future: FuturePromise<void>): boolean;
|
41
47
|
emit(eventName: string | symbol, ...args: any[]): boolean {
|
42
48
|
return super.emit(eventName, ...args);
|
43
49
|
}
|
@@ -56,12 +62,12 @@ export class PageQueue extends EventEmitter {
|
|
56
62
|
overwrite: boolean = false
|
57
63
|
) {
|
58
64
|
if (!overwrite && this.references.has(initialPrompt.path)) {
|
59
|
-
console.log('Ignoring duplicate prompt', initialPrompt.path);
|
65
|
+
//console.log('Ignoring duplicate prompt', initialPrompt.path);
|
60
66
|
return Promise.resolve();
|
61
67
|
}
|
62
68
|
|
63
69
|
if (!overwrite && hasPageOnDisk(this.systemId, initialPrompt.method, initialPrompt.path)) {
|
64
|
-
console.log('Ignoring prompt with existing page', initialPrompt.path);
|
70
|
+
//console.log('Ignoring prompt with existing page', initialPrompt.path);
|
65
71
|
return Promise.resolve();
|
66
72
|
}
|
67
73
|
|
@@ -79,9 +85,8 @@ export class PageQueue extends EventEmitter {
|
|
79
85
|
|
80
86
|
private async addPageGenerator(generator: PageGenerator) {
|
81
87
|
generator.on('event', (event: StormEvent) => this.emit('event', event));
|
82
|
-
generator.on('page_refs', ({ event, references }) => {
|
83
|
-
|
84
|
-
references.forEach((reference) => {
|
88
|
+
generator.on('page_refs', async ({ event, references }) => {
|
89
|
+
const promises = references.map(async (reference) => {
|
85
90
|
if (
|
86
91
|
reference.url.startsWith('#') ||
|
87
92
|
reference.url.startsWith('javascript:') ||
|
@@ -93,14 +98,17 @@ export class PageQueue extends EventEmitter {
|
|
93
98
|
|
94
99
|
switch (reference.type) {
|
95
100
|
case 'image':
|
96
|
-
|
101
|
+
await this.addImagePrompt({
|
102
|
+
...reference,
|
103
|
+
content: event.payload.content,
|
104
|
+
});
|
97
105
|
break;
|
98
106
|
case 'css':
|
99
107
|
case 'javascript':
|
100
108
|
//console.log('Ignoring reference', reference);
|
101
109
|
break;
|
102
110
|
case 'html':
|
103
|
-
console.log('Adding page generator for', reference);
|
111
|
+
//console.log('Adding page generator for', reference);
|
104
112
|
const paths = Array.from(this.references.keys());
|
105
113
|
this.addPrompt({
|
106
114
|
name: reference.name,
|
@@ -121,6 +129,9 @@ export class PageQueue extends EventEmitter {
|
|
121
129
|
break;
|
122
130
|
}
|
123
131
|
});
|
132
|
+
|
133
|
+
await Promise.allSettled(promises);
|
134
|
+
this.emit('page', event);
|
124
135
|
});
|
125
136
|
return this.queue.add(() => generator.generate());
|
126
137
|
}
|
@@ -132,6 +143,31 @@ export class PageQueue extends EventEmitter {
|
|
132
143
|
public wait() {
|
133
144
|
return this.queue.wait();
|
134
145
|
}
|
146
|
+
|
147
|
+
private async addImagePrompt(prompt: ImagePrompt) {
|
148
|
+
const result = await stormClient.createImage(
|
149
|
+
`Create an image for the url "${prompt.url}" with this description: ${prompt.description}`.trim()
|
150
|
+
);
|
151
|
+
|
152
|
+
const futures: FuturePromise<void>[] = [];
|
153
|
+
|
154
|
+
result.on('data', async (event: StormEvent) => {
|
155
|
+
if (event.type === 'IMAGE') {
|
156
|
+
const future = createFuture();
|
157
|
+
futures.push(future);
|
158
|
+
this.emit('image', event, prompt, future);
|
159
|
+
setTimeout(() => {
|
160
|
+
if (!future.isResolved()) {
|
161
|
+
console.log('Image prompt timed out', prompt);
|
162
|
+
future.reject(new Error('Image prompt timed out'));
|
163
|
+
}
|
164
|
+
}, 30000);
|
165
|
+
}
|
166
|
+
});
|
167
|
+
|
168
|
+
await result.waitForDone();
|
169
|
+
await Promise.allSettled(futures.map((f) => f.promise));
|
170
|
+
}
|
135
171
|
}
|
136
172
|
|
137
173
|
export class PageGenerator extends EventEmitter {
|
@@ -4,6 +4,40 @@
|
|
4
4
|
*/
|
5
5
|
export type Future<T> = () => Promise<T>;
|
6
6
|
|
7
|
+
export type FuturePromise<T> = {
|
8
|
+
promise: Promise<T>;
|
9
|
+
resolve: (value: T) => void;
|
10
|
+
reject: (reason: any) => void;
|
11
|
+
isResolved: () => boolean;
|
12
|
+
};
|
13
|
+
|
14
|
+
export function createFuture<T = void>(): FuturePromise<T> {
|
15
|
+
let resolved = false;
|
16
|
+
let resolve: (value: T) => void = () => {
|
17
|
+
resolved = true;
|
18
|
+
};
|
19
|
+
let reject: (reason: any) => void = () => {
|
20
|
+
resolved = true;
|
21
|
+
};
|
22
|
+
|
23
|
+
const promise = new Promise<T>((res, rej) => {
|
24
|
+
resolve = (value: T) => {
|
25
|
+
resolved = true;
|
26
|
+
res(value);
|
27
|
+
};
|
28
|
+
reject = (reason: any) => {
|
29
|
+
resolved = true;
|
30
|
+
rej(reason);
|
31
|
+
};
|
32
|
+
});
|
33
|
+
return {
|
34
|
+
promise,
|
35
|
+
resolve,
|
36
|
+
reject,
|
37
|
+
isResolved: () => resolved,
|
38
|
+
};
|
39
|
+
}
|
40
|
+
|
7
41
|
type InternalFuture<T> = {
|
8
42
|
execute: Future<T>;
|
9
43
|
promise: Promise<T>;
|
package/src/storm/events.ts
CHANGED
@@ -311,6 +311,15 @@ export interface StormEventDone {
|
|
311
311
|
created: number;
|
312
312
|
}
|
313
313
|
|
314
|
+
export interface StormImage {
|
315
|
+
type: 'IMAGE';
|
316
|
+
reason: string;
|
317
|
+
created: number;
|
318
|
+
payload: {
|
319
|
+
href: string;
|
320
|
+
};
|
321
|
+
}
|
322
|
+
|
314
323
|
export interface StormEventDefinitionChange {
|
315
324
|
type: 'DEFINITION_CHANGE';
|
316
325
|
reason: string;
|
@@ -494,4 +503,5 @@ export type StormEvent =
|
|
494
503
|
| StormEventLandingPage
|
495
504
|
| StormEventReferenceClassification
|
496
505
|
| StormEventApiBase
|
497
|
-
| StormEventUIStarted
|
506
|
+
| StormEventUIStarted
|
507
|
+
| StormImage;
|
package/src/storm/page-utils.ts
CHANGED
@@ -2,7 +2,8 @@
|
|
2
2
|
* Copyright 2023 Kapeta Inc.
|
3
3
|
* SPDX-License-Identifier: BUSL-1.1
|
4
4
|
*/
|
5
|
-
import { StormEventFileDone, StormEventPage } from './events';
|
5
|
+
import { StormEventFileDone, StormEventPage, StormImage } from './events';
|
6
|
+
|
6
7
|
import { Response } from 'express';
|
7
8
|
import os from 'node:os';
|
8
9
|
import Path from 'path';
|
@@ -10,6 +11,7 @@ import FS from 'fs-extra';
|
|
10
11
|
import FSExtra from 'fs-extra';
|
11
12
|
import { ConversationItem } from './stream';
|
12
13
|
import exp from 'node:constants';
|
14
|
+
import { ImagePrompt } from './PageGenerator';
|
13
15
|
|
14
16
|
export const SystemIdHeader = 'System-Id';
|
15
17
|
|
@@ -45,6 +47,27 @@ export async function writeAssetToDisk(systemId: string, event: StormEventFileDo
|
|
45
47
|
};
|
46
48
|
}
|
47
49
|
|
50
|
+
export async function writeImageToDisk(systemId: string, event: StormImage, prompt: ImagePrompt) {
|
51
|
+
const baseDir = getSystemBaseDir(systemId);
|
52
|
+
const path = Path.join(baseDir, normalizePath(prompt.url));
|
53
|
+
|
54
|
+
const response = await fetch(event.payload.href);
|
55
|
+
if (!response.ok || !response.body) {
|
56
|
+
throw new Error(`Failed to fetch image: ${event.payload.href}`);
|
57
|
+
}
|
58
|
+
|
59
|
+
await FS.ensureDir(Path.dirname(path));
|
60
|
+
|
61
|
+
const buffer = await response.arrayBuffer();
|
62
|
+
await FS.writeFile(path, Buffer.from(buffer));
|
63
|
+
|
64
|
+
console.log(`Image written to disk: ${event.payload.href} > ${path}`);
|
65
|
+
|
66
|
+
return {
|
67
|
+
path,
|
68
|
+
};
|
69
|
+
}
|
70
|
+
|
48
71
|
export function hasPageOnDisk(systemId: string, method: string, path: string) {
|
49
72
|
const baseDir = getSystemBaseDir(systemId);
|
50
73
|
const filePath = getFilePath(method);
|
@@ -65,9 +88,20 @@ export function resolveReadPath(systemId: string, path: string, method: string)
|
|
65
88
|
|
66
89
|
path = normalizePath(path);
|
67
90
|
|
68
|
-
|
91
|
+
let fullPath = Path.join(baseDir, path);
|
69
92
|
|
70
|
-
|
93
|
+
//First check if there is a file at the exact path
|
94
|
+
try {
|
95
|
+
const stat = FS.statSync(fullPath);
|
96
|
+
if (stat && stat.isFile()) {
|
97
|
+
return fullPath;
|
98
|
+
}
|
99
|
+
} catch (e) {
|
100
|
+
// Ignore
|
101
|
+
}
|
102
|
+
|
103
|
+
const filePath = getFilePath(method);
|
104
|
+
fullPath = Path.join(baseDir, path, filePath);
|
71
105
|
|
72
106
|
if (FS.existsSync(fullPath)) {
|
73
107
|
return fullPath;
|
@@ -119,7 +153,7 @@ export function readPageFromDisk(systemId: string, path: string, method: string,
|
|
119
153
|
|
120
154
|
res.type(filePath.split('.').pop() as string);
|
121
155
|
|
122
|
-
const content = FS.readFileSync(filePath
|
156
|
+
const content = FS.readFileSync(filePath);
|
123
157
|
res.write(content);
|
124
158
|
res.end();
|
125
159
|
}
|
package/src/storm/routes.ts
CHANGED
@@ -23,7 +23,7 @@ import {
|
|
23
23
|
UIPageVoteRequest,
|
24
24
|
UIPageGetVoteRequest,
|
25
25
|
} from './stormClient';
|
26
|
-
import { Page, StormEvent, StormEventPage, StormEventPhaseType, UserJourneyScreen } from './events';
|
26
|
+
import { Page, StormEvent, StormEventPage, StormEventPhaseType, StormImage, UserJourneyScreen } from './events';
|
27
27
|
|
28
28
|
import {
|
29
29
|
createPhaseEndEvent,
|
@@ -40,11 +40,12 @@ import {
|
|
40
40
|
readPageFromDiskAsString,
|
41
41
|
SystemIdHeader,
|
42
42
|
writeAssetToDisk,
|
43
|
+
writeImageToDisk,
|
43
44
|
writePageToDisk,
|
44
45
|
} from './page-utils';
|
45
46
|
import { UIServer } from './UIServer';
|
46
|
-
import { PageQueue } from './PageGenerator';
|
47
47
|
import { randomUUID } from 'crypto';
|
48
|
+
import { ImagePrompt, PageQueue } from './PageGenerator';
|
48
49
|
|
49
50
|
const UI_SERVERS: { [key: string]: UIServer } = {};
|
50
51
|
const router = Router();
|
@@ -114,6 +115,21 @@ router.post('/ui/screen', async (req: KapetaBodyRequest, res: Response) => {
|
|
114
115
|
}
|
115
116
|
});
|
116
117
|
|
118
|
+
queue.on('image', async (screenData, prompt, future) => {
|
119
|
+
if (!systemId) {
|
120
|
+
return;
|
121
|
+
}
|
122
|
+
try {
|
123
|
+
const promise = handleImageEvent(systemId, screenData, prompt);
|
124
|
+
promises.push(promise);
|
125
|
+
await promise;
|
126
|
+
future.resolve();
|
127
|
+
} catch (e) {
|
128
|
+
console.error('Failed to handle image event', e);
|
129
|
+
future.reject(e);
|
130
|
+
}
|
131
|
+
});
|
132
|
+
|
117
133
|
await queue.addPrompt(aiRequest, conversationId, true);
|
118
134
|
|
119
135
|
await queue.wait();
|
@@ -211,6 +227,18 @@ router.post('/:handle/ui/iterative', async (req: KapetaBodyRequest, res: Respons
|
|
211
227
|
pageEventPromises.push(sendPageEvent(landingPagesStream.getConversationId(), screenData, res));
|
212
228
|
});
|
213
229
|
|
230
|
+
pageQueue.on('image', async (screenData, prompt, future) => {
|
231
|
+
try {
|
232
|
+
const promise = handleImageEvent(landingPagesStream.getConversationId(), screenData, prompt);
|
233
|
+
pageEventPromises.push(promise);
|
234
|
+
await promise;
|
235
|
+
future.resolve();
|
236
|
+
} catch (e) {
|
237
|
+
console.error('Failed to handle image event', e);
|
238
|
+
future.reject(e);
|
239
|
+
}
|
240
|
+
});
|
241
|
+
|
214
242
|
pageQueue.on('event', (screenData: StormEvent) => {
|
215
243
|
sendEvent(res, screenData);
|
216
244
|
});
|
@@ -347,7 +375,7 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
|
|
347
375
|
queue.setUiTheme(theme);
|
348
376
|
|
349
377
|
shellsStream.on('data', (data: StormEvent) => {
|
350
|
-
console.log('Processing shell event', data);
|
378
|
+
//console.log('Processing shell event', data);
|
351
379
|
sendEvent(res, data);
|
352
380
|
|
353
381
|
if (data.type !== 'UI_SHELL') {
|
@@ -394,6 +422,18 @@ router.post('/:handle/ui', async (req: KapetaBodyRequest, res: Response) => {
|
|
394
422
|
pageEventPromises.push(sendPageEvent(outerConversationId, screenData, res));
|
395
423
|
});
|
396
424
|
|
425
|
+
queue.on('image', async (screenData, prompt, future) => {
|
426
|
+
try {
|
427
|
+
const promise = handleImageEvent(outerConversationId, screenData, prompt);
|
428
|
+
pageEventPromises.push(promise);
|
429
|
+
await promise;
|
430
|
+
future.resolve();
|
431
|
+
} catch (e) {
|
432
|
+
console.error('Failed to handle image event', e);
|
433
|
+
future.reject(e);
|
434
|
+
}
|
435
|
+
});
|
436
|
+
|
397
437
|
queue.on('event', (screenData: StormEvent) => {
|
398
438
|
sendEvent(res, screenData);
|
399
439
|
});
|
@@ -766,4 +806,12 @@ async function sendPageEvent(mainConversationId: string, data: StormEventPage, r
|
|
766
806
|
sendEvent(res, convertPageEvent(data, data.payload.conversationId, mainConversationId));
|
767
807
|
}
|
768
808
|
|
809
|
+
async function handleImageEvent(mainConversationId: string, data: StormImage, prompt: ImagePrompt) {
|
810
|
+
try {
|
811
|
+
await writeImageToDisk(mainConversationId, data, prompt);
|
812
|
+
} catch (err) {
|
813
|
+
console.error('Failed to write image to disk', err);
|
814
|
+
}
|
815
|
+
}
|
816
|
+
|
769
817
|
export default router;
|
package/src/storm/stormClient.ts
CHANGED
@@ -263,6 +263,13 @@ class StormClient {
|
|
263
263
|
});
|
264
264
|
}
|
265
265
|
|
266
|
+
public createImage(prompt: string, conversationId?: string) {
|
267
|
+
return this.send('/v2/ui/image', {
|
268
|
+
prompt,
|
269
|
+
conversationId,
|
270
|
+
});
|
271
|
+
}
|
272
|
+
|
266
273
|
public createUIImplementation(prompt: StormUIImplementationPrompt, conversationId?: string) {
|
267
274
|
return this.send('/v2/ui/merge', {
|
268
275
|
prompt,
|