@kapeta/local-cluster-service 0.43.3 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/dist/cjs/index.js +2 -0
- package/dist/cjs/src/codeGeneratorManager.d.ts +1 -0
- package/dist/cjs/src/codeGeneratorManager.js +12 -6
- package/dist/cjs/src/middleware/cors.d.ts +1 -0
- package/dist/cjs/src/middleware/kapeta.d.ts +1 -0
- package/dist/cjs/src/middleware/stringBody.d.ts +1 -0
- package/dist/cjs/src/proxy/routes.js +19 -19
- package/dist/cjs/src/proxy/types/rest.d.ts +2 -4
- package/dist/cjs/src/proxy/types/rest.js +45 -6
- package/dist/cjs/src/proxy/types/web.d.ts +2 -4
- package/dist/cjs/src/proxy/types/web.js +2 -2
- package/dist/cjs/src/storm/codegen.d.ts +36 -0
- package/dist/cjs/src/storm/codegen.js +160 -0
- package/dist/cjs/src/storm/event-parser.d.ts +70 -0
- package/dist/cjs/src/storm/event-parser.js +543 -0
- package/dist/cjs/src/storm/events.d.ts +127 -0
- package/dist/cjs/src/storm/events.js +6 -0
- package/dist/cjs/src/storm/routes.d.ts +7 -0
- package/dist/cjs/src/storm/routes.js +109 -0
- package/dist/cjs/src/storm/stormClient.d.ts +13 -0
- package/dist/cjs/src/storm/stormClient.js +87 -0
- package/dist/cjs/src/storm/stream.d.ts +38 -0
- package/dist/cjs/src/storm/stream.js +57 -0
- package/dist/cjs/src/types.d.ts +5 -2
- package/dist/esm/index.js +2 -0
- package/dist/esm/src/codeGeneratorManager.d.ts +1 -0
- package/dist/esm/src/codeGeneratorManager.js +12 -6
- package/dist/esm/src/middleware/cors.d.ts +1 -0
- package/dist/esm/src/middleware/kapeta.d.ts +1 -0
- package/dist/esm/src/middleware/stringBody.d.ts +1 -0
- package/dist/esm/src/proxy/routes.js +19 -19
- package/dist/esm/src/proxy/types/rest.d.ts +2 -4
- package/dist/esm/src/proxy/types/rest.js +45 -6
- package/dist/esm/src/proxy/types/web.d.ts +2 -4
- package/dist/esm/src/proxy/types/web.js +2 -2
- package/dist/esm/src/storm/codegen.d.ts +36 -0
- package/dist/esm/src/storm/codegen.js +160 -0
- package/dist/esm/src/storm/event-parser.d.ts +70 -0
- package/dist/esm/src/storm/event-parser.js +543 -0
- package/dist/esm/src/storm/events.d.ts +127 -0
- package/dist/esm/src/storm/events.js +6 -0
- package/dist/esm/src/storm/routes.d.ts +7 -0
- package/dist/esm/src/storm/routes.js +109 -0
- package/dist/esm/src/storm/stormClient.d.ts +13 -0
- package/dist/esm/src/storm/stormClient.js +87 -0
- package/dist/esm/src/storm/stream.d.ts +38 -0
- package/dist/esm/src/storm/stream.js +57 -0
- package/dist/esm/src/types.d.ts +5 -2
- package/index.ts +2 -0
- package/package.json +6 -3
- package/src/codeGeneratorManager.ts +17 -8
- package/src/proxy/routes.ts +26 -20
- package/src/proxy/types/rest.ts +73 -11
- package/src/proxy/types/web.ts +3 -5
- package/src/storm/codegen.ts +210 -0
- package/src/storm/event-parser.ts +688 -0
- package/src/storm/events.ts +169 -0
- package/src/storm/routes.ts +143 -0
- package/src/storm/stormClient.ts +114 -0
- package/src/storm/stream.ts +88 -0
- package/src/types.ts +5 -2
- package/src/utils/BlockInstanceRunner.ts +4 -2
@@ -0,0 +1,109 @@
|
|
1
|
+
"use strict";
|
2
|
+
/**
|
3
|
+
* Copyright 2023 Kapeta Inc.
|
4
|
+
* SPDX-License-Identifier: BUSL-1.1
|
5
|
+
*/
|
6
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
7
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
8
|
+
};
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
10
|
+
const express_promise_router_1 = __importDefault(require("express-promise-router"));
|
11
|
+
const cors_1 = require("../middleware/cors");
|
12
|
+
const stringBody_1 = require("../middleware/stringBody");
|
13
|
+
const stormClient_1 = require("./stormClient");
|
14
|
+
const event_parser_1 = require("./event-parser");
|
15
|
+
const codegen_1 = require("./codegen");
|
16
|
+
const router = (0, express_promise_router_1.default)();
|
17
|
+
router.use('/', cors_1.corsHandler);
|
18
|
+
router.use('/', stringBody_1.stringBody);
|
19
|
+
router.post('/:handle/all', async (req, res) => {
|
20
|
+
const handle = req.params.handle;
|
21
|
+
try {
|
22
|
+
const stormOptions = await (0, event_parser_1.resolveOptions)();
|
23
|
+
const eventParser = new event_parser_1.StormEventParser(stormOptions);
|
24
|
+
console.log('Got prompt', req.stringBody);
|
25
|
+
const aiRequest = JSON.parse(req.stringBody ?? '{}');
|
26
|
+
const metaStream = await stormClient_1.stormClient.createMetadata(aiRequest.prompt, aiRequest.history);
|
27
|
+
res.set('Content-Type', 'application/x-ndjson');
|
28
|
+
metaStream.on('data', (data) => {
|
29
|
+
eventParser.addEvent(data);
|
30
|
+
});
|
31
|
+
await streamStormPartialResponse(metaStream, res);
|
32
|
+
if (!eventParser.isValid()) {
|
33
|
+
// We can't continue if the meta stream is invalid
|
34
|
+
res.write({
|
35
|
+
type: 'ERROR_INTERNAL',
|
36
|
+
payload: { error: eventParser.getError() },
|
37
|
+
reason: 'Failed to generate system',
|
38
|
+
created: Date.now(),
|
39
|
+
});
|
40
|
+
res.end();
|
41
|
+
return;
|
42
|
+
}
|
43
|
+
const result = eventParser.toResult(handle);
|
44
|
+
console.log('RESULT\n', JSON.stringify(result, null, 2));
|
45
|
+
const stormCodegen = new codegen_1.StormCodegen(aiRequest.prompt, result.blocks);
|
46
|
+
const codegenStream = streamStormPartialResponse(stormCodegen.getStream(), res);
|
47
|
+
await stormCodegen.process();
|
48
|
+
await codegenStream;
|
49
|
+
sendDone(res);
|
50
|
+
}
|
51
|
+
catch (err) {
|
52
|
+
sendError(err, res);
|
53
|
+
}
|
54
|
+
});
|
55
|
+
router.post('/metadata', async (req, res) => {
|
56
|
+
const aiRequest = JSON.parse(req.stringBody ?? '{}');
|
57
|
+
const result = await stormClient_1.stormClient.createMetadata(aiRequest.prompt, aiRequest.history);
|
58
|
+
await streamStormResponse(result, res);
|
59
|
+
});
|
60
|
+
router.post('/services/implement', async (req, res) => {
|
61
|
+
const aiRequest = JSON.parse(req.stringBody ?? '{}');
|
62
|
+
const result = await stormClient_1.stormClient.createServiceImplementation(aiRequest.prompt, aiRequest.history);
|
63
|
+
await streamStormResponse(result, res);
|
64
|
+
});
|
65
|
+
router.post('/ui/implement', async (req, res) => {
|
66
|
+
const aiRequest = JSON.parse(req.stringBody ?? '{}');
|
67
|
+
const result = await stormClient_1.stormClient.createUIImplementation(aiRequest.prompt, aiRequest.history);
|
68
|
+
await streamStormResponse(result, res);
|
69
|
+
});
|
70
|
+
async function streamStormResponse(result, res) {
|
71
|
+
res.set('Content-Type', 'application/x-ndjson');
|
72
|
+
await streamStormPartialResponse(result, res);
|
73
|
+
sendDone(res);
|
74
|
+
}
|
75
|
+
function sendDone(res) {
|
76
|
+
res.write(JSON.stringify({
|
77
|
+
type: 'DONE',
|
78
|
+
created: Date.now(),
|
79
|
+
}) + '\n');
|
80
|
+
res.end();
|
81
|
+
}
|
82
|
+
function sendError(err, res) {
|
83
|
+
console.error('Failed to send prompt', err);
|
84
|
+
if (res.headersSent) {
|
85
|
+
res.write(JSON.stringify({
|
86
|
+
type: 'ERROR_INTERNAL',
|
87
|
+
created: Date.now(),
|
88
|
+
payload: { error: err.message },
|
89
|
+
reason: 'Failed while sending prompt',
|
90
|
+
}) + '\n');
|
91
|
+
}
|
92
|
+
else {
|
93
|
+
res.status(400).send({ error: err.message });
|
94
|
+
}
|
95
|
+
}
|
96
|
+
function streamStormPartialResponse(result, res) {
|
97
|
+
return new Promise((resolve, reject) => {
|
98
|
+
result.on('data', (data) => {
|
99
|
+
res.write(JSON.stringify(data) + '\n');
|
100
|
+
});
|
101
|
+
result.on('error', (err) => {
|
102
|
+
reject(err);
|
103
|
+
});
|
104
|
+
result.on('end', () => {
|
105
|
+
resolve();
|
106
|
+
});
|
107
|
+
});
|
108
|
+
}
|
109
|
+
exports.default = router;
|
@@ -0,0 +1,13 @@
|
|
1
|
+
import { ConversationItem, StormFileImplementationPrompt, StormStream } from './stream';
|
2
|
+
export declare const STORM_ID = "storm";
|
3
|
+
declare class StormClient {
|
4
|
+
private readonly _baseUrl;
|
5
|
+
constructor();
|
6
|
+
private createOptions;
|
7
|
+
private send;
|
8
|
+
createMetadata(prompt: string, history?: ConversationItem[]): Promise<StormStream>;
|
9
|
+
createUIImplementation(prompt: StormFileImplementationPrompt, history?: ConversationItem[]): Promise<StormStream>;
|
10
|
+
createServiceImplementation(prompt: StormFileImplementationPrompt, history?: ConversationItem[]): Promise<StormStream>;
|
11
|
+
}
|
12
|
+
export declare const stormClient: StormClient;
|
13
|
+
export {};
|
@@ -0,0 +1,87 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
4
|
+
};
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
6
|
+
exports.stormClient = exports.STORM_ID = void 0;
|
7
|
+
/**
|
8
|
+
* Copyright 2023 Kapeta Inc.
|
9
|
+
* SPDX-License-Identifier: BUSL-1.1
|
10
|
+
*/
|
11
|
+
const nodejs_api_client_1 = require("@kapeta/nodejs-api-client");
|
12
|
+
const utils_1 = require("../utils/utils");
|
13
|
+
const promises_1 = __importDefault(require("node:readline/promises"));
|
14
|
+
const node_stream_1 = require("node:stream");
|
15
|
+
const stream_1 = require("./stream");
|
16
|
+
exports.STORM_ID = 'storm';
|
17
|
+
class StormClient {
|
18
|
+
_baseUrl;
|
19
|
+
constructor() {
|
20
|
+
this._baseUrl = (0, utils_1.getRemoteUrl)('ai-service', 'https://ai.kapeta.com');
|
21
|
+
}
|
22
|
+
createOptions(path, method, body) {
|
23
|
+
const url = `${this._baseUrl}${path}`;
|
24
|
+
const headers = {
|
25
|
+
'Content-Type': 'application/json',
|
26
|
+
};
|
27
|
+
const api = new nodejs_api_client_1.KapetaAPI();
|
28
|
+
if (api.hasToken()) {
|
29
|
+
//headers['Authorization'] = `Bearer ${api.getAccessToken()}`; //TODO: Enable authentication
|
30
|
+
}
|
31
|
+
return {
|
32
|
+
url,
|
33
|
+
method: method,
|
34
|
+
body: JSON.stringify(body),
|
35
|
+
headers,
|
36
|
+
};
|
37
|
+
}
|
38
|
+
async send(path, body, method = 'POST') {
|
39
|
+
const stringPrompt = typeof body.prompt === 'string' ? body.prompt : JSON.stringify(body.prompt);
|
40
|
+
const options = this.createOptions(path, method, {
|
41
|
+
prompt: stringPrompt,
|
42
|
+
history: body.history,
|
43
|
+
});
|
44
|
+
const out = new stream_1.StormStream(stringPrompt, body.history);
|
45
|
+
const response = await fetch(options.url, options);
|
46
|
+
if (response.status !== 200) {
|
47
|
+
throw new Error(`Got error response from ${options.url}: ${response.status}\nContent: ${await response.text()}`);
|
48
|
+
}
|
49
|
+
const jsonLStream = promises_1.default.createInterface(node_stream_1.Readable.fromWeb(response.body));
|
50
|
+
jsonLStream.on('line', (line) => {
|
51
|
+
out.addJSONLine(line);
|
52
|
+
});
|
53
|
+
jsonLStream.on('error', (error) => {
|
54
|
+
out.emit('error', error);
|
55
|
+
});
|
56
|
+
jsonLStream.on('pause', () => {
|
57
|
+
console.log('paused');
|
58
|
+
});
|
59
|
+
jsonLStream.on('resume', () => {
|
60
|
+
console.log('resumed');
|
61
|
+
});
|
62
|
+
jsonLStream.on('close', () => {
|
63
|
+
out.end();
|
64
|
+
});
|
65
|
+
return out;
|
66
|
+
}
|
67
|
+
createMetadata(prompt, history) {
|
68
|
+
return this.send('/v2/all', {
|
69
|
+
history: history ?? [],
|
70
|
+
prompt: prompt,
|
71
|
+
});
|
72
|
+
}
|
73
|
+
createUIImplementation(prompt, history) {
|
74
|
+
return this.send('/v2/ui/merge', {
|
75
|
+
history: history ?? [],
|
76
|
+
prompt,
|
77
|
+
});
|
78
|
+
}
|
79
|
+
createServiceImplementation(prompt, history) {
|
80
|
+
console.log('SENDING SERVICE PROMPT', JSON.stringify(prompt, null, 2));
|
81
|
+
return this.send('/v2/services/merge', {
|
82
|
+
history: history ?? [],
|
83
|
+
prompt,
|
84
|
+
});
|
85
|
+
}
|
86
|
+
}
|
87
|
+
exports.stormClient = new StormClient();
|
@@ -0,0 +1,38 @@
|
|
1
|
+
/// <reference types="node" />
|
2
|
+
/**
|
3
|
+
* Copyright 2023 Kapeta Inc.
|
4
|
+
* SPDX-License-Identifier: BUSL-1.1
|
5
|
+
*/
|
6
|
+
import { EventEmitter } from 'node:events';
|
7
|
+
import { StormEvent } from './events';
|
8
|
+
import { AIFileTypes, GeneratedFile } from '@kapeta/codegen';
|
9
|
+
export declare class StormStream extends EventEmitter {
|
10
|
+
private history;
|
11
|
+
private lines;
|
12
|
+
constructor(prompt?: string, history?: ConversationItem[]);
|
13
|
+
addJSONLine(line: string): void;
|
14
|
+
end(): void;
|
15
|
+
on(event: 'end', listener: () => void): this;
|
16
|
+
on(event: 'error', listener: (e: Error) => void): this;
|
17
|
+
on(event: 'data', listener: (data: StormEvent) => void): this;
|
18
|
+
emit(event: 'end'): boolean;
|
19
|
+
emit(event: 'error', e: Error): boolean;
|
20
|
+
emit(event: 'data', data: StormEvent): boolean;
|
21
|
+
waitForDone(): Promise<void>;
|
22
|
+
}
|
23
|
+
export interface ConversationItem {
|
24
|
+
role: 'user' | 'model';
|
25
|
+
content: string;
|
26
|
+
}
|
27
|
+
export interface StormContextRequest<T = string> {
|
28
|
+
history?: ConversationItem[];
|
29
|
+
prompt: T;
|
30
|
+
}
|
31
|
+
export interface StormFileInfo extends GeneratedFile {
|
32
|
+
type: AIFileTypes;
|
33
|
+
}
|
34
|
+
export interface StormFileImplementationPrompt {
|
35
|
+
context: StormFileInfo[];
|
36
|
+
template: StormFileInfo;
|
37
|
+
prompt: string;
|
38
|
+
}
|
@@ -0,0 +1,57 @@
|
|
1
|
+
"use strict";
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
+
exports.StormStream = void 0;
|
4
|
+
/**
|
5
|
+
* Copyright 2023 Kapeta Inc.
|
6
|
+
* SPDX-License-Identifier: BUSL-1.1
|
7
|
+
*/
|
8
|
+
const node_events_1 = require("node:events");
|
9
|
+
class StormStream extends node_events_1.EventEmitter {
|
10
|
+
history;
|
11
|
+
lines = [];
|
12
|
+
constructor(prompt = '', history) {
|
13
|
+
super();
|
14
|
+
this.history = history ?? [];
|
15
|
+
this.history.push({
|
16
|
+
role: 'user',
|
17
|
+
content: prompt,
|
18
|
+
});
|
19
|
+
}
|
20
|
+
addJSONLine(line) {
|
21
|
+
try {
|
22
|
+
this.lines.push(line);
|
23
|
+
const event = JSON.parse(line);
|
24
|
+
if (!event.created) {
|
25
|
+
event.created = Date.now();
|
26
|
+
}
|
27
|
+
this.emit('data', event);
|
28
|
+
}
|
29
|
+
catch (e) {
|
30
|
+
this.emit('error', e);
|
31
|
+
}
|
32
|
+
}
|
33
|
+
end() {
|
34
|
+
this.history.push({
|
35
|
+
role: 'model',
|
36
|
+
content: this.lines.join('\n'),
|
37
|
+
});
|
38
|
+
this.emit('end');
|
39
|
+
}
|
40
|
+
on(event, listener) {
|
41
|
+
return super.on(event, listener);
|
42
|
+
}
|
43
|
+
emit(eventName, ...args) {
|
44
|
+
return super.emit(eventName, ...args);
|
45
|
+
}
|
46
|
+
waitForDone() {
|
47
|
+
return new Promise((resolve, reject) => {
|
48
|
+
this.on('error', (err) => {
|
49
|
+
reject(err);
|
50
|
+
});
|
51
|
+
this.on('end', () => {
|
52
|
+
resolve();
|
53
|
+
});
|
54
|
+
});
|
55
|
+
}
|
56
|
+
}
|
57
|
+
exports.StormStream = StormStream;
|
package/dist/esm/src/types.d.ts
CHANGED
@@ -6,6 +6,7 @@ import express from 'express';
|
|
6
6
|
import { Connection, Resource } from '@kapeta/schemas';
|
7
7
|
import { StringBodyRequest } from './middleware/stringBody';
|
8
8
|
import { KapetaRequest } from './middleware/kapeta';
|
9
|
+
import { EnrichedAsset } from './assetManager';
|
9
10
|
export declare const KIND_RESOURCE_OPERATOR = "core/resource-type-operator";
|
10
11
|
export declare const KIND_BLOCK_TYPE = "core/block-type";
|
11
12
|
export declare const KIND_BLOCK_TYPE_OPERATOR = "core/block-type-operator";
|
@@ -80,9 +81,11 @@ export type ProxyRequestHandler = (req: StringBodyRequest, res: express.Response
|
|
80
81
|
export interface ProxyRequestInfo {
|
81
82
|
address: string;
|
82
83
|
connection: Connection;
|
83
|
-
providerResource: Resource;
|
84
|
-
consumerResource: Resource;
|
85
84
|
consumerPath: string;
|
85
|
+
consumerResource: Resource;
|
86
|
+
consumerBlockAsset: EnrichedAsset;
|
87
|
+
providerResource: Resource;
|
88
|
+
providerBlockAsset: EnrichedAsset;
|
86
89
|
}
|
87
90
|
export interface SimpleResponse {
|
88
91
|
code: number;
|
package/index.ts
CHANGED
@@ -24,6 +24,7 @@ import AttachmentRoutes from './src/attachments/routes';
|
|
24
24
|
import TaskRoutes from './src/tasks/routes';
|
25
25
|
import APIRoutes from './src/api';
|
26
26
|
import AIRoutes from './src/ai/routes';
|
27
|
+
import StormRoutes from './src/storm/routes';
|
27
28
|
import { isLinux } from './src/utils/utils';
|
28
29
|
import request from 'request';
|
29
30
|
import { repositoryManager } from './src/repositoryManager';
|
@@ -74,6 +75,7 @@ function createServer() {
|
|
74
75
|
app.use('/tasks', TaskRoutes);
|
75
76
|
app.use('/api', APIRoutes);
|
76
77
|
app.use('/ai', AIRoutes);
|
78
|
+
app.use('/storm', StormRoutes);
|
77
79
|
|
78
80
|
app.get('/status', async (req, res) => {
|
79
81
|
res.send({
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "@kapeta/local-cluster-service",
|
3
|
-
"version": "0.
|
3
|
+
"version": "0.45.0",
|
4
4
|
"description": "Manages configuration, ports and service discovery for locally running Kapeta systems",
|
5
5
|
"type": "commonjs",
|
6
6
|
"exports": {
|
@@ -43,13 +43,16 @@
|
|
43
43
|
"build:cjs": "tsc --outDir ./dist/cjs && echo '{\"type\":\"commonjs\"}' > ./dist/cjs/package.json",
|
44
44
|
"build": "npm run clean && npm run build:esm && npm run build:cjs",
|
45
45
|
"format": "prettier --write .",
|
46
|
-
"lint": "tsc
|
46
|
+
"lint": "npm run lint:tsc && npm run lint:eslint",
|
47
|
+
"lint:tsc": "tsc --noEmit",
|
48
|
+
"lint:eslint": "eslint src/**/*.ts",
|
47
49
|
"prepublishOnly": "npm run build"
|
48
50
|
},
|
49
51
|
"homepage": "https://github.com/kapetacom/local-cluster-service#readme",
|
50
52
|
"dependencies": {
|
51
|
-
"@kapeta/codegen": "^1.
|
53
|
+
"@kapeta/codegen": "^1.4.0",
|
52
54
|
"@kapeta/config-mapper": "^1.2.1",
|
55
|
+
"@kapeta/kaplang-core": "^1.16.0",
|
53
56
|
"@kapeta/local-cluster-config": "^0.4.2",
|
54
57
|
"@kapeta/nodejs-api-client": ">=0.2.0 <2",
|
55
58
|
"@kapeta/nodejs-process": "^1.2.0",
|
@@ -16,6 +16,22 @@ const TARGET_KIND = 'core/language-target';
|
|
16
16
|
const BLOCK_TYPE_REGEX = /^core\/block-type.*/;
|
17
17
|
|
18
18
|
class CodeGeneratorManager {
|
19
|
+
public async ensureTarget(targetKind: string) {
|
20
|
+
const targetRef = normalizeKapetaUri(targetKind);
|
21
|
+
|
22
|
+
// Automatically downloads target if not available
|
23
|
+
const targetAsset = await assetManager.getAsset(targetRef);
|
24
|
+
|
25
|
+
if (!targetAsset) {
|
26
|
+
console.error('Language target not found: %s', targetKind);
|
27
|
+
return false;
|
28
|
+
}
|
29
|
+
|
30
|
+
await this.ensureLanguageTargetInRegistry(targetAsset?.path, targetAsset?.version, targetAsset?.data);
|
31
|
+
|
32
|
+
return true;
|
33
|
+
}
|
34
|
+
|
19
35
|
private async ensureLanguageTargetInRegistry(path: string, version: string, definition: Definition) {
|
20
36
|
const key = `${definition.metadata.name}:${version}`;
|
21
37
|
|
@@ -81,17 +97,10 @@ class CodeGeneratorManager {
|
|
81
97
|
return;
|
82
98
|
}
|
83
99
|
|
84
|
-
|
85
|
-
|
86
|
-
// Automatically downloads target if not available
|
87
|
-
const targetAsset = await assetManager.getAsset(targetRef);
|
88
|
-
|
89
|
-
if (!targetAsset) {
|
90
|
-
console.error('Language target not found: %s', yamlContent.spec.target?.kind);
|
100
|
+
if (!(await this.ensureTarget(yamlContent.spec.target?.kind))) {
|
91
101
|
return;
|
92
102
|
}
|
93
103
|
|
94
|
-
await this.ensureLanguageTargetInRegistry(targetAsset?.path, targetAsset?.version, targetAsset?.data);
|
95
104
|
const baseDir = Path.dirname(yamlFile);
|
96
105
|
console.log('Generating code for path: %s', baseDir);
|
97
106
|
const codeGenerator = new BlockCodeGenerator(yamlContent as BlockDefinition);
|
package/src/proxy/routes.ts
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
*/
|
5
5
|
|
6
6
|
import Router from 'express-promise-router';
|
7
|
-
import {
|
7
|
+
import { Response } from 'express';
|
8
8
|
import { Resource } from '@kapeta/schemas';
|
9
9
|
import { proxyRestRequest } from './types/rest';
|
10
10
|
import { proxyHttpRequest } from './types/web';
|
@@ -61,23 +61,26 @@ router.all(
|
|
61
61
|
return;
|
62
62
|
}
|
63
63
|
|
64
|
-
const
|
64
|
+
const consumerBlockInstance = _.find(plan.spec.blocks, (blockInstance) => {
|
65
65
|
return blockInstance.id.toLowerCase() === connection.consumer.blockId.toLowerCase();
|
66
66
|
});
|
67
67
|
|
68
|
-
if (!
|
68
|
+
if (!consumerBlockInstance) {
|
69
69
|
res.status(401).send({ error: `Block instance not found "${req.params.consumerInstanceId}` });
|
70
70
|
return;
|
71
71
|
}
|
72
72
|
|
73
|
-
const
|
73
|
+
const consumerBlockAsset = await assetManager.getAsset(consumerBlockInstance.block.ref);
|
74
74
|
|
75
|
-
if (!
|
76
|
-
res.status(401).send({ error: `Block asset not found "${
|
75
|
+
if (!consumerBlockAsset) {
|
76
|
+
res.status(401).send({ error: `Block asset not found "${consumerBlockInstance.block.ref}` });
|
77
77
|
return;
|
78
78
|
}
|
79
79
|
|
80
|
-
const consumerResource = getResource(
|
80
|
+
const consumerResource = getResource(
|
81
|
+
consumerBlockAsset.data.spec.consumers,
|
82
|
+
req.params.consumerResourceName
|
83
|
+
);
|
81
84
|
|
82
85
|
if (!consumerResource) {
|
83
86
|
res.status(401).send({
|
@@ -93,23 +96,26 @@ router.all(
|
|
93
96
|
req.params.type
|
94
97
|
);
|
95
98
|
|
96
|
-
const
|
99
|
+
const providerBlockInstance = _.find(plan.spec.blocks, (blockInstance) => {
|
97
100
|
return blockInstance.id.toLowerCase() === connection.provider.blockId.toLowerCase();
|
98
101
|
});
|
99
102
|
|
100
|
-
if (!
|
103
|
+
if (!providerBlockInstance) {
|
101
104
|
res.status(401).send({ error: `Block instance not found "${connection.provider.blockId}` });
|
102
105
|
return;
|
103
106
|
}
|
104
107
|
|
105
|
-
const
|
108
|
+
const providerBlockAsset = await assetManager.getAsset(providerBlockInstance.block.ref);
|
106
109
|
|
107
|
-
if (!
|
108
|
-
res.status(401).send({ error: `Block asset not found "${
|
110
|
+
if (!providerBlockAsset) {
|
111
|
+
res.status(401).send({ error: `Block asset not found "${providerBlockInstance.block.ref}` });
|
109
112
|
return;
|
110
113
|
}
|
111
114
|
|
112
|
-
const providerResource = getResource(
|
115
|
+
const providerResource = getResource(
|
116
|
+
providerBlockAsset.data.spec.providers,
|
117
|
+
connection.provider.resourceName
|
118
|
+
);
|
113
119
|
|
114
120
|
if (!providerResource) {
|
115
121
|
res.status(401).send({
|
@@ -129,19 +135,19 @@ router.all(
|
|
129
135
|
address = address.substring(0, address.length - 1);
|
130
136
|
}
|
131
137
|
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
that is being called and identify the destination path from the connection.
|
136
|
-
*/
|
138
|
+
// Get the path the consumer requested. Note that this might not match the path the
|
139
|
+
// destination is expecting so we need to identify the method that is being called and
|
140
|
+
// identify the destination path from the connection.
|
137
141
|
const consumerPath = req.originalUrl.substring(basePath.length - 1);
|
138
142
|
|
139
143
|
typeHandler(req, res, {
|
140
|
-
consumerPath,
|
141
144
|
address,
|
145
|
+
connection,
|
146
|
+
consumerPath,
|
142
147
|
consumerResource,
|
148
|
+
consumerBlockAsset,
|
143
149
|
providerResource,
|
144
|
-
|
150
|
+
providerBlockAsset,
|
145
151
|
});
|
146
152
|
} catch (err: any) {
|
147
153
|
console.warn('Failed to process proxy request', err);
|
package/src/proxy/types/rest.ts
CHANGED
@@ -10,12 +10,13 @@ import { pathTemplateParser } from '../../utils/pathTemplateParser';
|
|
10
10
|
import { networkManager } from '../../networkManager';
|
11
11
|
|
12
12
|
import { socketManager } from '../../socketManager';
|
13
|
-
import { Request
|
14
|
-
import { ProxyRequestInfo, SimpleRequest, StringMap } from '../../types';
|
15
|
-
import { StringBodyRequest } from '../../middleware/stringBody';
|
13
|
+
import { Request } from 'express';
|
14
|
+
import { ProxyRequestHandler, ProxyRequestInfo, SimpleRequest, SimpleResponse, StringMap } from '../../types';
|
16
15
|
import { Resource } from '@kapeta/schemas';
|
17
16
|
import { stringify } from 'qs';
|
18
17
|
import { proxyHttpRequest } from './web';
|
18
|
+
import { DSLConverters, HTTPTransport, RESTMethod, maskSensitiveData } from '@kapeta/kaplang-core';
|
19
|
+
import { EnrichedAsset } from '../../assetManager';
|
19
20
|
|
20
21
|
export function getRestMethodId(restResource: Resource, httpMethod: string, httpPath: string) {
|
21
22
|
return _.findKey(restResource.spec.methods, (method) => {
|
@@ -37,6 +38,10 @@ export function getRestMethodId(restResource: Resource, httpMethod: string, http
|
|
37
38
|
});
|
38
39
|
}
|
39
40
|
|
41
|
+
interface RESTMethodWithId extends RESTMethod {
|
42
|
+
id: string;
|
43
|
+
}
|
44
|
+
|
40
45
|
/**
|
41
46
|
*
|
42
47
|
* @param req {Request}
|
@@ -52,7 +57,7 @@ function resolveMethods(req: Request, opts: ProxyRequestInfo) {
|
|
52
57
|
);
|
53
58
|
}
|
54
59
|
|
55
|
-
const consumerMethod = _.cloneDeep(opts.consumerResource.spec.methods[consumerMethodId]);
|
60
|
+
const consumerMethod = _.cloneDeep(opts.consumerResource.spec.methods[consumerMethodId] as RESTMethodWithId);
|
56
61
|
|
57
62
|
if (!consumerMethod) {
|
58
63
|
throw new Error(
|
@@ -70,7 +75,7 @@ function resolveMethods(req: Request, opts: ProxyRequestInfo) {
|
|
70
75
|
throw new Error(`Connection contained no mapping for consumer method "${consumerMethodId}`);
|
71
76
|
}
|
72
77
|
|
73
|
-
const providerMethod = _.cloneDeep(opts.providerResource.spec.methods[providerMethodId]);
|
78
|
+
const providerMethod = _.cloneDeep(opts.providerResource.spec.methods[providerMethodId] as RESTMethodWithId);
|
74
79
|
|
75
80
|
if (!providerMethod) {
|
76
81
|
throw new Error(
|
@@ -86,12 +91,67 @@ function resolveMethods(req: Request, opts: ProxyRequestInfo) {
|
|
86
91
|
};
|
87
92
|
}
|
88
93
|
|
89
|
-
|
94
|
+
function resolveEntitiesSource(blockAsset: EnrichedAsset) {
|
95
|
+
return (blockAsset.data.spec?.entities?.source?.value as string) || '';
|
96
|
+
}
|
97
|
+
|
98
|
+
const MASK_STRING = '*******';
|
99
|
+
|
100
|
+
function maskSimpleRequest(req: SimpleRequest, consumerMethod: RESTMethod, entitiesSource: string) {
|
101
|
+
const maskedRequest = _.cloneDeep(req);
|
102
|
+
|
103
|
+
Object.entries(consumerMethod.arguments || {}).forEach(([key, value]) => {
|
104
|
+
if (value.transport === HTTPTransport.BODY && typeof maskedRequest.body === 'string') {
|
105
|
+
try {
|
106
|
+
maskedRequest.body = maskSensitiveData(
|
107
|
+
maskedRequest.body,
|
108
|
+
DSLConverters.fromSchemaType(value),
|
109
|
+
entitiesSource,
|
110
|
+
MASK_STRING
|
111
|
+
);
|
112
|
+
} catch (error) {
|
113
|
+
// Ignore errors masking the request body
|
114
|
+
}
|
115
|
+
}
|
116
|
+
// TODO: Mask HEADER
|
117
|
+
// TODO: Mask QUERY
|
118
|
+
});
|
119
|
+
|
120
|
+
return maskedRequest;
|
121
|
+
}
|
122
|
+
|
123
|
+
function maskSimpleResponse(res: SimpleResponse, consumerMethod: RESTMethod, entitiesSource: string) {
|
124
|
+
const maskedResponse = _.cloneDeep(res);
|
125
|
+
|
126
|
+
Object.entries(consumerMethod.arguments || {}).forEach(([key, value]) => {
|
127
|
+
// TODO: Mask HEADER
|
128
|
+
});
|
129
|
+
|
130
|
+
try {
|
131
|
+
if (typeof maskedResponse.body === 'string') {
|
132
|
+
maskedResponse.body = maskSensitiveData(
|
133
|
+
maskedResponse.body,
|
134
|
+
DSLConverters.fromSchemaType(consumerMethod.responseType),
|
135
|
+
entitiesSource,
|
136
|
+
MASK_STRING
|
137
|
+
);
|
138
|
+
}
|
139
|
+
} catch (error) {
|
140
|
+
// Ignore errors masking the response body
|
141
|
+
}
|
142
|
+
|
143
|
+
return maskedResponse;
|
144
|
+
}
|
145
|
+
|
146
|
+
export const proxyRestRequest: ProxyRequestHandler = (req, res, opts) => {
|
90
147
|
if (_.isEmpty(opts.consumerResource.spec.methods) && _.isEmpty(opts.providerResource.spec.methods)) {
|
91
148
|
// If there are no methods defined, we assume the user controls the path and we just proxy the raw request
|
92
149
|
return proxyHttpRequest(req, res, opts);
|
93
150
|
}
|
94
|
-
|
151
|
+
const { consumerMethod, providerMethod } = resolveMethods(req, opts);
|
152
|
+
|
153
|
+
const consumerEntitiesSource = resolveEntitiesSource(opts.consumerBlockAsset);
|
154
|
+
const providerEntitiesSource = resolveEntitiesSource(opts.providerBlockAsset);
|
95
155
|
|
96
156
|
const consumerPathTemplate = pathTemplateParser(consumerMethod.path);
|
97
157
|
const providerPathTemplate = pathTemplateParser(providerMethod.path);
|
@@ -134,7 +194,7 @@ export function proxyRestRequest(req: StringBodyRequest, res: Response, opts: Pr
|
|
134
194
|
const traffic = networkManager.addRequest(
|
135
195
|
req.params.systemId,
|
136
196
|
opts.connection,
|
137
|
-
reqOpts,
|
197
|
+
maskSimpleRequest(reqOpts, consumerMethod, consumerEntitiesSource),
|
138
198
|
consumerMethod.id,
|
139
199
|
providerMethod.id
|
140
200
|
);
|
@@ -160,11 +220,13 @@ export function proxyRestRequest(req: StringBodyRequest, res: Response, opts: Pr
|
|
160
220
|
|
161
221
|
res.status(response.statusCode);
|
162
222
|
|
163
|
-
|
223
|
+
const simpleResponse = {
|
164
224
|
code: response.statusCode,
|
165
225
|
headers: response.headers as StringMap,
|
166
226
|
body: responseBody,
|
167
|
-
}
|
227
|
+
};
|
228
|
+
|
229
|
+
traffic.withResponse(maskSimpleResponse(simpleResponse, consumerMethod, consumerEntitiesSource));
|
168
230
|
|
169
231
|
socketManager.emit(traffic.connectionId, 'traffic_end', traffic);
|
170
232
|
|
@@ -174,4 +236,4 @@ export function proxyRestRequest(req: StringBodyRequest, res: Response, opts: Pr
|
|
174
236
|
|
175
237
|
res.end();
|
176
238
|
});
|
177
|
-
}
|
239
|
+
};
|