@scrypted/server 0.1.7 → 0.1.10

package/src/plugin/plugin-http.ts

@@ -5,6 +5,7 @@ import { ServerResponse, IncomingHttpHeaders } from 'http';
 import WebSocket, { Server as WebSocketServer } from "ws";
 
 export function isConnectionUpgrade(headers: IncomingHttpHeaders) {
+    // connection:'keep-alive, Upgrade'
     return headers.connection?.toLowerCase().includes('upgrade');
 }
 
@@ -34,7 +35,7 @@ export abstract class PluginHttp<T> {
             this.endpointHandler(req, res, false, false, this.handleRequestEndpoint.bind(this))
         });
     }
-    
+
     abstract handleEngineIOEndpoint(req: Request, res: ServerResponse, endpointRequest: HttpRequest, pluginData: T): void;
     abstract handleRequestEndpoint(req: Request, res: Response, endpointRequest: HttpRequest, pluginData: T): void;
     abstract getEndpointPluginData(endpoint: string, isUpgrade: boolean, isEngineIOEndpoint: boolean): Promise<T>;
@@ -43,7 +44,6 @@ export abstract class PluginHttp<T> {
     async endpointHandler(req: Request, res: Response, isPublicEndpoint: boolean, isEngineIOEndpoint: boolean,
         handler: (req: Request, res: Response, endpointRequest: HttpRequest, pluginData: T) => void) {
 
-        // connection:'keep-alive, Upgrade'
         const isUpgrade = isConnectionUpgrade(req.headers);
 
         const end = (code: number, message: string) => {

package/src/plugin/plugin-npm-dependencies.ts

@@ -5,10 +5,12 @@ import path from 'path';
 import { once } from 'events';
 import process from 'process';
 import mkdirp from "mkdirp";
+import semver from 'semver';
 
 export function getPluginNodePath(name: string) {
     const pluginVolume = ensurePluginVolume(name);
-    const nodePrefix = path.join(pluginVolume, `${process.platform}-${process.arch}`);
+    const nodeMajorVersion = semver.parse(process.version).major;
+    const nodePrefix = path.join(pluginVolume, `${process.platform}-${process.arch}-${nodeMajorVersion}`);
     return nodePrefix;
 }
 

package/src/plugin/plugin-repl.ts

@@ -1,4 +1,4 @@
-import { listenZero } from './listen-zero';
+import { listenZero } from '../listen-zero';
 import { Server } from 'net';
 import { once } from 'events';
 import repl from 'repl';

package/src/plugin/runtime/node-fork-worker.ts

@@ -30,7 +30,14 @@ export class NodeForkWorker extends ChildProcessWorker {
     }
 
     setupRpcPeer(peer: RpcPeer): void {
-        this.worker.on('message', message => peer.handleMessage(message as any));
+        this.worker.on('message', (message, sendHandle) => {
+            if (sendHandle) {
+                this.emit('rpc', message, sendHandle);
+            }
+            else {
+                peer.handleMessage(message as any);
+            }
+        });
         peer.transportSafeArgumentTypes.add(Buffer.name);
     }
 

package/src/plugin/runtime/runtime-worker.ts

@@ -1,6 +1,7 @@
 import { RpcMessage, RpcPeer } from "../../rpc";
 import { PluginDebug } from "../plugin-debug";
 import {Readable} from "stream";
+import net from "net";
 
 export interface RuntimeWorkerOptions {
     pluginDebug: PluginDebug;
@@ -15,6 +16,7 @@ export interface RuntimeWorker {
 
     kill(): void;
 
+    on(event: 'rpc', listener: (message: any, sendHandle: net.Socket) => void): this;
     on(event: 'error', listener: (err: Error) => void): this;
     on(event: 'close', listener: (code: number | null, signal: NodeJS.Signals | null) => void): this;
     on(event: 'disconnect', listener: () => void): this;
@@ -25,4 +27,3 @@ export interface RuntimeWorker {
 
     setupRpcPeer(peer: RpcPeer): void;
 }
-

package/src/rpc-serializer.ts

@@ -0,0 +1,141 @@
+import type { Readable, Writable } from "stream";
+import { SidebandBufferSerializer } from "./plugin/buffer-serializer";
+import { RpcPeer } from "./rpc";
+
+export function createDuplexRpcPeer(selfName: string, peerName: string, readable: Readable, writable: Writable) {
+    const serializer = createRpcDuplexSerializer(readable, writable);
+
+    const rpcPeer = new RpcPeer(selfName, peerName, (message, reject, serializationContext) => {
+        try {
+            serializer.sendMessage(message, reject, serializationContext);
+        }
+        catch (e) {
+            reject?.(e);
+            readable.destroy();
+        }
+    });
+
+    serializer.setupRpcPeer(rpcPeer);
+    readable.on('close', serializer.onDisconnected);
+    readable.on('error', serializer.onDisconnected);
+    return rpcPeer;
+}
+
+export function createRpcSerializer(options: {
+    sendMessageBuffer: (buffer: Buffer) => void,
+    sendMessageFinish: (message: any) => void,
+}) {
+    let rpcPeer: RpcPeer;
+
+    const { sendMessageBuffer, sendMessageFinish } = options;
+    let connected = true;
+    const onDisconnected = () => {
+        connected = false;
+        rpcPeer.kill('connection closed.');
+    }
+
+    const sendMessage = (message: any, reject: (e: Error) => void, serializationContext: any, ) => {
+        if (!connected) {
+            reject?.(new Error('peer disconnected'));
+            return;
+        }
+
+        const buffers = serializationContext?.buffers;
+        if (buffers) {
+            for (const buffer of buffers) {
+                sendMessageBuffer(buffer);
+            }
+        }
+        sendMessageFinish(message);
+    }
+
+    let pendingSerializationContext: any = {};
+    const setupRpcPeer = (peer: RpcPeer) => {
+        rpcPeer = peer;
+        rpcPeer.addSerializer(Buffer, 'Buffer', new SidebandBufferSerializer());
+    }
+
+    const onMessageBuffer = (buffer: Buffer) => {
+        pendingSerializationContext = pendingSerializationContext || {
+            buffers: [],
+        };
+        const buffers: Buffer[] = pendingSerializationContext.buffers;
+        buffers.push(buffer);
+    };
+
+    const onMessageFinish = (message: any) => {
+        const messageSerializationContext = pendingSerializationContext;
+        pendingSerializationContext = undefined;
+        rpcPeer.handleMessage(message, messageSerializationContext);
+    }
+
+    return {
+        sendMessage,
+        setupRpcPeer,
+        onMessageBuffer,
+        onMessageFinish,
+        onDisconnected,
+    };
+}
+
+export function createRpcDuplexSerializer(readable: Readable, writable: Writable) {
+    const socketSend = (type: number, data: Buffer) => {
+        const header = Buffer.alloc(5);
+        header.writeUInt32BE(data.length + 1, 0);
+        header.writeUInt8(type, 4);
+
+        writable.write(Buffer.concat([header, data]));
+    }
+
+    const createSocketSend = (type: number) => {
+        return (data: Buffer) => {
+            return socketSend(type, data);
+        }
+    }
+
+    const sendMessageBuffer = createSocketSend(1);
+    const sendMessageFinish = createSocketSend(0);
+
+    const serializer = createRpcSerializer({
+        sendMessageBuffer,
+        sendMessageFinish: (message) => sendMessageFinish(Buffer.from(JSON.stringify(message))),
+    });
+
+    let header: Buffer;
+    const readMessages = () => {
+        while (true) {
+            if (!header) {
+                header = readable.read(5);
+                if (!header)
+                    return;
+            }
+
+            const length = header.readUInt32BE(0);
+            const type = header.readUInt8(4);
+            const payload: Buffer = readable.read(length - 1);
+            if (!payload)
+                return;
+
+            header = undefined;
+
+            const data = payload;
+
+            if (type === 0) {
+                const message = JSON.parse(data.toString());
+                serializer.onMessageFinish(message);
+            }
+            else {
+                serializer.onMessageBuffer(data);
+            }
+        }
+    }
+
+    readable.on('readable', readMessages);
+    readMessages();
+
+    return {
+        setupRpcPeer: serializer.setupRpcPeer,
+        sendMessage: serializer.sendMessage,
+        onDisconnected: serializer.onDisconnected,
+    };
+}

package/src/runtime.ts

@@ -78,46 +78,6 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
 
         app.disable('x-powered-by');
 
-        this.app.options(['/endpoint/@:owner/:pkg/engine.io/api/activate', '/endpoint/@:owner/:pkg/engine.io/api/activate'], (req, res) => {
-            this.addAccessControlHeaders(req, res);
-            res.setHeader('Access-Control-Allow-Methods', 'POST, OPTIONS');
-            res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, Content-Length, X-Requested-With');
-            res.send(200);
-        });
-
-        this.app.post(['/endpoint/@:owner/:pkg/engine.io/api/activate', '/endpoint/@:owner/:pkg/engine.io/api/activate'], (req, res) => {
-            const { username } = (req as any);
-            if (!username) {
-                res.status(401);
-                res.send('Not Authorized');
-                return;
-            }
-
-            const { owner, pkg } = req.params;
-            let endpoint = pkg;
-            if (owner)
-                endpoint = `@${owner}/${endpoint}`;
-
-            const { id } = req.body;
-            try {
-                const host = this.plugins?.[endpoint];
-                if (!host)
-                    throw new Error('invalid plugin');
-                // @ts-expect-error
-                const socket: IOServerSocket = host.io.clients[id];
-                if (!socket)
-                    throw new Error('invalid socket');
-                socket.emit('/api/activate');
-                res.send({
-                    id,
-                })
-            }
-            catch (e) {
-                res.status(500);
-                res.end();
-            }
-        });
-
         this.addMiddleware();
 
         app.get('/web/oauth/callback', (req, res) => {

package/src/scrypted-server-main.ts

@@ -411,7 +411,6 @@ async function start() {
                 secure: req.secure,
                 signed: true,
                 httpOnly: true,
-                sameSite: !req.secure ? true : 'none',
             });
 
             if (change_password) {
@@ -453,7 +452,6 @@ async function start() {
             secure: req.secure,
             signed: true,
             httpOnly: true,
-            sameSite: !req.secure ? true : 'none',
         });
 
         res.send({

package/src/services/plugin.ts

@@ -51,26 +51,20 @@ export class PluginComponent {
         this.scrypted.getDevice(id);
         await this.scrypted.devices[id]?.handler?.ensureProxy();
     }
-    async getMixins(id: string) {
-        console.warn('legacy use of getMixins, use the mixins property');
-        const pluginDevice = this.scrypted.findPluginDeviceById(id);
-        return getState(pluginDevice, ScryptedInterfaceProperty.mixins) || [];
-    }
     async getIdForPluginId(pluginId: string) {
         return this.scrypted.findPluginDevice(pluginId)?._id;
     }
     async getIdForNativeId(pluginId: string, nativeId: ScryptedNativeId) {
         return this.scrypted.findPluginDevice(pluginId, nativeId)?._id;
     }
+    /**
+     * @deprecated available as device.pluginId now.
+     * Remove at some point after core/ui rolls out 6/20/2022.
+     */
     async getPluginId(id: string) {
         const pluginDevice = this.scrypted.findPluginDeviceById(id);
         return pluginDevice.pluginId;
     }
-    async getPluginProcessId(pluginId: string) {
-        if (this.scrypted.plugins[pluginId]?.worker?.killed)
-            return 'killed';
-        return this.scrypted.plugins[pluginId]?.worker?.pid;
-    }
     async reload(pluginId: string) {
         const plugin = await this.scrypted.datastore.tryGet(Plugin, pluginId);
         await this.scrypted.runPlugin(plugin);
@@ -94,14 +88,17 @@ export class PluginComponent {
         const packageJson = await this.getPackageJson(pluginId);
         const host = this.scrypted.plugins[pluginId];
         let rpcObjects = 0;
+        let pendingResults = 0;
         if (host.peer) {
             rpcObjects = host.peer.localProxied.size + Object.keys(host.peer.remoteWeakProxies).length;
+            pendingResults = Object.keys(host.peer.pendingResults).length;
         }
         return {
             pid: host?.worker?.pid,
             stats: host?.stats,
             rpcObjects,
             packageJson,
+            pendingResults,
             id: this.scrypted.findPluginDevice(pluginId)._id,
         }
     }

package/src/threading.ts

@@ -1,6 +1,7 @@
 import worker_threads from 'worker_threads';
 import { getEvalSource, RpcPeer } from './rpc';
 import v8 from 'v8';
+import vm from 'vm';
 
 export async function newThread<T>(thread: () => Promise<T>): Promise<T>;
 export async function newThread<V, T>(params: V, thread: (params: V) => Promise<T>): Promise<T>;
@@ -28,22 +29,22 @@ export async function newThread<T>(...args: any[]): Promise<T> {
             const g = global as any;
             g[customRequire] = g.require;
         }
-        const v8 = global.require('v8');
-        const worker_threads = global.require('worker_threads');
-        const vm = global.require('vm');
-        const mainPeer = new RpcPeer('thread', 'main', (message: any, reject: any) => {
+        const thread_v8: typeof v8 = global.require('v8');
+        const thread_worker_threads: typeof worker_threads = global.require('worker_threads');
+        const thread_vm: typeof vm = global.require('vm');
+        const mainPeer: RpcPeer = new RpcPeer('thread', 'main', (message: any, reject: any) => {
             try {
-                worker_threads.parentPort.postMessage(v8.serialize(message));
+                thread_worker_threads.parentPort.postMessage(thread_v8.serialize(message));
             }
             catch (e) {
                 reject?.(e);
             }
         });
         mainPeer.transportSafeArgumentTypes.add(Buffer.name);
-        worker_threads.parentPort.on('message', (message: any) => mainPeer.handleMessage(v8.deserialize(message)));
+        thread_worker_threads.parentPort.on('message', (message: any) => mainPeer.handleMessage(thread_v8.deserialize(message)));
 
         mainPeer.params.eval = async (script: string, moduleNames: string[], paramNames: string[], ...paramValues: any[]) => {
-            const f = vm.compileFunction(`return (${script})`, paramNames, {
+            const f = thread_vm.compileFunction(`return (${script})`, paramNames, {
                 filename: 'script.js',
             });
             const params: any = {};
@@ -92,31 +93,3 @@ export async function newThread<T>(...args: any[]): Promise<T> {
         worker.terminate();
     }
 }
-
-async function test() {
-    const foo = 5;
-    const bar = 6;
-
-    console.log(await newThread({
-        foo, bar,
-    }, async () => {
-        return foo + bar;
-    }));
-
-
-    console.log(await newThread({
-        foo, bar,
-    }, async ({foo,bar}) => {
-        return foo + bar;
-    }));
-
-    const sayHelloInMainThread = () => console.log('hello! main thread:', worker_threads.isMainThread);
-    await newThread({
-        sayHelloInMainThread,
-    }, async () => {
-        sayHelloInMainThread();
-    })
-}
-
-// if (true)
-//     test();

package/test/rpc-duplex-test.ts

@@ -0,0 +1,30 @@
+import net from 'net';
+import { listenZeroSingleClient } from "../src/listen-zero";
+import { createDuplexRpcPeer } from "../src/rpc-serializer";
+
+async function test() {
+    const { port, clientPromise } = await listenZeroSingleClient();
+
+
+    const n1 = net.connect({
+        port,
+        host: '127.0.0.1',
+    });
+
+    const n2 = await clientPromise;
+    console.log('connected');
+
+    const p1 = createDuplexRpcPeer('p1', 'p2', n1, n1);
+    const p2 = createDuplexRpcPeer('p2', 'p1', n2, n2);
+
+    p1.params.test = () => console.log('p1 test');
+    p2.params.test = () => console.log('p2 test');
+
+    await (await p1.getParam('test'))();
+    await (await p2.getParam('test'))();
+
+    n1.destroy();
+    n2.destroy();
+}
+
+test();

package/test/threading-test.ts

@@ -0,0 +1,29 @@
+import worker_threads from 'worker_threads';
+import { newThread } from "../src/threading";
+
+async function test() {
+    const foo = 5;
+    const bar = 6;
+
+    console.log(await newThread({
+        foo, bar,
+    }, async () => {
+        return foo + bar;
+    }));
+
+
+    console.log(await newThread({
+        foo, bar,
+    }, async ({ foo, bar }) => {
+        return foo + bar;
+    }));
+
+    const sayHelloInMainThread = () => console.log('hello! main thread:', worker_threads.isMainThread);
+    await newThread({
+        sayHelloInMainThread,
+    }, async () => {
+        sayHelloInMainThread();
+    })
+}
+
+test();

package/dist/plugin/listen-zero.js

@@ -1,23 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.listenZeroExpress = exports.listenZero = void 0;
-const events_1 = require("events");
-async function listenZero(server) {
-    server.listen(0);
-    await (0, events_1.once)(server, 'listening');
-    return server.address().port;
-}
-exports.listenZero = listenZero;
-function listenZeroExpress(app) {
-    const server = app.listen(0);
-    return {
-        server,
-        port: (async () => {
-            await (0, events_1.once)(server, 'listening');
-            const { port } = server.address();
-            return port;
-        })()
-    };
-}
-exports.listenZeroExpress = listenZeroExpress;
-//# sourceMappingURL=listen-zero.js.map

package/dist/plugin/listen-zero.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"listen-zero.js","sourceRoot":"","sources":["../../src/plugin/listen-zero.ts"],"names":[],"mappings":";;;AACA,mCAA8B;AAGvB,KAAK,UAAU,UAAU,CAAC,MAAkB;IAC/C,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACjB,MAAM,IAAA,aAAI,EAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAChC,OAAQ,MAAM,CAAC,OAAO,EAAsB,CAAC,IAAI,CAAC;AACtD,CAAC;AAJD,gCAIC;AAED,SAAgB,iBAAiB,CAAC,GAAoB;IAClD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC7B,OAAO;QACH,MAAM;QACN,IAAI,EAAE,CAAC,KAAK,IAAI,EAAE;YACd,MAAM,IAAA,aAAI,EAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAChC,MAAM,EAAE,IAAI,EAAE,GAAI,MAAM,CAAC,OAAO,EAAsB,CAAC;YACvD,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC,EAAE;KACP,CAAA;AACL,CAAC;AAVD,8CAUC"}

package/src/plugin/listen-zero.ts

@@ -1,21 +0,0 @@
-import net from 'net';
-import { once } from 'events';
-import express from 'express';
-
-export async function listenZero(server: net.Server) {
-    server.listen(0);
-    await once(server, 'listening');
-    return (server.address() as net.AddressInfo).port;
-}
-
-export function listenZeroExpress(app: express.Express) {
-    const server = app.listen(0);
-    return {
-        server,
-        port: (async () => {
-            await once(server, 'listening');
-            const { port } = (server.address() as net.AddressInfo);
-            return port;
-        })()
-    }
-}