@scrypted/server 0.1.5 → 0.1.8

package/src/plugin/plugin-host.ts

@@ -4,13 +4,16 @@ import crypto from 'crypto';
 import * as io from 'engine.io';
 import fs from 'fs';
 import mkdirp from 'mkdirp';
+import net from 'net';
 import path from 'path';
 import rimraf from 'rimraf';
+import { Duplex } from 'stream';
 import WebSocket, { once } from 'ws';
 import { Plugin } from '../db-types';
 import { IOServer, IOServerSocket } from '../io';
 import { Logger } from '../logger';
 import { RpcPeer } from '../rpc';
+import { createDuplexRpcPeer, createRpcSerializer } from '../rpc-serializer';
 import { ScryptedRuntime } from '../runtime';
 import { sleep } from '../sleep';
 import { SidebandBufferSerializer } from './buffer-serializer';
@@ -134,23 +137,8 @@ export class PluginHost {
                 try {
                     if (socket.request.url.indexOf('/api') !== -1) {
                         if (socket.request.url.indexOf('/public') !== -1) {
-                            socket.send(JSON.stringify({
-                                // @ts-expect-error
-                                id: socket.id,
-                            }));
-                            const timeout = new Promise((_, rj) => setTimeout(() => rj(new Error('timeout')), 10000));
-                            try {
-                                await Promise.race([
-                                    once(socket, '/api/activate'),
-                                    timeout,
-                                ]);
-                                // client will send a start request when it's ready to process events.
-                                await once(socket, 'message');
-                            }
-                            catch (e) {
-                                socket.close();
-                                return;
-                            }
+                            socket.close();
+                            return;
                         }
 
                         await this.createRpcIoPeer(socket);
@@ -344,51 +332,62 @@ export class PluginHost {
             disconnect();
         });
 
+        this.worker.on('rpc', (message, sendHandle) => {
+            this.createRpcPeer(sendHandle as net.Socket);
+        });
+
         this.peer.params.updateStats = (stats: any) => {
             this.stats = stats;
         }
     }
 
     async createRpcIoPeer(socket: IOServerSocket) {
-        let connected = true;
-        const rpcPeer = new RpcPeer(`api/${this.pluginId}`, 'web', (message, reject, serializationContext) => {
-            if (!connected) {
-                reject?.(new Error('peer disconnected'));
-                return;
-            }
-            const buffers = serializationContext?.buffers;
-            if (buffers) {
-                for (const buffer of buffers) {
-                    socket.send(buffer);
-                }
-            }
-            socket.send(JSON.stringify(message))
+        const serializer = createRpcSerializer({
+            sendMessageBuffer: buffer => socket.send(buffer),
+            sendMessageFinish: message => socket.send(JSON.stringify(message)),
         });
-        let pendingSerializationContext: any = {};
+
         socket.on('message', data => {
             if (data.constructor === Buffer || data.constructor === ArrayBuffer) {
-                pendingSerializationContext = pendingSerializationContext || {
-                    buffers: [],
-                };
-                const buffers: Buffer[] = pendingSerializationContext.buffers;
-                buffers.push(Buffer.from(data));
-                return;
+                serializer.onMessageBuffer(Buffer.from(data));
+            }
+            else {
+                serializer.onMessageFinish(JSON.parse(data as string));
+            }
+        });
+
+        const rpcPeer = new RpcPeer(`api/${this.pluginId}`, 'engine.io', (message, reject, serializationContext) => {
+            try {
+                serializer.sendMessage(message, reject, serializationContext);
+            }
+            catch (e) {
+                reject?.(e);
             }
-            const messageSerializationContext = pendingSerializationContext;
-            pendingSerializationContext = undefined;
-            rpcPeer.handleMessage(JSON.parse(data as string), messageSerializationContext);
         });
+        serializer.setupRpcPeer(rpcPeer);
+
         // wrap the host api with a connection specific api that can be torn down on disconnect
         const api = new PluginAPIProxy(this.api, await this.peer.getParam('mediaManager'));
         const kill = () => {
-            connected = false;
-            rpcPeer.kill('engine.io connection closed.')
+            serializer.onDisconnected();
             api.removeListeners();
         }
         socket.on('close', kill);
         socket.on('error', kill);
 
-        rpcPeer.addSerializer(Buffer, 'Buffer', new SidebandBufferSerializer());
+        return setupPluginRemote(rpcPeer, api, null, () => this.scrypted.stateManager.getSystemState());
+    }
+
+    async createRpcPeer(duplex: Duplex) {
+        const rpcPeer = createDuplexRpcPeer(`api/${this.pluginId}`, 'duplex', duplex, duplex);
+
+        // wrap the host api with a connection specific api that can be torn down on disconnect
+        const api = new PluginAPIProxy(this.api, await this.peer.getParam('mediaManager'));
+        const kill = () => {
+            api.removeListeners();
+        };
+        duplex.on('close', kill);
+
         return setupPluginRemote(rpcPeer, api, null, () => this.scrypted.stateManager.getSystemState());
     }
 }

package/src/plugin/plugin-http.ts

@@ -1,9 +1,14 @@
 import { HttpRequest } from '@scrypted/types';
 import bodyParser from 'body-parser';
 import { Request, Response, Router } from 'express';
-import { ServerResponse } from 'http';
+import { ServerResponse, IncomingHttpHeaders } from 'http';
 import WebSocket, { Server as WebSocketServer } from "ws";
 
+export function isConnectionUpgrade(headers: IncomingHttpHeaders) {
+    // connection:'keep-alive, Upgrade'
+    return headers.connection?.toLowerCase().includes('upgrade');
+}
+
 export abstract class PluginHttp<T> {
     wss = new WebSocketServer({ noServer: true });
 
@@ -30,7 +35,7 @@ export abstract class PluginHttp<T> {
             this.endpointHandler(req, res, false, false, this.handleRequestEndpoint.bind(this))
         });
     }
-    
+
     abstract handleEngineIOEndpoint(req: Request, res: ServerResponse, endpointRequest: HttpRequest, pluginData: T): void;
     abstract handleRequestEndpoint(req: Request, res: Response, endpointRequest: HttpRequest, pluginData: T): void;
     abstract getEndpointPluginData(endpoint: string, isUpgrade: boolean, isEngineIOEndpoint: boolean): Promise<T>;
@@ -39,7 +44,7 @@ export abstract class PluginHttp<T> {
     async endpointHandler(req: Request, res: Response, isPublicEndpoint: boolean, isEngineIOEndpoint: boolean,
         handler: (req: Request, res: Response, endpointRequest: HttpRequest, pluginData: T) => void) {
 
-        const isUpgrade = req.headers.connection?.toLowerCase() === 'upgrade';
+        const isUpgrade = isConnectionUpgrade(req.headers);
 
         const end = (code: number, message: string) => {
             if (isUpgrade) {

package/src/plugin/plugin-npm-dependencies.ts

@@ -5,10 +5,12 @@ import path from 'path';
 import { once } from 'events';
 import process from 'process';
 import mkdirp from "mkdirp";
+import semver from 'semver';
 
 export function getPluginNodePath(name: string) {
     const pluginVolume = ensurePluginVolume(name);
-    const nodePrefix = path.join(pluginVolume, `${process.platform}-${process.arch}`);
+    const nodeMajorVersion = semver.parse(process.version).major;
+    const nodePrefix = path.join(pluginVolume, `${process.platform}-${process.arch}-${nodeMajorVersion}`);
     return nodePrefix;
 }
 

package/src/plugin/plugin-repl.ts

@@ -1,4 +1,4 @@
-import { listenZero } from './listen-zero';
+import { listenZero } from '../listen-zero';
 import { Server } from 'net';
 import { once } from 'events';
 import repl from 'repl';

package/src/plugin/runtime/node-fork-worker.ts

@@ -30,7 +30,14 @@ export class NodeForkWorker extends ChildProcessWorker {
     }
 
     setupRpcPeer(peer: RpcPeer): void {
-        this.worker.on('message', message => peer.handleMessage(message as any));
+        this.worker.on('message', (message, sendHandle) => {
+            if (sendHandle) {
+                this.emit('rpc', message, sendHandle);
+            }
+            else {
+                peer.handleMessage(message as any);
+            }
+        });
         peer.transportSafeArgumentTypes.add(Buffer.name);
     }
 

package/src/plugin/runtime/runtime-worker.ts

@@ -1,6 +1,7 @@
 import { RpcMessage, RpcPeer } from "../../rpc";
 import { PluginDebug } from "../plugin-debug";
 import {Readable} from "stream";
+import net from "net";
 
 export interface RuntimeWorkerOptions {
     pluginDebug: PluginDebug;
@@ -15,6 +16,7 @@ export interface RuntimeWorker {
 
     kill(): void;
 
+    on(event: 'rpc', listener: (message: any, sendHandle: net.Socket) => void): this;
     on(event: 'error', listener: (err: Error) => void): this;
     on(event: 'close', listener: (code: number | null, signal: NodeJS.Signals | null) => void): this;
     on(event: 'disconnect', listener: () => void): this;
@@ -25,4 +27,3 @@ export interface RuntimeWorker {
 
     setupRpcPeer(peer: RpcPeer): void;
 }
-

package/src/rpc-serializer.ts

@@ -0,0 +1,141 @@
+import type { Readable, Writable } from "stream";
+import { SidebandBufferSerializer } from "./plugin/buffer-serializer";
+import { RpcPeer } from "./rpc";
+
+export function createDuplexRpcPeer(selfName: string, peerName: string, readable: Readable, writable: Writable) {
+    const serializer = createRpcDuplexSerializer(readable, writable);
+
+    const rpcPeer = new RpcPeer(selfName, peerName, (message, reject, serializationContext) => {
+        try {
+            serializer.sendMessage(message, reject, serializationContext);
+        }
+        catch (e) {
+            reject?.(e);
+            readable.destroy();
+        }
+    });
+
+    serializer.setupRpcPeer(rpcPeer);
+    readable.on('close', serializer.onDisconnected);
+    readable.on('error', serializer.onDisconnected);
+    return rpcPeer;
+}
+
+export function createRpcSerializer(options: {
+    sendMessageBuffer: (buffer: Buffer) => void,
+    sendMessageFinish: (message: any) => void,
+}) {
+    let rpcPeer: RpcPeer;
+
+    const { sendMessageBuffer, sendMessageFinish } = options;
+    let connected = true;
+    const onDisconnected = () => {
+        connected = false;
+        rpcPeer.kill('connection closed.');
+    }
+
+    const sendMessage = (message: any, reject: (e: Error) => void, serializationContext: any, ) => {
+        if (!connected) {
+            reject?.(new Error('peer disconnected'));
+            return;
+        }
+
+        const buffers = serializationContext?.buffers;
+        if (buffers) {
+            for (const buffer of buffers) {
+                sendMessageBuffer(buffer);
+            }
+        }
+        sendMessageFinish(message);
+    }
+
+    let pendingSerializationContext: any = {};
+    const setupRpcPeer = (peer: RpcPeer) => {
+        rpcPeer = peer;
+        rpcPeer.addSerializer(Buffer, 'Buffer', new SidebandBufferSerializer());
+    }
+
+    const onMessageBuffer = (buffer: Buffer) => {
+        pendingSerializationContext = pendingSerializationContext || {
+            buffers: [],
+        };
+        const buffers: Buffer[] = pendingSerializationContext.buffers;
+        buffers.push(buffer);
+    };
+
+    const onMessageFinish = (message: any) => {
+        const messageSerializationContext = pendingSerializationContext;
+        pendingSerializationContext = undefined;
+        rpcPeer.handleMessage(message, messageSerializationContext);
+    }
+
+    return {
+        sendMessage,
+        setupRpcPeer,
+        onMessageBuffer,
+        onMessageFinish,
+        onDisconnected,
+    };
+}
+
+export function createRpcDuplexSerializer(readable: Readable, writable: Writable) {
+    const socketSend = (type: number, data: Buffer) => {
+        const header = Buffer.alloc(5);
+        header.writeUInt32BE(data.length + 1, 0);
+        header.writeUInt8(type, 4);
+
+        writable.write(Buffer.concat([header, data]));
+    }
+
+    const createSocketSend = (type: number) => {
+        return (data: Buffer) => {
+            return socketSend(type, data);
+        }
+    }
+
+    const sendMessageBuffer = createSocketSend(1);
+    const sendMessageFinish = createSocketSend(0);
+
+    const serializer = createRpcSerializer({
+        sendMessageBuffer,
+        sendMessageFinish: (message) => sendMessageFinish(Buffer.from(JSON.stringify(message))),
+    });
+
+    let header: Buffer;
+    const readMessages = () => {
+        while (true) {
+            if (!header) {
+                header = readable.read(5);
+                if (!header)
+                    return;
+            }
+
+            const length = header.readUInt32BE(0);
+            const type = header.readUInt8(4);
+            const payload: Buffer = readable.read(length - 1);
+            if (!payload)
+                return;
+
+            header = undefined;
+
+            const data = payload;
+
+            if (type === 0) {
+                const message = JSON.parse(data.toString());
+                serializer.onMessageFinish(message);
+            }
+            else {
+                serializer.onMessageBuffer(data);
+            }
+        }
+    }
+
+    readable.on('readable', readMessages);
+    readMessages();
+
+    return {
+        setupRpcPeer: serializer.setupRpcPeer,
+        sendMessage: serializer.sendMessage,
+        onDisconnected: serializer.onDisconnected,
+    };
+}

package/src/runtime.ts

@@ -29,7 +29,7 @@ import * as io from 'engine.io';
 import { spawn as ptySpawn } from 'node-pty';
 import rimraf from 'rimraf';
 import { getPluginVolume } from './plugin/plugin-volume';
-import { PluginHttp } from './plugin/plugin-http';
+import { isConnectionUpgrade, PluginHttp } from './plugin/plugin-http';
 import AdmZip from 'adm-zip';
 import path from 'path';
 import { CORSControl, CORSServer } from './services/cors';
@@ -78,46 +78,6 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
 
         app.disable('x-powered-by');
 
-        this.app.options(['/endpoint/@:owner/:pkg/engine.io/api/activate', '/endpoint/@:owner/:pkg/engine.io/api/activate'], (req, res) => {
-            this.addAccessControlHeaders(req, res);
-            res.setHeader('Access-Control-Allow-Methods', 'POST, OPTIONS');
-            res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, Content-Length, X-Requested-With');
-            res.send(200);
-        });
-
-        this.app.post(['/endpoint/@:owner/:pkg/engine.io/api/activate', '/endpoint/@:owner/:pkg/engine.io/api/activate'], (req, res) => {
-            const { username } = (req as any);
-            if (!username) {
-                res.status(401);
-                res.send('Not Authorized');
-                return;
-            }
-
-            const { owner, pkg } = req.params;
-            let endpoint = pkg;
-            if (owner)
-                endpoint = `@${owner}/${endpoint}`;
-
-            const { id } = req.body;
-            try {
-                const host = this.plugins?.[endpoint];
-                if (!host)
-                    throw new Error('invalid plugin');
-                // @ts-expect-error
-                const socket: IOServerSocket = host.io.clients[id];
-                if (!socket)
-                    throw new Error('invalid socket');
-                socket.emit('/api/activate');
-                res.send({
-                    id,
-                })
-            }
-            catch (e) {
-                res.status(500);
-                res.end();
-            }
-        });
-
         this.addMiddleware();
 
         app.get('/web/oauth/callback', (req, res) => {
@@ -297,7 +257,7 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
     }
 
     async shellHandler(req: Request, res: Response) {
-        const isUpgrade = req.headers.connection?.toLowerCase() === 'upgrade';
+        const isUpgrade = isConnectionUpgrade(req.headers);
 
         const end = (code: number, message: string) => {
             if (isUpgrade) {
@@ -432,7 +392,7 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
     handleRequestEndpoint(req: Request, res: Response, endpointRequest: HttpRequest, pluginData: HttpPluginData) {
         const { pluginHost, pluginDevice } = pluginData;
         const handler = this.getDevice<HttpRequestHandler>(pluginDevice._id);
-        if (handler.interfaces.includes(ScryptedInterface.EngineIOHandler) && req.headers.connection === 'upgrade' && req.headers.upgrade?.toLowerCase() === 'websocket') {
+        if (handler.interfaces.includes(ScryptedInterface.EngineIOHandler) && isConnectionUpgrade(req.headers) && req.headers.upgrade?.toLowerCase() === 'websocket') {
             this.wss.handleUpgrade(req, req.socket, null, ws => {
                 console.log(ws);
             });

package/src/scrypted-server-main.ts

@@ -2,7 +2,7 @@ import path from 'path';
 import process from 'process';
 import http from 'http';
 import https from 'https';
-import express from 'express';
+import express, { Request } from 'express';
 import bodyParser from 'body-parser';
 import net from 'net';
 import { ScryptedRuntime } from './runtime';
@@ -160,7 +160,7 @@ async function start() {
         // only basic auth will fail with 401. it is up to the endpoints to manage
         // lack of login from cookie auth.
 
-        const { login_user_token } = req.signedCookies;
+        const login_user_token = getSignedLoginUserToken(req);
         if (login_user_token) {
             const userTokenParts = login_user_token.split('#');
             const username = userTokenParts[0];
@@ -353,8 +353,16 @@ async function start() {
         });
     });
 
+    const getLoginUserToken = (reqSecure: boolean) => {
+        return reqSecure ? 'login_user_token' : 'login_user_token_inseucre';
+    };
+
+    const getSignedLoginUserToken = (req: Request<any>): string => {
+        return req.signedCookies[getLoginUserToken(req.secure)];
+    };
+
     app.get('/logout', (req, res) => {
-        res.clearCookie('login_user_token');
+        res.clearCookie(getLoginUserToken(req.secure));
         res.send({});
     });
 
@@ -398,13 +406,11 @@ async function start() {
             }
 
             const login_user_token = `${username}#${timestamp}`;
-            res.cookie('login_user_token', login_user_token, {
+            res.cookie(getLoginUserToken(req.secure), login_user_token, {
                 maxAge,
-                // enabling this will disable insecure http login...
-                // secure: true,
+                secure: req.secure,
                 signed: true,
                 httpOnly: true,
-                sameSite: 'none',
             });
 
             if (change_password) {
@@ -441,13 +447,11 @@ async function start() {
         hasLogin = true;
 
         const login_user_token = `${username}#${timestamp}`
-        res.cookie('login_user_token', login_user_token, {
+        res.cookie(getLoginUserToken(req.secure), login_user_token, {
             maxAge,
-            // enabling this will disable insecure http login...
-            // secure: true,
+            secure: req.secure,
             signed: true,
             httpOnly: true,
-            sameSite: 'none',
         });
 
         res.send({
@@ -484,7 +488,7 @@ async function start() {
             return;
         }
 
-        const { login_user_token } = req.signedCookies;
+        const login_user_token = getSignedLoginUserToken(req);
         if (!login_user_token) {
             res.send({
                 error: 'Not logged in.',

package/src/services/plugin.ts

@@ -51,26 +51,20 @@ export class PluginComponent {
         this.scrypted.getDevice(id);
         await this.scrypted.devices[id]?.handler?.ensureProxy();
     }
-    async getMixins(id: string) {
-        console.warn('legacy use of getMixins, use the mixins property');
-        const pluginDevice = this.scrypted.findPluginDeviceById(id);
-        return getState(pluginDevice, ScryptedInterfaceProperty.mixins) || [];
-    }
     async getIdForPluginId(pluginId: string) {
         return this.scrypted.findPluginDevice(pluginId)?._id;
     }
     async getIdForNativeId(pluginId: string, nativeId: ScryptedNativeId) {
         return this.scrypted.findPluginDevice(pluginId, nativeId)?._id;
     }
+    /**
+     * @deprecated available as device.pluginId now.
+     * Remove at some point after core/ui rolls out 6/20/2022.
+     */
     async getPluginId(id: string) {
         const pluginDevice = this.scrypted.findPluginDeviceById(id);
         return pluginDevice.pluginId;
     }
-    async getPluginProcessId(pluginId: string) {
-        if (this.scrypted.plugins[pluginId]?.worker?.killed)
-            return 'killed';
-        return this.scrypted.plugins[pluginId]?.worker?.pid;
-    }
     async reload(pluginId: string) {
         const plugin = await this.scrypted.datastore.tryGet(Plugin, pluginId);
         await this.scrypted.runPlugin(plugin);

package/src/threading.ts

@@ -1,6 +1,7 @@
 import worker_threads from 'worker_threads';
 import { getEvalSource, RpcPeer } from './rpc';
 import v8 from 'v8';
+import vm from 'vm';
 
 export async function newThread<T>(thread: () => Promise<T>): Promise<T>;
 export async function newThread<V, T>(params: V, thread: (params: V) => Promise<T>): Promise<T>;
@@ -28,22 +29,22 @@ export async function newThread<T>(...args: any[]): Promise<T> {
             const g = global as any;
             g[customRequire] = g.require;
         }
-        const v8 = global.require('v8');
-        const worker_threads = global.require('worker_threads');
-        const vm = global.require('vm');
-        const mainPeer = new RpcPeer('thread', 'main', (message: any, reject: any) => {
+        const thread_v8: typeof v8 = global.require('v8');
+        const thread_worker_threads: typeof worker_threads = global.require('worker_threads');
+        const thread_vm: typeof vm = global.require('vm');
+        const mainPeer: RpcPeer = new RpcPeer('thread', 'main', (message: any, reject: any) => {
             try {
-                worker_threads.parentPort.postMessage(v8.serialize(message));
+                thread_worker_threads.parentPort.postMessage(thread_v8.serialize(message));
             }
             catch (e) {
                 reject?.(e);
             }
         });
         mainPeer.transportSafeArgumentTypes.add(Buffer.name);
-        worker_threads.parentPort.on('message', (message: any) => mainPeer.handleMessage(v8.deserialize(message)));
+        thread_worker_threads.parentPort.on('message', (message: any) => mainPeer.handleMessage(thread_v8.deserialize(message)));
 
         mainPeer.params.eval = async (script: string, moduleNames: string[], paramNames: string[], ...paramValues: any[]) => {
-            const f = vm.compileFunction(`return (${script})`, paramNames, {
+            const f = thread_vm.compileFunction(`return (${script})`, paramNames, {
                 filename: 'script.js',
             });
             const params: any = {};
@@ -92,31 +93,3 @@ export async function newThread<T>(...args: any[]): Promise<T> {
         worker.terminate();
     }
 }
-
-async function test() {
-    const foo = 5;
-    const bar = 6;
-
-    console.log(await newThread({
-        foo, bar,
-    }, async () => {
-        return foo + bar;
-    }));
-
-
-    console.log(await newThread({
-        foo, bar,
-    }, async ({foo,bar}) => {
-        return foo + bar;
-    }));
-
-    const sayHelloInMainThread = () => console.log('hello! main thread:', worker_threads.isMainThread);
-    await newThread({
-        sayHelloInMainThread,
-    }, async () => {
-        sayHelloInMainThread();
-    })
-}
-
-// if (true)
-//     test();

package/test/rpc-duplex-test.ts

@@ -0,0 +1,30 @@
+import net from 'net';
+import { listenZeroSingleClient } from "../src/listen-zero";
+import { createDuplexRpcPeer } from "../src/rpc-serializer";
+
+async function test() {
+    const { port, clientPromise } = await listenZeroSingleClient();
+
+
+    const n1 = net.connect({
+        port,
+        host: '127.0.0.1',
+    });
+
+    const n2 = await clientPromise;
+    console.log('connected');
+
+    const p1 = createDuplexRpcPeer('p1', 'p2', n1, n1);
+    const p2 = createDuplexRpcPeer('p2', 'p1', n2, n2);
+
+    p1.params.test = () => console.log('p1 test');
+    p2.params.test = () => console.log('p2 test');
+
+    await (await p1.getParam('test'))();
+    await (await p2.getParam('test'))();
+
+    n1.destroy();
+    n2.destroy();
+}
+
+test();

package/test/threading-test.ts

@@ -0,0 +1,29 @@
+import worker_threads from 'worker_threads';
+import { newThread } from "../src/threading";
+
+async function test() {
+    const foo = 5;
+    const bar = 6;
+
+    console.log(await newThread({
+        foo, bar,
+    }, async () => {
+        return foo + bar;
+    }));
+
+
+    console.log(await newThread({
+        foo, bar,
+    }, async ({ foo, bar }) => {
+        return foo + bar;
+    }));
+
+    const sayHelloInMainThread = () => console.log('hello! main thread:', worker_threads.isMainThread);
+    await newThread({
+        sayHelloInMainThread,
+    }, async () => {
+        sayHelloInMainThread();
+    })
+}
+
+test();