@scrypted/server 0.1.14 → 0.2.1

package/src/rpc-serializer.ts

@@ -3,7 +3,7 @@ import { SidebandBufferSerializer } from "./plugin/buffer-serializer";
 import { RpcPeer } from "./rpc";
 
 export function createDuplexRpcPeer(selfName: string, peerName: string, readable: Readable, writable: Writable) {
-    const serializer = createRpcDuplexSerializer(readable, writable);
+    const serializer = createRpcDuplexSerializer(writable);
 
     const rpcPeer = new RpcPeer(selfName, peerName, (message, reject, serializationContext) => {
         try {
@@ -16,6 +16,7 @@ export function createDuplexRpcPeer(selfName: string, peerName: string, readable
     });
 
     serializer.setupRpcPeer(rpcPeer);
+    readable.on('data', data => serializer.onData(data));
     readable.on('close', serializer.onDisconnected);
     readable.on('error', serializer.onDisconnected);
     return rpcPeer;
@@ -34,7 +35,7 @@ export function createRpcSerializer(options: {
         rpcPeer.kill('connection closed.');
     }
 
-    const sendMessage = (message: any, reject: (e: Error) => void, serializationContext: any, ) => {
+    const sendMessage = (message: any, reject: (e: Error) => void, serializationContext: any,) => {
         if (!connected) {
             reject?.(new Error('peer disconnected'));
             return;
@@ -78,7 +79,9 @@ export function createRpcSerializer(options: {
     };
 }
 
-export function createRpcDuplexSerializer(readable: Readable, writable: Writable) {
+export function createRpcDuplexSerializer(writable: {
+    write: (data: Buffer) => void;
+}) {
     const socketSend = (type: number, data: Buffer) => {
         const header = Buffer.alloc(5);
         header.writeUInt32BE(data.length + 1, 0);
@@ -102,38 +105,52 @@ export function createRpcDuplexSerializer(readable: Readable, writable: Writable
     });
 
     let header: Buffer;
-    const readMessages = () => {
+    let pending: Buffer;
+
+    const readPending = (length: number) => {
+        if (!pending || pending.length < length)
+            return;
+
+        const ret = pending.slice(0, length);
+        pending = pending.slice(length);
+        if (!pending.length)
+            pending = undefined;
+        return ret;
+    }
+
+    const onData = (data: Buffer) => {
+        if (!pending)
+            pending = data;
+        else
+            pending = Buffer.concat([pending, data]);
+
         while (true) {
             if (!header) {
-                header = readable.read(5);
+                header = readPending(5);
                 if (!header)
                     return;
             }
 
             const length = header.readUInt32BE(0);
             const type = header.readUInt8(4);
-            const payload: Buffer = readable.read(length - 1);
+            const payload: Buffer = readPending(length - 1);
             if (!payload)
                 return;
 
             header = undefined;
 
-            const data = payload;
-
             if (type === 0) {
-                const message = JSON.parse(data.toString());
+                const message = JSON.parse(payload.toString());
                 serializer.onMessageFinish(message);
             }
             else {
-                serializer.onMessageBuffer(data);
+                serializer.onMessageBuffer(payload);
             }
         }
     }
 
-    readable.on('readable', readMessages);
-    readMessages();
-
     return {
+        onData,
         setupRpcPeer: serializer.setupRpcPeer,
         sendMessage: serializer.sendMessage,
         onDisconnected: serializer.onDisconnected,

package/src/rpc.ts

@@ -138,7 +138,7 @@ class RpcProxy implements PrimitiveProxyHandler<any> {
 
         if (this.proxyOneWayMethods?.includes?.(method)) {
             rpcApply.oneway = true;
-            this.peer.send(rpcApply);
+            this.peer.send(rpcApply, undefined, serializationContext);
             return Promise.resolve();
         }
 
@@ -321,7 +321,8 @@ export class RpcPeer {
         const params = Object.assign({}, this.params, coercedParams);
         let compile: CompileFunction;
         try {
-            compile = require('vm').compileFunction;;
+            // prevent bundlers from trying to include non-existent vm module.
+            compile = module[`require`]('vm').compileFunction;
         }
         catch (e) {
             compile = compileFunction;

package/src/runtime.ts

@@ -1,39 +1,37 @@
-import { Level } from './level';
-import { PluginHost } from './plugin/plugin-host';
-import { ScryptedNativeId, Device, EngineIOHandler, HttpRequest, HttpRequestHandler, OauthClient, PushHandler, ScryptedDevice, ScryptedInterface, ScryptedInterfaceProperty, DeviceInformation } from '@scrypted/types';
-import { PluginDeviceProxyHandler } from './plugin/plugin-device';
-import { Plugin, PluginDevice, ScryptedAlert } from './db-types';
-import { getState, ScryptedStateManager, setState } from './state';
-import { Request, Response } from 'express';
-import { createResponseInterface } from './http-interfaces';
-import http, { ServerResponse, IncomingHttpHeaders } from 'http';
-import https from 'https';
-import express from 'express';
-import { LogEntry, Logger, makeAlertId } from './logger';
-import { getDisplayName, getDisplayRoom, getDisplayType, getProvidedNameOrDefault, getProvidedRoomOrDefault, getProvidedTypeOrDefault } from './infer-defaults';
-import { URL } from "url";
-import qs from "query-string";
-import { PluginComponent } from './services/plugin';
-import WebSocket, { Server as WebSocketServer } from "ws";
+import { Device, DeviceInformation, EngineIOHandler, HttpRequest, HttpRequestHandler, OauthClient, PushHandler, ScryptedDevice, ScryptedInterface, ScryptedInterfaceProperty, ScryptedNativeId } from '@scrypted/types';
+import AdmZip from 'adm-zip';
 import axios from 'axios';
-import tar from 'tar';
+import * as io from 'engine.io';
 import { once } from 'events';
+import express, { Request, Response } from 'express';
+import http, { ServerResponse } from 'http';
+import https from 'https';
+import { spawn as ptySpawn } from 'node-pty-prebuilt-multiarch';
+import path from 'path';
+import rimraf from 'rimraf';
+import semver from 'semver';
 import { PassThrough } from 'stream';
+import tar from 'tar';
+import { URL } from "url";
+import WebSocket, { Server as WebSocketServer } from "ws";
+import { Plugin, PluginDevice, ScryptedAlert } from './db-types';
+import { createResponseInterface } from './http-interfaces';
+import { getDisplayName, getDisplayRoom, getDisplayType, getProvidedNameOrDefault, getProvidedRoomOrDefault, getProvidedTypeOrDefault } from './infer-defaults';
+import { IOServer } from './io';
+import { Level } from './level';
+import { LogEntry, Logger, makeAlertId } from './logger';
 import { PluginDebug } from './plugin/plugin-debug';
+import { PluginDeviceProxyHandler } from './plugin/plugin-device';
+import { PluginHost } from './plugin/plugin-host';
+import { isConnectionUpgrade, PluginHttp } from './plugin/plugin-http';
+import { getPluginVolume } from './plugin/plugin-volume';
 import { getIpAddress, SCRYPTED_INSECURE_PORT, SCRYPTED_SECURE_PORT } from './server-settings';
-import semver from 'semver';
-import { ServiceControl } from './services/service-control';
 import { Alerts } from './services/alerts';
-import { Info } from './services/info';
-import * as io from 'engine.io';
-import { spawn as ptySpawn } from 'node-pty';
-import rimraf from 'rimraf';
-import { getPluginVolume } from './plugin/plugin-volume';
-import { isConnectionUpgrade, PluginHttp } from './plugin/plugin-http';
-import AdmZip from 'adm-zip';
-import path from 'path';
 import { CORSControl, CORSServer } from './services/cors';
-import { IOServer, IOServerSocket } from './io';
+import { Info } from './services/info';
+import { PluginComponent } from './services/plugin';
+import { ServiceControl } from './services/service-control';
+import { getState, ScryptedStateManager, setState } from './state';
 
 interface DeviceProxyPair {
     handler: PluginDeviceProxyHandler;
@@ -185,12 +183,10 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
 
             const url = new URL(callback_url as string);
             if (url.search) {
-                const search = qs.parse(url.search);
-                const state = search.state as string;
+                const state = url.searchParams.get('state');
                 if (state) {
                     const { s, d, r } = JSON.parse(state);
-                    search.state = s;
-                    url.search = '?' + qs.stringify(search);
+                    url.searchParams.set('state', s);
                     const oauthClient: ScryptedDevice & OauthClient = this.getDevice(d);
                     await oauthClient.onOauthCallback(url.toString()).catch();
                     res.redirect(r);
@@ -198,12 +194,12 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
                 }
             }
             if (url.hash) {
-                const hash = qs.parse(url.hash);
-                const state = hash.state as string;
+                const hash = new URLSearchParams(url.hash.substring(1));
+                const state = hash.get('state');
                 if (state) {
                     const { s, d, r } = JSON.parse(state);
-                    hash.state = s;
-                    url.hash = '#' + qs.stringify(hash);
+                    hash.set('state', s);
+                    url.hash = '#' + hash.toString();
                     const oauthClient: ScryptedDevice & OauthClient = this.getDevice(d);
                     await oauthClient.onOauthCallback(url.toString());
                     res.redirect(r);
@@ -283,8 +279,11 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
             this.shellio.handleRequest(req, res);
     }
 
-    async getEndpointPluginData(endpoint: string, isUpgrade: boolean, isEngineIOEndpoint: boolean): Promise<HttpPluginData> {
+    async getEndpointPluginData(req: Request, endpoint: string, isUpgrade: boolean, isEngineIOEndpoint: boolean): Promise<HttpPluginData> {
         const ret = await this.getPluginForEndpoint(endpoint);
+        if (req.url.indexOf('/engine.io/api') !== -1)
+            return ret;
+
         const { pluginDevice } = ret;
 
         // check if upgrade requests can be handled. must be websocket.

package/src/scrypted-plugin-main.ts

@@ -2,6 +2,8 @@ import { startPluginRemote } from "./plugin/plugin-remote-worker";
 import { RpcMessage } from "./rpc";
 import worker_threads from "worker_threads";
 import v8 from 'v8';
+import net from 'net';
+import { SidebandSocketSerializer } from "./plugin/socket-serializer";
 
 if (process.argv[2] === 'child-thread') {
     const peer = startPluginRemote(process.argv[3], (message, reject) => {
@@ -16,7 +18,7 @@ if (process.argv[2] === 'child-thread') {
     worker_threads.parentPort.on('message', message => peer.handleMessage(v8.deserialize(message)));
 }
 else {
-    const peer = startPluginRemote(process.argv[3], (message, reject) => process.send(message, undefined, {
+    const peer = startPluginRemote(process.argv[3], (message, reject, serializationContext) => process.send(message, serializationContext?.sendHandle, {
         swallowErrors: !reject,
     }, e => {
         if (e)
@@ -24,6 +26,7 @@ else {
     }));
 
     peer.transportSafeArgumentTypes.add(Buffer.name);
+    peer.addSerializer(net.Socket, net.Socket.name, new SidebandSocketSerializer());
     process.on('message', message => peer.handleMessage(message as RpcMessage));
     process.on('disconnect', () => {
         console.error('peer host disconnected, exiting.');

package/src/scrypted-server-main.ts

@@ -12,7 +12,6 @@ import { getHostAddresses, SCRYPTED_DEBUG_PORT, SCRYPTED_INSECURE_PORT, SCRYPTED
 import crypto from 'crypto';
 import cookieParser from 'cookie-parser';
 import axios from 'axios';
-import qs from 'query-string';
 import { RPCResultError } from './rpc';
 import fs from 'fs';
 import mkdirp from 'mkdirp';
@@ -155,7 +154,30 @@ async function start() {
     // use a hash of the private key as the cookie secret.
     app.use(cookieParser(crypto.createHash('sha256').update(certSetting.value.serviceKey).digest().toString('hex')));
 
-    app.all('*', async (req, res, next) => {
+    // trap to add access control headers.
+    app.use((req, res, next) => {
+        if (!req.headers.upgrade)
+            scrypted.addAccessControlHeaders(req, res);
+        next();
+    })
+
+    app.options('*', (req, res) => {
+        // add more?
+        res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+        res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, Content-Length, X-Requested-With');
+        res.send(200);
+    });
+
+    const authSalt = crypto.randomBytes(16);
+    const createAuthorizationToken = (login_user_token: string) => {
+        const salted = login_user_token + authSalt;
+        const hash = crypto.createHash('sha256');
+        hash.update(salted);
+        const sha = hash.digest().toString('hex');
+        return `Bearer ${sha}#${login_user_token}`;
+    }
+
+    app.use(async (req, res, next) => {
         // this is a trap for all auth.
         // only basic auth will fail with 401. it is up to the endpoints to manage
         // lack of login from cookie auth.
@@ -165,10 +187,8 @@ async function start() {
             const userTokenParts = login_user_token.split('#');
             const username = userTokenParts[0];
             const timestamp = parseInt(userTokenParts[1]);
-            if (timestamp + 86400000 < Date.now()) {
-                console.warn('login expired');
+            if (timestamp + 86400000 < Date.now())
                 return next();
-            }
 
             // this database lookup on every web request is not necessary, the cookie
             // itself is the auth, and is signed. furthermore, this is currently
@@ -182,7 +202,27 @@ async function start() {
             // }
 
             res.locals.username = username;
-            (req as any).username = username;
+        }
+        else if (req.headers.authorization?.startsWith('Bearer ')) {
+            const splits = req.headers.authorization.substring('Bearer '.length).split('#');
+            const login_user_token = splits[1] + '#' + splits[2];
+            if (login_user_token) {
+                const check = splits[0];
+
+                const salted = login_user_token + authSalt;
+                const hash = crypto.createHash('sha256');
+                hash.update(salted);
+                const sha = hash.digest().toString('hex');
+
+                if (check === sha) {
+                    const splits2 = login_user_token.split('#');
+                    const username = splits2[0];
+                    const timestamp = parseInt(splits2[1]);
+                    if (timestamp + 86400000 < Date.now())
+                        return next();
+                    res.locals.username = username;
+                }
+            }
         }
         next();
     });
@@ -277,8 +317,8 @@ async function start() {
 
     app.get('/web/component/script/search', async (req, res) => {
         try {
-            const query = qs.stringify({
-                text: req.query.text,
+            const query = new URLSearchParams({
+                text: req.query.text.toString(),
             })
             const response = await axios(`https://registry.npmjs.org/-/v1/search?${query}`);
             res.send(response.data);
@@ -355,7 +395,7 @@ async function start() {
     });
 
     const getLoginUserToken = (reqSecure: boolean) => {
-        return reqSecure ? 'login_user_token' : 'login_user_token_inseucre';
+        return reqSecure ? 'login_user_token' : 'login_user_token_insecure';
     };
 
     const getSignedLoginUserToken = (req: Request<any>): string => {
@@ -364,21 +404,23 @@ async function start() {
 
     app.get('/logout', (req, res) => {
         res.clearCookie(getLoginUserToken(req.secure));
-        res.send({});
+        if (req.headers['accept']?.startsWith('application/json')) {
+            res.send({});
+        }
+        else {
+            res.redirect('/endpoint/@scrypted/core/public/');
+        }
     });
 
     let hasLogin = await db.getCount(ScryptedUser) > 0;
 
     app.options('/login', (req, res) => {
-        scrypted.addAccessControlHeaders(req, res);
         res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
         res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, Content-Length, X-Requested-With');
         res.send(200);
     });
 
     app.post('/login', async (req, res) => {
-        scrypted.addAccessControlHeaders(req, res);
-
         const { username, password, change_password } = req.body;
         const timestamp = Date.now();
         const maxAge = 86400000;
@@ -422,6 +464,7 @@ async function start() {
             }
 
             res.send({
+                authorization: createAuthorizationToken(login_user_token),
                 username,
                 expiration: maxAge,
                 addresses,
@@ -456,6 +499,7 @@ async function start() {
         });
 
         res.send({
+            authorization: createAuthorizationToken(login_user_token),
             username,
             token: user.token,
             expiration: maxAge,
@@ -463,6 +507,7 @@ async function start() {
         });
     });
 
+
     app.get('/login', async (req, res) => {
         scrypted.addAccessControlHeaders(req, res);
 
@@ -510,6 +555,7 @@ async function start() {
         }
 
         res.send({
+            authorization: createAuthorizationToken(login_user_token),
             expiration: 86400000 - (Date.now() - timestamp),
             username,
             addresses,

package/src/state.ts

@@ -116,7 +116,7 @@ export class ScryptedStateManager extends EventRegistry {
         let cb = (eventDetails: EventDetails, eventData: any) => {
             if (denoise && lastData === eventData)
                 return;
-            callback(eventDetails, eventData);
+            callback?.(eventDetails, eventData);
         };
 
         const wrappedRegister = super.listenDevice(id, options, cb);
@@ -124,6 +124,7 @@ export class ScryptedStateManager extends EventRegistry {
         return new EventListenerRegisterImpl(() => {
             wrappedRegister.removeListener();
             cb = undefined;
+            callback = undefined;
             polling = false;
         });
     }