@scrypted/server 0.113.0 → 0.115.0

package/package.json

@@ -1,17 +1,17 @@
 {
   "name": "@scrypted/server",
-  "version": "0.113.0",
+  "version": "0.115.0",
   "description": "",
   "dependencies": {
     "@mapbox/node-pre-gyp": "^1.0.11",
     "@scrypted/ffmpeg-static": "^6.1.0-build1",
-    "@scrypted/node-pty": "^1.0.14",
-    "@scrypted/types": "^0.3.30",
+    "@scrypted/node-pty": "^1.0.18",
+    "@scrypted/types": "^0.3.43",
     "adm-zip": "^0.5.14",
     "body-parser": "^1.20.2",
     "cookie-parser": "^1.4.6",
     "dotenv": "^16.4.5",
-    "engine.io": "^6.5.4",
+    "engine.io": "^6.6.0",
     "express": "^4.19.2",
     "follow-redirects": "^1.15.6",
     "http-auth": "^4.2.0",
@@ -21,17 +21,17 @@
     "nan": "^2.20.0",
     "node-dijkstra": "^2.5.0",
     "node-forge": "^1.3.1",
-    "node-gyp": "^10.1.0",
-    "py": "npm:@bjia56/portable-python@^0.1.52",
+    "node-gyp": "^10.2.0",
+    "py": "npm:@bjia56/portable-python@^0.1.54",
     "router": "^1.3.8",
-    "semver": "^7.6.2",
+    "semver": "^7.6.3",
     "sharp": "^0.33.4",
     "source-map-support": "^0.5.21",
-    "tar": "^7.4.0",
+    "tar": "^7.4.3",
     "tslib": "^2.6.3",
-    "typescript": "^5.5.2",
+    "typescript": "^5.5.4",
     "whatwg-mimetype": "^4.0.0",
-    "ws": "^8.17.1"
+    "ws": "^8.18.0"
   },
   "devDependencies": {
     "@types/adm-zip": "^0.5.5",
@@ -40,13 +40,13 @@
     "@types/follow-redirects": "^1.14.4",
     "@types/http-auth": "^4.1.4",
     "@types/ip": "^1.1.3",
-    "@types/lodash": "^4.17.5",
+    "@types/lodash": "^4.17.7",
     "@types/node-dijkstra": "^2.5.6",
     "@types/node-forge": "^1.3.11",
     "@types/semver": "^7.5.8",
     "@types/source-map-support": "^0.5.10",
     "@types/whatwg-mimetype": "^3.0.2",
-    "@types/ws": "^8.5.10"
+    "@types/ws": "^8.5.11"
   },
   "bin": {
     "scrypted-serve": "bin/scrypted-serve"

package/python/plugin_remote.py

@@ -29,6 +29,7 @@ from plugin_pip import install_with_pip, need_requirements, remove_pip_dirs
 from scrypted_python.scrypted_sdk import PluginFork, ScryptedStatic
 from scrypted_python.scrypted_sdk.types import (Device, DeviceManifest,
                                                 EventDetails,
+                                                ScryptedInterface,
                                                 ScryptedInterfaceMethods,
                                                 ScryptedInterfaceProperty,
                                                 Storage)
@@ -139,6 +140,19 @@ class SystemManager(scrypted_python.scrypted_sdk.types.SystemManager):
             state = self.systemState.get(check, None)
             if not state:
                 continue
+            checkInterfaces = state.get('interfaces', None)
+            if not checkInterfaces:
+                continue
+            interfaces = checkInterfaces.get('value', [])
+            if ScryptedInterface.ScryptedPlugin.value in interfaces:
+                checkPluginId = state.get('pluginId', None)
+                if not checkPluginId:
+                    continue
+                pluginId = checkPluginId.get('value', None)
+                if not pluginId:
+                    continue
+                if pluginId == name:
+                    return self.getDeviceById(check)
             checkName = state.get('name', None)
             if not checkName:
                 continue
@@ -390,7 +404,7 @@ class PluginRemote:
 
         def computeClusterObjectHash(o: ClusterObject) -> str:
             m = hashlib.sha256()
-            m.update(bytes(f"{o['id']}{o['port']}{o.get('sourcePort', '')}{o['proxyId']}{clusterSecret}", 'utf8'))
+            m.update(bytes(f"{o['id']}{o['port']}{o.get('sourcePort') or ''}{o['proxyId']}{clusterSecret}", 'utf8'))
             return base64.b64encode(m.digest()).decode('utf-8')
 
         def onProxySerialization(value: Any, proxyId: str, sourcePeerPort: int = None):

package/src/fetch/http-fetch.ts

@@ -89,11 +89,11 @@ export async function httpFetch<T extends HttpFetchOptions<Readable>>(options: T
         controller = new AbortController();
         timeout = setTimeout(() => controller.abort(), options.timeout);
 
-        options.signal?.addEventListener('abort', () => controller.abort('abort'));
+        options.signal?.addEventListener('abort', () => controller.abort(options.signal?.reason));
     }
 
     const signal = controller?.signal || options.signal;
-    signal?.addEventListener('abort', () => request.destroy(new Error('abort')));
+    signal?.addEventListener('abort', () => request.destroy(new Error(options.signal?.reason || 'abort')));
 
     const nodeHeaders: Record<string, string[]> = {};
     for (const [k, v] of headers) {
@@ -122,9 +122,12 @@ export async function httpFetch<T extends HttpFetchOptions<Readable>>(options: T
     try {
         const [response] = await once(request, 'response') as [IncomingMessage];
 
-        if (!options?.ignoreStatusCode) {
+
+        if (options?.checkStatusCode === undefined || options?.checkStatusCode) {
             try {
-                checkStatus(response.statusCode);
+                const checker = typeof options?.checkStatusCode === 'function' ? options.checkStatusCode : checkStatus;
+                if (!checker(response.statusCode))
+                    throw new Error(`http response statusCode ${response.statusCode}`);
             }
             catch (e) {
                 readMessageBuffer(response).catch(() => { });
@@ -150,31 +153,3 @@ export async function httpFetch<T extends HttpFetchOptions<Readable>>(options: T
     }
 }
 
-function ensureType<T>(v: T) {
-}
-
-async function test() {
-    const a = await httpFetch({
-        url: 'http://example.com',
-    });
-
-    ensureType<Buffer>(a.body);
-
-    const b = await httpFetch({
-        url: 'http://example.com',
-        responseType: 'json',
-    });
-    ensureType<any>(b.body);
-
-    const c = await httpFetch({
-        url: 'http://example.com',
-        responseType: 'readable',
-    });
-    ensureType<IncomingMessage>(c.body);
-
-    const d = await httpFetch({
-        url: 'http://example.com',
-        responseType: 'buffer',
-    });
-    ensureType<Buffer>(d.body);
-}

package/src/fetch/index.ts

@@ -7,7 +7,10 @@ export interface HttpFetchOptionsBase<B> {
     signal?: AbortSignal,
     timeout?: number;
     rejectUnauthorized?: boolean;
-    ignoreStatusCode?: boolean;
+    /**
+     * Checks the status code. Defaults to true.
+     */
+    checkStatusCode?: boolean | ((statusCode: number) => boolean);
     body?: B | string | ArrayBufferView | any;
     withCredentials?: boolean;
 }
@@ -40,6 +43,7 @@ export function fetchStatusCodeOk(statusCode: number) {
 export function checkStatus(statusCode: number) {
     if (!fetchStatusCodeOk(statusCode))
         throw new Error(`http response statusCode ${statusCode}`);
+    return true;
 }
 
 export function getFetchMethod(options: HttpFetchOptions<any>) {
@@ -177,7 +181,7 @@ export async function domFetch<T extends HttpFetchOptions<BodyInit>>(options: T)
         controller = new AbortController();
         timeout = setTimeout(() => controller.abort(), options.timeout);
 
-        options.signal?.addEventListener('abort', () => controller.abort('abort'));
+        options.signal?.addEventListener('abort', () => controller.abort(options.signal?.reason));
     }
 
     try {
@@ -190,9 +194,11 @@ export async function domFetch<T extends HttpFetchOptions<BodyInit>>(options: T)
             body,
         });
 
-        if (!options?.ignoreStatusCode) {
+        if (options?.checkStatusCode === undefined || options?.checkStatusCode) {
             try {
-                checkStatus(response.status);
+                const checker = typeof options?.checkStatusCode === 'function' ? options.checkStatusCode : checkStatus;
+                if (!checker(response.status))
+                    throw new Error(`http response statusCode ${response.status}`);
             }
             catch (e) {
                 response.arrayBuffer().catch(() => { });
@@ -210,32 +216,3 @@ export async function domFetch<T extends HttpFetchOptions<BodyInit>>(options: T)
         clearTimeout(timeout);
     }
 }
-
-function ensureType<T>(v: T) {
-}
-
-async function test() {
-    const a = await domFetch({
-        url: 'http://example.com',
-    });
-
-    ensureType<Buffer>(a.body);
-
-    const b = await domFetch({
-        url: 'http://example.com',
-        responseType: 'json',
-    });
-    ensureType<any>(b.body);
-
-    const c = await domFetch({
-        url: 'http://example.com',
-        responseType: 'readable',
-    });
-    ensureType<Response>(c.body);
-
-    const d = await domFetch({
-        url: 'http://example.com',
-        responseType: 'buffer',
-    });
-    ensureType<Buffer>(d.body);
-}

package/src/listen-zero.ts

@@ -7,13 +7,13 @@ export class ListenZeroSingleClientTimeoutError extends Error {
     }
 }
 
-export async function listenZero(server: net.Server, hostname?: string) {
-    server.listen(0, hostname);
+export async function listenZero(server: net.Server, hostname: string) {
+    server.listen(0, hostname || '127.0.0.1');
     await once(server, 'listening');
     return (server.address() as net.AddressInfo).port;
 }
 
-export async function listenZeroSingleClient(hostname?: string, options?: net.ServerOpts, listenTimeout = 30000) {
+export async function listenZeroSingleClient(hostname: string, options?: net.ServerOpts, listenTimeout = 30000) {
     const server = new net.Server(options);
     const port = await listenZero(server, hostname);
 

package/src/plugin/plugin-console.ts

@@ -295,8 +295,8 @@ export async function createConsoleServer(remoteStdout: Readable, remoteStderr:
         socket.once('error', cleanup);
         socket.once('end', cleanup);
     });
-    const readPort = await listenZero(readServer);
-    const writePort = await listenZero(writeServer);
+    const readPort = await listenZero(readServer, '127.0.0.1');
+    const writePort = await listenZero(writeServer, '127.0.0.1');
 
     return {
         clear(nativeId: ScryptedNativeId) {

package/src/plugin/plugin-npm-dependencies.ts

@@ -9,8 +9,12 @@ import { ensurePluginVolume } from "./plugin-volume";
 
 export function defaultNpmExec(args: string[], options: child_process.SpawnOptions) {
     let npm = 'npm';
-    if (os.platform() === 'win32')
+    if (os.platform() === 'win32') {
         npm += '.cmd';
+        // wrap each argument in a quote to handle spaces in paths
+        // https://github.com/nodejs/node/issues/38490#issuecomment-927330248
+        args = args.map(arg => '"' + arg + '"');
+    }
     const cp = child_process.spawn(npm, args, options);
     return cp;
 }

package/src/plugin/plugin-remote-worker.ts

@@ -9,6 +9,7 @@ import { computeClusterObjectHash } from '../cluster/cluster-hash';
 import { ClusterObject, ConnectRPCObject } from '../cluster/connect-rpc-object';
 import { listenZero } from '../listen-zero';
 import { RpcMessage, RpcPeer } from '../rpc';
+import { evalLocal } from '../rpc-peer-eval';
 import { createDuplexRpcPeer } from '../rpc-serializer';
 import { MediaManagerImpl } from './media';
 import { PluginAPI, PluginAPIProxy, PluginRemote, PluginRemoteLoadZipOptions } from './plugin-api';
@@ -269,11 +270,11 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
 
                 process.on('uncaughtException', e => {
                     getPluginConsole().error('uncaughtException', e);
-                    scrypted.log.e('uncaughtException ' + e?.toString());
+                    scrypted.log.e('uncaughtException ' + (e.stack || e?.toString()));
                 });
                 process.on('unhandledRejection', e => {
                     getPluginConsole().error('unhandledRejection', e);
-                    scrypted.log.e('unhandledRejection ' + e?.toString());
+                    scrypted.log.e('unhandledRejection ' + ((e as Error).stack || e?.toString()));
                 });
 
                 installSourceMapSupport({
@@ -376,7 +377,7 @@ export function startPluginRemote(mainFilename: string, pluginId: string, peerSe
 
             try {
                 const filename = zipOptions?.debug ? '/plugin/main.nodejs.js' : `/${pluginId}/main.nodejs.js`;
-                peer.evalLocal(script, filename, params);
+                evalLocal(peer, script, filename, params);
 
                 if (zipOptions?.fork) {
                     // pluginConsole?.log('plugin forked');

package/src/plugin/plugin-remote.ts

@@ -179,7 +179,6 @@ class DeviceStateProxyHandler implements ProxyHandler<any> {
 
     set?(target: any, p: PropertyKey, value: any, receiver: any) {
         checkProperty(p.toString(), value);
-        const now = Date.now();
         this.deviceManager.systemManager.state[this.id][p as string] = {
             value,
         };
@@ -446,7 +445,7 @@ export async function setupPluginRemote(peer: RpcPeer, api: PluginAPI, pluginId:
         return remote;
     }
     catch (e) {
-        throw new RPCResultError(peer, 'error while retrieving PluginRemote', e);
+        throw new RPCResultError(peer, 'error while retrieving PluginRemote', e as Error);
     }
 }
 

package/src/plugin/plugin-repl.ts

@@ -75,5 +75,5 @@ export async function createREPLServer(scrypted: ScryptedStatic, params: any, pl
         socket.on('error', cleanup);
         socket.on('end', cleanup);
     });
-    return listenZero(server);
+    return listenZero(server, '127.0.0.1');
 }

package/src/plugin/runtime/node-worker-common.ts

@@ -15,6 +15,7 @@ function prep(pluginVolume: string, hash: string) {
     const zipFile = path.join(zipDir, zipFilename);
     const unzippedPath = path.join(zipDir, 'unzipped')
     const zipDirTmp = zipDir + '.tmp';
+    const zipDirTmp2 = zipDir + '.tmp2';
 
     return {
         unzippedPath,
@@ -22,6 +23,7 @@ function prep(pluginVolume: string, hash: string) {
         zipDir,
         zipFile,
         zipDirTmp,
+        zipDirTmp2,
     };
 }
 
@@ -52,16 +54,39 @@ export function prepareZipSync(pluginVolume: string, h: string, getZip: () => Bu
 }
 
 export function extractZip(pluginVolume: string, h: string, zipBuffer: Buffer) {
-    const { zipDir, zipDirTmp, zipFilename, zipFile, unzippedPath } = prep(pluginVolume, h);
+    const { zipDir, zipDirTmp, zipDirTmp2, zipFilename, zipFile, unzippedPath } = prep(pluginVolume, h);
 
+    // stage the plugin zip in a tmp directory, then move it to the final location.
+    // so if the zip extraction is corrupt, it won't be used.
     fs.rmSync(zipDirTmp, {
         recursive: true,
         force: true,
     });
-    fs.rmSync(zipDir, {
-        recursive: true,
-        force: true,
-    });
+
+    let zipDirTmp2Exists = false;
+    try {
+        fs.rmSync(zipDirTmp2, {
+            recursive: true,
+            force: true,
+        });
+    }
+    catch (e) {
+        zipDirTmp2Exists = true;
+    }
+
+    try {
+        fs.rmSync(zipDir, {
+            recursive: true,
+            force: true,
+        });
+    }
+    catch (e) {
+        if (zipDirTmp2Exists)
+            throw e;
+        // file lock from dangling plugin process may have prevented the recursive rm from completing.
+        // try renaming it. it will get cleaned up eventually.
+        fs.renameSync(zipDir, zipDirTmp2);
+    }
     fs.mkdirSync(zipDirTmp, {
         recursive: true,
     });

package/src/rpc-peer-eval.ts

@@ -0,0 +1,27 @@
+import type { CompileFunctionOptions } from 'vm';
+import { RpcPeer } from "./rpc";
+
+type CompileFunction = (code: string, params?: ReadonlyArray<string>, options?: CompileFunctionOptions) => Function;
+
+function compileFunction(code: string, params?: ReadonlyArray<string>, options?: CompileFunctionOptions): any {
+    params = params || [];
+    const f = `(function(${params.join(',')}) {;${code};})`;
+    return eval(f);
+}
+
+export function evalLocal<T>(peer: RpcPeer, script: string, filename?: string, coercedParams?: { [name: string]: any }): T {
+    const params = Object.assign({}, peer.params, coercedParams);
+    let compile: CompileFunction;
+    try {
+        // prevent bundlers from trying to include non-existent vm module.
+        compile = module[`require`]('vm').compileFunction;
+    }
+    catch (e) {
+        compile = compileFunction;
+    }
+    const f = compile(script, Object.keys(params), {
+        filename,
+    });
+    const value = f(...Object.values(params));
+    return value;
+}

package/src/rpc-serializer.ts

@@ -10,7 +10,7 @@ export function createDuplexRpcPeer(selfName: string, peerName: string, readable
             serializer.sendMessage(message, reject, serializationContext);
         }
         catch (e) {
-            reject?.(e);
+            reject?.(e as Error);
             readable.destroy();
         }
     });
@@ -165,7 +165,7 @@ export function createRpcDuplexSerializer(writable: {
                     serializer.onMessageFinish(message);
                 }
                 catch (e) {
-                    serializer.kill('message parse failure ' + e.message);
+                    serializer.kill('message parse failure ' + (e as Error).message);
                 }
             }
             else {

package/src/rpc.ts

@@ -1,6 +1,3 @@
-import type { CompileFunctionOptions } from 'vm';
-type CompileFunction = (code: string, params?: ReadonlyArray<string>, options?: CompileFunctionOptions) => Function;
-
 export function startPeriodicGarbageCollection() {
     if (!global.gc) {
         console.warn('rpc peer garbage collection not available: global.gc is not exposed.');
@@ -253,12 +250,6 @@ export class RPCResultError extends Error {
     }
 }
 
-function compileFunction(code: string, params?: ReadonlyArray<string>, options?: CompileFunctionOptions): any {
-    params = params || [];
-    const f = `(function(${params.join(',')}) {;${code};})`;
-    return eval(f);
-}
-
 declare class WeakRef<T> {
     target: T;
     constructor(target: any);
@@ -499,23 +490,6 @@ export class RpcPeer {
         });
     }
 
-    evalLocal<T>(script: string, filename?: string, coercedParams?: { [name: string]: any }): T {
-        const params = Object.assign({}, this.params, coercedParams);
-        let compile: CompileFunction;
-        try {
-            // prevent bundlers from trying to include non-existent vm module.
-            compile = module[`require`]('vm').compileFunction;
-        }
-        catch (e) {
-            compile = compileFunction;
-        }
-        const f = compile(script, Object.keys(params), {
-            filename,
-        });
-        const value = f(...Object.values(params));
-        return value;
-    }
-
     /**
      * @deprecated
      * @param result
@@ -723,7 +697,7 @@ export class RpcPeer {
                     }
                     catch (e) {
                         // console.error('failure', rpcApply.method, e);
-                        this.createErrorResult(result, e);
+                        this.createErrorResult(result, e as Error);
                     }
 
                     this.send(result, undefined, serializationContext);
@@ -785,7 +759,7 @@ export class RpcPeer {
                     }
                     catch (e) {
                         // console.error('failure', rpcApply.method, e);
-                        this.createErrorResult(result, e);
+                        this.createErrorResult(result, e as Error);
                     }
 
                     if (!rpcApply.oneway)

package/src/runtime.ts

@@ -275,10 +275,11 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
             return;
         }
 
+        const reqany = req as any;
         if ((req as any).upgradeHead)
-            this.connectRPCObjectIO.handleUpgrade(req, res.socket, (req as any).upgradeHead)
+            this.connectRPCObjectIO.handleUpgrade(reqany, res.socket, reqany.upgradeHead)
         else
-            this.connectRPCObjectIO.handleRequest(req, res);
+            this.connectRPCObjectIO.handleRequest(reqany, res);
     }
 
     async getEndpointPluginData(req: Request, endpoint: string, isUpgrade: boolean, isEngineIOEndpoint: boolean): Promise<HttpPluginData> {
@@ -422,15 +423,18 @@ export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
             return;
         }
 
-        (req as any).scrypted = {
+        const reqany = req as any;
+
+        reqany.scrypted = {
             endpointRequest,
             pluginDevice,
             accessControls,
         };
+
         if ((req as any).upgradeHead)
-            pluginHost.io.handleUpgrade(req, res.socket, (req as any).upgradeHead)
+            pluginHost.io.handleUpgrade(reqany, res.socket, reqany.upgradeHead)
         else
-            pluginHost.io.handleRequest(req, res);
+            pluginHost.io.handleRequest(reqany, res);
     }
 
     handleRequestEndpoint(req: Request, res: Response, endpointRequest: HttpRequest, pluginData: HttpPluginData) {

package/src/scrypted-server-main.ts

@@ -161,6 +161,16 @@ async function start(mainFilename: string, options?: {
         callback(sha === user.passwordHash || password === user.token);
     });
 
+    // the default http-auth will returns a WWW-Authenticate header if login fails.
+    // this causes the Safari to prompt for login.
+    // https://github.com/gevorg/http-auth/blob/4158fa75f58de70fd44aa68876a8674725e0556e/src/auth/base.js#L81
+    // override the ask function to return a bare 401 instead.
+    // @ts-expect-error
+    basicAuth.ask = (res) => {
+        res.statusCode = 401;
+        res.end();
+    };
+
     const httpsServerOptions = process.env.SCRYPTED_HTTPS_OPTIONS_FILE
         ? JSON.parse(fs.readFileSync(process.env.SCRYPTED_HTTPS_OPTIONS_FILE).toString())
         : {};
@@ -219,6 +229,12 @@ async function start(mainFilename: string, options?: {
     }
 
     app.use(async (req, res, next) => {
+        // /web/component requires basic auth admin access.
+        if (req.url.startsWith('/web/component/')) {
+            next();
+            return;
+        }
+
         // the remote address may be ipv6 prefixed so use a fuzzy match.
         // eg ::ffff:192.168.2.124
         if (process.env.SCRYPTED_ADMIN_USERNAME