@scrypted/server 0.6.26 → 0.7.2

package/src/rpc.ts

@@ -48,8 +48,18 @@ interface RpcApply extends RpcMessage {
 
 interface RpcResult extends RpcMessage {
     id: string;
+    // TODO 3/2/2023
+    // deprecate these properties from rpc protocol. treat error results like any other result
+    // and auto serialize them.
+    /**
+     * @deprecated
+     */
     stack?: string;
+    /**
+     * @deprecated
+     */
     message?: string;
+    throw?: boolean;
     result?: any;
 }
 
@@ -81,6 +91,12 @@ export interface PrimitiveProxyHandler<T extends object> extends ProxyHandler<T>
 }
 
 class RpcProxy implements PrimitiveProxyHandler<any> {
+    static iteratorMethods = new Set([
+        'next',
+        'throw',
+        'return',
+    ]);
+
     constructor(public peer: RpcPeer,
         public entry: LocalProxiedEntry,
         public constructorName: string,
@@ -94,6 +110,18 @@ class RpcProxy implements PrimitiveProxyHandler<any> {
     }
 
     get(target: any, p: PropertyKey, receiver: any): any {
+        if (p === Symbol.asyncIterator) {
+            if (!this.proxyProps?.[Symbol.asyncIterator.toString()])
+                return;
+            return () => {
+                return new Proxy(() => { }, this);
+            };
+        }
+        if (RpcProxy.iteratorMethods.has(p?.toString())) {
+            const asyncIteratorMethod = this.proxyProps?.[Symbol.asyncIterator.toString()]?.[p];
+            if (asyncIteratorMethod)
+                return new Proxy(() => asyncIteratorMethod, this);
+        }
         if (p === RpcPeer.PROPERTY_PROXY_ID)
             return this.entry.id;
         if (p === '__proxy_constructor')
@@ -131,13 +159,18 @@ class RpcProxy implements PrimitiveProxyHandler<any> {
     }
 
     apply(target: any, thisArg: any, argArray?: any): any {
-        if (Object.isFrozen(this.peer.pendingResults))
-            return Promise.reject(new RPCResultError(this.peer, 'RpcPeer has been killed'));
+        const method = target() || null;
+        const oneway = this.proxyOneWayMethods?.includes?.(method);
+
+        if (Object.isFrozen(this.peer.pendingResults)) {
+            if (oneway)
+                return Promise.resolve();
+            return Promise.reject(new RPCResultError(this.peer, 'RpcPeer has been killed (apply) ' + target()));
+        }
 
         // rpc objects can be functions. if the function is a oneway method,
         // it will have a null in the oneway method list. this is because
         // undefined is not JSON serializable.
-        const method = target() || null;
         const args: any[] = [];
         const serializationContext: any = {};
         for (const arg of (argArray || [])) {
@@ -152,7 +185,7 @@ class RpcProxy implements PrimitiveProxyHandler<any> {
             method,
         };
 
-        if (this.proxyOneWayMethods?.includes?.(method)) {
+        if (oneway) {
             rpcApply.oneway = true;
             // a oneway callable object doesn't need to be in the JSON payload.
             if (method === null)
@@ -161,13 +194,46 @@ class RpcProxy implements PrimitiveProxyHandler<any> {
             return Promise.resolve();
         }
 
-        return this.peer.createPendingResult((id, reject) => {
+        const pendingResult = this.peer.createPendingResult((id, reject) => {
             rpcApply.id = id;
             this.peer.send(rpcApply, reject, serializationContext);
-        })
+        });
+
+        const asyncIterator = this.proxyProps?.[Symbol.asyncIterator.toString()];
+        if (!asyncIterator || (method !== asyncIterator.next && method !== asyncIterator.return))
+            return pendingResult;
+
+        return pendingResult
+            .then(value => {
+                if (method === asyncIterator.return) {
+                    return {
+                        done: true,
+                        value: undefined,
+                    }
+                }
+                return ({
+                    value,
+                    done: false,
+                });
+            })
+            .catch(e => {
+                if (e.name === 'StopAsyncIteration') {
+                    return {
+                        done: true,
+                        value: undefined,
+                    }
+                }
+                throw e;
+            })
     }
 }
 
+interface SerialiedRpcResultError {
+    name: string;
+    stack: string;
+    message: string;
+}
+
 // todo: error constructor adds a "cause" variable in Chrome 93, Node v??
 export class RPCResultError extends Error {
     constructor(peer: RpcPeer, message: string, public cause?: Error, options?: { name: string, stack: string | undefined }) {
@@ -177,7 +243,7 @@ export class RPCResultError extends Error {
             this.name = options?.name;
         }
         if (options?.stack) {
-            this.stack = `${peer.peerName}:${peer.selfName}\n${cause?.stack || options.stack}`;
+            this.stack = `${cause?.stack || options.stack}\n${peer.peerName}:${peer.selfName}`;
         }
     }
 }
@@ -188,10 +254,10 @@ function compileFunction(code: string, params?: ReadonlyArray<string>, options?:
     return eval(f);
 }
 
-declare class WeakRef {
-    target: any;
+declare class WeakRef<T> {
+    target: T;
     constructor(target: any);
-    deref(): any;
+    deref(): T;
 }
 
 try {
@@ -225,19 +291,26 @@ interface LocalProxiedEntry {
     finalizerId: string | undefined;
 }
 
+interface ErrorType {
+    name: string;
+    message: string;
+    stack?: string;
+}
+
 export class RpcPeer {
     idCounter = 1;
     params: { [name: string]: any } = {};
     pendingResults: { [id: string]: Deferred } = {};
     proxyCounter = 1;
     localProxied = new Map<any, LocalProxiedEntry>();
-    localProxyMap: { [id: string]: any } = {};
+    localProxyMap = new Map<string, any>();
     // @ts-ignore
     remoteWeakProxies: { [id: string]: WeakRef<any> } = {};
     // @ts-ignore
     finalizers = new FinalizationRegistry(entry => this.finalize(entry as LocalProxiedEntry));
     nameDeserializerMap = new Map<string, RpcSerializer>();
-    onSerialization = new Map<string, (value: any) => void>();
+    onProxyTypeSerialization = new Map<string, (value: any) => void>();
+    onProxySerialization: (value: any, proxyId: string) => any;
     constructorSerializerMap = new Map<any, string>();
     transportSafeArgumentTypes = RpcPeer.getDefaultTransportSafeArgumentTypes();
     killed: Promise<string>;
@@ -281,6 +354,37 @@ export class RpcPeer {
         }
     }
 
+    // static setProxyProperties(value: any, properties: any) {
+    //     value[RpcPeer.PROPERTY_PROXY_PROPERTIES] = properties;
+    // }
+
+    // static getProxyProperties(value: any) {
+    //     return value?.[RpcPeer.PROPERTY_PROXY_PROPERTIES];
+    // }
+
+    static getIteratorNext(target: any): string {
+        if (!target[Symbol.asyncIterator])
+            return;
+        const proxyProps = target[this.PROPERTY_PROXY_PROPERTIES]?.[Symbol.asyncIterator.toString()];
+        return proxyProps?.next || 'next';
+    }
+
+    static prepareProxyProperties(value: any) {
+        let props = value?.[RpcPeer.PROPERTY_PROXY_PROPERTIES];
+        if (!value[Symbol.asyncIterator])
+            return props;
+        props ||= {};
+        if (!props[Symbol.asyncIterator.toString()]) {
+            props[Symbol.asyncIterator.toString()] = {
+                next: 'next',
+                throw: 'throw',
+                return: 'return',
+            };
+        }
+        return props;
+    }
+
+    static readonly RPC_RESULT_ERROR_NAME = 'RPCResultError';
     static readonly PROPERTY_PROXY_ID = '__proxy_id';
     static readonly PROPERTY_PROXY_ONEWAY_METHODS = '__proxy_oneway_methods';
     static readonly PROPERTY_JSON_DISABLE_SERIALIZATION = '__json_disable_serialization';
@@ -304,9 +408,17 @@ export class RpcPeer {
         }).catch(e => e.message || 'Unknown Error');
     }
 
+    static isTransportSafe(value: any) {
+        return !value || (!value[RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION] && this.getDefaultTransportSafeArgumentTypes().has(value.constructor?.name));
+    }
+
+    isTransportSafe(value: any) {
+        return !value || (!value[RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION] && this.transportSafeArgumentTypes.has(value.constructor?.name));
+    }
+
     createPendingResult(cb: (id: string, reject: (e: Error) => void) => void): Promise<any> {
         if (Object.isFrozen(this.pendingResults))
-            return Promise.reject(new RPCResultError(this, 'RpcPeer has been killed'));
+            return Promise.reject(new RPCResultError(this, 'RpcPeer has been killed (createPendingResult)'));
 
         const promise = new Promise((resolve, reject) => {
             const id = (this.idCounter++).toString();
@@ -332,7 +444,7 @@ export class RpcPeer {
         this.pendingResults = Object.freeze({});
         this.params = Object.freeze({});
         this.remoteWeakProxies = Object.freeze({});
-        this.localProxyMap = Object.freeze({});
+        this.localProxyMap.clear()
         this.localProxied.clear();
     }
 
@@ -383,10 +495,16 @@ export class RpcPeer {
         return value;
     }
 
-    createErrorResult(result: RpcResult, e: any) {
-        result.stack = e.stack || 'no stack';
-        result.result = (e as Error).name || 'no name';
+    /**
+     * @deprecated
+     * @param result
+     * @param e
+     */
+    createErrorResult(result: RpcResult, e: ErrorType) {
+        result.result = this.serializeError(e);
+        result.throw = true;
         result.message = (e as Error).message || 'no message';
+        result.stack = e.stack || 'no stack';
     }
 
     deserialize(value: any, deserializationContext: any): any {
@@ -403,6 +521,9 @@ export class RpcPeer {
         }
 
         const { __remote_proxy_id, __remote_proxy_finalizer_id, __local_proxy_id, __remote_constructor_name, __serialized_value, __remote_proxy_props, __remote_proxy_oneway_methods } = value;
+        if (__remote_constructor_name === RpcPeer.RPC_RESULT_ERROR_NAME)
+            return this.deserializeError(__serialized_value);
+
         if (__remote_proxy_id) {
             let proxy = this.remoteWeakProxies[__remote_proxy_id]?.deref();
             if (!proxy)
@@ -418,7 +539,7 @@ export class RpcPeer {
         }
 
         if (__local_proxy_id) {
-            const ret = this.localProxyMap[__local_proxy_id];
+            const ret = this.localProxyMap.get(__local_proxy_id);
             if (!ret)
                 throw new RPCResultError(this, `invalid local proxy id ${__local_proxy_id}`);
             return ret;
@@ -432,6 +553,28 @@ export class RpcPeer {
         return value;
     }
 
+    deserializeError(e: SerialiedRpcResultError): RPCResultError {
+        const { name, stack, message } = e;
+        return new RPCResultError(this, message, undefined, { name, stack });
+    }
+
+    serializeError(e: ErrorType): RpcRemoteProxyValue {
+        const __serialized_value: SerialiedRpcResultError = {
+            stack: e.stack || '[no stack]',
+            name: e.name || '[no name]',
+            message: e.message || '[no message]',
+        }
+        return {
+            // probably not safe to use constructor.name
+            __remote_constructor_name: RpcPeer.RPC_RESULT_ERROR_NAME,
+            __remote_proxy_id: undefined,
+            __remote_proxy_finalizer_id: undefined,
+            __remote_proxy_oneway_methods: undefined,
+            __remote_proxy_props: undefined,
+            __serialized_value,
+        };
+    }
+
     serialize(value: any, serializationContext: any): any {
         if (value?.[RpcPeer.PROPERTY_JSON_COPY_SERIALIZE_CHILDREN] === true) {
             const ret: any = {};
@@ -441,12 +584,32 @@ export class RpcPeer {
             return ret;
         }
 
-        if (!value || (!value[RpcPeer.PROPERTY_JSON_DISABLE_SERIALIZATION] && this.transportSafeArgumentTypes.has(value.constructor?.name))) {
+        if (this.isTransportSafe(value)) {
             return value;
         }
 
         let __remote_constructor_name = value.__proxy_constructor || value.constructor?.name?.toString();
-        this.onSerialization.get(__remote_constructor_name)?.(value);
+
+        if (value instanceof Error)
+            return this.serializeError(value);
+
+        const serializerMapName = this.constructorSerializerMap.get(value.constructor);
+        if (serializerMapName) {
+            __remote_constructor_name = serializerMapName;
+            const serializer = this.nameDeserializerMap.get(serializerMapName);
+            if (!serializer)
+                throw new Error('serializer not found for ' + serializerMapName);
+            const serialized = serializer.serialize(value, serializationContext);
+            const ret: RpcRemoteProxyValue = {
+                __remote_proxy_id: undefined,
+                __remote_proxy_finalizer_id: undefined,
+                __remote_constructor_name,
+                __remote_proxy_props: RpcPeer.prepareProxyProperties(value),
+                __remote_proxy_oneway_methods: value?.[RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS],
+                __serialized_value: serialized,
+            }
+            return ret;
+        }
 
         let proxiedEntry = this.localProxied.get(value);
         if (proxiedEntry) {
@@ -456,7 +619,7 @@ export class RpcPeer {
                 __remote_proxy_id: proxiedEntry.id,
                 __remote_proxy_finalizer_id,
                 __remote_constructor_name,
-                __remote_proxy_props: value?.[RpcPeer.PROPERTY_PROXY_PROPERTIES],
+                __remote_proxy_props: RpcPeer.prepareProxyProperties(value),
                 __remote_proxy_oneway_methods: value?.[RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS],
             }
             return ret;
@@ -470,23 +633,7 @@ export class RpcPeer {
             return ret;
         }
 
-        const serializerMapName = this.constructorSerializerMap.get(value.constructor);
-        if (serializerMapName) {
-            __remote_constructor_name = serializerMapName;
-            const serializer = this.nameDeserializerMap.get(serializerMapName);
-            if (!serializer)
-                throw new Error('serializer not found for ' + serializerMapName);
-            const serialized = serializer.serialize(value, serializationContext);
-            const ret: RpcRemoteProxyValue = {
-                __remote_proxy_id: undefined,
-                __remote_proxy_finalizer_id: undefined,
-                __remote_constructor_name,
-                __remote_proxy_props: value?.[RpcPeer.PROPERTY_PROXY_PROPERTIES],
-                __remote_proxy_oneway_methods: value?.[RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS],
-                __serialized_value: serialized,
-            }
-            return ret;
-        }
+        this.onProxyTypeSerialization.get(__remote_constructor_name)?.(value);
 
         const __remote_proxy_id = (this.proxyCounter++).toString();
         proxiedEntry = {
@@ -494,13 +641,15 @@ export class RpcPeer {
             finalizerId: __remote_proxy_id,
         };
         this.localProxied.set(value, proxiedEntry);
-        this.localProxyMap[__remote_proxy_id] = value;
+        this.localProxyMap.set(__remote_proxy_id, value);
+
+        const __remote_proxy_props = this.onProxySerialization ? this.onProxySerialization(value, __remote_proxy_id) : RpcPeer.prepareProxyProperties(value);
 
         const ret: RpcRemoteProxyValue = {
             __remote_proxy_id,
             __remote_proxy_finalizer_id: __remote_proxy_id,
             __remote_constructor_name,
-            __remote_proxy_props: value?.[RpcPeer.PROPERTY_PROXY_PROPERTIES],
+            __remote_proxy_props,
             __remote_proxy_oneway_methods: value?.[RpcPeer.PROPERTY_PROXY_ONEWAY_METHODS],
         }
 
@@ -543,11 +692,19 @@ export class RpcPeer {
                 case 'param': {
                     const rpcParam = message as RpcParam;
                     const serializationContext: any = {};
-                    const result: RpcResult = {
-                        type: 'result',
-                        id: rpcParam.id,
-                        result: this.serialize(this.params[rpcParam.param], serializationContext)
-                    };
+                    let result: RpcResult;
+                    try {
+                        result = {
+                            type: 'result',
+                            id: rpcParam.id,
+                            result: this.serialize(this.params[rpcParam.param], serializationContext)
+                        };
+                    }
+                    catch (e) {
+                        // console.error('failure', rpcApply.method, e);
+                        this.createErrorResult(result, e);
+                    }
+
                     this.send(result, undefined, serializationContext);
                     break;
                 }
@@ -560,7 +717,7 @@ export class RpcPeer {
                     const serializationContext: any = {};
 
                     try {
-                        const target = this.localProxyMap[rpcApply.proxyId];
+                        const target = this.localProxyMap.get(rpcApply.proxyId);
                         if (!target)
                             throw new Error(`proxy id ${rpcApply.proxyId} not found`);
 
@@ -575,6 +732,19 @@ export class RpcPeer {
                             if (!method)
                                 throw new Error(`target ${target?.constructor?.name} does not have method ${rpcApply.method}`);
                             value = await target[rpcApply.method](...args);
+
+                            if (RpcPeer.getIteratorNext(target) === rpcApply.method) {
+                                if (value.done) {
+                                    const errorType: ErrorType = {
+                                        name: 'StopAsyncIteration',
+                                        message: undefined,
+                                    };
+                                    throw errorType;
+                                }
+                                else {
+                                    value = value.value;
+                                }
+                            }
                         }
                         else {
                             value = await target(...args);
@@ -592,12 +762,13 @@ export class RpcPeer {
                     break;
                 }
                 case 'result': {
+                    // console.log(message)
                     const rpcResult = message as RpcResult;
                     const deferred = this.pendingResults[rpcResult.id];
                     delete this.pendingResults[rpcResult.id];
                     if (!deferred)
                         throw new Error(`unknown result ${rpcResult.id}`);
-                    if (rpcResult.message || rpcResult.stack) {
+                    if ((rpcResult.message || rpcResult.stack) && !rpcResult.throw) {
                         const e = new RPCResultError(this, rpcResult.message || 'no message', undefined, {
                             name: rpcResult.result,
                             stack: rpcResult.stack,
@@ -605,19 +776,23 @@ export class RpcPeer {
                         deferred.reject(e);
                         return;
                     }
-                    deferred.resolve(this.deserialize(rpcResult.result, deserializationContext));
+                    const deserialized = this.deserialize(rpcResult.result, deserializationContext);
+                    if (rpcResult.throw)
+                        deferred.reject(deserialized);
+                    else
+                        deferred.resolve(deserialized);
                     break;
                 }
                 case 'finalize': {
                     const rpcFinalize = message as RpcFinalize;
-                    const local = this.localProxyMap[rpcFinalize.__local_proxy_id];
+                    const local = this.localProxyMap.get(rpcFinalize.__local_proxy_id);
                     if (local) {
                         const localProxiedEntry = this.localProxied.get(local);
                         // if a finalizer id is specified, it must match.
                         if (rpcFinalize.__local_proxy_finalizer_id && rpcFinalize.__local_proxy_finalizer_id !== localProxiedEntry?.finalizerId) {
                             break;
                         }
-                        delete this.localProxyMap[rpcFinalize.__local_proxy_id];
+                        this.localProxyMap.delete(rpcFinalize.__local_proxy_id);
                         this.localProxied.delete(local);
                     }
                     break;

package/src/runtime.ts

@@ -39,6 +39,7 @@ import { PluginComponent } from './services/plugin';
 import { ServiceControl } from './services/service-control';
 import { UsersService } from './services/users';
 import { getState, ScryptedStateManager, setState } from './state';
+import crypto from 'crypto';
 
 interface DeviceProxyPair {
     handler: PluginDeviceProxyHandler;
@@ -54,6 +55,8 @@ interface HttpPluginData {
 }
 
 export class ScryptedRuntime extends PluginHttp<HttpPluginData> {
+    clusterId = crypto.randomBytes(3).toString('hex');
+    clusterSecret = crypto.randomBytes(16).toString('hex');
     datastore: Level;
     plugins: { [id: string]: PluginHost } = {};
     pluginDevices: { [id: string]: PluginDevice } = {};

package/src/threading.ts

@@ -1,3 +1,5 @@
+// WARNING: threading.ts does not work because RpcPeer transpilation does not include readonly static properties in the class definition
+
 import worker_threads from 'worker_threads';
 import { getEvalSource, RpcPeer } from './rpc';
 import v8 from 'v8';

package/test/rpc-iterator-test.ts

@@ -0,0 +1,46 @@
+import { RpcPeer } from "../src/rpc";
+import { sleep } from '../src/sleep';
+
+const p1 = new RpcPeer('p1', 'p2', message => {
+    // console.log('message p1 p2', message);
+    p2.handleMessage(message);
+});
+
+const p2 = new RpcPeer('p2', 'p1', message => {
+    // console.log('message p2 p1', message);
+    p1.handleMessage(message);
+});
+
+async function* generator() {
+    try {
+        yield 2;
+        yield 3;
+    }
+    catch (e) {
+        console.log('caught', e)
+    }
+}
+
+p1.params['thing'] = generator();
+
+async function test() {
+    const foo = await p2.getParam('thing') as AsyncGenerator<number>;
+    if (true) {
+        for await (const c of foo) {
+            console.log(c);
+        }
+    }
+    else {
+        await sleep(0);
+        console.log(await foo.next());
+        await sleep(0);
+        // await foo.throw(new Error('barf'));
+        await foo.return(44);
+        await sleep(0);
+        console.log(await foo.next());
+        console.log(await foo.next());
+    }
+
+}
+
+test();

package/test/rpc-python-test.ts

@@ -0,0 +1,44 @@
+import child_process from 'child_process';
+import path from 'path';
+import type { Readable, Writable } from "stream";
+import { createDuplexRpcPeer } from '../src/rpc-serializer';
+import assert from 'assert';
+import net from 'net';
+
+async function main() {
+    const server = net.createServer(client => {
+        console.log('got client');
+        client.on('data', b => console.log('data', b.toString()));
+    });
+    server.listen(6666);
+
+    const cp = child_process.spawn('python3', [path.join(__dirname, '../python/rpc-iterator-test.py')], {
+        stdio: ['pipe', 'inherit', 'inherit', 'pipe', 'pipe'],
+    });
+
+    cp.on('exit', code => console.log('exited', code))
+
+    const rpcPeer = createDuplexRpcPeer('node', 'python', cp.stdio[3] as Readable, cp.stdio[4] as Writable);
+
+    async function* test() {
+        yield 1;
+        yield 2;
+        yield 3;
+    }
+
+    rpcPeer.params['test'] = test();
+
+    // const foo = await rpcPeer.getParam('foo');
+    // assert.equal(foo, 3);
+
+    // const bar = await rpcPeer.getParam('bar');
+    // console.log(bar);
+
+    // const ticker = await rpcPeer.getParam('ticker');
+    // for await (const v of ticker) {
+    //     console.log(v);
+    // }
+    // process.exit();
+}
+
+main();