@streamr/node 100.0.0-rc.0

package/dist/src/plugins/operator/config.schema.json

@@ -0,0 +1,152 @@
+{
+    "$id": "config.schema.json",
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "type": "object",
+    "description": "Operator plugin configuration",
+    "additionalProperties": false,
+    "required": [
+        "operatorContractAddress"
+    ],
+    "properties": {
+        "operatorContractAddress": {
+            "type": "string",
+            "description": "Operator contract Ethereum address",
+            "format": "ethereum-address"
+        },
+        "heartbeatUpdateIntervalInMs": {
+            "type": "integer",
+            "description": "The interval (in milliseconds) at which heartbeats get published to coordination stream",
+            "minimum": 0,
+            "default": 10000
+        },
+        "heartbeatTimeoutInMs": {
+            "type": "integer",
+            "description": "The maximum time (in milliseconds) to wait for the 1st heartbeat of a target operator's nodes",
+            "minimum": 0,
+            "default": 120000
+        },
+        "fleetState": {
+            "type": "object",
+            "description": "Operator fleet state settings (both for me and targets)",
+            "additionalProperties": false,
+            "default": {},
+            "properties": {
+                "pruneAgeInMs": {
+                    "type": "integer",
+                    "description": "The maximum time (in milliseconds) of a heartbeat to count towards a node being online",
+                    "minimum": 0,
+                    "default": 180000
+                },
+                "pruneIntervalInMs": {
+                    "type": "integer",
+                    "description": "The interval (in milliseconds) at which to prune old heartbeats",
+                    "minimum": 0,
+                    "default": 30000
+                },
+                "latencyExtraInMs": {
+                    "type": "integer",
+                    "description": "Account for extra latency (milliseconds) due to networking",
+                    "minimum": 0,
+                    "default": 2000
+                },
+                "warmupPeriodInMs": {
+                    "type": "integer",
+                    "description": "The time (in milliseconds) to wait before starting to count heartbeats",
+                    "minimum": 0,
+                    "default": 10000
+                }
+            }
+        },
+        "checkOperatorValueBreachIntervalInMs": {
+            "type": "integer",
+            "description": "The interval (in milliseconds) at which to check for operator value breach",
+            "minimum": 0,
+            "default": 3600000
+        },
+        "announceNodeToContract": {
+            "type": "object",
+            "description": "Announce node to contract settings",
+            "additionalProperties": false,
+            "default": {},
+            "properties": {
+                "pollIntervalInMs": {
+                    "type": "integer",
+                    "description": "The interval (in milliseconds) at which to poll for need to write new value to contract",
+                    "minimum": 0,
+                    "default": 600000
+                },
+                "writeIntervalInMs": {
+                    "type": "integer",
+                    "description": "How often (in milliseconds) should a new value be written to contract",
+                    "minimum": 0,
+                    "default": 86400000
+                }
+            }
+        },
+        "maintainOperatorValue": {
+            "type": "object",
+            "description": "Maintain operator value settings",
+            "additionalProperties": false,
+            "default": {},
+            "properties": {
+                "intervalInMs": {
+                    "type": "integer",
+                    "description": "How often to run (in milliseconds)",
+                    "minimum": 0,
+                    "default": 3600000
+                },
+                "withdrawLimitSafetyFraction": {
+                    "type": "number",
+                    "description": "The fraction of the withdraw limit to use as a safety margin",
+                    "minimum": 0,
+                    "maximum": 1,
+                    "default": 0.5
+                },
+                "minSponsorshipEarningsInWithdraw": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "default": 1
+                },
+                "maxSponsorshipsInWithdraw": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "default": 20
+                }
+            }
+        },
+        "inspectRandomNode": {
+            "type": "object",
+            "description": "Inspect random node settings",
+            "additionalProperties": false,
+            "default": {},
+            "properties": {
+                "intervalInMs": {
+                    "type": "integer",
+                    "description": "How often to run (in milliseconds)",
+                    "minimum": 0,
+                    "default": 900000
+                }
+            }
+        },
+        "closeExpiredFlags": {
+            "type": "object",
+            "description": "Close expired flags settings",
+            "additionalProperties": false,
+            "default": {},
+            "properties": {
+                "intervalInMs": {
+                    "type": "integer",
+                    "description": "How often to run (in milliseconds)",
+                    "minimum": 0,
+                    "default": 600000
+                },
+                "maxAgeInMs": {
+                    "type": "integer",
+                    "description": "The maximum age (in milliseconds) of a flag before it is expired",
+                    "minimum": 0,
+                    "default": 86400000
+                }
+            }
+        }
+    }
+}

package/dist/src/plugins/operator/createIsLeaderFn.d.ts

@@ -0,0 +1,4 @@
+import { StreamrClient } from '@streamr/sdk';
+import { OperatorFleetState } from './OperatorFleetState';
+import { Logger } from '@streamr/utils';
+export declare function createIsLeaderFn(streamrClient: StreamrClient, operatorFleetState: OperatorFleetState, logger?: Logger): Promise<() => boolean>;

package/dist/src/plugins/operator/createIsLeaderFn.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createIsLeaderFn = void 0;
+async function createIsLeaderFn(streamrClient, operatorFleetState, logger) {
+    const myNodeId = await streamrClient.getNodeId();
+    return () => {
+        const leaderNodeId = operatorFleetState.getLeaderNodeId();
+        const isLeader = myNodeId === leaderNodeId;
+        logger?.debug('Check whether leader', { isLeader, leaderNodeId, myNodeId });
+        return isLeader;
+    };
+}
+exports.createIsLeaderFn = createIsLeaderFn;
+//# sourceMappingURL=createIsLeaderFn.js.map

package/dist/src/plugins/operator/createIsLeaderFn.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"createIsLeaderFn.js","sourceRoot":"","sources":["../../../../src/plugins/operator/createIsLeaderFn.ts"],"names":[],"mappings":";;;AAIO,KAAK,UAAU,gBAAgB,CAClC,aAA4B,EAC5B,kBAAsC,EACtC,MAAe;IAEf,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,SAAS,EAAE,CAAA;IAChD,OAAO,GAAG,EAAE;QACR,MAAM,YAAY,GAAG,kBAAkB,CAAC,eAAe,EAAE,CAAA;QACzD,MAAM,QAAQ,GAAG,QAAQ,KAAK,YAAY,CAAA;QAC1C,MAAM,EAAE,KAAK,CAAC,sBAAsB,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC,CAAA;QAC3E,OAAO,QAAQ,CAAA;IACnB,CAAC,CAAA;AACL,CAAC;AAZD,4CAYC"}

package/dist/src/plugins/operator/fetchRedundancyFactor.d.ts

@@ -0,0 +1,2 @@
+import { OperatorServiceConfig } from './OperatorPlugin';
+export declare function fetchRedundancyFactor({ operatorContractAddress, signer }: Pick<OperatorServiceConfig, 'operatorContractAddress' | 'signer'>): Promise<number | undefined>;

package/dist/src/plugins/operator/fetchRedundancyFactor.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchRedundancyFactor = void 0;
+const utils_1 = require("@streamr/utils");
+const ethers_1 = require("ethers");
+const network_contracts_1 = require("@streamr/network-contracts");
+const zod_1 = require("zod");
+const logger = new utils_1.Logger(module);
+const MetadataSchema = zod_1.z.object({
+    redundancyFactor: zod_1.z.number()
+        .int()
+        .gte(1)
+});
+async function fetchRedundancyFactor({ operatorContractAddress, signer }) {
+    const operator = new ethers_1.Contract(operatorContractAddress, network_contracts_1.operatorABI, signer);
+    const metadataAsString = await operator.metadata();
+    if (metadataAsString.length === 0) {
+        return 1;
+    }
+    let metadata;
+    try {
+        metadata = JSON.parse(metadataAsString);
+    }
+    catch {
+        logger.warn('Encountered malformed metadata', { operatorContractAddress, metadataAsString });
+        return undefined;
+    }
+    let validatedMetadata;
+    try {
+        validatedMetadata = MetadataSchema.parse(metadata);
+    }
+    catch (err) {
+        logger.warn('Encountered invalid metadata', {
+            operatorContractAddress,
+            metadataAsString,
+            reason: err?.reason
+        });
+        return undefined;
+    }
+    return validatedMetadata.redundancyFactor;
+}
+exports.fetchRedundancyFactor = fetchRedundancyFactor;
+//# sourceMappingURL=fetchRedundancyFactor.js.map

package/dist/src/plugins/operator/fetchRedundancyFactor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetchRedundancyFactor.js","sourceRoot":"","sources":["../../../../src/plugins/operator/fetchRedundancyFactor.ts"],"names":[],"mappings":";;;AAAA,0CAAuC;AACvC,mCAAiC;AACjC,kEAAkE;AAElE,6BAAuB;AAEvB,MAAM,MAAM,GAAG,IAAI,cAAM,CAAC,MAAM,CAAC,CAAA;AAEjC,MAAM,cAAc,GAAG,OAAC,CAAC,MAAM,CAAC;IAC5B,gBAAgB,EAAE,OAAC,CAAC,MAAM,EAAE;SACvB,GAAG,EAAE;SACL,GAAG,CAAC,CAAC,CAAC;CACd,CAAC,CAAA;AAEK,KAAK,UAAU,qBAAqB,CAAC,EACxC,uBAAuB,EACvB,MAAM,EAC0D;IAChE,MAAM,QAAQ,GAAG,IAAI,iBAAQ,CAAC,uBAAuB,EAAE,+BAAW,EAAE,MAAM,CAAwB,CAAA;IAClG,MAAM,gBAAgB,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,CAAA;IAElD,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,CAAA;IACZ,CAAC;IAED,IAAI,QAAiC,CAAA;IACrC,IAAI,CAAC;QACD,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAA;IAC3C,CAAC;IAAC,MAAM,CAAC;QACL,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,CAAC,CAAA;QAC5F,OAAO,SAAS,CAAA;IACpB,CAAC;IAED,IAAI,iBAAiD,CAAA;IACrD,IAAI,CAAC;QACD,iBAAiB,GAAG,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IACtD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,MAAM,CAAC,IAAI,CAAC,8BAA8B,EAAE;YACxC,uBAAuB;YACvB,gBAAgB;YAChB,MAAM,EAAE,GAAG,EAAE,MAAM;SACtB,CAAC,CAAA;QACF,OAAO,SAAS,CAAA;IACpB,CAAC;IACD,OAAO,iBAAiB,CAAC,gBAAgB,CAAA;AAC7C,CAAC;AA/BD,sDA+BC"}

package/dist/src/plugins/operator/formCoordinationStreamId.d.ts

@@ -0,0 +1,3 @@
+import { EthereumAddress } from '@streamr/utils';
+import { StreamID } from '@streamr/protocol';
+export declare function formCoordinationStreamId(operatorContractAddress: EthereumAddress): StreamID;

package/dist/src/plugins/operator/formCoordinationStreamId.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formCoordinationStreamId = void 0;
+const protocol_1 = require("@streamr/protocol");
+function formCoordinationStreamId(operatorContractAddress) {
+    return (0, protocol_1.toStreamID)('/operator/coordination', operatorContractAddress);
+}
+exports.formCoordinationStreamId = formCoordinationStreamId;
+//# sourceMappingURL=formCoordinationStreamId.js.map

package/dist/src/plugins/operator/formCoordinationStreamId.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"formCoordinationStreamId.js","sourceRoot":"","sources":["../../../../src/plugins/operator/formCoordinationStreamId.ts"],"names":[],"mappings":";;;AACA,gDAAwD;AAExD,SAAgB,wBAAwB,CAAC,uBAAwC;IAC7E,OAAO,IAAA,qBAAU,EAAC,wBAAwB,EAAE,uBAAuB,CAAC,CAAA;AACxE,CAAC;AAFD,4DAEC"}

package/dist/src/plugins/operator/heartbeatUtils.d.ts

@@ -0,0 +1,67 @@
+import { z } from 'zod';
+import { NetworkNodeType, NetworkPeerDescriptor } from '@streamr/sdk';
+export declare const HeartbeatMessageSchema: z.ZodObject<{
+    msgType: z.ZodEnum<["heartbeat"]>;
+    peerDescriptor: z.ZodObject<{
+        nodeId: z.ZodString;
+        type: z.ZodOptional<z.ZodNativeEnum<typeof NetworkNodeType>>;
+        websocket: z.ZodOptional<z.ZodObject<{
+            host: z.ZodString;
+            port: z.ZodNumber;
+            tls: z.ZodBoolean;
+        }, "strip", z.ZodTypeAny, {
+            port: number;
+            host: string;
+            tls: boolean;
+        }, {
+            port: number;
+            host: string;
+            tls: boolean;
+        }>>;
+        region: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        nodeId: string;
+        type?: NetworkNodeType | undefined;
+        websocket?: {
+            port: number;
+            host: string;
+            tls: boolean;
+        } | undefined;
+        region?: number | undefined;
+    }, {
+        nodeId: string;
+        type?: NetworkNodeType | undefined;
+        websocket?: {
+            port: number;
+            host: string;
+            tls: boolean;
+        } | undefined;
+        region?: number | undefined;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    msgType: "heartbeat";
+    peerDescriptor: {
+        nodeId: string;
+        type?: NetworkNodeType | undefined;
+        websocket?: {
+            port: number;
+            host: string;
+            tls: boolean;
+        } | undefined;
+        region?: number | undefined;
+    };
+}, {
+    msgType: "heartbeat";
+    peerDescriptor: {
+        nodeId: string;
+        type?: NetworkNodeType | undefined;
+        websocket?: {
+            port: number;
+            host: string;
+            tls: boolean;
+        } | undefined;
+        region?: number | undefined;
+    };
+}>;
+export type HeartbeatMessage = z.infer<typeof HeartbeatMessageSchema>;
+export declare function createHeartbeatMessage(peerDescriptor: NetworkPeerDescriptor): HeartbeatMessage;

package/dist/src/plugins/operator/heartbeatUtils.js

@@ -0,0 +1,26 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createHeartbeatMessage = exports.HeartbeatMessageSchema = void 0;
+const zod_1 = require("zod");
+const sdk_1 = require("@streamr/sdk");
+exports.HeartbeatMessageSchema = zod_1.z.object({
+    msgType: zod_1.z.enum(['heartbeat']),
+    peerDescriptor: zod_1.z.object({
+        nodeId: zod_1.z.string(),
+        type: zod_1.z.optional(zod_1.z.nativeEnum(sdk_1.NetworkNodeType)),
+        websocket: zod_1.z.optional(zod_1.z.object({
+            host: zod_1.z.string(),
+            port: zod_1.z.number(),
+            tls: zod_1.z.boolean()
+        })),
+        region: zod_1.z.optional(zod_1.z.number())
+    })
+});
+function createHeartbeatMessage(peerDescriptor) {
+    return {
+        msgType: 'heartbeat',
+        peerDescriptor
+    };
+}
+exports.createHeartbeatMessage = createHeartbeatMessage;
+//# sourceMappingURL=heartbeatUtils.js.map

package/dist/src/plugins/operator/heartbeatUtils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"heartbeatUtils.js","sourceRoot":"","sources":["../../../../src/plugins/operator/heartbeatUtils.ts"],"names":[],"mappings":";;;AAAA,6BAAuB;AACvB,sCAAqE;AAExD,QAAA,sBAAsB,GAAG,OAAC,CAAC,MAAM,CAAC;IAC3C,OAAO,EAAE,OAAC,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,CAAC;IAC9B,cAAc,EAAE,OAAC,CAAC,MAAM,CAAC;QACrB,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE;QAClB,IAAI,EAAE,OAAC,CAAC,QAAQ,CAAC,OAAC,CAAC,UAAU,CAAC,qBAAe,CAAC,CAAC;QAC/C,SAAS,EAAE,OAAC,CAAC,QAAQ,CAAC,OAAC,CAAC,MAAM,CAAC;YAC3B,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE;YAChB,IAAI,EAAE,OAAC,CAAC,MAAM,EAAE;YAChB,GAAG,EAAE,OAAC,CAAC,OAAO,EAAE;SACnB,CAAC,CAAC;QACH,MAAM,EAAE,OAAC,CAAC,QAAQ,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC;KACjC,CAAC;CACL,CAAC,CAAA;AAIF,SAAgB,sBAAsB,CAAC,cAAqC;IACxE,OAAO;QACH,OAAO,EAAE,WAAW;QACpB,cAAc;KACjB,CAAA;AACL,CAAC;AALD,wDAKC"}

package/dist/src/plugins/operator/inspectOverTime.d.ts

@@ -0,0 +1,22 @@
+/// <reference types="node" />
+import { EthereumAddress } from '@streamr/utils';
+import { StreamrClient } from '@streamr/sdk';
+import { CreateOperatorFleetStateFn } from './OperatorFleetState';
+import { FindNodesForTargetGivenFleetStateFn, InspectTargetFn, Target } from './inspectionUtils';
+interface InspectOverTimeOpts {
+    target: Target;
+    streamrClient: StreamrClient;
+    createOperatorFleetState: CreateOperatorFleetStateFn;
+    getRedundancyFactor: (operatorContractAddress: EthereumAddress) => Promise<number | undefined>;
+    sleepTimeInMsBeforeFirstInspection: number;
+    heartbeatTimeoutInMs: number;
+    inspectionIntervalInMs: number;
+    maxInspections: number;
+    waitUntilPassOrDone: boolean;
+    abortSignal: AbortSignal;
+    traceId: string;
+    findNodesForTargetGivenFleetStateFn?: FindNodesForTargetGivenFleetStateFn;
+    inspectTargetFn?: InspectTargetFn;
+}
+export declare function inspectOverTime(opts: InspectOverTimeOpts): () => Promise<boolean[]>;
+export {};

package/dist/src/plugins/operator/inspectOverTime.js

@@ -0,0 +1,146 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.inspectOverTime = void 0;
+const utils_1 = require("@streamr/utils");
+const inspectionUtils_1 = require("./inspectionUtils");
+const formCoordinationStreamId_1 = require("./formCoordinationStreamId");
+const range_1 = __importDefault(require("lodash/range"));
+function inspectOverTime(opts) {
+    const task = new InspectionOverTimeTask(opts);
+    task.start();
+    return async () => {
+        if (opts.waitUntilPassOrDone) {
+            await task.waitUntilPassOrDone();
+        }
+        task.destroy();
+        return task.calculateResult();
+    };
+}
+exports.inspectOverTime = inspectOverTime;
+class InspectionOverTimeTask {
+    target;
+    streamrClient;
+    createOperatorFleetState;
+    getRedundancyFactor;
+    sleepTimeInMsBeforeFirstInspection;
+    heartbeatTimeoutInMs;
+    inspectionIntervalInMs;
+    maxInspections;
+    abortSignal;
+    findNodesForTargetGivenFleetStateFn;
+    inspectTargetFn;
+    logger;
+    fleetState;
+    inspectionResults = new Array();
+    abortController = new AbortController();
+    passedSingleInspectionGate = new utils_1.Gate(false);
+    doneGate = new utils_1.Gate(false);
+    constructor({ target, streamrClient, createOperatorFleetState, getRedundancyFactor, sleepTimeInMsBeforeFirstInspection, heartbeatTimeoutInMs, inspectionIntervalInMs, maxInspections, abortSignal: userAbortSignal, traceId, findNodesForTargetGivenFleetStateFn = inspectionUtils_1.findNodesForTargetGivenFleetState, inspectTargetFn = inspectionUtils_1.inspectTarget, }) {
+        this.target = target;
+        this.streamrClient = streamrClient;
+        this.createOperatorFleetState = createOperatorFleetState;
+        this.getRedundancyFactor = getRedundancyFactor;
+        this.sleepTimeInMsBeforeFirstInspection = sleepTimeInMsBeforeFirstInspection;
+        this.heartbeatTimeoutInMs = heartbeatTimeoutInMs;
+        this.inspectionIntervalInMs = inspectionIntervalInMs;
+        this.maxInspections = maxInspections;
+        this.abortSignal = (0, utils_1.composeAbortSignals)(userAbortSignal, this.abortController.signal);
+        this.findNodesForTargetGivenFleetStateFn = findNodesForTargetGivenFleetStateFn;
+        this.inspectTargetFn = inspectTargetFn;
+        this.logger = new utils_1.Logger(module, { traceId });
+        this.abortSignal.addEventListener('abort', async () => {
+            await this.fleetState?.destroy();
+        });
+    }
+    calculateResult() {
+        const passCount = this.inspectionResults.filter((pass) => pass).length;
+        this.logger.info('Inspection done', {
+            passFraction: `${passCount}/${this.inspectionResults.length}`,
+            inspectionResults: this.inspectionResults
+        });
+        return this.inspectionResults;
+    }
+    start() {
+        this.run().catch((err) => {
+            if (!(err instanceof utils_1.AbortError) && err?.reason !== 'AbortError') {
+                this.logger.warn('Error encountered', { err });
+                this.destroy();
+            }
+        });
+    }
+    waitUntilPassOrDone() {
+        return Promise.race([
+            this.doneGate.waitUntilOpen(),
+            this.passedSingleInspectionGate.waitUntilOpen()
+        ]);
+    }
+    destroy() {
+        this.abortController.abort();
+    }
+    async run() {
+        this.logger.info('Start', {
+            target: this.target,
+            heartbeatTimeoutInMs: this.heartbeatTimeoutInMs,
+            inspectionIntervalInMs: this.inspectionIntervalInMs,
+            maxInspections: this.maxInspections
+        });
+        await this.initializeNewOperatorFleetState();
+        this.logger.info('Sleep', { timeInMs: this.sleepTimeInMsBeforeFirstInspection });
+        await (0, utils_1.wait)(this.sleepTimeInMsBeforeFirstInspection, this.abortSignal);
+        for (const attemptNo of (0, range_1.default)(1, this.maxInspections + 1)) {
+            const startTime = Date.now();
+            this.logger.info('Inspecting target', { attemptNo, target: this.target });
+            const onlineNodeDescriptors = await this.findNodesForTargetGivenFleetStateFn(this.target, this.fleetState, this.getRedundancyFactor, this.logger);
+            this.abortSignal.throwIfAborted();
+            const pass = await this.inspectTargetFn({
+                target: this.target,
+                targetPeerDescriptors: onlineNodeDescriptors,
+                streamrClient: this.streamrClient,
+                abortSignal: this.abortSignal,
+                logger: this.logger
+            });
+            this.inspectionResults.push(pass);
+            if (pass) {
+                this.passedSingleInspectionGate.open();
+            }
+            const timeElapsedInMs = Date.now() - startTime;
+            this.logger.info('Inspected target', {
+                attemptNo,
+                pass,
+                timeElapsedInMs,
+                target: this.target
+            });
+            if (attemptNo !== this.maxInspections) {
+                // TODO: remove when NET-1169 landed;
+                //  workaround subscribe bug in @streamr/sdk (sometimes messages don't come thru to heartbeat stream)
+                if (this.fleetState?.getNodeIds().length === 0) {
+                    this.logger.info('Destroying and re-creating fleet state');
+                    if (this.fleetState !== undefined) {
+                        await this.fleetState.destroy();
+                    }
+                    this.abortSignal.throwIfAborted();
+                    await this.initializeNewOperatorFleetState();
+                    this.abortSignal.throwIfAborted();
+                }
+                const sleepTime = Math.max(this.inspectionIntervalInMs - timeElapsedInMs, 0);
+                this.logger.info('Sleep', { timeInMs: sleepTime });
+                await (0, utils_1.wait)(sleepTime, this.abortSignal);
+            }
+        }
+        this.doneGate.open();
+    }
+    async initializeNewOperatorFleetState() {
+        this.fleetState = this.createOperatorFleetState((0, formCoordinationStreamId_1.formCoordinationStreamId)(this.target.operatorAddress));
+        await this.fleetState.start();
+        this.logger.info('Waiting for fleet state');
+        await Promise.race([
+            this.fleetState.waitUntilReady(),
+            (0, utils_1.wait)(this.heartbeatTimeoutInMs, this.abortSignal)
+        ]);
+        this.logger.info('Wait done for fleet state', { onlineNodeCount: this.fleetState.getNodeIds().length });
+    }
+}
+//# sourceMappingURL=inspectOverTime.js.map

package/dist/src/plugins/operator/inspectOverTime.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"inspectOverTime.js","sourceRoot":"","sources":["../../../../src/plugins/operator/inspectOverTime.ts"],"names":[],"mappings":";;;;;;AAAA,0CAAqG;AAGrG,uDAM0B;AAC1B,yEAAqE;AACrE,yDAAgC;AAkBhC,SAAgB,eAAe,CAAC,IAAyB;IACrD,MAAM,IAAI,GAAG,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAA;IAC7C,IAAI,CAAC,KAAK,EAAE,CAAA;IACZ,OAAO,KAAK,IAAI,EAAE;QACd,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAA;QACpC,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAA;QACd,OAAO,IAAI,CAAC,eAAe,EAAE,CAAA;IACjC,CAAC,CAAA;AACL,CAAC;AAVD,0CAUC;AAED,MAAM,sBAAsB;IACP,MAAM,CAAQ;IACd,aAAa,CAAe;IAC5B,wBAAwB,CAA4B;IACpD,mBAAmB,CAA2E;IAC9F,kCAAkC,CAAQ;IAC1C,oBAAoB,CAAQ;IAC5B,sBAAsB,CAAQ;IAC9B,cAAc,CAAQ;IACtB,WAAW,CAAa;IACxB,mCAAmC,CAAqC;IACxE,eAAe,CAAiB;IAChC,MAAM,CAAQ;IAEvB,UAAU,CAAqB;IACtB,iBAAiB,GAAG,IAAI,KAAK,EAAW,CAAA;IACxC,eAAe,GAAG,IAAI,eAAe,EAAE,CAAA;IACvC,0BAA0B,GAAG,IAAI,YAAI,CAAC,KAAK,CAAC,CAAA;IAC5C,QAAQ,GAAG,IAAI,YAAI,CAAC,KAAK,CAAC,CAAA;IAE3C,YAAY,EACR,MAAM,EACN,aAAa,EACb,wBAAwB,EACxB,mBAAmB,EACnB,kCAAkC,EAClC,oBAAoB,EACpB,sBAAsB,EACtB,cAAc,EACd,WAAW,EAAE,eAAe,EAC5B,OAAO,EACP,mCAAmC,GAAG,mDAAiC,EACvE,eAAe,GAAG,+BAAa,GACb;QAClB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAA;QAClC,IAAI,CAAC,wBAAwB,GAAG,wBAAwB,CAAA;QACxD,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAC9C,IAAI,CAAC,kCAAkC,GAAG,kCAAkC,CAAA;QAC5E,IAAI,CAAC,oBAAoB,GAAG,oBAAoB,CAAA;QAChD,IAAI,CAAC,sBAAsB,GAAG,sBAAsB,CAAA;QACpD,IAAI,CAAC,cAAc,GAAG,cAAc,CAAA;QACpC,IAAI,CAAC,WAAW,GAAG,IAAA,2BAAmB,EAAC,eAAe,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAA;QACpF,IAAI,CAAC,mCAAmC,GAAG,mCAAmC,CAAA;QAC9E,IAAI,CAAC,eAAe,GAAG,eAAe,CAAA;QACtC,IAAI,CAAC,MAAM,GAAG,IAAI,cAAM,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;QAC7C,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,CAAA;QACpC,CAAC,CAAC,CAAA;IACN,CAAC;IAED,eAAe;QACX,MAAM,SAAS,GAAG,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,CAAA;QACtE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE;YAChC,YAAY,EAAE,GAAG,SAAS,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,EAAE;YAC7D,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;SAC5C,CAAC,CAAA;QACF,OAAO,IAAI,CAAC,iBAAiB,CAAA;IACjC,CAAC;IAED,KAAK;QACD,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACrB,IAAI,CAAC,CAAC,GAAG,YAAY,kBAAU,CAAC,IAAI,GAAG,EAAE,MAAM,KAAK,YAAY,EAAE,CAAC;gBAC/D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAA;gBAC9C,IAAI,CAAC,OAAO,EAAE,CAAA;YAClB,CAAC;QACL,CAAC,CAAC,CAAA;IACN,CAAC;IAED,mBAAmB;QACf,OAAO,OAAO,CAAC,IAAI,CAAC;YAChB,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE;YAC7B,IAAI,CAAC,0BAA0B,CAAC,aAAa,EAAE;SAClD,CAAC,CAAA;IACN,CAAC;IAED,OAAO;QACH,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAA;IAChC,CAAC;IAEO,KAAK,CAAC,GAAG;QACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;YAC/C,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;YACnD,cAAc,EAAE,IAAI,CAAC,cAAc;SACtC,CAAC,CAAA;QAEF,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAA;QAE5C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,kCAAkC,EAAE,CAAC,CAAA;QAChF,MAAM,IAAA,YAAI,EAAC,IAAI,CAAC,kCAAkC,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;QAErE,KAAK,MAAM,SAAS,IAAI,IAAA,eAAK,EAAC,CAAC,EAAE,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,EAAE,CAAC;YACxD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;YAEzE,MAAM,qBAAqB,GAAG,MAAM,IAAI,CAAC,mCAAmC,CACxE,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,UAAW,EAChB,IAAI,CAAC,mBAAmB,EACxB,IAAI,CAAC,MAAM,CACd,CAAA;YACD,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAA;YACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC;gBACpC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,qBAAqB,EAAE,qBAAqB;gBAC5C,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;aACtB,CAAC,CAAA;YACF,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACjC,IAAI,IAAI,EAAE,CAAC;gBACP,IAAI,CAAC,0BAA0B,CAAC,IAAI,EAAE,CAAA;YAC1C,CAAC;YACD,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;YAC9C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE;gBACjC,SAAS;gBACT,IAAI;gBACJ,eAAe;gBACf,MAAM,EAAE,IAAI,CAAC,MAAM;aACtB,CAAC,CAAA;YAEF,IAAI,SAAS,KAAK,IAAI,CAAC,cAAc,EAAE,CAAC;gBACpC,qCAAqC;gBACrC,qGAAqG;gBACrG,IAAI,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC7C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAA;oBAC1D,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;wBAChC,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAA;oBACnC,CAAC;oBACD,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAA;oBACjC,MAAM,IAAI,CAAC,+BAA+B,EAAE,CAAA;oBAC5C,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAA;gBACrC,CAAC;gBAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,sBAAsB,GAAG,eAAe,EAAE,CAAC,CAAC,CAAA;gBAC5E,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAA;gBAClD,MAAM,IAAA,YAAI,EAAC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,CAAA;YAC3C,CAAC;QACL,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;IACxB,CAAC;IAEO,KAAK,CAAC,+BAA+B;QACzC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,wBAAwB,CAAC,IAAA,mDAAwB,EAAC,IAAI,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAA;QACtG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAA;QAC7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAA;QAC3C,MAAM,OAAO,CAAC,IAAI,CAAC;YACf,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE;YAChC,IAAA,YAAI,EAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,CAAC,WAAW,CAAC;SACpD,CAAC,CAAA;QACF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,eAAe,EAAE,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3G,CAAC;CACJ"}

package/dist/src/plugins/operator/inspectRandomNode.d.ts

@@ -0,0 +1,8 @@
+/// <reference types="node" />
+import { EthereumAddress } from '@streamr/utils';
+import { StreamPartAssignments } from './StreamPartAssignments';
+import { StreamrClient } from '@streamr/sdk';
+import { findTarget } from './inspectionUtils';
+import { ContractFacade } from './ContractFacade';
+import { CreateOperatorFleetStateFn } from './OperatorFleetState';
+export declare function inspectRandomNode(operatorContractAddress: EthereumAddress, contractFacade: ContractFacade, assignments: StreamPartAssignments, streamrClient: StreamrClient, heartbeatTimeoutInMs: number, getRedundancyFactor: (operatorContractAddress: EthereumAddress) => Promise<number | undefined>, createOperatorFleetState: CreateOperatorFleetStateFn, abortSignal: AbortSignal, findTargetFn?: typeof findTarget): Promise<void>;

package/dist/src/plugins/operator/inspectRandomNode.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.inspectRandomNode = void 0;
+const utils_1 = require("@streamr/utils");
+const protocol_1 = require("@streamr/protocol");
+const inspectionUtils_1 = require("./inspectionUtils");
+const inspectOverTime_1 = require("./inspectOverTime");
+async function inspectRandomNode(operatorContractAddress, contractFacade, assignments, streamrClient, heartbeatTimeoutInMs, getRedundancyFactor, createOperatorFleetState, abortSignal, findTargetFn = inspectionUtils_1.findTarget) {
+    const traceId = (0, utils_1.randomString)(6);
+    const logger = new utils_1.Logger(module, { traceId });
+    logger.info('Select a random operator to inspect');
+    const target = await findTargetFn(operatorContractAddress, contractFacade, assignments, logger);
+    if (target === undefined) {
+        return;
+    }
+    logger.debug('Target established', { target });
+    const consumeResults = (0, inspectOverTime_1.inspectOverTime)({
+        target,
+        streamrClient,
+        createOperatorFleetState,
+        getRedundancyFactor,
+        sleepTimeInMsBeforeFirstInspection: 0,
+        heartbeatTimeoutInMs,
+        inspectionIntervalInMs: 8 * 60 * 1000,
+        maxInspections: 10,
+        waitUntilPassOrDone: true,
+        abortSignal,
+        traceId
+    });
+    const results = await consumeResults();
+    if (results.some((pass) => pass)) {
+        logger.info('Not raising flag', { target });
+        return;
+    }
+    const flagAlreadyRaised = await contractFacade.hasOpenFlag(target.operatorAddress, target.sponsorshipAddress);
+    if (flagAlreadyRaised) {
+        logger.info('Not raising flag (target already has open flag)', { target });
+        return;
+    }
+    logger.info('Raise flag', { target });
+    await contractFacade.flag(target.sponsorshipAddress, target.operatorAddress, protocol_1.StreamPartIDUtils.getStreamPartition(target.streamPart));
+}
+exports.inspectRandomNode = inspectRandomNode;
+//# sourceMappingURL=inspectRandomNode.js.map

package/dist/src/plugins/operator/inspectRandomNode.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"inspectRandomNode.js","sourceRoot":"","sources":["../../../../src/plugins/operator/inspectRandomNode.ts"],"names":[],"mappings":";;;AAAA,0CAAsE;AAGtE,gDAAqD;AACrD,uDAA8C;AAG9C,uDAAmD;AAE5C,KAAK,UAAU,iBAAiB,CACnC,uBAAwC,EACxC,cAA8B,EAC9B,WAAkC,EAClC,aAA4B,EAC5B,oBAA4B,EAC5B,mBAA8F,EAC9F,wBAAoD,EACpD,WAAwB,EACxB,YAAY,GAAG,4BAAU;IAEzB,MAAM,OAAO,GAAG,IAAA,oBAAY,EAAC,CAAC,CAAC,CAAA;IAC/B,MAAM,MAAM,GAAG,IAAI,cAAM,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;IAC9C,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAA;IAElD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,uBAAuB,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,CAAC,CAAA;IAC/F,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACvB,OAAM;IACV,CAAC;IACD,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;IAE9C,MAAM,cAAc,GAAG,IAAA,iCAAe,EAAC;QACnC,MAAM;QACN,aAAa;QACb,wBAAwB;QACxB,mBAAmB;QACnB,kCAAkC,EAAE,CAAC;QACrC,oBAAoB;QACpB,sBAAsB,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI;QACrC,cAAc,EAAE,EAAE;QAClB,mBAAmB,EAAE,IAAI;QACzB,WAAW;QACX,OAAO;KACV,CAAC,CAAA;IAEF,MAAM,OAAO,GAAG,MAAM,cAAc,EAAE,CAAA;IACtC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;QAC3C,OAAM;IACV,CAAC;IAED,MAAM,iBAAiB,GAAG,MAAM,cAAc,CAAC,WAAW,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,kBAAkB,CAAC,CAAA;IAC7G,IAAI,iBAAiB,EAAE,CAAC;QACpB,MAAM,CAAC,IAAI,CAAC,iDAAiD,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;QAC1E,OAAM;IACV,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,MAAM,EAAE,CAAC,CAAA;IACrC,MAAM,cAAc,CAAC,IAAI,CACrB,MAAM,CAAC,kBAAkB,EACzB,MAAM,CAAC,eAAe,EACtB,4BAAiB,CAAC,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,CAC1D,CAAA;AACL,CAAC;AArDD,8CAqDC"}

package/dist/src/plugins/operator/inspectionUtils.d.ts

@@ -0,0 +1,23 @@
+/// <reference types="node" />
+import { NetworkPeerDescriptor, StreamrClient } from '@streamr/sdk';
+import { OperatorFleetState } from './OperatorFleetState';
+import { StreamPartID } from '@streamr/protocol';
+import { EthereumAddress, Logger } from '@streamr/utils';
+import { StreamPartAssignments } from './StreamPartAssignments';
+import { ContractFacade } from './ContractFacade';
+export type FindNodesForTargetGivenFleetStateFn = typeof findNodesForTargetGivenFleetState;
+export type InspectTargetFn = typeof inspectTarget;
+export interface Target {
+    sponsorshipAddress: EthereumAddress;
+    operatorAddress: EthereumAddress;
+    streamPart: StreamPartID;
+}
+export declare function findTarget(myOperatorContractAddress: EthereumAddress, contractFacade: ContractFacade, assignments: StreamPartAssignments, logger: Logger): Promise<Target | undefined>;
+export declare function findNodesForTargetGivenFleetState(target: Target, targetOperatorFleetState: OperatorFleetState, getRedundancyFactor: (operatorContractAddress: EthereumAddress) => Promise<number | undefined>, logger: Logger): Promise<NetworkPeerDescriptor[]>;
+export declare function inspectTarget({ target, targetPeerDescriptors, streamrClient, abortSignal, logger }: {
+    target: Target;
+    targetPeerDescriptors: NetworkPeerDescriptor[];
+    streamrClient: StreamrClient;
+    abortSignal: AbortSignal;
+    logger: Logger;
+}): Promise<boolean>;