@gabrielhicks/solv 5.6.5 → 5.6.7

package/dist/cli/monitoring/scripts/common.py

@@ -0,0 +1,63 @@
+from pprint import pprint
+import json
+import numpy
+import time
+
+
+class ValidatorConfig:
+    def __init__(self,
+                 validator_name: str,
+                 secrets_path: str,
+                 local_rpc_address: str,
+                 remote_rpc_address: str,
+                 cluster_environment: str,
+                 debug_mode: bool):
+        self.validator_name = validator_name
+        self.secrets_path = secrets_path
+        self.local_rpc_address = local_rpc_address
+        self.remote_rpc_address = remote_rpc_address
+        self.cluster_environment = cluster_environment
+        self.debug_mode = debug_mode
+
+
+class JsonEncoder(json.JSONEncoder):
+    """ Special json encoder for numpy types """
+    def default(self, obj):
+        if isinstance(obj, (numpy.int_, numpy.intc, numpy.intp, numpy.int8,
+                            numpy.int16, numpy.int32, numpy.int64, numpy.uint8,
+                            numpy.uint16, numpy.uint32, numpy.uint64)):
+            return int(obj)
+        elif isinstance(obj, (numpy.float_, numpy.float16, numpy.float32,
+                              numpy.float64)):
+            return float(obj)
+        elif isinstance(obj, (numpy.ndarray,)):
+            return obj.tolist()
+        return json.JSONEncoder.default(self, obj)
+
+
+def debug(config: ValidatorConfig, data):
+    if config.debug_mode:
+        pprint(data)
+
+
+def print_json(data):
+    print(json.dumps(data, cls=JsonEncoder))
+
+
+def measurement_from_fields(name, data, tags, config, legacy_tags=None):
+    if legacy_tags is None:
+        legacy_tags = {}
+    data.update({"cluster_environment": config.cluster_environment})
+    measurement = {
+        "measurement": name,
+        "time": round(time.time() * 1000),
+        "monitoring_version": "3.2.0",
+        "cluster_environment": config.cluster_environment,
+        "fields": data,
+        "tags": tags
+    }
+    
+    measurement.update(legacy_tags)
+
+    return measurement
+

package/dist/cli/monitoring/scripts/measurement_validator_info.py

@@ -0,0 +1,353 @@
+import time
+import solana_rpc as rpc
+from common import debug
+from common import ValidatorConfig
+import statistics
+import numpy as np
+from common import measurement_from_fields
+
+
+def get_metrics_from_vote_account_item(item):
+    return {
+        'epoch_number': item['epochCredits'][-1][0],
+        'credits_epoch': item['epochCredits'][-1][1],
+        'credits_previous_epoch': item['epochCredits'][-1][2],
+        'activated_stake': item['activatedStake'],
+        'credits_epoch_delta': item['epochCredits'][-1][1] - item['epochCredits'][-1][2],
+        'commission': item['commission']
+    }
+
+
+def find_item_in_vote_accounts_section(identity_account_pubkey, section_parent, section_name):
+    if section_name in section_parent:
+        section = section_parent[section_name]
+        for item in section:
+            if item['nodePubkey'] == identity_account_pubkey:
+                return get_metrics_from_vote_account_item(item)
+
+    return None
+
+
+def get_vote_account_metrics(vote_accounts_data, identity_account_pubkey):
+    """
+    get vote metrics from vote account
+    :return:
+    voting_status: 0 if validator not found in voting accounts
+    voting_status: 1 if validator is current
+    voting_status: 2 if validator is delinquent
+
+    """
+    result = find_item_in_vote_accounts_section(identity_account_pubkey, vote_accounts_data, 'current')
+    if result is not None:
+        result.update({'voting_status': 1})
+    else:
+        result = find_item_in_vote_accounts_section(identity_account_pubkey, vote_accounts_data, 'delinquent')
+        if result is not None:
+            result.update({'voting_status': 2})
+        else:
+            result = {'voting_status': 0}
+    return result
+
+
+def get_leader_schedule_metrics(leader_schedule_data, identity_account_pubkey):
+    """
+    get metrics about leader slots
+    """
+    if identity_account_pubkey in leader_schedule_data:
+        return {"leader_slots_this_epoch": len(leader_schedule_data[identity_account_pubkey])}
+    else:
+        return {"leader_slots_this_epoch": 0}
+
+
+def get_block_production_metrics(block_production_data, identity_account_pubkey):
+    try:
+        item = block_production_data['value']['byIdentity'][identity_account_pubkey]
+        return {
+            "slots_done": item[0],
+            "slots_skipped": item[0] - item[1],
+            "blocks_produced": item[1]
+
+        }
+    except:
+        return {"slots_done": 0, "slots_skipped": 0, "blocks_produced": 0}
+
+
+def get_block_production_cli_metrics(block_production_data_cli, identity_account_pubkey: str):
+    if 'leaders' in block_production_data_cli:
+        leaders = block_production_data_cli['leaders']
+        skip_rate = []
+        my_skip_rate = 0
+        for leader in leaders:
+            leader_slots = leader.get('leaderSlots', 0)
+            if leader_slots > 0:
+                current_skip_rate = leader.get('skippedSlots', 0) / leader_slots
+                skip_rate.append(current_skip_rate)
+                if leader['identityPubkey'] == identity_account_pubkey:
+                    my_skip_rate = current_skip_rate
+
+        result = {
+            'leader_skip_rate': my_skip_rate,
+            'cluster_min_leader_skip_rate': min(skip_rate),
+            'cluster_max_leader_skip_rate': max(skip_rate),
+            'cluster_mean_leader_skip_rate': statistics.mean(skip_rate),
+            'cluster_median_leader_skip_rate': statistics.median(skip_rate),
+        }
+    else:
+        result = {}
+
+    return result
+
+
+def get_performance_metrics(performance_sample_data, epoch_info_data, leader_schedule_by_identity):
+    if len(performance_sample_data) > 0:
+        sample = performance_sample_data[0]
+        if sample['numSlots'] > 0:
+            mid_slot_time = sample['samplePeriodSecs'] / sample['numSlots']
+        else:
+            mid_slot_time = 0
+        current_slot_index = epoch_info_data['slotIndex']
+        remaining_time = (epoch_info_data["slotsInEpoch"] - current_slot_index) * mid_slot_time
+        epoch_end_time = round(time.time()) + remaining_time
+        time_until_next_slot = -1
+        if leader_schedule_by_identity is not None:
+            for slot in leader_schedule_by_identity:
+                if current_slot_index < slot:
+                    next_slot = slot
+                    time_until_next_slot = (next_slot - current_slot_index) * mid_slot_time
+                    break
+        else:
+            time_until_next_slot = None
+
+        result = {
+            "epoch_endtime": epoch_end_time,
+            "epoch_remaining_sec": remaining_time
+        }
+
+        if time_until_next_slot is not None:
+            result.update({"time_until_next_slot": time_until_next_slot})
+    else:
+        result = {}
+
+    return result
+
+
+def get_balance_metric(balance_data, key: str):
+    if 'value' in balance_data:
+        result = {key: balance_data['value']}
+    else:
+        result = {}
+
+    return result
+
+
+def get_solana_version_metric(solana_version_data):
+    if solana_version_data is not None:
+        if 'solana-core' in solana_version_data:
+            return {'solana_version': solana_version_data['solana-core']}
+
+    return {}
+
+
+def get_validators_metric(validators, identity_account_pubkey):
+    if validators is not None:
+        epoch_credits_l = []
+        last_vote_l = []
+        root_slot_l = []
+        current_last_vote = -1
+        current_root_slot = -1
+
+        for v in validators:
+            if not v['delinquent']:
+                epoch_credits_l.append(v['epochCredits'])
+                last_vote_l.append(v['lastVote'])
+                root_slot_l.append(v['rootSlot'])
+            if identity_account_pubkey == v['identityPubkey']:
+                current_last_vote = v['lastVote']
+                current_root_slot = v['rootSlot']
+
+        epoch_credits = np.array(epoch_credits_l, dtype=np.int32)
+        last_vote = np.array(last_vote_l, dtype=np.int32)
+        root_slot = np.array(root_slot_l, dtype=np.int32)
+
+        last_vote = last_vote[last_vote > 0]
+        root_slot = root_slot[root_slot > 0]
+
+        cluster_max_last_vote = np.amax(last_vote)
+        cluster_min_last_vote = np.amin(last_vote)
+        cluster_mean_last_vote = abs((last_vote - cluster_max_last_vote).mean())
+        cluster_median_last_vote = abs(np.median(last_vote - cluster_max_last_vote))
+
+        cluster_max_root_slot = np.amax(root_slot)
+        cluster_min_root_slot = np.amin(root_slot)
+        cluster_mean_root_slot = abs((root_slot - cluster_max_root_slot).mean())
+        cluster_median_root_slot = abs(np.median(root_slot - cluster_max_root_slot))
+
+        result = {
+            'cluster_mean_epoch_credits': epoch_credits.mean(),
+            'cluster_min_epoch_credits': np.amin(epoch_credits),
+            'cluster_max_epoch_credits': np.amax(epoch_credits),
+            'cluster_median_epoch_credits': np.median(epoch_credits),
+
+            'cluster_max_last_vote': cluster_max_last_vote,
+            'cluster_min_last_vote_v2': cluster_min_last_vote,
+            'cluster_mean_last_vote_v2': cluster_mean_last_vote,
+            'cluster_median_last_vote': cluster_median_last_vote,
+            'current_last_vote': current_last_vote,
+
+            'cluster_max_root_slot': cluster_max_root_slot,
+            'cluster_min_root_slot_v2': cluster_min_root_slot,
+            'cluster_mean_root_slot_v2': cluster_mean_root_slot,
+            'cluster_median_root_slot': cluster_median_root_slot,
+            'current_root_slot': current_root_slot
+        }
+
+    else:
+        result = {}
+
+    return result
+
+
+def get_current_stake_metric(stake_data):
+    active = 0
+    activating = 0
+    deactivating = 0
+    active_cnt = 0
+    activating_cnt = 0
+    deactivating_cnt = 0
+    for item in stake_data:
+        if 'activeStake' in item:
+            active = active + item.get('activeStake', 0)
+            active_cnt = active_cnt + 1
+        if 'activatingStake' in item:
+            activating = activating + item.get('activatingStake', 0)
+            activating_cnt = activating_cnt + 1
+        if 'deactivatingStake' in item:
+            deactivating = deactivating + item.get('deactivatingStake', 0)
+            deactivating_cnt = deactivating_cnt + 1
+
+    return {
+        'active_stake': active,
+        'activating_stake': activating,
+        'deactivating_stake': deactivating,
+        'stake_holders': len(stake_data),
+        'active_cnt': active_cnt,
+        'activating_cnt': activating_cnt,
+        'deactivating_cnt': deactivating_cnt
+    }
+
+
+def load_data(config: ValidatorConfig):
+    identity_account_pubkey = rpc.load_identity_account_pubkey(config)
+    vote_account_pubkey = rpc.load_vote_account_pubkey(config)
+
+    epoch_info_data = rpc.load_epoch_info(config)
+    block_production_cli = rpc.load_block_production_cli(config)
+    performance_sample_data = rpc.load_recent_performance_sample(config)
+    solana_version_data = rpc.load_solana_version(config)
+    validators_data = rpc.load_solana_validators(config)
+
+    default = []
+
+    identity_account_balance_data = default
+    leader_schedule_data = default
+    block_production_data = default   
+
+    vote_account_balance_data = default
+    vote_accounts_data = default
+    stakes_data = default
+
+    if identity_account_pubkey is not None:
+        identity_account_balance_data = rpc.load_identity_account_balance(config, identity_account_pubkey)
+        leader_schedule_data = rpc.load_leader_schedule(config, identity_account_pubkey)
+        block_production_data = rpc.load_block_production(config, identity_account_pubkey)      
+
+    if vote_account_pubkey is not None:
+        vote_account_balance_data = rpc.load_vote_account_balance(config, vote_account_pubkey)
+        vote_accounts_data = rpc.load_vote_accounts(config, vote_account_pubkey)
+        stakes_data = rpc.load_stakes(config, vote_account_pubkey)
+
+    result = {
+        'identity_account_pubkey': identity_account_pubkey,
+        'vote_account_pubkey': vote_account_pubkey,
+        'identity_account_balance': identity_account_balance_data,
+        'vote_account_balance': vote_account_balance_data,
+        'epoch_info': epoch_info_data,
+        'leader_schedule': leader_schedule_data,
+        'block_production': block_production_data,
+        'load_block_production_cli': block_production_cli,
+        'vote_accounts': vote_accounts_data,
+        'performance_sample': performance_sample_data,
+        'solana_version_data': solana_version_data,
+        'stakes_data': stakes_data,
+        'validators_data': validators_data,   
+        'cpu_model': rpc.load_cpu_model(config)
+    }
+
+    debug(config, str(result))
+
+    return result
+
+
+def calculate_influx_fields(data):
+    if data is None:
+        result = {"validator_status": 0}
+    else:
+        identity_account_pubkey = data['identity_account_pubkey']
+
+        vote_account_metrics = get_vote_account_metrics(data['vote_accounts'], identity_account_pubkey)
+        leader_schedule_metrics = get_leader_schedule_metrics(data['leader_schedule'], identity_account_pubkey)
+        epoch_metrics = data['epoch_info']
+        block_production_metrics = get_block_production_metrics(data['block_production'], identity_account_pubkey)
+        if identity_account_pubkey in data['leader_schedule']:
+            leader_schedule_by_identity = data['leader_schedule'][identity_account_pubkey]
+        else:
+            leader_schedule_by_identity = None
+
+        performance_metrics = get_performance_metrics(
+            data['performance_sample'], epoch_metrics, leader_schedule_by_identity)
+
+        result = {"validator_status": 1}
+        result.update(vote_account_metrics)
+        result.update(leader_schedule_metrics)
+        result.update(epoch_metrics)
+        result.update(block_production_metrics)
+        result.update(performance_metrics)
+        result.update(get_balance_metric(data['identity_account_balance'], 'identity_account_balance'))
+        result.update(get_balance_metric(data['vote_account_balance'], 'vote_account_balance'))
+        result.update(get_current_stake_metric(data['stakes_data']))
+        result.update(get_validators_metric(data['validators_data'], identity_account_pubkey))
+        result.update(get_block_production_cli_metrics(data['load_block_production_cli'], identity_account_pubkey))
+        result.update({"cpu_model": data['cpu_model']})
+
+    return result
+
+
+def calculate_output_data(config: ValidatorConfig):
+    data = load_data(config)
+
+    tags = {
+        "validator_identity_pubkey": data['identity_account_pubkey'],
+        "validator_vote_pubkey": data['vote_account_pubkey'],
+        "validator_name": config.validator_name,
+        "cluster_environment": config.cluster_environment
+    }
+
+    legacy_tags = {
+        "validator_identity_pubkey": data['identity_account_pubkey'],
+        "validator_vote_pubkey": data['vote_account_pubkey'],
+        "validator_name": config.validator_name,
+    }
+
+    measurement = measurement_from_fields(
+        "validators_info",
+        calculate_influx_fields(data),
+        tags,
+        config,
+        legacy_tags
+    )
+    measurement.update({"cpu_model": data['cpu_model']})
+    if data is not None and 'solana_version_data' in data:
+        measurement.update(get_solana_version_metric(data['solana_version_data']))
+
+    return measurement
+

package/dist/cli/monitoring/scripts/output_validator_measurements.py

@@ -0,0 +1,6 @@
+from monitoring_config import config
+from measurement_validator_info import calculate_output_data
+from common import print_json
+
+print_json(calculate_output_data(config))
+

package/dist/cli/monitoring/scripts/request_utils.py

@@ -0,0 +1,76 @@
+from common import ValidatorConfig
+import subprocess
+import requests
+import json
+from common import debug
+
+
+def execute_cmd_str(config: ValidatorConfig, cmd: str, convert_to_json: bool, default=None):
+    """
+    executes shell command and return string result
+    :param default:
+    :param config:
+    :param convert_to_json:
+    :param cmd: shell command
+    :return: returns string result or None
+    """
+    try:
+        debug(config, cmd)
+        result: str = subprocess.check_output(cmd, shell=True, stderr=subprocess.DEVNULL, timeout=10).decode().strip()
+
+        if convert_to_json:
+            result = json.loads(result)
+
+        debug(config, result)
+
+        return result
+    except:
+        return default
+
+
+def rpc_call(config: ValidatorConfig, address: str, method: str, params, error_result, except_result):
+    """
+    calls solana rpc (https://docs.solana.com/developing/clients/jsonrpc-api)
+    and returns result or default
+    :param config:
+    :param except_result:
+    :param error_result:
+    :param address: local or remote rpc server address
+    :param method: rpc method
+    :param params: rpc call parameters
+    :return: result or default
+    """
+    try:
+        json_request = {
+            "jsonrpc": "2.0",
+            "id": 1,
+            "method": method,
+            "params": params
+        }
+        debug(config, json_request)
+        debug(config, address)
+
+        json_response = requests.post(address, json=json_request).json()
+        if 'result' not in json_response:
+            result = error_result
+        else:
+            result = json_response['result']
+    except:
+        result = except_result
+
+    debug(config, result)
+
+    return result
+
+
+def smart_rpc_call(config: ValidatorConfig, method: str, params, default_result):
+    """
+    tries to call local rpc, if it fails tries to call remote rpc
+    """
+    result = rpc_call(config, config.local_rpc_address, method, params, None, None)
+
+    if result is None:
+        result = rpc_call(config, config.remote_rpc_address, method, params, default_result, default_result)
+
+    return result
+

package/dist/cli/monitoring/scripts/solana_rpc.py

@@ -0,0 +1,202 @@
+from common import ValidatorConfig
+from typing import Optional
+from common import debug
+from request_utils import execute_cmd_str, smart_rpc_call, rpc_call
+
+
+def load_identity_account_pubkey(config: ValidatorConfig) -> Optional[str]:
+    """
+    loads validator identity account pubkey
+    :param config: Validator Configuration
+    :return: returns validator identity pubkey or None
+    """
+    identity_cmd = f'solana address -u localhost --keypair ' + config.secrets_path + '/validator-keypair.json'
+    debug(config, identity_cmd)
+    return execute_cmd_str(config, identity_cmd, convert_to_json=False)
+
+
+def load_vote_account_pubkey(config: ValidatorConfig) -> Optional[str]:
+    """
+    loads vote account pubkey
+    :param config: Validator Configuration
+    :return: returns vote account pubkey  or None
+    """
+    vote_pubkey_cmd = f'solana address -u localhost --keypair ' + config.secrets_path + '/vote-account-keypair.json'
+    debug(config, vote_pubkey_cmd)
+    return execute_cmd_str(config, vote_pubkey_cmd, convert_to_json=False)
+
+
+def load_vote_account_balance(config: ValidatorConfig, vote_account_pubkey: str):
+    """
+    loads vote account balance
+    https://docs.solana.com/developing/clients/jsonrpc-api#getbalance
+    """
+    return smart_rpc_call(config, "getBalance", [vote_account_pubkey], {})
+
+
+def load_identity_account_balance(config: ValidatorConfig, identity_account_pubkey: str):
+    """
+    loads identity account balance
+    https://docs.solana.com/developing/clients/jsonrpc-api#getbalance
+    """
+    return smart_rpc_call(config, "getBalance", [identity_account_pubkey], {})
+
+
+def load_epoch_info(config: ValidatorConfig):
+    """
+    loads epoch info
+    https://docs.solana.com/developing/clients/jsonrpc-api#getepochinfo
+    """
+    return smart_rpc_call(config, "getEpochInfo", [], {})
+
+
+def load_leader_schedule(config: ValidatorConfig, identity_account_pubkey: str):
+    """
+    loads leader schedule
+    https://docs.solana.com/developing/clients/jsonrpc-api#getleaderschedule
+    """
+    params = [
+        None,
+        {
+            'identity': identity_account_pubkey
+        }
+    ]
+    return smart_rpc_call(config, "getLeaderSchedule", params, {})
+
+
+def load_block_production(config: ValidatorConfig, identity_account_pubkey: str):
+    """
+    loads block production
+    https://docs.solana.com/developing/clients/jsonrpc-api#getblockproduction
+    """
+    params = [
+        {
+            'identity': identity_account_pubkey
+        }
+    ]
+    return smart_rpc_call(config, "getBlockProduction", params, {})
+
+
+def load_block_production_cli(config: ValidatorConfig):
+    cmd = f'solana block-production -u l --output json-compact'
+    return execute_cmd_str(config, cmd, convert_to_json=True, default={})
+
+
+def load_vote_accounts(config: ValidatorConfig, vote_account_pubkey: str):
+    """
+    loads block production
+    https://docs.solana.com/developing/clients/jsonrpc-api#getvoteaccounts
+    """
+    params = [
+        {
+            'votePubkey': vote_account_pubkey
+        }
+    ]
+    return smart_rpc_call(config, "getVoteAccounts", params, {})
+
+
+def load_recent_performance_sample(config: ValidatorConfig):
+    """
+    loads recent performance sample
+    https://docs.solana.com/developing/clients/jsonrpc-api#getrecentperformancesamples
+    """
+    params = [1]
+    return rpc_call(config, config.remote_rpc_address, "getRecentPerformanceSamples", params, [], [])
+
+
+def load_solana_version(config: ValidatorConfig):
+    """
+    loads solana version
+    https://docs.solana.com/developing/clients/jsonrpc-api#getversion
+    """
+    return rpc_call(config, config.local_rpc_address, "getVersion", [], [], [])
+
+
+def load_stake_account_rewards(config: ValidatorConfig, stake_account):
+    cmd = f'solana stake-account ' + stake_account + ' --num-rewards-epochs=1 --with-rewards --output json-compact'
+    return execute_cmd_str(config, cmd, convert_to_json=True)
+
+
+def load_solana_validators(config: ValidatorConfig):
+    cmd = f'solana validators -ul --output json-compact'
+    data = execute_cmd_str(config, cmd, convert_to_json=True)
+
+    if (data is not None) and ('validators' in data):
+        return data['validators']
+    else:
+        return None
+
+
+def load_stakes(config: ValidatorConfig, vote_account):
+    cmd = f'solana stakes ' + vote_account + ' --output json-compact'
+    return execute_cmd_str(config, cmd, convert_to_json=True, default=[])
+
+
+def load_block_time(config: ValidatorConfig, block):
+    """
+    loads solana version
+    https://docs.solana.com/developing/clients/jsonrpc-api#getblocktime
+    """
+    params = [block]
+    return rpc_call(config, config.local_rpc_address, "getBlockTime", params, None, None)
+
+#    cmd = f'solana block-time -u l ' + str(block) + ' --output json-compact'
+#    return execute_cmd_str(cmd, convert_to_json=True)
+
+
+def try_to_load_current_block_info(config: ValidatorConfig):
+    epoch_info_data = load_epoch_info(config)
+
+    if epoch_info_data is not None:
+        slot_index = epoch_info_data['slotIndex']
+        absolute_slot = epoch_info_data['absoluteSlot']
+
+        block_time_data = load_block_time(config, absolute_slot)
+
+        if block_time_data is not None:
+            return {
+                'slot_index':  slot_index,
+                'absolute_block': absolute_slot,
+                'block_time': block_time_data['timestamp']
+            }
+
+    return None
+
+
+def load_current_block_info(config: ValidatorConfig):
+    result = None
+    max_tries = 10
+    current_try = 0
+    while result is None and current_try < max_tries:
+        result = try_to_load_current_block_info(config)
+        current_try = current_try + 1
+
+    return result
+
+
+def load_cpu_model(config: ValidatorConfig):
+    cmd = 'cat /proc/cpuinfo  | grep name| uniq'
+    cpu_info = execute_cmd_str(config, cmd, False).split(":")
+    cpu_model = cpu_info[1].strip()
+
+    if cpu_model is not None:
+        return cpu_model
+    else:
+        return 'Unknown'
+
+
+def load_solana_validators_full(config: ValidatorConfig):
+    cmd = f'solana validators -ul --output json-compact'
+    return execute_cmd_str(config, cmd, convert_to_json=True)
+
+
+def load_solana_validators_info(config: ValidatorConfig):
+    cmd = f'solana validator-info get --url ' + config.remote_rpc_address + ' --output json-compact'
+    data = execute_cmd_str(config, cmd, convert_to_json=True)
+    return data
+
+
+def load_solana_gossip(config: ValidatorConfig):
+    cmd = f'solana gossip -ul --output json-compact'
+    return execute_cmd_str(config, cmd, convert_to_json=True)
+