@oro-ai/sdk 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +220 -1
- package/dist/index.d.ts +220 -1
- package/dist/index.js +16 -0
- package/dist/index.mjs +14 -0
- package/package.json +1 -1
- package/src/generated/sdk.gen.ts +23 -1
- package/src/generated/types.gen.ts +221 -1
package/dist/index.d.mts
CHANGED
|
@@ -370,6 +370,23 @@ type AgentVersionPublic = {
|
|
|
370
370
|
*/
|
|
371
371
|
latest_final_score?: (number | null);
|
|
372
372
|
};
|
|
373
|
+
/**
|
|
374
|
+
* Per-validator score for an agent version.
|
|
375
|
+
*/
|
|
376
|
+
type AgentVersionScoreEntry = {
|
|
377
|
+
/**
|
|
378
|
+
* Validator hotkey
|
|
379
|
+
*/
|
|
380
|
+
validator_hotkey: string;
|
|
381
|
+
/**
|
|
382
|
+
* Score from this validator
|
|
383
|
+
*/
|
|
384
|
+
score: number;
|
|
385
|
+
/**
|
|
386
|
+
* Evaluation run ID
|
|
387
|
+
*/
|
|
388
|
+
run_id: string;
|
|
389
|
+
};
|
|
373
390
|
/**
|
|
374
391
|
* State of an agent version evaluation.
|
|
375
392
|
*/
|
|
@@ -441,6 +458,64 @@ type AgentVersionStatus = {
|
|
|
441
458
|
[key: string]: (number);
|
|
442
459
|
} | null);
|
|
443
460
|
};
|
|
461
|
+
/**
|
|
462
|
+
* Score variance across validators for a single agent version.
|
|
463
|
+
*/
|
|
464
|
+
type AgentVersionVariance = {
|
|
465
|
+
/**
|
|
466
|
+
* Agent version ID
|
|
467
|
+
*/
|
|
468
|
+
agent_version_id: string;
|
|
469
|
+
/**
|
|
470
|
+
* Agent name
|
|
471
|
+
*/
|
|
472
|
+
agent_name: string;
|
|
473
|
+
/**
|
|
474
|
+
* Miner hotkey
|
|
475
|
+
*/
|
|
476
|
+
miner_hotkey: string;
|
|
477
|
+
/**
|
|
478
|
+
* Number of validators that scored this version
|
|
479
|
+
*/
|
|
480
|
+
validator_count: number;
|
|
481
|
+
/**
|
|
482
|
+
* Mean score
|
|
483
|
+
*/
|
|
484
|
+
avg_score: number;
|
|
485
|
+
/**
|
|
486
|
+
* Minimum score
|
|
487
|
+
*/
|
|
488
|
+
min_score: number;
|
|
489
|
+
/**
|
|
490
|
+
* Maximum score
|
|
491
|
+
*/
|
|
492
|
+
max_score: number;
|
|
493
|
+
/**
|
|
494
|
+
* max_score - min_score
|
|
495
|
+
*/
|
|
496
|
+
spread: number;
|
|
497
|
+
/**
|
|
498
|
+
* True if spread exceeds threshold (default 10%)
|
|
499
|
+
*/
|
|
500
|
+
is_high_variance: boolean;
|
|
501
|
+
/**
|
|
502
|
+
* Individual validator scores
|
|
503
|
+
*/
|
|
504
|
+
per_validator: Array<AgentVersionScoreEntry>;
|
|
505
|
+
};
|
|
506
|
+
/**
|
|
507
|
+
* Response for agent version score variance analytics.
|
|
508
|
+
*/
|
|
509
|
+
type AgentVersionVarianceResponse = {
|
|
510
|
+
/**
|
|
511
|
+
* Per-version variance data
|
|
512
|
+
*/
|
|
513
|
+
agent_versions: Array<AgentVersionVariance>;
|
|
514
|
+
/**
|
|
515
|
+
* Spread threshold used for flagging
|
|
516
|
+
*/
|
|
517
|
+
variance_threshold: number;
|
|
518
|
+
};
|
|
444
519
|
/**
|
|
445
520
|
* 409 - Resource is already invalidated.
|
|
446
521
|
*/
|
|
@@ -2078,6 +2153,64 @@ type ValidatorPublic = {
|
|
|
2078
2153
|
*/
|
|
2079
2154
|
identity_description?: (string | null);
|
|
2080
2155
|
};
|
|
2156
|
+
/**
|
|
2157
|
+
* Response for validator scoring analytics.
|
|
2158
|
+
*/
|
|
2159
|
+
type ValidatorScoresResponse = {
|
|
2160
|
+
/**
|
|
2161
|
+
* Per-validator summaries
|
|
2162
|
+
*/
|
|
2163
|
+
validators: Array<ValidatorScoreSummary>;
|
|
2164
|
+
/**
|
|
2165
|
+
* Global average score across all validators
|
|
2166
|
+
*/
|
|
2167
|
+
global_avg_score: number;
|
|
2168
|
+
/**
|
|
2169
|
+
* Global standard deviation
|
|
2170
|
+
*/
|
|
2171
|
+
global_stddev: number;
|
|
2172
|
+
};
|
|
2173
|
+
/**
|
|
2174
|
+
* Aggregated scoring stats for a single validator.
|
|
2175
|
+
*/
|
|
2176
|
+
type ValidatorScoreSummary = {
|
|
2177
|
+
/**
|
|
2178
|
+
* Validator hotkey
|
|
2179
|
+
*/
|
|
2180
|
+
validator_hotkey: string;
|
|
2181
|
+
/**
|
|
2182
|
+
* Total completed runs
|
|
2183
|
+
*/
|
|
2184
|
+
total_runs: number;
|
|
2185
|
+
/**
|
|
2186
|
+
* Mean score across runs
|
|
2187
|
+
*/
|
|
2188
|
+
avg_score: number;
|
|
2189
|
+
/**
|
|
2190
|
+
* Median score
|
|
2191
|
+
*/
|
|
2192
|
+
median_score: number;
|
|
2193
|
+
/**
|
|
2194
|
+
* Standard deviation of scores
|
|
2195
|
+
*/
|
|
2196
|
+
stddev_score: number;
|
|
2197
|
+
/**
|
|
2198
|
+
* Minimum score
|
|
2199
|
+
*/
|
|
2200
|
+
min_score: number;
|
|
2201
|
+
/**
|
|
2202
|
+
* Maximum score
|
|
2203
|
+
*/
|
|
2204
|
+
max_score: number;
|
|
2205
|
+
/**
|
|
2206
|
+
* Percentage deviation from the global average (negative = below)
|
|
2207
|
+
*/
|
|
2208
|
+
deviation_from_global: number;
|
|
2209
|
+
/**
|
|
2210
|
+
* True if deviation exceeds 1.5 standard deviations from global mean
|
|
2211
|
+
*/
|
|
2212
|
+
is_outlier: boolean;
|
|
2213
|
+
};
|
|
2081
2214
|
/**
|
|
2082
2215
|
* Status of a validator.
|
|
2083
2216
|
*/
|
|
@@ -2482,6 +2615,10 @@ type GetAuditEventsData = {
|
|
|
2482
2615
|
* Offset for pagination
|
|
2483
2616
|
*/
|
|
2484
2617
|
offset?: number;
|
|
2618
|
+
/**
|
|
2619
|
+
* Filter: at or after (ISO 8601)
|
|
2620
|
+
*/
|
|
2621
|
+
since?: (string | null);
|
|
2485
2622
|
/**
|
|
2486
2623
|
* Filter by target ID
|
|
2487
2624
|
*/
|
|
@@ -2490,6 +2627,10 @@ type GetAuditEventsData = {
|
|
|
2490
2627
|
* Filter by target type
|
|
2491
2628
|
*/
|
|
2492
2629
|
target_type?: (string | null);
|
|
2630
|
+
/**
|
|
2631
|
+
* Filter: at or before (ISO 8601)
|
|
2632
|
+
*/
|
|
2633
|
+
until?: (string | null);
|
|
2493
2634
|
};
|
|
2494
2635
|
};
|
|
2495
2636
|
type GetAuditEventsResponse = (AuditEventsResponse);
|
|
@@ -2522,6 +2663,14 @@ type ListMinersData = {
|
|
|
2522
2663
|
* Offset for pagination
|
|
2523
2664
|
*/
|
|
2524
2665
|
offset?: number;
|
|
2666
|
+
/**
|
|
2667
|
+
* Last submitted at or after
|
|
2668
|
+
*/
|
|
2669
|
+
since?: (string | null);
|
|
2670
|
+
/**
|
|
2671
|
+
* Last submitted at or before
|
|
2672
|
+
*/
|
|
2673
|
+
until?: (string | null);
|
|
2525
2674
|
};
|
|
2526
2675
|
};
|
|
2527
2676
|
type ListMinersResponse = (AdminMinersResponse);
|
|
@@ -2548,10 +2697,18 @@ type ListAgentVersions1Data = {
|
|
|
2548
2697
|
* Offset for pagination
|
|
2549
2698
|
*/
|
|
2550
2699
|
offset?: number;
|
|
2700
|
+
/**
|
|
2701
|
+
* Filter: created at or after (ISO 8601)
|
|
2702
|
+
*/
|
|
2703
|
+
since?: (string | null);
|
|
2551
2704
|
/**
|
|
2552
2705
|
* Suite ID (defaults to active suite)
|
|
2553
2706
|
*/
|
|
2554
2707
|
suite_id?: (number | null);
|
|
2708
|
+
/**
|
|
2709
|
+
* Filter: created at or before (ISO 8601)
|
|
2710
|
+
*/
|
|
2711
|
+
until?: (string | null);
|
|
2555
2712
|
};
|
|
2556
2713
|
};
|
|
2557
2714
|
type ListAgentVersions1Response = (AdminAgentVersionsResponse);
|
|
@@ -2570,10 +2727,18 @@ type ListEvaluationRunsData = {
|
|
|
2570
2727
|
* Offset for pagination
|
|
2571
2728
|
*/
|
|
2572
2729
|
offset?: number;
|
|
2730
|
+
/**
|
|
2731
|
+
* Filter: created at or after (ISO 8601)
|
|
2732
|
+
*/
|
|
2733
|
+
since?: (string | null);
|
|
2573
2734
|
/**
|
|
2574
2735
|
* Filter by run status (e.g., RUNNING, SUCCESS)
|
|
2575
2736
|
*/
|
|
2576
2737
|
status?: (EvaluationRunStatus | null);
|
|
2738
|
+
/**
|
|
2739
|
+
* Filter: created at or before (ISO 8601)
|
|
2740
|
+
*/
|
|
2741
|
+
until?: (string | null);
|
|
2577
2742
|
/**
|
|
2578
2743
|
* Filter by validator hotkey
|
|
2579
2744
|
*/
|
|
@@ -2582,6 +2747,50 @@ type ListEvaluationRunsData = {
|
|
|
2582
2747
|
};
|
|
2583
2748
|
type ListEvaluationRunsResponse = (AdminEvaluationRunsResponse);
|
|
2584
2749
|
type ListEvaluationRunsError = (HTTPValidationError);
|
|
2750
|
+
type GetValidatorScoresData = {
|
|
2751
|
+
query?: {
|
|
2752
|
+
/**
|
|
2753
|
+
* Only runs after this time
|
|
2754
|
+
*/
|
|
2755
|
+
since?: (string | null);
|
|
2756
|
+
/**
|
|
2757
|
+
* Suite ID (defaults to active suite)
|
|
2758
|
+
*/
|
|
2759
|
+
suite_id?: (number | null);
|
|
2760
|
+
/**
|
|
2761
|
+
* Only runs before this time
|
|
2762
|
+
*/
|
|
2763
|
+
until?: (string | null);
|
|
2764
|
+
};
|
|
2765
|
+
};
|
|
2766
|
+
type GetValidatorScoresResponse = (ValidatorScoresResponse);
|
|
2767
|
+
type GetValidatorScoresError = (HTTPValidationError);
|
|
2768
|
+
type GetAgentVersionVarianceData = {
|
|
2769
|
+
query?: {
|
|
2770
|
+
/**
|
|
2771
|
+
* Number of agent versions
|
|
2772
|
+
*/
|
|
2773
|
+
limit?: number;
|
|
2774
|
+
/**
|
|
2775
|
+
* Only versions after this time
|
|
2776
|
+
*/
|
|
2777
|
+
since?: (string | null);
|
|
2778
|
+
/**
|
|
2779
|
+
* Suite ID (defaults to active suite)
|
|
2780
|
+
*/
|
|
2781
|
+
suite_id?: (number | null);
|
|
2782
|
+
/**
|
|
2783
|
+
* Only versions before this time
|
|
2784
|
+
*/
|
|
2785
|
+
until?: (string | null);
|
|
2786
|
+
/**
|
|
2787
|
+
* Spread threshold for flagging
|
|
2788
|
+
*/
|
|
2789
|
+
variance_threshold?: number;
|
|
2790
|
+
};
|
|
2791
|
+
};
|
|
2792
|
+
type GetAgentVersionVarianceResponse = (AgentVersionVarianceResponse);
|
|
2793
|
+
type GetAgentVersionVarianceError = (HTTPValidationError);
|
|
2585
2794
|
|
|
2586
2795
|
declare const client: _hey_api_client_fetch.Client<Request, Response, unknown, _hey_api_client_fetch.RequestOptions<boolean, string>>;
|
|
2587
2796
|
/**
|
|
@@ -2854,6 +3063,16 @@ declare const listAgentVersions1: <ThrowOnError extends boolean = false>(options
|
|
|
2854
3063
|
* List evaluation runs with optional filtering and pagination.
|
|
2855
3064
|
*/
|
|
2856
3065
|
declare const listEvaluationRuns: <ThrowOnError extends boolean = false>(options?: OptionsLegacyParser<ListEvaluationRunsData, ThrowOnError>) => _hey_api_client_fetch.RequestResult<AdminEvaluationRunsResponse, HTTPValidationError, ThrowOnError>;
|
|
3066
|
+
/**
|
|
3067
|
+
* Aggregated scoring statistics per validator
|
|
3068
|
+
* Compute per-validator scoring statistics from completed runs.
|
|
3069
|
+
*/
|
|
3070
|
+
declare const getValidatorScores: <ThrowOnError extends boolean = false>(options?: OptionsLegacyParser<GetValidatorScoresData, ThrowOnError>) => _hey_api_client_fetch.RequestResult<ValidatorScoresResponse, HTTPValidationError, ThrowOnError>;
|
|
3071
|
+
/**
|
|
3072
|
+
* Score variance across validators for recent agent versions
|
|
3073
|
+
* Find agent versions with high score variance across validators.
|
|
3074
|
+
*/
|
|
3075
|
+
declare const getAgentVersionVariance: <ThrowOnError extends boolean = false>(options?: OptionsLegacyParser<GetAgentVersionVarianceData, ThrowOnError>) => _hey_api_client_fetch.RequestResult<AgentVersionVarianceResponse, HTTPValidationError, ThrowOnError>;
|
|
2857
3076
|
|
|
2858
3077
|
/**
|
|
2859
3078
|
* Auto-generated error code type.
|
|
@@ -3206,4 +3425,4 @@ declare class SessionAuthManager {
|
|
|
3206
3425
|
*/
|
|
3207
3426
|
declare function configureSessionAuth(baseUrl: string, config: SessionAuthConfig): SessionAuthManager;
|
|
3208
3427
|
|
|
3209
|
-
export { type ActivateSuiteData, type ActivateSuiteError, type ActivateSuiteResponse, type ActivateSuiteResponse2, type AdminAgentVersionEntry, type AdminAgentVersionsResponse, type AdminEvaluationRunEntry, type AdminEvaluationRunsResponse, type AdminMinerEntry, type AdminMinersResponse, type AdmissionReason, type AdmissionStatus, type AgentNotFoundError, type AgentPublic, type AgentVersionHistoryEntry, type AgentVersionNotFoundError, type AgentVersionProblemsResponse, type AgentVersionPublic, type AgentVersionState, type AgentVersionStatus, type AlreadyInvalidatedError, type ArtifactDownloadRequest, type ArtifactDownloadResponse, type ArtifactNotFoundError, type ArtifactNotReleasedError, type ArtifactReleaseState, type ArtifactType, type AtCapacityError, type AuditEventEntry, type AuditEventsResponse, type BanMinerData, type BanMinerError, type BanMinerResponse, type BanRequest, type BanResponse, type BanValidatorData, type BanValidatorError, type BanValidatorResponse, type BittensorAuthConfig, type Body_submit_agent, type CachedSession, type CancelAgentVersionData, type CancelAgentVersionError, type CancelAgentVersionResponse, type CancelRequest, type CancelResponse, type ChallengeRequest, type ChallengeResponse, type ChutesAuthStatusResponse, type ClaimWorkData, type ClaimWorkError, type ClaimWorkResponse, type ClaimWorkResponse2, type ClearMinerCooldownData, type ClearMinerCooldownError, type ClearMinerCooldownResponse, type CodeAnalysisError, type CompleteRunData, type CompleteRunError, type CompleteRunRequest, type CompleteRunResponse, type CompleteRunResponse2, type CooldownActiveError, type CreateSessionEndpointData, type CreateSessionEndpointError, type CreateSessionEndpointResponse, type CreateSuiteData, type CreateSuiteError, type CreateSuiteRequest, type CreateSuiteResponse, type CreateSuiteResponse2, type DiscardAgentVersionData, type DiscardAgentVersionError, type DiscardAgentVersionResponse, type DiscardRequest, type DiscardResponse, type ErrorCategory, type EvalRunNotFoundError, type EvaluationRunDetail, type EvaluationRunPublic, type EvaluationRunStatus, type EvaluationRunStatusPublic, type FileTooLargeError, type GetAgentVersionData, type GetAgentVersionError, type GetAgentVersionProblemsData, type GetAgentVersionProblemsError, type GetAgentVersionProblemsResponse, type GetAgentVersionResponse, type GetAgentVersionRunsData, type GetAgentVersionRunsError, type GetAgentVersionRunsResponse, type GetAgentVersionStatusData, type GetAgentVersionStatusError, type GetAgentVersionStatusResponse, type GetArtifactDownloadUrlData, type GetArtifactDownloadUrlError, type GetArtifactDownloadUrlResponse, type GetAuditEventsData, type GetAuditEventsError, type GetAuditEventsResponse, type GetChutesAuthStatusError, type GetChutesAuthStatusResponse, type GetCurrentSuiteError, type GetCurrentSuiteResponse, type GetEvaluationRunData, type GetEvaluationRunError, type GetEvaluationRunResponse, type GetLeaderboardData, type GetLeaderboardError, type GetLeaderboardResponse, type GetOwnedAgentVersionStatusData, type GetOwnedAgentVersionStatusError, type GetOwnedAgentVersionStatusResponse, type GetPendingEvaluationsData, type GetPendingEvaluationsError, type GetPendingEvaluationsResponse, type GetReaperStatsError, type GetReaperStatsResponse, type GetRunningEvaluationsError, type GetRunningEvaluationsResponse, type GetSuiteProblemsData, type GetSuiteProblemsError, type GetSuiteProblemsResponse, type GetTopAgentError, type GetTopAgentResponse, type GetValidatorsError, type GetValidatorsResponse, type HTTPValidationError, type HealthCheckError, type HealthCheckResponse, type HeartbeatData, type HeartbeatError, type HeartbeatRequest, type HeartbeatResponse, type HeartbeatResponse2, type InvalidAgentNameError, type InvalidArtifactTypeError, type InvalidFileError, type InvalidProblemIdError, type InvalidateEvaluationRunData, type InvalidateEvaluationRunError, type InvalidateEvaluationRunResponse, type InvalidateRunRequest, type JoinWaitlistData, type JoinWaitlistError, type JoinWaitlistResponse, type LeaderboardEntry, type LeaderboardResponse, type LeaseExpiredError, type ListAgentVersions1Data, type ListAgentVersions1Error, type ListAgentVersions1Response, type ListAgentVersionsData, type ListAgentVersionsError, type ListAgentVersionsResponse, type ListEvaluationRunsData, type ListEvaluationRunsError, type ListEvaluationRunsResponse, type ListMinerAgentsError, type ListMinerAgentsResponse, type ListMinersData, type ListMinersError, type ListMinersResponse, type LogoutData, type LogoutError, type LogoutResponse, type LogoutResponse2, type MinerAgentsResponse, type MinerNotFoundError, type MissingParameterError, type MissingScoreError, type NoActiveSuiteError, type NotRunOwnerError, type OroErrorCode, type PendingEvaluation, type PendingEvaluationSummary, type PendingEvaluationsResponse, type PresignUploadData, type PresignUploadError, type PresignUploadRequest, type PresignUploadResponse, type PresignUploadResponse2, type ProblemNotFoundError, type ProblemProgressEntry, type ProblemProgressUpdate, type ProblemPublic, type ProblemStatus, type ProgressUpdateRequest, type ProgressUpdateResponse, type RateLimitExceededError, type ReaperStatsResponse, type ReevaluateAgentVersionData, type ReevaluateAgentVersionError, type ReevaluateAgentVersionResponse, type ReevaluateRequest, type ReevaluateResponse, type ReinstateAgentVersionData, type ReinstateAgentVersionError, type ReinstateAgentVersionResponse, type ReinstateRequest, type RequestChallengeData, type RequestChallengeError, type RequestChallengeResponse, type RetryConfig, type RetryContext, type RunAlreadyCompleteError, type RunningEvaluation, type ScoreBelowThresholdError, type SessionAuthConfig, SessionAuthManager, type SessionInfo, type SessionRequest, type SessionResponse, type SetTopAgentData, type SetTopAgentError, type SetTopAgentResponse, type SetTopRequest, type SetTopResponse, type StoreChutesTokenData, type StoreChutesTokenError, type StoreChutesTokenRequest, type StoreChutesTokenResponse, type SubmitAgentData, type SubmitAgentError, type SubmitAgentResponse, type SubmitAgentResponse2, type SuiteNotFoundError, type SuitePublic, type SuiteWithProblemsResponse, type TerminalStatus, type TopAgentResponse, type UnbanMinerData, type UnbanMinerError, type UnbanMinerResponse, type UnbanValidatorData, type UnbanValidatorError, type UnbanValidatorResponse, type UpdateProgressData, type UpdateProgressError, type UpdateProgressResponse, type ValidationError, type ValidatorCurrentAgent, type ValidatorNotFoundError, type ValidatorProblemResult, type ValidatorPublic, type ValidatorStatus, type WaitlistSignupRequest, type WaitlistSignupResponse, type WorkItemStatus, activateSuite, banMiner, banValidator, cancelAgentVersion, claimWork, classifyError, classifyStatus, clearMinerCooldown, client, completeRun, computeDelay, configureBittensorAuth, configurePublicClient, configureSessionAuth, createRetryFetch, createSessionEndpoint, createSuite, discardAgentVersion, generateAuthHeaders, getAgentVersion, getAgentVersionProblems, getAgentVersionRuns, getAgentVersionStatus, getArtifactDownloadUrl, getAuditEvents, getChutesAuthStatus, getCurrentSuite, getErrorCode, getErrorDetail, getEvaluationRun, getLeaderboard, getOwnedAgentVersionStatus, getPendingEvaluations, getReaperStats, getRunningEvaluations, getSuiteProblems, getTopAgent, getValidators, hasDetail, hasErrorCode, healthCheck, heartbeat, invalidateEvaluationRun, isTransient, isTransientError, joinWaitlist, listAgentVersions, listAgentVersions1, listEvaluationRuns, listMinerAgents, listMiners, logout, parseRetryAfter, presignUpload, reevaluateAgentVersion, reinstateAgentVersion, requestChallenge, setTopAgent, storeChutesToken, submitAgent, unbanMiner, unbanValidator, updateProgress };
|
|
3428
|
+
export { type ActivateSuiteData, type ActivateSuiteError, type ActivateSuiteResponse, type ActivateSuiteResponse2, type AdminAgentVersionEntry, type AdminAgentVersionsResponse, type AdminEvaluationRunEntry, type AdminEvaluationRunsResponse, type AdminMinerEntry, type AdminMinersResponse, type AdmissionReason, type AdmissionStatus, type AgentNotFoundError, type AgentPublic, type AgentVersionHistoryEntry, type AgentVersionNotFoundError, type AgentVersionProblemsResponse, type AgentVersionPublic, type AgentVersionScoreEntry, type AgentVersionState, type AgentVersionStatus, type AgentVersionVariance, type AgentVersionVarianceResponse, type AlreadyInvalidatedError, type ArtifactDownloadRequest, type ArtifactDownloadResponse, type ArtifactNotFoundError, type ArtifactNotReleasedError, type ArtifactReleaseState, type ArtifactType, type AtCapacityError, type AuditEventEntry, type AuditEventsResponse, type BanMinerData, type BanMinerError, type BanMinerResponse, type BanRequest, type BanResponse, type BanValidatorData, type BanValidatorError, type BanValidatorResponse, type BittensorAuthConfig, type Body_submit_agent, type CachedSession, type CancelAgentVersionData, type CancelAgentVersionError, type CancelAgentVersionResponse, type CancelRequest, type CancelResponse, type ChallengeRequest, type ChallengeResponse, type ChutesAuthStatusResponse, type ClaimWorkData, type ClaimWorkError, type ClaimWorkResponse, type ClaimWorkResponse2, type ClearMinerCooldownData, type ClearMinerCooldownError, type ClearMinerCooldownResponse, type CodeAnalysisError, type CompleteRunData, type CompleteRunError, type CompleteRunRequest, type CompleteRunResponse, type CompleteRunResponse2, type CooldownActiveError, type CreateSessionEndpointData, type CreateSessionEndpointError, type CreateSessionEndpointResponse, type CreateSuiteData, type CreateSuiteError, type CreateSuiteRequest, type CreateSuiteResponse, type CreateSuiteResponse2, type DiscardAgentVersionData, type DiscardAgentVersionError, type DiscardAgentVersionResponse, type DiscardRequest, type DiscardResponse, type ErrorCategory, type EvalRunNotFoundError, type EvaluationRunDetail, type EvaluationRunPublic, type EvaluationRunStatus, type EvaluationRunStatusPublic, type FileTooLargeError, type GetAgentVersionData, type GetAgentVersionError, type GetAgentVersionProblemsData, type GetAgentVersionProblemsError, type GetAgentVersionProblemsResponse, type GetAgentVersionResponse, type GetAgentVersionRunsData, type GetAgentVersionRunsError, type GetAgentVersionRunsResponse, type GetAgentVersionStatusData, type GetAgentVersionStatusError, type GetAgentVersionStatusResponse, type GetAgentVersionVarianceData, type GetAgentVersionVarianceError, type GetAgentVersionVarianceResponse, type GetArtifactDownloadUrlData, type GetArtifactDownloadUrlError, type GetArtifactDownloadUrlResponse, type GetAuditEventsData, type GetAuditEventsError, type GetAuditEventsResponse, type GetChutesAuthStatusError, type GetChutesAuthStatusResponse, type GetCurrentSuiteError, type GetCurrentSuiteResponse, type GetEvaluationRunData, type GetEvaluationRunError, type GetEvaluationRunResponse, type GetLeaderboardData, type GetLeaderboardError, type GetLeaderboardResponse, type GetOwnedAgentVersionStatusData, type GetOwnedAgentVersionStatusError, type GetOwnedAgentVersionStatusResponse, type GetPendingEvaluationsData, type GetPendingEvaluationsError, type GetPendingEvaluationsResponse, type GetReaperStatsError, type GetReaperStatsResponse, type GetRunningEvaluationsError, type GetRunningEvaluationsResponse, type GetSuiteProblemsData, type GetSuiteProblemsError, type GetSuiteProblemsResponse, type GetTopAgentError, type GetTopAgentResponse, type GetValidatorScoresData, type GetValidatorScoresError, type GetValidatorScoresResponse, type GetValidatorsError, type GetValidatorsResponse, type HTTPValidationError, type HealthCheckError, type HealthCheckResponse, type HeartbeatData, type HeartbeatError, type HeartbeatRequest, type HeartbeatResponse, type HeartbeatResponse2, type InvalidAgentNameError, type InvalidArtifactTypeError, type InvalidFileError, type InvalidProblemIdError, type InvalidateEvaluationRunData, type InvalidateEvaluationRunError, type InvalidateEvaluationRunResponse, type InvalidateRunRequest, type JoinWaitlistData, type JoinWaitlistError, type JoinWaitlistResponse, type LeaderboardEntry, type LeaderboardResponse, type LeaseExpiredError, type ListAgentVersions1Data, type ListAgentVersions1Error, type ListAgentVersions1Response, type ListAgentVersionsData, type ListAgentVersionsError, type ListAgentVersionsResponse, type ListEvaluationRunsData, type ListEvaluationRunsError, type ListEvaluationRunsResponse, type ListMinerAgentsError, type ListMinerAgentsResponse, type ListMinersData, type ListMinersError, type ListMinersResponse, type LogoutData, type LogoutError, type LogoutResponse, type LogoutResponse2, type MinerAgentsResponse, type MinerNotFoundError, type MissingParameterError, type MissingScoreError, type NoActiveSuiteError, type NotRunOwnerError, type OroErrorCode, type PendingEvaluation, type PendingEvaluationSummary, type PendingEvaluationsResponse, type PresignUploadData, type PresignUploadError, type PresignUploadRequest, type PresignUploadResponse, type PresignUploadResponse2, type ProblemNotFoundError, type ProblemProgressEntry, type ProblemProgressUpdate, type ProblemPublic, type ProblemStatus, type ProgressUpdateRequest, type ProgressUpdateResponse, type RateLimitExceededError, type ReaperStatsResponse, type ReevaluateAgentVersionData, type ReevaluateAgentVersionError, type ReevaluateAgentVersionResponse, type ReevaluateRequest, type ReevaluateResponse, type ReinstateAgentVersionData, type ReinstateAgentVersionError, type ReinstateAgentVersionResponse, type ReinstateRequest, type RequestChallengeData, type RequestChallengeError, type RequestChallengeResponse, type RetryConfig, type RetryContext, type RunAlreadyCompleteError, type RunningEvaluation, type ScoreBelowThresholdError, type SessionAuthConfig, SessionAuthManager, type SessionInfo, type SessionRequest, type SessionResponse, type SetTopAgentData, type SetTopAgentError, type SetTopAgentResponse, type SetTopRequest, type SetTopResponse, type StoreChutesTokenData, type StoreChutesTokenError, type StoreChutesTokenRequest, type StoreChutesTokenResponse, type SubmitAgentData, type SubmitAgentError, type SubmitAgentResponse, type SubmitAgentResponse2, type SuiteNotFoundError, type SuitePublic, type SuiteWithProblemsResponse, type TerminalStatus, type TopAgentResponse, type UnbanMinerData, type UnbanMinerError, type UnbanMinerResponse, type UnbanValidatorData, type UnbanValidatorError, type UnbanValidatorResponse, type UpdateProgressData, type UpdateProgressError, type UpdateProgressResponse, type ValidationError, type ValidatorCurrentAgent, type ValidatorNotFoundError, type ValidatorProblemResult, type ValidatorPublic, type ValidatorScoreSummary, type ValidatorScoresResponse, type ValidatorStatus, type WaitlistSignupRequest, type WaitlistSignupResponse, type WorkItemStatus, activateSuite, banMiner, banValidator, cancelAgentVersion, claimWork, classifyError, classifyStatus, clearMinerCooldown, client, completeRun, computeDelay, configureBittensorAuth, configurePublicClient, configureSessionAuth, createRetryFetch, createSessionEndpoint, createSuite, discardAgentVersion, generateAuthHeaders, getAgentVersion, getAgentVersionProblems, getAgentVersionRuns, getAgentVersionStatus, getAgentVersionVariance, getArtifactDownloadUrl, getAuditEvents, getChutesAuthStatus, getCurrentSuite, getErrorCode, getErrorDetail, getEvaluationRun, getLeaderboard, getOwnedAgentVersionStatus, getPendingEvaluations, getReaperStats, getRunningEvaluations, getSuiteProblems, getTopAgent, getValidatorScores, getValidators, hasDetail, hasErrorCode, healthCheck, heartbeat, invalidateEvaluationRun, isTransient, isTransientError, joinWaitlist, listAgentVersions, listAgentVersions1, listEvaluationRuns, listMinerAgents, listMiners, logout, parseRetryAfter, presignUpload, reevaluateAgentVersion, reinstateAgentVersion, requestChallenge, setTopAgent, storeChutesToken, submitAgent, unbanMiner, unbanValidator, updateProgress };
|
package/dist/index.d.ts
CHANGED
|
@@ -370,6 +370,23 @@ type AgentVersionPublic = {
|
|
|
370
370
|
*/
|
|
371
371
|
latest_final_score?: (number | null);
|
|
372
372
|
};
|
|
373
|
+
/**
|
|
374
|
+
* Per-validator score for an agent version.
|
|
375
|
+
*/
|
|
376
|
+
type AgentVersionScoreEntry = {
|
|
377
|
+
/**
|
|
378
|
+
* Validator hotkey
|
|
379
|
+
*/
|
|
380
|
+
validator_hotkey: string;
|
|
381
|
+
/**
|
|
382
|
+
* Score from this validator
|
|
383
|
+
*/
|
|
384
|
+
score: number;
|
|
385
|
+
/**
|
|
386
|
+
* Evaluation run ID
|
|
387
|
+
*/
|
|
388
|
+
run_id: string;
|
|
389
|
+
};
|
|
373
390
|
/**
|
|
374
391
|
* State of an agent version evaluation.
|
|
375
392
|
*/
|
|
@@ -441,6 +458,64 @@ type AgentVersionStatus = {
|
|
|
441
458
|
[key: string]: (number);
|
|
442
459
|
} | null);
|
|
443
460
|
};
|
|
461
|
+
/**
|
|
462
|
+
* Score variance across validators for a single agent version.
|
|
463
|
+
*/
|
|
464
|
+
type AgentVersionVariance = {
|
|
465
|
+
/**
|
|
466
|
+
* Agent version ID
|
|
467
|
+
*/
|
|
468
|
+
agent_version_id: string;
|
|
469
|
+
/**
|
|
470
|
+
* Agent name
|
|
471
|
+
*/
|
|
472
|
+
agent_name: string;
|
|
473
|
+
/**
|
|
474
|
+
* Miner hotkey
|
|
475
|
+
*/
|
|
476
|
+
miner_hotkey: string;
|
|
477
|
+
/**
|
|
478
|
+
* Number of validators that scored this version
|
|
479
|
+
*/
|
|
480
|
+
validator_count: number;
|
|
481
|
+
/**
|
|
482
|
+
* Mean score
|
|
483
|
+
*/
|
|
484
|
+
avg_score: number;
|
|
485
|
+
/**
|
|
486
|
+
* Minimum score
|
|
487
|
+
*/
|
|
488
|
+
min_score: number;
|
|
489
|
+
/**
|
|
490
|
+
* Maximum score
|
|
491
|
+
*/
|
|
492
|
+
max_score: number;
|
|
493
|
+
/**
|
|
494
|
+
* max_score - min_score
|
|
495
|
+
*/
|
|
496
|
+
spread: number;
|
|
497
|
+
/**
|
|
498
|
+
* True if spread exceeds threshold (default 10%)
|
|
499
|
+
*/
|
|
500
|
+
is_high_variance: boolean;
|
|
501
|
+
/**
|
|
502
|
+
* Individual validator scores
|
|
503
|
+
*/
|
|
504
|
+
per_validator: Array<AgentVersionScoreEntry>;
|
|
505
|
+
};
|
|
506
|
+
/**
|
|
507
|
+
* Response for agent version score variance analytics.
|
|
508
|
+
*/
|
|
509
|
+
type AgentVersionVarianceResponse = {
|
|
510
|
+
/**
|
|
511
|
+
* Per-version variance data
|
|
512
|
+
*/
|
|
513
|
+
agent_versions: Array<AgentVersionVariance>;
|
|
514
|
+
/**
|
|
515
|
+
* Spread threshold used for flagging
|
|
516
|
+
*/
|
|
517
|
+
variance_threshold: number;
|
|
518
|
+
};
|
|
444
519
|
/**
|
|
445
520
|
* 409 - Resource is already invalidated.
|
|
446
521
|
*/
|
|
@@ -2078,6 +2153,64 @@ type ValidatorPublic = {
|
|
|
2078
2153
|
*/
|
|
2079
2154
|
identity_description?: (string | null);
|
|
2080
2155
|
};
|
|
2156
|
+
/**
|
|
2157
|
+
* Response for validator scoring analytics.
|
|
2158
|
+
*/
|
|
2159
|
+
type ValidatorScoresResponse = {
|
|
2160
|
+
/**
|
|
2161
|
+
* Per-validator summaries
|
|
2162
|
+
*/
|
|
2163
|
+
validators: Array<ValidatorScoreSummary>;
|
|
2164
|
+
/**
|
|
2165
|
+
* Global average score across all validators
|
|
2166
|
+
*/
|
|
2167
|
+
global_avg_score: number;
|
|
2168
|
+
/**
|
|
2169
|
+
* Global standard deviation
|
|
2170
|
+
*/
|
|
2171
|
+
global_stddev: number;
|
|
2172
|
+
};
|
|
2173
|
+
/**
|
|
2174
|
+
* Aggregated scoring stats for a single validator.
|
|
2175
|
+
*/
|
|
2176
|
+
type ValidatorScoreSummary = {
|
|
2177
|
+
/**
|
|
2178
|
+
* Validator hotkey
|
|
2179
|
+
*/
|
|
2180
|
+
validator_hotkey: string;
|
|
2181
|
+
/**
|
|
2182
|
+
* Total completed runs
|
|
2183
|
+
*/
|
|
2184
|
+
total_runs: number;
|
|
2185
|
+
/**
|
|
2186
|
+
* Mean score across runs
|
|
2187
|
+
*/
|
|
2188
|
+
avg_score: number;
|
|
2189
|
+
/**
|
|
2190
|
+
* Median score
|
|
2191
|
+
*/
|
|
2192
|
+
median_score: number;
|
|
2193
|
+
/**
|
|
2194
|
+
* Standard deviation of scores
|
|
2195
|
+
*/
|
|
2196
|
+
stddev_score: number;
|
|
2197
|
+
/**
|
|
2198
|
+
* Minimum score
|
|
2199
|
+
*/
|
|
2200
|
+
min_score: number;
|
|
2201
|
+
/**
|
|
2202
|
+
* Maximum score
|
|
2203
|
+
*/
|
|
2204
|
+
max_score: number;
|
|
2205
|
+
/**
|
|
2206
|
+
* Percentage deviation from the global average (negative = below)
|
|
2207
|
+
*/
|
|
2208
|
+
deviation_from_global: number;
|
|
2209
|
+
/**
|
|
2210
|
+
* True if deviation exceeds 1.5 standard deviations from global mean
|
|
2211
|
+
*/
|
|
2212
|
+
is_outlier: boolean;
|
|
2213
|
+
};
|
|
2081
2214
|
/**
|
|
2082
2215
|
* Status of a validator.
|
|
2083
2216
|
*/
|
|
@@ -2482,6 +2615,10 @@ type GetAuditEventsData = {
|
|
|
2482
2615
|
* Offset for pagination
|
|
2483
2616
|
*/
|
|
2484
2617
|
offset?: number;
|
|
2618
|
+
/**
|
|
2619
|
+
* Filter: at or after (ISO 8601)
|
|
2620
|
+
*/
|
|
2621
|
+
since?: (string | null);
|
|
2485
2622
|
/**
|
|
2486
2623
|
* Filter by target ID
|
|
2487
2624
|
*/
|
|
@@ -2490,6 +2627,10 @@ type GetAuditEventsData = {
|
|
|
2490
2627
|
* Filter by target type
|
|
2491
2628
|
*/
|
|
2492
2629
|
target_type?: (string | null);
|
|
2630
|
+
/**
|
|
2631
|
+
* Filter: at or before (ISO 8601)
|
|
2632
|
+
*/
|
|
2633
|
+
until?: (string | null);
|
|
2493
2634
|
};
|
|
2494
2635
|
};
|
|
2495
2636
|
type GetAuditEventsResponse = (AuditEventsResponse);
|
|
@@ -2522,6 +2663,14 @@ type ListMinersData = {
|
|
|
2522
2663
|
* Offset for pagination
|
|
2523
2664
|
*/
|
|
2524
2665
|
offset?: number;
|
|
2666
|
+
/**
|
|
2667
|
+
* Last submitted at or after
|
|
2668
|
+
*/
|
|
2669
|
+
since?: (string | null);
|
|
2670
|
+
/**
|
|
2671
|
+
* Last submitted at or before
|
|
2672
|
+
*/
|
|
2673
|
+
until?: (string | null);
|
|
2525
2674
|
};
|
|
2526
2675
|
};
|
|
2527
2676
|
type ListMinersResponse = (AdminMinersResponse);
|
|
@@ -2548,10 +2697,18 @@ type ListAgentVersions1Data = {
|
|
|
2548
2697
|
* Offset for pagination
|
|
2549
2698
|
*/
|
|
2550
2699
|
offset?: number;
|
|
2700
|
+
/**
|
|
2701
|
+
* Filter: created at or after (ISO 8601)
|
|
2702
|
+
*/
|
|
2703
|
+
since?: (string | null);
|
|
2551
2704
|
/**
|
|
2552
2705
|
* Suite ID (defaults to active suite)
|
|
2553
2706
|
*/
|
|
2554
2707
|
suite_id?: (number | null);
|
|
2708
|
+
/**
|
|
2709
|
+
* Filter: created at or before (ISO 8601)
|
|
2710
|
+
*/
|
|
2711
|
+
until?: (string | null);
|
|
2555
2712
|
};
|
|
2556
2713
|
};
|
|
2557
2714
|
type ListAgentVersions1Response = (AdminAgentVersionsResponse);
|
|
@@ -2570,10 +2727,18 @@ type ListEvaluationRunsData = {
|
|
|
2570
2727
|
* Offset for pagination
|
|
2571
2728
|
*/
|
|
2572
2729
|
offset?: number;
|
|
2730
|
+
/**
|
|
2731
|
+
* Filter: created at or after (ISO 8601)
|
|
2732
|
+
*/
|
|
2733
|
+
since?: (string | null);
|
|
2573
2734
|
/**
|
|
2574
2735
|
* Filter by run status (e.g., RUNNING, SUCCESS)
|
|
2575
2736
|
*/
|
|
2576
2737
|
status?: (EvaluationRunStatus | null);
|
|
2738
|
+
/**
|
|
2739
|
+
* Filter: created at or before (ISO 8601)
|
|
2740
|
+
*/
|
|
2741
|
+
until?: (string | null);
|
|
2577
2742
|
/**
|
|
2578
2743
|
* Filter by validator hotkey
|
|
2579
2744
|
*/
|
|
@@ -2582,6 +2747,50 @@ type ListEvaluationRunsData = {
|
|
|
2582
2747
|
};
|
|
2583
2748
|
type ListEvaluationRunsResponse = (AdminEvaluationRunsResponse);
|
|
2584
2749
|
type ListEvaluationRunsError = (HTTPValidationError);
|
|
2750
|
+
type GetValidatorScoresData = {
|
|
2751
|
+
query?: {
|
|
2752
|
+
/**
|
|
2753
|
+
* Only runs after this time
|
|
2754
|
+
*/
|
|
2755
|
+
since?: (string | null);
|
|
2756
|
+
/**
|
|
2757
|
+
* Suite ID (defaults to active suite)
|
|
2758
|
+
*/
|
|
2759
|
+
suite_id?: (number | null);
|
|
2760
|
+
/**
|
|
2761
|
+
* Only runs before this time
|
|
2762
|
+
*/
|
|
2763
|
+
until?: (string | null);
|
|
2764
|
+
};
|
|
2765
|
+
};
|
|
2766
|
+
type GetValidatorScoresResponse = (ValidatorScoresResponse);
|
|
2767
|
+
type GetValidatorScoresError = (HTTPValidationError);
|
|
2768
|
+
type GetAgentVersionVarianceData = {
|
|
2769
|
+
query?: {
|
|
2770
|
+
/**
|
|
2771
|
+
* Number of agent versions
|
|
2772
|
+
*/
|
|
2773
|
+
limit?: number;
|
|
2774
|
+
/**
|
|
2775
|
+
* Only versions after this time
|
|
2776
|
+
*/
|
|
2777
|
+
since?: (string | null);
|
|
2778
|
+
/**
|
|
2779
|
+
* Suite ID (defaults to active suite)
|
|
2780
|
+
*/
|
|
2781
|
+
suite_id?: (number | null);
|
|
2782
|
+
/**
|
|
2783
|
+
* Only versions before this time
|
|
2784
|
+
*/
|
|
2785
|
+
until?: (string | null);
|
|
2786
|
+
/**
|
|
2787
|
+
* Spread threshold for flagging
|
|
2788
|
+
*/
|
|
2789
|
+
variance_threshold?: number;
|
|
2790
|
+
};
|
|
2791
|
+
};
|
|
2792
|
+
type GetAgentVersionVarianceResponse = (AgentVersionVarianceResponse);
|
|
2793
|
+
type GetAgentVersionVarianceError = (HTTPValidationError);
|
|
2585
2794
|
|
|
2586
2795
|
declare const client: _hey_api_client_fetch.Client<Request, Response, unknown, _hey_api_client_fetch.RequestOptions<boolean, string>>;
|
|
2587
2796
|
/**
|
|
@@ -2854,6 +3063,16 @@ declare const listAgentVersions1: <ThrowOnError extends boolean = false>(options
|
|
|
2854
3063
|
* List evaluation runs with optional filtering and pagination.
|
|
2855
3064
|
*/
|
|
2856
3065
|
declare const listEvaluationRuns: <ThrowOnError extends boolean = false>(options?: OptionsLegacyParser<ListEvaluationRunsData, ThrowOnError>) => _hey_api_client_fetch.RequestResult<AdminEvaluationRunsResponse, HTTPValidationError, ThrowOnError>;
|
|
3066
|
+
/**
|
|
3067
|
+
* Aggregated scoring statistics per validator
|
|
3068
|
+
* Compute per-validator scoring statistics from completed runs.
|
|
3069
|
+
*/
|
|
3070
|
+
declare const getValidatorScores: <ThrowOnError extends boolean = false>(options?: OptionsLegacyParser<GetValidatorScoresData, ThrowOnError>) => _hey_api_client_fetch.RequestResult<ValidatorScoresResponse, HTTPValidationError, ThrowOnError>;
|
|
3071
|
+
/**
|
|
3072
|
+
* Score variance across validators for recent agent versions
|
|
3073
|
+
* Find agent versions with high score variance across validators.
|
|
3074
|
+
*/
|
|
3075
|
+
declare const getAgentVersionVariance: <ThrowOnError extends boolean = false>(options?: OptionsLegacyParser<GetAgentVersionVarianceData, ThrowOnError>) => _hey_api_client_fetch.RequestResult<AgentVersionVarianceResponse, HTTPValidationError, ThrowOnError>;
|
|
2857
3076
|
|
|
2858
3077
|
/**
|
|
2859
3078
|
* Auto-generated error code type.
|
|
@@ -3206,4 +3425,4 @@ declare class SessionAuthManager {
|
|
|
3206
3425
|
*/
|
|
3207
3426
|
declare function configureSessionAuth(baseUrl: string, config: SessionAuthConfig): SessionAuthManager;
|
|
3208
3427
|
|
|
3209
|
-
export { type ActivateSuiteData, type ActivateSuiteError, type ActivateSuiteResponse, type ActivateSuiteResponse2, type AdminAgentVersionEntry, type AdminAgentVersionsResponse, type AdminEvaluationRunEntry, type AdminEvaluationRunsResponse, type AdminMinerEntry, type AdminMinersResponse, type AdmissionReason, type AdmissionStatus, type AgentNotFoundError, type AgentPublic, type AgentVersionHistoryEntry, type AgentVersionNotFoundError, type AgentVersionProblemsResponse, type AgentVersionPublic, type AgentVersionState, type AgentVersionStatus, type AlreadyInvalidatedError, type ArtifactDownloadRequest, type ArtifactDownloadResponse, type ArtifactNotFoundError, type ArtifactNotReleasedError, type ArtifactReleaseState, type ArtifactType, type AtCapacityError, type AuditEventEntry, type AuditEventsResponse, type BanMinerData, type BanMinerError, type BanMinerResponse, type BanRequest, type BanResponse, type BanValidatorData, type BanValidatorError, type BanValidatorResponse, type BittensorAuthConfig, type Body_submit_agent, type CachedSession, type CancelAgentVersionData, type CancelAgentVersionError, type CancelAgentVersionResponse, type CancelRequest, type CancelResponse, type ChallengeRequest, type ChallengeResponse, type ChutesAuthStatusResponse, type ClaimWorkData, type ClaimWorkError, type ClaimWorkResponse, type ClaimWorkResponse2, type ClearMinerCooldownData, type ClearMinerCooldownError, type ClearMinerCooldownResponse, type CodeAnalysisError, type CompleteRunData, type CompleteRunError, type CompleteRunRequest, type CompleteRunResponse, type CompleteRunResponse2, type CooldownActiveError, type CreateSessionEndpointData, type CreateSessionEndpointError, type CreateSessionEndpointResponse, type CreateSuiteData, type CreateSuiteError, type CreateSuiteRequest, type CreateSuiteResponse, type CreateSuiteResponse2, type DiscardAgentVersionData, type DiscardAgentVersionError, type DiscardAgentVersionResponse, type DiscardRequest, type DiscardResponse, type ErrorCategory, type EvalRunNotFoundError, type EvaluationRunDetail, type EvaluationRunPublic, type EvaluationRunStatus, type EvaluationRunStatusPublic, type FileTooLargeError, type GetAgentVersionData, type GetAgentVersionError, type GetAgentVersionProblemsData, type GetAgentVersionProblemsError, type GetAgentVersionProblemsResponse, type GetAgentVersionResponse, type GetAgentVersionRunsData, type GetAgentVersionRunsError, type GetAgentVersionRunsResponse, type GetAgentVersionStatusData, type GetAgentVersionStatusError, type GetAgentVersionStatusResponse, type GetArtifactDownloadUrlData, type GetArtifactDownloadUrlError, type GetArtifactDownloadUrlResponse, type GetAuditEventsData, type GetAuditEventsError, type GetAuditEventsResponse, type GetChutesAuthStatusError, type GetChutesAuthStatusResponse, type GetCurrentSuiteError, type GetCurrentSuiteResponse, type GetEvaluationRunData, type GetEvaluationRunError, type GetEvaluationRunResponse, type GetLeaderboardData, type GetLeaderboardError, type GetLeaderboardResponse, type GetOwnedAgentVersionStatusData, type GetOwnedAgentVersionStatusError, type GetOwnedAgentVersionStatusResponse, type GetPendingEvaluationsData, type GetPendingEvaluationsError, type GetPendingEvaluationsResponse, type GetReaperStatsError, type GetReaperStatsResponse, type GetRunningEvaluationsError, type GetRunningEvaluationsResponse, type GetSuiteProblemsData, type GetSuiteProblemsError, type GetSuiteProblemsResponse, type GetTopAgentError, type GetTopAgentResponse, type GetValidatorsError, type GetValidatorsResponse, type HTTPValidationError, type HealthCheckError, type HealthCheckResponse, type HeartbeatData, type HeartbeatError, type HeartbeatRequest, type HeartbeatResponse, type HeartbeatResponse2, type InvalidAgentNameError, type InvalidArtifactTypeError, type InvalidFileError, type InvalidProblemIdError, type InvalidateEvaluationRunData, type InvalidateEvaluationRunError, type InvalidateEvaluationRunResponse, type InvalidateRunRequest, type JoinWaitlistData, type JoinWaitlistError, type JoinWaitlistResponse, type LeaderboardEntry, type LeaderboardResponse, type LeaseExpiredError, type ListAgentVersions1Data, type ListAgentVersions1Error, type ListAgentVersions1Response, type ListAgentVersionsData, type ListAgentVersionsError, type ListAgentVersionsResponse, type ListEvaluationRunsData, type ListEvaluationRunsError, type ListEvaluationRunsResponse, type ListMinerAgentsError, type ListMinerAgentsResponse, type ListMinersData, type ListMinersError, type ListMinersResponse, type LogoutData, type LogoutError, type LogoutResponse, type LogoutResponse2, type MinerAgentsResponse, type MinerNotFoundError, type MissingParameterError, type MissingScoreError, type NoActiveSuiteError, type NotRunOwnerError, type OroErrorCode, type PendingEvaluation, type PendingEvaluationSummary, type PendingEvaluationsResponse, type PresignUploadData, type PresignUploadError, type PresignUploadRequest, type PresignUploadResponse, type PresignUploadResponse2, type ProblemNotFoundError, type ProblemProgressEntry, type ProblemProgressUpdate, type ProblemPublic, type ProblemStatus, type ProgressUpdateRequest, type ProgressUpdateResponse, type RateLimitExceededError, type ReaperStatsResponse, type ReevaluateAgentVersionData, type ReevaluateAgentVersionError, type ReevaluateAgentVersionResponse, type ReevaluateRequest, type ReevaluateResponse, type ReinstateAgentVersionData, type ReinstateAgentVersionError, type ReinstateAgentVersionResponse, type ReinstateRequest, type RequestChallengeData, type RequestChallengeError, type RequestChallengeResponse, type RetryConfig, type RetryContext, type RunAlreadyCompleteError, type RunningEvaluation, type ScoreBelowThresholdError, type SessionAuthConfig, SessionAuthManager, type SessionInfo, type SessionRequest, type SessionResponse, type SetTopAgentData, type SetTopAgentError, type SetTopAgentResponse, type SetTopRequest, type SetTopResponse, type StoreChutesTokenData, type StoreChutesTokenError, type StoreChutesTokenRequest, type StoreChutesTokenResponse, type SubmitAgentData, type SubmitAgentError, type SubmitAgentResponse, type SubmitAgentResponse2, type SuiteNotFoundError, type SuitePublic, type SuiteWithProblemsResponse, type TerminalStatus, type TopAgentResponse, type UnbanMinerData, type UnbanMinerError, type UnbanMinerResponse, type UnbanValidatorData, type UnbanValidatorError, type UnbanValidatorResponse, type UpdateProgressData, type UpdateProgressError, type UpdateProgressResponse, type ValidationError, type ValidatorCurrentAgent, type ValidatorNotFoundError, type ValidatorProblemResult, type ValidatorPublic, type ValidatorStatus, type WaitlistSignupRequest, type WaitlistSignupResponse, type WorkItemStatus, activateSuite, banMiner, banValidator, cancelAgentVersion, claimWork, classifyError, classifyStatus, clearMinerCooldown, client, completeRun, computeDelay, configureBittensorAuth, configurePublicClient, configureSessionAuth, createRetryFetch, createSessionEndpoint, createSuite, discardAgentVersion, generateAuthHeaders, getAgentVersion, getAgentVersionProblems, getAgentVersionRuns, getAgentVersionStatus, getArtifactDownloadUrl, getAuditEvents, getChutesAuthStatus, getCurrentSuite, getErrorCode, getErrorDetail, getEvaluationRun, getLeaderboard, getOwnedAgentVersionStatus, getPendingEvaluations, getReaperStats, getRunningEvaluations, getSuiteProblems, getTopAgent, getValidators, hasDetail, hasErrorCode, healthCheck, heartbeat, invalidateEvaluationRun, isTransient, isTransientError, joinWaitlist, listAgentVersions, listAgentVersions1, listEvaluationRuns, listMinerAgents, listMiners, logout, parseRetryAfter, presignUpload, reevaluateAgentVersion, reinstateAgentVersion, requestChallenge, setTopAgent, storeChutesToken, submitAgent, unbanMiner, unbanValidator, updateProgress };
|
|
3428
|
+
export { type ActivateSuiteData, type ActivateSuiteError, type ActivateSuiteResponse, type ActivateSuiteResponse2, type AdminAgentVersionEntry, type AdminAgentVersionsResponse, type AdminEvaluationRunEntry, type AdminEvaluationRunsResponse, type AdminMinerEntry, type AdminMinersResponse, type AdmissionReason, type AdmissionStatus, type AgentNotFoundError, type AgentPublic, type AgentVersionHistoryEntry, type AgentVersionNotFoundError, type AgentVersionProblemsResponse, type AgentVersionPublic, type AgentVersionScoreEntry, type AgentVersionState, type AgentVersionStatus, type AgentVersionVariance, type AgentVersionVarianceResponse, type AlreadyInvalidatedError, type ArtifactDownloadRequest, type ArtifactDownloadResponse, type ArtifactNotFoundError, type ArtifactNotReleasedError, type ArtifactReleaseState, type ArtifactType, type AtCapacityError, type AuditEventEntry, type AuditEventsResponse, type BanMinerData, type BanMinerError, type BanMinerResponse, type BanRequest, type BanResponse, type BanValidatorData, type BanValidatorError, type BanValidatorResponse, type BittensorAuthConfig, type Body_submit_agent, type CachedSession, type CancelAgentVersionData, type CancelAgentVersionError, type CancelAgentVersionResponse, type CancelRequest, type CancelResponse, type ChallengeRequest, type ChallengeResponse, type ChutesAuthStatusResponse, type ClaimWorkData, type ClaimWorkError, type ClaimWorkResponse, type ClaimWorkResponse2, type ClearMinerCooldownData, type ClearMinerCooldownError, type ClearMinerCooldownResponse, type CodeAnalysisError, type CompleteRunData, type CompleteRunError, type CompleteRunRequest, type CompleteRunResponse, type CompleteRunResponse2, type CooldownActiveError, type CreateSessionEndpointData, type CreateSessionEndpointError, type CreateSessionEndpointResponse, type CreateSuiteData, type CreateSuiteError, type CreateSuiteRequest, type CreateSuiteResponse, type CreateSuiteResponse2, type DiscardAgentVersionData, type DiscardAgentVersionError, type DiscardAgentVersionResponse, type DiscardRequest, type DiscardResponse, type ErrorCategory, type EvalRunNotFoundError, type EvaluationRunDetail, type EvaluationRunPublic, type EvaluationRunStatus, type EvaluationRunStatusPublic, type FileTooLargeError, type GetAgentVersionData, type GetAgentVersionError, type GetAgentVersionProblemsData, type GetAgentVersionProblemsError, type GetAgentVersionProblemsResponse, type GetAgentVersionResponse, type GetAgentVersionRunsData, type GetAgentVersionRunsError, type GetAgentVersionRunsResponse, type GetAgentVersionStatusData, type GetAgentVersionStatusError, type GetAgentVersionStatusResponse, type GetAgentVersionVarianceData, type GetAgentVersionVarianceError, type GetAgentVersionVarianceResponse, type GetArtifactDownloadUrlData, type GetArtifactDownloadUrlError, type GetArtifactDownloadUrlResponse, type GetAuditEventsData, type GetAuditEventsError, type GetAuditEventsResponse, type GetChutesAuthStatusError, type GetChutesAuthStatusResponse, type GetCurrentSuiteError, type GetCurrentSuiteResponse, type GetEvaluationRunData, type GetEvaluationRunError, type GetEvaluationRunResponse, type GetLeaderboardData, type GetLeaderboardError, type GetLeaderboardResponse, type GetOwnedAgentVersionStatusData, type GetOwnedAgentVersionStatusError, type GetOwnedAgentVersionStatusResponse, type GetPendingEvaluationsData, type GetPendingEvaluationsError, type GetPendingEvaluationsResponse, type GetReaperStatsError, type GetReaperStatsResponse, type GetRunningEvaluationsError, type GetRunningEvaluationsResponse, type GetSuiteProblemsData, type GetSuiteProblemsError, type GetSuiteProblemsResponse, type GetTopAgentError, type GetTopAgentResponse, type GetValidatorScoresData, type GetValidatorScoresError, type GetValidatorScoresResponse, type GetValidatorsError, type GetValidatorsResponse, type HTTPValidationError, type HealthCheckError, type HealthCheckResponse, type HeartbeatData, type HeartbeatError, type HeartbeatRequest, type HeartbeatResponse, type HeartbeatResponse2, type InvalidAgentNameError, type InvalidArtifactTypeError, type InvalidFileError, type InvalidProblemIdError, type InvalidateEvaluationRunData, type InvalidateEvaluationRunError, type InvalidateEvaluationRunResponse, type InvalidateRunRequest, type JoinWaitlistData, type JoinWaitlistError, type JoinWaitlistResponse, type LeaderboardEntry, type LeaderboardResponse, type LeaseExpiredError, type ListAgentVersions1Data, type ListAgentVersions1Error, type ListAgentVersions1Response, type ListAgentVersionsData, type ListAgentVersionsError, type ListAgentVersionsResponse, type ListEvaluationRunsData, type ListEvaluationRunsError, type ListEvaluationRunsResponse, type ListMinerAgentsError, type ListMinerAgentsResponse, type ListMinersData, type ListMinersError, type ListMinersResponse, type LogoutData, type LogoutError, type LogoutResponse, type LogoutResponse2, type MinerAgentsResponse, type MinerNotFoundError, type MissingParameterError, type MissingScoreError, type NoActiveSuiteError, type NotRunOwnerError, type OroErrorCode, type PendingEvaluation, type PendingEvaluationSummary, type PendingEvaluationsResponse, type PresignUploadData, type PresignUploadError, type PresignUploadRequest, type PresignUploadResponse, type PresignUploadResponse2, type ProblemNotFoundError, type ProblemProgressEntry, type ProblemProgressUpdate, type ProblemPublic, type ProblemStatus, type ProgressUpdateRequest, type ProgressUpdateResponse, type RateLimitExceededError, type ReaperStatsResponse, type ReevaluateAgentVersionData, type ReevaluateAgentVersionError, type ReevaluateAgentVersionResponse, type ReevaluateRequest, type ReevaluateResponse, type ReinstateAgentVersionData, type ReinstateAgentVersionError, type ReinstateAgentVersionResponse, type ReinstateRequest, type RequestChallengeData, type RequestChallengeError, type RequestChallengeResponse, type RetryConfig, type RetryContext, type RunAlreadyCompleteError, type RunningEvaluation, type ScoreBelowThresholdError, type SessionAuthConfig, SessionAuthManager, type SessionInfo, type SessionRequest, type SessionResponse, type SetTopAgentData, type SetTopAgentError, type SetTopAgentResponse, type SetTopRequest, type SetTopResponse, type StoreChutesTokenData, type StoreChutesTokenError, type StoreChutesTokenRequest, type StoreChutesTokenResponse, type SubmitAgentData, type SubmitAgentError, type SubmitAgentResponse, type SubmitAgentResponse2, type SuiteNotFoundError, type SuitePublic, type SuiteWithProblemsResponse, type TerminalStatus, type TopAgentResponse, type UnbanMinerData, type UnbanMinerError, type UnbanMinerResponse, type UnbanValidatorData, type UnbanValidatorError, type UnbanValidatorResponse, type UpdateProgressData, type UpdateProgressError, type UpdateProgressResponse, type ValidationError, type ValidatorCurrentAgent, type ValidatorNotFoundError, type ValidatorProblemResult, type ValidatorPublic, type ValidatorScoreSummary, type ValidatorScoresResponse, type ValidatorStatus, type WaitlistSignupRequest, type WaitlistSignupResponse, type WorkItemStatus, activateSuite, banMiner, banValidator, cancelAgentVersion, claimWork, classifyError, classifyStatus, clearMinerCooldown, client, completeRun, computeDelay, configureBittensorAuth, configurePublicClient, configureSessionAuth, createRetryFetch, createSessionEndpoint, createSuite, discardAgentVersion, generateAuthHeaders, getAgentVersion, getAgentVersionProblems, getAgentVersionRuns, getAgentVersionStatus, getAgentVersionVariance, getArtifactDownloadUrl, getAuditEvents, getChutesAuthStatus, getCurrentSuite, getErrorCode, getErrorDetail, getEvaluationRun, getLeaderboard, getOwnedAgentVersionStatus, getPendingEvaluations, getReaperStats, getRunningEvaluations, getSuiteProblems, getTopAgent, getValidatorScores, getValidators, hasDetail, hasErrorCode, healthCheck, heartbeat, invalidateEvaluationRun, isTransient, isTransientError, joinWaitlist, listAgentVersions, listAgentVersions1, listEvaluationRuns, listMinerAgents, listMiners, logout, parseRetryAfter, presignUpload, reevaluateAgentVersion, reinstateAgentVersion, requestChallenge, setTopAgent, storeChutesToken, submitAgent, unbanMiner, unbanValidator, updateProgress };
|
package/dist/index.js
CHANGED
|
@@ -54,6 +54,7 @@ __export(index_exports, {
|
|
|
54
54
|
getAgentVersionProblems: () => getAgentVersionProblems,
|
|
55
55
|
getAgentVersionRuns: () => getAgentVersionRuns,
|
|
56
56
|
getAgentVersionStatus: () => getAgentVersionStatus,
|
|
57
|
+
getAgentVersionVariance: () => getAgentVersionVariance,
|
|
57
58
|
getArtifactDownloadUrl: () => getArtifactDownloadUrl,
|
|
58
59
|
getAuditEvents: () => getAuditEvents,
|
|
59
60
|
getChutesAuthStatus: () => getChutesAuthStatus,
|
|
@@ -68,6 +69,7 @@ __export(index_exports, {
|
|
|
68
69
|
getRunningEvaluations: () => getRunningEvaluations,
|
|
69
70
|
getSuiteProblems: () => getSuiteProblems,
|
|
70
71
|
getTopAgent: () => getTopAgent,
|
|
72
|
+
getValidatorScores: () => getValidatorScores,
|
|
71
73
|
getValidators: () => getValidators,
|
|
72
74
|
hasDetail: () => hasDetail,
|
|
73
75
|
hasErrorCode: () => hasErrorCode,
|
|
@@ -387,6 +389,18 @@ var listEvaluationRuns = (options) => {
|
|
|
387
389
|
url: "/v1/admin/evaluation-runs"
|
|
388
390
|
});
|
|
389
391
|
};
|
|
392
|
+
var getValidatorScores = (options) => {
|
|
393
|
+
return (options?.client ?? client).get({
|
|
394
|
+
...options,
|
|
395
|
+
url: "/v1/admin/analytics/validator-scores"
|
|
396
|
+
});
|
|
397
|
+
};
|
|
398
|
+
var getAgentVersionVariance = (options) => {
|
|
399
|
+
return (options?.client ?? client).get({
|
|
400
|
+
...options,
|
|
401
|
+
url: "/v1/admin/analytics/agent-version-variance"
|
|
402
|
+
});
|
|
403
|
+
};
|
|
390
404
|
|
|
391
405
|
// src/errors.ts
|
|
392
406
|
function classifyStatus(status) {
|
|
@@ -773,6 +787,7 @@ function configureSessionAuth(baseUrl, config) {
|
|
|
773
787
|
getAgentVersionProblems,
|
|
774
788
|
getAgentVersionRuns,
|
|
775
789
|
getAgentVersionStatus,
|
|
790
|
+
getAgentVersionVariance,
|
|
776
791
|
getArtifactDownloadUrl,
|
|
777
792
|
getAuditEvents,
|
|
778
793
|
getChutesAuthStatus,
|
|
@@ -787,6 +802,7 @@ function configureSessionAuth(baseUrl, config) {
|
|
|
787
802
|
getRunningEvaluations,
|
|
788
803
|
getSuiteProblems,
|
|
789
804
|
getTopAgent,
|
|
805
|
+
getValidatorScores,
|
|
790
806
|
getValidators,
|
|
791
807
|
hasDetail,
|
|
792
808
|
hasErrorCode,
|
package/dist/index.mjs
CHANGED
|
@@ -288,6 +288,18 @@ var listEvaluationRuns = (options) => {
|
|
|
288
288
|
url: "/v1/admin/evaluation-runs"
|
|
289
289
|
});
|
|
290
290
|
};
|
|
291
|
+
var getValidatorScores = (options) => {
|
|
292
|
+
return (options?.client ?? client).get({
|
|
293
|
+
...options,
|
|
294
|
+
url: "/v1/admin/analytics/validator-scores"
|
|
295
|
+
});
|
|
296
|
+
};
|
|
297
|
+
var getAgentVersionVariance = (options) => {
|
|
298
|
+
return (options?.client ?? client).get({
|
|
299
|
+
...options,
|
|
300
|
+
url: "/v1/admin/analytics/agent-version-variance"
|
|
301
|
+
});
|
|
302
|
+
};
|
|
291
303
|
|
|
292
304
|
// src/errors.ts
|
|
293
305
|
function classifyStatus(status) {
|
|
@@ -673,6 +685,7 @@ export {
|
|
|
673
685
|
getAgentVersionProblems,
|
|
674
686
|
getAgentVersionRuns,
|
|
675
687
|
getAgentVersionStatus,
|
|
688
|
+
getAgentVersionVariance,
|
|
676
689
|
getArtifactDownloadUrl,
|
|
677
690
|
getAuditEvents,
|
|
678
691
|
getChutesAuthStatus,
|
|
@@ -687,6 +700,7 @@ export {
|
|
|
687
700
|
getRunningEvaluations,
|
|
688
701
|
getSuiteProblems,
|
|
689
702
|
getTopAgent,
|
|
703
|
+
getValidatorScores,
|
|
690
704
|
getValidators,
|
|
691
705
|
hasDetail,
|
|
692
706
|
hasErrorCode,
|
package/package.json
CHANGED
package/src/generated/sdk.gen.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// This file is auto-generated by @hey-api/openapi-ts
|
|
2
2
|
|
|
3
3
|
import { createClient, createConfig, type OptionsLegacyParser, formDataBodySerializer } from '@hey-api/client-fetch';
|
|
4
|
-
import type { HealthCheckError, HealthCheckResponse, GetCurrentSuiteError, GetCurrentSuiteResponse, GetSuiteProblemsData, GetSuiteProblemsError, GetSuiteProblemsResponse, GetLeaderboardData, GetLeaderboardError, GetLeaderboardResponse, GetTopAgentError, GetTopAgentResponse, GetAgentVersionStatusData, GetAgentVersionStatusError, GetAgentVersionStatusResponse, GetAgentVersionRunsData, GetAgentVersionRunsError, GetAgentVersionRunsResponse, GetAgentVersionProblemsData, GetAgentVersionProblemsError, GetAgentVersionProblemsResponse, GetAgentVersionData, GetAgentVersionError, GetAgentVersionResponse, GetArtifactDownloadUrlData, GetArtifactDownloadUrlError, GetArtifactDownloadUrlResponse, GetEvaluationRunData, GetEvaluationRunError, GetEvaluationRunResponse, GetValidatorsError, GetValidatorsResponse, GetRunningEvaluationsError, GetRunningEvaluationsResponse, GetPendingEvaluationsData, GetPendingEvaluationsError, GetPendingEvaluationsResponse, JoinWaitlistData, JoinWaitlistError, JoinWaitlistResponse, RequestChallengeData, RequestChallengeError, RequestChallengeResponse, CreateSessionEndpointData, CreateSessionEndpointError, CreateSessionEndpointResponse, LogoutData, LogoutError, LogoutResponse2, SubmitAgentData, SubmitAgentError, SubmitAgentResponse2, StoreChutesTokenData, StoreChutesTokenError, StoreChutesTokenResponse, GetChutesAuthStatusError, GetChutesAuthStatusResponse, ListMinerAgentsError, ListMinerAgentsResponse, ListAgentVersionsData, ListAgentVersionsError, ListAgentVersionsResponse, GetOwnedAgentVersionStatusData, GetOwnedAgentVersionStatusError, GetOwnedAgentVersionStatusResponse, ClaimWorkData, ClaimWorkError, ClaimWorkResponse2, HeartbeatData, HeartbeatError, HeartbeatResponse2, UpdateProgressData, UpdateProgressError, UpdateProgressResponse, PresignUploadData, PresignUploadError, PresignUploadResponse2, CompleteRunData, CompleteRunError, CompleteRunResponse2, BanMinerData, BanMinerError, BanMinerResponse, UnbanMinerData, UnbanMinerError, UnbanMinerResponse, BanValidatorData, BanValidatorError, BanValidatorResponse, UnbanValidatorData, UnbanValidatorError, UnbanValidatorResponse, DiscardAgentVersionData, DiscardAgentVersionError, DiscardAgentVersionResponse, ReinstateAgentVersionData, ReinstateAgentVersionError, ReinstateAgentVersionResponse, SetTopAgentData, SetTopAgentError, SetTopAgentResponse, InvalidateEvaluationRunData, InvalidateEvaluationRunError, InvalidateEvaluationRunResponse, ReevaluateAgentVersionData, ReevaluateAgentVersionError, ReevaluateAgentVersionResponse, CancelAgentVersionData, CancelAgentVersionError, CancelAgentVersionResponse, CreateSuiteData, CreateSuiteError, CreateSuiteResponse2, ActivateSuiteData, ActivateSuiteError, ActivateSuiteResponse2, GetAuditEventsData, GetAuditEventsError, GetAuditEventsResponse, GetReaperStatsError, GetReaperStatsResponse, ClearMinerCooldownData, ClearMinerCooldownError, ClearMinerCooldownResponse, ListMinersData, ListMinersError, ListMinersResponse, ListAgentVersions1Data, ListAgentVersions1Error, ListAgentVersions1Response, ListEvaluationRunsData, ListEvaluationRunsError, ListEvaluationRunsResponse } from './types.gen';
|
|
4
|
+
import type { HealthCheckError, HealthCheckResponse, GetCurrentSuiteError, GetCurrentSuiteResponse, GetSuiteProblemsData, GetSuiteProblemsError, GetSuiteProblemsResponse, GetLeaderboardData, GetLeaderboardError, GetLeaderboardResponse, GetTopAgentError, GetTopAgentResponse, GetAgentVersionStatusData, GetAgentVersionStatusError, GetAgentVersionStatusResponse, GetAgentVersionRunsData, GetAgentVersionRunsError, GetAgentVersionRunsResponse, GetAgentVersionProblemsData, GetAgentVersionProblemsError, GetAgentVersionProblemsResponse, GetAgentVersionData, GetAgentVersionError, GetAgentVersionResponse, GetArtifactDownloadUrlData, GetArtifactDownloadUrlError, GetArtifactDownloadUrlResponse, GetEvaluationRunData, GetEvaluationRunError, GetEvaluationRunResponse, GetValidatorsError, GetValidatorsResponse, GetRunningEvaluationsError, GetRunningEvaluationsResponse, GetPendingEvaluationsData, GetPendingEvaluationsError, GetPendingEvaluationsResponse, JoinWaitlistData, JoinWaitlistError, JoinWaitlistResponse, RequestChallengeData, RequestChallengeError, RequestChallengeResponse, CreateSessionEndpointData, CreateSessionEndpointError, CreateSessionEndpointResponse, LogoutData, LogoutError, LogoutResponse2, SubmitAgentData, SubmitAgentError, SubmitAgentResponse2, StoreChutesTokenData, StoreChutesTokenError, StoreChutesTokenResponse, GetChutesAuthStatusError, GetChutesAuthStatusResponse, ListMinerAgentsError, ListMinerAgentsResponse, ListAgentVersionsData, ListAgentVersionsError, ListAgentVersionsResponse, GetOwnedAgentVersionStatusData, GetOwnedAgentVersionStatusError, GetOwnedAgentVersionStatusResponse, ClaimWorkData, ClaimWorkError, ClaimWorkResponse2, HeartbeatData, HeartbeatError, HeartbeatResponse2, UpdateProgressData, UpdateProgressError, UpdateProgressResponse, PresignUploadData, PresignUploadError, PresignUploadResponse2, CompleteRunData, CompleteRunError, CompleteRunResponse2, BanMinerData, BanMinerError, BanMinerResponse, UnbanMinerData, UnbanMinerError, UnbanMinerResponse, BanValidatorData, BanValidatorError, BanValidatorResponse, UnbanValidatorData, UnbanValidatorError, UnbanValidatorResponse, DiscardAgentVersionData, DiscardAgentVersionError, DiscardAgentVersionResponse, ReinstateAgentVersionData, ReinstateAgentVersionError, ReinstateAgentVersionResponse, SetTopAgentData, SetTopAgentError, SetTopAgentResponse, InvalidateEvaluationRunData, InvalidateEvaluationRunError, InvalidateEvaluationRunResponse, ReevaluateAgentVersionData, ReevaluateAgentVersionError, ReevaluateAgentVersionResponse, CancelAgentVersionData, CancelAgentVersionError, CancelAgentVersionResponse, CreateSuiteData, CreateSuiteError, CreateSuiteResponse2, ActivateSuiteData, ActivateSuiteError, ActivateSuiteResponse2, GetAuditEventsData, GetAuditEventsError, GetAuditEventsResponse, GetReaperStatsError, GetReaperStatsResponse, ClearMinerCooldownData, ClearMinerCooldownError, ClearMinerCooldownResponse, ListMinersData, ListMinersError, ListMinersResponse, ListAgentVersions1Data, ListAgentVersions1Error, ListAgentVersions1Response, ListEvaluationRunsData, ListEvaluationRunsError, ListEvaluationRunsResponse, GetValidatorScoresData, GetValidatorScoresError, GetValidatorScoresResponse, GetAgentVersionVarianceData, GetAgentVersionVarianceError, GetAgentVersionVarianceResponse } from './types.gen';
|
|
5
5
|
|
|
6
6
|
export const client = createClient(createConfig());
|
|
7
7
|
|
|
@@ -560,4 +560,26 @@ export const listEvaluationRuns = <ThrowOnError extends boolean = false>(options
|
|
|
560
560
|
...options,
|
|
561
561
|
url: '/v1/admin/evaluation-runs'
|
|
562
562
|
});
|
|
563
|
+
};
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* Aggregated scoring statistics per validator
|
|
567
|
+
* Compute per-validator scoring statistics from completed runs.
|
|
568
|
+
*/
|
|
569
|
+
export const getValidatorScores = <ThrowOnError extends boolean = false>(options?: OptionsLegacyParser<GetValidatorScoresData, ThrowOnError>) => {
|
|
570
|
+
return (options?.client ?? client).get<GetValidatorScoresResponse, GetValidatorScoresError, ThrowOnError>({
|
|
571
|
+
...options,
|
|
572
|
+
url: '/v1/admin/analytics/validator-scores'
|
|
573
|
+
});
|
|
574
|
+
};
|
|
575
|
+
|
|
576
|
+
/**
|
|
577
|
+
* Score variance across validators for recent agent versions
|
|
578
|
+
* Find agent versions with high score variance across validators.
|
|
579
|
+
*/
|
|
580
|
+
export const getAgentVersionVariance = <ThrowOnError extends boolean = false>(options?: OptionsLegacyParser<GetAgentVersionVarianceData, ThrowOnError>) => {
|
|
581
|
+
return (options?.client ?? client).get<GetAgentVersionVarianceResponse, GetAgentVersionVarianceError, ThrowOnError>({
|
|
582
|
+
...options,
|
|
583
|
+
url: '/v1/admin/analytics/agent-version-variance'
|
|
584
|
+
});
|
|
563
585
|
};
|
|
@@ -384,6 +384,24 @@ export type AgentVersionPublic = {
|
|
|
384
384
|
latest_final_score?: (number | null);
|
|
385
385
|
};
|
|
386
386
|
|
|
387
|
+
/**
|
|
388
|
+
* Per-validator score for an agent version.
|
|
389
|
+
*/
|
|
390
|
+
export type AgentVersionScoreEntry = {
|
|
391
|
+
/**
|
|
392
|
+
* Validator hotkey
|
|
393
|
+
*/
|
|
394
|
+
validator_hotkey: string;
|
|
395
|
+
/**
|
|
396
|
+
* Score from this validator
|
|
397
|
+
*/
|
|
398
|
+
score: number;
|
|
399
|
+
/**
|
|
400
|
+
* Evaluation run ID
|
|
401
|
+
*/
|
|
402
|
+
run_id: string;
|
|
403
|
+
};
|
|
404
|
+
|
|
387
405
|
/**
|
|
388
406
|
* State of an agent version evaluation.
|
|
389
407
|
*/
|
|
@@ -457,6 +475,66 @@ export type AgentVersionStatus = {
|
|
|
457
475
|
} | null);
|
|
458
476
|
};
|
|
459
477
|
|
|
478
|
+
/**
|
|
479
|
+
* Score variance across validators for a single agent version.
|
|
480
|
+
*/
|
|
481
|
+
export type AgentVersionVariance = {
|
|
482
|
+
/**
|
|
483
|
+
* Agent version ID
|
|
484
|
+
*/
|
|
485
|
+
agent_version_id: string;
|
|
486
|
+
/**
|
|
487
|
+
* Agent name
|
|
488
|
+
*/
|
|
489
|
+
agent_name: string;
|
|
490
|
+
/**
|
|
491
|
+
* Miner hotkey
|
|
492
|
+
*/
|
|
493
|
+
miner_hotkey: string;
|
|
494
|
+
/**
|
|
495
|
+
* Number of validators that scored this version
|
|
496
|
+
*/
|
|
497
|
+
validator_count: number;
|
|
498
|
+
/**
|
|
499
|
+
* Mean score
|
|
500
|
+
*/
|
|
501
|
+
avg_score: number;
|
|
502
|
+
/**
|
|
503
|
+
* Minimum score
|
|
504
|
+
*/
|
|
505
|
+
min_score: number;
|
|
506
|
+
/**
|
|
507
|
+
* Maximum score
|
|
508
|
+
*/
|
|
509
|
+
max_score: number;
|
|
510
|
+
/**
|
|
511
|
+
* max_score - min_score
|
|
512
|
+
*/
|
|
513
|
+
spread: number;
|
|
514
|
+
/**
|
|
515
|
+
* True if spread exceeds threshold (default 10%)
|
|
516
|
+
*/
|
|
517
|
+
is_high_variance: boolean;
|
|
518
|
+
/**
|
|
519
|
+
* Individual validator scores
|
|
520
|
+
*/
|
|
521
|
+
per_validator: Array<AgentVersionScoreEntry>;
|
|
522
|
+
};
|
|
523
|
+
|
|
524
|
+
/**
|
|
525
|
+
* Response for agent version score variance analytics.
|
|
526
|
+
*/
|
|
527
|
+
export type AgentVersionVarianceResponse = {
|
|
528
|
+
/**
|
|
529
|
+
* Per-version variance data
|
|
530
|
+
*/
|
|
531
|
+
agent_versions: Array<AgentVersionVariance>;
|
|
532
|
+
/**
|
|
533
|
+
* Spread threshold used for flagging
|
|
534
|
+
*/
|
|
535
|
+
variance_threshold: number;
|
|
536
|
+
};
|
|
537
|
+
|
|
460
538
|
/**
|
|
461
539
|
* 409 - Resource is already invalidated.
|
|
462
540
|
*/
|
|
@@ -2181,6 +2259,66 @@ export type ValidatorPublic = {
|
|
|
2181
2259
|
identity_description?: (string | null);
|
|
2182
2260
|
};
|
|
2183
2261
|
|
|
2262
|
+
/**
|
|
2263
|
+
* Response for validator scoring analytics.
|
|
2264
|
+
*/
|
|
2265
|
+
export type ValidatorScoresResponse = {
|
|
2266
|
+
/**
|
|
2267
|
+
* Per-validator summaries
|
|
2268
|
+
*/
|
|
2269
|
+
validators: Array<ValidatorScoreSummary>;
|
|
2270
|
+
/**
|
|
2271
|
+
* Global average score across all validators
|
|
2272
|
+
*/
|
|
2273
|
+
global_avg_score: number;
|
|
2274
|
+
/**
|
|
2275
|
+
* Global standard deviation
|
|
2276
|
+
*/
|
|
2277
|
+
global_stddev: number;
|
|
2278
|
+
};
|
|
2279
|
+
|
|
2280
|
+
/**
|
|
2281
|
+
* Aggregated scoring stats for a single validator.
|
|
2282
|
+
*/
|
|
2283
|
+
export type ValidatorScoreSummary = {
|
|
2284
|
+
/**
|
|
2285
|
+
* Validator hotkey
|
|
2286
|
+
*/
|
|
2287
|
+
validator_hotkey: string;
|
|
2288
|
+
/**
|
|
2289
|
+
* Total completed runs
|
|
2290
|
+
*/
|
|
2291
|
+
total_runs: number;
|
|
2292
|
+
/**
|
|
2293
|
+
* Mean score across runs
|
|
2294
|
+
*/
|
|
2295
|
+
avg_score: number;
|
|
2296
|
+
/**
|
|
2297
|
+
* Median score
|
|
2298
|
+
*/
|
|
2299
|
+
median_score: number;
|
|
2300
|
+
/**
|
|
2301
|
+
* Standard deviation of scores
|
|
2302
|
+
*/
|
|
2303
|
+
stddev_score: number;
|
|
2304
|
+
/**
|
|
2305
|
+
* Minimum score
|
|
2306
|
+
*/
|
|
2307
|
+
min_score: number;
|
|
2308
|
+
/**
|
|
2309
|
+
* Maximum score
|
|
2310
|
+
*/
|
|
2311
|
+
max_score: number;
|
|
2312
|
+
/**
|
|
2313
|
+
* Percentage deviation from the global average (negative = below)
|
|
2314
|
+
*/
|
|
2315
|
+
deviation_from_global: number;
|
|
2316
|
+
/**
|
|
2317
|
+
* True if deviation exceeds 1.5 standard deviations from global mean
|
|
2318
|
+
*/
|
|
2319
|
+
is_outlier: boolean;
|
|
2320
|
+
};
|
|
2321
|
+
|
|
2184
2322
|
/**
|
|
2185
2323
|
* Status of a validator.
|
|
2186
2324
|
*/
|
|
@@ -2705,6 +2843,10 @@ export type GetAuditEventsData = {
|
|
|
2705
2843
|
* Offset for pagination
|
|
2706
2844
|
*/
|
|
2707
2845
|
offset?: number;
|
|
2846
|
+
/**
|
|
2847
|
+
* Filter: at or after (ISO 8601)
|
|
2848
|
+
*/
|
|
2849
|
+
since?: (string | null);
|
|
2708
2850
|
/**
|
|
2709
2851
|
* Filter by target ID
|
|
2710
2852
|
*/
|
|
@@ -2713,6 +2855,10 @@ export type GetAuditEventsData = {
|
|
|
2713
2855
|
* Filter by target type
|
|
2714
2856
|
*/
|
|
2715
2857
|
target_type?: (string | null);
|
|
2858
|
+
/**
|
|
2859
|
+
* Filter: at or before (ISO 8601)
|
|
2860
|
+
*/
|
|
2861
|
+
until?: (string | null);
|
|
2716
2862
|
};
|
|
2717
2863
|
};
|
|
2718
2864
|
|
|
@@ -2753,6 +2899,14 @@ export type ListMinersData = {
|
|
|
2753
2899
|
* Offset for pagination
|
|
2754
2900
|
*/
|
|
2755
2901
|
offset?: number;
|
|
2902
|
+
/**
|
|
2903
|
+
* Last submitted at or after
|
|
2904
|
+
*/
|
|
2905
|
+
since?: (string | null);
|
|
2906
|
+
/**
|
|
2907
|
+
* Last submitted at or before
|
|
2908
|
+
*/
|
|
2909
|
+
until?: (string | null);
|
|
2756
2910
|
};
|
|
2757
2911
|
};
|
|
2758
2912
|
|
|
@@ -2782,10 +2936,18 @@ export type ListAgentVersions1Data = {
|
|
|
2782
2936
|
* Offset for pagination
|
|
2783
2937
|
*/
|
|
2784
2938
|
offset?: number;
|
|
2939
|
+
/**
|
|
2940
|
+
* Filter: created at or after (ISO 8601)
|
|
2941
|
+
*/
|
|
2942
|
+
since?: (string | null);
|
|
2785
2943
|
/**
|
|
2786
2944
|
* Suite ID (defaults to active suite)
|
|
2787
2945
|
*/
|
|
2788
2946
|
suite_id?: (number | null);
|
|
2947
|
+
/**
|
|
2948
|
+
* Filter: created at or before (ISO 8601)
|
|
2949
|
+
*/
|
|
2950
|
+
until?: (string | null);
|
|
2789
2951
|
};
|
|
2790
2952
|
};
|
|
2791
2953
|
|
|
@@ -2807,10 +2969,18 @@ export type ListEvaluationRunsData = {
|
|
|
2807
2969
|
* Offset for pagination
|
|
2808
2970
|
*/
|
|
2809
2971
|
offset?: number;
|
|
2972
|
+
/**
|
|
2973
|
+
* Filter: created at or after (ISO 8601)
|
|
2974
|
+
*/
|
|
2975
|
+
since?: (string | null);
|
|
2810
2976
|
/**
|
|
2811
2977
|
* Filter by run status (e.g., RUNNING, SUCCESS)
|
|
2812
2978
|
*/
|
|
2813
2979
|
status?: (EvaluationRunStatus | null);
|
|
2980
|
+
/**
|
|
2981
|
+
* Filter: created at or before (ISO 8601)
|
|
2982
|
+
*/
|
|
2983
|
+
until?: (string | null);
|
|
2814
2984
|
/**
|
|
2815
2985
|
* Filter by validator hotkey
|
|
2816
2986
|
*/
|
|
@@ -2820,4 +2990,54 @@ export type ListEvaluationRunsData = {
|
|
|
2820
2990
|
|
|
2821
2991
|
export type ListEvaluationRunsResponse = (AdminEvaluationRunsResponse);
|
|
2822
2992
|
|
|
2823
|
-
export type ListEvaluationRunsError = (HTTPValidationError);
|
|
2993
|
+
export type ListEvaluationRunsError = (HTTPValidationError);
|
|
2994
|
+
|
|
2995
|
+
export type GetValidatorScoresData = {
|
|
2996
|
+
query?: {
|
|
2997
|
+
/**
|
|
2998
|
+
* Only runs after this time
|
|
2999
|
+
*/
|
|
3000
|
+
since?: (string | null);
|
|
3001
|
+
/**
|
|
3002
|
+
* Suite ID (defaults to active suite)
|
|
3003
|
+
*/
|
|
3004
|
+
suite_id?: (number | null);
|
|
3005
|
+
/**
|
|
3006
|
+
* Only runs before this time
|
|
3007
|
+
*/
|
|
3008
|
+
until?: (string | null);
|
|
3009
|
+
};
|
|
3010
|
+
};
|
|
3011
|
+
|
|
3012
|
+
export type GetValidatorScoresResponse = (ValidatorScoresResponse);
|
|
3013
|
+
|
|
3014
|
+
export type GetValidatorScoresError = (HTTPValidationError);
|
|
3015
|
+
|
|
3016
|
+
export type GetAgentVersionVarianceData = {
|
|
3017
|
+
query?: {
|
|
3018
|
+
/**
|
|
3019
|
+
* Number of agent versions
|
|
3020
|
+
*/
|
|
3021
|
+
limit?: number;
|
|
3022
|
+
/**
|
|
3023
|
+
* Only versions after this time
|
|
3024
|
+
*/
|
|
3025
|
+
since?: (string | null);
|
|
3026
|
+
/**
|
|
3027
|
+
* Suite ID (defaults to active suite)
|
|
3028
|
+
*/
|
|
3029
|
+
suite_id?: (number | null);
|
|
3030
|
+
/**
|
|
3031
|
+
* Only versions before this time
|
|
3032
|
+
*/
|
|
3033
|
+
until?: (string | null);
|
|
3034
|
+
/**
|
|
3035
|
+
* Spread threshold for flagging
|
|
3036
|
+
*/
|
|
3037
|
+
variance_threshold?: number;
|
|
3038
|
+
};
|
|
3039
|
+
};
|
|
3040
|
+
|
|
3041
|
+
export type GetAgentVersionVarianceResponse = (AgentVersionVarianceResponse);
|
|
3042
|
+
|
|
3043
|
+
export type GetAgentVersionVarianceError = (HTTPValidationError);
|