@opensoha/contracts 0.1.24 → 0.1.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/auth/catalog_generated.go +2 -2
- package/auth/gateway_permissions_test.go +48 -0
- package/auth/permission-catalog.json +300 -1
- package/compat/consumer-revisions.json +1 -1
- package/compat/json-schema-ecosystem-0.1.0.json +1 -1
- package/compat/openapi-loose-boundaries-0.1.0.json +1 -1
- package/fixtures/openapi/KubernetesAgentActionCheck/invalid/unproven-state.json +6 -0
- package/fixtures/openapi/KubernetesAgentActionCheck/valid/configured-denied.json +9 -0
- package/fixtures/openapi/KubernetesAgentActionProfile/invalid/wildcard.json +1 -0
- package/fixtures/openapi/KubernetesAgentActionProfile/valid/workbench.json +1 -0
- package/fixtures/openapi/KubernetesClusterRegistrationRequest/invalid/action-profile.json +3 -0
- package/fixtures/openapi/KubernetesClusterRegistrationRequest/valid/action-profile.json +5 -0
- package/fixtures/openapi/KubernetesNodeListEnvelope/invalid/condition-status-boolean.json +17 -0
- package/fixtures/openapi/KubernetesNodeListEnvelope/invalid/taint-effect-missing.json +16 -0
- package/fixtures/openapi/KubernetesNodeListEnvelope/valid/legacy-fields-absent.json +11 -0
- package/fixtures/openapi/KubernetesNodeListEnvelope/valid/no-taints.json +13 -0
- package/fixtures/openapi/KubernetesNodeListEnvelope/valid/ready-cordoned-tainted.json +28 -0
- package/fixtures/openapi/KubernetesNodeListEnvelope/valid/unknown-fields-null.json +13 -0
- package/fixtures/openapi/KubernetesTopologyCoverage/invalid/empty-as-success.json +5 -0
- package/fixtures/openapi/KubernetesTopologyCoverage/valid/not-installed.json +6 -0
- package/fixtures/openapi/KubernetesTopologyGraphEdge/invalid/negative-weight.json +8 -0
- package/fixtures/openapi/KubernetesTopologyGraphEdge/valid/zero-weight.json +20 -0
- package/fixtures/openapi/KubernetesTopologyGraphNode/invalid/fake-flow-kind.json +42 -0
- package/fixtures/openapi/KubernetesTopologyGraphNode/valid/rule.json +42 -0
- package/fixtures/openapi/KubernetesTopologySnapshot/invalid/raw-source-field.json +33 -0
- package/fixtures/openapi/KubernetesTopologySnapshot/valid/projected-source.json +32 -0
- package/fixtures/openapi/KubernetesTopologySnapshotRequest/invalid/arbitrary-gvr.json +8 -0
- package/fixtures/openapi/KubernetesTopologySnapshotRequest/valid/bounded-sources.json +11 -0
- package/fixtures/openapi/KubernetesTopologySourceQuery/invalid/cluster-namespace.json +4 -0
- package/fixtures/openapi/KubernetesTopologySourceQuery/valid/all-namespaces.json +4 -0
- package/fixtures/openapi/ResourceMonitoringPanel/invalid/unknown-quality.json +7 -0
- package/fixtures/openapi/ResourceMonitoringPanel/valid/not-collected.json +7 -0
- package/fixtures/openapi/ResourceMonitoringPoint/invalid/string-zero.json +4 -0
- package/fixtures/openapi/ResourceMonitoringPoint/valid/missing.json +4 -0
- package/fixtures/openapi/ResourceMonitoringPoint/valid/zero.json +4 -0
- package/fixtures/openapi/ResourceMonitoringView/invalid/raw-query.json +12 -0
- package/fixtures/openapi/ResourceMonitoringView/valid/unconfigured.json +12 -0
- package/gen/go/sohaapi/client.go +1 -1
- package/gen/go/sohaapi/kubernetes_monitoring_test.go +67 -0
- package/gen/go/sohaapi/types.go +1265 -228
- package/gen/ts/sohaapi/index.ts +1116 -1
- package/openapi/soha-api.json +2497 -302
- package/openapi/soha-api.yaml +732 -3
- package/openapi/spec_generated.go +3 -3
- package/openapi/spec_test.go +34 -0
- package/package.json +1 -1
- package/resource/agent_access.go +12 -0
- package/resource/agent_profiles.go +62 -0
- package/resource/agent_profiles_test.go +36 -0
- package/resource/kubernetes_kinds.go +73 -0
- package/resource/kubernetes_kinds_test.go +38 -0
- package/resource/models.go +23 -14
- package/resource/network_topology.go +166 -0
- package/resource/network_topology_projection.go +208 -0
- package/resource/network_topology_projection_test.go +53 -0
- package/resource/network_topology_test.go +51 -0
- package/resource/network_topology_validation.go +29 -0
- package/resource/node_drain.go +34 -0
- package/resource/resource_monitoring.go +99 -0
- package/resource/runtime/network_topology.go +232 -0
- package/resource/runtime/network_topology_test.go +62 -0
- package/resource/runtime/node_drain.go +124 -0
- package/resource/runtime/node_drain_test.go +95 -0
- package/resource/runtime/resource_edit.go +128 -0
- package/resource/runtime/resource_edit_test.go +183 -0
- package/resource/runtime/resource_monitoring.go +243 -0
- package/resource/runtime/resource_monitoring_test.go +88 -0
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
package runtime
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"context"
|
|
5
|
+
"encoding/json"
|
|
6
|
+
"errors"
|
|
7
|
+
"fmt"
|
|
8
|
+
"io"
|
|
9
|
+
"strings"
|
|
10
|
+
"sync"
|
|
11
|
+
"time"
|
|
12
|
+
|
|
13
|
+
resource "github.com/opensoha/soha-contracts/resource"
|
|
14
|
+
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
15
|
+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
16
|
+
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
17
|
+
"k8s.io/client-go/rest"
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
// ReadNetworkTopologySnapshot performs only fixed, discovery-verified, bounded reads.
|
|
21
|
+
// The caller authenticates and authorizes each query before invoking this reader.
|
|
22
|
+
func ReadNetworkTopologySnapshot(ctx context.Context, client rest.Interface, clusterID string, input resource.NetworkTopologySnapshotRequest) (resource.NetworkTopologySnapshot, error) {
|
|
23
|
+
if err := resource.ValidateNetworkTopologySnapshotRequest(input); err != nil {
|
|
24
|
+
return resource.NetworkTopologySnapshot{}, err
|
|
25
|
+
}
|
|
26
|
+
if client == nil {
|
|
27
|
+
return resource.NetworkTopologySnapshot{}, fmt.Errorf("kubernetes REST client unavailable")
|
|
28
|
+
}
|
|
29
|
+
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
30
|
+
defer cancel()
|
|
31
|
+
output := resource.NetworkTopologySnapshot{Version: resource.NetworkTopologySnapshotFeature, Sources: make([]resource.NetworkTopologySourceResult, len(input.Queries))}
|
|
32
|
+
budget := &topologyReadBudget{remaining: resource.NetworkTopologyMaxTotalBytes}
|
|
33
|
+
sem := make(chan struct{}, 4)
|
|
34
|
+
var wg sync.WaitGroup
|
|
35
|
+
for index, query := range input.Queries {
|
|
36
|
+
wg.Add(1)
|
|
37
|
+
go func(index int, query resource.NetworkTopologySourceQuery) {
|
|
38
|
+
defer wg.Done()
|
|
39
|
+
select {
|
|
40
|
+
case sem <- struct{}{}:
|
|
41
|
+
defer func() { <-sem }()
|
|
42
|
+
case <-ctx.Done():
|
|
43
|
+
output.Sources[index] = topologySourceFailure(query, "error", "timeout")
|
|
44
|
+
return
|
|
45
|
+
}
|
|
46
|
+
output.Sources[index] = readTopologySource(ctx, client, clusterID, query, budget)
|
|
47
|
+
}(index, query)
|
|
48
|
+
}
|
|
49
|
+
wg.Wait()
|
|
50
|
+
total := 0
|
|
51
|
+
for i := range output.Sources {
|
|
52
|
+
source := &output.Sources[i]
|
|
53
|
+
kept := source.Objects[:0]
|
|
54
|
+
for _, object := range source.Objects {
|
|
55
|
+
size := len(object.Content)
|
|
56
|
+
if total+size > resource.NetworkTopologyMaxTotalBytes {
|
|
57
|
+
source.Coverage.State = "partial"
|
|
58
|
+
source.Coverage.Truncated = true
|
|
59
|
+
source.Coverage.ReasonCode = "total_budget"
|
|
60
|
+
continue
|
|
61
|
+
}
|
|
62
|
+
total += size
|
|
63
|
+
kept = append(kept, object)
|
|
64
|
+
}
|
|
65
|
+
source.Objects = kept
|
|
66
|
+
}
|
|
67
|
+
return output, nil
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
func topologySourceFailure(query resource.NetworkTopologySourceQuery, state, reason string) resource.NetworkTopologySourceResult {
|
|
71
|
+
definition, _ := resource.LookupNetworkTopologySource(query.SourceID)
|
|
72
|
+
return resource.NetworkTopologySourceResult{Coverage: resource.NetworkTopologyCoverage{Provider: definition.Provider, SourceID: query.SourceID, Namespace: query.Namespace, State: state, ReasonCode: reason, CheckedAt: time.Now().UTC().Format(time.RFC3339), SupportedKinds: []string{definition.Kind}}, Objects: []resource.NetworkTopologySourceObject{}}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
func readTopologySource(ctx context.Context, client rest.Interface, clusterID string, query resource.NetworkTopologySourceQuery, budget *topologyReadBudget) resource.NetworkTopologySourceResult {
|
|
76
|
+
definition, _ := resource.LookupNetworkTopologySource(query.SourceID)
|
|
77
|
+
result := topologySourceFailure(query, "available", "")
|
|
78
|
+
version, err := discoverTopologySource(ctx, client, definition, budget)
|
|
79
|
+
if err != nil {
|
|
80
|
+
result.Coverage.State, result.Coverage.ReasonCode = topologyReadError(err)
|
|
81
|
+
return result
|
|
82
|
+
}
|
|
83
|
+
base := topologyAPIBase(definition.Group, version)
|
|
84
|
+
if query.Namespace != "" {
|
|
85
|
+
base += "/namespaces/" + query.Namespace
|
|
86
|
+
}
|
|
87
|
+
raw, err := boundedTopologyGET(ctx, client, base+"/"+definition.Resource, true, budget)
|
|
88
|
+
if err != nil {
|
|
89
|
+
result.Coverage.State, result.Coverage.ReasonCode = topologyReadError(err)
|
|
90
|
+
return result
|
|
91
|
+
}
|
|
92
|
+
var list struct {
|
|
93
|
+
Metadata struct {
|
|
94
|
+
Continue string `json:"continue"`
|
|
95
|
+
} `json:"metadata"`
|
|
96
|
+
Items []map[string]any `json:"items"`
|
|
97
|
+
}
|
|
98
|
+
if json.Unmarshal(raw, &list) != nil {
|
|
99
|
+
result.Coverage.State = "error"
|
|
100
|
+
result.Coverage.ReasonCode = "invalid_response"
|
|
101
|
+
return result
|
|
102
|
+
}
|
|
103
|
+
result.Coverage.ObservedAt = time.Now().UTC().Format(time.RFC3339)
|
|
104
|
+
if len(list.Items) > resource.NetworkTopologyMaxSourceItems || list.Metadata.Continue != "" {
|
|
105
|
+
result.Coverage.Truncated = true
|
|
106
|
+
result.Coverage.State = "partial"
|
|
107
|
+
result.Coverage.ReasonCode = "item_budget"
|
|
108
|
+
}
|
|
109
|
+
for index, object := range list.Items {
|
|
110
|
+
if index >= resource.NetworkTopologyMaxSourceItems {
|
|
111
|
+
break
|
|
112
|
+
}
|
|
113
|
+
projected, err := resource.ProjectNetworkTopologyObject(query.SourceID, object)
|
|
114
|
+
if err != nil {
|
|
115
|
+
result.Coverage.Truncated = true
|
|
116
|
+
result.Coverage.State = "partial"
|
|
117
|
+
result.Coverage.ReasonCode = "object_budget"
|
|
118
|
+
continue
|
|
119
|
+
}
|
|
120
|
+
metadata, _ := projected["metadata"].(map[string]any)
|
|
121
|
+
name, _ := metadata["name"].(string)
|
|
122
|
+
namespace, _ := metadata["namespace"].(string)
|
|
123
|
+
if name == "" || (query.Namespace != "" && namespace != query.Namespace) {
|
|
124
|
+
result.Coverage.State = "partial"
|
|
125
|
+
result.Coverage.ReasonCode = "invalid_identity"
|
|
126
|
+
continue
|
|
127
|
+
}
|
|
128
|
+
uid, _ := metadata["uid"].(string)
|
|
129
|
+
rv, _ := metadata["resourceVersion"].(string)
|
|
130
|
+
generation, _ := metadata["generation"].(float64)
|
|
131
|
+
content, _ := json.Marshal(projected)
|
|
132
|
+
result.Objects = append(result.Objects, resource.NetworkTopologySourceObject{Resource: resource.ResourceRef{ClusterID: clusterID, APIVersion: strings.TrimPrefix(definition.Group+"/"+version, "/"), ScopeMode: topologyScope(definition.Namespaced), Kind: definition.Kind, Namespace: namespace, Name: name, UID: uid, ResourceVersion: rv}, Generation: int64(generation), Content: string(content)})
|
|
133
|
+
}
|
|
134
|
+
return result
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
func topologyAPIBase(group, version string) string {
|
|
138
|
+
if group == "" {
|
|
139
|
+
return "/api/" + version
|
|
140
|
+
}
|
|
141
|
+
return "/apis/" + group + "/" + version
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
func discoverTopologySource(ctx context.Context, client rest.Interface, definition resource.NetworkTopologySourceDefinition, budget *topologyReadBudget) (string, error) {
|
|
145
|
+
versions := []string{definition.Version}
|
|
146
|
+
if definition.FallbackVersion != "" {
|
|
147
|
+
versions = append(versions, definition.FallbackVersion)
|
|
148
|
+
}
|
|
149
|
+
for _, version := range versions {
|
|
150
|
+
raw, err := boundedTopologyGET(ctx, client, topologyAPIBase(definition.Group, version), false, budget)
|
|
151
|
+
if apierrors.IsNotFound(err) {
|
|
152
|
+
continue
|
|
153
|
+
}
|
|
154
|
+
if err != nil {
|
|
155
|
+
return "", err
|
|
156
|
+
}
|
|
157
|
+
var list metav1.APIResourceList
|
|
158
|
+
if err := json.Unmarshal(raw, &list); err != nil {
|
|
159
|
+
return "", fmt.Errorf("invalid discovery")
|
|
160
|
+
}
|
|
161
|
+
for _, item := range list.APIResources {
|
|
162
|
+
if item.Name == definition.Resource && item.Kind == definition.Kind && item.Namespaced == definition.Namespaced {
|
|
163
|
+
return version, nil
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return "", apierrors.NewNotFound(schema.GroupResource{Group: definition.Group, Resource: definition.Resource}, "")
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
func boundedTopologyGET(ctx context.Context, client rest.Interface, path string, list bool, budget *topologyReadBudget) ([]byte, error) {
|
|
171
|
+
request := client.Get().AbsPath(path).SetHeader("Accept", "application/json")
|
|
172
|
+
if list {
|
|
173
|
+
request = request.Param("limit", fmt.Sprint(resource.NetworkTopologyMaxSourceItems))
|
|
174
|
+
}
|
|
175
|
+
stream, err := request.Stream(ctx)
|
|
176
|
+
if err != nil {
|
|
177
|
+
return nil, err
|
|
178
|
+
}
|
|
179
|
+
defer func() { _ = stream.Close() }()
|
|
180
|
+
body, err := io.ReadAll(io.LimitReader(&topologyBudgetReader{reader: stream, budget: budget}, resource.NetworkTopologyMaxSourceBytes+1))
|
|
181
|
+
if err != nil {
|
|
182
|
+
return nil, err
|
|
183
|
+
}
|
|
184
|
+
if len(body) > resource.NetworkTopologyMaxSourceBytes {
|
|
185
|
+
return nil, fmt.Errorf("topology byte budget")
|
|
186
|
+
}
|
|
187
|
+
return body, nil
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
func topologyReadError(err error) (string, string) {
|
|
191
|
+
switch {
|
|
192
|
+
case apierrors.IsForbidden(err) || apierrors.IsUnauthorized(err):
|
|
193
|
+
return "denied", "kubernetes_rbac"
|
|
194
|
+
case apierrors.IsNotFound(err):
|
|
195
|
+
return "not_installed", "source_not_served"
|
|
196
|
+
case strings.Contains(err.Error(), "budget"):
|
|
197
|
+
return "partial", "byte_budget"
|
|
198
|
+
default:
|
|
199
|
+
return "error", "source_unavailable"
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
func topologyScope(namespaced bool) resource.ResourceScopeMode {
|
|
204
|
+
if namespaced {
|
|
205
|
+
return resource.ResourceScopeModeNamespace
|
|
206
|
+
}
|
|
207
|
+
return resource.ResourceScopeModeCluster
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
type topologyReadBudget struct {
|
|
211
|
+
mu sync.Mutex
|
|
212
|
+
remaining int
|
|
213
|
+
}
|
|
214
|
+
type topologyBudgetReader struct {
|
|
215
|
+
reader io.Reader
|
|
216
|
+
budget *topologyReadBudget
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
func (r *topologyBudgetReader) Read(data []byte) (int, error) {
|
|
220
|
+
r.budget.mu.Lock()
|
|
221
|
+
available := min(len(data), r.budget.remaining)
|
|
222
|
+
r.budget.remaining -= available
|
|
223
|
+
r.budget.mu.Unlock()
|
|
224
|
+
if available <= 0 {
|
|
225
|
+
return 0, errors.New("topology total byte budget")
|
|
226
|
+
}
|
|
227
|
+
count, err := r.reader.Read(data[:available])
|
|
228
|
+
r.budget.mu.Lock()
|
|
229
|
+
r.budget.remaining += available - count
|
|
230
|
+
r.budget.mu.Unlock()
|
|
231
|
+
return count, err
|
|
232
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
package runtime
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"context"
|
|
5
|
+
"encoding/json"
|
|
6
|
+
"fmt"
|
|
7
|
+
"net/http"
|
|
8
|
+
"net/http/httptest"
|
|
9
|
+
"strings"
|
|
10
|
+
"testing"
|
|
11
|
+
|
|
12
|
+
cr "github.com/opensoha/soha-contracts/resource"
|
|
13
|
+
"k8s.io/client-go/kubernetes"
|
|
14
|
+
"k8s.io/client-go/rest"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
func TestTopologySnapshotBoundsDiscoveryAndProjection(t *testing.T) {
|
|
18
|
+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
19
|
+
switch r.URL.Path {
|
|
20
|
+
case "/api/v1":
|
|
21
|
+
_, _ = fmt.Fprint(w, `{"resources":[{"name":"pods","kind":"Pod","namespaced":true},{"name":"services","kind":"Service","namespaced":true}]}`)
|
|
22
|
+
case "/api/v1/namespaces/team/pods":
|
|
23
|
+
if r.URL.Query().Get("limit") != "200" {
|
|
24
|
+
t.Error("missing item bound")
|
|
25
|
+
}
|
|
26
|
+
_, _ = fmt.Fprint(w, `{"metadata":{"continue":"next"},"items":[{"metadata":{"name":"p","namespace":"team","uid":"uid","resourceVersion":"9"},"spec":{"secret":"credential"},"status":{"phase":"Running"}}]}`)
|
|
27
|
+
case "/api/v1/namespaces/team/services":
|
|
28
|
+
http.Error(w, `{"kind":"Status","apiVersion":"v1","reason":"Forbidden","code":403}`, http.StatusForbidden)
|
|
29
|
+
case "/apis/traefik.io/v1alpha1":
|
|
30
|
+
http.NotFound(w, r)
|
|
31
|
+
default:
|
|
32
|
+
t.Errorf("unexpected path: %s", r.URL.Path)
|
|
33
|
+
http.NotFound(w, r)
|
|
34
|
+
}
|
|
35
|
+
}))
|
|
36
|
+
defer server.Close()
|
|
37
|
+
client, err := kubernetes.NewForConfig(&rest.Config{Host: server.URL})
|
|
38
|
+
if err != nil {
|
|
39
|
+
t.Fatal(err)
|
|
40
|
+
}
|
|
41
|
+
output, err := ReadNetworkTopologySnapshot(context.Background(), client.CoreV1().RESTClient(), "cluster", cr.NetworkTopologySnapshotRequest{Queries: []cr.NetworkTopologySourceQuery{{SourceID: "kubernetes.pods", Namespace: "team"}, {SourceID: "kubernetes.services", Namespace: "team"}, {SourceID: "traefik.ingressroutes", Namespace: "team"}}})
|
|
42
|
+
if err != nil {
|
|
43
|
+
t.Fatal(err)
|
|
44
|
+
}
|
|
45
|
+
if output.Sources[0].Coverage.State != "partial" || !output.Sources[0].Coverage.Truncated || output.Sources[1].Coverage.State != "denied" || output.Sources[2].Coverage.State != "not_installed" {
|
|
46
|
+
t.Fatalf("coverage: %+v", output.Sources)
|
|
47
|
+
}
|
|
48
|
+
raw, _ := json.Marshal(output)
|
|
49
|
+
if strings.Contains(string(raw), "credential") {
|
|
50
|
+
t.Fatal("full manifest escaped")
|
|
51
|
+
}
|
|
52
|
+
if output.Sources[0].Objects[0].Resource.ResourceVersion != "9" {
|
|
53
|
+
t.Fatal("identity lost")
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
func TestTopologySnapshotDoesNotReadArbitrarySources(t *testing.T) {
|
|
58
|
+
_, err := ReadNetworkTopologySnapshot(context.Background(), nil, "c", cr.NetworkTopologySnapshotRequest{Queries: []cr.NetworkTopologySourceQuery{{SourceID: "secrets"}}})
|
|
59
|
+
if err == nil {
|
|
60
|
+
t.Fatal("unknown source accepted")
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
package runtime
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"context"
|
|
5
|
+
"errors"
|
|
6
|
+
"sync"
|
|
7
|
+
|
|
8
|
+
"github.com/opensoha/soha-contracts/resource"
|
|
9
|
+
policyv1 "k8s.io/api/policy/v1"
|
|
10
|
+
policyv1beta1 "k8s.io/api/policy/v1beta1"
|
|
11
|
+
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
12
|
+
"k8s.io/client-go/kubernetes"
|
|
13
|
+
policyclient "k8s.io/client-go/kubernetes/typed/policy/v1"
|
|
14
|
+
policybetaclient "k8s.io/client-go/kubernetes/typed/policy/v1beta1"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
// DrainClient observes eviction failures while leaving kubectl drain semantics intact.
|
|
18
|
+
type DrainClient struct {
|
|
19
|
+
kubernetes.Interface
|
|
20
|
+
failures sync.Map
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
func NewDrainClient(client kubernetes.Interface) *DrainClient {
|
|
24
|
+
return &DrainClient{Interface: client}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
func (c *DrainClient) PolicyV1() policyclient.PolicyV1Interface {
|
|
28
|
+
return drainPolicy{PolicyV1Interface: c.Interface.PolicyV1(), observer: c}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
func (c *DrainClient) PolicyV1beta1() policybetaclient.PolicyV1beta1Interface {
|
|
32
|
+
return drainBetaPolicy{PolicyV1beta1Interface: c.Interface.PolicyV1beta1(), observer: c}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
func (c *DrainClient) observe(namespace, name string, err error) error {
|
|
36
|
+
key := namespace + "/" + name
|
|
37
|
+
if err == nil || apierrors.IsNotFound(err) {
|
|
38
|
+
c.failures.Delete(key)
|
|
39
|
+
} else {
|
|
40
|
+
c.failures.Store(key, err)
|
|
41
|
+
}
|
|
42
|
+
return err
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
func (c *DrainClient) Failure(ctx context.Context, err error) error {
|
|
46
|
+
if err == nil {
|
|
47
|
+
return nil
|
|
48
|
+
}
|
|
49
|
+
code := "node_drain_failed"
|
|
50
|
+
var denied error
|
|
51
|
+
c.failures.Range(func(_, value any) bool {
|
|
52
|
+
failure, ok := value.(error)
|
|
53
|
+
if !ok {
|
|
54
|
+
return true
|
|
55
|
+
}
|
|
56
|
+
switch {
|
|
57
|
+
case apierrors.IsForbidden(failure) || apierrors.IsUnauthorized(failure):
|
|
58
|
+
denied = failure
|
|
59
|
+
case apierrors.HasStatusCause(failure, policyv1.DisruptionBudgetCause):
|
|
60
|
+
code = "node_drain_pdb_blocked"
|
|
61
|
+
case apierrors.IsTooManyRequests(failure) && code != "node_drain_pdb_blocked":
|
|
62
|
+
code = "node_drain_eviction_blocked"
|
|
63
|
+
}
|
|
64
|
+
return true
|
|
65
|
+
})
|
|
66
|
+
if denied != nil {
|
|
67
|
+
return denied
|
|
68
|
+
}
|
|
69
|
+
if errors.Is(ctx.Err(), context.Canceled) {
|
|
70
|
+
code = "node_drain_canceled"
|
|
71
|
+
} else if code == "node_drain_failed" && errors.Is(ctx.Err(), context.DeadlineExceeded) {
|
|
72
|
+
code = "node_drain_timeout"
|
|
73
|
+
}
|
|
74
|
+
return &resource.NodeDrainError{Code: code, Cause: err}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
type drainPolicy struct {
|
|
78
|
+
policyclient.PolicyV1Interface
|
|
79
|
+
observer *DrainClient
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
func (c *DrainClient) PodSelectionFailure(ctx context.Context, failures []error) error {
|
|
83
|
+
for _, err := range failures {
|
|
84
|
+
if apierrors.IsForbidden(err) || apierrors.IsUnauthorized(err) || apierrors.IsNotFound(err) {
|
|
85
|
+
return err
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
err := errors.Join(failures...)
|
|
89
|
+
if ctx.Err() != nil {
|
|
90
|
+
return c.Failure(ctx, err)
|
|
91
|
+
}
|
|
92
|
+
return &resource.NodeDrainError{Code: "node_drain_pod_selection", Cause: err}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
func (c drainPolicy) Evictions(namespace string) policyclient.EvictionInterface {
|
|
96
|
+
return drainEvictions{EvictionInterface: c.PolicyV1Interface.Evictions(namespace), observer: c.observer}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
type drainEvictions struct {
|
|
100
|
+
policyclient.EvictionInterface
|
|
101
|
+
observer *DrainClient
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
func (c drainEvictions) Evict(ctx context.Context, eviction *policyv1.Eviction) error {
|
|
105
|
+
return c.observer.observe(eviction.Namespace, eviction.Name, c.EvictionInterface.Evict(ctx, eviction))
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
type drainBetaPolicy struct {
|
|
109
|
+
policybetaclient.PolicyV1beta1Interface
|
|
110
|
+
observer *DrainClient
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
func (c drainBetaPolicy) Evictions(namespace string) policybetaclient.EvictionInterface {
|
|
114
|
+
return drainBetaEvictions{EvictionInterface: c.PolicyV1beta1Interface.Evictions(namespace), observer: c.observer}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
type drainBetaEvictions struct {
|
|
118
|
+
policybetaclient.EvictionInterface
|
|
119
|
+
observer *DrainClient
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
func (c drainBetaEvictions) Evict(ctx context.Context, eviction *policyv1beta1.Eviction) error {
|
|
123
|
+
return c.observer.observe(eviction.Namespace, eviction.Name, c.EvictionInterface.Evict(ctx, eviction))
|
|
124
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
package runtime
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"context"
|
|
5
|
+
"errors"
|
|
6
|
+
"strings"
|
|
7
|
+
"testing"
|
|
8
|
+
"time"
|
|
9
|
+
|
|
10
|
+
"github.com/opensoha/soha-contracts/resource"
|
|
11
|
+
policyv1 "k8s.io/api/policy/v1"
|
|
12
|
+
policyv1beta1 "k8s.io/api/policy/v1beta1"
|
|
13
|
+
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
14
|
+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
15
|
+
"k8s.io/apimachinery/pkg/runtime"
|
|
16
|
+
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
17
|
+
"k8s.io/client-go/kubernetes/fake"
|
|
18
|
+
ktesting "k8s.io/client-go/testing"
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
func TestDrainFeedbackClassifiesTypedEvictionCausesAndClearsRecoveredBlockers(t *testing.T) {
|
|
22
|
+
pdb := &apierrors.StatusError{ErrStatus: metav1.Status{Code: 429, Reason: metav1.StatusReasonTooManyRequests, Message: "private provider message", Details: &metav1.StatusDetails{Causes: []metav1.StatusCause{{Type: policyv1.DisruptionBudgetCause}}}}}
|
|
23
|
+
for _, beta := range []bool{false, true} {
|
|
24
|
+
t.Run(map[bool]string{false: "v1", true: "v1beta1"}[beta], func(t *testing.T) {
|
|
25
|
+
client := fake.NewClientset()
|
|
26
|
+
var evictionError error = pdb
|
|
27
|
+
client.PrependReactor("create", "pods", func(ktesting.Action) (bool, runtime.Object, error) { return true, nil, evictionError })
|
|
28
|
+
observed := NewDrainClient(client)
|
|
29
|
+
evict := func() error {
|
|
30
|
+
meta := metav1.ObjectMeta{Name: "pod", Namespace: "team"}
|
|
31
|
+
if beta {
|
|
32
|
+
return observed.PolicyV1beta1().Evictions("team").Evict(context.Background(), &policyv1beta1.Eviction{ObjectMeta: meta})
|
|
33
|
+
}
|
|
34
|
+
return observed.PolicyV1().Evictions("team").Evict(context.Background(), &policyv1.Eviction{ObjectMeta: meta})
|
|
35
|
+
}
|
|
36
|
+
_ = evict()
|
|
37
|
+
assertDrainCode(t, observed.Failure(context.Background(), errors.New("private token")), "node_drain_pdb_blocked")
|
|
38
|
+
evictionError = apierrors.NewTooManyRequests("private throttle", 1)
|
|
39
|
+
_ = evict()
|
|
40
|
+
assertDrainCode(t, observed.Failure(context.Background(), errors.New("private")), "node_drain_eviction_blocked")
|
|
41
|
+
evictionError = nil
|
|
42
|
+
_ = evict()
|
|
43
|
+
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
|
|
44
|
+
defer cancel()
|
|
45
|
+
assertDrainCode(t, observed.Failure(ctx, errors.New("private")), "node_drain_timeout")
|
|
46
|
+
evictionError = apierrors.NewForbidden(schema.GroupResource{Resource: "pods"}, "pod", errors.New("private"))
|
|
47
|
+
_ = evict()
|
|
48
|
+
if !apierrors.IsForbidden(observed.Failure(context.Background(), errors.New("private"))) {
|
|
49
|
+
t.Fatal("eviction RBAC denial must retain forbidden semantics")
|
|
50
|
+
}
|
|
51
|
+
})
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
func TestDrainFeedbackKeepsCancellationAndSuccessDistinct(t *testing.T) {
|
|
56
|
+
client := NewDrainClient(fake.NewClientset())
|
|
57
|
+
ctx, cancel := context.WithCancel(context.Background())
|
|
58
|
+
cancel()
|
|
59
|
+
assertDrainCode(t, client.Failure(ctx, ctx.Err()), "node_drain_canceled")
|
|
60
|
+
if err := client.Failure(ctx, nil); err != nil {
|
|
61
|
+
t.Fatalf("success changed: %v", err)
|
|
62
|
+
}
|
|
63
|
+
if resource.IsNodeDrainErrorCode("provider_arbitrary_error") {
|
|
64
|
+
t.Fatal("untrusted error code accepted")
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
func TestDrainFeedbackDoesNotHidePermissionDenialBehindPDB(t *testing.T) {
|
|
69
|
+
client := NewDrainClient(fake.NewClientset())
|
|
70
|
+
client.failures.Store("invalid", struct{}{})
|
|
71
|
+
assertDrainCode(t, client.Failure(context.Background(), errors.New("private")), "node_drain_failed")
|
|
72
|
+
blocked := &apierrors.StatusError{ErrStatus: metav1.Status{Code: 429, Details: &metav1.StatusDetails{Causes: []metav1.StatusCause{{Type: policyv1.DisruptionBudgetCause}}}}}
|
|
73
|
+
_ = client.observe("team", "blocked", blocked)
|
|
74
|
+
canceled, cancel := context.WithCancel(context.Background())
|
|
75
|
+
cancel()
|
|
76
|
+
assertDrainCode(t, client.Failure(canceled, canceled.Err()), "node_drain_canceled")
|
|
77
|
+
_ = client.observe("team", "denied", apierrors.NewForbidden(schema.GroupResource{Resource: "pods/eviction"}, "denied", errors.New("private")))
|
|
78
|
+
if err := client.Failure(canceled, canceled.Err()); !apierrors.IsForbidden(err) {
|
|
79
|
+
t.Fatalf("permission denial hidden: %v", err)
|
|
80
|
+
}
|
|
81
|
+
if err := client.PodSelectionFailure(context.Background(), []error{apierrors.NewForbidden(schema.GroupResource{Resource: "pods"}, "", errors.New("private"))}); !apierrors.IsForbidden(err) {
|
|
82
|
+
t.Fatalf("pod list permission hidden: %v", err)
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
func assertDrainCode(t *testing.T, err error, want string) {
|
|
87
|
+
t.Helper()
|
|
88
|
+
var failure *resource.NodeDrainError
|
|
89
|
+
if !errors.As(err, &failure) || failure.Code != want {
|
|
90
|
+
t.Fatalf("got %v; want %s", err, want)
|
|
91
|
+
}
|
|
92
|
+
if strings.Contains(err.Error(), "private") {
|
|
93
|
+
t.Fatal("provider text leaked")
|
|
94
|
+
}
|
|
95
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
package runtime
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"context"
|
|
5
|
+
"encoding/json"
|
|
6
|
+
"errors"
|
|
7
|
+
"fmt"
|
|
8
|
+
"strings"
|
|
9
|
+
|
|
10
|
+
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
11
|
+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
12
|
+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
|
13
|
+
"k8s.io/apimachinery/pkg/types"
|
|
14
|
+
"k8s.io/apimachinery/pkg/util/sets"
|
|
15
|
+
"k8s.io/client-go/dynamic"
|
|
16
|
+
"k8s.io/client-go/util/csaupgrade"
|
|
17
|
+
"k8s.io/utils/ptr"
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
// ApplyResourceEdit preserves SSA conflicts while migrating this editor's own Create/Update fields.
|
|
21
|
+
// Dry-run migration validates a complete manifest with Update; it never changes live ownership.
|
|
22
|
+
func ApplyResourceEdit(ctx context.Context, resource dynamic.ResourceInterface, current, desired *unstructured.Unstructured, manager string, dryRun bool) (*unstructured.Unstructured, error) {
|
|
23
|
+
patch, err := json.Marshal(desired.Object)
|
|
24
|
+
if err != nil {
|
|
25
|
+
return nil, err
|
|
26
|
+
}
|
|
27
|
+
options := metav1.PatchOptions{FieldManager: manager, Force: ptr.To(false)}
|
|
28
|
+
if dryRun {
|
|
29
|
+
options.DryRun = []string{metav1.DryRunAll}
|
|
30
|
+
}
|
|
31
|
+
migration, err := resourceEditMigrationPatch(current, manager)
|
|
32
|
+
if err != nil {
|
|
33
|
+
return nil, err
|
|
34
|
+
}
|
|
35
|
+
if len(migration) == 0 {
|
|
36
|
+
return resource.Patch(ctx, desired.GetName(), types.ApplyPatchType, patch, options)
|
|
37
|
+
}
|
|
38
|
+
if dryRun {
|
|
39
|
+
// Probe foreign ownership first; an initial list merge can also fail validation
|
|
40
|
+
// until this creator's Update ownership is migrated.
|
|
41
|
+
probed, probeErr := resource.Patch(ctx, desired.GetName(), types.ApplyPatchType, patch, options)
|
|
42
|
+
if probeErr == nil {
|
|
43
|
+
return probed, nil
|
|
44
|
+
}
|
|
45
|
+
if !resourceEditPortMergeInvalid(current, probeErr) && !onlyResourceEditConflicts(current, probeErr, manager) {
|
|
46
|
+
return nil, probeErr
|
|
47
|
+
}
|
|
48
|
+
return resource.Update(ctx, desired, metav1.UpdateOptions{FieldManager: manager, DryRun: []string{metav1.DryRunAll}})
|
|
49
|
+
}
|
|
50
|
+
migrated, err := resource.Patch(ctx, desired.GetName(), types.JSONPatchType, migration, metav1.PatchOptions{})
|
|
51
|
+
if err != nil {
|
|
52
|
+
return nil, err
|
|
53
|
+
}
|
|
54
|
+
desired = desired.DeepCopy()
|
|
55
|
+
desired.SetResourceVersion(migrated.GetResourceVersion())
|
|
56
|
+
patch, err = json.Marshal(desired.Object)
|
|
57
|
+
if err != nil {
|
|
58
|
+
return nil, err
|
|
59
|
+
}
|
|
60
|
+
return resource.Patch(ctx, desired.GetName(), types.ApplyPatchType, patch, options)
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
func onlyResourceEditConflicts(current *unstructured.Unstructured, conflict error, manager string) bool {
|
|
64
|
+
var status apierrors.APIStatus
|
|
65
|
+
if !apierrors.IsConflict(conflict) || !errors.As(conflict, &status) {
|
|
66
|
+
return false
|
|
67
|
+
}
|
|
68
|
+
details := status.Status().Details
|
|
69
|
+
if details == nil || len(details.Causes) == 0 {
|
|
70
|
+
return false
|
|
71
|
+
}
|
|
72
|
+
prefix := fmt.Sprintf("conflict with %q using %s", manager, current.GetAPIVersion())
|
|
73
|
+
for _, cause := range details.Causes {
|
|
74
|
+
if cause.Type != metav1.CauseTypeFieldManagerConflict || cause.Field == "" || (cause.Message != prefix && !strings.HasPrefix(cause.Message, prefix+" at ")) {
|
|
75
|
+
return false
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return true
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
func resourceEditMigrationPatch(current *unstructured.Unstructured, manager string) ([]byte, error) {
|
|
82
|
+
if manager == "" || current.GetUID() == "" || current.GetResourceVersion() == "" {
|
|
83
|
+
return nil, nil
|
|
84
|
+
}
|
|
85
|
+
updates := 0
|
|
86
|
+
for _, entry := range current.GetManagedFields() {
|
|
87
|
+
if entry.Manager != manager || entry.Subresource != "" {
|
|
88
|
+
continue
|
|
89
|
+
}
|
|
90
|
+
// client-go's upgrader discards other Update versions; fail closed instead.
|
|
91
|
+
if entry.APIVersion != current.GetAPIVersion() {
|
|
92
|
+
return nil, nil
|
|
93
|
+
}
|
|
94
|
+
if entry.Operation == metav1.ManagedFieldsOperationUpdate {
|
|
95
|
+
updates++
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
if updates != 1 {
|
|
99
|
+
return nil, nil
|
|
100
|
+
}
|
|
101
|
+
patch, err := csaupgrade.UpgradeManagedFieldsPatch(current, sets.New(manager), manager)
|
|
102
|
+
if err != nil || len(patch) == 0 {
|
|
103
|
+
return patch, err
|
|
104
|
+
}
|
|
105
|
+
var operations []map[string]any
|
|
106
|
+
if err := json.Unmarshal(patch, &operations); err != nil {
|
|
107
|
+
return nil, err
|
|
108
|
+
}
|
|
109
|
+
operations = append([]map[string]any{{"op": "test", "path": "/metadata/uid", "value": string(current.GetUID())}}, operations...)
|
|
110
|
+
return json.Marshal(operations)
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
func resourceEditPortMergeInvalid(current *unstructured.Unstructured, err error) bool {
|
|
114
|
+
var status apierrors.APIStatus
|
|
115
|
+
if current.GetKind() != "Service" || !apierrors.IsInvalid(err) || !errors.As(err, &status) {
|
|
116
|
+
return false
|
|
117
|
+
}
|
|
118
|
+
details := status.Status().Details
|
|
119
|
+
if details == nil || len(details.Causes) == 0 {
|
|
120
|
+
return false
|
|
121
|
+
}
|
|
122
|
+
for _, cause := range details.Causes {
|
|
123
|
+
if cause.Type != metav1.CauseTypeFieldValueRequired || !strings.HasPrefix(cause.Field, "spec.ports[") || !strings.HasSuffix(cause.Field, "].name") {
|
|
124
|
+
return false
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return true
|
|
128
|
+
}
|