@synopackageland/cli 0.5.0 → 0.6.0

package/dist/templates/native-go-delegate/backend/internal/delegate/service_test.go

@@ -0,0 +1,318 @@
+package delegate
+
+import (
+	"context"
+	"dsmapp/internal/api"
+	"dsmapp/internal/background"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+type identity struct{ user string }
+
+func (i identity) Authenticate(*http.Request) (api.Identity, error) {
+	return api.Identity{User: i.user}, nil
+}
+func service(t *testing.T, c *Client) *Service {
+	dir := t.TempDir()
+	s := &Service{Client: c, Store: background.NewStore(filepath.Join(dir, "background")), Scope: "home", StagingDir: filepath.Join(dir, "executor-staging"), LookupUID: func(string) string { return "1205" }}
+	return s
+}
+func rpc(t *testing.T, s *Service, user, method, params string) map[string]any {
+	t.Helper()
+	routes := s.Routes()
+	server, e := api.NewServerWithPermissions(identity{user}, []byte(`{"schema":1,"default":"administrators","methods":{}}`), routes...)
+	if e != nil {
+		t.Fatal(e)
+	}
+	r := httptest.NewRequest("POST", "http://localhost/api", strings.NewReader(`{"method":"`+method+`","params":`+params+`}`))
+	r.Header.Set("Content-Type", "application/json")
+	r.AddCookie(&http.Cookie{Name: "id", Value: "cookie-only"})
+	r.Header.Set("X-SYNO-TOKEN", "request-token")
+	w := httptest.NewRecorder()
+	server.ServeHTTP(w, r)
+	var out map[string]any
+	if json.Unmarshal(w.Body.Bytes(), &out) != nil {
+		t.Fatal(w.Body.String())
+	}
+	return out
+}
+func TestCGIForwardsSessionPairNeverUsername(t *testing.T) {
+	c := executor(t, func(q Request) string {
+		wire, _ := json.Marshal(q)
+		if strings.Contains(string(wire), "username") || strings.Contains(string(wire), "alice") {
+			t.Errorf("username leaked: %s", wire)
+		}
+		if q.Auth["dsm_cookie"] != "cookie-only" || q.Auth["dsm_synotoken"] != "request-token" {
+			t.Errorf("auth %+v", q.Auth)
+		}
+		return `{"ok":true,"result":{"delegation_id":"d1","subject":{"username":"alice","uid":1205},"expires_at":"2030-01-01T00:00:00Z"}}` + "\n"
+	})
+	s := service(t, c)
+	for _, method := range []string{"delegate.create", "delegate.list", "delegate.revoke"} {
+		params := "{}"
+		if method == "delegate.create" {
+			params = `{"job_id":"j","ops":["files.write"],"path":"/home/Downloads","ttl_seconds":600}`
+		}
+		if method == "delegate.revoke" {
+			params = `{"delegation_id":"d1"}`
+		}
+		out := rpc(t, s, "alice", method, params)
+		if out["success"] != true {
+			t.Fatalf("%s: %+v", method, out)
+		}
+	}
+	out := rpc(t, s, "alice", "delegate.create", `{"job_id":"j","ops":["files.write"],"path":"/home","ttl_seconds":600,"username":"victim"}`)
+	if out["success"] != false {
+		t.Fatal("accepted username")
+	}
+}
+func TestDownloadJobsFinishCancelOwnerAndRecovery(t *testing.T) {
+	var mu sync.Mutex
+	revokes := 0
+	writes := 0
+	c := executor(t, func(q Request) string {
+		switch q.Op {
+		case "delegate.create":
+			return `{"ok":true,"result":{"delegation_id":"d1","subject":{"username":"alice","uid":1205},"expires_at":"2030-01-01T00:00:00Z"}}` + "\n"
+		case "delegate.revoke":
+			mu.Lock()
+			revokes++
+			mu.Unlock()
+			var p map[string]any
+			json.Unmarshal(q.Params, &p)
+			if q.Auth == nil && p["job_id"] == nil {
+				t.Error("missing job binding")
+			}
+		case "files.write":
+			mu.Lock()
+			writes++
+			mu.Unlock()
+		}
+		return `{"ok":true,"result":{}}` + "\n"
+	})
+	s := service(t, c)
+	web := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("download")) }))
+	defer web.Close()
+	submit := func(key string) string {
+		out := rpc(t, s, "alice", "delegate.jobs.submit", `{"key":"`+key+`","url":"`+web.URL+`/file","dest":"/home/Downloads","name":"file.txt"}`)
+		if out["success"] != true {
+			t.Fatal(out)
+		}
+		return out["data"].(map[string]any)["id"].(string)
+	}
+	id := submit("first")
+	if e := s.RunOnce(context.Background()); e != nil {
+		t.Fatal(e)
+	}
+	j, _ := s.Store.Get(context.Background(), id)
+	if j.DelegationID != "d1" {
+		t.Fatalf("delegation not stored on job: %+v", j)
+	}
+	if j.State != background.StateSucceeded {
+		t.Fatalf("job %+v", j)
+	}
+	id = submit("second")
+	out := rpc(t, s, "bob", "delegate.jobs.cancel", `{"id":"`+id+`"}`)
+	if out["success"] != false {
+		t.Fatal("other owner canceled")
+	}
+	out = rpc(t, s, "alice", "delegate.jobs.cancel", `{"id":"`+id+`"}`)
+	if out["success"] != true {
+		t.Fatal(out)
+	}
+	mu.Lock()
+	if revokes != 2 || writes != 1 {
+		t.Errorf("revokes %d writes %d", revokes, writes)
+	}
+	mu.Unlock()
+	id = submit("restart")
+	s.Store.Claim(context.Background())
+	if e := s.Recover(context.Background()); e != nil {
+		t.Fatal(e)
+	}
+	j, _ = s.Store.Get(context.Background(), id)
+	if j.State != background.StateInterrupted {
+		t.Fatal(j)
+	}
+	if pending, e := s.Store.DelegationCleanup(context.Background()); e != nil || len(pending) != 0 {
+		t.Fatalf("cleanup %v %v", pending, e)
+	}
+}
+func TestCapabilityUnavailableAndBlocked(t *testing.T) {
+	s := service(t, &Client{Socket: filepath.Join(t.TempDir(), "absent")})
+	out := rpc(t, s, "alice", "delegate.status", "{}")
+	if out["data"].(map[string]any)["state"] != "needs-admin-approval" {
+		t.Fatal(out)
+	}
+	s.Client = executor(t, func(Request) string { return `{"ok":false,"code":"POLICY_DENIED"}` + "\n" })
+	out = rpc(t, s, "alice", "delegate.status", "{}")
+	if out["data"].(map[string]any)["state"] != "blocked" {
+		t.Fatal(out)
+	}
+}
+func TestDownloadLimitTimeoutAndStagingCleanup(t *testing.T) {
+	c := executor(t, func(Request) string { return `{"ok":true,"result":{}}` + "\n" })
+	s := service(t, c)
+	web := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/slow" {
+			<-r.Context().Done()
+			return
+		}
+		w.Header().Set("Content-Length", "67108865")
+		w.WriteHeader(200)
+	}))
+	defer web.Close()
+	for _, p := range []string{"/big", "/slow"} {
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
+		e := s.download(ctx, DownloadInput{URL: web.URL + p, Dest: "/home", Name: "file"}, JobContext{Client: c, JobID: "j", DelegationID: "d", StagingDir: s.StagingDir})
+		cancel()
+		if e == nil {
+			t.Fatal("unbounded download")
+		}
+		files, _ := os.ReadDir(s.StagingDir)
+		if len(files) != 0 {
+			t.Fatal("staging leaked")
+		}
+	}
+}
+
+func TestRevocationFailureIsDurableAndRetried(t *testing.T) {
+	var mu sync.Mutex
+	reject := true
+	creates := 0
+	c := executor(t, func(q Request) string {
+		switch q.Op {
+		case "delegate.create":
+			mu.Lock()
+			creates++
+			mu.Unlock()
+			return `{"ok":true,"result":{"delegation_id":"d1","subject":{"username":"alice","uid":"1205"},"expires_at":"2030-01-01T00:00:00Z"}}` + "\n"
+		case "delegate.revoke":
+			mu.Lock()
+			bad := reject
+			mu.Unlock()
+			if bad {
+				return `{"ok":false,"code":"TEMPORARY_FAILURE"}` + "\n"
+			}
+		}
+		return `{"ok":true,"result":{}}` + "\n"
+	})
+	s := service(t, c)
+	params := `{"key":"same","url":"https://example.com/file","dest":"/home/Downloads","name":"file"}`
+	a := rpc(t, s, "alice", "delegate.jobs.submit", params)
+	b := rpc(t, s, "alice", "delegate.jobs.submit", params)
+	if a["success"] != true || b["success"] != true || a["data"].(map[string]any)["id"] != b["data"].(map[string]any)["id"] {
+		t.Fatal(a, b)
+	}
+	mu.Lock()
+	if creates != 1 {
+		t.Errorf("duplicate created %d grants", creates)
+	}
+	mu.Unlock()
+	id := a["data"].(map[string]any)["id"].(string)
+	out := rpc(t, s, "alice", "delegate.jobs.cancel", `{"id":"`+id+`"}`)
+	if out["data"].(map[string]any)["revocation_pending"] != true {
+		t.Fatal(out)
+	}
+	if list, e := s.Store.DelegationCleanup(context.Background()); e != nil || len(list) != 1 {
+		t.Fatal(list, e)
+	}
+	mu.Lock()
+	reject = false
+	mu.Unlock()
+	// A fresh process can retry using only durable job metadata.
+	s.Store = background.NewStore(filepath.Join(filepath.Dir(s.StagingDir), "background"))
+	if e := s.Sweep(context.Background()); e != nil {
+		t.Fatal(e)
+	}
+	if list, e := s.Store.DelegationCleanup(context.Background()); e != nil || len(list) != 0 {
+		t.Fatal(list, e)
+	}
+}
+func TestRunningJobCancellationStopsDownloadAndRevokes(t *testing.T) {
+	var mu sync.Mutex
+	revokes := 0
+	c := executor(t, func(q Request) string {
+		if q.Op == "delegate.create" {
+			return `{"ok":true,"result":{"delegation_id":"d1","subject":{"username":"alice","uid":1205},"expires_at":"2030-01-01T00:00:00Z"}}` + "\n"
+		}
+		if q.Op == "delegate.revoke" {
+			mu.Lock()
+			revokes++
+			mu.Unlock()
+		}
+		if q.Op == "files.write" {
+			t.Error("canceled download wrote destination")
+		}
+		return `{"ok":true,"result":{}}` + "\n"
+	})
+	started := make(chan struct{})
+	web := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { close(started); <-r.Context().Done() }))
+	defer web.Close()
+	s := service(t, c)
+	out := rpc(t, s, "alice", "delegate.jobs.submit", `{"key":"cancel-running","url":"`+web.URL+`","dest":"/home","name":"file"}`)
+	if out["success"] != true {
+		t.Fatal(out)
+	}
+	id := out["data"].(map[string]any)["id"].(string)
+	done := make(chan error, 1)
+	go func() { done <- s.RunOnce(context.Background()) }()
+	select {
+	case <-started:
+	case <-time.After(3 * time.Second):
+		t.Fatal("download did not start")
+	}
+	out = rpc(t, s, "alice", "delegate.jobs.cancel", `{"id":"`+id+`"}`)
+	if out["success"] != true {
+		t.Fatal(out)
+	}
+	select {
+	case e := <-done:
+		if e != nil {
+			t.Fatal(e)
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatal("cancellation did not stop worker")
+	}
+	job, _ := s.Store.Get(context.Background(), id)
+	if job.State != background.StateCanceled {
+		t.Fatal(job)
+	}
+	mu.Lock()
+	if revokes < 1 {
+		t.Error("not revoked")
+	}
+	mu.Unlock()
+}
+func TestSessionCookieMustBeSingleAndRequestScoped(t *testing.T) {
+	for _, r := range []*http.Request{nil, httptest.NewRequest("POST", "http://localhost", nil)} {
+		if _, e := api.DSMSession(r); e == nil {
+			t.Fatal("missing cookie accepted")
+		}
+	}
+	r := httptest.NewRequest("POST", "http://localhost", nil)
+	r.AddCookie(&http.Cookie{Name: "id", Value: "a"})
+	r.AddCookie(&http.Cookie{Name: "id", Value: "b"})
+	if _, e := api.DSMSession(r); e == nil {
+		t.Fatal("ambiguous cookie accepted")
+	}
+}
+
+func TestStatusRejectsPreviousExecutorPolicy(t *testing.T) {
+	c := executor(t, func(Request) string {
+		return `{"ok":true,"result":{"ready":true,"version":"1","executor_digest":"old","policy_digest":"old"}}` + "\n"
+	})
+	s := service(t, c)
+	s.ExecutorDigest, s.PolicyDigest = "current", "current"
+	if got := s.Status(context.Background()); got.State != "blocked" {
+		t.Fatalf("stale executor ready: %+v", got)
+	}
+}

package/dist/templates/native-go-delegate/backend/internal/delegate/wait_cancel_test.go

@@ -0,0 +1,101 @@
+package delegate
+
+import (
+	"context"
+	"dsmapp/internal/background"
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestWaitingCancellationRevocationAndOtherOwnerAccess(t *testing.T) {
+	for _, action := range []string{"cancel", "revoke", "parent-stop"} {
+		t.Run(action, func(t *testing.T) {
+			s := service(t, absentExecutor(t))
+			id := retryJob(t, s, time.Now().Add(time.Hour))
+			bob := background.Submitter{User: "bob", UID: "1206"}
+			other, _, err := s.Store.Submit(context.Background(), background.SubmitRequest{Key: "other", Name: "other", Data: []byte("other"), Submitter: bob})
+			if err != nil {
+				t.Fatal(err)
+			}
+			cgi := &Service{Client: s.Client, Store: background.NewRequestStore(filepath.Join(filepath.Dir(s.StagingDir), "background")), LookupUID: func(user string) string {
+				if user == "bob" {
+					return "1206"
+				}
+				return "1205"
+			}}
+			waiting := make(chan struct{})
+			s.wait = func(ctx context.Context, _ time.Duration) error { close(waiting); <-ctx.Done(); return ctx.Err() }
+			ctx, stop := context.WithCancel(context.Background())
+			defer stop()
+			done := make(chan error, 1)
+			go func() { done <- s.RunOnce(ctx) }()
+			select {
+			case <-waiting:
+			case <-time.After(3 * time.Second):
+				t.Fatal("worker did not wait")
+			}
+			// A separate CGI store/process shares the same lock file with the worker.
+			accessDone := make(chan error, 1)
+			go func() {
+				list, e := cgi.Store.ListFor(context.Background(), background.Access{Owner: bob})
+				if e != nil {
+					accessDone <- e
+					return
+				}
+				if len(list) != 1 || list[0].ID != other.ID {
+					accessDone <- errors.New("owner list leaked waiting job")
+					return
+				}
+				_, e = cgi.Store.CancelFor(context.Background(), other.ID, background.Access{Owner: bob})
+				accessDone <- e
+			}()
+			select {
+			case e := <-accessDone:
+				if e != nil {
+					t.Fatal(e)
+				}
+			case <-time.After(time.Second):
+				t.Fatal("executor timer holds journal lock")
+			}
+			denied := rpc(t, cgi, "bob", "delegate.jobs.cancel", `{"id":"`+id+`"}`)
+			if denied["success"] != false {
+				t.Fatal("other owner canceled waiting job", denied)
+			}
+			switch action {
+			case "cancel":
+				out := rpc(t, cgi, "alice", "delegate.jobs.cancel", `{"id":"`+id+`"}`)
+				if out["success"] != true || out["data"].(map[string]any)["revocation_pending"] != true {
+					t.Fatal(out)
+				}
+			case "revoke":
+				out := rpc(t, cgi, "alice", "delegate.revoke", `{"delegation_id":"grant"}`)
+				if out["success"] != false {
+					t.Fatal("absent executor falsely acknowledged revocation", out)
+				}
+			case "parent-stop":
+				stop()
+			}
+			select {
+			case <-done:
+			case <-time.After(time.Second):
+				t.Fatal("waiting worker ignored cancellation")
+			}
+			if action == "parent-stop" {
+				assertJob(t, s, id, background.StateInterrupted, "worker-stopped")
+			} else {
+				assertJob(t, s, id, background.StateCanceled, "")
+			}
+			pending, e := s.Store.DelegationCleanup(context.Background())
+			if e != nil || len(pending) != 1 || pending[0].JobID != id {
+				t.Fatalf("cleanup not durable: %v %v", pending, e)
+			}
+			entries, e := os.ReadDir(s.StagingDir)
+			if e != nil || len(entries) != 0 {
+				t.Fatalf("cancellation leaked staging: %v %v", entries, e)
+			}
+		})
+	}
+}

package/dist/templates/native-go-delegate/backend/internal/delegate/worker.go

@@ -0,0 +1,225 @@
+package delegate
+
+import (
+	"bytes"
+	"context"
+	"dsmapp/internal/api"
+	"dsmapp/internal/background"
+	"encoding/json"
+	"errors"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+)
+
+const DownloadTimeout = 60 * time.Second
+
+func downloadClient() *http.Client {
+	return &http.Client{Timeout: DownloadTimeout, Transport: &http.Transport{Proxy: nil, DisableKeepAlives: true, DialContext: (&net.Dialer{Timeout: 5 * time.Second}).DialContext, ResponseHeaderTimeout: 10 * time.Second, MaxResponseHeaderBytes: 32 << 10, DisableCompression: true}, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
+}
+func (s *Service) download(parent context.Context, in DownloadInput, job JobContext) error {
+	if !validDownload(in) {
+		return fail("invalid-request")
+	}
+	ctx, cancel := context.WithTimeout(parent, DownloadTimeout)
+	defer cancel()
+	request, e := http.NewRequestWithContext(ctx, "GET", in.URL, nil)
+	if e != nil {
+		return fail("invalid-request")
+	}
+	client := s.HTTP
+	if client == nil {
+		client = downloadClient()
+	}
+	response, e := client.Do(request)
+	if e != nil {
+		if ctx.Err() != nil {
+			return fail("timeout")
+		}
+		return fail("download_failed")
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return fail("download_failed")
+	}
+	if response.ContentLength > MaxDownloadBytes {
+		return fail("limit-exceeded")
+	}
+	if e = os.MkdirAll(job.StagingDir, 0700); e != nil {
+		return fail("STAGING_UNAVAILABLE")
+	}
+	info, e := os.Lstat(job.StagingDir)
+	if e != nil || !info.IsDir() || info.Mode().Perm()&0077 != 0 {
+		return fail("STAGING_UNAVAILABLE")
+	}
+	tmp, e := os.CreateTemp(job.StagingDir, "download-")
+	if e != nil {
+		return fail("STAGING_UNAVAILABLE")
+	}
+	defer os.Remove(tmp.Name())
+	defer tmp.Close()
+	n, e := io.Copy(tmp, io.LimitReader(contextReader{ctx, response.Body}, MaxDownloadBytes+1))
+	if e != nil {
+		return fail("download_failed")
+	}
+	if n > MaxDownloadBytes {
+		return fail("limit-exceeded")
+	}
+	if _, e = tmp.Seek(0, io.SeekStart); e != nil {
+		return fail("STAGING_UNAVAILABLE")
+	}
+	return job.Files().Write(parent, in.Dest, in.Name, tmp, n)
+}
+func (s *Service) Recover(ctx context.Context) error {
+	if e := s.Store.Recover(ctx); e != nil {
+		return e
+	}
+	return s.Sweep(ctx)
+}
+
+// RunOnce processes either the original SHA256 worker or the fixed delegated
+// download worker. No registry or operation ID comes from HTTP input.
+func (s *Service) RunOnce(parent context.Context) error {
+	job, e := s.Store.Claim(parent)
+	if e != nil || job == nil {
+		return e
+	}
+	ctx, cancel := context.WithCancel(parent)
+	if job.Worker != background.WorkerDownloadV1 {
+		cancel()
+		ctx, cancel = context.WithTimeout(parent, background.JobTimeout)
+	}
+	defer cancel()
+	watched := make(chan struct{})
+	go func() {
+		defer close(watched)
+		ticker := time.NewTicker(100 * time.Millisecond)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				state, e := s.Store.Get(ctx, job.ID)
+				if e == nil && (state.CancelRequested || state.State != background.StateRunning) {
+					cancel()
+					return
+				}
+			}
+		}
+	}()
+	outcome := background.Outcome{State: background.StateSucceeded}
+	input, e := s.Store.OpenInput(job.ID)
+	if e == nil {
+		switch job.Worker {
+		case background.WorkerDownloadV1:
+			var in DownloadInput
+			data, readErr := io.ReadAll(io.LimitReader(input, background.MaxInputBytes+1))
+			if readErr != nil || json.Unmarshal(data, &in) != nil {
+				e = fail("invalid-request")
+			} else {
+				// Verify immutable input before contacting any remote host or executor.
+				hash, e2 := background.SHA256V1(ctx, bytes.NewReader(data))
+				if e2 != nil || hash.SHA256 != job.InputDigest || hash.Bytes != job.InputBytes {
+					e = fail("input-changed")
+				} else {
+					binding, e2 := s.Store.Delegation(ctx, job.ID)
+					e = e2
+					if e == nil {
+						e = s.download(ctx, in, JobContext{Client: s.Client, JobID: job.ID, DelegationID: binding.DelegationID, StagingDir: s.StagingDir, callExecutor: func(ctx context.Context, op string, params map[string]any, out any) error {
+							return s.callWaitingExecutor(ctx, job.ID, binding.ExpiresAt, op, params, out)
+						}})
+					}
+				}
+			}
+		case background.WorkerSHA256V1:
+			result, e2 := background.SHA256V1(ctx, io.LimitReader(input, background.MaxInputBytes+1))
+			e = e2
+			if e == nil && (result.SHA256 != job.InputDigest || result.Bytes != job.InputBytes) {
+				e = fail("input-changed")
+			}
+			outcome.Result = &result
+		default:
+			e = fail("unsupported")
+		}
+		input.Close()
+	}
+	if e != nil {
+		outcome = background.Outcome{State: background.StateFailed, ErrorCode: "worker-error"}
+		outcome.ErrorCode = errorCode(e)
+	}
+	if parent.Err() != nil {
+		outcome = background.Outcome{State: background.StateInterrupted, ErrorCode: "worker-stopped"}
+	} else if ctx.Err() == context.DeadlineExceeded {
+		outcome = background.Outcome{State: background.StateFailed, ErrorCode: "timeout"}
+	}
+	cancel()
+	<-watched
+	finish, done := context.WithTimeout(context.Background(), 20*time.Second)
+	defer done()
+	if e = s.Store.Finish(finish, job.ID, outcome); e != nil {
+		return e
+	}
+	return s.Sweep(finish)
+}
+func (s *Service) Run(ctx context.Context, info background.WorkerInfo) error {
+	if e := background.RefuseRoot(info.UID); e != nil {
+		return e
+	}
+	if e := s.Store.Recover(ctx); e != nil {
+		return e
+	}
+	ctx, stop := context.WithCancel(ctx)
+	var heartbeat sync.WaitGroup
+	heartbeat.Add(1)
+	go func() {
+		defer heartbeat.Done()
+		beat := time.NewTicker(5 * time.Second)
+		defer beat.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-beat.C:
+				if e := s.Store.Heartbeat(ctx, info); e != nil && ctx.Err() == nil {
+					log.Print("worker heartbeat unavailable")
+				}
+			}
+		}
+	}()
+	defer func() { stop(); heartbeat.Wait() }()
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+	for ctx.Err() == nil {
+		if e := s.Store.Heartbeat(ctx, info); e != nil {
+			return e
+		}
+		if e := s.Sweep(ctx); e != nil {
+			log.Printf("delegation cleanup pending: %s", errorCode(e))
+		}
+		if e := s.Store.Tick(ctx); e != nil {
+			log.Print("schedule tick unavailable")
+		}
+		if e := s.RunOnce(ctx); e != nil && ctx.Err() == nil {
+			log.Printf("worker operation: %s", errorCode(e))
+		}
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-ticker.C:
+		}
+	}
+	return nil
+}
+
+func errorCode(e error) string {
+	var a *api.Error
+	if errors.As(e, &a) {
+		return a.Code
+	}
+	return "unavailable"
+}