@synopackageland/cli 0.4.1 → 0.5.0

package/dist/templates/native-go/backend/internal/background/owner_test.go

@@ -0,0 +1,126 @@
+package background
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+)
+
+func TestOwnerAdmissionIsAtomicAndGatePersists(t *testing.T) {
+	s, dir := newStore(t)
+	ctx := context.Background()
+	owner := Submitter{User: "normal", UID: "1025"}
+	req := SubmitRequest{Key: "one", Name: "a", Data: []byte("x"), Submitter: owner}
+	if _, _, err := s.Submit(ctx, req); !errors.Is(err, ErrOrdinaryDisabled) {
+		t.Fatalf("default gate: %v", err)
+	}
+	if err := s.SetOrdinaryUsers(ctx, true); err != nil {
+		t.Fatal(err)
+	}
+	fresh := NewStore(dir)
+	st, _ := fresh.Status(ctx)
+	if !st.AllowOrdinaryUsers {
+		t.Fatal("gate not durable")
+	}
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	accepted := 0
+	for i := 0; i < 8; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			r := req
+			r.Key = fmt.Sprint(i)
+			_, _, err := fresh.Submit(ctx, r)
+			mu.Lock()
+			defer mu.Unlock()
+			if err == nil {
+				accepted++
+			} else if !errors.Is(err, ErrOwnerQueueFull) {
+				t.Errorf("admission: %v", err)
+			}
+		}(i)
+	}
+	wg.Wait()
+	if accepted != MaxOwnerPending {
+		t.Fatalf("concurrent owner jobs %d", accepted)
+	}
+	if err := fresh.SetOrdinaryUsers(ctx, false); err != nil {
+		t.Fatal(err)
+	}
+	if _, _, err := s.Submit(ctx, req); !errors.Is(err, ErrOrdinaryDisabled) {
+		t.Fatalf("gate changed after caller admission: %v", err)
+	}
+}
+func TestOwnerByteQuotaIncludesReservedResult(t *testing.T) {
+	s, _ := newStore(t)
+	ctx := context.Background()
+	submit(t, s, "one", make([]byte, MaxInputBytes))
+	_, _, err := s.Submit(ctx, SubmitRequest{Key: "two", Name: "a", Data: make([]byte, MaxInputBytes), Submitter: admin, Admin: true})
+	if !errors.Is(err, ErrOwnerQuota) {
+		t.Fatalf("owner byte quota: %v", err)
+	}
+	if _, _, err = s.Submit(ctx, SubmitRequest{Key: "two", Name: "a", Data: make([]byte, MaxInputBytes-2*resultReserve), Submitter: admin, Admin: true}); err != nil {
+		t.Fatal(err)
+	}
+}
+func TestPurgeRunningJobDrainsWithoutResultAndAuditSurvives(t *testing.T) {
+	s, dir := newStore(t)
+	ctx := context.Background()
+	job := submit(t, s, "one", []byte("x"))
+	_, _ = s.Claim(ctx)
+	audit, err := s.PurgeOwner(ctx, admin, "manager")
+	if err != nil || audit.Draining != 1 || audit.Removed != 0 {
+		t.Fatalf("%+v %v", audit, err)
+	}
+	if _, err := s.GetFor(ctx, job.ID, Access{Admin: true}); !errors.Is(err, ErrNotFound) {
+		t.Fatal("purged job is visible")
+	}
+	if job, err := s.Get(ctx, job.ID); err != nil || !job.CancelRequested {
+		t.Fatal("worker cannot see drain request")
+	}
+	if err := s.Finish(ctx, job.ID, Outcome{State: StateSucceeded, Result: &Result{Bytes: 1}}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := s.Get(ctx, job.ID); !errors.Is(err, ErrNotFound) {
+		t.Fatal("purged job retained after drain")
+	}
+	if _, err := os.Stat(filepath.Join(dir, "inputs", job.ID)); !errors.Is(err, os.ErrNotExist) {
+		t.Fatal("purged input remains")
+	}
+	fresh := NewStore(dir)
+	if err := fresh.withJournal(ctx, func(_ *os.Root, j *journal) (bool, error) {
+		if len(j.PurgeAudit) != 1 || j.PurgeAudit[0].Actor != "manager" {
+			t.Fatal("audit lost")
+		}
+		return false, nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+func TestLegacyUnknownOwnerIsAdminOnlyAndRecoveryCompletesPurge(t *testing.T) {
+	s, dir := newStore(t)
+	ctx := context.Background()
+	job := submit(t, s, "one", []byte("x"))
+	_, _ = s.Claim(ctx)
+	_ = s.withJournal(ctx, func(_ *os.Root, j *journal) (bool, error) { j.Jobs[0].Submitter.UID = ""; return true, nil })
+	if list, _ := s.ListFor(ctx, Access{Owner: Submitter{User: "admin", UID: "1024"}}); len(list) != 0 {
+		t.Fatal("legacy owner visible")
+	}
+	if list, _ := s.List(ctx); len(list) != 1 {
+		t.Fatal("legacy owner hidden from admin")
+	}
+	if _, err := s.PurgeOwner(ctx, Submitter{User: "admin"}, "manager"); err != nil {
+		t.Fatal(err)
+	}
+	if err := NewStore(dir).Recover(ctx); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := s.Get(ctx, job.ID); !errors.Is(err, ErrNotFound) {
+		t.Fatal("crashed purge recovered a result")
+	}
+}

package/dist/templates/native-go/backend/internal/background/runner.go

@@ -0,0 +1,185 @@
+package background
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"io"
+	"log"
+	"sync"
+	"time"
+)
+
+// Work is one fixed worker algorithm. It receives only the job's input bytes.
+type Work func(ctx context.Context, in io.Reader) (Result, error)
+
+// SHA256V1 is the only worker in this version: it hashes the imported copy.
+func SHA256V1(ctx context.Context, in io.Reader) (Result, error) {
+	h := sha256.New()
+	buf := make([]byte, 32*1024)
+	total := 0
+	for {
+		if err := ctx.Err(); err != nil {
+			return Result{}, err
+		}
+		n, err := in.Read(buf)
+		h.Write(buf[:n])
+		total += n
+		if errors.Is(err, io.EOF) {
+			return Result{SHA256: hex.EncodeToString(h.Sum(nil)), Bytes: total}, nil
+		}
+		if err != nil {
+			return Result{}, err
+		}
+	}
+}
+
+var errRefusedRoot = errors.New("the background worker refuses to run as root")
+
+// RefuseRoot keeps the worker on the package account.
+func RefuseRoot(uid int) error {
+	if uid == 0 {
+		return errRefusedRoot
+	}
+	return nil
+}
+
+var (
+	errCancelRequested = errors.New("cancel requested")
+	errTimeout         = errors.New("job time limit reached")
+)
+
+// Runner claims jobs from the Store and runs them one at a time.
+type Runner struct {
+	Store      *Store
+	Work       Work
+	Timeout    time.Duration
+	Poll       time.Duration
+	CancelPoll time.Duration
+	Beat       time.Duration
+	Info       WorkerInfo
+}
+
+// Run returns nil after ctx ends and the job in flight, if any, is recorded
+// as interrupted. Any other return is a spool failure.
+func (r *Runner) Run(ctx context.Context) error {
+	if err := r.Store.Recover(ctx); err != nil {
+		return err
+	}
+	if err := r.Store.Heartbeat(ctx, r.Info); err != nil {
+		return err
+	}
+	var beats sync.WaitGroup
+	defer beats.Wait()
+	beats.Add(1)
+	go func() { defer beats.Done(); r.heartbeat(ctx) }()
+	for ctx.Err() == nil {
+		if err := r.Store.Tick(ctx); err != nil && ctx.Err() == nil {
+			log.Printf("schedule tick failed: %v", err)
+		}
+		job, err := r.Store.Claim(ctx)
+		if err != nil && ctx.Err() == nil {
+			log.Printf("claim failed: %v", err)
+		}
+		if job == nil || err != nil {
+			sleep(ctx, r.Poll)
+			continue
+		}
+		r.process(ctx, job)
+	}
+	return nil
+}
+
+func sleep(ctx context.Context, d time.Duration) {
+	select {
+	case <-ctx.Done():
+	case <-time.After(d):
+	}
+}
+
+func (r *Runner) heartbeat(ctx context.Context) {
+	for {
+		sleep(ctx, r.Beat)
+		if ctx.Err() != nil {
+			return
+		}
+		if err := r.Store.Heartbeat(ctx, r.Info); err != nil && ctx.Err() == nil {
+			log.Printf("heartbeat failed: %v", err)
+		}
+	}
+}
+
+func (r *Runner) process(parent context.Context, job *Job) {
+	ctx, cancel := context.WithCancelCause(parent)
+	defer cancel(nil)
+	timer := time.AfterFunc(r.Timeout, func() { cancel(errTimeout) })
+	defer timer.Stop()
+	var watcher sync.WaitGroup
+	watcher.Add(1)
+	go func() { defer watcher.Done(); r.watchCancel(ctx, job.ID, cancel) }()
+
+	result, err := r.run(ctx, job)
+	outcome := Outcome{State: StateSucceeded, Result: &result}
+	switch {
+	case parent.Err() != nil:
+		outcome = Outcome{State: StateInterrupted, ErrorCode: "worker-stopped"}
+	case errors.Is(context.Cause(ctx), errCancelRequested):
+		outcome = Outcome{State: StateCanceled}
+	case errors.Is(context.Cause(ctx), errTimeout):
+		outcome = Outcome{State: StateFailed, ErrorCode: "timeout"}
+	case errors.Is(err, errInputChanged):
+		outcome = Outcome{State: StateFailed, ErrorCode: "input-changed"}
+	case err != nil:
+		log.Printf("job %s failed: %v", job.ID, err)
+		outcome = Outcome{State: StateFailed, ErrorCode: "worker-error"}
+	}
+	cancel(nil)
+	watcher.Wait()
+	// The stop signal that ended parent must not prevent recording the outcome.
+	fin, done := context.WithTimeout(context.Background(), 10*time.Second)
+	defer done()
+	if err := r.Store.Finish(fin, job.ID, outcome); err != nil {
+		log.Printf("job %s outcome not recorded: %v", job.ID, err)
+	}
+}
+
+var errInputChanged = errors.New("stored input no longer matches its digest")
+
+func (r *Runner) run(ctx context.Context, job *Job) (Result, error) {
+	f, err := r.Store.OpenInput(job.ID)
+	if err != nil {
+		return Result{}, err
+	}
+	defer f.Close()
+	result, err := r.Work(ctx, io.LimitReader(f, MaxInputBytes+1))
+	if err != nil {
+		return Result{}, err
+	}
+	if result.SHA256 != job.InputDigest || result.Bytes != job.InputBytes {
+		return Result{}, errInputChanged
+	}
+	return result, nil
+}
+
+func (r *Runner) watchCancel(ctx context.Context, id string, cancel context.CancelCauseFunc) {
+	for {
+		sleep(ctx, r.CancelPoll)
+		if ctx.Err() != nil {
+			return
+		}
+		if job, err := r.Store.Get(ctx, id); err == nil && (job.CancelRequested || job.State != StateRunning) {
+			cancel(errCancelRequested)
+			return
+		}
+	}
+}
+
+// NewRunner is the production assembly: the fixed sha256-v1 worker over the
+// package-private spool at dir.
+func NewRunner(dir string, info WorkerInfo) *Runner {
+	return &Runner{
+		Store: NewStore(dir), Work: SHA256V1, Timeout: JobTimeout,
+		Poll: 500 * time.Millisecond, CancelPoll: 250 * time.Millisecond, Beat: 5 * time.Second, Info: info,
+	}
+}

package/dist/templates/native-go/backend/internal/background/runner_test.go

@@ -0,0 +1,209 @@
+package background
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func fastRunner(s *Store, work Work) *Runner {
+	return &Runner{Store: s, Work: work, Timeout: 2 * time.Second, Poll: 5 * time.Millisecond, CancelPoll: 5 * time.Millisecond,
+		Beat: 10 * time.Millisecond, Info: WorkerInfo{PID: 7, UID: 1000, Username: "pkg"}}
+}
+
+func start(t *testing.T, r *Runner) (stop func() error) {
+	t.Helper()
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- r.Run(ctx) }()
+	return func() error { cancel(); return <-done }
+}
+
+func waitState(t *testing.T, s *Store, id string, want State) Job {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		job, err := s.Get(context.Background(), id)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if job.State == want {
+			return job
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	job, _ := s.Get(context.Background(), id)
+	t.Fatalf("job %s never reached %s: %+v", id, want, job)
+	return job
+}
+
+func TestSHA256V1WorkHashesInput(t *testing.T) {
+	res, err := SHA256V1(context.Background(), strings.NewReader("abc"))
+	if err != nil || res.Bytes != 3 || res.SHA256 != digest([]byte("abc")) {
+		t.Fatalf("%+v %v", res, err)
+	}
+}
+
+func TestSHA256V1WorkStopsWhenContextEnds(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if _, err := SHA256V1(ctx, strings.NewReader(strings.Repeat("x", 1<<20))); !errors.Is(err, context.Canceled) {
+		t.Fatalf("%v", err)
+	}
+}
+
+func TestRefuseRoot(t *testing.T) {
+	if RefuseRoot(0) == nil || RefuseRoot(1000) != nil {
+		t.Fatal("the worker must refuse to run as root")
+	}
+}
+
+func TestRunnerProcessesQueuedJobEndToEnd(t *testing.T) {
+	s, _ := newStore(t)
+	job := submit(t, s, "k", []byte("hello"))
+	stop := start(t, fastRunner(s, SHA256V1))
+	done := waitState(t, s, job.ID, StateSucceeded)
+	if err := stop(); err != nil {
+		t.Fatal(err)
+	}
+	if done.Result == nil || done.Result.SHA256 != digest([]byte("hello")) || done.Result.Bytes != 5 {
+		t.Fatalf("%+v", done)
+	}
+	st, _ := s.Status(context.Background())
+	if st.Worker == nil || st.Worker.UID != 1000 {
+		t.Fatalf("the worker must publish its identity: %+v", st.Worker)
+	}
+}
+
+func TestRunnerFailsJobWhoseInputChanged(t *testing.T) {
+	s, dir := newStore(t)
+	job := submit(t, s, "k", []byte("hello"))
+	if err := os.WriteFile(dir+"/inputs/"+job.ID, []byte("tampered"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	stop := start(t, fastRunner(s, SHA256V1))
+	got := waitState(t, s, job.ID, StateFailed)
+	_ = stop()
+	if got.ErrorCode != "input-changed" {
+		t.Fatalf("%+v", got)
+	}
+}
+
+func TestRunnerTimesOutLongJob(t *testing.T) {
+	s, _ := newStore(t)
+	job := submit(t, s, "k", []byte("x"))
+	r := fastRunner(s, func(ctx context.Context, _ io.Reader) (Result, error) { <-ctx.Done(); return Result{}, ctx.Err() })
+	r.Timeout = 50 * time.Millisecond
+	stop := start(t, r)
+	got := waitState(t, s, job.ID, StateFailed)
+	_ = stop()
+	if got.ErrorCode != "timeout" {
+		t.Fatalf("%+v", got)
+	}
+}
+
+func TestRunnerHonorsCancelOfRunningJob(t *testing.T) {
+	s, _ := newStore(t)
+	job := submit(t, s, "k", []byte("x"))
+	var started atomic.Bool
+	r := fastRunner(s, func(ctx context.Context, _ io.Reader) (Result, error) {
+		started.Store(true)
+		<-ctx.Done()
+		return Result{}, ctx.Err()
+	})
+	stop := start(t, r)
+	for !started.Load() {
+		time.Sleep(time.Millisecond)
+	}
+	if _, err := s.Cancel(context.Background(), job.ID); err != nil {
+		t.Fatal(err)
+	}
+	waitState(t, s, job.ID, StateCanceled)
+	_ = stop()
+}
+
+func TestRunnerStopInterruptsRunningJobWithoutReplay(t *testing.T) {
+	s, _ := newStore(t)
+	job := submit(t, s, "k", []byte("x"))
+	var started atomic.Bool
+	r := fastRunner(s, func(ctx context.Context, _ io.Reader) (Result, error) {
+		started.Store(true)
+		<-ctx.Done()
+		return Result{}, ctx.Err()
+	})
+	stop := start(t, r)
+	for !started.Load() {
+		time.Sleep(time.Millisecond)
+	}
+	if err := stop(); err != nil {
+		t.Fatal(err)
+	}
+	got, _ := s.Get(context.Background(), job.ID)
+	if got.State != StateInterrupted || got.ErrorCode != "worker-stopped" {
+		t.Fatalf("%+v", got)
+	}
+	// Starting again must not run it again.
+	stop = start(t, fastRunner(s, SHA256V1))
+	time.Sleep(50 * time.Millisecond)
+	_ = stop()
+	if again, _ := s.Get(context.Background(), job.ID); again.State != StateInterrupted {
+		t.Fatalf("interrupted jobs are never replayed: %+v", again)
+	}
+}
+
+func TestRunnerRecoversStaleRunningJobOnStart(t *testing.T) {
+	s, _ := newStore(t)
+	job := submit(t, s, "k", []byte("x"))
+	if _, err := s.Claim(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	stop := start(t, fastRunner(s, SHA256V1))
+	waitState(t, s, job.ID, StateInterrupted)
+	_ = stop()
+}
+
+func TestRunnerRunsJobsOneAfterAnother(t *testing.T) {
+	s, _ := newStore(t)
+	var running, peak atomic.Int32
+	r := fastRunner(s, func(ctx context.Context, in io.Reader) (Result, error) {
+		n := running.Add(1)
+		defer running.Add(-1)
+		if n > peak.Load() {
+			peak.Store(n)
+		}
+		time.Sleep(10 * time.Millisecond)
+		return SHA256V1(ctx, in)
+	})
+	var ids []string
+	for _, k := range []string{"ka", "kb", "kc"} {
+		job, _, err := s.Submit(context.Background(), SubmitRequest{Key: k, Name: "a.txt", Data: []byte(k), Submitter: Submitter{User: k, UID: "1025"}, Admin: true})
+		if err != nil {
+			t.Fatal(err)
+		}
+		ids = append(ids, job.ID)
+	}
+	stop := start(t, r)
+	for _, id := range ids {
+		waitState(t, s, id, StateSucceeded)
+	}
+	_ = stop()
+	if peak.Load() != 1 {
+		t.Fatalf("at most one job may run at a time, saw %d", peak.Load())
+	}
+}
+
+func TestNewRunnerUsesProductLimitsAndTheFixedWorker(t *testing.T) {
+	r := NewRunner(t.TempDir(), WorkerInfo{UID: 5})
+	if r.Timeout != JobTimeout || r.Work == nil || r.Info.UID != 5 || r.Poll <= 0 || r.CancelPoll <= 0 || r.Beat <= 0 {
+		t.Fatalf("%+v", r)
+	}
+	res, err := r.Work(context.Background(), strings.NewReader("abc"))
+	if err != nil || res.SHA256 != digest([]byte("abc")) {
+		t.Fatalf("the production worker must be sha256-v1: %+v %v", res, err)
+	}
+}