@synopackageland/cli 0.4.0 → 0.5.0

package/templates/native-go/backend/internal/background/runner_test.go

@@ -0,0 +1,209 @@
+package background
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func fastRunner(s *Store, work Work) *Runner {
+	return &Runner{Store: s, Work: work, Timeout: 2 * time.Second, Poll: 5 * time.Millisecond, CancelPoll: 5 * time.Millisecond,
+		Beat: 10 * time.Millisecond, Info: WorkerInfo{PID: 7, UID: 1000, Username: "pkg"}}
+}
+
+func start(t *testing.T, r *Runner) (stop func() error) {
+	t.Helper()
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- r.Run(ctx) }()
+	return func() error { cancel(); return <-done }
+}
+
+func waitState(t *testing.T, s *Store, id string, want State) Job {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		job, err := s.Get(context.Background(), id)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if job.State == want {
+			return job
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	job, _ := s.Get(context.Background(), id)
+	t.Fatalf("job %s never reached %s: %+v", id, want, job)
+	return job
+}
+
+func TestSHA256V1WorkHashesInput(t *testing.T) {
+	res, err := SHA256V1(context.Background(), strings.NewReader("abc"))
+	if err != nil || res.Bytes != 3 || res.SHA256 != digest([]byte("abc")) {
+		t.Fatalf("%+v %v", res, err)
+	}
+}
+
+func TestSHA256V1WorkStopsWhenContextEnds(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if _, err := SHA256V1(ctx, strings.NewReader(strings.Repeat("x", 1<<20))); !errors.Is(err, context.Canceled) {
+		t.Fatalf("%v", err)
+	}
+}
+
+func TestRefuseRoot(t *testing.T) {
+	if RefuseRoot(0) == nil || RefuseRoot(1000) != nil {
+		t.Fatal("the worker must refuse to run as root")
+	}
+}
+
+func TestRunnerProcessesQueuedJobEndToEnd(t *testing.T) {
+	s, _ := newStore(t)
+	job := submit(t, s, "k", []byte("hello"))
+	stop := start(t, fastRunner(s, SHA256V1))
+	done := waitState(t, s, job.ID, StateSucceeded)
+	if err := stop(); err != nil {
+		t.Fatal(err)
+	}
+	if done.Result == nil || done.Result.SHA256 != digest([]byte("hello")) || done.Result.Bytes != 5 {
+		t.Fatalf("%+v", done)
+	}
+	st, _ := s.Status(context.Background())
+	if st.Worker == nil || st.Worker.UID != 1000 {
+		t.Fatalf("the worker must publish its identity: %+v", st.Worker)
+	}
+}
+
+func TestRunnerFailsJobWhoseInputChanged(t *testing.T) {
+	s, dir := newStore(t)
+	job := submit(t, s, "k", []byte("hello"))
+	if err := os.WriteFile(dir+"/inputs/"+job.ID, []byte("tampered"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	stop := start(t, fastRunner(s, SHA256V1))
+	got := waitState(t, s, job.ID, StateFailed)
+	_ = stop()
+	if got.ErrorCode != "input-changed" {
+		t.Fatalf("%+v", got)
+	}
+}
+
+func TestRunnerTimesOutLongJob(t *testing.T) {
+	s, _ := newStore(t)
+	job := submit(t, s, "k", []byte("x"))
+	r := fastRunner(s, func(ctx context.Context, _ io.Reader) (Result, error) { <-ctx.Done(); return Result{}, ctx.Err() })
+	r.Timeout = 50 * time.Millisecond
+	stop := start(t, r)
+	got := waitState(t, s, job.ID, StateFailed)
+	_ = stop()
+	if got.ErrorCode != "timeout" {
+		t.Fatalf("%+v", got)
+	}
+}
+
+func TestRunnerHonorsCancelOfRunningJob(t *testing.T) {
+	s, _ := newStore(t)
+	job := submit(t, s, "k", []byte("x"))
+	var started atomic.Bool
+	r := fastRunner(s, func(ctx context.Context, _ io.Reader) (Result, error) {
+		started.Store(true)
+		<-ctx.Done()
+		return Result{}, ctx.Err()
+	})
+	stop := start(t, r)
+	for !started.Load() {
+		time.Sleep(time.Millisecond)
+	}
+	if _, err := s.Cancel(context.Background(), job.ID); err != nil {
+		t.Fatal(err)
+	}
+	waitState(t, s, job.ID, StateCanceled)
+	_ = stop()
+}
+
+func TestRunnerStopInterruptsRunningJobWithoutReplay(t *testing.T) {
+	s, _ := newStore(t)
+	job := submit(t, s, "k", []byte("x"))
+	var started atomic.Bool
+	r := fastRunner(s, func(ctx context.Context, _ io.Reader) (Result, error) {
+		started.Store(true)
+		<-ctx.Done()
+		return Result{}, ctx.Err()
+	})
+	stop := start(t, r)
+	for !started.Load() {
+		time.Sleep(time.Millisecond)
+	}
+	if err := stop(); err != nil {
+		t.Fatal(err)
+	}
+	got, _ := s.Get(context.Background(), job.ID)
+	if got.State != StateInterrupted || got.ErrorCode != "worker-stopped" {
+		t.Fatalf("%+v", got)
+	}
+	// Starting again must not run it again.
+	stop = start(t, fastRunner(s, SHA256V1))
+	time.Sleep(50 * time.Millisecond)
+	_ = stop()
+	if again, _ := s.Get(context.Background(), job.ID); again.State != StateInterrupted {
+		t.Fatalf("interrupted jobs are never replayed: %+v", again)
+	}
+}
+
+func TestRunnerRecoversStaleRunningJobOnStart(t *testing.T) {
+	s, _ := newStore(t)
+	job := submit(t, s, "k", []byte("x"))
+	if _, err := s.Claim(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	stop := start(t, fastRunner(s, SHA256V1))
+	waitState(t, s, job.ID, StateInterrupted)
+	_ = stop()
+}
+
+func TestRunnerRunsJobsOneAfterAnother(t *testing.T) {
+	s, _ := newStore(t)
+	var running, peak atomic.Int32
+	r := fastRunner(s, func(ctx context.Context, in io.Reader) (Result, error) {
+		n := running.Add(1)
+		defer running.Add(-1)
+		if n > peak.Load() {
+			peak.Store(n)
+		}
+		time.Sleep(10 * time.Millisecond)
+		return SHA256V1(ctx, in)
+	})
+	var ids []string
+	for _, k := range []string{"ka", "kb", "kc"} {
+		job, _, err := s.Submit(context.Background(), SubmitRequest{Key: k, Name: "a.txt", Data: []byte(k), Submitter: Submitter{User: k, UID: "1025"}, Admin: true})
+		if err != nil {
+			t.Fatal(err)
+		}
+		ids = append(ids, job.ID)
+	}
+	stop := start(t, r)
+	for _, id := range ids {
+		waitState(t, s, id, StateSucceeded)
+	}
+	_ = stop()
+	if peak.Load() != 1 {
+		t.Fatalf("at most one job may run at a time, saw %d", peak.Load())
+	}
+}
+
+func TestNewRunnerUsesProductLimitsAndTheFixedWorker(t *testing.T) {
+	r := NewRunner(t.TempDir(), WorkerInfo{UID: 5})
+	if r.Timeout != JobTimeout || r.Work == nil || r.Info.UID != 5 || r.Poll <= 0 || r.CancelPoll <= 0 || r.Beat <= 0 {
+		t.Fatalf("%+v", r)
+	}
+	res, err := r.Work(context.Background(), strings.NewReader("abc"))
+	if err != nil || res.SHA256 != digest([]byte("abc")) {
+		t.Fatalf("the production worker must be sha256-v1: %+v %v", res, err)
+	}
+}

package/templates/native-go/backend/internal/background/schedule.go

@@ -0,0 +1,300 @@
+package background
+
+import (
+	"context"
+	"errors"
+	"golang.org/x/sys/unix"
+	"io"
+	"os"
+	"time"
+)
+
+const MaxSchedules = 8
+const ManualQueueReserve = 4
+
+// A tick delayed past this window is missed, never a catch-up submission.
+const ScheduleWindow = 5 * time.Second
+
+var ErrScheduleFull = errors.New("background schedule limit reached")
+var ErrInputUnavailable = errors.New("the imported private copy is unavailable")
+
+type Schedule struct {
+	ID              string    `json:"id"`
+	Key             string    `json:"key"`
+	Owner           Submitter `json:"owner"`
+	SourceJobID     string    `json:"sourceJobId"`
+	Name            string    `json:"name"`
+	InputDigest     string    `json:"inputDigest"`
+	InputBytes      int       `json:"inputBytes"`
+	WorkerRevision  int       `json:"workerRevision"`
+	IntervalSeconds int       `json:"intervalSeconds"`
+	Enabled         bool      `json:"enabled"`
+	CreatedAt       time.Time `json:"createdAt"`
+	NextAt          time.Time `json:"nextAt"`
+	LastTriggeredAt time.Time `json:"lastTriggeredAt,omitzero"`
+	Missed          uint64    `json:"missed"`
+	LastMissedAt    time.Time `json:"lastMissedAt,omitzero"`
+	Reason          string    `json:"reason,omitempty"`
+}
+type ScheduleRequest struct {
+	Key             string `json:"key"`
+	JobID           string `json:"jobId"`
+	IntervalSeconds int    `json:"intervalSeconds"`
+	Consent         bool   `json:"consent"`
+}
+
+func validInterval(seconds int) bool { return seconds == 3600 || seconds == 86400 || seconds == 604800 }
+func (a Access) seesSchedule(sc Schedule) bool {
+	return a.Admin || (a.Owner.Valid() && sc.Owner == a.Owner)
+}
+func findSchedule(j *journal, id string) *Schedule {
+	if !jobID.MatchString(id) {
+		return nil
+	}
+	for i := range j.Schedules {
+		if j.Schedules[i].ID == id {
+			return &j.Schedules[i]
+		}
+	}
+	return nil
+}
+func (s *Store) Schedules(ctx context.Context, a Access) ([]Schedule, error) {
+	out := []Schedule{}
+	err := s.withJournal(ctx, func(_ *os.Root, j *journal) (bool, error) {
+		for _, sc := range j.Schedules {
+			if a.seesSchedule(sc) {
+				out = append(out, sc)
+			}
+		}
+		return false, nil
+	})
+	return out, err
+}
+func readCopy(root *os.Root, id, digest string, size int) ([]byte, error) {
+	if !jobID.MatchString(id) || size < 1 || size > MaxInputBytes {
+		return nil, ErrInputUnavailable
+	}
+	f, err := root.OpenFile(inputs+"/"+id, os.O_RDONLY|unix.O_NOFOLLOW, 0)
+	if err != nil {
+		return nil, ErrInputUnavailable
+	}
+	defer f.Close()
+	if checkPrivate(f) != nil {
+		return nil, ErrInputUnavailable
+	}
+	b, err := io.ReadAll(io.LimitReader(f, MaxInputBytes+1))
+	if err != nil || len(b) != size || sha256hex(b) != digest {
+		return nil, ErrInputUnavailable
+	}
+	return b, nil
+}
+func (s *Store) CreateSchedule(ctx context.Context, r ScheduleRequest, a Access) (Schedule, bool, error) {
+	var out Schedule
+	dup := false
+	if !a.Owner.Valid() || !r.Consent || !validInterval(r.IntervalSeconds) || !keyRE.MatchString(r.Key) {
+		return out, false, ErrInvalid
+	}
+	err := s.withJournal(ctx, func(root *os.Root, j *journal) (bool, error) {
+		if !j.Enabled {
+			return false, ErrDisabled
+		}
+		if !a.Admin && !j.AllowOrdinaryUsers {
+			return false, ErrOrdinaryDisabled
+		}
+		for _, sc := range j.Schedules {
+			if sc.Owner == a.Owner && sc.Key == r.Key {
+				if sc.SourceJobID != r.JobID || sc.IntervalSeconds != r.IntervalSeconds {
+					return false, ErrKeyConflict
+				}
+				out, dup = sc, true
+				return false, nil
+			}
+		}
+		job := find(j, r.JobID)
+		if job == nil || job.PurgeRequested || job.Submitter != a.Owner {
+			return false, ErrNotFound
+		}
+		// Scheduling only a completed import avoids racing a mutable/incomplete source.
+		if job.State != StateSucceeded || job.ScheduleID != "" || job.WorkerRevision != s.revision {
+			return false, ErrInputUnavailable
+		}
+		data, err := readCopy(root, job.ID, job.InputDigest, job.InputBytes)
+		if err != nil {
+			return false, err
+		}
+		if len(j.Schedules) >= MaxSchedules {
+			return false, ErrScheduleFull
+		}
+		_, ownerBytes := j.ownerUsage(a.Owner)
+		if ownerBytes+int64(len(data)) > MaxOwnerBytes {
+			return false, ErrOwnerQuota
+		}
+		_, used := j.pending()
+		if used+int64(len(data)) > s.limits.MaxTotalBytes {
+			return false, ErrQuota
+		}
+		id, err := newID()
+		if err != nil {
+			return false, err
+		}
+		if err := writeInput(root, id, data); err != nil {
+			return false, err
+		}
+		now := s.now().UTC()
+		out = Schedule{ID: id, Key: r.Key, Owner: a.Owner, SourceJobID: job.ID, Name: job.Name, InputDigest: job.InputDigest, InputBytes: job.InputBytes, WorkerRevision: s.revision, IntervalSeconds: r.IntervalSeconds, Enabled: true, CreatedAt: now, NextAt: now.Add(time.Duration(r.IntervalSeconds) * time.Second)}
+		j.Schedules = append(j.Schedules, out)
+		return true, nil
+	})
+	return out, dup, err
+}
+
+// SetSchedule(false) journals revocation before the cancel watcher can observe it.
+// Enable is administrator-only and requires fresh explicit consent at the route.
+func (s *Store) SetSchedule(ctx context.Context, id string, enabled bool, a Access) (Schedule, error) {
+	var out Schedule
+	err := s.withJournal(ctx, func(root *os.Root, j *journal) (bool, error) {
+		sc := findSchedule(j, id)
+		if sc == nil || !a.seesSchedule(*sc) {
+			return false, ErrNotFound
+		}
+		if enabled {
+			if !a.Admin {
+				return false, ErrInvalid
+			}
+			if !j.Enabled {
+				return false, ErrDisabled
+			}
+			if _, err := readCopy(root, sc.ID, sc.InputDigest, sc.InputBytes); err != nil {
+				return false, err
+			}
+			sc.WorkerRevision = s.revision
+			sc.Enabled = true
+			sc.Reason = ""
+			sc.NextAt = s.now().UTC().Add(time.Duration(sc.IntervalSeconds) * time.Second)
+		} else {
+			sc.Enabled = false
+			sc.Reason = "disabled"
+			s.cancelScheduleJobs(root, j, id)
+		}
+		out = *sc
+		return true, nil
+	})
+	return out, err
+}
+func (s *Store) cancelScheduleJobs(root *os.Root, j *journal, id string) {
+	for i := range j.Jobs {
+		job := &j.Jobs[i]
+		if job.ScheduleID != id {
+			continue
+		}
+		if job.State == StateQueued {
+			s.finishJob(root, job, StateCanceled, "schedule-disabled")
+		} else if job.State == StateRunning {
+			job.CancelRequested = true
+		}
+	}
+}
+func (s *Store) DeleteSchedule(ctx context.Context, id string, a Access) error {
+	return s.withJournal(ctx, func(root *os.Root, j *journal) (bool, error) {
+		sc := findSchedule(j, id)
+		if sc == nil || !a.seesSchedule(*sc) {
+			return false, ErrNotFound
+		}
+		s.cancelScheduleJobs(root, j, id)
+		kept := j.Schedules[:0]
+		for _, v := range j.Schedules {
+			if v.ID != id {
+				kept = append(kept, v)
+			}
+		}
+		j.Schedules = kept
+		return true, nil
+	})
+}
+func (s *Store) reconcileSchedules(root *os.Root, j *journal, recovering bool) bool {
+	changed := false
+	now := s.now().UTC()
+	for i := range j.Schedules {
+		sc := &j.Schedules[i]
+		if !validInterval(sc.IntervalSeconds) || sc.NextAt.IsZero() {
+			sc.Enabled = false
+			sc.Reason = "invalid-schedule"
+			s.cancelScheduleJobs(root, j, sc.ID)
+			changed = true
+			continue
+		}
+		if sc.WorkerRevision != s.revision {
+			sc.Enabled = false
+			sc.Reason = "worker-changed"
+			s.cancelScheduleJobs(root, j, sc.ID)
+			changed = true
+			continue
+		}
+		if !sc.Enabled || sc.NextAt.After(now) {
+			continue
+		}
+		due := sc.NextAt
+		period := time.Duration(sc.IntervalSeconds) * time.Second
+		count := uint64(now.Sub(due)/period) + 1
+		sc.NextAt = due.Add(time.Duration(count) * period)
+		changed = true
+		// Recovery deliberately skips even a just-due tick. There is no startup burst.
+		if recovering || !j.Enabled || count > 1 || now.Sub(due) > ScheduleWindow {
+			sc.Missed += count
+			sc.LastMissedAt = due
+			sc.Reason = "missed"
+			continue
+		}
+		data, err := readCopy(root, sc.ID, sc.InputDigest, sc.InputBytes)
+		if err != nil {
+			sc.Enabled = false
+			sc.Reason = "input-unavailable"
+			sc.Missed++
+			sc.LastMissedAt = due
+			continue
+		}
+		pending, ownerBytes := j.ownerUsage(sc.Owner)
+		countPending, used := j.pending()
+		reason := ""
+		switch {
+		case pending >= MaxOwnerPending:
+			reason = "owner-queue-full"
+		case ownerBytes+int64(len(data)+resultReserve) > MaxOwnerBytes:
+			reason = "owner-quota-exceeded"
+		case countPending >= MaxQueued-ManualQueueReserve:
+			reason = "queue-full"
+		case used+int64(len(data)+resultReserve) > s.limits.MaxTotalBytes:
+			reason = "quota-exceeded"
+		}
+		if reason != "" {
+			sc.Missed++
+			sc.LastMissedAt = due
+			sc.Reason = reason
+			continue
+		}
+		id, err := newID()
+		if err != nil {
+			sc.Missed++
+			sc.LastMissedAt = due
+			sc.Reason = "unavailable"
+			continue
+		}
+		if err := writeInput(root, id, data); err != nil {
+			sc.Missed++
+			sc.LastMissedAt = due
+			sc.Reason = "unavailable"
+			continue
+		}
+		sc.LastTriggeredAt = due
+		sc.Reason = ""
+		j.Jobs = append(j.Jobs, Job{ID: id, Key: "schedule-" + id, Name: sc.Name, State: StateQueued, Submitter: sc.Owner, Worker: WorkerSHA256V1, WorkerRevision: s.revision, InputDigest: sc.InputDigest, InputBytes: sc.InputBytes, SubmittedAt: now, ScheduleID: sc.ID, ScheduledAt: due})
+	}
+	j.trim()
+	return changed
+}
+
+// Tick advances each durable checkpoint and creates a job in the same commit.
+// Multiple processes therefore cannot admit the same schedule trigger twice.
+func (s *Store) Tick(ctx context.Context) error {
+	return s.withJournal(ctx, func(root *os.Root, j *journal) (bool, error) { return s.reconcileSchedules(root, j, false), nil })
+}