@synopackageland/cli 0.8.0 → 0.9.0

package/templates/native-go-delegate/backend/internal/background/delegated.go

@@ -1,277 +0,0 @@
-package background
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"io"
-	"os"
-	"strings"
-	"time"
-
-	"golang.org/x/sys/unix"
-)
-
-const WorkerDownloadV1 = "download-to-folder-v1"
-
-// DelegationBinding is job metadata, never a session credential. It stays until
-// executor revocation succeeds, even if cancellation removes the job input.
-type DelegationBinding struct {
-	JobID           string    `json:"job_id"`
-	ExpiresAt       time.Time `json:"expires_at,omitzero"`
-	WaitingExecutor bool      `json:"waiting_executor,omitempty"`
-	DelegationID    string    `json:"delegation_id"`
-}
-
-func bindingName(id string) string { return "delegate-" + id + ".json" }
-
-// SubmitDelegated reserves a job under the spool lock, creates its grant without
-// the lock, then commits the binding. Workers cannot claim a pending reservation.
-func (s *Store) SubmitDelegated(ctx context.Context, req SubmitRequest, create func(string) (DelegationBinding, error), rollback func(DelegationBinding)) (Job, error) {
-	if len(req.Data) == 0 || len(req.Data) > MaxInputBytes || !keyRE.MatchString(req.Key) || !validName(req.Name) || !req.Submitter.Valid() {
-		return Job{}, ErrInvalid
-	}
-	digest := sha256hex(req.Data)
-	var out Job
-	reserved := false
-	err := s.withJournal(ctx, func(root *os.Root, j *journal) (bool, error) {
-		for _, job := range j.Jobs {
-			if job.Submitter == req.Submitter && job.Key == req.Key && !job.PurgeRequested {
-				if job.InputDigest != digest || job.Worker != WorkerDownloadV1 {
-					return false, ErrKeyConflict
-				}
-				if job.AdmissionPending {
-					return false, ErrBusy
-				}
-				out = job
-				return false, nil
-			}
-		}
-		pending, ownerBytes := j.ownerUsage(req.Submitter)
-		if pending >= MaxOwnerPending {
-			return false, ErrOwnerQueueFull
-		}
-		if ownerBytes+int64(len(req.Data)+resultReserve) > MaxOwnerBytes {
-			return false, ErrOwnerQuota
-		}
-		count, used := j.pending()
-		if count >= MaxQueued {
-			return false, ErrQueueFull
-		}
-		if used+int64(len(req.Data)+resultReserve) > s.limits.MaxTotalBytes {
-			return false, ErrQuota
-		}
-		// Executor outages must not create an unbounded cleanup journal.
-		dir, e := root.Open(".")
-		if e != nil {
-			return false, e
-		}
-		entries, e := dir.ReadDir(-1)
-		dir.Close()
-		if e != nil {
-			return false, e
-		}
-		bindings := 0
-		for _, entry := range entries {
-			if strings.HasPrefix(entry.Name(), "delegate-") && strings.HasSuffix(entry.Name(), ".json") {
-				bindings++
-			}
-		}
-		// Pending admissions can each create one cleanup record outside this lock.
-		for _, job := range j.Jobs {
-			if job.AdmissionPending {
-				bindings++
-			}
-		}
-		if bindings >= MaxHistory+MaxQueued {
-			return false, ErrQuota
-		}
-
-		id, e := newID()
-		if e != nil {
-			return false, e
-		}
-
-		if e = writeInput(root, id, req.Data); e != nil {
-			return false, e
-		}
-		reserved = true
-		out = Job{AdmissionPending: true, ID: id, Key: req.Key, Name: req.Name, State: StateQueued, Submitter: req.Submitter, Worker: WorkerDownloadV1, WorkerRevision: s.revision, InputDigest: digest, InputBytes: len(req.Data), SubmittedAt: s.now().UTC()}
-		j.Jobs = append(j.Jobs, out)
-		j.trim()
-		return true, nil
-	})
-	if err != nil || !reserved {
-		return out, err
-	}
-	binding, err := create(out.ID)
-	binding.JobID = out.ID
-	grant := binding.DelegationID
-	if err == nil && grant == "" {
-		err = ErrInvalid
-	}
-	if err == nil {
-		err = s.withJournal(ctx, func(root *os.Root, j *journal) (bool, error) {
-			if e := writeAtomic(root, bindingName(out.ID), binding); e != nil {
-				return false, e
-			}
-			job := find(j, out.ID)
-			if job == nil || !job.AdmissionPending || job.State != StateQueued || job.CancelRequested || job.PurgeRequested {
-				return false, ErrNotCancelable
-			}
-			job.AdmissionPending = false
-			job.DelegationID = grant
-			out = *job
-			return true, nil
-		})
-	}
-	if err != nil {
-		// Retain failed revocations independently of the request's canceled context.
-		cleanup, cancel := context.WithTimeout(context.Background(), RequestLockTimeout)
-		_ = s.withJournal(cleanup, func(root *os.Root, j *journal) (bool, error) {
-			if grant != "" {
-				if e := writeAtomic(root, bindingName(out.ID), binding); e != nil {
-					return false, e
-				}
-			}
-			job := find(j, out.ID)
-			if job == nil || !job.AdmissionPending {
-				return false, nil
-			}
-			job.AdmissionPending = false
-			if job.State == StateQueued {
-				s.finishJob(root, job, StateFailed, "delegation-uncommitted")
-			}
-			return true, nil
-		})
-		cancel()
-		if grant != "" {
-			rollback(binding)
-		}
-		return Job{}, err
-	}
-	return out, nil
-}
-func readBinding(root *os.Root, id string) (DelegationBinding, error) {
-	var b DelegationBinding
-	if !jobID.MatchString(id) {
-		return b, ErrInvalid
-	}
-	f, e := root.OpenFile(bindingName(id), os.O_RDONLY|unix.O_NOFOLLOW, 0)
-	if e != nil {
-		return b, e
-	}
-	defer f.Close()
-	if e = checkPrivate(f); e != nil {
-		return b, e
-	}
-	data, e := io.ReadAll(io.LimitReader(f, 4097))
-	if e != nil || len(data) > 4096 || json.Unmarshal(data, &b) != nil || b.JobID != id || b.DelegationID == "" {
-		return b, ErrInvalid
-	}
-	return b, nil
-}
-func (s *Store) Delegation(ctx context.Context, id string) (DelegationBinding, error) {
-	var b DelegationBinding
-	e := s.withJournal(ctx, func(root *os.Root, _ *journal) (bool, error) {
-		var e error
-		b, e = readBinding(root, id)
-		return false, e
-	})
-	return b, e
-}
-
-// DelegationCleanup includes orphaned grants and canceled/running jobs. The
-// independent metadata record makes restart and executor outages retryable.
-func (s *Store) DelegationCleanup(ctx context.Context) ([]DelegationBinding, error) {
-	out := []DelegationBinding{}
-	e := s.withJournal(ctx, func(root *os.Root, j *journal) (bool, error) {
-		dir, e := root.Open(".")
-		if e != nil {
-			return false, e
-		}
-		defer dir.Close()
-		entries, e := dir.ReadDir(-1)
-		if e != nil {
-			return false, e
-		}
-		for _, entry := range entries {
-			n := entry.Name()
-			if !strings.HasPrefix(n, "delegate-") || !strings.HasSuffix(n, ".json") {
-				continue
-			}
-			id := strings.TrimSuffix(strings.TrimPrefix(n, "delegate-"), ".json")
-			b, e := readBinding(root, id)
-			if e != nil {
-				return false, e
-			}
-			job := find(j, id)
-			if job == nil || job.State.terminal() || job.CancelRequested || job.PurgeRequested {
-				out = append(out, b)
-			}
-		}
-		return false, nil
-	})
-	return out, e
-}
-func (s *Store) ForgetDelegation(ctx context.Context, id string) error {
-	return s.withJournal(ctx, func(root *os.Root, _ *journal) (bool, error) {
-		if !jobID.MatchString(id) {
-			return false, ErrInvalid
-		}
-		e := root.Remove(bindingName(id))
-		if errors.Is(e, os.ErrNotExist) {
-			return false, nil
-		}
-		if e != nil {
-			return false, e
-		}
-		return false, syncDir(root, ".")
-	})
-}
-
-// SetExecutorWaiting updates only the delegation sidecar. The claimed journal
-// slot remains reserved, preserving quotas and cancellation/recovery semantics.
-// No journal lock survives this method, a socket call, or a retry timer.
-func (s *Store) SetExecutorWaiting(ctx context.Context, id string, waiting bool) error {
-	return s.withJournal(ctx, func(root *os.Root, j *journal) (bool, error) {
-		job := find(j, id)
-		if job == nil || job.State != StateRunning {
-			return false, ErrNotRunning
-		}
-		if job.CancelRequested || job.PurgeRequested {
-			return false, context.Canceled
-		}
-		binding, err := readBinding(root, id)
-		if err != nil {
-			return false, err
-		}
-		binding.WaitingExecutor = waiting
-		return false, writeAtomic(root, bindingName(id), binding)
-	})
-}
-
-// ListDelegatedFor reads job state and waiting metadata in one short snapshot,
-// avoiding a race with terminal cleanup removing the delegation sidecar.
-func (s *Store) ListDelegatedFor(ctx context.Context, access Access) ([]Job, error) {
-	out := []Job{}
-	err := s.withJournal(ctx, func(root *os.Root, j *journal) (bool, error) {
-		for _, job := range j.Jobs {
-			if job.Worker != WorkerDownloadV1 || !access.sees(job) {
-				continue
-			}
-			if job.State == StateRunning {
-				binding, err := readBinding(root, job.ID)
-				if err != nil {
-					return false, err
-				}
-				if binding.WaitingExecutor {
-					job.State = "waiting_executor"
-				}
-			}
-			out = append(out, job)
-		}
-		return false, nil
-	})
-	return out, err
-}

package/templates/native-go-delegate/backend/internal/background/delegated_moves.go

@@ -1,270 +0,0 @@
-package background
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"io"
-	"os"
-	"strings"
-	"time"
-
-	"golang.org/x/sys/unix"
-)
-
-// StagingIdentity identifies a package-owned source for cleanup only. It is
-// never a grant, a UID selector for destination writes, or protected authority.
-type StagingIdentity struct {
-	Device, Inode, ParentDevice, ParentInode, RootDevice, RootInode uint64
-	UID                                                             uint32
-	Size, ModifiedNS                                                int64
-}
-
-type MoveBinding struct {
-	JobID        string          `json:"job_id"`
-	DelegationID string          `json:"delegation_id"`
-	Params       json.RawMessage `json:"params"`
-	Source       StagingIdentity `json:"source"`
-	Settled      bool            `json:"settled,omitempty"`
-	CleanupDone  bool            `json:"cleanup_done,omitempty"`
-	Refusal      string          `json:"refusal,omitempty"`
-	// Phase records how far admission got. Older mirrors have none and are
-	// treated as possibly sent. Attempts counts sends that may have arrived.
-	Phase       string `json:"phase,omitempty"`
-	Attempts    int    `json:"attempts,omitempty"`
-	NextAttempt int64  `json:"next_attempt,omitempty"`
-	// CancelRequested is the producer's durable cancel: it is forwarded until
-	// the executor answers, and admission is never sent again.
-	CancelRequested bool `json:"cancel_requested,omitempty"`
-}
-
-const (
-	MoveReserved = "reserved" // durable before transport; no request sent
-	MoveSent     = "sent"     // a request may have reached the executor
-	MoveAccepted = "accepted" // the executor returned the operation
-)
-
-const maxMoveMirrorBytes = 10 << 20
-
-func moveBindingName(id string) string { return "move-" + sha256hex([]byte(id)) + ".json" }
-
-func readMoveBinding(root *os.Root, id string) (MoveBinding, error) {
-	var out MoveBinding
-	f, err := root.OpenFile(moveBindingName(id), os.O_RDONLY|unix.O_NOFOLLOW, 0)
-	if err != nil {
-		return out, err
-	}
-	defer f.Close()
-	if err = checkPrivate(f); err != nil {
-		return out, err
-	}
-	data, err := io.ReadAll(io.LimitReader(f, 16385))
-	if err != nil || len(data) > 16384 || json.Unmarshal(data, &out) != nil || out.JobID != id || out.DelegationID == "" || !json.Valid(out.Params) {
-		return MoveBinding{}, ErrInvalid
-	}
-	return out, nil
-}
-
-func (s *Store) MoveBinding(ctx context.Context, id string) (MoveBinding, error) {
-	var out MoveBinding
-	err := s.withJournal(ctx, func(root *os.Root, _ *journal) (bool, error) {
-		var err error
-		out, err = readMoveBinding(root, id)
-		return false, err
-	})
-	return out, err
-}
-
-// ReserveMove fsyncs a mirror before admission. A process crash or ambiguous
-// response leaves this reference for a status query first; only an operation
-// the executor does not know is admitted again, through ClaimAdmission.
-// A short spool lock covers metadata only, not a socket call or file copy.
-func (s *Store) ReserveMove(ctx context.Context, in MoveBinding) (MoveBinding, bool, error) {
-	out := in
-	fresh := false
-	err := s.withJournal(ctx, func(root *os.Root, _ *journal) (bool, error) {
-		existing, err := readMoveBinding(root, in.JobID)
-		if err == nil {
-			if existing.DelegationID != in.DelegationID || !bytes.Equal(existing.Params, in.Params) {
-				return false, ErrKeyConflict
-			}
-			out = existing
-			return false, nil
-		}
-		if !errors.Is(err, os.ErrNotExist) {
-			return false, err
-		}
-		dir, err := root.Open(".")
-		if err != nil {
-			return false, err
-		}
-		defer dir.Close()
-		entries, err := dir.ReadDir(-1)
-		if err != nil {
-			return false, err
-		}
-		var used int64
-		for _, e := range entries {
-			if strings.HasPrefix(e.Name(), "move-") && strings.HasSuffix(e.Name(), ".json") {
-				info, err := e.Info()
-				if err != nil {
-					return false, err
-				}
-				used += info.Size()
-			}
-		}
-		// Completed/uncertain mirrors are retained for local replay prevention.
-		// The protected executor owns the authoritative operation/history quota.
-		encoded, err := json.Marshal(in)
-		if err != nil || len(encoded) > 16384 {
-			return false, ErrInvalid
-		}
-		if used+int64(len(encoded)) > maxMoveMirrorBytes {
-			return false, ErrQuota
-		}
-		if err = writeAtomic(root, moveBindingName(in.JobID), in); err != nil {
-			return false, err
-		}
-		fresh = true
-		return false, nil
-	})
-	return out, fresh, err
-}
-
-// updateMoveBinding rewrites one existing mirror under the spool lock when
-// change reports a modification.
-func (s *Store) updateMoveBinding(ctx context.Context, id string, change func(*MoveBinding) bool) error {
-	return s.withJournal(ctx, func(root *os.Root, _ *journal) (bool, error) {
-		in, err := readMoveBinding(root, id)
-		if err != nil || !change(&in) {
-			return false, err
-		}
-		return false, writeAtomic(root, moveBindingName(id), in)
-	})
-}
-
-// ClaimAdmission durably records one send before it happens. Concurrent
-// callers serialize here: a claim is refused for accepted, settled, refused
-// or exhausted mirrors, and before the backoff of the previous send ends.
-func (s *Store) ClaimAdmission(ctx context.Context, id string, now time.Time, limit int, backoff func(int) time.Duration) (MoveBinding, bool, error) {
-	var out MoveBinding
-	claimed := false
-	err := s.updateMoveBinding(ctx, id, func(in *MoveBinding) bool {
-		out = *in
-		if in.Phase == MoveAccepted || in.Settled || in.CleanupDone || in.Refusal != "" || in.CancelRequested || in.Attempts >= limit || now.UnixNano() < in.NextAttempt {
-			return false
-		}
-		in.Phase, in.Attempts = MoveSent, in.Attempts+1
-		in.NextAttempt = now.Add(backoff(in.Attempts)).UnixNano()
-		out, claimed = *in, true
-		return true
-	})
-	return out, claimed, err
-}
-
-// ReleaseAdmission returns a claim the executor proved it did not admit, so
-// the bound counts only sends that might have created an operation.
-func (s *Store) ReleaseAdmission(ctx context.Context, id string) error {
-	return s.updateMoveBinding(ctx, id, func(in *MoveBinding) bool {
-		if in.Phase != MoveSent {
-			return false
-		}
-		in.Phase, in.Attempts, in.NextAttempt = MoveReserved, in.Attempts-1, 0
-		return true
-	})
-}
-
-// RequestMoveCancel durably records a cancel of the mirror bound to the grant.
-func (s *Store) RequestMoveCancel(ctx context.Context, id, delegationID string) error {
-	return s.updateMoveBinding(ctx, id, func(in *MoveBinding) bool {
-		changed := in.DelegationID == delegationID && !in.CancelRequested
-		in.CancelRequested = in.CancelRequested || changed
-		return changed
-	})
-}
-
-func (s *Store) AcceptMove(ctx context.Context, id string) error {
-	return s.updateMoveBinding(ctx, id, func(in *MoveBinding) bool {
-		changed := in.Phase != MoveAccepted
-		in.Phase = MoveAccepted
-		return changed
-	})
-}
-
-func (s *Store) FinishMoveMirror(ctx context.Context, id string, cleaned bool, refusal string) error {
-	return s.updateMoveBinding(ctx, id, func(in *MoveBinding) bool {
-		in.CleanupDone = in.CleanupDone || cleaned
-		in.Settled = in.Settled || cleaned
-		if refusal != "" {
-			in.Refusal = refusal
-		}
-		return true
-	})
-}
-
-func (s *Store) moveBindings(ctx context.Context) ([]MoveBinding, error) {
-	out := []MoveBinding{}
-	err := s.withJournal(ctx, func(root *os.Root, _ *journal) (bool, error) {
-		dir, err := root.Open(".")
-		if err != nil {
-			return false, err
-		}
-		defer dir.Close()
-		entries, err := dir.ReadDir(-1)
-		if err != nil {
-			return false, err
-		}
-		for _, entry := range entries {
-			if !strings.HasPrefix(entry.Name(), "move-") || !strings.HasSuffix(entry.Name(), ".json") {
-				continue
-			}
-			// Decode the bounded record to recover its non-path job key; verify it
-			// again through readMoveBinding before trusting even this local mirror.
-			f, err := root.OpenFile(entry.Name(), os.O_RDONLY|unix.O_NOFOLLOW, 0)
-			if err != nil {
-				return false, err
-			}
-			err = checkPrivate(f)
-			data, readErr := io.ReadAll(io.LimitReader(f, 16385))
-			f.Close()
-			var key struct {
-				JobID string `json:"job_id"`
-			}
-			if err != nil || readErr != nil || len(data) > 16384 || json.Unmarshal(data, &key) != nil || entry.Name() != moveBindingName(key.JobID) {
-				return false, ErrInvalid
-			}
-			binding, err := readMoveBinding(root, key.JobID)
-			if err != nil {
-				return false, err
-			}
-			out = append(out, binding)
-		}
-		return false, nil
-	})
-	return out, err
-}
-
-func (s *Store) PendingMoves(ctx context.Context) ([]MoveBinding, error) {
-	bindings, err := s.moveBindings(ctx)
-	out := []MoveBinding{}
-	for _, binding := range bindings {
-		if !binding.Settled && !binding.CleanupDone && binding.Refusal == "" {
-			out = append(out, binding)
-		}
-	}
-	return out, err
-}
-
-// MoveBindings includes settled/refused history. Callers must apply fresh
-// owner authorization; package mirrors never confer destination authority.
-func (s *Store) MoveBindings(ctx context.Context) ([]MoveBinding, error) { return s.moveBindings(ctx) }
-
-// Settled mirrors remain available to explicit status calls and idempotency,
-// but terminal operations do not make the worker poll history indefinitely.
-func (s *Store) SettleMoveMirror(ctx context.Context, id string) error {
-	return s.updateMoveBinding(ctx, id, func(in *MoveBinding) bool {
-		in.Settled = true
-		return true
-	})
-}

package/templates/native-go-delegate/backend/internal/background/delegated_test.go

@@ -1,34 +0,0 @@
-package background
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"testing"
-)
-
-func TestPendingRevocationJournalBoundsAdmission(t *testing.T) {
-	s := NewStore(filepath.Join(t.TempDir(), "spool"))
-	ctx := context.Background()
-	e := s.withJournal(ctx, func(root *os.Root, _ *journal) (bool, error) {
-		for i := 0; i < MaxHistory+MaxQueued; i++ {
-			id := fmt.Sprintf("%032x", i)
-			if e := writeAtomic(root, bindingName(id), DelegationBinding{JobID: id, DelegationID: "grant"}); e != nil {
-				return false, e
-			}
-		}
-		return false, nil
-	})
-	if e != nil {
-		t.Fatal(e)
-	}
-	_, e = s.SubmitDelegated(ctx, SubmitRequest{Key: "key", Name: "file", Data: []byte("input"), Submitter: Submitter{User: "alice", UID: "1205"}}, func(string) (DelegationBinding, error) {
-		t.Error("created grant despite cleanup backlog")
-		return DelegationBinding{DelegationID: "grant"}, nil
-	}, func(DelegationBinding) {})
-	if !errors.Is(e, ErrQuota) {
-		t.Fatalf("got %v, want revocation backlog quota", e)
-	}
-}