@stackonward/cli 0.0.1 → 1.0.0

package/dist/resources/templates/go-backend/internal/core/commandreplay/service_test.go

@@ -0,0 +1,891 @@
+package commandreplay
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+
+	"{{module}}/internal/common/canonicaljson"
+	"{{module}}/internal/core/database"
+	"{{module}}/pkg/apierror"
+)
+
+const commandReplayTestNamespace = "test.command.replay"
+
+var commandReplayTestNow = time.Date(2026, 5, 28, 10, 0, 0, 0, time.UTC)
+
+func TestCommandReplayExecuteSucceeds(t *testing.T) {
+	service, repo := newTestCommandReplayService(t)
+	principalType := "identity"
+	principalRef := "user_123"
+	fnCalled := 0
+
+	result, err := service.Execute(context.Background(), CommandReplayBeginRequest{
+		Namespace:      " test.command.replay ",
+		CommandScope:   " quote:123 ",
+		IdempotencyKey: " key-123 ",
+		RequestPayload: json.RawMessage(`{"currency":"USD","amount":100}`),
+		PrincipalType:  &principalType,
+		PrincipalRef:   &principalRef,
+	}, func(ctx context.Context, record *CommandReplayRecord) (*CommandReplayCommandResult, error) {
+		fnCalled++
+		if record.ID == 0 {
+			t.Fatalf("record.ID = 0, want persisted id")
+		}
+		return &CommandReplayCommandResult{
+			Status:          CommandReplayStatusSucceeded,
+			ResponsePayload: json.RawMessage(`{"b":2,"a":1}`),
+		}, nil
+	})
+	if err != nil {
+		t.Fatalf("Execute() error = %v", err)
+	}
+	if fnCalled != 1 {
+		t.Fatalf("fnCalled = %d, want 1", fnCalled)
+	}
+	if result.Replayed {
+		t.Fatalf("Execute() Replayed = true, want false")
+	}
+	if result.Status != CommandReplayStatusSucceeded {
+		t.Fatalf("Execute() Status = %q, want succeeded", result.Status)
+	}
+	if string(result.ResponsePayload) != `{"a":1,"b":2}` {
+		t.Fatalf("Execute() ResponsePayload = %s, want canonical object", result.ResponsePayload)
+	}
+
+	stored := repo.mustFind(t, commandReplayTestNamespace, "quote:123", "key-123")
+	if stored.Status != CommandReplayStatusSucceeded {
+		t.Fatalf("stored Status = %q, want succeeded", stored.Status)
+	}
+	if string(stored.Metadata) != "{}" {
+		t.Fatalf("stored Metadata = %s, want {}", stored.Metadata)
+	}
+	if string(stored.ResponsePayload) != `{"a":1,"b":2}` {
+		t.Fatalf("stored ResponsePayload = %s, want canonical object", stored.ResponsePayload)
+	}
+	if !stored.ExpiresAt.Equal(commandReplayTestNow.Add(defaultCommandReplayTTL)) {
+		t.Fatalf("stored ExpiresAt = %s, want %s", stored.ExpiresAt, commandReplayTestNow.Add(defaultCommandReplayTTL))
+	}
+	if stored.CompletedAt == nil || !stored.CompletedAt.Equal(commandReplayTestNow) {
+		t.Fatalf("stored CompletedAt = %v, want %s", stored.CompletedAt, commandReplayTestNow)
+	}
+	if stored.PrincipalType == nil || *stored.PrincipalType != principalType {
+		t.Fatalf("stored PrincipalType = %v, want %q", stored.PrincipalType, principalType)
+	}
+	if stored.PrincipalRef == nil || *stored.PrincipalRef != principalRef {
+		t.Fatalf("stored PrincipalRef = %v, want %q", stored.PrincipalRef, principalRef)
+	}
+}
+
+func TestCommandReplayExecuteReplaysTerminalRecordWithoutInvokingFunc(t *testing.T) {
+	service, repo := newTestCommandReplayService(t)
+	errorCode := "payment_failed"
+	repo.seed(seedCommandReplayRecord(t, json.RawMessage(`{"amount":100}`), func(record *CommandReplayRecord) {
+		record.Status = CommandReplayStatusFailed
+		record.ResponsePayload = json.RawMessage(`{"error":{"code":"payment_failed","message":"Payment failed","type":"invalid_request_error"}}`)
+		record.ErrorCode = &errorCode
+		record.ErrorStatus = ptrInt(http.StatusConflict)
+		record.ExpiresAt = commandReplayTestNow.Add(time.Hour)
+	}))
+
+	result, err := service.Execute(context.Background(), beginRequest(json.RawMessage(`{"amount":100}`)), func(ctx context.Context, record *CommandReplayRecord) (*CommandReplayCommandResult, error) {
+		t.Fatalf("fn should not be invoked for terminal replay")
+		return nil, nil
+	})
+	assertAPIError(t, err, http.StatusConflict, errorCode)
+	if !result.Replayed {
+		t.Fatalf("Execute() Replayed = false, want true")
+	}
+	if result.Status != CommandReplayStatusFailed {
+		t.Fatalf("Execute() Status = %q, want failed", result.Status)
+	}
+	if string(result.ResponsePayload) != `{"error":{"code":"payment_failed","message":"Payment failed","type":"invalid_request_error"}}` {
+		t.Fatalf("Execute() ResponsePayload = %s", result.ResponsePayload)
+	}
+	if result.ErrorCode == nil || *result.ErrorCode != errorCode {
+		t.Fatalf("Execute() ErrorCode = %v, want %q", result.ErrorCode, errorCode)
+	}
+	if result.ErrorStatus == nil || *result.ErrorStatus != http.StatusConflict {
+		t.Fatalf("Execute() ErrorStatus = %v, want 409", result.ErrorStatus)
+	}
+}
+
+func TestCommandReplayExecuteRejectsInProgressScopeLockConflict(t *testing.T) {
+	service, repo := newTestCommandReplayService(t)
+	repo.lockAcquired = false
+
+	_, err := service.Execute(context.Background(), beginRequest(json.RawMessage(`{"amount":100}`)), func(ctx context.Context, record *CommandReplayRecord) (*CommandReplayCommandResult, error) {
+		t.Fatalf("fn should not be invoked when scope lock is unavailable")
+		return nil, nil
+	})
+	assertAPIError(t, err, http.StatusConflict, errCodeCommandReplayInProgress)
+}
+
+func TestCommandReplayExecuteRejectsHashConflict(t *testing.T) {
+	service, repo := newTestCommandReplayService(t)
+	repo.seed(seedCommandReplayRecord(t, json.RawMessage(`{"amount":100}`), func(record *CommandReplayRecord) {
+		record.Status = CommandReplayStatusSucceeded
+		record.ExpiresAt = commandReplayTestNow.Add(time.Hour)
+	}))
+
+	_, err := service.Execute(context.Background(), beginRequest(json.RawMessage(`{"amount":200}`)), successCommandReplayFunc(t))
+	assertAPIError(t, err, http.StatusConflict, errCodeCommandReplayConflict)
+}
+
+func TestCommandReplayExecuteRejectsExpiredRecord(t *testing.T) {
+	service, repo := newTestCommandReplayService(t)
+	repo.seed(seedCommandReplayRecord(t, json.RawMessage(`{"amount":100}`), func(record *CommandReplayRecord) {
+		record.Status = CommandReplayStatusSucceeded
+		record.ExpiresAt = commandReplayTestNow.Add(-time.Second)
+	}))
+
+	_, err := service.Execute(context.Background(), beginRequest(json.RawMessage(`{"amount":100}`)), successCommandReplayFunc(t))
+	assertAPIError(t, err, http.StatusConflict, errCodeCommandReplayExpired)
+}
+
+func TestCommandReplayExecuteBusinessFailureCommitsAndReturnsBusinessError(t *testing.T) {
+	service, repo := newTestCommandReplayService(t)
+	businessErr := apierror.Conflict("payment_declined", "Payment was declined")
+
+	result, err := service.Execute(context.Background(), beginRequest(json.RawMessage(`{"amount":100}`)), func(ctx context.Context, record *CommandReplayRecord) (*CommandReplayCommandResult, error) {
+		return &CommandReplayCommandResult{
+			Status:      CommandReplayStatusFailed,
+			ErrorCode:   " payment_declined ",
+			ReturnError: businessErr,
+		}, nil
+	})
+	if !errors.Is(err, businessErr) {
+		t.Fatalf("Execute() error = %v, want businessErr", err)
+	}
+	if result == nil || result.Status != CommandReplayStatusFailed {
+		t.Fatalf("Execute() result = %#v, want failed result", result)
+	}
+
+	stored := repo.mustFind(t, commandReplayTestNamespace, "quote:123", "key-123")
+	if stored.Status != CommandReplayStatusFailed {
+		t.Fatalf("stored Status = %q, want failed", stored.Status)
+	}
+	if stored.ErrorCode == nil || *stored.ErrorCode != "payment_declined" {
+		t.Fatalf("stored ErrorCode = %v, want payment_declined", stored.ErrorCode)
+	}
+	if stored.ErrorStatus == nil || *stored.ErrorStatus != http.StatusConflict {
+		t.Fatalf("stored ErrorStatus = %v, want 409", stored.ErrorStatus)
+	}
+	if string(stored.ResponsePayload) != `{"error":{"code":"payment_declined","message":"Payment was declined","type":"invalid_request_error"}}` {
+		t.Fatalf("stored ResponsePayload = %s, want canonical object", stored.ResponsePayload)
+	}
+	if stored.CompletedAt == nil || !stored.CompletedAt.Equal(commandReplayTestNow) {
+		t.Fatalf("stored CompletedAt = %v, want %s", stored.CompletedAt, commandReplayTestNow)
+	}
+}
+
+func TestCommandReplayExecuteInfrastructureErrorDoesNotMarkTerminal(t *testing.T) {
+	service, repo := newTestCommandReplayService(t)
+	infraErr := errors.New("database unavailable")
+
+	_, err := service.Execute(context.Background(), beginRequest(json.RawMessage(`{"amount":100}`)), func(ctx context.Context, record *CommandReplayRecord) (*CommandReplayCommandResult, error) {
+		return nil, infraErr
+	})
+	if !errors.Is(err, infraErr) {
+		t.Fatalf("Execute() error = %v, want infraErr", err)
+	}
+
+	stored := repo.mustFind(t, commandReplayTestNamespace, "quote:123", "key-123")
+	if stored.Status.IsTerminal() {
+		t.Fatalf("stored Status = %q, want non-terminal", stored.Status)
+	}
+}
+
+func TestCommandReplayExecuteRereadsDuplicateCreateAndClassifies(t *testing.T) {
+	service, repo := newTestCommandReplayService(t)
+	payload := json.RawMessage(`{"amount":100}`)
+	repo.createHook = func(record *CommandReplayRecord) error {
+		repo.seed(seedCommandReplayRecord(t, payload, func(existing *CommandReplayRecord) {
+			existing.ID = 99
+			existing.Status = CommandReplayStatusSucceeded
+			existing.ResponsePayload = json.RawMessage(`{"order_id":"ord_123"}`)
+			existing.ExpiresAt = commandReplayTestNow.Add(time.Hour)
+		}))
+		return ErrCommandReplayRecordExists
+	}
+
+	result, err := service.Execute(context.Background(), beginRequest(payload), successCommandReplayFunc(t))
+	if err != nil {
+		t.Fatalf("Execute() error = %v", err)
+	}
+	if !result.Replayed {
+		t.Fatalf("Execute() Replayed = false, want true")
+	}
+	if result.Record.ID != 99 {
+		t.Fatalf("Execute() Record.ID = %d, want 99", result.Record.ID)
+	}
+	if string(result.ResponsePayload) != `{"order_id":"ord_123"}` {
+		t.Fatalf("Execute() ResponsePayload = %s", result.ResponsePayload)
+	}
+}
+
+func TestCommandReplayExecuteRejectsInvalidMetadata(t *testing.T) {
+	service, _ := newTestCommandReplayService(t)
+	req := beginRequest(json.RawMessage(`{"amount":100}`))
+	req.Metadata = json.RawMessage(`[]`)
+
+	_, err := service.Execute(context.Background(), req, successCommandReplayFunc(t))
+	assertAPIError(t, err, http.StatusBadRequest, apierror.CodeValidationFailed)
+}
+
+func TestCommandReplayExecuteRejectsNonObjectPayload(t *testing.T) {
+	service, _ := newTestCommandReplayService(t)
+	req := beginRequest(json.RawMessage(`["not","object"]`))
+
+	_, err := service.Execute(context.Background(), req, successCommandReplayFunc(t))
+	assertAPIError(t, err, http.StatusBadRequest, apierror.CodeValidationFailed)
+}
+
+func TestCommandReplayExecuteRejectsInvalidNamespace(t *testing.T) {
+	service, _ := newTestCommandReplayService(t)
+	req := beginRequest(json.RawMessage(`{"amount":100}`))
+	req.Namespace = "Onex.product.product_beta.checkout"
+
+	_, err := service.Execute(context.Background(), req, successCommandReplayFunc(t))
+	assertAPIError(t, err, http.StatusBadRequest, apierror.CodeValidationFailed)
+}
+
+func TestCommandReplayExecuteRejectsIncompletePrincipal(t *testing.T) {
+	principalType := "identity"
+	principalRef := "ipr_123"
+	blank := " \t "
+	tests := []struct {
+		name          string
+		principalType *string
+		principalRef  *string
+	}{
+		{name: "missing principal ref", principalType: &principalType},
+		{name: "missing principal type", principalRef: &principalRef},
+		{name: "blank principal pair", principalType: &blank, principalRef: &blank},
+		{name: "blank principal ref", principalType: &principalType, principalRef: &blank},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			service, _ := newTestCommandReplayService(t)
+			req := beginRequest(json.RawMessage(`{"amount":100}`))
+			req.PrincipalType = tt.principalType
+			req.PrincipalRef = tt.principalRef
+
+			_, err := service.Execute(context.Background(), req, successCommandReplayFunc(t))
+			assertAPIError(t, err, http.StatusBadRequest, apierror.CodeValidationFailed)
+		})
+	}
+}
+
+func TestCommandReplayExecuteRejectsOverlongFields(t *testing.T) {
+	tests := []struct {
+		name   string
+		mutate func(*CommandReplayBeginRequest)
+	}{
+		{
+			name: "namespace",
+			mutate: func(req *CommandReplayBeginRequest) {
+				req.Namespace = "onex.platform." + strings.Repeat("a", maxNamespaceLength+1)
+			},
+		},
+		{
+			name: "command scope",
+			mutate: func(req *CommandReplayBeginRequest) {
+				req.CommandScope = strings.Repeat("a", maxCommandScopeLength+1)
+			},
+		},
+		{
+			name: "idempotency key",
+			mutate: func(req *CommandReplayBeginRequest) {
+				req.IdempotencyKey = strings.Repeat("a", maxIdempotencyKeyLength+1)
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			service, _ := newTestCommandReplayService(t)
+			req := beginRequest(json.RawMessage(`{"amount":100}`))
+			tt.mutate(&req)
+
+			_, err := service.Execute(context.Background(), req, successCommandReplayFunc(t))
+			assertAPIError(t, err, http.StatusBadRequest, apierror.CodeValidationFailed)
+		})
+	}
+}
+
+func TestCommandReplayExecuteRejectsOverlongErrorCode(t *testing.T) {
+	service, _ := newTestCommandReplayService(t)
+	errorCode := strings.Repeat("a", maxErrorCodeLength+1)
+
+	_, err := service.Execute(context.Background(), beginRequest(json.RawMessage(`{"amount":100}`)), func(ctx context.Context, record *CommandReplayRecord) (*CommandReplayCommandResult, error) {
+		return &CommandReplayCommandResult{
+			Status:      CommandReplayStatusFailed,
+			ErrorCode:   errorCode,
+			ReturnError: apierror.Conflict(errorCode, "Payment was declined"),
+		}, nil
+	})
+	assertAPIError(t, err, http.StatusBadRequest, apierror.CodeValidationFailed)
+}
+
+func TestCommandReplayCanonicalHashStableAcrossJSONKeyOrder(t *testing.T) {
+	first := mustCanonicalHash(t, json.RawMessage(`{"b":2.0,"a":{"d":4,"c":3}}`))
+	second := mustCanonicalHash(t, json.RawMessage(`{"a":{"c":3,"d":4},"b":2}`))
+
+	if first != second {
+		t.Fatalf("canonical hash mismatch: %s != %s", first, second)
+	}
+}
+
+func TestCommandReplayExecuteRejectsDuplicateJSONKeys(t *testing.T) {
+	service, _ := newTestCommandReplayService(t)
+	req := beginRequest(json.RawMessage(`{"amount":100,"amount":200}`))
+
+	_, err := service.Execute(context.Background(), req, successCommandReplayFunc(t))
+	assertAPIError(t, err, http.StatusBadRequest, apierror.CodeValidationFailed)
+}
+
+func TestCommandReplayExternalLifecycle(t *testing.T) {
+	service, repo := newTestCommandReplayService(t)
+	lease, err := service.BeginExternal(context.Background(), beginRequest(json.RawMessage(`{"amount":100}`)), 5*time.Minute, func(ctx context.Context, record *CommandReplayRecord) (CommandReplayExternalReservation, error) {
+		if record.ID == 0 {
+			t.Fatal("reservation record was not persisted")
+		}
+		return CommandReplayExternalReservation{
+			ReferenceType:           "checkout.provider_operation",
+			ReferenceNo:             "cpo_1",
+			PreparedResponsePayload: json.RawMessage(`{"operation_no":"cpo_1","state":"prepared"}`),
+		}, nil
+	})
+	if err != nil {
+		t.Fatalf("BeginExternal() error = %v", err)
+	}
+	if lease.ExecutionToken == "" || lease.AttemptCount != 1 || lease.Reservation.ReferenceNo != "cpo_1" {
+		t.Fatalf("lease = %#v", lease)
+	}
+	stored := repo.mustFind(t, commandReplayTestNamespace, "quote:123", "key-123")
+	if !stored.HasExternalReservation() || stored.ExecutionTokenHash == nil || stored.ExternalAttemptCount != 1 {
+		t.Fatalf("stored external record = %#v", stored)
+	}
+	if string(stored.ResponsePayload) != `{"operation_no":"cpo_1","state":"prepared"}` {
+		t.Fatalf("stored prepared response = %s", stored.ResponsePayload)
+	}
+
+	result, err := service.CompleteExternal(context.Background(), lease.RecordID, lease.ExecutionToken, func(ctx context.Context, record *CommandReplayRecord) (*CommandReplayCommandResult, error) {
+		if record.ExternalReferenceNo == nil || *record.ExternalReferenceNo != "cpo_1" {
+			t.Fatalf("completion record = %#v", record)
+		}
+		return &CommandReplayCommandResult{Status: CommandReplayStatusSucceeded, ResponsePayload: json.RawMessage(`{"operation_no":"cpo_1"}`)}, nil
+	})
+	if err != nil || result.Status != CommandReplayStatusSucceeded {
+		t.Fatalf("CompleteExternal() = %#v, %v", result, err)
+	}
+	stored = repo.mustFind(t, commandReplayTestNamespace, "quote:123", "key-123")
+	if stored.HasExternalReservation() || stored.ExecutionTokenHash != nil || stored.Status != CommandReplayStatusSucceeded {
+		t.Fatalf("completed stored record = %#v", stored)
+	}
+}
+
+func TestCommandReplayExternalRecoveryCarriesPreparedResponse(t *testing.T) {
+	service, _ := newTestCommandReplayService(t)
+	current := commandReplayTestNow
+	service.now = func() time.Time { return current }
+	lease, err := service.BeginExternal(
+		context.Background(),
+		beginRequest(json.RawMessage(`{"amount":100}`)),
+		time.Minute,
+		func(context.Context, *CommandReplayRecord) (CommandReplayExternalReservation, error) {
+			return CommandReplayExternalReservation{
+				ReferenceType:           "checkout.provider_operation",
+				ReferenceNo:             "cpo_1",
+				PreparedResponsePayload: json.RawMessage(`{"b":2,"a":1}`),
+			}, nil
+		},
+	)
+	if err != nil {
+		t.Fatalf("BeginExternal() error = %v", err)
+	}
+	if string(lease.Reservation.PreparedResponsePayload) != `{"a":1,"b":2}` {
+		t.Fatalf("prepared response = %s", lease.Reservation.PreparedResponsePayload)
+	}
+	nextRecoveryAt := current.Add(2 * time.Minute)
+	if err := service.DeferExternal(context.Background(), lease.RecordID, lease.ExecutionToken, "provider_timeout", nextRecoveryAt); err != nil {
+		t.Fatalf("DeferExternal() error = %v", err)
+	}
+	current = nextRecoveryAt
+	recoveryLease, err := service.BeginExternal(
+		context.Background(),
+		beginRequest(json.RawMessage(`{"amount":100}`)),
+		time.Minute,
+		func(context.Context, *CommandReplayRecord) (CommandReplayExternalReservation, error) {
+			t.Fatal("reservation callback ran during recovery")
+			return CommandReplayExternalReservation{}, nil
+		},
+	)
+	if err != nil {
+		t.Fatalf("BeginExternal(recovery) error = %v", err)
+	}
+	if string(recoveryLease.Reservation.PreparedResponsePayload) != `{"a":1,"b":2}` {
+		t.Fatalf("recovered prepared response = %s", recoveryLease.Reservation.PreparedResponsePayload)
+	}
+}
+
+func TestCommandReplayExternalDeferAndRecoveryClaim(t *testing.T) {
+	service, repo := newTestCommandReplayService(t)
+	current := commandReplayTestNow
+	service.now = func() time.Time { return current }
+	lease, err := service.BeginExternal(context.Background(), beginRequest(json.RawMessage(`{"amount":100}`)), time.Minute, func(ctx context.Context, record *CommandReplayRecord) (CommandReplayExternalReservation, error) {
+		return CommandReplayExternalReservation{ReferenceType: "checkout.provider_operation", ReferenceNo: "cpo_1"}, nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	nextRecoveryAt := current.Add(2 * time.Minute)
+	if err := service.DeferExternal(context.Background(), lease.RecordID, lease.ExecutionToken, "provider_timeout", nextRecoveryAt); err != nil {
+		t.Fatalf("DeferExternal() error = %v", err)
+	}
+	stored := repo.mustFind(t, commandReplayTestNamespace, "quote:123", "key-123")
+	if stored.ExecutionTokenHash != nil || stored.ExternalLastErrorCode == nil || *stored.ExternalLastErrorCode != "provider_timeout" {
+		t.Fatalf("deferred record = %#v", stored)
+	}
+	current = nextRecoveryAt
+	recoveryLease, err := service.ClaimExternalRecovery(context.Background(), lease.RecordID, time.Minute)
+	if err != nil {
+		t.Fatalf("ClaimExternalRecovery() error = %v", err)
+	}
+	if recoveryLease.AttemptCount != 2 || recoveryLease.ExecutionToken == lease.ExecutionToken {
+		t.Fatalf("recovery lease = %#v", recoveryLease)
+	}
+}
+
+func TestCommandReplayListDueExternalRecoveriesRejectsUnboundedLimit(t *testing.T) {
+	service, _ := newTestCommandReplayService(t)
+
+	_, err := service.ListDueExternalRecoveries(
+		context.Background(),
+		"checkout.provider_operation",
+		commandReplayTestNow,
+		501,
+	)
+	assertAPIError(t, err, http.StatusBadRequest, apierror.CodeValidationFailed)
+}
+
+func TestCommandReplayExternalRejectsOverlongReferenceFields(t *testing.T) {
+	tests := []struct {
+		name        string
+		reservation CommandReplayExternalReservation
+	}{
+		{
+			name: "reference type",
+			reservation: CommandReplayExternalReservation{
+				ReferenceType: strings.Repeat("a", maxExternalReferenceTypeLength+1),
+				ReferenceNo:   "cpo_1",
+			},
+		},
+		{
+			name: "reference number",
+			reservation: CommandReplayExternalReservation{
+				ReferenceType: "checkout.provider_operation",
+				ReferenceNo:   strings.Repeat("a", maxExternalReferenceNoLength+1),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			service, _ := newTestCommandReplayService(t)
+			_, err := service.BeginExternal(
+				context.Background(),
+				beginRequest(json.RawMessage(`{"amount":100}`)),
+				time.Minute,
+				func(context.Context, *CommandReplayRecord) (CommandReplayExternalReservation, error) {
+					return tt.reservation, nil
+				},
+			)
+			assertAPIError(t, err, http.StatusBadRequest, apierror.CodeValidationFailed)
+		})
+	}
+
+	service, _ := newTestCommandReplayService(t)
+	_, err := service.ListDueExternalRecoveries(
+		context.Background(),
+		strings.Repeat("a", maxExternalReferenceTypeLength+1),
+		commandReplayTestNow,
+		1,
+	)
+	assertAPIError(t, err, http.StatusBadRequest, apierror.CodeValidationFailed)
+}
+
+func TestCommandReplayExternalRejectsStaleExecutionToken(t *testing.T) {
+	service, _ := newTestCommandReplayService(t)
+	lease, err := service.BeginExternal(context.Background(), beginRequest(json.RawMessage(`{"amount":100}`)), time.Minute, func(ctx context.Context, record *CommandReplayRecord) (CommandReplayExternalReservation, error) {
+		return CommandReplayExternalReservation{ReferenceType: "checkout.provider_operation", ReferenceNo: "cpo_1"}, nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	called := false
+	_, err = service.CompleteExternal(context.Background(), lease.RecordID, strings.Repeat("A", 43), func(ctx context.Context, record *CommandReplayRecord) (*CommandReplayCommandResult, error) {
+		called = true
+		return nil, nil
+	})
+	assertAPIError(t, err, http.StatusConflict, errCodeCommandReplayInProgress)
+	if called {
+		t.Fatal("completion callback ran under stale token")
+	}
+}
+
+func TestCommandReplayExternalDeferWithRejectsStaleTokenBeforeBusinessMutation(t *testing.T) {
+	service, _ := newTestCommandReplayService(t)
+	lease, err := service.BeginExternal(context.Background(), beginRequest(json.RawMessage(`{"amount":100}`)), time.Minute, func(context.Context, *CommandReplayRecord) (CommandReplayExternalReservation, error) {
+		return CommandReplayExternalReservation{ReferenceType: "provider_price_mapping", ReferenceNo: "501"}, nil
+	})
+	if err != nil {
+		t.Fatalf("BeginExternal() error = %v", err)
+	}
+	called := false
+	err = service.DeferExternalWith(
+		context.Background(),
+		lease.RecordID,
+		strings.Repeat("A", 43),
+		"provider_timeout",
+		commandReplayTestNow.Add(time.Minute),
+		func(context.Context, *CommandReplayRecord) error {
+			called = true
+			return nil
+		},
+	)
+	assertAPIError(t, err, http.StatusConflict, errCodeCommandReplayInProgress)
+	if called {
+		t.Fatal("defer callback ran under stale token")
+	}
+}
+
+func newTestCommandReplayService(t *testing.T) (*CommandReplayService, *fakeCommandReplayRepository) {
+	t.Helper()
+
+	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+	if err != nil {
+		t.Fatalf("open sqlite: %v", err)
+	}
+	repo := newFakeCommandReplayRepository()
+	service := NewCommandReplayService(repo, database.NewTransactionManager(db))
+	service.now = func() time.Time { return commandReplayTestNow }
+	return service, repo
+}
+
+func beginRequest(payload json.RawMessage) CommandReplayBeginRequest {
+	return CommandReplayBeginRequest{
+		Namespace:      commandReplayTestNamespace,
+		CommandScope:   "quote:123",
+		IdempotencyKey: "key-123",
+		RequestPayload: payload,
+		Metadata:       json.RawMessage(`{"source":"test"}`),
+		TTL:            time.Hour,
+	}
+}
+
+func seedCommandReplayRecord(t *testing.T, payload json.RawMessage, mutate func(*CommandReplayRecord)) *CommandReplayRecord {
+	t.Helper()
+
+	record := &CommandReplayRecord{
+		Namespace:       commandReplayTestNamespace,
+		CommandScope:    "quote:123",
+		IdempotencyKey:  "key-123",
+		RequestHash:     mustCanonicalHash(t, payload),
+		Status:          CommandReplayStatusStarted,
+		ResponsePayload: json.RawMessage(`{}`),
+		ExpiresAt:       commandReplayTestNow.Add(time.Hour),
+		Metadata:        json.RawMessage(`{}`),
+	}
+	if mutate != nil {
+		mutate(record)
+	}
+	return record
+}
+
+func successCommandReplayFunc(t *testing.T) CommandReplayExecuteFunc {
+	t.Helper()
+
+	return func(ctx context.Context, record *CommandReplayRecord) (*CommandReplayCommandResult, error) {
+		return &CommandReplayCommandResult{
+			Status:          CommandReplayStatusSucceeded,
+			ResponsePayload: json.RawMessage(`{"ok":true}`),
+		}, nil
+	}
+}
+
+func mustCanonicalHash(t *testing.T, payload json.RawMessage) string {
+	t.Helper()
+
+	raw, err := canonicalRawJSONObject(payload, false)
+	if err != nil {
+		t.Fatalf("canonicalRawJSONObject() error = %v", err)
+	}
+	hash, err := canonicaljson.Hash(raw)
+	if err != nil {
+		t.Fatalf("canonicaljson.Hash() error = %v", err)
+	}
+	return hash
+}
+
+func assertAPIError(t *testing.T, err error, status int, code string) {
+	t.Helper()
+
+	var apiErr *apierror.APIError
+	if !errors.As(err, &apiErr) {
+		t.Fatalf("error = %v, want APIError", err)
+	}
+	if apiErr.Status != status {
+		t.Fatalf("APIError.Status = %d, want %d", apiErr.Status, status)
+	}
+	if apiErr.Detail.Code != code {
+		t.Fatalf("APIError.Code = %q, want %q", apiErr.Detail.Code, code)
+	}
+}
+
+type fakeCommandReplayRepository struct {
+	mu           sync.Mutex
+	nextID       int64
+	records      map[string]*CommandReplayRecord
+	lockAcquired bool
+	createHook   func(*CommandReplayRecord) error
+}
+
+func newFakeCommandReplayRepository() *fakeCommandReplayRepository {
+	return &fakeCommandReplayRepository{
+		nextID:       1,
+		records:      make(map[string]*CommandReplayRecord),
+		lockAcquired: true,
+	}
+}
+
+func (r *fakeCommandReplayRepository) TryAcquireScopeLock(ctx context.Context, namespace, commandScope, idempotencyKey string) (bool, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	return r.lockAcquired, nil
+}
+
+func (r *fakeCommandReplayRepository) FindByScope(ctx context.Context, namespace, commandScope, idempotencyKey string) (*CommandReplayRecord, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	record := r.records[commandReplayRecordKey(namespace, commandScope, idempotencyKey)]
+	return cloneCommandReplayRecord(record), nil
+}
+
+func (r *fakeCommandReplayRepository) Create(ctx context.Context, record *CommandReplayRecord) error {
+	if r.createHook != nil {
+		return r.createHook(record)
+	}
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	key := commandReplayRecordKey(record.Namespace, record.CommandScope, record.IdempotencyKey)
+	if _, exists := r.records[key]; exists {
+		return ErrCommandReplayRecordExists
+	}
+	stored := cloneCommandReplayRecord(record)
+	if stored.ID == 0 {
+		stored.ID = r.nextID
+		r.nextID++
+	}
+	record.ID = stored.ID
+	r.records[key] = stored
+	return nil
+}
+
+func (r *fakeCommandReplayRepository) MarkSucceeded(ctx context.Context, id int64, responsePayload json.RawMessage, completedAt time.Time) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	record := r.findByIDLocked(id)
+	if record == nil {
+		return errors.New("record not found")
+	}
+	record.Status = CommandReplayStatusSucceeded
+	record.ResponsePayload = append(json.RawMessage(nil), responsePayload...)
+	record.ErrorCode = nil
+	record.ErrorStatus = nil
+	record.CompletedAt = cloneTimePtr(&completedAt)
+	return nil
+}
+
+func (r *fakeCommandReplayRepository) MarkFailed(ctx context.Context, id int64, errorCode string, errorStatus int, responsePayload json.RawMessage, completedAt time.Time) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	record := r.findByIDLocked(id)
+	if record == nil {
+		return errors.New("record not found")
+	}
+	record.Status = CommandReplayStatusFailed
+	record.ErrorCode = &errorCode
+	record.ErrorStatus = &errorStatus
+	record.ResponsePayload = append(json.RawMessage(nil), responsePayload...)
+	record.CompletedAt = cloneTimePtr(&completedAt)
+	return nil
+}
+
+func (r *fakeCommandReplayRepository) FindByIDForUpdate(ctx context.Context, id int64) (*CommandReplayRecord, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return cloneCommandReplayRecord(r.findByIDLocked(id)), nil
+}
+
+func (r *fakeCommandReplayRepository) StartExternalExecution(ctx context.Context, id int64, referenceType, referenceNo, tokenHash string, preparedResponsePayload json.RawMessage, leaseExpiresAt time.Time) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	record := r.findByIDLocked(id)
+	if record == nil || record.Status != CommandReplayStatusStarted || record.ExternalReferenceNo != nil {
+		return errors.New("external execution cannot start")
+	}
+	record.ExternalReferenceType = cloneStringPtr(&referenceType)
+	record.ExternalReferenceNo = cloneStringPtr(&referenceNo)
+	record.ResponsePayload = append(json.RawMessage(nil), preparedResponsePayload...)
+	record.ExecutionTokenHash = cloneStringPtr(&tokenHash)
+	record.LeaseExpiresAt = cloneTimePtr(&leaseExpiresAt)
+	record.ExternalNextRecoveryAt = cloneTimePtr(&leaseExpiresAt)
+	record.ExternalAttemptCount = 1
+	record.ExternalLastErrorCode = nil
+	return nil
+}
+
+func (r *fakeCommandReplayRepository) ClaimExternalRecovery(ctx context.Context, id int64, expectedRecoveryAt time.Time, tokenHash string, leaseExpiresAt time.Time) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	record := r.findByIDLocked(id)
+	if record == nil || record.Status != CommandReplayStatusStarted || record.ExternalNextRecoveryAt == nil ||
+		!record.ExternalNextRecoveryAt.Equal(expectedRecoveryAt) {
+		return errors.New("external recovery cannot be claimed")
+	}
+	record.ExecutionTokenHash = cloneStringPtr(&tokenHash)
+	record.LeaseExpiresAt = cloneTimePtr(&leaseExpiresAt)
+	record.ExternalNextRecoveryAt = cloneTimePtr(&leaseExpiresAt)
+	record.ExternalAttemptCount++
+	record.ExternalLastErrorCode = nil
+	return nil
+}
+
+func (r *fakeCommandReplayRepository) DeferExternalExecution(ctx context.Context, id int64, tokenHash, errorCode string, nextRecoveryAt time.Time) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	record := r.findByIDLocked(id)
+	if record == nil || record.ExecutionTokenHash == nil || *record.ExecutionTokenHash != tokenHash {
+		return errors.New("external execution cannot be deferred")
+	}
+	record.ExecutionTokenHash = nil
+	record.LeaseExpiresAt = nil
+	record.ExternalNextRecoveryAt = cloneTimePtr(&nextRecoveryAt)
+	record.ExternalLastErrorCode = cloneStringPtr(&errorCode)
+	return nil
+}
+
+func (r *fakeCommandReplayRepository) ClearExternalExecution(ctx context.Context, id int64, tokenHash string) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	record := r.findByIDLocked(id)
+	if record == nil || record.ExecutionTokenHash == nil || *record.ExecutionTokenHash != tokenHash {
+		return errors.New("external execution cannot be cleared")
+	}
+	record.ExternalReferenceType = nil
+	record.ExternalReferenceNo = nil
+	record.ExecutionTokenHash = nil
+	record.LeaseExpiresAt = nil
+	record.ExternalNextRecoveryAt = nil
+	record.ExternalAttemptCount = 0
+	record.ExternalLastErrorCode = nil
+	return nil
+}
+
+func (r *fakeCommandReplayRepository) ListDueExternalRecoveries(ctx context.Context, referenceType string, at time.Time, limit int) ([]CommandReplayRecord, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	records := make([]CommandReplayRecord, 0, limit)
+	for _, record := range r.records {
+		if record.ExternalReferenceType != nil && *record.ExternalReferenceType == referenceType && record.Status == CommandReplayStatusStarted &&
+			record.HasExternalReservation() && record.ExternalNextRecoveryAt != nil && !record.ExternalNextRecoveryAt.After(at) && !record.HasActiveExternalLease(at) {
+			records = append(records, *cloneCommandReplayRecord(record))
+			if len(records) == limit {
+				break
+			}
+		}
+	}
+	return records, nil
+}
+
+func (r *fakeCommandReplayRepository) seed(record *CommandReplayRecord) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	stored := cloneCommandReplayRecord(record)
+	if stored.ID == 0 {
+		stored.ID = r.nextID
+		r.nextID++
+	}
+	record.ID = stored.ID
+	key := commandReplayRecordKey(stored.Namespace, stored.CommandScope, stored.IdempotencyKey)
+	r.records[key] = stored
+}
+
+func (r *fakeCommandReplayRepository) mustFind(t *testing.T, namespace, commandScope, idempotencyKey string) *CommandReplayRecord {
+	t.Helper()
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	record := r.records[commandReplayRecordKey(namespace, commandScope, idempotencyKey)]
+	if record == nil {
+		t.Fatalf("record not found")
+	}
+	return cloneCommandReplayRecord(record)
+}
+
+func (r *fakeCommandReplayRepository) findByIDLocked(id int64) *CommandReplayRecord {
+	for _, record := range r.records {
+		if record.ID == id {
+			return record
+		}
+	}
+	return nil
+}
+
+func commandReplayRecordKey(namespace, commandScope, idempotencyKey string) string {
+	return namespace + "\x00" + commandScope + "\x00" + idempotencyKey
+}
+
+func cloneCommandReplayRecord(record *CommandReplayRecord) *CommandReplayRecord {
+	if record == nil {
+		return nil
+	}
+	cloned := *record
+	cloned.PrincipalType = cloneStringPtr(record.PrincipalType)
+	cloned.PrincipalRef = cloneStringPtr(record.PrincipalRef)
+	cloned.ErrorCode = cloneStringPtr(record.ErrorCode)
+	cloned.ErrorStatus = cloneIntPtr(record.ErrorStatus)
+	cloned.ResponsePayload = append(json.RawMessage(nil), record.ResponsePayload...)
+	cloned.Metadata = append(json.RawMessage(nil), record.Metadata...)
+	cloned.CompletedAt = cloneTimePtr(record.CompletedAt)
+	cloned.ExternalReferenceType = cloneStringPtr(record.ExternalReferenceType)
+	cloned.ExternalReferenceNo = cloneStringPtr(record.ExternalReferenceNo)
+	cloned.ExecutionTokenHash = cloneStringPtr(record.ExecutionTokenHash)
+	cloned.LeaseExpiresAt = cloneTimePtr(record.LeaseExpiresAt)
+	cloned.ExternalNextRecoveryAt = cloneTimePtr(record.ExternalNextRecoveryAt)
+	cloned.ExternalLastErrorCode = cloneStringPtr(record.ExternalLastErrorCode)
+	return &cloned
+}
+
+func ptrInt(value int) *int {
+	return &value
+}