@johnnyboy11234/esg-idea-contest 0.2.4 → 0.3.1

package/backend/internal/storage/internal/store/pending.go

@@ -0,0 +1,41 @@
+// Package store holds the SQL of the storage module (pending_file_deletion,
+// V008). Only the storage module may import it.
+package store
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+)
+
+// Execer is satisfied by *sql.DB, *sql.Tx and *sql.Conn.
+type Execer interface {
+	ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error)
+}
+
+// InsertPending records key for deletion; the job may act on it from database
+// time NOW + delay. Returns the row id.
+func InsertPending(ctx context.Context, q Execer, key, reason string, delay time.Duration) (int64, error) {
+	res, err := q.ExecContext(ctx,
+		`INSERT INTO pending_file_deletion (object_key, reason, next_attempt_at)
+		 VALUES (?, ?, NOW(6) + INTERVAL ? MICROSECOND)`,
+		key, reason, delay.Microseconds())
+	if err != nil {
+		return 0, fmt.Errorf("insert pending_file_deletion: %w", err)
+	}
+	id, err := res.LastInsertId()
+	if err != nil {
+		return 0, fmt.Errorf("insert pending_file_deletion: %w", err)
+	}
+	return id, nil
+}
+
+// DeletePending removes one row after its object was deleted. A missing row
+// is not an error.
+func DeletePending(ctx context.Context, q Execer, id int64) error {
+	if _, err := q.ExecContext(ctx, `DELETE FROM pending_file_deletion WHERE id = ?`, id); err != nil {
+		return fmt.Errorf("delete pending_file_deletion %d: %w", id, err)
+	}
+	return nil
+}

package/backend/internal/storage/minio.go

@@ -0,0 +1,139 @@
+package storage
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+
+	"github.com/brian-huanggg/AwardRegistration/backend/internal/shared/web"
+)
+
+const (
+	// DialTimeout bounds connection setup to MinIO (Backend.md §4.4, D50).
+	DialTimeout = 2 * time.Second
+	// CallTimeout bounds one whole MinIO call, including a streamed Get.
+	CallTimeout = 30 * time.Second
+)
+
+// ErrObjectNotFound is returned by Get when the key does not exist. It is a
+// consistency problem (a row points at a missing object), not an outage.
+var ErrObjectNotFound = errors.New("storage: object not found")
+
+// SlideStore is the MinIO port. Failures other than ErrObjectNotFound are
+// *web.Error with code STORAGE_UNAVAILABLE (503).
+type SlideStore interface {
+	// Put writes size bytes from r under key. size must be exact (>= 0).
+	Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) error
+	// Get opens key; the caller must Close the reader. size is the object length.
+	Get(ctx context.Context, key string) (rc io.ReadCloser, size int64, err error)
+	// Delete removes key. Deleting a missing key succeeds.
+	Delete(ctx context.Context, key string) error
+}
+
+// MinIOConfig configures NewMinIO.
+type MinIOConfig struct {
+	Endpoint  string // host:port
+	AccessKey string
+	SecretKey string
+	Bucket    string
+	Secure    bool
+}
+
+type minioStore struct {
+	c      *minio.Client
+	bucket string
+}
+
+// NewMinIO builds the client: 2 s dial timeout, and MaxRetries: 1 — minio-go
+// treats 0 as "default of 10", so 1 is the only way to get a single attempt
+// (Backend.md §4.4, A3c). It makes no network call.
+func NewMinIO(cfg MinIOConfig) (SlideStore, error) {
+	tr := http.DefaultTransport.(*http.Transport).Clone() //nolint:forcetypeassert // stdlib guarantee
+	tr.DialContext = (&net.Dialer{Timeout: DialTimeout, KeepAlive: 30 * time.Second}).DialContext
+	c, err := minio.New(cfg.Endpoint, &minio.Options{
+		Creds:      credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
+		Secure:     cfg.Secure,
+		Transport:  tr,
+		MaxRetries: 1,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("storage: minio client: %w", err)
+	}
+	return &minioStore{c: c, bucket: cfg.Bucket}, nil
+}
+
+func unavailable(op string, err error) error {
+	e := web.NewError(web.CodeStorageUnavailable, "object storage is unavailable")
+	e.Cause = fmt.Errorf("minio %s: %w", op, err)
+	return e
+}
+
+func isNotFound(err error) bool {
+	return minio.ToErrorResponse(err).Code == "NoSuchKey"
+}
+
+func (s *minioStore) Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) error {
+	ctx, cancel := context.WithTimeout(ctx, CallTimeout)
+	defer cancel()
+	_, err := s.c.PutObject(ctx, s.bucket, key, r, size, minio.PutObjectOptions{
+		ContentType: contentType,
+		// Single PUT for slide-sized objects; no multipart retries to reason about.
+		DisableMultipart: true,
+	})
+	if err != nil {
+		return unavailable("put", err)
+	}
+	return nil
+}
+
+func (s *minioStore) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) {
+	ctx, cancel := context.WithTimeout(ctx, CallTimeout)
+	obj, err := s.c.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
+	if err != nil {
+		cancel()
+		return nil, 0, unavailable("get", err)
+	}
+	// GetObject is lazy; Stat performs the request, so a down server or a
+	// missing key surfaces here, before the caller writes a first byte.
+	st, err := obj.Stat()
+	if err != nil {
+		_ = obj.Close()
+		cancel()
+		if isNotFound(err) {
+			return nil, 0, ErrObjectNotFound
+		}
+		return nil, 0, unavailable("stat", err)
+	}
+	return &cancelCloser{ReadCloser: obj, cancel: cancel}, st.Size, nil
+}
+
+func (s *minioStore) Delete(ctx context.Context, key string) error {
+	ctx, cancel := context.WithTimeout(ctx, CallTimeout)
+	defer cancel()
+	if err := s.c.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}); err != nil {
+		if isNotFound(err) {
+			return nil
+		}
+		return unavailable("delete", err)
+	}
+	return nil
+}
+
+// cancelCloser releases the call context when the stream is closed.
+type cancelCloser struct {
+	io.ReadCloser
+	cancel context.CancelFunc
+}
+
+func (c *cancelCloser) Close() error {
+	err := c.ReadCloser.Close()
+	c.cancel()
+	return err
+}

package/backend/internal/storage/minio_integration_test.go

@@ -0,0 +1,87 @@
+//go:build integration
+
+package storage_test
+
+import (
+	"context"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	tcminio "github.com/testcontainers/testcontainers-go/modules/minio"
+
+	"github.com/brian-huanggg/AwardRegistration/backend/internal/shared/web"
+	"github.com/brian-huanggg/AwardRegistration/backend/internal/storage"
+)
+
+func startMinIO(t *testing.T) (storage.MinIOConfig, *tcminio.MinioContainer) {
+	t.Helper()
+	ctx := context.Background()
+	ctr, err := tcminio.Run(ctx, "cgr.dev/chainguard/minio:latest") // same image as docker-compose.yml
+	if err != nil {
+		t.Fatalf("start minio: %v", err)
+	}
+	t.Cleanup(func() { _ = ctr.Terminate(context.WithoutCancel(ctx)) })
+	ep, err := ctr.ConnectionString(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg := storage.MinIOConfig{Endpoint: ep, AccessKey: ctr.Username, SecretKey: ctr.Password, Bucket: "slides"}
+	admin, err := minio.New(ep, &minio.Options{Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, "")})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := admin.MakeBucket(ctx, cfg.Bucket, minio.MakeBucketOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	return cfg, ctr
+}
+
+func TestMinIORoundTrip(t *testing.T) {
+	cfg, ctr := startMinIO(t)
+	ctx := context.Background()
+	s, err := storage.NewMinIO(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	body := "hello slides"
+	if err := s.Put(ctx, "a/b.pdf", strings.NewReader(body), int64(len(body)), "application/pdf"); err != nil {
+		t.Fatal(err)
+	}
+	rc, size, err := s.Get(ctx, "a/b.pdf")
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, _ := io.ReadAll(rc)
+	_ = rc.Close()
+	if string(got) != body || size != int64(len(body)) {
+		t.Fatalf("got %q size %d", got, size)
+	}
+
+	if _, _, err := s.Get(ctx, "missing"); !errors.Is(err, storage.ErrObjectNotFound) {
+		t.Fatalf("missing key: %v", err)
+	}
+	if err := s.Delete(ctx, "a/b.pdf"); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.Delete(ctx, "a/b.pdf"); err != nil {
+		t.Fatalf("delete of missing key must succeed: %v", err)
+	}
+	if _, _, err := s.Get(ctx, "a/b.pdf"); !errors.Is(err, storage.ErrObjectNotFound) {
+		t.Fatalf("after delete: %v", err)
+	}
+
+	// MinIO down → 503 STORAGE_UNAVAILABLE.
+	if err := ctr.Terminate(ctx); err != nil {
+		t.Fatal(err)
+	}
+	err = s.Put(ctx, "x", strings.NewReader("x"), 1, "application/pdf")
+	var we *web.Error
+	if !errors.As(err, &we) || we.Code != web.CodeStorageUnavailable {
+		t.Fatalf("down: %v", err)
+	}
+}

package/backend/internal/storage/minio_test.go

@@ -0,0 +1,84 @@
+package storage
+
+import (
+	"context"
+	"errors"
+	"net"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"github.com/brian-huanggg/AwardRegistration/backend/internal/shared/web"
+)
+
+func newStore(t *testing.T, endpoint string) SlideStore {
+	t.Helper()
+	s, err := NewMinIO(MinIOConfig{Endpoint: endpoint, AccessKey: "a", SecretKey: "bbbbbbbb", Bucket: "slides"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return s
+}
+
+func assertUnavailable(t *testing.T, err error) {
+	t.Helper()
+	var we *web.Error
+	if !errors.As(err, &we) || we.Code != web.CodeStorageUnavailable || we.Status != 503 {
+		t.Fatalf("want 503 STORAGE_UNAVAILABLE, got %v", err)
+	}
+}
+
+// refusing accepts a connection and drops it at once, counting attempts.
+func refusing(t *testing.T) (addr string, attempts *atomic.Int32) {
+	t.Helper()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = ln.Close() })
+	attempts = new(atomic.Int32)
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			attempts.Add(1)
+			_ = c.Close()
+		}
+	}()
+	return ln.Addr().String(), attempts
+}
+
+// OI-52 / A3c: retries are off, so a failing server sees exactly one attempt.
+func TestSingleAttemptPerCall(t *testing.T) {
+	ctx := context.Background()
+	calls := map[string]func(SlideStore) error{
+		"put":    func(s SlideStore) error { return s.Put(ctx, "k", strings.NewReader("abc"), 3, "application/pdf") },
+		"get":    func(s SlideStore) error { _, _, err := s.Get(ctx, "k"); return err },
+		"delete": func(s SlideStore) error { return s.Delete(ctx, "k") },
+	}
+	for name, call := range calls {
+		t.Run(name, func(t *testing.T) {
+			addr, attempts := refusing(t)
+			assertUnavailable(t, call(newStore(t, addr)))
+			if n := attempts.Load(); n != 1 {
+				t.Fatalf("connection attempts = %d, want 1", n)
+			}
+		})
+	}
+}
+
+func TestUnreachableIsUnavailable(t *testing.T) {
+	ln, _ := net.Listen("tcp", "127.0.0.1:0")
+	addr := ln.Addr().String()
+	_ = ln.Close() // nothing listens any more
+	err := newStore(t, addr).Put(context.Background(), "k", strings.NewReader("x"), 1, "application/pdf")
+	assertUnavailable(t, err)
+}
+
+func TestNewMinIORejectsBadEndpoint(t *testing.T) {
+	if _, err := NewMinIO(MinIOConfig{Endpoint: "http://host/path"}); err == nil {
+		t.Fatal("want error")
+	}
+}

package/backend/internal/storage/pending.go

@@ -0,0 +1,144 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/brian-huanggg/AwardRegistration/backend/internal/shared/db"
+	"github.com/brian-huanggg/AwardRegistration/backend/internal/storage/internal/store"
+)
+
+// Reason is why an object key is in pending_file_deletion (Sequence.md §2).
+type Reason string
+
+const (
+	// ReasonReplaced: old key, recorded in the transaction that switched the pointer.
+	ReasonReplaced Reason = "REPLACED"
+	// ReasonDeleted: key of a deleted Idea, recorded in the delete transaction.
+	ReasonDeleted Reason = "DELETED"
+	// ReasonUploadAborted: new key whose transaction rolled back (and the
+	// immediate delete failed) or ended with an unknown outcome.
+	ReasonUploadAborted Reason = "UPLOAD_ABORTED"
+)
+
+// UploadAbortedGrace delays the clean-up of a key recorded after an unknown
+// outcome: longer than the maximum lock wait plus transaction time, so a COMMIT
+// the server is still finishing is never raced (D36). [Proposed] OI-19.
+const UploadAbortedGrace = 10 * time.Minute
+
+// bookkeepingTimeout bounds one database write that runs outside the request's
+// own transaction (separate UPLOAD_ABORTED insert, pending row removal).
+const bookkeepingTimeout = 5 * time.Second
+
+// PendingID identifies a pending_file_deletion row.
+type PendingID int64
+
+// PendingDeletion writes pending_file_deletion (PRD §6.8, D25) and makes the
+// best-effort post-commit delete. The table is drained by the clean-up job
+// (next phase); nothing here waits for it.
+type PendingDeletion struct {
+	db    *sql.DB
+	store SlideStore
+	log   *slog.Logger
+}
+
+// NewPendingDeletion builds the service. log nil discards.
+func NewPendingDeletion(database *sql.DB, s SlideStore, log *slog.Logger) *PendingDeletion {
+	if log == nil {
+		log = slog.New(slog.DiscardHandler)
+	}
+	return &PendingDeletion{db: database, store: s, log: log}
+}
+
+// RecordInTx records key with REPLACED or DELETED inside the caller's
+// transaction, so the row exists exactly when the change committed. After a
+// Committed outcome the caller passes the id to DeleteAfterCommit.
+func (p *PendingDeletion) RecordInTx(ctx context.Context, tx *sql.Tx, key string, reason Reason) (PendingID, error) {
+	if reason != ReasonReplaced && reason != ReasonDeleted {
+		return 0, fmt.Errorf("storage: RecordInTx with reason %q", reason)
+	}
+	id, err := store.InsertPending(ctx, tx, key, string(reason), 0)
+	return PendingID(id), err
+}
+
+// RecordUploadAborted records a newly uploaded key whose transaction did not
+// commit, in its own transaction (best effort; the request is failing anyway).
+//
+//   - RolledBack (the immediate delete failed): due at once.
+//   - Unknown (the COMMIT may have been applied): due after UploadAbortedGrace;
+//     the job keeps the object if an idea row references the key.
+//
+// Committed is a programming error: a committed key is referenced.
+func (p *PendingDeletion) RecordUploadAborted(ctx context.Context, key string, outcome db.Outcome) error {
+	var delay time.Duration
+	switch outcome {
+	case db.RolledBack:
+	case db.Unknown:
+		delay = UploadAbortedGrace
+	default:
+		return fmt.Errorf("storage: RecordUploadAborted after %v", outcome)
+	}
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), bookkeepingTimeout)
+	defer cancel()
+	if _, err := store.InsertPending(ctx, p.db, key, string(ReasonUploadAborted), delay); err != nil {
+		p.log.ErrorContext(ctx, "pending_file_deletion insert failed; object may be orphaned",
+			slog.String("objectKey", key), slog.String("outcome", outcome.String()), slog.Any("error", err))
+		return err
+	}
+	return nil
+}
+
+// AbortUpload compensates a newly uploaded key whose transaction did not
+// commit (Sequence.md §2). It never fails the request, which is failing
+// anyway; every failure is logged.
+//
+//   - RolledBack: delete the object now; if that fails, record UPLOAD_ABORTED
+//     due at once.
+//   - Unknown: keep the object (the COMMIT may have been applied) and record
+//     UPLOAD_ABORTED due after UploadAbortedGrace.
+//
+// The caller's cancellation is ignored, so a client that disconnects does not
+// skip it. Committed is a programming error and is only logged.
+func (p *PendingDeletion) AbortUpload(ctx context.Context, key string, outcome db.Outcome) {
+	switch outcome {
+	case db.RolledBack:
+		dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), CallTimeout)
+		err := p.store.Delete(dctx, key)
+		cancel()
+		if err == nil {
+			return
+		}
+		p.log.WarnContext(ctx, "delete of a rolled-back upload failed; recording UPLOAD_ABORTED",
+			slog.String("objectKey", key), slog.Any("error", err))
+	case db.Unknown:
+	default:
+		p.log.ErrorContext(ctx, "AbortUpload called for a committed key; object kept",
+			slog.String("objectKey", key), slog.String("outcome", outcome.String()))
+		return
+	}
+	// RecordUploadAborted logs its own failure.
+	_ = p.RecordUploadAborted(ctx, key, outcome)
+}
+
+// DeleteAfterCommit deletes key from storage and then its pending row. It runs
+// after the response outcome is fixed and never fails the request: a storage
+// or database failure is logged and the row stays for the clean-up job
+// (Sequence.md §4, §5). The caller's cancellation is ignored, so a client that
+// disconnects does not skip it.
+func (p *PendingDeletion) DeleteAfterCommit(ctx context.Context, id PendingID, key string) {
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), CallTimeout+bookkeepingTimeout)
+	defer cancel()
+	if err := p.store.Delete(ctx, key); err != nil {
+		p.log.WarnContext(ctx, "post-commit object delete failed; left for the clean-up job",
+			slog.String("objectKey", key), slog.Int64("pendingId", int64(id)), slog.Any("error", err))
+		return
+	}
+	if err := store.DeletePending(ctx, p.db, int64(id)); err != nil {
+		// The object is gone; the job's delete of a missing key succeeds.
+		p.log.WarnContext(ctx, "pending_file_deletion row not removed after delete",
+			slog.String("objectKey", key), slog.Int64("pendingId", int64(id)), slog.Any("error", err))
+	}
+}