@stackonward/cli 0.0.1 → 1.0.0

package/dist/resources/templates/go-backend/internal/core/schedule/scheduler.go

@@ -0,0 +1,1589 @@
+package schedule
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+	"unicode/utf8"
+
+	"github.com/robfig/cron/v3"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
+	"go.uber.org/zap"
+)
+
+// ═══════════════════════════════════════════════════════════════
+// 任务注册信息
+// ═══════════════════════════════════════════════════════════════
+
+// TaskType 任务类型
+type TaskType string
+
+const (
+	TaskTypeCron     TaskType = "cron"
+	TaskTypeInterval TaskType = "interval"
+	TaskTypeManual   TaskType = "manual"
+)
+
+// TaskHandler 是调度器执行任务的统一处理器契约。
+type TaskHandler func(context.Context) error
+
+// RegisteredTask 已注册任务的信息
+type RegisteredTask struct {
+	// 任务名称
+	Name string
+	// 任务类型
+	Type TaskType
+	// 默认 Cron 表达式
+	DefaultPattern string
+	// Cron 表达式（仅 cron 类型）
+	Pattern string
+	// 上次运行时间
+	LastRun *time.Time
+	// 下次运行时间
+	NextRun *time.Time
+	// 运行次数
+	RunCount int64
+	// 错误次数
+	ErrorCount int64
+	// 上次错误信息
+	LastError string
+	// 是否正在运行
+	IsRunning bool
+	// 注册时间
+	RegisteredAt time.Time
+	// cron 入口 ID
+	entryID cron.EntryID
+	// 当前控制面定义版本
+	controlRevision int64
+	// 任务处理器
+	handler TaskHandler
+}
+
+// ExecutionResult 任务执行结果
+type ExecutionResult struct {
+	// 任务名称
+	TaskName string
+	// 开始时间
+	StartTime time.Time
+	// 结束时间
+	EndTime time.Time
+	// 执行时长（毫秒）
+	Duration int64
+	// 是否成功
+	Success bool
+	// 错误信息
+	Error string
+}
+
+const taskRunMetadataSchemaVersion = "scheduler.run.v1"
+
+const dispatchHeartbeatInterval = 10 * time.Second
+
+const taskErrorMaxLength = 512
+
+const (
+	taskDisplayNameMaxRunes      = 191
+	taskDescriptionMaxRunes      = 1000
+	taskMaxRuntimeSecondsLimit   = 86400
+	taskRetryMaxAttemptsLimit    = 10
+	taskRetryBackoffSecondsLimit = 86400
+)
+
+type taskHandlerDisposition uint8
+
+const (
+	taskHandlerSucceeded taskHandlerDisposition = iota
+	taskHandlerFailed
+	taskHandlerSkipped
+)
+
+type taskHandlerOutcome struct {
+	disposition taskHandlerDisposition
+	message     string
+}
+
+var taskErrorSensitivePairPattern = regexp.MustCompile(`(?i)\b(password|passwd|pwd|secret|token|api[_-]?key|access[_-]?token|refresh[_-]?token|authorization|credential)\b\s*[:=]\s*("[^"]*"|'[^']*'|[^\s,;]+)`)
+
+var taskErrorBearerPattern = regexp.MustCompile(`(?i)\bbearer\s+[A-Za-z0-9._~+/\-=]+`)
+
+// ═══════════════════════════════════════════════════════════════
+// 调度器
+// ═══════════════════════════════════════════════════════════════
+
+// Scheduler 定时任务调度器
+type Scheduler struct {
+	cron   *cron.Cron
+	logger *zap.Logger
+
+	mu               sync.RWMutex
+	tasks            map[string]*RegisteredTask
+	executionHistory map[string][]ExecutionResult
+	maxHistorySize   int
+	// executionCtx 是 cron 执行的根生命周期，Stop 通过它向运行中任务广播退出信号。
+	executionCtx    context.Context
+	executionCancel context.CancelFunc
+
+	// 可观测性钩子在启动期注入，保持 schedule 包不反向依赖 observability。
+	tracer   trace.Tracer
+	metrics  jobObserver
+	recorder RunRecorder
+	controls ControlStore
+}
+
+// jobObserver 是 Scheduler 对 metrics 的最小依赖。
+// 抽出 interface 避免 schedule 包反向 import observability（A4 防循环）。
+type jobObserver interface {
+	JobSucceeded(name string, ts time.Time)
+	JobFailed(name string)
+}
+
+// New 创建调度器
+func New(logger *zap.Logger) *Scheduler {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &Scheduler{
+		cron:             cron.New(cron.WithSeconds()),
+		logger:           logger.Named("Scheduler"),
+		tasks:            make(map[string]*RegisteredTask),
+		executionHistory: make(map[string][]ExecutionResult),
+		maxHistorySize:   100,
+		executionCtx:     ctx,
+		executionCancel:  cancel,
+	}
+}
+
+// SetTracer 注入 OTel tracer，使所有 cron 任务自动获得 root span。
+// 应在 cmd 启动期 wire 阶段调一次；运行时不应反复变更。
+func (s *Scheduler) SetTracer(tracer trace.Tracer) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.tracer = tracer
+}
+
+// SetObserver 注入 metrics observer，使所有 cron 任务写入成功时间和连续失败计数。
+// 同 SetTracer 仅启动期注入。
+func (s *Scheduler) SetObserver(obs jobObserver) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.metrics = obs
+}
+
+// SetRunRecorder 注入运行持久化 recorder。
+// recorder 失败只影响运维观测，不阻断业务任务执行。
+func (s *Scheduler) SetRunRecorder(recorder RunRecorder) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.recorder = recorder
+}
+
+// SetControlStore 注入调度控制面存储。
+func (s *Scheduler) SetControlStore(store ControlStore) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.controls = store
+}
+
+// Start 启动调度器
+func (s *Scheduler) Start() {
+	s.mu.Lock()
+	if s.executionCtx == nil || s.executionCtx.Err() != nil {
+		s.executionCtx, s.executionCancel = context.WithCancel(context.Background())
+	}
+	s.mu.Unlock()
+	s.cron.Start()
+	s.logger.Info("scheduler started")
+}
+
+// Stop 停止调度器
+func (s *Scheduler) Stop() {
+	s.mu.RLock()
+	cancel := s.executionCancel
+	s.mu.RUnlock()
+	if cancel != nil {
+		cancel()
+	}
+	ctx := s.cron.Stop()
+	<-ctx.Done()
+	s.logger.Info("scheduler stopped")
+}
+
+// ═══════════════════════════════════════════════════════════════
+// 任务注册
+// ═══════════════════════════════════════════════════════════════
+
+// RegisterCron 注册 cron 定时任务
+func (s *Scheduler) RegisterCron(name, pattern string, fn TaskHandler) error {
+	return s.RegisterCronWithOptions(name, pattern, TaskRegistrationOptions{
+		DisplayName:         name,
+		ConcurrencyPolicy:   ConcurrencyPolicyForbid,
+		ManualRunEnabled:    true,
+		MaxRuntimeSeconds:   3600,
+		RetryMaxAttempts:    0,
+		RetryBackoffSeconds: 60,
+	}, fn)
+}
+
+// RegisterCronWithOptions 注册带首次控制面默认策略的 cron 定时任务。
+func (s *Scheduler) RegisterCronWithOptions(name, pattern string, options TaskRegistrationOptions, fn TaskHandler) error {
+	name = strings.TrimSpace(name)
+	pattern = strings.TrimSpace(pattern)
+	options.DisplayName = strings.TrimSpace(options.DisplayName)
+	options.Description = strings.TrimSpace(options.Description)
+	if err := validateTaskRegistration(name, pattern, options, fn); err != nil {
+		return err
+	}
+	registeredAt := time.Now()
+	effectivePattern := pattern
+	effectiveTimezone := "UTC"
+	var controlRevision int64
+
+	s.mu.RLock()
+	controlStore := s.controls
+	s.mu.RUnlock()
+
+	if controlStore != nil {
+		control, err := controlStore.RegisterTaskControl(context.Background(), TaskControlRegistration{
+			TaskName:            name,
+			TaskType:            TaskTypeCron,
+			DisplayName:         options.DisplayName,
+			Description:         options.Description,
+			DefaultPattern:      pattern,
+			Pattern:             pattern,
+			SourceRuntime:       "worker",
+			ConcurrencyPolicy:   options.ConcurrencyPolicy,
+			ManualRunEnabled:    options.ManualRunEnabled,
+			MaxRuntimeSeconds:   options.MaxRuntimeSeconds,
+			RetryMaxAttempts:    options.RetryMaxAttempts,
+			RetryBackoffSeconds: options.RetryBackoffSeconds,
+			RegisteredAt:        registeredAt,
+		})
+		if err != nil {
+			return fmt.Errorf("register scheduler task control '%s': %w", name, err)
+		}
+		if control != nil && control.Pattern != "" {
+			effectivePattern = control.Pattern
+			effectiveTimezone = control.Timezone
+			controlRevision = control.DefinitionRevision
+		}
+	}
+
+	s.mu.Lock()
+	if _, exists := s.tasks[name]; exists {
+		s.logger.Warn("task already registered, replacing", zap.String("name", name))
+		s.removeLocked(name)
+	}
+
+	wrappedFn := s.wrapTask(name, fn)
+
+	entryID, err := s.cron.AddFunc(schedulerCronSpec(effectivePattern, effectiveTimezone), wrappedFn)
+	if err != nil {
+		s.mu.Unlock()
+		return fmt.Errorf("register cron task '%s': %w", name, err)
+	}
+
+	// 计算下次运行时间
+	entry := s.cron.Entry(entryID)
+	var nextRun *time.Time
+	if !entry.Next.IsZero() {
+		t := entry.Next
+		nextRun = &t
+	}
+
+	task := &RegisteredTask{
+		Name:            name,
+		Type:            TaskTypeCron,
+		DefaultPattern:  pattern,
+		Pattern:         effectivePattern,
+		NextRun:         nextRun,
+		RegisteredAt:    registeredAt,
+		entryID:         entryID,
+		controlRevision: controlRevision,
+		handler:         fn,
+	}
+
+	s.tasks[name] = task
+	s.executionHistory[name] = []ExecutionResult{}
+	recorder := s.recorder
+	taskSnapshot := *task
+	s.mu.Unlock()
+
+	if recorder != nil {
+		if err := recorder.UpsertTask(context.Background(), taskSnapshot); err != nil {
+			s.logger.Error("record scheduler task registration failed",
+				zap.String("name", name),
+				zap.Error(err),
+			)
+		}
+	}
+	s.logger.Info("cron task registered", zap.String("name", name), zap.String("pattern", effectivePattern))
+
+	return nil
+}
+
+func validateTaskRegistration(name, pattern string, options TaskRegistrationOptions, fn TaskHandler) error {
+	if name == "" || utf8.RuneCountInString(name) > taskDisplayNameMaxRunes {
+		return fmt.Errorf("scheduler task name must contain 1-%d characters", taskDisplayNameMaxRunes)
+	}
+	if pattern == "" {
+		return fmt.Errorf("scheduler task %q cron pattern is required", name)
+	}
+	if fn == nil {
+		return fmt.Errorf("scheduler task %q handler is required", name)
+	}
+	if options.DisplayName == "" || utf8.RuneCountInString(options.DisplayName) > taskDisplayNameMaxRunes {
+		return fmt.Errorf("scheduler task %q display name must contain 1-%d characters", name, taskDisplayNameMaxRunes)
+	}
+	if utf8.RuneCountInString(options.Description) > taskDescriptionMaxRunes {
+		return fmt.Errorf("scheduler task %q description exceeds %d characters", name, taskDescriptionMaxRunes)
+	}
+	switch options.ConcurrencyPolicy {
+	case ConcurrencyPolicyForbid, ConcurrencyPolicyAllow:
+	default:
+		return fmt.Errorf("scheduler task %q concurrency policy %q is invalid", name, options.ConcurrencyPolicy)
+	}
+	if options.MaxRuntimeSeconds < 1 || options.MaxRuntimeSeconds > taskMaxRuntimeSecondsLimit {
+		return fmt.Errorf("scheduler task %q max runtime seconds must be between 1 and %d", name, taskMaxRuntimeSecondsLimit)
+	}
+	if options.RetryMaxAttempts < 0 || options.RetryMaxAttempts > taskRetryMaxAttemptsLimit {
+		return fmt.Errorf("scheduler task %q retry max attempts must be between 0 and %d", name, taskRetryMaxAttemptsLimit)
+	}
+	if options.RetryBackoffSeconds < 1 || options.RetryBackoffSeconds > taskRetryBackoffSecondsLimit {
+		return fmt.Errorf("scheduler task %q retry backoff seconds must be between 1 and %d", name, taskRetryBackoffSecondsLimit)
+	}
+	return nil
+}
+
+// RegisterInterval 注册固定间隔任务
+func (s *Scheduler) RegisterInterval(name string, interval time.Duration, fn TaskHandler) error {
+	// 将间隔转为 cron 表达式（秒级）
+	seconds := int(interval.Seconds())
+	if seconds < 1 {
+		return fmt.Errorf("interval must be at least 1 second")
+	}
+
+	pattern := fmt.Sprintf("@every %ds", seconds)
+	return s.RegisterCron(name, pattern, fn)
+}
+
+// Unregister 注销任务
+func (s *Scheduler) Unregister(name string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.removeLocked(name)
+}
+
+func (s *Scheduler) removeLocked(name string) bool {
+	task, exists := s.tasks[name]
+	if !exists {
+		return false
+	}
+
+	s.cron.Remove(task.entryID)
+	delete(s.tasks, name)
+	delete(s.executionHistory, name)
+	s.logger.Info("task unregistered", zap.String("name", name))
+	return true
+}
+
+// ═══════════════════════════════════════════════════════════════
+// 任务包装（自动记录执行）
+// ═══════════════════════════════════════════════════════════════
+
+func (s *Scheduler) wrapTask(name string, fn TaskHandler) func() {
+	return func() {
+		ctx, cancel := s.scheduledRunContext()
+		defer cancel()
+		_, _ = s.executeTaskRun(ctx, name, fn, TaskRunOptions{
+			Trigger: RunTriggerCron,
+			Runtime: "worker",
+		})
+	}
+}
+
+func (s *Scheduler) scheduledRunContext() (context.Context, context.CancelFunc) {
+	s.mu.RLock()
+	parent := s.executionCtx
+	s.mu.RUnlock()
+	if parent == nil {
+		parent = context.Background()
+	}
+	return context.WithCancel(parent)
+}
+
+// ExecuteTask 由控制面命令触发一次已注册任务。
+func (s *Scheduler) ExecuteTask(ctx context.Context, name string, opts TaskRunOptions) (TaskRunResult, error) {
+	s.mu.RLock()
+	task, ok := s.tasks[name]
+	var handler TaskHandler
+	if ok {
+		handler = task.handler
+	}
+	s.mu.RUnlock()
+	if !ok || handler == nil {
+		return TaskRunResult{Status: RunStatusFailed, Error: "task is not registered"}, fmt.Errorf("task %q is not registered", name)
+	}
+	if opts.Trigger == "" {
+		opts.Trigger = RunTriggerManualRun
+	}
+	if opts.Runtime == "" {
+		opts.Runtime = "worker"
+	}
+	return s.executeTaskRun(ctx, name, handler, opts)
+}
+
+// ProcessDispatches 认领后台写入的手动调度命令并执行。
+func (s *Scheduler) ProcessDispatches(ctx context.Context, store DispatchStore, runtime string, limit int) error {
+	if store == nil {
+		return nil
+	}
+	if runtime == "" {
+		runtime = "worker"
+	}
+	if limit <= 0 {
+		limit = 10
+	}
+	dispatches, err := store.ClaimPendingDispatches(ctx, runtime, limit, time.Now().UTC())
+	if err != nil {
+		return fmt.Errorf("claim scheduler dispatches: %w", err)
+	}
+	for _, dispatch := range dispatches {
+		runCtx, stopHeartbeat, err := s.startDispatchHeartbeat(ctx, store, dispatch.ID, dispatch.ClaimToken, runtime)
+		if err != nil {
+			return fmt.Errorf("heartbeat scheduler dispatch %d: %w", dispatch.ID, err)
+		}
+		result, runErr := s.ExecuteTask(runCtx, dispatch.TaskName, TaskRunOptions{
+			Trigger:        RunTriggerManualRun,
+			DispatchID:     dispatch.ID,
+			CorrelationID:  dispatch.CorrelationID,
+			OperatorUserID: dispatch.OperatorUserID,
+			Reason:         dispatch.Reason,
+			Runtime:        runtime,
+			Metadata:       dispatch.Payload,
+		})
+		stopHeartbeat(result.RunID)
+		finishedAt := time.Now().UTC()
+		if runErr != nil && result.Error == "" {
+			result.Error = sanitizeTaskError(runErr.Error())
+			result.Status = RunStatusFailed
+		}
+		if err := store.CompleteDispatch(ctx, dispatch.ID, result.RunID, dispatch.ClaimToken, result, finishedAt); err != nil {
+			return fmt.Errorf("complete scheduler dispatch %d: %w", dispatch.ID, err)
+		}
+	}
+	return nil
+}
+
+func (s *Scheduler) startDispatchHeartbeat(ctx context.Context, store DispatchStore, dispatchID int64, claimToken string, runtime string) (context.Context, func(int64), error) {
+	runCtx, cancel := context.WithCancelCause(ctx)
+	state, err := store.HeartbeatDispatch(ctx, dispatchID, 0, runtime, claimToken, time.Now().UTC())
+	if err != nil {
+		cancel(err)
+		return runCtx, func(int64) {}, err
+	}
+	if state != nil && state.CancelRequested {
+		cancel(dispatchCancellationError(state))
+	}
+	done := make(chan int64, 1)
+	finished := make(chan struct{})
+	go func() {
+		defer close(finished)
+		ticker := time.NewTicker(dispatchHeartbeatInterval)
+		defer ticker.Stop()
+		runID := int64(0)
+		for {
+			select {
+			case id := <-done:
+				runID = id
+				if state, err := store.HeartbeatDispatch(ctx, dispatchID, runID, runtime, claimToken, time.Now().UTC()); err != nil {
+					s.logger.Error("scheduler dispatch heartbeat failed",
+						zap.Int64("dispatchID", dispatchID),
+						zap.Error(err),
+					)
+				} else if state != nil && state.CancelRequested {
+					cancel(dispatchCancellationError(state))
+				}
+				return
+			case <-ctx.Done():
+				cancel(ctx.Err())
+				return
+			case <-ticker.C:
+				state, err := store.HeartbeatDispatch(ctx, dispatchID, runID, runtime, claimToken, time.Now().UTC())
+				if err != nil {
+					s.logger.Error("scheduler dispatch heartbeat failed",
+						zap.Int64("dispatchID", dispatchID),
+						zap.Error(err),
+					)
+					continue
+				}
+				if state != nil && state.CancelRequested {
+					cancel(dispatchCancellationError(state))
+				}
+			}
+		}
+	}()
+	return runCtx, func(runID int64) {
+		select {
+		case done <- runID:
+		case <-finished:
+		}
+		<-finished
+		cancel(context.Canceled)
+	}, nil
+}
+
+type dispatchCancellationCauseError struct {
+	requestedAt time.Time
+	reason      string
+}
+
+func (e *dispatchCancellationCauseError) Error() string {
+	if e == nil || e.reason == "" {
+		return context.Canceled.Error()
+	}
+	return e.reason
+}
+
+func dispatchCancellationError(state *DispatchLeaseState) error {
+	if state == nil {
+		return context.Canceled
+	}
+	requestedAt := time.Now().UTC()
+	if state.CancelRequestedAt != nil && !state.CancelRequestedAt.IsZero() {
+		requestedAt = state.CancelRequestedAt.UTC()
+	}
+	reason := sanitizeTaskError(state.CancelReason)
+	if reason == "" {
+		reason = context.Canceled.Error()
+	}
+	return &dispatchCancellationCauseError{
+		requestedAt: requestedAt,
+		reason:      reason,
+	}
+}
+
+func schedulerContextError(ctx context.Context) string {
+	reason, _ := schedulerCancellationDetails(ctx)
+	return reason
+}
+
+func schedulerCancellationDetails(ctx context.Context) (string, *time.Time) {
+	if cause := context.Cause(ctx); cause != nil {
+		var cancelCause *dispatchCancellationCauseError
+		if errors.As(cause, &cancelCause) {
+			cancelRequestedAt := cancelCause.requestedAt
+			return cancelCause.Error(), &cancelRequestedAt
+		}
+		return sanitizeTaskError(cause.Error()), nil
+	}
+	if err := ctx.Err(); err != nil {
+		return sanitizeTaskError(err.Error()), nil
+	}
+	return "", nil
+}
+
+// ReconcileControls 让运行中 cron entry 跟随控制面表达式变更。
+func (s *Scheduler) ReconcileControls(ctx context.Context) error {
+	s.mu.RLock()
+	store := s.controls
+	s.mu.RUnlock()
+	if store == nil {
+		return nil
+	}
+	controls, err := store.ListTaskControls(ctx)
+	if err != nil {
+		return fmt.Errorf("list scheduler controls: %w", err)
+	}
+	for _, control := range controls {
+		if control.Pattern == "" {
+			continue
+		}
+		if err := s.reconcileControl(control); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Scheduler) reconcileControl(control TaskControl) error {
+	s.mu.Lock()
+	task, exists := s.tasks[control.TaskName]
+	if !exists || task.Pattern == control.Pattern && task.controlRevision == control.DefinitionRevision {
+		s.mu.Unlock()
+		return nil
+	}
+	handler := task.handler
+	s.cron.Remove(task.entryID)
+	wrappedFn := s.wrapTask(task.Name, handler)
+	entryID, err := s.cron.AddFunc(schedulerCronSpec(control.Pattern, control.Timezone), wrappedFn)
+	if err != nil {
+		s.mu.Unlock()
+		return fmt.Errorf("reconcile scheduler task %q: %w", control.TaskName, err)
+	}
+	entry := s.cron.Entry(entryID)
+	var nextRun *time.Time
+	if !entry.Next.IsZero() {
+		t := entry.Next
+		nextRun = &t
+	}
+	task.Pattern = control.Pattern
+	task.NextRun = nextRun
+	task.entryID = entryID
+	task.controlRevision = control.DefinitionRevision
+	taskSnapshot := *task
+	recorder := s.recorder
+	s.mu.Unlock()
+
+	if recorder != nil {
+		if err := recorder.UpsertTask(context.Background(), taskSnapshot); err != nil {
+			s.logger.Error("record reconciled scheduler task failed",
+				zap.String("name", control.TaskName),
+				zap.Error(err),
+			)
+		}
+	}
+	return nil
+}
+
+func schedulerCronSpec(pattern, timezone string) string {
+	if timezone == "" || timezone == "UTC" || len(pattern) > 0 && pattern[0] == '@' {
+		return pattern
+	}
+	return "CRON_TZ=" + timezone + " " + pattern
+}
+
+// ExpireRuntimeExceededRuns 关闭进程异常退出后遗留的超时运行记录。
+func (s *Scheduler) ExpireRuntimeExceededRuns(ctx context.Context) (int64, error) {
+	s.mu.RLock()
+	recorder := s.recorder
+	s.mu.RUnlock()
+	expirer, ok := recorder.(RuntimeExceededRunExpirer)
+	if !ok || expirer == nil {
+		return 0, nil
+	}
+	return expirer.ExpireRuntimeExceededRuns(ctx, time.Now().UTC())
+}
+
+func (s *Scheduler) executeTaskRun(ctx context.Context, name string, fn TaskHandler, opts TaskRunOptions) (TaskRunResult, error) {
+	startTime := time.Now()
+	if opts.Trigger == "" {
+		opts.Trigger = RunTriggerCron
+	}
+	if opts.Runtime == "" {
+		opts.Runtime = "worker"
+	}
+
+	control, controlErr := s.getControl(ctx, name)
+	if controlErr != nil {
+		return TaskRunResult{Status: RunStatusFailed, Error: controlErr.Error()}, controlErr
+	}
+	taskSnapshot, recorder, skipReason := s.markStartedWithControl(name, startTime, control)
+	if taskSnapshot == nil {
+		return TaskRunResult{Status: RunStatusFailed, Error: "task is not registered"}, fmt.Errorf("task %q is not registered", name)
+	}
+	runMetadata := buildTaskRunMetadata(*taskSnapshot, control, opts, startTime)
+	if skipReason != "" {
+		runMetadata["skip"] = map[string]any{"reason": skipReason}
+		opts.Metadata = runMetadata
+		result := s.recordSkipped(ctx, taskSnapshot, recorder, opts, startTime, skipReason)
+		return result, nil
+	}
+
+	s.mu.RLock()
+	tracer := s.tracer
+	obs := s.metrics
+	s.mu.RUnlock()
+	var span trace.Span
+	if tracer != nil {
+		ctx, span = tracer.Start(ctx, "cron."+name,
+			trace.WithSpanKind(trace.SpanKindInternal),
+			trace.WithAttributes(attribute.String("job.name", name)),
+		)
+	}
+
+	s.logger.Info("task started", zap.String("name", name))
+
+	retryMaxAttempts, retryBackoff, maxRuntime := resolveTaskRuntimePolicy(control)
+	runID := int64(0)
+	startRecordCtx := ctx
+	if ctx.Err() != nil {
+		startRecordCtx = context.WithoutCancel(ctx)
+	}
+	if recorder != nil {
+		var err error
+		runID, err = recorder.StartRun(startRecordCtx, TaskRunStart{
+			TaskName:       name,
+			TaskType:       taskSnapshot.Type,
+			Pattern:        taskSnapshot.Pattern,
+			Trigger:        opts.Trigger,
+			DispatchID:     opts.DispatchID,
+			CorrelationID:  opts.CorrelationID,
+			OperatorUserID: opts.OperatorUserID,
+			Reason:         opts.Reason,
+			Runtime:        opts.Runtime,
+			StartedAt:      startTime,
+			Metadata:       runMetadata,
+		})
+		if err != nil {
+			if errors.Is(err, ErrTaskAlreadyRunning) {
+				finishedAt := time.Now()
+				reason := ErrTaskAlreadyRunning.Error()
+				s.markCompleted(name, startTime, finishedAt, RunStatusSkipped, reason)
+				if span != nil {
+					span.SetAttributes(attribute.Float64("job.duration_seconds", finishedAt.Sub(startTime).Seconds()))
+					span.End()
+				}
+				s.logger.Info("task skipped", zap.String("name", name), zap.String("reason", reason))
+				return TaskRunResult{
+					Status:     RunStatusSkipped,
+					Error:      reason,
+					Skipped:    true,
+					SkipReason: reason,
+				}, nil
+			}
+			s.logger.Error("record scheduler task start failed",
+				zap.String("name", name),
+				zap.Error(err),
+			)
+		}
+	}
+	if recorder != nil && runID > 0 {
+		s.recordRunEvent(startRecordCtx, recorder, TaskRunEventRecord{
+			RunID:     runID,
+			EventType: RunEventPolicyEvaluated,
+			Level:     "info",
+			Stage:     "policy",
+			Message:   "Task policy evaluated",
+			Payload:   buildTaskRunPolicyPayload(control, opts, retryMaxAttempts, retryBackoff, maxRuntime),
+			CreatedAt: time.Now(),
+		})
+	}
+	if ctx.Err() != nil {
+		taskErr, cancelRequestedAt := schedulerCancellationDetails(ctx)
+		recordCtx := context.WithoutCancel(ctx)
+		finishedAt := time.Now()
+		duration := finishedAt.Sub(startTime)
+		if recorder != nil && runID > 0 {
+			payload := map[string]any{
+				"dispatch_id": opts.DispatchID,
+				"reason":      taskErr,
+			}
+			if cancelRequestedAt != nil {
+				payload["cancel_requested_at"] = cancelRequestedAt.UTC()
+			}
+			s.recordRunEvent(recordCtx, recorder, TaskRunEventRecord{
+				RunID:     runID,
+				EventType: RunEventCancelRequested,
+				Level:     "warning",
+				Stage:     "control",
+				Message:   "Task cancellation requested",
+				Payload:   payload,
+				CreatedAt: finishedAt,
+			})
+		}
+		s.markCompleted(name, startTime, finishedAt, RunStatusCanceled, taskErr)
+		if recorder != nil {
+			if err := recorder.CompleteRun(recordCtx, TaskRunCompletion{
+				RunID:             runID,
+				TaskName:          name,
+				TaskType:          taskSnapshot.Type,
+				Pattern:           taskSnapshot.Pattern,
+				Trigger:           opts.Trigger,
+				DispatchID:        opts.DispatchID,
+				CorrelationID:     opts.CorrelationID,
+				OperatorUserID:    opts.OperatorUserID,
+				Reason:            opts.Reason,
+				Runtime:           opts.Runtime,
+				StartedAt:         startTime,
+				FinishedAt:        finishedAt,
+				DurationMs:        duration.Milliseconds(),
+				Status:            RunStatusCanceled,
+				Error:             taskErr,
+				CancelRequestedAt: cancelRequestedAt,
+				Metadata:          runMetadata,
+			}); err != nil {
+				s.logger.Error("record scheduler task cancellation failed",
+					zap.String("name", name),
+					zap.Error(err),
+				)
+			}
+		}
+		if span != nil {
+			span.SetStatus(codes.Error, taskErr)
+			span.SetAttributes(attribute.Float64("job.duration_seconds", duration.Seconds()))
+			span.End()
+		}
+		return TaskRunResult{
+			RunID:             runID,
+			Status:            RunStatusCanceled,
+			Error:             taskErr,
+			Canceled:          true,
+			CancelRequestedAt: cancelRequestedAt,
+		}, nil
+	}
+
+	handlerCtx := withRunEventReporter(ctx, runID, recorder, s.logger)
+	cancelHandlerTimeout := func() {}
+	if maxRuntime > 0 {
+		handlerCtx, cancelHandlerTimeout = context.WithTimeoutCause(handlerCtx, maxRuntime, ErrTaskRuntimeExceeded)
+	}
+	taskErr, disposition := s.runTaskHandlerWithRetry(handlerCtx, name, fn, recorder, runID, retryMaxAttempts, retryBackoff)
+	cancelHandlerTimeout()
+
+	finishedAt := time.Now()
+	duration := finishedAt.Sub(startTime)
+	status := RunStatusSucceeded
+	switch disposition {
+	case taskHandlerSkipped:
+		status = RunStatusSkipped
+	case taskHandlerFailed:
+		status = RunStatusFailed
+	}
+	runtimeExceeded := maxRuntime > 0 && (errors.Is(context.Cause(handlerCtx), ErrTaskRuntimeExceeded) || duration > maxRuntime)
+	if runtimeExceeded {
+		status = RunStatusFailed
+		taskErr = ErrTaskRuntimeExceeded.Error()
+	}
+	var cancelRequestedAt *time.Time
+	if ctx.Err() != nil && !runtimeExceeded {
+		status = RunStatusCanceled
+		taskErr, cancelRequestedAt = schedulerCancellationDetails(ctx)
+		recordCtx := context.WithoutCancel(ctx)
+		if recorder != nil && runID > 0 {
+			payload := map[string]any{
+				"dispatch_id": opts.DispatchID,
+				"duration_ms": duration.Milliseconds(),
+				"reason":      taskErr,
+			}
+			if cancelRequestedAt != nil {
+				payload["cancel_requested_at"] = cancelRequestedAt.UTC()
+			}
+			s.recordRunEvent(recordCtx, recorder, TaskRunEventRecord{
+				RunID:     runID,
+				EventType: RunEventCancelRequested,
+				Level:     "warning",
+				Stage:     "control",
+				Message:   "Task cancellation requested",
+				Payload:   payload,
+				CreatedAt: finishedAt,
+			})
+		}
+	}
+	if runtimeExceeded && recorder != nil && runID > 0 {
+		s.recordRunEvent(ctx, recorder, TaskRunEventRecord{
+			RunID:     runID,
+			EventType: RunEventRuntimeExceeded,
+			Level:     "warning",
+			Message:   "Task runtime exceeded control threshold",
+			Payload: map[string]any{
+				"duration_ms":         duration.Milliseconds(),
+				"max_runtime_seconds": int64(maxRuntime / time.Second),
+			},
+			CreatedAt: finishedAt,
+		})
+	}
+	if status == RunStatusSkipped && recorder != nil && runID > 0 {
+		s.recordRunEvent(ctx, recorder, TaskRunEventRecord{
+			RunID:     runID,
+			EventType: RunEventSkipped,
+			Level:     "info",
+			Stage:     "handler",
+			Message:   "Task handler skipped execution",
+			Payload:   map[string]any{"reason": taskErr},
+			CreatedAt: finishedAt,
+		})
+	}
+	s.markCompleted(name, startTime, finishedAt, status, taskErr)
+
+	if recorder != nil {
+		completeCtx := ctx
+		if status == RunStatusCanceled {
+			completeCtx = context.WithoutCancel(ctx)
+		}
+		if err := recorder.CompleteRun(completeCtx, TaskRunCompletion{
+			RunID:             runID,
+			TaskName:          name,
+			TaskType:          taskSnapshot.Type,
+			Pattern:           taskSnapshot.Pattern,
+			Trigger:           opts.Trigger,
+			DispatchID:        opts.DispatchID,
+			CorrelationID:     opts.CorrelationID,
+			OperatorUserID:    opts.OperatorUserID,
+			Reason:            opts.Reason,
+			Runtime:           opts.Runtime,
+			StartedAt:         startTime,
+			FinishedAt:        finishedAt,
+			DurationMs:        duration.Milliseconds(),
+			Status:            status,
+			Error:             taskErr,
+			CancelRequestedAt: cancelRequestedAt,
+			Metadata:          runMetadata,
+		}); err != nil {
+			s.logger.Error("record scheduler task completion failed",
+				zap.String("name", name),
+				zap.Error(err),
+			)
+		}
+	}
+
+	if span != nil {
+		if status == RunStatusFailed || status == RunStatusCanceled {
+			span.SetStatus(codes.Error, taskErr)
+		}
+		span.SetAttributes(attribute.Float64("job.duration_seconds", duration.Seconds()))
+		span.End()
+	}
+
+	if obs != nil {
+		switch status {
+		case RunStatusSucceeded:
+			obs.JobSucceeded(name, time.Now())
+		case RunStatusFailed:
+			obs.JobFailed(name)
+		}
+	}
+
+	if status == RunStatusCanceled {
+		s.logger.Warn("task canceled", zap.String("name", name), zap.Duration("duration", duration), zap.String("reason", taskErr))
+	} else if status == RunStatusSkipped {
+		s.logger.Info("task skipped", zap.String("name", name), zap.Duration("duration", duration), zap.String("reason", taskErr))
+	} else if status == RunStatusSucceeded {
+		s.logger.Info("task completed", zap.String("name", name), zap.Duration("duration", duration))
+	} else {
+		s.logger.Error("task failed", zap.String("name", name), zap.Duration("duration", duration), zap.String("error", taskErr))
+	}
+
+	handlerSkipReason := ""
+	if status == RunStatusSkipped {
+		handlerSkipReason = taskErr
+	}
+	return TaskRunResult{
+		RunID:             runID,
+		Status:            status,
+		Error:             taskErr,
+		Skipped:           status == RunStatusSkipped,
+		SkipReason:        handlerSkipReason,
+		Canceled:          status == RunStatusCanceled,
+		CancelRequestedAt: cancelRequestedAt,
+	}, nil
+}
+
+func buildTaskRunMetadata(task RegisteredTask, control *TaskControl, opts TaskRunOptions, startedAt time.Time) map[string]any {
+	taskSnapshot := map[string]any{
+		"name":             task.Name,
+		"type":             string(task.Type),
+		"pattern":          task.Pattern,
+		"default_pattern":  task.DefaultPattern,
+		"registered_at":    task.RegisteredAt.UTC(),
+		"run_count_before": task.RunCount,
+		"error_count":      task.ErrorCount,
+	}
+	if task.LastRun != nil {
+		taskSnapshot["last_run_at"] = task.LastRun.UTC()
+	}
+	if task.NextRun != nil {
+		taskSnapshot["next_run_at"] = task.NextRun.UTC()
+	}
+	if task.LastError != "" {
+		taskSnapshot["last_error"] = task.LastError
+	}
+
+	triggerSnapshot := map[string]any{
+		"type": string(opts.Trigger),
+	}
+	if opts.DispatchID > 0 {
+		triggerSnapshot["dispatch_id"] = opts.DispatchID
+	}
+	if opts.CorrelationID != "" {
+		triggerSnapshot["correlation_id"] = opts.CorrelationID
+	}
+	if opts.OperatorUserID != nil {
+		triggerSnapshot["operator_user_id"] = *opts.OperatorUserID
+	}
+	if opts.Reason != "" {
+		triggerSnapshot["reason"] = opts.Reason
+	}
+
+	runtimeSnapshot := map[string]any{
+		"name":       opts.Runtime,
+		"started_at": startedAt.UTC(),
+	}
+
+	controlSnapshot := map[string]any{
+		"configured": false,
+	}
+	if control != nil {
+		controlSnapshot = map[string]any{
+			"configured":            true,
+			"status":                string(control.ControlStatus),
+			"concurrency_policy":    string(control.ConcurrencyPolicy),
+			"manual_run_enabled":    control.ManualRunEnabled,
+			"max_runtime_seconds":   control.MaxRuntimeSeconds,
+			"retry_max_attempts":    control.RetryMaxAttempts,
+			"retry_backoff_seconds": control.RetryBackoffSeconds,
+			"definition_revision":   control.DefinitionRevision,
+		}
+		if control.Pattern != "" {
+			controlSnapshot["pattern"] = control.Pattern
+		}
+		if control.Timezone != "" {
+			controlSnapshot["timezone"] = control.Timezone
+		}
+	}
+
+	metadata := map[string]any{
+		"schema_version": taskRunMetadataSchemaVersion,
+		"task":           taskSnapshot,
+		"trigger":        triggerSnapshot,
+		"runtime":        runtimeSnapshot,
+		"control":        controlSnapshot,
+	}
+	if input := cloneTaskRunMetadata(opts.Metadata); len(input) > 0 {
+		metadata["input"] = input
+	}
+	return metadata
+}
+
+func cloneTaskRunMetadata(value map[string]any) map[string]any {
+	if len(value) == 0 {
+		return nil
+	}
+	clone := make(map[string]any, len(value))
+	for key, item := range value {
+		clone[key] = item
+	}
+	return clone
+}
+
+func buildTaskRunPolicyPayload(control *TaskControl, opts TaskRunOptions, retryMaxAttempts int, retryBackoff time.Duration, maxRuntime time.Duration) map[string]any {
+	payload := map[string]any{
+		"trigger_type":          string(opts.Trigger),
+		"retry_max_attempts":    retryMaxAttempts,
+		"retry_backoff_seconds": int64(retryBackoff / time.Second),
+		"max_runtime_seconds":   int64(maxRuntime / time.Second),
+	}
+	if opts.DispatchID > 0 {
+		payload["dispatch_id"] = opts.DispatchID
+	}
+	if opts.CorrelationID != "" {
+		payload["correlation_id"] = opts.CorrelationID
+	}
+	if control == nil {
+		payload["control_configured"] = false
+		return payload
+	}
+	payload["control_configured"] = true
+	payload["control_status"] = string(control.ControlStatus)
+	payload["concurrency_policy"] = string(control.ConcurrencyPolicy)
+	payload["manual_run_enabled"] = control.ManualRunEnabled
+	payload["definition_revision"] = control.DefinitionRevision
+	if control.Pattern != "" {
+		payload["pattern"] = control.Pattern
+	}
+	if control.Timezone != "" {
+		payload["timezone"] = control.Timezone
+	}
+	return payload
+}
+
+func resolveTaskRuntimePolicy(control *TaskControl) (int, time.Duration, time.Duration) {
+	if control == nil {
+		return 0, 60 * time.Second, 0
+	}
+	retryMaxAttempts := control.RetryMaxAttempts
+	if retryMaxAttempts < 0 {
+		retryMaxAttempts = 0
+	}
+	backoffSeconds := control.RetryBackoffSeconds
+	if backoffSeconds <= 0 {
+		backoffSeconds = 60
+	}
+	var maxRuntime time.Duration
+	if control.MaxRuntimeSeconds > 0 {
+		maxRuntime = time.Duration(control.MaxRuntimeSeconds) * time.Second
+	}
+	return retryMaxAttempts, time.Duration(backoffSeconds) * time.Second, maxRuntime
+}
+
+func (s *Scheduler) runTaskHandlerWithRetry(ctx context.Context, name string, fn TaskHandler, recorder RunRecorder, runID int64, retryMaxAttempts int, retryBackoff time.Duration) (string, taskHandlerDisposition) {
+	var taskErr string
+	for attempt := 0; ; attempt++ {
+		if ctx.Err() != nil {
+			return schedulerContextError(ctx), taskHandlerFailed
+		}
+		attemptNumber := attempt + 1
+		attemptStartedAt := time.Now()
+		if recorder != nil && runID > 0 {
+			s.recordRunEvent(ctx, recorder, TaskRunEventRecord{
+				RunID:     runID,
+				EventType: RunEventAttemptStarted,
+				Level:     "info",
+				Stage:     "handler",
+				Message:   "Task attempt started",
+				Payload: map[string]any{
+					"attempt":     attemptNumber,
+					"max_retries": retryMaxAttempts,
+				},
+				CreatedAt: attemptStartedAt,
+			})
+		}
+		outcome := s.runTaskHandlerOnce(ctx, name, runID, fn)
+		taskErr = outcome.message
+		attemptDuration := time.Since(attemptStartedAt)
+		if outcome.disposition == taskHandlerSucceeded {
+			if recorder != nil && runID > 0 {
+				s.recordRunEvent(ctx, recorder, TaskRunEventRecord{
+					RunID:     runID,
+					EventType: RunEventAttemptSucceeded,
+					Level:     "info",
+					Stage:     "handler",
+					Message:   "Task attempt succeeded",
+					Payload: map[string]any{
+						"attempt":     attemptNumber,
+						"duration_ms": attemptDuration.Milliseconds(),
+						"max_retries": retryMaxAttempts,
+					},
+					CreatedAt: time.Now(),
+				})
+			}
+			return "", taskHandlerSucceeded
+		}
+		if outcome.disposition == taskHandlerSkipped {
+			return taskErr, taskHandlerSkipped
+		}
+		willRetry := attempt < retryMaxAttempts && ctx.Err() == nil
+		if recorder != nil && runID > 0 {
+			recordCtx := ctx
+			if ctx.Err() != nil {
+				recordCtx = context.WithoutCancel(ctx)
+			}
+			s.recordRunEvent(recordCtx, recorder, TaskRunEventRecord{
+				RunID:     runID,
+				EventType: RunEventAttemptFailed,
+				Level:     "error",
+				Stage:     "handler",
+				Message:   "Task attempt failed",
+				Payload: map[string]any{
+					"attempt":         attemptNumber,
+					"duration_ms":     attemptDuration.Milliseconds(),
+					"max_retries":     retryMaxAttempts,
+					"will_retry":      willRetry,
+					"backoff_seconds": int64(retryBackoff / time.Second),
+					"error":           taskErr,
+				},
+				CreatedAt: time.Now(),
+			})
+		}
+		if ctx.Err() != nil {
+			return schedulerContextError(ctx), taskHandlerFailed
+		}
+		if attempt >= retryMaxAttempts {
+			return taskErr, taskHandlerFailed
+		}
+		s.logger.Warn("task retry scheduled",
+			zap.String("name", name),
+			zap.Int("attempt", attempt+1),
+			zap.Int("nextAttempt", attempt+2),
+			zap.String("error", taskErr),
+		)
+		if recorder != nil && runID > 0 {
+			s.recordRunEvent(ctx, recorder, TaskRunEventRecord{
+				RunID:     runID,
+				EventType: RunEventRetrying,
+				Level:     "warning",
+				Stage:     "handler",
+				Message:   "Task retry scheduled",
+				Payload: map[string]any{
+					"attempt":         attemptNumber,
+					"next_attempt":    attemptNumber + 1,
+					"max_retries":     retryMaxAttempts,
+					"backoff_seconds": int64(retryBackoff / time.Second),
+					"error":           taskErr,
+				},
+				CreatedAt: time.Now(),
+			})
+		}
+		timer := time.NewTimer(retryBackoff)
+		select {
+		case <-ctx.Done():
+			if !timer.Stop() {
+				select {
+				case <-timer.C:
+				default:
+				}
+			}
+			return schedulerContextError(ctx), taskHandlerFailed
+		case <-timer.C:
+		}
+	}
+}
+
+func (s *Scheduler) runTaskHandlerOnce(ctx context.Context, name string, runID int64, fn TaskHandler) (outcome taskHandlerOutcome) {
+	outcome.disposition = taskHandlerSucceeded
+	defer func() {
+		if r := recover(); r != nil {
+			outcome = taskHandlerOutcome{disposition: taskHandlerFailed, message: safeTaskPanicMessage(r)}
+			s.logger.Error("task panic", schedulerErrorLogFields(name, runID, outcome.message)...)
+		}
+	}()
+	if err := fn(ctx); err != nil {
+		if reason, ok := runSkippedErrorReason(err); ok {
+			outcome = taskHandlerOutcome{disposition: taskHandlerSkipped, message: sanitizeTaskError(reason)}
+			if outcome.message == "" {
+				outcome.message = "task skipped"
+			}
+			return outcome
+		}
+		outcome = taskHandlerOutcome{disposition: taskHandlerFailed, message: sanitizeTaskError(err.Error())}
+		s.logger.Error("task returned error", schedulerErrorLogFields(name, runID, outcome.message)...)
+	}
+	return outcome
+}
+
+func runSkippedErrorReason(err error) (string, bool) {
+	var value RunSkippedError
+	if errors.As(err, &value) {
+		return value.Reason, true
+	}
+	var pointer *RunSkippedError
+	if errors.As(err, &pointer) && pointer != nil {
+		return pointer.Reason, true
+	}
+	return "", false
+}
+
+// recover 返回任意 panic 载荷；这里只保留类型类别，避免秘密值进入日志和持久化。
+func safeTaskPanicMessage(value any) string {
+	switch value.(type) {
+	case error:
+		return "task panic: error"
+	case string:
+		return "task panic: string"
+	default:
+		return "task panic: value"
+	}
+}
+
+func sanitizeTaskError(message string) string {
+	message = strings.TrimSpace(message)
+	if message == "" {
+		return ""
+	}
+	sanitized := taskErrorBearerPattern.ReplaceAllString(message, "Bearer [REDACTED]")
+	sanitized = taskErrorSensitivePairPattern.ReplaceAllString(sanitized, "$1=[REDACTED]")
+	sanitized = strings.Map(func(r rune) rune {
+		if r == '\n' || r == '\r' || r == '\t' {
+			return ' '
+		}
+		if r < 0x20 || r == 0x7f {
+			return -1
+		}
+		return r
+	}, sanitized)
+	sanitized = strings.Join(strings.Fields(sanitized), " ")
+	runes := []rune(sanitized)
+	if len(runes) > taskErrorMaxLength {
+		return string(runes[:taskErrorMaxLength]) + "..."
+	}
+	return sanitized
+}
+
+func schedulerErrorLogFields(name string, runID int64, taskErr string) []zap.Field {
+	fields := []zap.Field{
+		zap.String("name", name),
+		zap.String("error", taskErr),
+	}
+	if runID > 0 {
+		fields = append(fields, zap.Int64("runID", runID))
+	}
+	return fields
+}
+
+func (s *Scheduler) recordRunEvent(ctx context.Context, recorder RunRecorder, event TaskRunEventRecord) {
+	if err := recorder.RecordRunEvent(ctx, event); err != nil {
+		s.logger.Error("record scheduler task event failed",
+			zap.Int64("runID", event.RunID),
+			zap.String("eventType", string(event.EventType)),
+			zap.Error(err),
+		)
+	}
+}
+
+func (s *Scheduler) getControl(ctx context.Context, name string) (*TaskControl, error) {
+	s.mu.RLock()
+	store := s.controls
+	s.mu.RUnlock()
+	if store == nil {
+		return nil, nil
+	}
+	return store.GetTaskControl(ctx, name)
+}
+
+func (s *Scheduler) recordSkipped(ctx context.Context, task *RegisteredTask, recorder RunRecorder, opts TaskRunOptions, startedAt time.Time, reason string) TaskRunResult {
+	finishedAt := time.Now()
+	duration := finishedAt.Sub(startedAt)
+	runID := int64(0)
+	if recorder != nil {
+		var err error
+		runID, err = recorder.StartRun(ctx, TaskRunStart{
+			TaskName:       task.Name,
+			TaskType:       task.Type,
+			Pattern:        task.Pattern,
+			Trigger:        opts.Trigger,
+			DispatchID:     opts.DispatchID,
+			CorrelationID:  opts.CorrelationID,
+			OperatorUserID: opts.OperatorUserID,
+			Reason:         opts.Reason,
+			Runtime:        opts.Runtime,
+			StartedAt:      startedAt,
+			Metadata:       opts.Metadata,
+		})
+		if err != nil {
+			s.logger.Error("record skipped scheduler task start failed",
+				zap.String("name", task.Name),
+				zap.Error(err),
+			)
+		}
+		if err := recorder.CompleteRun(ctx, TaskRunCompletion{
+			RunID:          runID,
+			TaskName:       task.Name,
+			TaskType:       task.Type,
+			Pattern:        task.Pattern,
+			Trigger:        opts.Trigger,
+			DispatchID:     opts.DispatchID,
+			CorrelationID:  opts.CorrelationID,
+			OperatorUserID: opts.OperatorUserID,
+			Reason:         opts.Reason,
+			Runtime:        opts.Runtime,
+			StartedAt:      startedAt,
+			FinishedAt:     finishedAt,
+			DurationMs:     duration.Milliseconds(),
+			Status:         RunStatusSkipped,
+			Error:          reason,
+			Metadata:       opts.Metadata,
+		}); err != nil {
+			s.logger.Error("record skipped scheduler task completion failed",
+				zap.String("name", task.Name),
+				zap.Error(err),
+			)
+		}
+	}
+	s.markCompleted(task.Name, startedAt, finishedAt, RunStatusSkipped, reason)
+	s.logger.Info("task skipped", zap.String("name", task.Name), zap.String("reason", reason))
+	return TaskRunResult{
+		RunID:      runID,
+		Status:     RunStatusSkipped,
+		Error:      reason,
+		Skipped:    true,
+		SkipReason: reason,
+	}
+}
+
+func (s *Scheduler) markStartedWithControl(name string, t time.Time, control *TaskControl) (*RegisteredTask, RunRecorder, string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	task, exists := s.tasks[name]
+	if !exists {
+		return nil, s.recorder, ""
+	}
+	if control != nil {
+		if control.ControlStatus == ControlStatusPaused {
+			taskCopy := *task
+			return &taskCopy, s.recorder, "task is paused"
+		}
+		if control.ConcurrencyPolicy != ConcurrencyPolicyAllow && task.IsRunning {
+			taskCopy := *task
+			return &taskCopy, s.recorder, "task is already running"
+		}
+	} else if task.IsRunning {
+		taskCopy := *task
+		return &taskCopy, s.recorder, "task is already running"
+	}
+	task.IsRunning = true
+	task.LastRun = &t
+	taskCopy := *task
+	return &taskCopy, s.recorder, ""
+}
+
+func (s *Scheduler) markCompleted(name string, startTime, endTime time.Time, status RunStatus, errMsg string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	task, exists := s.tasks[name]
+	if !exists {
+		return
+	}
+
+	task.IsRunning = false
+	task.RunCount++
+
+	if status == RunStatusFailed {
+		task.ErrorCount++
+		task.LastError = errMsg
+	}
+
+	// 更新下次运行时间
+	entry := s.cron.Entry(task.entryID)
+	if !entry.Next.IsZero() {
+		t := entry.Next
+		task.NextRun = &t
+	}
+
+	// 记录执行结果
+	result := ExecutionResult{
+		TaskName:  name,
+		StartTime: startTime,
+		EndTime:   endTime,
+		Duration:  endTime.Sub(startTime).Milliseconds(),
+		Success:   status == RunStatusSucceeded,
+		Error:     errMsg,
+	}
+
+	history := s.executionHistory[name]
+	history = append(history, result)
+	if len(history) > s.maxHistorySize {
+		history = history[1:]
+	}
+	s.executionHistory[name] = history
+}
+
+// ═══════════════════════════════════════════════════════════════
+// 查询
+// ═══════════════════════════════════════════════════════════════
+
+// GetAllTasks 获取所有已注册任务
+func (s *Scheduler) GetAllTasks() []RegisteredTask {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	tasks := make([]RegisteredTask, 0, len(s.tasks))
+	for _, t := range s.tasks {
+		tasks = append(tasks, *t)
+	}
+	return tasks
+}
+
+// GetTask 获取特定任务
+func (s *Scheduler) GetTask(name string) (*RegisteredTask, bool) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	t, ok := s.tasks[name]
+	if !ok {
+		return nil, false
+	}
+	taskCopy := *t
+	return &taskCopy, true
+}
+
+// GetTaskHistory 获取任务执行历史
+func (s *Scheduler) GetTaskHistory(name string, limit int) []ExecutionResult {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	history, exists := s.executionHistory[name]
+	if !exists {
+		return nil
+	}
+
+	if limit > 0 && limit < len(history) {
+		return history[len(history)-limit:]
+	}
+	result := make([]ExecutionResult, len(history))
+	copy(result, history)
+	return result
+}
+
+// GetTaskStatistics 获取任务统计信息
+func (s *Scheduler) GetTaskStatistics(name string) *TaskStatistics {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	task, exists := s.tasks[name]
+	if !exists {
+		return nil
+	}
+
+	history := s.executionHistory[name]
+
+	var totalDuration int64
+	var successCount int64
+	for _, h := range history {
+		totalDuration += h.Duration
+		if h.Success {
+			successCount++
+		}
+	}
+
+	var avgDuration float64
+	var successRate float64
+
+	if len(history) > 0 {
+		avgDuration = float64(totalDuration) / float64(len(history))
+		successRate = float64(successCount) / float64(len(history)) * 100
+	} else {
+		successRate = 100
+	}
+
+	// 最近 10 条
+	recent := history
+	if len(recent) > 10 {
+		recent = recent[len(recent)-10:]
+	}
+
+	return &TaskStatistics{
+		Task:             *task,
+		AverageDuration:  avgDuration,
+		SuccessRate:      successRate,
+		RecentExecutions: recent,
+	}
+}
+
+// TaskStatistics 任务统计
+type TaskStatistics struct {
+	Task             RegisteredTask
+	AverageDuration  float64
+	SuccessRate      float64
+	RecentExecutions []ExecutionResult
+}
+
+// HealthStatus 调度器健康状态
+type HealthStatus struct {
+	Healthy      bool
+	TotalTasks   int
+	RunningTasks int
+	FailedTasks  int
+	CronJobs     int
+}
+
+// GetHealthStatus 获取调度器健康状态
+func (s *Scheduler) GetHealthStatus() HealthStatus {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	var running, failed, cronJobs int
+	for _, t := range s.tasks {
+		if t.IsRunning {
+			running++
+		}
+		if t.ErrorCount > 0 {
+			failed++
+		}
+		if t.Type == TaskTypeCron {
+			cronJobs++
+		}
+	}
+
+	return HealthStatus{
+		Healthy:      true,
+		TotalTasks:   len(s.tasks),
+		RunningTasks: running,
+		FailedTasks:  failed,
+		CronJobs:     cronJobs,
+	}
+}