orch-code 0.1.1

package/internal/confidence/scorer.go

@@ -0,0 +1,253 @@
+package confidence
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/furkanbeydemir/orch/internal/models"
+)
+
+type Scorer struct{}
+
+func New() *Scorer {
+	return &Scorer{}
+}
+
+func (s *Scorer) Score(state *models.RunState) *models.ConfidenceReport {
+	if state == nil {
+		return nil
+	}
+
+	reasons := []string{}
+	warnings := []string{}
+
+	planScore := s.planCompleteness(state, &reasons, &warnings)
+	scopeScore := s.scopeCompliance(state, &reasons, &warnings)
+	validationScore := s.validationQuality(state, &reasons, &warnings)
+	testScore := s.testQuality(state, &reasons, &warnings)
+	reviewScore := s.reviewQuality(state, &reasons, &warnings)
+	retryPenalty := s.retryPenalty(state, &warnings)
+
+	score := planScore*0.10 + scopeScore*0.20 + validationScore*0.20 + testScore*0.25 + reviewScore*0.20 - retryPenalty*0.05
+	if score < 0 {
+		score = 0
+	}
+	if score > 1 {
+		score = 1
+	}
+
+	band := confidenceBand(score)
+	if band == "high" {
+		reasons = append(reasons, "confidence is high because plan, validation, tests, and review signals are aligned")
+	}
+	if band == "low" || band == "very_low" {
+		warnings = append(warnings, "confidence is below the preferred threshold for hands-off trust")
+	}
+
+	return &models.ConfidenceReport{
+		Score:    round2(score),
+		Band:     band,
+		Reasons:  uniqueNonEmpty(reasons),
+		Warnings: uniqueNonEmpty(warnings),
+	}
+}
+
+func (s *Scorer) planCompleteness(state *models.RunState, reasons, warnings *[]string) float64 {
+	if state.Plan == nil {
+		*warnings = append(*warnings, "structured plan is missing")
+		return 0.2
+	}
+	score := 0.3
+	if strings.TrimSpace(state.Plan.Summary) != "" {
+		score += 0.2
+	}
+	if len(state.Plan.FilesToInspect) > 0 {
+		score += 0.15
+	}
+	if len(state.Plan.FilesToModify) > 0 {
+		score += 0.15
+	}
+	if len(state.Plan.AcceptanceCriteria) > 0 {
+		score += 0.1
+		*reasons = append(*reasons, "structured plan includes explicit acceptance criteria")
+	} else {
+		*warnings = append(*warnings, "structured plan acceptance criteria are missing")
+	}
+	if len(state.Plan.TestRequirements) > 0 || strings.TrimSpace(state.Plan.TestStrategy) != "" {
+		score += 0.1
+	}
+	return clamp01(score)
+}
+
+func (s *Scorer) scopeCompliance(state *models.RunState, reasons, warnings *[]string) float64 {
+	for _, result := range state.ValidationResults {
+		if result.Name != "scope_compliance" {
+			continue
+		}
+		switch result.Status {
+		case models.ValidationPass:
+			*reasons = append(*reasons, "scope compliance gate passed")
+			return 1.0
+		case models.ValidationWarn:
+			*warnings = append(*warnings, result.Summary)
+			return 0.6
+		default:
+			*warnings = append(*warnings, result.Summary)
+			return 0.1
+		}
+	}
+	*warnings = append(*warnings, "scope compliance gate result is missing")
+	return 0.4
+}
+
+func (s *Scorer) validationQuality(state *models.RunState, reasons, warnings *[]string) float64 {
+	if len(state.ValidationResults) == 0 {
+		*warnings = append(*warnings, "validation results are missing")
+		return 0.3
+	}
+	passWeight := 0.0
+	totalWeight := 0.0
+	for _, result := range state.ValidationResults {
+		weight := validationWeight(result.Severity)
+		totalWeight += weight
+		switch result.Status {
+		case models.ValidationPass:
+			passWeight += weight
+		case models.ValidationWarn:
+			passWeight += weight * 0.5
+			*warnings = append(*warnings, result.Summary)
+		default:
+			*warnings = append(*warnings, result.Summary)
+		}
+	}
+	if totalWeight == 0 {
+		return 0.3
+	}
+	ratio := passWeight / totalWeight
+	if ratio >= 0.9 {
+		*reasons = append(*reasons, "validation gates passed with high coverage")
+	}
+	return clamp01(ratio)
+}
+
+func (s *Scorer) testQuality(state *models.RunState, reasons, warnings *[]string) float64 {
+	if strings.TrimSpace(state.TestResults) == "" {
+		*warnings = append(*warnings, "test output is missing")
+		return 0.25
+	}
+	score := 0.8
+	lower := strings.ToLower(state.TestResults)
+	if len(state.TestFailures) > 0 {
+		score = 0.2
+		for _, failure := range state.TestFailures {
+			*warnings = append(*warnings, failure.Code+": "+failure.Summary)
+			if failure.Flaky {
+				score = 0.35
+			}
+		}
+	} else if strings.Contains(lower, "fail") || strings.Contains(lower, "panic") {
+		*warnings = append(*warnings, "test output indicates instability or failure history")
+		score = 0.2
+	} else {
+		*reasons = append(*reasons, "test output was recorded for the run")
+	}
+	if state.Retries.Testing > 0 {
+		score -= 0.2
+		*warnings = append(*warnings, fmt.Sprintf("tests required %d retry attempt(s)", state.Retries.Testing))
+	}
+	return clamp01(score)
+}
+
+func (s *Scorer) reviewQuality(state *models.RunState, reasons, warnings *[]string) float64 {
+	if state.ReviewScorecard == nil {
+		*warnings = append(*warnings, "review scorecard is missing")
+		return 0.3
+	}
+	avg := float64(
+		state.ReviewScorecard.RequirementCoverage+
+			state.ReviewScorecard.ScopeControl+
+			state.ReviewScorecard.RegressionRisk+
+			state.ReviewScorecard.Readability+
+			state.ReviewScorecard.Maintainability+
+			state.ReviewScorecard.TestAdequacy,
+	) / 60.0
+	if state.ReviewScorecard.Decision == models.ReviewAccept {
+		*reasons = append(*reasons, "review rubric accepted the patch")
+	} else {
+		*warnings = append(*warnings, "review rubric did not fully accept the patch")
+		avg *= 0.7
+	}
+	return clamp01(avg)
+}
+
+func (s *Scorer) retryPenalty(state *models.RunState, warnings *[]string) float64 {
+	total := state.Retries.Validation + state.Retries.Testing + state.Retries.Review
+	if total == 0 {
+		return 0
+	}
+	*warnings = append(*warnings, fmt.Sprintf("run used %d total retry attempt(s)", total))
+	if total >= 3 {
+		return 1
+	}
+	if total == 2 {
+		return 0.6
+	}
+	return 0.3
+}
+
+func validationWeight(severity models.ValidationSeverity) float64 {
+	switch severity {
+	case models.SeverityCritical:
+		return 1.5
+	case models.SeverityHigh:
+		return 1.2
+	case models.SeverityMedium:
+		return 1.0
+	default:
+		return 0.8
+	}
+}
+
+func confidenceBand(score float64) string {
+	switch {
+	case score >= 0.85:
+		return "high"
+	case score >= 0.70:
+		return "medium"
+	case score >= 0.50:
+		return "low"
+	default:
+		return "very_low"
+	}
+}
+
+func clamp01(v float64) float64 {
+	if v < 0 {
+		return 0
+	}
+	if v > 1 {
+		return 1
+	}
+	return v
+}
+
+func round2(v float64) float64 {
+	return float64(int(v*100+0.5)) / 100
+}
+
+func uniqueNonEmpty(values []string) []string {
+	result := make([]string, 0, len(values))
+	seen := map[string]struct{}{}
+	for _, value := range values {
+		trimmed := strings.TrimSpace(value)
+		if trimmed == "" {
+			continue
+		}
+		if _, ok := seen[trimmed]; ok {
+			continue
+		}
+		seen[trimmed] = struct{}{}
+		result = append(result, trimmed)
+	}
+	return result
+}

package/internal/confidence/scorer_test.go

@@ -0,0 +1,83 @@
+package confidence
+
+import (
+	"testing"
+	"time"
+
+	"github.com/furkanbeydemir/orch/internal/models"
+)
+
+func TestScoreHighConfidenceRun(t *testing.T) {
+	scorer := New()
+	state := &models.RunState{
+		Task: models.Task{ID: "task-1", Description: "fix auth bug", CreatedAt: time.Now()},
+		Plan: &models.Plan{
+			Summary:            "Fix auth bug",
+			FilesToInspect:     []string{"auth.go"},
+			FilesToModify:      []string{"auth.go"},
+			AcceptanceCriteria: []models.AcceptanceCriterion{{ID: "ac-1", Description: "Bug fixed"}},
+			TestRequirements:   []string{"Run go test ./..."},
+		},
+		ValidationResults: []models.ValidationResult{
+			{Name: "scope_compliance", Status: models.ValidationPass, Severity: models.SeverityLow},
+			{Name: "patch_hygiene", Status: models.ValidationPass, Severity: models.SeverityLow},
+			{Name: "plan_compliance", Status: models.ValidationPass, Severity: models.SeverityLow},
+		},
+		ReviewScorecard: &models.ReviewScorecard{
+			RequirementCoverage: 9,
+			ScopeControl:        9,
+			RegressionRisk:      8,
+			Readability:         8,
+			Maintainability:     8,
+			TestAdequacy:        9,
+			Decision:            models.ReviewAccept,
+		},
+		TestResults: "ok   auth",
+	}
+
+	report := scorer.Score(state)
+	if report == nil {
+		t.Fatalf("expected confidence report")
+	}
+	if report.Band != "high" && report.Band != "medium" {
+		t.Fatalf("expected healthy confidence band, got %s", report.Band)
+	}
+	if report.Score <= 0.69 {
+		t.Fatalf("expected higher score, got %f", report.Score)
+	}
+}
+
+func TestScoreLowConfidenceRun(t *testing.T) {
+	scorer := New()
+	state := &models.RunState{
+		Task: models.Task{ID: "task-2", Description: "feature", CreatedAt: time.Now()},
+		Plan: &models.Plan{},
+		ValidationResults: []models.ValidationResult{
+			{Name: "scope_compliance", Status: models.ValidationFail, Severity: models.SeverityHigh, Summary: "scope fail"},
+		},
+		ReviewScorecard: &models.ReviewScorecard{
+			RequirementCoverage: 4,
+			ScopeControl:        3,
+			RegressionRisk:      4,
+			Readability:         5,
+			Maintainability:     5,
+			TestAdequacy:        2,
+			Decision:            models.ReviewRevise,
+		},
+		Retries: models.RetryState{Validation: 2, Testing: 1},
+	}
+
+	report := scorer.Score(state)
+	if report == nil {
+		t.Fatalf("expected confidence report")
+	}
+	if report.Band != "low" && report.Band != "very_low" {
+		t.Fatalf("expected low confidence band, got %s", report.Band)
+	}
+	if report.Score >= 0.70 {
+		t.Fatalf("expected low score, got %f", report.Score)
+	}
+	if len(report.Warnings) == 0 {
+		t.Fatalf("expected warnings for low confidence")
+	}
+}

package/internal/config/config.go

@@ -0,0 +1,331 @@
+package config
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+const (
+	OrchDir     = ".orch"
+	ConfigFile  = "config.json"
+	RepoMapFile = "repo-map.json"
+	RunsDir     = "runs"
+)
+
+type Config struct {
+	Version  int            `json:"version"`
+	Models   ModelConfig    `json:"models"`
+	Commands CommandConfig  `json:"commands"`
+	Patch    PatchConfig    `json:"patch"`
+	Safety   SafetyConfig   `json:"safety"`
+	Provider ProviderConfig `json:"provider"`
+}
+
+type ModelConfig struct {
+	Planner  string `json:"planner"`
+	Coder    string `json:"coder"`
+	Reviewer string `json:"reviewer"`
+}
+
+type CommandConfig struct {
+	Test string `json:"test"`
+	Lint string `json:"lint"`
+}
+
+type PatchConfig struct {
+	MaxFiles int `json:"maxFiles"`
+	MaxLines int `json:"maxLines"`
+}
+
+type SafetyConfig struct {
+	DryRun                     bool                   `json:"dryRun"`
+	RequireDestructiveApproval bool                   `json:"requireDestructiveApproval"`
+	LockStaleAfterSeconds      int                    `json:"lockStaleAfterSeconds"`
+	Retry                      RetryPolicyConfig      `json:"retry"`
+	Confidence                 ConfidencePolicyConfig `json:"confidence"`
+	FeatureFlags               SafetyFeatureFlags     `json:"featureFlags"`
+}
+
+type RetryPolicyConfig struct {
+	ValidationMax int `json:"validationMax"`
+	TestMax       int `json:"testMax"`
+	ReviewMax     int `json:"reviewMax"`
+}
+
+type ConfidencePolicyConfig struct {
+	CompleteMin float64 `json:"completeMin"`
+	FailBelow   float64 `json:"failBelow"`
+}
+
+type SafetyFeatureFlags struct {
+	PermissionMode         bool `json:"permissionMode"`
+	RepoLock               bool `json:"repoLock"`
+	RetryLimits            bool `json:"retryLimits"`
+	PatchConflictReporting bool `json:"patchConflictReporting"`
+	ConfidenceEnforcement  bool `json:"confidenceEnforcement"`
+}
+
+type ProviderConfig struct {
+	Default string               `json:"default"`
+	OpenAI  OpenAIProviderConfig `json:"openai"`
+	Flags   ProviderFeatureFlags `json:"flags"`
+}
+
+type OpenAIProviderConfig struct {
+	APIKeyEnv       string             `json:"apiKeyEnv"`
+	AccountTokenEnv string             `json:"accountTokenEnv"`
+	AuthMode        string             `json:"authMode"`
+	BaseURL         string             `json:"baseURL"`
+	ReasoningEffort string             `json:"reasoningEffort"`
+	TimeoutSeconds  int                `json:"timeoutSeconds"`
+	MaxRetries      int                `json:"maxRetries"`
+	Models          ProviderRoleModels `json:"models"`
+}
+
+type ProviderRoleModels struct {
+	Planner  string `json:"planner"`
+	Coder    string `json:"coder"`
+	Reviewer string `json:"reviewer"`
+}
+
+type ProviderFeatureFlags struct {
+	OpenAIEnabled bool `json:"openaiEnabled"`
+}
+
+func DefaultConfig() *Config {
+	return &Config{
+		Version: 1,
+		Models: ModelConfig{
+			Planner:  "openai:gpt-4o-mini",
+			Coder:    "anthropic:claude-sonnet",
+			Reviewer: "openai:gpt-4o-mini",
+		},
+		Commands: CommandConfig{
+			Test: "go test ./...",
+			Lint: "go vet ./...",
+		},
+		Patch: PatchConfig{
+			MaxFiles: 10,
+			MaxLines: 800,
+		},
+		Safety: SafetyConfig{
+			DryRun:                     true,
+			RequireDestructiveApproval: true,
+			LockStaleAfterSeconds:      3600,
+			Retry: RetryPolicyConfig{
+				ValidationMax: 2,
+				TestMax:       2,
+				ReviewMax:     2,
+			},
+			Confidence: ConfidencePolicyConfig{
+				CompleteMin: 0.70,
+				FailBelow:   0.50,
+			},
+			FeatureFlags: SafetyFeatureFlags{
+				PermissionMode:         true,
+				RepoLock:               true,
+				RetryLimits:            true,
+				PatchConflictReporting: true,
+				ConfidenceEnforcement:  true,
+			},
+		},
+		Provider: ProviderConfig{
+			Default: "openai",
+			OpenAI: OpenAIProviderConfig{
+				APIKeyEnv:       "OPENAI_API_KEY",
+				AccountTokenEnv: "OPENAI_ACCOUNT_TOKEN",
+				AuthMode:        "api_key",
+				BaseURL:         "https://api.openai.com/v1",
+				ReasoningEffort: "medium",
+				TimeoutSeconds:  90,
+				MaxRetries:      3,
+				Models: ProviderRoleModels{
+					Planner:  "gpt-5.3-codex",
+					Coder:    "gpt-5.3-codex",
+					Reviewer: "gpt-5.3-codex",
+				},
+			},
+			Flags: ProviderFeatureFlags{
+				OpenAIEnabled: true,
+			},
+		},
+	}
+}
+
+func Load(repoRoot string) (*Config, error) {
+	configPath := filepath.Join(repoRoot, OrchDir, ConfigFile)
+
+	data, err := os.ReadFile(configPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read config file: %w", err)
+	}
+
+	var cfg Config
+	if err := json.Unmarshal(data, &cfg); err != nil {
+		return nil, fmt.Errorf("failed to parse config: %w", err)
+	}
+
+	applyDefaults(&cfg, data)
+
+	return &cfg, nil
+}
+
+func applyDefaults(cfg *Config, rawJSON []byte) {
+	defaults := DefaultConfig()
+	presence := parsePresence(rawJSON)
+
+	if cfg.Safety.LockStaleAfterSeconds <= 0 {
+		cfg.Safety.LockStaleAfterSeconds = defaults.Safety.LockStaleAfterSeconds
+	}
+
+	if cfg.Safety.Retry.ValidationMax <= 0 {
+		cfg.Safety.Retry.ValidationMax = defaults.Safety.Retry.ValidationMax
+	}
+	if cfg.Safety.Retry.TestMax <= 0 {
+		cfg.Safety.Retry.TestMax = defaults.Safety.Retry.TestMax
+	}
+	if cfg.Safety.Retry.ReviewMax <= 0 {
+		cfg.Safety.Retry.ReviewMax = defaults.Safety.Retry.ReviewMax
+	}
+	if cfg.Safety.Confidence.CompleteMin <= 0 {
+		cfg.Safety.Confidence.CompleteMin = defaults.Safety.Confidence.CompleteMin
+	}
+	if cfg.Safety.Confidence.FailBelow <= 0 {
+		cfg.Safety.Confidence.FailBelow = defaults.Safety.Confidence.FailBelow
+	}
+
+	if !presence.featureFlagsPresent {
+		cfg.Safety.FeatureFlags = defaults.Safety.FeatureFlags
+	}
+	if !presence.confidenceEnforcementPresent {
+		cfg.Safety.FeatureFlags.ConfidenceEnforcement = defaults.Safety.FeatureFlags.ConfidenceEnforcement
+	}
+
+	if !presence.requireDestructiveApprovalPresent {
+		cfg.Safety.RequireDestructiveApproval = defaults.Safety.RequireDestructiveApproval
+	}
+
+	if !presence.providerPresent {
+		cfg.Provider = defaults.Provider
+	} else {
+		if cfg.Provider.Default == "" {
+			cfg.Provider.Default = defaults.Provider.Default
+		}
+		if cfg.Provider.OpenAI.APIKeyEnv == "" {
+			cfg.Provider.OpenAI.APIKeyEnv = defaults.Provider.OpenAI.APIKeyEnv
+		}
+		if cfg.Provider.OpenAI.AccountTokenEnv == "" {
+			cfg.Provider.OpenAI.AccountTokenEnv = defaults.Provider.OpenAI.AccountTokenEnv
+		}
+		if cfg.Provider.OpenAI.AuthMode == "" {
+			cfg.Provider.OpenAI.AuthMode = defaults.Provider.OpenAI.AuthMode
+		}
+		if cfg.Provider.OpenAI.BaseURL == "" {
+			cfg.Provider.OpenAI.BaseURL = defaults.Provider.OpenAI.BaseURL
+		}
+		if cfg.Provider.OpenAI.ReasoningEffort == "" {
+			cfg.Provider.OpenAI.ReasoningEffort = defaults.Provider.OpenAI.ReasoningEffort
+		}
+		if cfg.Provider.OpenAI.TimeoutSeconds <= 0 {
+			cfg.Provider.OpenAI.TimeoutSeconds = defaults.Provider.OpenAI.TimeoutSeconds
+		}
+		if cfg.Provider.OpenAI.MaxRetries <= 0 {
+			cfg.Provider.OpenAI.MaxRetries = defaults.Provider.OpenAI.MaxRetries
+		}
+		if cfg.Provider.OpenAI.Models.Planner == "" {
+			cfg.Provider.OpenAI.Models.Planner = defaults.Provider.OpenAI.Models.Planner
+		}
+		if cfg.Provider.OpenAI.Models.Coder == "" {
+			cfg.Provider.OpenAI.Models.Coder = defaults.Provider.OpenAI.Models.Coder
+		}
+		if cfg.Provider.OpenAI.Models.Reviewer == "" {
+			cfg.Provider.OpenAI.Models.Reviewer = defaults.Provider.OpenAI.Models.Reviewer
+		}
+		if !presence.providerFlagsPresent {
+			cfg.Provider.Flags = defaults.Provider.Flags
+		}
+	}
+}
+
+type fieldPresence struct {
+	featureFlagsPresent               bool
+	confidenceEnforcementPresent      bool
+	requireDestructiveApprovalPresent bool
+	providerPresent                   bool
+	providerFlagsPresent              bool
+}
+
+func parsePresence(rawJSON []byte) fieldPresence {
+	result := fieldPresence{}
+
+	var root map[string]json.RawMessage
+	if err := json.Unmarshal(rawJSON, &root); err != nil {
+		return result
+	}
+
+	safetyRaw, ok := root["safety"]
+	if !ok {
+		return result
+	}
+
+	var safety map[string]json.RawMessage
+	if err := json.Unmarshal(safetyRaw, &safety); err != nil {
+		return result
+	}
+
+	if featureFlagsRaw, ok := safety["featureFlags"]; ok {
+		result.featureFlagsPresent = true
+		var featureFlags map[string]json.RawMessage
+		if err := json.Unmarshal(featureFlagsRaw, &featureFlags); err == nil {
+			_, result.confidenceEnforcementPresent = featureFlags["confidenceEnforcement"]
+		}
+	}
+	_, result.requireDestructiveApprovalPresent = safety["requireDestructiveApproval"]
+
+	providerRaw, ok := root["provider"]
+	if ok {
+		result.providerPresent = true
+		var provider map[string]json.RawMessage
+		if err := json.Unmarshal(providerRaw, &provider); err == nil {
+			_, result.providerFlagsPresent = provider["flags"]
+		}
+	}
+
+	return result
+}
+
+func Save(repoRoot string, cfg *Config) error {
+	orchDir := filepath.Join(repoRoot, OrchDir)
+	if err := os.MkdirAll(orchDir, 0o755); err != nil {
+		return fmt.Errorf("failed to create .orch directory: %w", err)
+	}
+
+	data, err := json.MarshalIndent(cfg, "", "  ")
+	if err != nil {
+		return fmt.Errorf("failed to serialize config: %w", err)
+	}
+
+	configPath := filepath.Join(orchDir, ConfigFile)
+	if err := os.WriteFile(configPath, data, 0o644); err != nil {
+		return fmt.Errorf("failed to write config file: %w", err)
+	}
+
+	return nil
+}
+
+func EnsureOrchDir(repoRoot string) error {
+	dirs := []string{
+		filepath.Join(repoRoot, OrchDir),
+		filepath.Join(repoRoot, OrchDir, RunsDir),
+	}
+
+	for _, dir := range dirs {
+		if err := os.MkdirAll(dir, 0o755); err != nil {
+			return fmt.Errorf("failed to create directory %s: %w", dir, err)
+		}
+	}
+
+	return nil
+}