orch-code 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/LICENSE +21 -0
- package/README.md +624 -0
- package/cmd/apply.go +111 -0
- package/cmd/auth.go +393 -0
- package/cmd/auth_test.go +100 -0
- package/cmd/diff.go +57 -0
- package/cmd/doctor.go +149 -0
- package/cmd/explain.go +192 -0
- package/cmd/explain_test.go +62 -0
- package/cmd/init.go +100 -0
- package/cmd/interactive.go +1372 -0
- package/cmd/interactive_input.go +45 -0
- package/cmd/interactive_input_test.go +55 -0
- package/cmd/logs.go +72 -0
- package/cmd/model.go +84 -0
- package/cmd/plan.go +149 -0
- package/cmd/provider.go +189 -0
- package/cmd/provider_model_doctor_test.go +91 -0
- package/cmd/root.go +67 -0
- package/cmd/run.go +123 -0
- package/cmd/run_engine.go +208 -0
- package/cmd/run_engine_test.go +30 -0
- package/cmd/session.go +589 -0
- package/cmd/session_helpers.go +54 -0
- package/cmd/session_integration_test.go +30 -0
- package/cmd/session_list_current_test.go +87 -0
- package/cmd/session_messages_test.go +163 -0
- package/cmd/session_runs_test.go +68 -0
- package/cmd/sprint1_integration_test.go +119 -0
- package/cmd/stats.go +173 -0
- package/cmd/stats_test.go +71 -0
- package/cmd/version.go +4 -0
- package/go.mod +45 -0
- package/go.sum +108 -0
- package/internal/agents/agent.go +31 -0
- package/internal/agents/coder.go +167 -0
- package/internal/agents/planner.go +155 -0
- package/internal/agents/reviewer.go +118 -0
- package/internal/agents/runtime.go +25 -0
- package/internal/agents/runtime_test.go +77 -0
- package/internal/auth/account.go +78 -0
- package/internal/auth/oauth.go +523 -0
- package/internal/auth/store.go +287 -0
- package/internal/confidence/policy.go +174 -0
- package/internal/confidence/policy_test.go +71 -0
- package/internal/confidence/scorer.go +253 -0
- package/internal/confidence/scorer_test.go +83 -0
- package/internal/config/config.go +331 -0
- package/internal/config/config_defaults_test.go +138 -0
- package/internal/execution/contract_builder.go +160 -0
- package/internal/execution/contract_builder_test.go +68 -0
- package/internal/execution/plan_compliance.go +161 -0
- package/internal/execution/plan_compliance_test.go +71 -0
- package/internal/execution/retry_directive.go +132 -0
- package/internal/execution/scope_guard.go +69 -0
- package/internal/logger/logger.go +120 -0
- package/internal/models/contracts_test.go +100 -0
- package/internal/models/models.go +269 -0
- package/internal/orchestrator/orchestrator.go +701 -0
- package/internal/orchestrator/orchestrator_retry_test.go +135 -0
- package/internal/orchestrator/review_engine_test.go +50 -0
- package/internal/orchestrator/state.go +42 -0
- package/internal/orchestrator/test_classifier_test.go +68 -0
- package/internal/patch/applier.go +131 -0
- package/internal/patch/applier_test.go +25 -0
- package/internal/patch/parser.go +89 -0
- package/internal/patch/patch.go +60 -0
- package/internal/patch/summary.go +30 -0
- package/internal/patch/validator.go +104 -0
- package/internal/planning/normalizer.go +416 -0
- package/internal/planning/normalizer_test.go +64 -0
- package/internal/providers/errors.go +35 -0
- package/internal/providers/openai/client.go +498 -0
- package/internal/providers/openai/client_test.go +187 -0
- package/internal/providers/provider.go +47 -0
- package/internal/providers/registry.go +32 -0
- package/internal/providers/registry_test.go +57 -0
- package/internal/providers/router.go +52 -0
- package/internal/providers/state.go +114 -0
- package/internal/providers/state_test.go +64 -0
- package/internal/repo/analyzer.go +188 -0
- package/internal/repo/context.go +83 -0
- package/internal/review/engine.go +267 -0
- package/internal/review/engine_test.go +103 -0
- package/internal/runstore/store.go +137 -0
- package/internal/runstore/store_test.go +59 -0
- package/internal/runtime/lock.go +150 -0
- package/internal/runtime/lock_test.go +57 -0
- package/internal/session/compaction.go +260 -0
- package/internal/session/compaction_test.go +36 -0
- package/internal/session/service.go +117 -0
- package/internal/session/service_test.go +113 -0
- package/internal/storage/storage.go +1498 -0
- package/internal/storage/storage_test.go +413 -0
- package/internal/testing/classifier.go +80 -0
- package/internal/testing/classifier_test.go +36 -0
- package/internal/tools/command.go +160 -0
- package/internal/tools/command_test.go +56 -0
- package/internal/tools/file.go +111 -0
- package/internal/tools/git.go +77 -0
- package/internal/tools/invalid_params_test.go +36 -0
- package/internal/tools/policy.go +98 -0
- package/internal/tools/policy_test.go +36 -0
- package/internal/tools/registry_test.go +52 -0
- package/internal/tools/result.go +30 -0
- package/internal/tools/search.go +86 -0
- package/internal/tools/tool.go +94 -0
- package/main.go +9 -0
- package/npm/orch.js +25 -0
- package/package.json +41 -0
- package/scripts/changelog.js +20 -0
- package/scripts/check-release-version.js +21 -0
- package/scripts/lib/release-utils.js +223 -0
- package/scripts/postinstall.js +157 -0
- package/scripts/release.js +52 -0
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
package cmd
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"encoding/json"
|
|
5
|
+
"testing"
|
|
6
|
+
|
|
7
|
+
"github.com/furkanbeydemir/orch/internal/storage"
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
func TestSessionListJSONOutput(t *testing.T) {
|
|
11
|
+
repoRoot := t.TempDir()
|
|
12
|
+
t.Chdir(repoRoot)
|
|
13
|
+
|
|
14
|
+
store, err := storage.Open(repoRoot)
|
|
15
|
+
if err != nil {
|
|
16
|
+
t.Fatalf("open storage: %v", err)
|
|
17
|
+
}
|
|
18
|
+
defer store.Close()
|
|
19
|
+
|
|
20
|
+
projectID, err := store.GetOrCreateProject()
|
|
21
|
+
if err != nil {
|
|
22
|
+
t.Fatalf("project: %v", err)
|
|
23
|
+
}
|
|
24
|
+
if _, err := store.EnsureDefaultSession(projectID); err != nil {
|
|
25
|
+
t.Fatalf("default session: %v", err)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
sessionListJSON = true
|
|
29
|
+
t.Cleanup(func() { sessionListJSON = false })
|
|
30
|
+
|
|
31
|
+
output := captureStdout(t, func() {
|
|
32
|
+
if err := runSessionList(nil, nil); err != nil {
|
|
33
|
+
t.Fatalf("run session list: %v", err)
|
|
34
|
+
}
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
var payload []map[string]any
|
|
38
|
+
if err := json.Unmarshal([]byte(output), &payload); err != nil {
|
|
39
|
+
t.Fatalf("invalid json output: %v\noutput=%s", err, output)
|
|
40
|
+
}
|
|
41
|
+
if len(payload) == 0 {
|
|
42
|
+
t.Fatalf("expected at least one session")
|
|
43
|
+
}
|
|
44
|
+
if payload[0]["name"] == "" {
|
|
45
|
+
t.Fatalf("expected name in first session payload")
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
func TestSessionCurrentJSONOutput(t *testing.T) {
|
|
50
|
+
repoRoot := t.TempDir()
|
|
51
|
+
t.Chdir(repoRoot)
|
|
52
|
+
|
|
53
|
+
store, err := storage.Open(repoRoot)
|
|
54
|
+
if err != nil {
|
|
55
|
+
t.Fatalf("open storage: %v", err)
|
|
56
|
+
}
|
|
57
|
+
defer store.Close()
|
|
58
|
+
|
|
59
|
+
projectID, err := store.GetOrCreateProject()
|
|
60
|
+
if err != nil {
|
|
61
|
+
t.Fatalf("project: %v", err)
|
|
62
|
+
}
|
|
63
|
+
sess, err := store.EnsureDefaultSession(projectID)
|
|
64
|
+
if err != nil {
|
|
65
|
+
t.Fatalf("default session: %v", err)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
sessionCurrentJSON = true
|
|
69
|
+
t.Cleanup(func() { sessionCurrentJSON = false })
|
|
70
|
+
|
|
71
|
+
output := captureStdout(t, func() {
|
|
72
|
+
if err := runSessionCurrent(nil, nil); err != nil {
|
|
73
|
+
t.Fatalf("run session current: %v", err)
|
|
74
|
+
}
|
|
75
|
+
})
|
|
76
|
+
|
|
77
|
+
var payload map[string]any
|
|
78
|
+
if err := json.Unmarshal([]byte(output), &payload); err != nil {
|
|
79
|
+
t.Fatalf("invalid json output: %v\noutput=%s", err, output)
|
|
80
|
+
}
|
|
81
|
+
if payload["id"] != sess.ID {
|
|
82
|
+
t.Fatalf("expected id %s, got %v", sess.ID, payload["id"])
|
|
83
|
+
}
|
|
84
|
+
if payload["is_active"] != true {
|
|
85
|
+
t.Fatalf("expected is_active=true, got %v", payload["is_active"])
|
|
86
|
+
}
|
|
87
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
package cmd
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"encoding/json"
|
|
5
|
+
"strings"
|
|
6
|
+
"testing"
|
|
7
|
+
|
|
8
|
+
"github.com/furkanbeydemir/orch/internal/session"
|
|
9
|
+
"github.com/furkanbeydemir/orch/internal/storage"
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
func TestSessionMessagesCommand(t *testing.T) {
|
|
13
|
+
repoRoot := t.TempDir()
|
|
14
|
+
t.Chdir(repoRoot)
|
|
15
|
+
|
|
16
|
+
store, err := storage.Open(repoRoot)
|
|
17
|
+
if err != nil {
|
|
18
|
+
t.Fatalf("open storage: %v", err)
|
|
19
|
+
}
|
|
20
|
+
defer store.Close()
|
|
21
|
+
|
|
22
|
+
projectID, err := store.GetOrCreateProject()
|
|
23
|
+
if err != nil {
|
|
24
|
+
t.Fatalf("project: %v", err)
|
|
25
|
+
}
|
|
26
|
+
sess, err := store.EnsureDefaultSession(projectID)
|
|
27
|
+
if err != nil {
|
|
28
|
+
t.Fatalf("default session: %v", err)
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
svc := session.NewService(store)
|
|
32
|
+
if _, err := svc.AppendText(session.MessageInput{
|
|
33
|
+
SessionID: sess.ID,
|
|
34
|
+
Role: "user",
|
|
35
|
+
ProviderID: "openai",
|
|
36
|
+
ModelID: "gpt-5.3-codex",
|
|
37
|
+
Text: "hello session",
|
|
38
|
+
}); err != nil {
|
|
39
|
+
t.Fatalf("append text: %v", err)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
sessionMessagesLimit = 10
|
|
43
|
+
sessionMessagesJSON = false
|
|
44
|
+
output := captureStdout(t, func() {
|
|
45
|
+
if err := runSessionMessages(nil, nil); err != nil {
|
|
46
|
+
t.Fatalf("run session messages: %v", err)
|
|
47
|
+
}
|
|
48
|
+
})
|
|
49
|
+
|
|
50
|
+
if !strings.Contains(output, "role=user") {
|
|
51
|
+
t.Fatalf("expected role in output, got: %s", output)
|
|
52
|
+
}
|
|
53
|
+
if !strings.Contains(output, "hello session") {
|
|
54
|
+
t.Fatalf("expected message text in output, got: %s", output)
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
func TestSessionMessagesCommandJSONOutput(t *testing.T) {
|
|
59
|
+
repoRoot := t.TempDir()
|
|
60
|
+
t.Chdir(repoRoot)
|
|
61
|
+
|
|
62
|
+
store, err := storage.Open(repoRoot)
|
|
63
|
+
if err != nil {
|
|
64
|
+
t.Fatalf("open storage: %v", err)
|
|
65
|
+
}
|
|
66
|
+
defer store.Close()
|
|
67
|
+
|
|
68
|
+
projectID, err := store.GetOrCreateProject()
|
|
69
|
+
if err != nil {
|
|
70
|
+
t.Fatalf("project: %v", err)
|
|
71
|
+
}
|
|
72
|
+
sess, err := store.EnsureDefaultSession(projectID)
|
|
73
|
+
if err != nil {
|
|
74
|
+
t.Fatalf("default session: %v", err)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
svc := session.NewService(store)
|
|
78
|
+
if _, err := svc.AppendText(session.MessageInput{SessionID: sess.ID, Role: "user", Text: "json output test"}); err != nil {
|
|
79
|
+
t.Fatalf("append text: %v", err)
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
sessionMessagesLimit = 10
|
|
83
|
+
sessionMessagesJSON = true
|
|
84
|
+
t.Cleanup(func() { sessionMessagesJSON = false })
|
|
85
|
+
|
|
86
|
+
output := captureStdout(t, func() {
|
|
87
|
+
if err := runSessionMessages(nil, nil); err != nil {
|
|
88
|
+
t.Fatalf("run session messages: %v", err)
|
|
89
|
+
}
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
var payload []map[string]any
|
|
93
|
+
if err := json.Unmarshal([]byte(output), &payload); err != nil {
|
|
94
|
+
t.Fatalf("invalid json output: %v\noutput=%s", err, output)
|
|
95
|
+
}
|
|
96
|
+
if len(payload) == 0 {
|
|
97
|
+
t.Fatalf("expected at least one message in json output")
|
|
98
|
+
}
|
|
99
|
+
if payload[0]["role"] != "user" {
|
|
100
|
+
t.Fatalf("expected first message role=user, got: %v", payload[0]["role"])
|
|
101
|
+
}
|
|
102
|
+
parts, ok := payload[0]["parts"].([]any)
|
|
103
|
+
if !ok || len(parts) == 0 {
|
|
104
|
+
t.Fatalf("expected parts in json output, got: %#v", payload[0]["parts"])
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
func TestSessionMessagesCommandRendersStageAndCompaction(t *testing.T) {
|
|
109
|
+
repoRoot := t.TempDir()
|
|
110
|
+
t.Chdir(repoRoot)
|
|
111
|
+
|
|
112
|
+
store, err := storage.Open(repoRoot)
|
|
113
|
+
if err != nil {
|
|
114
|
+
t.Fatalf("open storage: %v", err)
|
|
115
|
+
}
|
|
116
|
+
defer store.Close()
|
|
117
|
+
|
|
118
|
+
projectID, err := store.GetOrCreateProject()
|
|
119
|
+
if err != nil {
|
|
120
|
+
t.Fatalf("project: %v", err)
|
|
121
|
+
}
|
|
122
|
+
sess, err := store.EnsureDefaultSession(projectID)
|
|
123
|
+
if err != nil {
|
|
124
|
+
t.Fatalf("default session: %v", err)
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
svc := session.NewService(store)
|
|
128
|
+
stagePayload, _ := json.Marshal(map[string]any{
|
|
129
|
+
"actor": "planner",
|
|
130
|
+
"step": "plan",
|
|
131
|
+
"message": "Generating plan...",
|
|
132
|
+
"timestamp": "2026-03-09T00:00:00Z",
|
|
133
|
+
})
|
|
134
|
+
compactionPayload, _ := json.Marshal(map[string]any{
|
|
135
|
+
"estimated_tokens": 70000,
|
|
136
|
+
"usable_input": 56000,
|
|
137
|
+
"summary": "Compaction summary content.",
|
|
138
|
+
})
|
|
139
|
+
if _, err := svc.AppendMessage(session.MessageInput{SessionID: sess.ID, Role: "assistant"}, []storage.SessionPart{
|
|
140
|
+
{Type: "stage", Payload: string(stagePayload)},
|
|
141
|
+
{Type: "compaction", Payload: string(compactionPayload)},
|
|
142
|
+
{Type: "stage", Payload: "{not-json"},
|
|
143
|
+
}); err != nil {
|
|
144
|
+
t.Fatalf("append stage/compaction parts: %v", err)
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
sessionMessagesLimit = 10
|
|
148
|
+
output := captureStdout(t, func() {
|
|
149
|
+
if err := runSessionMessages(nil, nil); err != nil {
|
|
150
|
+
t.Fatalf("run session messages: %v", err)
|
|
151
|
+
}
|
|
152
|
+
})
|
|
153
|
+
|
|
154
|
+
if !strings.Contains(output, "part=stage actor=\"planner\" step=\"plan\"") {
|
|
155
|
+
t.Fatalf("expected structured stage render, got: %s", output)
|
|
156
|
+
}
|
|
157
|
+
if !strings.Contains(output, "part=compaction estimated_tokens=70000") {
|
|
158
|
+
t.Fatalf("expected structured compaction render, got: %s", output)
|
|
159
|
+
}
|
|
160
|
+
if !strings.Contains(output, "part=stage payload=") {
|
|
161
|
+
t.Fatalf("expected malformed stage fallback render, got: %s", output)
|
|
162
|
+
}
|
|
163
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
package cmd
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"encoding/json"
|
|
5
|
+
"testing"
|
|
6
|
+
"time"
|
|
7
|
+
|
|
8
|
+
"github.com/furkanbeydemir/orch/internal/models"
|
|
9
|
+
"github.com/furkanbeydemir/orch/internal/storage"
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
func TestSessionRunsJSONOutput(t *testing.T) {
|
|
13
|
+
repoRoot := t.TempDir()
|
|
14
|
+
t.Chdir(repoRoot)
|
|
15
|
+
|
|
16
|
+
store, err := storage.Open(repoRoot)
|
|
17
|
+
if err != nil {
|
|
18
|
+
t.Fatalf("open storage: %v", err)
|
|
19
|
+
}
|
|
20
|
+
defer store.Close()
|
|
21
|
+
|
|
22
|
+
projectID, err := store.GetOrCreateProject()
|
|
23
|
+
if err != nil {
|
|
24
|
+
t.Fatalf("project: %v", err)
|
|
25
|
+
}
|
|
26
|
+
sess, err := store.EnsureDefaultSession(projectID)
|
|
27
|
+
if err != nil {
|
|
28
|
+
t.Fatalf("default session: %v", err)
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
err = store.SaveRunState(&models.RunState{
|
|
32
|
+
ID: "run-json-1",
|
|
33
|
+
ProjectID: projectID,
|
|
34
|
+
SessionID: sess.ID,
|
|
35
|
+
Task: models.Task{
|
|
36
|
+
ID: "task-json-1",
|
|
37
|
+
Description: "verify runs json output",
|
|
38
|
+
CreatedAt: time.Now(),
|
|
39
|
+
},
|
|
40
|
+
Status: models.StatusCompleted,
|
|
41
|
+
Retries: models.RetryState{},
|
|
42
|
+
StartedAt: time.Now(),
|
|
43
|
+
})
|
|
44
|
+
if err != nil {
|
|
45
|
+
t.Fatalf("save run state: %v", err)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
sessionRunsLimit = 10
|
|
49
|
+
sessionRunsJSON = true
|
|
50
|
+
t.Cleanup(func() { sessionRunsJSON = false })
|
|
51
|
+
|
|
52
|
+
output := captureStdout(t, func() {
|
|
53
|
+
if err := runSessionRuns(nil, nil); err != nil {
|
|
54
|
+
t.Fatalf("run session runs: %v", err)
|
|
55
|
+
}
|
|
56
|
+
})
|
|
57
|
+
|
|
58
|
+
var payload []map[string]any
|
|
59
|
+
if err := json.Unmarshal([]byte(output), &payload); err != nil {
|
|
60
|
+
t.Fatalf("invalid json output: %v\noutput=%s", err, output)
|
|
61
|
+
}
|
|
62
|
+
if len(payload) == 0 {
|
|
63
|
+
t.Fatalf("expected at least one run in json output")
|
|
64
|
+
}
|
|
65
|
+
if payload[0]["id"] != "run-json-1" {
|
|
66
|
+
t.Fatalf("unexpected run id in output: %v", payload[0]["id"])
|
|
67
|
+
}
|
|
68
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
package cmd
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"fmt"
|
|
5
|
+
"os"
|
|
6
|
+
"path/filepath"
|
|
7
|
+
"strings"
|
|
8
|
+
"testing"
|
|
9
|
+
"time"
|
|
10
|
+
|
|
11
|
+
"github.com/furkanbeydemir/orch/internal/config"
|
|
12
|
+
"github.com/furkanbeydemir/orch/internal/models"
|
|
13
|
+
"github.com/furkanbeydemir/orch/internal/storage"
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
func TestRunBlockedByRepoLock(t *testing.T) {
|
|
17
|
+
repoRoot := t.TempDir()
|
|
18
|
+
t.Chdir(repoRoot)
|
|
19
|
+
|
|
20
|
+
if err := config.EnsureOrchDir(repoRoot); err != nil {
|
|
21
|
+
t.Fatalf("ensure orch dir: %v", err)
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
cfg := config.DefaultConfig()
|
|
25
|
+
if err := config.Save(repoRoot, cfg); err != nil {
|
|
26
|
+
t.Fatalf("save config: %v", err)
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
lockContent := fmt.Sprintf(`{"pid":%d,"run_id":"other-run","created_at":"%s"}`,
|
|
30
|
+
os.Getpid(), time.Now().UTC().Format(time.RFC3339Nano))
|
|
31
|
+
lockPath := filepath.Join(repoRoot, config.OrchDir, "lock")
|
|
32
|
+
if err := os.WriteFile(lockPath, []byte(lockContent), 0o644); err != nil {
|
|
33
|
+
t.Fatalf("write lock file: %v", err)
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
err := runRun(nil, []string{"dummy task"})
|
|
37
|
+
if err == nil {
|
|
38
|
+
t.Fatalf("expected run to be blocked by repo lock")
|
|
39
|
+
}
|
|
40
|
+
if !strings.Contains(err.Error(), "run blocked by repository lock") {
|
|
41
|
+
t.Fatalf("unexpected error: %v", err)
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
func TestApplyRequiresDestructiveApproval(t *testing.T) {
|
|
46
|
+
repoRoot := t.TempDir()
|
|
47
|
+
t.Chdir(repoRoot)
|
|
48
|
+
|
|
49
|
+
if err := config.EnsureOrchDir(repoRoot); err != nil {
|
|
50
|
+
t.Fatalf("ensure orch dir: %v", err)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
cfg := config.DefaultConfig()
|
|
54
|
+
cfg.Safety.DryRun = true
|
|
55
|
+
cfg.Safety.RequireDestructiveApproval = true
|
|
56
|
+
if err := config.Save(repoRoot, cfg); err != nil {
|
|
57
|
+
t.Fatalf("save config: %v", err)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
patchContent := strings.Join([]string{
|
|
61
|
+
"diff --git a/demo.txt b/demo.txt",
|
|
62
|
+
"--- a/demo.txt",
|
|
63
|
+
"+++ b/demo.txt",
|
|
64
|
+
"@@ -1 +1 @@",
|
|
65
|
+
"-hello",
|
|
66
|
+
"+world",
|
|
67
|
+
"",
|
|
68
|
+
}, "\n")
|
|
69
|
+
store, err := storage.Open(repoRoot)
|
|
70
|
+
if err != nil {
|
|
71
|
+
t.Fatalf("open storage: %v", err)
|
|
72
|
+
}
|
|
73
|
+
defer store.Close()
|
|
74
|
+
|
|
75
|
+
projectID, err := store.GetOrCreateProject()
|
|
76
|
+
if err != nil {
|
|
77
|
+
t.Fatalf("get project id: %v", err)
|
|
78
|
+
}
|
|
79
|
+
sess, err := store.EnsureDefaultSession(projectID)
|
|
80
|
+
if err != nil {
|
|
81
|
+
t.Fatalf("ensure default session: %v", err)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
err = store.SaveRunState(&models.RunState{
|
|
85
|
+
ID: "run-test-apply",
|
|
86
|
+
ProjectID: projectID,
|
|
87
|
+
SessionID: sess.ID,
|
|
88
|
+
Task: models.Task{
|
|
89
|
+
ID: "task-apply",
|
|
90
|
+
Description: "apply test task",
|
|
91
|
+
CreatedAt: time.Now(),
|
|
92
|
+
},
|
|
93
|
+
Status: models.StatusCompleted,
|
|
94
|
+
Patch: &models.Patch{
|
|
95
|
+
TaskID: "task-apply",
|
|
96
|
+
RawDiff: patchContent,
|
|
97
|
+
},
|
|
98
|
+
Retries: models.RetryState{},
|
|
99
|
+
StartedAt: time.Now(),
|
|
100
|
+
})
|
|
101
|
+
if err != nil {
|
|
102
|
+
t.Fatalf("save run state: %v", err)
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
forceApply = true
|
|
106
|
+
approveDestructive = false
|
|
107
|
+
t.Cleanup(func() {
|
|
108
|
+
forceApply = false
|
|
109
|
+
approveDestructive = false
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
err = runApply(nil, nil)
|
|
113
|
+
if err == nil {
|
|
114
|
+
t.Fatalf("expected destructive apply to be blocked")
|
|
115
|
+
}
|
|
116
|
+
if !strings.Contains(err.Error(), "destructive apply blocked") {
|
|
117
|
+
t.Fatalf("unexpected error: %v", err)
|
|
118
|
+
}
|
|
119
|
+
}
|
package/cmd/stats.go
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
package cmd
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"fmt"
|
|
5
|
+
"os"
|
|
6
|
+
"sort"
|
|
7
|
+
"strings"
|
|
8
|
+
|
|
9
|
+
"github.com/furkanbeydemir/orch/internal/models"
|
|
10
|
+
"github.com/spf13/cobra"
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
var statsLimit int
|
|
14
|
+
|
|
15
|
+
var statsCmd = &cobra.Command{
|
|
16
|
+
Use: "stats",
|
|
17
|
+
Short: "Show run quality statistics",
|
|
18
|
+
Long: `Summarizes recent runs using structured Orch artifacts such as status, review, confidence, retries, and classified test failures.`,
|
|
19
|
+
RunE: runStats,
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
func init() {
|
|
23
|
+
rootCmd.AddCommand(statsCmd)
|
|
24
|
+
statsCmd.Flags().IntVar(&statsLimit, "limit", 50, "Maximum number of recent runs to include")
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
func runStats(cmd *cobra.Command, args []string) error {
|
|
28
|
+
cwd, err := os.Getwd()
|
|
29
|
+
if err != nil {
|
|
30
|
+
return fmt.Errorf("failed to get working directory: %w", err)
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
ctx, err := loadSessionContext(cwd)
|
|
34
|
+
if err != nil {
|
|
35
|
+
return err
|
|
36
|
+
}
|
|
37
|
+
defer ctx.Store.Close()
|
|
38
|
+
|
|
39
|
+
states, err := ctx.Store.ListRunStatesByProject(ctx.ProjectID, statsLimit)
|
|
40
|
+
if err != nil {
|
|
41
|
+
return fmt.Errorf("failed to load run states: %w", err)
|
|
42
|
+
}
|
|
43
|
+
if len(states) == 0 {
|
|
44
|
+
fmt.Println("📊 No runs found yet.")
|
|
45
|
+
fmt.Println(" Run 'orch run <task>' first.")
|
|
46
|
+
return nil
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
summary := summarizeRunStats(states)
|
|
50
|
+
printRunStats(summary)
|
|
51
|
+
return nil
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
type runStatsSummary struct {
|
|
55
|
+
TotalRuns int
|
|
56
|
+
CompletedRuns int
|
|
57
|
+
FailedRuns int
|
|
58
|
+
InProgressRuns int
|
|
59
|
+
AcceptedReviews int
|
|
60
|
+
RevisedReviews int
|
|
61
|
+
AverageConfidence float64
|
|
62
|
+
ConfidenceRunCount int
|
|
63
|
+
AverageRetries float64
|
|
64
|
+
TotalRetryCount int
|
|
65
|
+
StatusCounts map[string]int
|
|
66
|
+
ConfidenceBandCounts map[string]int
|
|
67
|
+
TestFailureCodeCounts map[string]int
|
|
68
|
+
LatestRunID string
|
|
69
|
+
LatestRunStatus string
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
func summarizeRunStats(states []*models.RunState) runStatsSummary {
|
|
73
|
+
summary := runStatsSummary{
|
|
74
|
+
TotalRuns: len(states),
|
|
75
|
+
StatusCounts: map[string]int{},
|
|
76
|
+
ConfidenceBandCounts: map[string]int{},
|
|
77
|
+
TestFailureCodeCounts: map[string]int{},
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
var confidenceTotal float64
|
|
81
|
+
for i, state := range states {
|
|
82
|
+
if state == nil {
|
|
83
|
+
continue
|
|
84
|
+
}
|
|
85
|
+
if i == 0 {
|
|
86
|
+
summary.LatestRunID = state.ID
|
|
87
|
+
summary.LatestRunStatus = string(state.Status)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
summary.StatusCounts[string(state.Status)]++
|
|
91
|
+
switch state.Status {
|
|
92
|
+
case models.StatusCompleted:
|
|
93
|
+
summary.CompletedRuns++
|
|
94
|
+
case models.StatusFailed:
|
|
95
|
+
summary.FailedRuns++
|
|
96
|
+
default:
|
|
97
|
+
summary.InProgressRuns++
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if state.Review != nil {
|
|
101
|
+
switch state.Review.Decision {
|
|
102
|
+
case models.ReviewAccept:
|
|
103
|
+
summary.AcceptedReviews++
|
|
104
|
+
case models.ReviewRevise:
|
|
105
|
+
summary.RevisedReviews++
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if state.Confidence != nil {
|
|
110
|
+
confidenceTotal += state.Confidence.Score
|
|
111
|
+
summary.ConfidenceRunCount++
|
|
112
|
+
if strings.TrimSpace(state.Confidence.Band) != "" {
|
|
113
|
+
summary.ConfidenceBandCounts[state.Confidence.Band]++
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
retries := state.Retries.Validation + state.Retries.Testing + state.Retries.Review
|
|
118
|
+
summary.TotalRetryCount += retries
|
|
119
|
+
|
|
120
|
+
for _, failure := range state.TestFailures {
|
|
121
|
+
code := strings.TrimSpace(failure.Code)
|
|
122
|
+
if code == "" {
|
|
123
|
+
code = "unknown"
|
|
124
|
+
}
|
|
125
|
+
summary.TestFailureCodeCounts[code]++
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
if summary.ConfidenceRunCount > 0 {
|
|
130
|
+
summary.AverageConfidence = confidenceTotal / float64(summary.ConfidenceRunCount)
|
|
131
|
+
}
|
|
132
|
+
if summary.TotalRuns > 0 {
|
|
133
|
+
summary.AverageRetries = float64(summary.TotalRetryCount) / float64(summary.TotalRuns)
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
return summary
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
func printRunStats(summary runStatsSummary) {
|
|
140
|
+
fmt.Println("═══════════════════════════════════════")
|
|
141
|
+
fmt.Println("📊 ORCH RUN STATS")
|
|
142
|
+
fmt.Println("═══════════════════════════════════════")
|
|
143
|
+
fmt.Printf("Runs Analyzed: %d\n", summary.TotalRuns)
|
|
144
|
+
fmt.Printf("Latest Run: %s (%s)\n", summary.LatestRunID, summary.LatestRunStatus)
|
|
145
|
+
fmt.Printf("Completed: %d\n", summary.CompletedRuns)
|
|
146
|
+
fmt.Printf("Failed: %d\n", summary.FailedRuns)
|
|
147
|
+
fmt.Printf("In Progress/Other: %d\n", summary.InProgressRuns)
|
|
148
|
+
fmt.Printf("Review Accept: %d\n", summary.AcceptedReviews)
|
|
149
|
+
fmt.Printf("Review Revise: %d\n", summary.RevisedReviews)
|
|
150
|
+
fmt.Printf("Average Retries: %.2f\n", summary.AverageRetries)
|
|
151
|
+
if summary.ConfidenceRunCount > 0 {
|
|
152
|
+
fmt.Printf("Average Confidence: %.2f across %d run(s)\n", summary.AverageConfidence, summary.ConfidenceRunCount)
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
printCountMap("Status Breakdown", summary.StatusCounts)
|
|
156
|
+
printCountMap("Confidence Bands", summary.ConfidenceBandCounts)
|
|
157
|
+
printCountMap("Test Failure Codes", summary.TestFailureCodeCounts)
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
func printCountMap(title string, counts map[string]int) {
|
|
161
|
+
if len(counts) == 0 {
|
|
162
|
+
return
|
|
163
|
+
}
|
|
164
|
+
fmt.Printf("\n%s\n", title)
|
|
165
|
+
keys := make([]string, 0, len(counts))
|
|
166
|
+
for key := range counts {
|
|
167
|
+
keys = append(keys, key)
|
|
168
|
+
}
|
|
169
|
+
sort.Strings(keys)
|
|
170
|
+
for _, key := range keys {
|
|
171
|
+
fmt.Printf(" - %s: %d\n", key, counts[key])
|
|
172
|
+
}
|
|
173
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
package cmd
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"testing"
|
|
5
|
+
"time"
|
|
6
|
+
|
|
7
|
+
"github.com/furkanbeydemir/orch/internal/models"
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
func TestSummarizeRunStats(t *testing.T) {
|
|
11
|
+
now := time.Now()
|
|
12
|
+
states := []*models.RunState{
|
|
13
|
+
{
|
|
14
|
+
ID: "run-3",
|
|
15
|
+
Task: models.Task{ID: "task-3", Description: "latest", CreatedAt: now},
|
|
16
|
+
Status: models.StatusFailed,
|
|
17
|
+
StartedAt: now,
|
|
18
|
+
Review: &models.ReviewResult{Decision: models.ReviewRevise},
|
|
19
|
+
Confidence: &models.ConfidenceReport{Score: 0.40, Band: "very_low"},
|
|
20
|
+
Retries: models.RetryState{Validation: 1, Testing: 1, Review: 0},
|
|
21
|
+
TestFailures: []models.TestFailure{{Code: "test_assertion_failure", Summary: "boom"}},
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
ID: "run-2",
|
|
25
|
+
Task: models.Task{ID: "task-2", Description: "middle", CreatedAt: now.Add(-time.Hour)},
|
|
26
|
+
Status: models.StatusCompleted,
|
|
27
|
+
StartedAt: now.Add(-time.Hour),
|
|
28
|
+
Review: &models.ReviewResult{Decision: models.ReviewAccept},
|
|
29
|
+
Confidence: &models.ConfidenceReport{Score: 0.80, Band: "high"},
|
|
30
|
+
Retries: models.RetryState{Validation: 0, Testing: 1, Review: 0},
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
ID: "run-1",
|
|
34
|
+
Task: models.Task{ID: "task-1", Description: "old", CreatedAt: now.Add(-2 * time.Hour)},
|
|
35
|
+
Status: models.StatusReviewing,
|
|
36
|
+
StartedAt: now.Add(-2 * time.Hour),
|
|
37
|
+
Review: &models.ReviewResult{Decision: models.ReviewRevise},
|
|
38
|
+
Retries: models.RetryState{Validation: 0, Testing: 0, Review: 1},
|
|
39
|
+
},
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
summary := summarizeRunStats(states)
|
|
43
|
+
|
|
44
|
+
if summary.TotalRuns != 3 {
|
|
45
|
+
t.Fatalf("expected 3 runs, got %d", summary.TotalRuns)
|
|
46
|
+
}
|
|
47
|
+
if summary.LatestRunID != "run-3" {
|
|
48
|
+
t.Fatalf("expected latest run to be run-3, got %s", summary.LatestRunID)
|
|
49
|
+
}
|
|
50
|
+
if summary.CompletedRuns != 1 || summary.FailedRuns != 1 || summary.InProgressRuns != 1 {
|
|
51
|
+
t.Fatalf("unexpected status counts: %+v", summary)
|
|
52
|
+
}
|
|
53
|
+
if summary.AcceptedReviews != 1 || summary.RevisedReviews != 2 {
|
|
54
|
+
t.Fatalf("unexpected review counts: %+v", summary)
|
|
55
|
+
}
|
|
56
|
+
if summary.ConfidenceRunCount != 2 {
|
|
57
|
+
t.Fatalf("expected 2 confidence-bearing runs, got %d", summary.ConfidenceRunCount)
|
|
58
|
+
}
|
|
59
|
+
if summary.AverageConfidence < 0.599 || summary.AverageConfidence > 0.601 {
|
|
60
|
+
t.Fatalf("expected average confidence about 0.60, got %.4f", summary.AverageConfidence)
|
|
61
|
+
}
|
|
62
|
+
if summary.TotalRetryCount != 4 {
|
|
63
|
+
t.Fatalf("expected total retries 4, got %d", summary.TotalRetryCount)
|
|
64
|
+
}
|
|
65
|
+
if summary.TestFailureCodeCounts["test_assertion_failure"] != 1 {
|
|
66
|
+
t.Fatalf("expected test failure code count to be recorded")
|
|
67
|
+
}
|
|
68
|
+
if summary.ConfidenceBandCounts["high"] != 1 || summary.ConfidenceBandCounts["very_low"] != 1 {
|
|
69
|
+
t.Fatalf("unexpected confidence band counts: %+v", summary.ConfidenceBandCounts)
|
|
70
|
+
}
|
|
71
|
+
}
|
package/cmd/version.go
ADDED