draft-board 0.1.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (250) hide show
  1. package/app/backend/.env.example +9 -0
  2. package/app/backend/.smartkanban/evidence/8b383839-cbec-45af-86ee-c7708d075cbe/bddf2ed5-2e21-4d46-a62b-10b87f1642a6_patch.txt +195 -0
  3. package/app/backend/.smartkanban/evidence/8b383839-cbec-45af-86ee-c7708d075cbe/bddf2ed5-2e21-4d46-a62b-10b87f1642a6_stat.txt +6 -0
  4. package/app/backend/CURL_EXAMPLES.md +335 -0
  5. package/app/backend/ENV_SETUP.md +65 -0
  6. package/app/backend/alembic/env.py +71 -0
  7. package/app/backend/alembic/script.py.mako +28 -0
  8. package/app/backend/alembic/versions/001_initial_schema.py +104 -0
  9. package/app/backend/alembic/versions/002_add_jobs_table.py +52 -0
  10. package/app/backend/alembic/versions/003_add_workspace_table.py +48 -0
  11. package/app/backend/alembic/versions/004_add_evidence_table.py +56 -0
  12. package/app/backend/alembic/versions/005_add_verification_commands.py +32 -0
  13. package/app/backend/alembic/versions/006_add_planner_lock_table.py +39 -0
  14. package/app/backend/alembic/versions/007_add_revision_review_tables.py +126 -0
  15. package/app/backend/alembic/versions/008_add_revision_idempotency_and_traceability.py +52 -0
  16. package/app/backend/alembic/versions/009_add_job_health_fields.py +46 -0
  17. package/app/backend/alembic/versions/010_add_review_comment_line_content.py +36 -0
  18. package/app/backend/alembic/versions/011_add_analysis_cache.py +47 -0
  19. package/app/backend/alembic/versions/012_add_boards_table.py +102 -0
  20. package/app/backend/alembic/versions/013_add_ticket_blocking.py +45 -0
  21. package/app/backend/alembic/versions/014_add_agent_sessions.py +220 -0
  22. package/app/backend/alembic/versions/015_add_ticket_sort_order.py +33 -0
  23. package/app/backend/alembic/versions/03220f0b93ae_add_pr_fields_to_ticket.py +49 -0
  24. package/app/backend/alembic/versions/0c2d89fff3b1_seed_board_configs_from_yaml.py +206 -0
  25. package/app/backend/alembic/versions/3348e5cf54c1_add_merge_checklist_table.py +67 -0
  26. package/app/backend/alembic/versions/357c780ee445_add_goal_status.py +34 -0
  27. package/app/backend/alembic/versions/553340b7e26c_add_autonomy_fields_to_goal.py +65 -0
  28. package/app/backend/alembic/versions/774dc335c679_merge_migration_heads.py +23 -0
  29. package/app/backend/alembic/versions/7b307e847cbd_merge_heads.py +23 -0
  30. package/app/backend/alembic/versions/82ecd978cc70_add_missing_indexes.py +48 -0
  31. package/app/backend/alembic/versions/8ef5054dc280_add_normalized_log_entries.py +173 -0
  32. package/app/backend/alembic/versions/8f3e2bd8ea3b_merge_migration_heads.py +23 -0
  33. package/app/backend/alembic/versions/9d17f0698d3b_add_config_column_to_boards_table.py +30 -0
  34. package/app/backend/alembic/versions/add_agent_conversation_history.py +72 -0
  35. package/app/backend/alembic/versions/add_job_variant.py +34 -0
  36. package/app/backend/alembic/versions/add_performance_indexes.py +95 -0
  37. package/app/backend/alembic/versions/add_repos_and_board_repos.py +174 -0
  38. package/app/backend/alembic/versions/add_session_id_to_jobs.py +27 -0
  39. package/app/backend/alembic/versions/add_sqlite_backend_tables.py +104 -0
  40. package/app/backend/alembic/versions/b10fb0b62240_add_diff_content_to_revisions.py +34 -0
  41. package/app/backend/alembic.ini +89 -0
  42. package/app/backend/app/__init__.py +3 -0
  43. package/app/backend/app/data_dir.py +85 -0
  44. package/app/backend/app/database.py +70 -0
  45. package/app/backend/app/database_sync.py +64 -0
  46. package/app/backend/app/dependencies/__init__.py +5 -0
  47. package/app/backend/app/dependencies/auth.py +80 -0
  48. package/app/backend/app/dependencies.py +43 -0
  49. package/app/backend/app/exceptions.py +178 -0
  50. package/app/backend/app/executors/__init__.py +1 -0
  51. package/app/backend/app/executors/adapters/__init__.py +1 -0
  52. package/app/backend/app/executors/adapters/aider.py +152 -0
  53. package/app/backend/app/executors/adapters/amazon_q.py +103 -0
  54. package/app/backend/app/executors/adapters/amp.py +123 -0
  55. package/app/backend/app/executors/adapters/claude.py +177 -0
  56. package/app/backend/app/executors/adapters/cline.py +127 -0
  57. package/app/backend/app/executors/adapters/codex.py +167 -0
  58. package/app/backend/app/executors/adapters/copilot.py +202 -0
  59. package/app/backend/app/executors/adapters/cursor.py +87 -0
  60. package/app/backend/app/executors/adapters/droid.py +123 -0
  61. package/app/backend/app/executors/adapters/gemini.py +132 -0
  62. package/app/backend/app/executors/adapters/goose.py +131 -0
  63. package/app/backend/app/executors/adapters/opencode.py +123 -0
  64. package/app/backend/app/executors/adapters/qwen.py +123 -0
  65. package/app/backend/app/executors/plugins/__init__.py +1 -0
  66. package/app/backend/app/executors/registry.py +202 -0
  67. package/app/backend/app/executors/spec.py +226 -0
  68. package/app/backend/app/main.py +486 -0
  69. package/app/backend/app/middleware/__init__.py +13 -0
  70. package/app/backend/app/middleware/idempotency.py +426 -0
  71. package/app/backend/app/middleware/rate_limit.py +312 -0
  72. package/app/backend/app/middleware/security_headers.py +43 -0
  73. package/app/backend/app/middleware/timeout.py +37 -0
  74. package/app/backend/app/models/__init__.py +56 -0
  75. package/app/backend/app/models/agent_conversation_history.py +56 -0
  76. package/app/backend/app/models/agent_session.py +127 -0
  77. package/app/backend/app/models/analysis_cache.py +49 -0
  78. package/app/backend/app/models/base.py +9 -0
  79. package/app/backend/app/models/board.py +79 -0
  80. package/app/backend/app/models/board_repo.py +68 -0
  81. package/app/backend/app/models/cost_budget.py +42 -0
  82. package/app/backend/app/models/enums.py +40 -0
  83. package/app/backend/app/models/evidence.py +132 -0
  84. package/app/backend/app/models/goal.py +102 -0
  85. package/app/backend/app/models/idempotency_entry.py +30 -0
  86. package/app/backend/app/models/job.py +163 -0
  87. package/app/backend/app/models/job_queue.py +39 -0
  88. package/app/backend/app/models/kv_store.py +28 -0
  89. package/app/backend/app/models/merge_checklist.py +87 -0
  90. package/app/backend/app/models/normalized_log.py +100 -0
  91. package/app/backend/app/models/planner_lock.py +43 -0
  92. package/app/backend/app/models/rate_limit_entry.py +25 -0
  93. package/app/backend/app/models/repo.py +66 -0
  94. package/app/backend/app/models/review_comment.py +91 -0
  95. package/app/backend/app/models/review_summary.py +69 -0
  96. package/app/backend/app/models/revision.py +130 -0
  97. package/app/backend/app/models/ticket.py +223 -0
  98. package/app/backend/app/models/ticket_event.py +83 -0
  99. package/app/backend/app/models/user.py +47 -0
  100. package/app/backend/app/models/workspace.py +71 -0
  101. package/app/backend/app/redis_client.py +119 -0
  102. package/app/backend/app/routers/__init__.py +29 -0
  103. package/app/backend/app/routers/agents.py +296 -0
  104. package/app/backend/app/routers/auth.py +94 -0
  105. package/app/backend/app/routers/board.py +885 -0
  106. package/app/backend/app/routers/dashboard.py +351 -0
  107. package/app/backend/app/routers/debug.py +528 -0
  108. package/app/backend/app/routers/evidence.py +96 -0
  109. package/app/backend/app/routers/executors.py +324 -0
  110. package/app/backend/app/routers/goals.py +574 -0
  111. package/app/backend/app/routers/jobs.py +448 -0
  112. package/app/backend/app/routers/maintenance.py +172 -0
  113. package/app/backend/app/routers/merge.py +360 -0
  114. package/app/backend/app/routers/planner.py +537 -0
  115. package/app/backend/app/routers/pull_requests.py +382 -0
  116. package/app/backend/app/routers/repos.py +263 -0
  117. package/app/backend/app/routers/revisions.py +939 -0
  118. package/app/backend/app/routers/settings.py +267 -0
  119. package/app/backend/app/routers/tickets.py +2003 -0
  120. package/app/backend/app/routers/webhooks.py +143 -0
  121. package/app/backend/app/routers/websocket.py +249 -0
  122. package/app/backend/app/schemas/__init__.py +109 -0
  123. package/app/backend/app/schemas/board.py +87 -0
  124. package/app/backend/app/schemas/common.py +33 -0
  125. package/app/backend/app/schemas/evidence.py +87 -0
  126. package/app/backend/app/schemas/goal.py +90 -0
  127. package/app/backend/app/schemas/job.py +97 -0
  128. package/app/backend/app/schemas/merge.py +139 -0
  129. package/app/backend/app/schemas/planner.py +500 -0
  130. package/app/backend/app/schemas/repo.py +187 -0
  131. package/app/backend/app/schemas/review.py +137 -0
  132. package/app/backend/app/schemas/revision.py +114 -0
  133. package/app/backend/app/schemas/ticket.py +238 -0
  134. package/app/backend/app/schemas/ticket_event.py +72 -0
  135. package/app/backend/app/schemas/workspace.py +19 -0
  136. package/app/backend/app/services/__init__.py +31 -0
  137. package/app/backend/app/services/agent_memory_service.py +223 -0
  138. package/app/backend/app/services/agent_registry.py +346 -0
  139. package/app/backend/app/services/agent_session_manager.py +318 -0
  140. package/app/backend/app/services/agent_session_service.py +219 -0
  141. package/app/backend/app/services/agent_tools.py +379 -0
  142. package/app/backend/app/services/auth_service.py +98 -0
  143. package/app/backend/app/services/autonomy_service.py +380 -0
  144. package/app/backend/app/services/board_repo_service.py +201 -0
  145. package/app/backend/app/services/board_service.py +326 -0
  146. package/app/backend/app/services/cleanup_service.py +1085 -0
  147. package/app/backend/app/services/config_service.py +908 -0
  148. package/app/backend/app/services/context_gatherer.py +557 -0
  149. package/app/backend/app/services/cost_tracking_service.py +293 -0
  150. package/app/backend/app/services/cursor_log_normalizer.py +536 -0
  151. package/app/backend/app/services/delivery_pipeline.py +440 -0
  152. package/app/backend/app/services/executor_service.py +634 -0
  153. package/app/backend/app/services/git_host/__init__.py +11 -0
  154. package/app/backend/app/services/git_host/factory.py +87 -0
  155. package/app/backend/app/services/git_host/github.py +270 -0
  156. package/app/backend/app/services/git_host/gitlab.py +194 -0
  157. package/app/backend/app/services/git_host/protocol.py +75 -0
  158. package/app/backend/app/services/git_merge_simple.py +346 -0
  159. package/app/backend/app/services/git_ops.py +384 -0
  160. package/app/backend/app/services/github_service.py +233 -0
  161. package/app/backend/app/services/goal_service.py +113 -0
  162. package/app/backend/app/services/job_service.py +423 -0
  163. package/app/backend/app/services/job_watchdog_service.py +424 -0
  164. package/app/backend/app/services/langchain_adapter.py +122 -0
  165. package/app/backend/app/services/llm_provider_clients.py +351 -0
  166. package/app/backend/app/services/llm_service.py +285 -0
  167. package/app/backend/app/services/log_normalizer.py +342 -0
  168. package/app/backend/app/services/log_stream_service.py +276 -0
  169. package/app/backend/app/services/merge_checklist_service.py +264 -0
  170. package/app/backend/app/services/merge_service.py +784 -0
  171. package/app/backend/app/services/orchestrator_log.py +84 -0
  172. package/app/backend/app/services/planner_service.py +1662 -0
  173. package/app/backend/app/services/planner_tick_sync.py +1040 -0
  174. package/app/backend/app/services/queued_message_service.py +156 -0
  175. package/app/backend/app/services/reliability_wrapper.py +389 -0
  176. package/app/backend/app/services/repo_discovery_service.py +318 -0
  177. package/app/backend/app/services/review_service.py +334 -0
  178. package/app/backend/app/services/revision_service.py +389 -0
  179. package/app/backend/app/services/safe_autopilot.py +510 -0
  180. package/app/backend/app/services/sqlite_worker.py +372 -0
  181. package/app/backend/app/services/task_dispatch.py +135 -0
  182. package/app/backend/app/services/ticket_generation_service.py +1781 -0
  183. package/app/backend/app/services/ticket_service.py +486 -0
  184. package/app/backend/app/services/udar_planner_service.py +1007 -0
  185. package/app/backend/app/services/webhook_service.py +126 -0
  186. package/app/backend/app/services/workspace_service.py +465 -0
  187. package/app/backend/app/services/worktree_file_service.py +92 -0
  188. package/app/backend/app/services/worktree_validator.py +213 -0
  189. package/app/backend/app/sqlite_kv.py +278 -0
  190. package/app/backend/app/state_machine.py +128 -0
  191. package/app/backend/app/templates/__init__.py +5 -0
  192. package/app/backend/app/templates/registry.py +243 -0
  193. package/app/backend/app/utils/__init__.py +5 -0
  194. package/app/backend/app/utils/artifact_reader.py +87 -0
  195. package/app/backend/app/utils/circuit_breaker.py +229 -0
  196. package/app/backend/app/utils/db_retry.py +136 -0
  197. package/app/backend/app/utils/ignored_fields.py +123 -0
  198. package/app/backend/app/utils/validators.py +54 -0
  199. package/app/backend/app/websocket/__init__.py +5 -0
  200. package/app/backend/app/websocket/manager.py +179 -0
  201. package/app/backend/app/websocket/state_tracker.py +113 -0
  202. package/app/backend/app/worker.py +3190 -0
  203. package/app/backend/calculator_tickets.json +40 -0
  204. package/app/backend/canary_tests.sh +591 -0
  205. package/app/backend/celerybeat-schedule +0 -0
  206. package/app/backend/celerybeat-schedule-shm +0 -0
  207. package/app/backend/celerybeat-schedule-wal +0 -0
  208. package/app/backend/logs/.gitkeep +3 -0
  209. package/app/backend/multiplication_division_implementation_tickets.json +55 -0
  210. package/app/backend/multiplication_division_tickets.json +42 -0
  211. package/app/backend/pyproject.toml +45 -0
  212. package/app/backend/requirements-dev.txt +8 -0
  213. package/app/backend/requirements.txt +20 -0
  214. package/app/backend/run.sh +30 -0
  215. package/app/backend/run_with_logs.sh +10 -0
  216. package/app/backend/scientific_calculator_tickets.json +40 -0
  217. package/app/backend/scripts/extract_openapi.py +21 -0
  218. package/app/backend/scripts/seed_demo.py +187 -0
  219. package/app/backend/setup_demo_review.py +302 -0
  220. package/app/backend/test_actual_parse.py +41 -0
  221. package/app/backend/test_agent_streaming.py +61 -0
  222. package/app/backend/test_parse.py +51 -0
  223. package/app/backend/test_streaming.py +51 -0
  224. package/app/backend/test_subprocess_streaming.py +50 -0
  225. package/app/backend/tests/__init__.py +1 -0
  226. package/app/backend/tests/conftest.py +46 -0
  227. package/app/backend/tests/test_auth.py +341 -0
  228. package/app/backend/tests/test_autonomy_service.py +391 -0
  229. package/app/backend/tests/test_cleanup_service_safety.py +417 -0
  230. package/app/backend/tests/test_middleware.py +279 -0
  231. package/app/backend/tests/test_planner_providers.py +290 -0
  232. package/app/backend/tests/test_planner_unblock.py +183 -0
  233. package/app/backend/tests/test_revision_invariants.py +618 -0
  234. package/app/backend/tests/test_sqlite_kv.py +290 -0
  235. package/app/backend/tests/test_sqlite_worker.py +353 -0
  236. package/app/backend/tests/test_task_dispatch.py +100 -0
  237. package/app/backend/tests/test_ticket_validation.py +304 -0
  238. package/app/backend/tests/test_udar_agent.py +693 -0
  239. package/app/backend/tests/test_webhook_service.py +184 -0
  240. package/app/backend/tickets_output.json +59 -0
  241. package/app/backend/user_management_tickets.json +50 -0
  242. package/app/backend/uvicorn.log +0 -0
  243. package/app/draft.yaml +313 -0
  244. package/app/frontend/dist/assets/index-LcjCczu5.js +155 -0
  245. package/app/frontend/dist/assets/index-_FP_279e.css +1 -0
  246. package/app/frontend/dist/index.html +14 -0
  247. package/app/frontend/dist/vite.svg +1 -0
  248. package/app/frontend/package.json +101 -0
  249. package/bin/cli.js +527 -0
  250. package/package.json +37 -0
@@ -0,0 +1,3190 @@
1
+ """Worker task implementations for Draft.
2
+
3
+ These functions are called by the SQLiteWorker (in-process background job runner).
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import json
9
+ import logging
10
+ import os
11
+ import subprocess
12
+ import threading
13
+ import time
14
+ import uuid
15
+ from dataclasses import dataclass
16
+ from datetime import UTC, datetime
17
+ from pathlib import Path
18
+ from typing import TYPE_CHECKING
19
+
20
+ from sqlalchemy.orm import selectinload
21
+
22
+ if TYPE_CHECKING:
23
+ from app.services.config_service import PlannerConfig
24
+
25
+ logger = logging.getLogger(__name__)
26
+ from app.data_dir import get_evidence_dir as central_evidence_dir
27
+ from app.data_dir import get_log_path
28
+ from app.database_sync import get_sync_db
29
+ from app.exceptions import (
30
+ ExecutorNotFoundError,
31
+ NotAGitRepositoryError,
32
+ WorkspaceError,
33
+ )
34
+ from app.models.evidence import Evidence, EvidenceKind
35
+ from app.models.job import Job, JobStatus
36
+ from app.models.ticket import Ticket
37
+ from app.models.ticket_event import TicketEvent
38
+ from app.services.config_service import DraftConfig, YoloStatus
39
+ from app.services.cursor_log_normalizer import CursorLogNormalizer
40
+ from app.services.executor_service import (
41
+ ExecutorService,
42
+ ExecutorType,
43
+ PromptBundleBuilder,
44
+ )
45
+ from app.services.log_stream_service import LogLevel, log_stream_publisher
46
+ from app.services.workspace_service import WorkspaceService
47
+ from app.services.worktree_validator import WorktreeValidator
48
+ from app.state_machine import ActorType, EventType, TicketState
49
+
50
+ # Thread-local storage for job context (THREAD-SAFE)
51
+ _job_context = threading.local()
52
+
53
+ # Track active subprocesses for cancellation (THREAD-SAFE with lock)
54
+ _active_processes: dict[str, subprocess.Popen] = {}
55
+ _active_processes_lock = threading.Lock()
56
+
57
+ # Executor retry configuration
58
+ EXECUTOR_MAX_RETRIES = 2 # Retry up to 2 times (3 total attempts)
59
+ EXECUTOR_RETRY_DELAY_BASE = 5 # Base delay in seconds (exponential backoff)
60
+
61
+ # Retry-eligible exit codes (transient failures)
62
+ RETRYABLE_EXIT_CODES = {
63
+ -1, # Timeout or general failure
64
+ 124, # Timeout (from timeout command)
65
+ 137, # SIGKILL (OOM or external kill)
66
+ 143, # SIGTERM
67
+ }
68
+
69
+ # Retry-eligible error patterns in stderr (network/API issues)
70
+ RETRYABLE_ERROR_PATTERNS = [
71
+ "connection reset",
72
+ "connection refused",
73
+ "network error",
74
+ "rate limit",
75
+ "503 service unavailable",
76
+ "502 bad gateway",
77
+ "504 gateway timeout",
78
+ "temporary failure",
79
+ "timeout",
80
+ "timed out",
81
+ ]
82
+
83
+
84
+ def get_current_job() -> str | None:
85
+ """Get the current job ID for this thread."""
86
+ return getattr(_job_context, "job_id", None)
87
+
88
+
89
+ def write_log(log_path: Path, message: str, job_id: str | None = None) -> None:
90
+ """Write a timestamped message to the log file AND stream via Redis.
91
+
92
+ Args:
93
+ log_path: Path to the log file
94
+ message: The log message
95
+ job_id: Optional job ID for real-time streaming (uses thread-local if not set)
96
+ """
97
+ log_path.parent.mkdir(parents=True, exist_ok=True)
98
+ timestamp = datetime.now(UTC).isoformat()
99
+ with open(log_path, "a") as f:
100
+ f.write(f"[{timestamp}] {message}\n")
101
+
102
+ # Also stream to Redis for real-time SSE (THREAD-SAFE)
103
+ stream_job_id = job_id or get_current_job()
104
+ if stream_job_id:
105
+ try:
106
+ log_stream_publisher.push_info(stream_job_id, message)
107
+ except Exception:
108
+ pass # Don't fail job if streaming fails
109
+
110
+
111
+ def set_current_job(job_id: str | None) -> None:
112
+ """Set the current job ID for this thread (THREAD-SAFE)."""
113
+ _job_context.job_id = job_id
114
+
115
+
116
+ def is_retryable_error(exit_code: int, stderr: str) -> bool:
117
+ """Check if an executor error is retryable (transient failure).
118
+
119
+ Args:
120
+ exit_code: The process exit code
121
+ stderr: The stderr output from the process
122
+
123
+ Returns:
124
+ True if the error is likely transient and worth retrying
125
+ """
126
+ # Check exit code
127
+ if exit_code in RETRYABLE_EXIT_CODES:
128
+ return True
129
+
130
+ # Check stderr for known transient error patterns
131
+ stderr_lower = stderr.lower()
132
+ for pattern in RETRYABLE_ERROR_PATTERNS:
133
+ if pattern in stderr_lower:
134
+ return True
135
+
136
+ return False
137
+
138
+
139
+ def register_active_process(job_id: str, process: subprocess.Popen) -> None:
140
+ """Register an active subprocess for potential cancellation."""
141
+ with _active_processes_lock:
142
+ _active_processes[job_id] = process
143
+
144
+
145
+ def unregister_active_process(job_id: str) -> None:
146
+ """Unregister an active subprocess."""
147
+ with _active_processes_lock:
148
+ _active_processes.pop(job_id, None)
149
+
150
+
151
+ def kill_job_process(job_id: str) -> bool:
152
+ """Kill the subprocess for a job (for cancellation).
153
+
154
+ Returns:
155
+ True if process was found and killed, False otherwise
156
+ """
157
+ with _active_processes_lock:
158
+ process = _active_processes.get(job_id)
159
+ if process and process.poll() is None: # Still running
160
+ logger.info(f"Killing process {process.pid} for job {job_id}")
161
+ try:
162
+ process.kill()
163
+ process.wait(timeout=5)
164
+ return True
165
+ except Exception as e:
166
+ logger.error(f"Failed to kill process for job {job_id}: {e}")
167
+ return False
168
+ return False
169
+
170
+
171
+ def stream_finished(job_id: str) -> None:
172
+ """Signal that job has finished streaming."""
173
+ try:
174
+ log_stream_publisher.push_finished(job_id)
175
+ except Exception:
176
+ pass
177
+
178
+
179
+ def get_job_with_ticket(job_id: str) -> tuple[Job, Ticket] | None:
180
+ """Get a job and its associated ticket."""
181
+ with get_sync_db() as db:
182
+ job = (
183
+ db.query(Job)
184
+ .options(selectinload(Job.ticket).selectinload(Ticket.goal))
185
+ .filter(Job.id == job_id)
186
+ .first()
187
+ )
188
+ if job and job.ticket:
189
+ # Expunge to use outside session
190
+ db.expunge(job)
191
+ db.expunge(job.ticket)
192
+ if job.ticket.goal:
193
+ db.expunge(job.ticket.goal)
194
+ return job, job.ticket
195
+ return None
196
+
197
+
198
+ def ensure_workspace_for_ticket(
199
+ ticket_id: str, goal_id: str
200
+ ) -> tuple[Path | None, str | None]:
201
+ """
202
+ Ensure a workspace exists for a ticket and return paths.
203
+
204
+ Returns:
205
+ Tuple of (worktree_path, error_message).
206
+ If successful, worktree_path is set and error_message is None.
207
+ If failed, worktree_path is None and error_message describes the error.
208
+ """
209
+ with get_sync_db() as db:
210
+ workspace_service = WorkspaceService(db)
211
+ try:
212
+ workspace = workspace_service.ensure_workspace(ticket_id, goal_id)
213
+ return Path(workspace.worktree_path), None
214
+ except NotAGitRepositoryError as e:
215
+ return None, e.message
216
+ except WorkspaceError as e:
217
+ return None, e.message
218
+ except Exception as e:
219
+ return None, f"Failed to create workspace: {str(e)}"
220
+
221
+
222
+ def get_log_path_for_job(job_id: str, worktree_path: Path | None) -> tuple[Path, str]:
223
+ """
224
+ Get the log path for a job using central data directory.
225
+
226
+ Returns:
227
+ Tuple of (full_log_path, path_stored_in_db).
228
+ """
229
+ full_path = get_log_path(job_id)
230
+ # Store the full path in DB since logs are now in a central location
231
+ return full_path, str(full_path)
232
+
233
+
234
+ def update_job_started(
235
+ job_id: str, log_path: str, timeout_seconds: int | None = None
236
+ ) -> bool:
237
+ """Mark job as running. Returns False if job was canceled."""
238
+ with get_sync_db() as db:
239
+ job = db.query(Job).filter(Job.id == job_id).first()
240
+ if not job:
241
+ return False
242
+
243
+ # Check if job was canceled before it started
244
+ if job.status == JobStatus.CANCELED.value:
245
+ return False
246
+
247
+ now = datetime.now(UTC)
248
+ job.status = JobStatus.RUNNING.value
249
+ job.started_at = now
250
+ job.last_heartbeat_at = now
251
+ job.log_path = log_path
252
+ if timeout_seconds:
253
+ job.timeout_seconds = timeout_seconds
254
+ db.commit()
255
+ return True
256
+
257
+
258
+ def update_job_heartbeat(job_id: str) -> bool:
259
+ """Update job heartbeat timestamp. Returns False if job was canceled."""
260
+ with get_sync_db() as db:
261
+ job = db.query(Job).filter(Job.id == job_id).first()
262
+ if not job:
263
+ return False
264
+
265
+ # Check if job was canceled
266
+ if job.status == JobStatus.CANCELED.value:
267
+ return False
268
+
269
+ job.last_heartbeat_at = datetime.now(UTC)
270
+ db.commit()
271
+ return True
272
+
273
+
274
+ def update_job_finished(job_id: str, status: JobStatus, exit_code: int = 0) -> None:
275
+ """Mark job as finished with given status and exit code."""
276
+ with get_sync_db() as db:
277
+ job = db.query(Job).filter(Job.id == job_id).first()
278
+ if job:
279
+ job.status = status.value
280
+ job.finished_at = datetime.now(UTC)
281
+ job.exit_code = exit_code
282
+ db.commit()
283
+
284
+
285
+ def check_canceled(job_id: str) -> bool:
286
+ """Check if the job has been canceled."""
287
+ with get_sync_db() as db:
288
+ job = db.query(Job).filter(Job.id == job_id).first()
289
+ return job is not None and job.status == JobStatus.CANCELED.value
290
+
291
+
292
+ def get_evidence_dir(
293
+ worktree_path: Path | None, job_id: str, repo_root: Path | None = None
294
+ ) -> Path:
295
+ """Get the directory for storing evidence files.
296
+
297
+ Evidence is stored in a central location at ~/.draft/evidence/{job_id}/
298
+ so it survives worktree cleanup and doesn't pollute target repos.
299
+
300
+ Args:
301
+ worktree_path: Unused (kept for API compat)
302
+ job_id: UUID of the job
303
+ repo_root: Unused (kept for API compat)
304
+
305
+ Returns:
306
+ Path to evidence directory
307
+ """
308
+ return central_evidence_dir(job_id)
309
+
310
+
311
+ def run_verification_command(
312
+ command: str,
313
+ cwd: Path | None,
314
+ evidence_dir: Path,
315
+ evidence_id: str,
316
+ repo_root: Path,
317
+ timeout: int = 300,
318
+ extra_allowed_commands: list[str] | None = None,
319
+ ) -> tuple[int, str, str]:
320
+ """
321
+ Run a verification command and capture output (SECURE - no shell injection).
322
+
323
+ SECURITY: Uses shlex.split() to safely parse commands without shell=True.
324
+ Only allows commands from a predefined allowlist to prevent arbitrary execution.
325
+
326
+ Args:
327
+ command: The command string to parse and execute
328
+ cwd: Working directory for the command
329
+ evidence_dir: Directory to store stdout/stderr files
330
+ evidence_id: UUID for naming evidence files
331
+ repo_root: Path to repo root (for computing relative paths)
332
+ timeout: Command timeout in seconds
333
+ extra_allowed_commands: Additional commands to allow (from config)
334
+
335
+ Returns:
336
+ Tuple of (exit_code, stdout_relpath, stderr_relpath) - paths are relative to repo_root
337
+
338
+ Raises:
339
+ ValueError: If command is not in allowlist
340
+ """
341
+ import shlex
342
+ import shutil as _shutil
343
+
344
+ stdout_path = evidence_dir / f"{evidence_id}.stdout"
345
+ stderr_path = evidence_dir / f"{evidence_id}.stderr"
346
+
347
+ # Allowlist of permitted commands (prevents arbitrary code execution)
348
+ ALLOWED_COMMANDS = {
349
+ "pytest",
350
+ "python",
351
+ "python3",
352
+ "ruff",
353
+ "mypy",
354
+ "black",
355
+ "isort",
356
+ "npm",
357
+ "yarn",
358
+ "pnpm",
359
+ "node",
360
+ "cargo",
361
+ "rustc",
362
+ "go",
363
+ "make",
364
+ "eslint",
365
+ "tsc",
366
+ "jest",
367
+ "vitest",
368
+ "flake8",
369
+ "pylint",
370
+ }
371
+ if extra_allowed_commands:
372
+ ALLOWED_COMMANDS.update(extra_allowed_commands)
373
+
374
+ try:
375
+ # SECURE: Parse command string into argv array (prevents injection)
376
+ # shlex.split() handles quotes and escaping properly
377
+ cmd_argv = shlex.split(command)
378
+
379
+ if not cmd_argv:
380
+ raise ValueError("Empty command")
381
+
382
+ # SECURITY CHECK: Validate first argument is in allowlist
383
+ # Resolve the command to its full path to prevent PATH manipulation attacks
384
+ # (e.g., a malicious "pytest" script in the worktree shadowing the real one)
385
+ raw_cmd = cmd_argv[0]
386
+ base_command = Path(raw_cmd).name # Strip path, get command name
387
+ if base_command not in ALLOWED_COMMANDS:
388
+ raise ValueError(
389
+ f"Command '{base_command}' not in allowlist. "
390
+ f"Allowed: {', '.join(sorted(ALLOWED_COMMANDS))}"
391
+ )
392
+
393
+ # Resolve to absolute path via PATH lookup to ensure we run the real binary
394
+ resolved_path = _shutil.which(base_command)
395
+ if resolved_path:
396
+ cmd_argv[0] = resolved_path
397
+
398
+ # Run WITHOUT shell=True (SECURE - no command injection possible)
399
+ result = subprocess.run(
400
+ cmd_argv, # List, not string
401
+ shell=False, # CRITICAL: No shell metacharacter interpretation
402
+ cwd=cwd,
403
+ capture_output=True,
404
+ text=True,
405
+ timeout=timeout,
406
+ )
407
+
408
+ # Write stdout/stderr to files
409
+ stdout_path.write_text(result.stdout or "")
410
+ stderr_path.write_text(result.stderr or "")
411
+
412
+ # Return relative paths for DB storage (security: no absolute paths in DB)
413
+ stdout_rel = str(stdout_path)
414
+ stderr_rel = str(stderr_path)
415
+
416
+ return result.returncode, stdout_rel, stderr_rel
417
+
418
+ except subprocess.TimeoutExpired as e:
419
+ # Write partial output if available
420
+ stdout_path.write_text(e.stdout.decode() if e.stdout else "Command timed out")
421
+ stderr_path.write_text(e.stderr.decode() if e.stderr else "")
422
+ stdout_rel = str(stdout_path)
423
+ stderr_rel = str(stderr_path)
424
+ return -1, stdout_rel, stderr_rel
425
+
426
+ except ValueError as e:
427
+ # Command validation failed (not in allowlist or empty)
428
+ stdout_path.write_text("")
429
+ stderr_path.write_text(f"Command validation failed: {str(e)}")
430
+ stdout_rel = str(stdout_path)
431
+ stderr_rel = str(stderr_path)
432
+ return -1, stdout_rel, stderr_rel
433
+
434
+ except FileNotFoundError:
435
+ # Command not found in PATH
436
+ stdout_path.write_text("")
437
+ stderr_path.write_text(f"Command not found: {cmd_argv[0]}")
438
+ stdout_rel = str(stdout_path)
439
+ stderr_rel = str(stderr_path)
440
+ return -1, stdout_rel, stderr_rel
441
+
442
+ except Exception as e:
443
+ stdout_path.write_text("")
444
+ stderr_path.write_text(f"Error running command: {str(e)}")
445
+
446
+ stdout_rel = str(stdout_path)
447
+ stderr_rel = str(stderr_path)
448
+ return -1, stdout_rel, stderr_rel
449
+
450
+
451
+ def create_evidence_record(
452
+ ticket_id: str,
453
+ job_id: str,
454
+ command: str,
455
+ exit_code: int,
456
+ stdout_path: str,
457
+ stderr_path: str,
458
+ evidence_id: str,
459
+ kind: EvidenceKind = EvidenceKind.COMMAND_LOG,
460
+ ) -> Evidence:
461
+ """Create an Evidence record in the database.
462
+
463
+ Args:
464
+ ticket_id: UUID of the ticket
465
+ job_id: UUID of the job
466
+ command: Command that was executed
467
+ exit_code: Exit code from the command
468
+ stdout_path: Path to stdout file
469
+ stderr_path: Path to stderr file
470
+ evidence_id: UUID for this evidence record
471
+ kind: Type of evidence (executor_stdout, git_diff_stat, etc.)
472
+
473
+ Returns:
474
+ The created Evidence record
475
+ """
476
+ with get_sync_db() as db:
477
+ evidence = Evidence(
478
+ id=evidence_id,
479
+ ticket_id=ticket_id,
480
+ job_id=job_id,
481
+ kind=kind.value,
482
+ command=command,
483
+ exit_code=exit_code,
484
+ stdout_path=stdout_path,
485
+ stderr_path=stderr_path,
486
+ )
487
+ db.add(evidence)
488
+ db.commit()
489
+ db.refresh(evidence)
490
+ return evidence
491
+
492
+
493
+ def create_revision_for_job(
494
+ ticket_id: str,
495
+ job_id: str,
496
+ diff_stat_evidence_id: str | None = None,
497
+ diff_patch_evidence_id: str | None = None,
498
+ ) -> str | None:
499
+ """Create a Revision record for a job that produced changes.
500
+
501
+ This function is IDEMPOTENT - if the same job_id is retried, returns existing revision.
502
+ Automatically supersedes any existing open revisions for the ticket.
503
+
504
+ Args:
505
+ ticket_id: UUID of the ticket
506
+ job_id: UUID of the job
507
+ diff_stat_evidence_id: Optional evidence ID for git diff stat
508
+ diff_patch_evidence_id: Optional evidence ID for git diff patch
509
+
510
+ Returns:
511
+ The revision ID if created/found, None on error
512
+ """
513
+ from app.models.revision import Revision, RevisionStatus
514
+
515
+ with get_sync_db() as db:
516
+ try:
517
+ # IDEMPOTENCY CHECK: Return existing revision if job was already processed
518
+ existing = (
519
+ db.query(Revision)
520
+ .filter(
521
+ Revision.ticket_id == ticket_id,
522
+ Revision.job_id == job_id,
523
+ )
524
+ .first()
525
+ )
526
+ if existing:
527
+ import logging
528
+
529
+ logging.getLogger(__name__).info(
530
+ f"Revision already exists for job {job_id}: {existing.id}"
531
+ )
532
+ return existing.id
533
+
534
+ # Supersede any open revisions (in same transaction)
535
+ open_revisions = (
536
+ db.query(Revision)
537
+ .filter(
538
+ Revision.ticket_id == ticket_id,
539
+ Revision.status == RevisionStatus.OPEN.value,
540
+ )
541
+ .all()
542
+ )
543
+ for rev in open_revisions:
544
+ rev.status = RevisionStatus.SUPERSEDED.value
545
+
546
+ # Get next revision number
547
+ last_revision = (
548
+ db.query(Revision)
549
+ .filter(Revision.ticket_id == ticket_id)
550
+ .order_by(Revision.number.desc())
551
+ .first()
552
+ )
553
+ next_number = (last_revision.number if last_revision else 0) + 1
554
+
555
+ # Create new revision
556
+ revision = Revision(
557
+ ticket_id=ticket_id,
558
+ job_id=job_id,
559
+ number=next_number,
560
+ status=RevisionStatus.OPEN.value,
561
+ diff_stat_evidence_id=diff_stat_evidence_id,
562
+ diff_patch_evidence_id=diff_patch_evidence_id,
563
+ )
564
+ db.add(revision)
565
+ db.commit()
566
+ db.refresh(revision)
567
+ return revision.id
568
+ except Exception as e:
569
+ db.rollback()
570
+ # Log but don't fail the job
571
+ import logging
572
+
573
+ logging.getLogger(__name__).error(f"Failed to create revision: {e}")
574
+ return None
575
+
576
+
577
+ def get_feedback_bundle_for_ticket(ticket_id: str) -> dict | None:
578
+ """Get the feedback bundle from the most recent changes_requested revision.
579
+
580
+ This is used when re-running an execute job after changes were requested.
581
+
582
+ Args:
583
+ ticket_id: UUID of the ticket
584
+
585
+ Returns:
586
+ Feedback bundle dict if found, None otherwise
587
+ """
588
+ from app.models.review_comment import ReviewComment
589
+ from app.models.review_summary import ReviewSummary
590
+ from app.models.revision import Revision, RevisionStatus
591
+
592
+ with get_sync_db() as db:
593
+ try:
594
+ # Find the most recent revision with changes_requested status
595
+ revision = (
596
+ db.query(Revision)
597
+ .filter(
598
+ Revision.ticket_id == ticket_id,
599
+ Revision.status == RevisionStatus.CHANGES_REQUESTED.value,
600
+ )
601
+ .order_by(Revision.number.desc())
602
+ .first()
603
+ )
604
+
605
+ if not revision:
606
+ return None
607
+
608
+ # Get review summary
609
+ review_summary = (
610
+ db.query(ReviewSummary)
611
+ .filter(ReviewSummary.revision_id == revision.id)
612
+ .first()
613
+ )
614
+
615
+ # Get unresolved comments
616
+ comments = (
617
+ db.query(ReviewComment)
618
+ .filter(
619
+ ReviewComment.revision_id == revision.id,
620
+ ReviewComment.resolved == False, # noqa: E712
621
+ )
622
+ .order_by(ReviewComment.created_at)
623
+ .all()
624
+ )
625
+
626
+ # Build feedback bundle
627
+ return {
628
+ "ticket_id": ticket_id,
629
+ "revision_id": revision.id,
630
+ "revision_number": revision.number,
631
+ "decision": "changes_requested",
632
+ "summary": review_summary.body if review_summary else "",
633
+ "comments": [
634
+ {
635
+ "file_path": c.file_path,
636
+ "line_number": c.line_number,
637
+ "anchor": c.anchor,
638
+ "body": c.body,
639
+ "line_content": c.line_content,
640
+ }
641
+ for c in comments
642
+ ],
643
+ }
644
+ except Exception as e:
645
+ import logging
646
+
647
+ logging.getLogger(__name__).error(f"Failed to get feedback bundle: {e}")
648
+ return None
649
+
650
+
651
+ def transition_ticket_sync(
652
+ ticket_id: str,
653
+ to_state: TicketState,
654
+ reason: str | None = None,
655
+ payload: dict | None = None,
656
+ actor_id: str = "worker",
657
+ auto_verify: bool = True,
658
+ ) -> None:
659
+ """
660
+ Transition a ticket to a new state synchronously.
661
+
662
+ Args:
663
+ ticket_id: The UUID of the ticket
664
+ to_state: The target state
665
+ reason: Optional reason for the transition
666
+ payload: Optional payload for the event
667
+ actor_id: The ID of the actor performing the transition
668
+ auto_verify: If True, auto-enqueue verify job when entering verifying state
669
+ """
670
+ with get_sync_db() as db:
671
+ ticket = db.query(Ticket).filter(Ticket.id == ticket_id).first()
672
+ if not ticket:
673
+ return
674
+
675
+ from_state = ticket.state
676
+
677
+ # Skip no-op transitions (already in target state)
678
+ if from_state == to_state.value:
679
+ logger.info(
680
+ f"Skipping no-op transition for ticket {ticket_id}: already in {from_state}"
681
+ )
682
+ return
683
+
684
+ # Validate state transition against the state machine
685
+ from app.state_machine import validate_transition as sm_validate
686
+
687
+ try:
688
+ from_state_enum = TicketState(from_state)
689
+ except ValueError:
690
+ logger.error(f"Invalid current state '{from_state}' for ticket {ticket_id}")
691
+ return
692
+
693
+ if not sm_validate(from_state_enum, to_state):
694
+ logger.error(
695
+ f"Invalid state transition {from_state} -> {to_state.value} "
696
+ f"for ticket {ticket_id}, skipping"
697
+ )
698
+ return
699
+
700
+ ticket.state = to_state.value
701
+ board_id = ticket.board_id
702
+
703
+ # Create transition event
704
+ event = TicketEvent(
705
+ ticket_id=ticket_id,
706
+ event_type=EventType.TRANSITIONED.value,
707
+ from_state=from_state,
708
+ to_state=to_state.value,
709
+ actor_type=ActorType.EXECUTOR.value,
710
+ actor_id=actor_id,
711
+ reason=reason,
712
+ payload_json=json.dumps(payload) if payload else None,
713
+ )
714
+ db.add(event)
715
+ db.commit()
716
+
717
+ # Broadcast board invalidation via WebSocket
718
+ if board_id:
719
+ from app.websocket.manager import broadcast_sync
720
+
721
+ broadcast_sync(
722
+ f"board:{board_id}",
723
+ {"type": "invalidate", "reason": "ticket_transition"},
724
+ )
725
+
726
+ # Auto-trigger verification when entering verifying state
727
+ if auto_verify and to_state == TicketState.VERIFYING:
728
+ _enqueue_verify_job_sync(ticket_id)
729
+
730
+
731
+ def _enqueue_verify_job_sync(ticket_id: str) -> str | None:
732
+ """
733
+ Synchronously enqueue a verify job for a ticket (idempotent).
734
+
735
+ Idempotency: Only creates a new verify job if there is no active
736
+ (queued or running) verify job for this ticket. This prevents
737
+ duplicate verify jobs from race conditions or retries.
738
+
739
+ Returns:
740
+ The job ID if created, None if skipped (already active).
741
+ """
742
+ from app.models.job import Job, JobKind, JobStatus
743
+
744
+ with get_sync_db() as db:
745
+ # IDEMPOTENCY CHECK: Is there already an active verify job?
746
+ active_verify = (
747
+ db.query(Job)
748
+ .filter(
749
+ Job.ticket_id == ticket_id,
750
+ Job.kind == JobKind.VERIFY.value,
751
+ Job.status.in_([JobStatus.QUEUED.value, JobStatus.RUNNING.value]),
752
+ )
753
+ .first()
754
+ )
755
+
756
+ if active_verify:
757
+ # Already has an active verify job - skip to avoid duplicates
758
+ return None
759
+
760
+ # Get the ticket to inherit board_id
761
+ ticket = db.query(Ticket).filter(Ticket.id == ticket_id).first()
762
+ board_id = ticket.board_id if ticket else None
763
+
764
+ # Create the job record with board_id for permission scoping
765
+ job = Job(
766
+ ticket_id=ticket_id,
767
+ board_id=board_id,
768
+ kind=JobKind.VERIFY.value,
769
+ status=JobStatus.QUEUED.value,
770
+ )
771
+ db.add(job)
772
+ db.flush()
773
+ job_id = job.id
774
+ # Commit BEFORE enqueue_task to release the SQLite write lock.
775
+ # enqueue_task opens a separate sqlite3 connection which would
776
+ # deadlock if this session still holds the write lock.
777
+ db.commit()
778
+
779
+ # Enqueue the verify task (outside the db session to avoid deadlock)
780
+ from app.services.task_dispatch import enqueue_task
781
+
782
+ task = enqueue_task("verify_ticket", args=[job_id])
783
+
784
+ # Update the job with the task ID
785
+ with get_sync_db() as db:
786
+ job = db.query(Job).filter(Job.id == job_id).first()
787
+ if job:
788
+ job.celery_task_id = task.id
789
+
790
+ return job_id
791
+
792
+
793
+ def run_executor_cli(
794
+ command: list[str],
795
+ cwd: Path,
796
+ evidence_dir: Path,
797
+ evidence_id: str,
798
+ repo_root: Path,
799
+ timeout: int = 600,
800
+ job_id: str | None = None,
801
+ normalize_logs: bool = False,
802
+ stdin_content: str | None = None,
803
+ ) -> tuple[int, str, str]:
804
+ """
805
+ Run the executor CLI and capture output with real-time streaming.
806
+
807
+ Args:
808
+ command: The CLI command to run as a list of arguments
809
+ cwd: Working directory for the command
810
+ evidence_dir: Directory to store stdout/stderr files
811
+ evidence_id: UUID for naming evidence files
812
+ repo_root: Path to repo root (for computing relative paths)
813
+ timeout: Command timeout in seconds
814
+ job_id: Optional job ID for real-time log streaming
815
+ normalize_logs: If True, parse cursor-agent JSON and stream normalized entries
816
+ stdin_content: Optional content to pipe to the process via stdin
817
+
818
+ Returns:
819
+ Tuple of (exit_code, stdout_relpath, stderr_relpath) - paths are relative to repo_root
820
+ """
821
+ import json as json_module
822
+ import threading
823
+
824
+ stdout_path = evidence_dir / f"{evidence_id}.stdout"
825
+ stderr_path = evidence_dir / f"{evidence_id}.stderr"
826
+
827
+ stdout_lines: list[str] = []
828
+ stderr_lines: list[str] = []
829
+
830
+ # Create normalizer if needed
831
+ normalizer = CursorLogNormalizer(str(cwd)) if normalize_logs else None
832
+
833
+ # Strip Claude Code session env vars to avoid "nested session" errors
834
+ # when spawning claude CLI from within a Claude Code session
835
+ clean_env = {
836
+ k: v
837
+ for k, v in os.environ.items()
838
+ if k not in ("CLAUDECODE", "CLAUDE_CODE_ENTRYPOINT")
839
+ }
840
+
841
+ try:
842
+ # Use Popen for real-time streaming instead of blocking run()
843
+ process = subprocess.Popen(
844
+ command,
845
+ cwd=cwd,
846
+ stdin=subprocess.PIPE if stdin_content else None,
847
+ stdout=subprocess.PIPE,
848
+ stderr=subprocess.PIPE,
849
+ text=True,
850
+ bufsize=1, # Line buffered
851
+ env=clean_env,
852
+ )
853
+
854
+ # Write stdin content and close to signal EOF
855
+ if stdin_content and process.stdin:
856
+ process.stdin.write(stdin_content)
857
+ process.stdin.close()
858
+
859
+ def stream_output(pipe, lines_list, is_stderr=False, stop_event=None):
860
+ """Read and stream output line by line with stop event support."""
861
+ try:
862
+ for line in iter(pipe.readline, ""):
863
+ # Check if we should stop
864
+ if stop_event and stop_event.is_set():
865
+ logger.debug(
866
+ f"Stream thread stopping due to stop_event ({'stderr' if is_stderr else 'stdout'})"
867
+ )
868
+ break
869
+ if not line:
870
+ break
871
+ line = line.rstrip("\n")
872
+ lines_list.append(line)
873
+
874
+ # Stream to Redis for real-time SSE
875
+ if job_id:
876
+ try:
877
+ if is_stderr:
878
+ log_stream_publisher.push_stderr(job_id, line)
879
+ elif normalizer:
880
+ # Parse and stream normalized entries
881
+ entries = normalizer.process_line(line)
882
+ for entry in entries:
883
+ # Serialize normalized entry as JSON
884
+ entry_data = {
885
+ "entry_type": entry.entry_type.value,
886
+ "content": entry.content,
887
+ "sequence": entry.sequence,
888
+ "tool_name": entry.tool_name,
889
+ "action_type": entry.action_type.value
890
+ if entry.action_type
891
+ else None,
892
+ "tool_status": entry.tool_status.value
893
+ if entry.tool_status
894
+ else None,
895
+ "metadata": entry.metadata,
896
+ }
897
+ log_stream_publisher.push(
898
+ job_id,
899
+ LogLevel.NORMALIZED,
900
+ json_module.dumps(entry_data),
901
+ )
902
+ else:
903
+ log_stream_publisher.push_stdout(job_id, line)
904
+ except Exception:
905
+ pass # Don't fail execution if streaming fails
906
+ finally:
907
+ pipe.close()
908
+
909
+ # Create stop events for graceful thread termination
910
+ stdout_stop_event = threading.Event()
911
+ stderr_stop_event = threading.Event()
912
+
913
+ # Stream stdout and stderr in parallel threads
914
+ # Use daemon=True so threads don't block process exit if they get stuck
915
+ stdout_thread = threading.Thread(
916
+ target=stream_output,
917
+ args=(process.stdout, stdout_lines, False, stdout_stop_event),
918
+ daemon=True,
919
+ name=f"stdout-{job_id[:8] if job_id else 'unknown'}",
920
+ )
921
+ stderr_thread = threading.Thread(
922
+ target=stream_output,
923
+ args=(process.stderr, stderr_lines, True, stderr_stop_event),
924
+ daemon=True,
925
+ name=f"stderr-{job_id[:8] if job_id else 'unknown'}",
926
+ )
927
+
928
+ stdout_thread.start()
929
+ stderr_thread.start()
930
+
931
+ # Register process for cancellation support
932
+ if job_id:
933
+ register_active_process(job_id, process)
934
+
935
+ # Wait for process with timeout AND poll for cancellation
936
+ try:
937
+ start_time = time.time()
938
+ last_heartbeat_time = start_time
939
+ while True:
940
+ # Check if process finished
941
+ exit_code = process.poll()
942
+ if exit_code is not None:
943
+ break
944
+
945
+ # Update heartbeat every 30 seconds to prevent watchdog from killing long-running jobs
946
+ now = time.time()
947
+ if job_id and now - last_heartbeat_time >= 30:
948
+ update_job_heartbeat(job_id)
949
+ last_heartbeat_time = now
950
+
951
+ # Check if job was canceled
952
+ if job_id and check_canceled(job_id):
953
+ logger.info(
954
+ f"Job {job_id} canceled, attempting graceful shutdown of process {process.pid}"
955
+ )
956
+ # Try SIGTERM first for graceful shutdown
957
+ process.terminate()
958
+ try:
959
+ process.wait(timeout=5)
960
+ logger.info(f"Process {process.pid} terminated gracefully")
961
+ except subprocess.TimeoutExpired:
962
+ logger.warning(
963
+ f"Process {process.pid} did not terminate gracefully, using SIGKILL"
964
+ )
965
+ process.kill() # Force kill if still alive
966
+ process.wait()
967
+ stdout_lines.append("\n[CANCELED] Job canceled by user")
968
+ exit_code = -2 # Special exit code for cancellation
969
+ break
970
+
971
+ # Check timeout
972
+ if time.time() - start_time > timeout:
973
+ logger.warning(
974
+ f"Process {process.pid} exceeded timeout {timeout}s, attempting graceful shutdown"
975
+ )
976
+ # Try SIGTERM first for graceful shutdown
977
+ process.terminate()
978
+ try:
979
+ process.wait(timeout=5)
980
+ logger.info(
981
+ f"Process {process.pid} terminated gracefully after timeout"
982
+ )
983
+ except subprocess.TimeoutExpired:
984
+ logger.warning(
985
+ f"Process {process.pid} did not terminate gracefully, using SIGKILL"
986
+ )
987
+ process.kill()
988
+ process.wait()
989
+ stdout_lines.append(
990
+ f"\n[TIMEOUT] Process killed after {timeout} seconds"
991
+ )
992
+ exit_code = -1
993
+ break
994
+
995
+ # Poll every second
996
+ time.sleep(1)
997
+
998
+ finally:
999
+ # Unregister process
1000
+ if job_id:
1001
+ unregister_active_process(job_id)
1002
+
1003
+ # Signal threads to stop gracefully
1004
+ stdout_stop_event.set()
1005
+ stderr_stop_event.set()
1006
+
1007
+ # Close pipes to unblock threads waiting on readline()
1008
+ if process.stdout:
1009
+ try:
1010
+ process.stdout.close()
1011
+ except Exception:
1012
+ pass
1013
+ if process.stderr:
1014
+ try:
1015
+ process.stderr.close()
1016
+ except Exception:
1017
+ pass
1018
+
1019
+ # Wait for output threads to finish (with timeout to prevent blocking)
1020
+ # Try longer timeout first (10s)
1021
+ stdout_thread.join(timeout=10)
1022
+ stderr_thread.join(timeout=10)
1023
+
1024
+ # CRITICAL: If threads didn't stop, force cleanup
1025
+ threads_stuck = False
1026
+ if stdout_thread.is_alive():
1027
+ logger.error(
1028
+ f"stdout thread for job {job_id[:8] if job_id else 'unknown'} did not stop after 10s - forcing cleanup"
1029
+ )
1030
+ threads_stuck = True
1031
+ # Thread will be GC'd because daemon=True, but we lost some output
1032
+ # This is acceptable - better than blocking indefinitely
1033
+
1034
+ if stderr_thread.is_alive():
1035
+ logger.error(
1036
+ f"stderr thread for job {job_id[:8] if job_id else 'unknown'} did not stop after 10s - forcing cleanup"
1037
+ )
1038
+ threads_stuck = True
1039
+
1040
+ if threads_stuck:
1041
+ # Add warning to output that some logs may be incomplete
1042
+ stdout_lines.append(
1043
+ "\n[WARNING] Output streaming threads did not terminate cleanly - some logs may be incomplete"
1044
+ )
1045
+
1046
+ # Write captured output to files
1047
+ stdout_path.write_text("\n".join(stdout_lines))
1048
+ stderr_path.write_text("\n".join(stderr_lines))
1049
+
1050
+ # Return relative paths for secure DB storage
1051
+ stdout_rel = str(stdout_path)
1052
+ stderr_rel = str(stderr_path)
1053
+ return exit_code, stdout_rel, stderr_rel
1054
+
1055
+ except subprocess.TimeoutExpired as e:
1056
+ # Write partial output if available
1057
+ stdout_path.write_text(
1058
+ e.stdout.decode()
1059
+ if e.stdout
1060
+ else f"Command timed out after {timeout} seconds"
1061
+ )
1062
+ stderr_path.write_text(e.stderr.decode() if e.stderr else "")
1063
+ stdout_rel = str(stdout_path)
1064
+ stderr_rel = str(stderr_path)
1065
+ return -1, stdout_rel, stderr_rel
1066
+
1067
+ except FileNotFoundError as e:
1068
+ stdout_path.write_text("")
1069
+ stderr_path.write_text(f"Executor CLI not found: {str(e)}")
1070
+ stdout_rel = str(stdout_path)
1071
+ stderr_rel = str(stderr_path)
1072
+ return -1, stdout_rel, stderr_rel
1073
+
1074
+ except Exception as e:
1075
+ stdout_path.write_text("")
1076
+ stderr_path.write_text(f"Error running executor CLI: {str(e)}")
1077
+ stdout_rel = str(stdout_path)
1078
+ stderr_rel = str(stderr_path)
1079
+ return -1, stdout_rel, stderr_rel
1080
+
1081
+
1082
+ def run_executor_cli_with_retry(
1083
+ command: list[str],
1084
+ cwd: Path,
1085
+ evidence_dir: Path,
1086
+ evidence_id: str,
1087
+ repo_root: Path,
1088
+ timeout: int = 600,
1089
+ job_id: str | None = None,
1090
+ normalize_logs: bool = False,
1091
+ stdin_content: str | None = None,
1092
+ log_path: Path | None = None,
1093
+ ) -> tuple[int, str, str]:
1094
+ """
1095
+ Wrapper for run_executor_cli with retry logic for transient failures.
1096
+
1097
+ Retries up to EXECUTOR_MAX_RETRIES times with exponential backoff when:
1098
+ - Exit code is in RETRYABLE_EXIT_CODES (timeout, kill, term)
1099
+ - Stderr contains known transient error patterns (network, rate limit, etc.)
1100
+
1101
+ Args:
1102
+ Same as run_executor_cli, plus:
1103
+ log_path: Optional path to write retry messages
1104
+
1105
+ Returns:
1106
+ Same as run_executor_cli: (exit_code, stdout_relpath, stderr_relpath)
1107
+ """
1108
+ last_exit_code = -1
1109
+ last_stdout_path = ""
1110
+ last_stderr_path = ""
1111
+
1112
+ for attempt in range(EXECUTOR_MAX_RETRIES + 1): # 0, 1, 2 = 3 total attempts
1113
+ if attempt > 0:
1114
+ # This is a retry - calculate backoff delay
1115
+ delay = EXECUTOR_RETRY_DELAY_BASE * (
1116
+ 2 ** (attempt - 1)
1117
+ ) # Exponential backoff: 5s, 10s
1118
+ retry_msg = f"Retry attempt {attempt}/{EXECUTOR_MAX_RETRIES} after {delay}s delay..."
1119
+ logger.info(retry_msg)
1120
+ if log_path:
1121
+ write_log(log_path, retry_msg, job_id)
1122
+ time.sleep(delay)
1123
+
1124
+ # Run the executor
1125
+ exit_code, stdout_rel, stderr_rel = run_executor_cli(
1126
+ command=command,
1127
+ cwd=cwd,
1128
+ evidence_dir=evidence_dir,
1129
+ evidence_id=f"{evidence_id}_attempt{attempt}"
1130
+ if attempt > 0
1131
+ else evidence_id,
1132
+ repo_root=repo_root,
1133
+ timeout=timeout,
1134
+ job_id=job_id,
1135
+ normalize_logs=normalize_logs,
1136
+ stdin_content=stdin_content,
1137
+ )
1138
+
1139
+ last_exit_code = exit_code
1140
+ last_stdout_path = stdout_rel
1141
+ last_stderr_path = stderr_rel
1142
+
1143
+ # Success - return immediately
1144
+ if exit_code == 0:
1145
+ if attempt > 0:
1146
+ success_msg = f"Executor succeeded on retry attempt {attempt}"
1147
+ logger.info(success_msg)
1148
+ if log_path:
1149
+ write_log(log_path, success_msg, job_id)
1150
+ return exit_code, stdout_rel, stderr_rel
1151
+
1152
+ # Check if this is the last attempt
1153
+ if attempt >= EXECUTOR_MAX_RETRIES:
1154
+ break
1155
+
1156
+ # Check if error is retryable
1157
+ try:
1158
+ stderr_content = (repo_root / stderr_rel).read_text()
1159
+ except Exception:
1160
+ stderr_content = ""
1161
+
1162
+ if is_retryable_error(exit_code, stderr_content):
1163
+ retry_reason = (
1164
+ f"Executor failed with retryable error (exit_code={exit_code})"
1165
+ )
1166
+ logger.warning(retry_reason)
1167
+ if log_path:
1168
+ write_log(log_path, retry_reason, job_id)
1169
+ # Continue to next iteration (retry)
1170
+ else:
1171
+ # Not retryable - fail immediately
1172
+ non_retry_msg = f"Executor failed with non-retryable error (exit_code={exit_code}) - not retrying"
1173
+ logger.info(non_retry_msg)
1174
+ if log_path:
1175
+ write_log(log_path, non_retry_msg, job_id)
1176
+ break
1177
+
1178
+ # All retries exhausted or non-retryable error
1179
+ if attempt > 0:
1180
+ exhausted_msg = f"Executor failed after {attempt + 1} attempts (final exit_code={last_exit_code})"
1181
+ logger.error(exhausted_msg)
1182
+ if log_path:
1183
+ write_log(log_path, exhausted_msg, job_id)
1184
+
1185
+ return last_exit_code, last_stdout_path, last_stderr_path
1186
+
1187
+
1188
+ def capture_git_diff(
1189
+ cwd: Path,
1190
+ evidence_dir: Path,
1191
+ evidence_id: str,
1192
+ repo_root: Path,
1193
+ ) -> tuple[int, str, str, str, bool]:
1194
+ """
1195
+ Capture git diff output for changes made in the worktree.
1196
+
1197
+ This function captures both:
1198
+ 1. Changes to tracked files (via git diff)
1199
+ 2. New untracked files (via git status --porcelain)
1200
+
1201
+ Args:
1202
+ cwd: Working directory (worktree path)
1203
+ evidence_dir: Directory to store diff files
1204
+ evidence_id: UUID for naming evidence files
1205
+ repo_root: Path to repo root (for computing relative paths)
1206
+
1207
+ Returns:
1208
+ Tuple of (exit_code, diff_stat_relpath, diff_patch_relpath, diff_stat_text, has_changes)
1209
+ has_changes is True if there are uncommitted changes OR new untracked files.
1210
+ Paths are relative to repo_root.
1211
+ """
1212
+ diff_stat_path = evidence_dir / f"{evidence_id}.diff_stat"
1213
+ diff_patch_path = evidence_dir / f"{evidence_id}.diff_patch"
1214
+ stderr_path = evidence_dir / f"{evidence_id}.stderr"
1215
+
1216
+ diff_stat = ""
1217
+ has_changes = False
1218
+ has_tracked_changes = False
1219
+ has_untracked_files = False
1220
+ untracked_files: list[str] = []
1221
+
1222
+ try:
1223
+ # First get the diff stat for tracked file changes (both staged and unstaged)
1224
+ stat_result = subprocess.run(
1225
+ ["git", "diff", "HEAD", "--stat"],
1226
+ cwd=cwd,
1227
+ capture_output=True,
1228
+ text=True,
1229
+ timeout=60,
1230
+ )
1231
+ diff_stat = stat_result.stdout.strip() if stat_result.stdout else ""
1232
+
1233
+ # Then get the full patch for tracked files (both staged and unstaged)
1234
+ patch_result = subprocess.run(
1235
+ ["git", "diff", "HEAD"],
1236
+ cwd=cwd,
1237
+ capture_output=True,
1238
+ text=True,
1239
+ timeout=60,
1240
+ )
1241
+ diff_patch = patch_result.stdout.strip() if patch_result.stdout else ""
1242
+ has_tracked_changes = bool(diff_patch)
1243
+
1244
+ # Also check for untracked files (new files created by executor)
1245
+ # These won't show up in git diff but represent real work done
1246
+ status_result = subprocess.run(
1247
+ ["git", "status", "--porcelain"],
1248
+ cwd=cwd,
1249
+ capture_output=True,
1250
+ text=True,
1251
+ timeout=60,
1252
+ )
1253
+ if status_result.stdout:
1254
+ for line in status_result.stdout.strip().split("\n"):
1255
+ if line.startswith("??"):
1256
+ # Untracked file - extract path (skip "?? " prefix)
1257
+ file_path = line[3:].strip()
1258
+ # Skip .draft directory (internal files)
1259
+ if not file_path.startswith(".draft"):
1260
+ untracked_files.append(file_path)
1261
+ has_untracked_files = len(untracked_files) > 0
1262
+
1263
+ # Determine if there are actual changes (tracked OR untracked)
1264
+ has_changes = has_tracked_changes or has_untracked_files
1265
+
1266
+ # Build comprehensive diff stat that includes untracked files
1267
+ stat_parts = []
1268
+ if diff_stat:
1269
+ stat_parts.append(diff_stat)
1270
+ if untracked_files:
1271
+ stat_parts.append("\nNew files (untracked):")
1272
+ for f in untracked_files[:20]: # Limit to 20 files in summary
1273
+ stat_parts.append(f" + {f}")
1274
+ if len(untracked_files) > 20:
1275
+ stat_parts.append(f" ... and {len(untracked_files) - 20} more")
1276
+
1277
+ final_diff_stat = "\n".join(stat_parts) if stat_parts else "(no changes)"
1278
+ diff_stat_path.write_text(final_diff_stat)
1279
+
1280
+ # For patch, also include untracked file contents if any
1281
+ patch_parts = []
1282
+ if diff_patch:
1283
+ patch_parts.append(diff_patch)
1284
+ if untracked_files:
1285
+ patch_parts.append("\n\n# === New untracked files ===\n")
1286
+ for f in untracked_files[:10]: # Limit to 10 files to avoid huge patches
1287
+ file_full_path = cwd / f
1288
+ if file_full_path.is_file():
1289
+ try:
1290
+ content = file_full_path.read_text()
1291
+ # Truncate large files
1292
+ if len(content) > 5000:
1293
+ content = content[:5000] + "\n... (truncated)"
1294
+ patch_parts.append(f"\n# +++ {f}\n{content}")
1295
+ except Exception:
1296
+ patch_parts.append(f"\n# +++ {f} (could not read)")
1297
+
1298
+ final_patch = "\n".join(patch_parts) if patch_parts else "(no changes)"
1299
+ diff_patch_path.write_text(final_patch)
1300
+
1301
+ # Combine stderr from commands
1302
+ combined_stderr = ""
1303
+ if stat_result.stderr:
1304
+ combined_stderr += f"git diff --stat stderr:\n{stat_result.stderr}\n"
1305
+ if patch_result.stderr:
1306
+ combined_stderr += f"git diff stderr:\n{patch_result.stderr}\n"
1307
+ if status_result.stderr:
1308
+ combined_stderr += f"git status stderr:\n{status_result.stderr}\n"
1309
+ stderr_path.write_text(combined_stderr)
1310
+
1311
+ # Return relative paths for secure DB storage
1312
+ diff_stat_rel = str(diff_stat_path)
1313
+ diff_patch_rel = str(diff_patch_path)
1314
+ return 0, diff_stat_rel, diff_patch_rel, final_diff_stat, has_changes
1315
+
1316
+ except subprocess.TimeoutExpired:
1317
+ diff_stat_path.write_text("Git diff timed out")
1318
+ diff_patch_path.write_text("")
1319
+ stderr_path.write_text("Git diff command timed out after 60 seconds")
1320
+ diff_stat_rel = str(diff_stat_path)
1321
+ diff_patch_rel = str(diff_patch_path)
1322
+ return -1, diff_stat_rel, diff_patch_rel, "(timeout)", False
1323
+
1324
+ except Exception as e:
1325
+ diff_stat_path.write_text("")
1326
+ diff_patch_path.write_text("")
1327
+ stderr_path.write_text(f"Error running git diff: {str(e)}")
1328
+ diff_stat_rel = str(diff_stat_path)
1329
+ diff_patch_rel = str(diff_patch_path)
1330
+ return -1, diff_stat_rel, diff_patch_rel, "(error)", False
1331
+
1332
+
1333
+ # =============================================================================
1334
+ # NO-CHANGES ANALYSIS (LLM-powered)
1335
+ # =============================================================================
1336
+
1337
+
1338
+ @dataclass
1339
+ class NoChangesAnalysis:
1340
+ """Result of analyzing why no code changes were produced."""
1341
+
1342
+ reason: str # Human-readable explanation
1343
+ needs_code_changes: bool # True if code changes are actually needed
1344
+ requires_manual_work: bool # True if manual human intervention is required
1345
+ manual_work_description: str | None # Description of manual work needed (if any)
1346
+
1347
+
1348
+ def analyze_no_changes_reason(
1349
+ ticket_title: str,
1350
+ ticket_description: str | None,
1351
+ executor_stdout: str,
1352
+ planner_config: PlannerConfig,
1353
+ ) -> NoChangesAnalysis:
1354
+ """
1355
+ Analyze executor output to determine why no code changes were produced.
1356
+
1357
+ Uses LLM to understand the executor's reasoning and categorize the result:
1358
+ 1. No changes needed - the task doesn't require code modifications
1359
+ 2. Manual work required - needs human intervention (config, external tools, etc.)
1360
+ 3. Unclear/error - couldn't determine, needs investigation
1361
+
1362
+ Args:
1363
+ ticket_title: Title of the ticket being executed
1364
+ ticket_description: Description of the ticket
1365
+ executor_stdout: The stdout output from the executor
1366
+ planner_config: Planner configuration for LLM settings
1367
+
1368
+ Returns:
1369
+ NoChangesAnalysis with categorized result
1370
+ """
1371
+ import logging
1372
+
1373
+ from app.services.llm_service import LLMService
1374
+
1375
+ logger = logging.getLogger(__name__)
1376
+
1377
+ # Truncate executor output to avoid token limits
1378
+ max_output_chars = 8000
1379
+ truncated_stdout = executor_stdout[:max_output_chars]
1380
+ if len(executor_stdout) > max_output_chars:
1381
+ truncated_stdout += "\n... (output truncated)"
1382
+
1383
+ system_prompt = """You are a technical analyst reviewing why a coding agent completed a task without making any code changes.
1384
+
1385
+ Analyze the executor output and categorize the result into ONE of these categories:
1386
+
1387
+ 1. NO_CHANGES_NEEDED - The task genuinely doesn't require code changes. Examples:
1388
+ - The requested functionality already exists
1389
+ - The code is already correct as-is
1390
+ - The task was a review/analysis that doesn't need modifications
1391
+
1392
+ 2. MANUAL_WORK_REQUIRED - The task requires human intervention that the agent cannot do. Examples:
1393
+ - Configuration changes in external systems
1394
+ - Running commands that require special permissions
1395
+ - Setting up environment variables or secrets
1396
+ - Installing system packages
1397
+ - Deploying or running external services
1398
+ - Manual testing or verification steps
1399
+
1400
+ 3. NEEDS_INVESTIGATION - Unable to determine clearly, needs human review
1401
+
1402
+ Your response MUST be valid JSON with this exact structure:
1403
+ {
1404
+ "category": "NO_CHANGES_NEEDED" | "MANUAL_WORK_REQUIRED" | "NEEDS_INVESTIGATION",
1405
+ "reason": "Brief explanation of why no code changes were made",
1406
+ "manual_work_description": "If MANUAL_WORK_REQUIRED, describe exactly what needs to be done manually. Otherwise null."
1407
+ }"""
1408
+
1409
+ user_prompt = f"""A coding agent was asked to work on this ticket but produced no code changes.
1410
+
1411
+ TICKET TITLE: {ticket_title}
1412
+
1413
+ TICKET DESCRIPTION:
1414
+ {ticket_description or "(no description)"}
1415
+
1416
+ EXECUTOR OUTPUT:
1417
+ {truncated_stdout}
1418
+
1419
+ Analyze why no code changes were produced and categorize the result."""
1420
+
1421
+ try:
1422
+ llm_service = LLMService(planner_config)
1423
+ response = llm_service.call_completion(
1424
+ messages=[{"role": "user", "content": user_prompt}],
1425
+ max_tokens=500,
1426
+ system_prompt=system_prompt,
1427
+ timeout=30,
1428
+ )
1429
+ data = llm_service.safe_parse_json(response.content, {})
1430
+
1431
+ category = data.get("category", "NEEDS_INVESTIGATION")
1432
+ reason = data.get("reason", "Unable to determine why no changes were produced")
1433
+ manual_work_desc = data.get("manual_work_description")
1434
+
1435
+ return NoChangesAnalysis(
1436
+ reason=reason,
1437
+ needs_code_changes=(category == "NEEDS_INVESTIGATION"),
1438
+ requires_manual_work=(category == "MANUAL_WORK_REQUIRED"),
1439
+ manual_work_description=manual_work_desc
1440
+ if category == "MANUAL_WORK_REQUIRED"
1441
+ else None,
1442
+ )
1443
+
1444
+ except Exception as e:
1445
+ logger.error(f"Failed to analyze no-changes reason: {e}")
1446
+ # Fallback: treat as needs investigation
1447
+ return NoChangesAnalysis(
1448
+ reason=f"Analysis failed: {str(e)}",
1449
+ needs_code_changes=True,
1450
+ requires_manual_work=False,
1451
+ manual_work_description=None,
1452
+ )
1453
+
1454
+
1455
+ def create_manual_work_followup_sync(
1456
+ parent_ticket_id: str,
1457
+ parent_ticket_title: str,
1458
+ manual_work_description: str,
1459
+ goal_id: str,
1460
+ board_id: str | None = None,
1461
+ ) -> str | None:
1462
+ """
1463
+ Create a follow-up ticket for manual work that the agent cannot perform.
1464
+
1465
+ The ticket is created in PROPOSED state with a [Manual Work] prefix.
1466
+
1467
+ Args:
1468
+ parent_ticket_id: ID of the blocked ticket
1469
+ parent_ticket_title: Title of the blocked ticket
1470
+ manual_work_description: Description of the manual work needed
1471
+ goal_id: Goal ID to link the follow-up ticket to
1472
+ board_id: Optional board ID for permission scoping
1473
+
1474
+ Returns:
1475
+ The ID of the created follow-up ticket, or None if creation failed
1476
+ """
1477
+ import logging
1478
+
1479
+ logger = logging.getLogger(__name__)
1480
+
1481
+ try:
1482
+ with get_sync_db() as db:
1483
+ # Get the parent ticket for priority inheritance
1484
+ parent_ticket = (
1485
+ db.query(Ticket).filter(Ticket.id == parent_ticket_id).first()
1486
+ )
1487
+ priority = parent_ticket.priority if parent_ticket else None
1488
+
1489
+ # Create follow-up ticket with [Manual Work] prefix
1490
+ followup_title = f"[Manual Work] {parent_ticket_title}"
1491
+ # Truncate if too long (max 255 chars)
1492
+ if len(followup_title) > 255:
1493
+ followup_title = followup_title[:252] + "..."
1494
+
1495
+ followup_description = f"""This ticket requires manual human intervention that the automated agent cannot perform.
1496
+
1497
+ **Original Ticket:** {parent_ticket_title}
1498
+
1499
+ **Manual Work Required:**
1500
+ {manual_work_description}
1501
+
1502
+ **Instructions:**
1503
+ 1. Review the manual work description above
1504
+ 2. Perform the required actions manually
1505
+ 3. Mark this ticket as done when complete
1506
+ """
1507
+
1508
+ followup_ticket = Ticket(
1509
+ goal_id=goal_id,
1510
+ board_id=board_id,
1511
+ title=followup_title,
1512
+ description=followup_description,
1513
+ state=TicketState.PROPOSED.value,
1514
+ priority=priority,
1515
+ )
1516
+ db.add(followup_ticket)
1517
+ db.flush()
1518
+ followup_id = followup_ticket.id
1519
+
1520
+ # Create creation event for follow-up ticket
1521
+ creation_event = TicketEvent(
1522
+ ticket_id=followup_id,
1523
+ event_type=EventType.CREATED.value,
1524
+ from_state=None,
1525
+ to_state=TicketState.PROPOSED.value,
1526
+ actor_type=ActorType.EXECUTOR.value,
1527
+ actor_id="execute_worker",
1528
+ reason=f"Manual work follow-up for blocked ticket: {parent_ticket_title}",
1529
+ payload_json=json.dumps(
1530
+ {
1531
+ "parent_ticket_id": parent_ticket_id,
1532
+ "manual_work": True,
1533
+ "auto_generated": True,
1534
+ }
1535
+ ),
1536
+ )
1537
+ db.add(creation_event)
1538
+
1539
+ # Create link event on the parent ticket
1540
+ link_event = TicketEvent(
1541
+ ticket_id=parent_ticket_id,
1542
+ event_type=EventType.COMMENT.value,
1543
+ from_state=TicketState.BLOCKED.value,
1544
+ to_state=TicketState.BLOCKED.value,
1545
+ actor_type=ActorType.EXECUTOR.value,
1546
+ actor_id="execute_worker",
1547
+ reason=f"Created manual work follow-up ticket: {followup_title}",
1548
+ payload_json=json.dumps(
1549
+ {
1550
+ "followup_ticket_id": followup_id,
1551
+ "manual_work_followup": True,
1552
+ }
1553
+ ),
1554
+ )
1555
+ db.add(link_event)
1556
+
1557
+ db.commit()
1558
+
1559
+ logger.info(
1560
+ f"Created manual work follow-up ticket {followup_id} for blocked ticket {parent_ticket_id}"
1561
+ )
1562
+ return followup_id
1563
+
1564
+ except Exception as e:
1565
+ logger.error(f"Failed to create manual work follow-up ticket: {e}")
1566
+ return None
1567
+
1568
+
1569
+ def _get_related_tickets_context_sync(ticket_id: str) -> dict | None:
1570
+ """
1571
+ Get context about related tickets for better prompt building.
1572
+
1573
+ Returns dict with:
1574
+ - dependencies: list of tickets this ticket depends on
1575
+ - completed_tickets: list of DONE tickets in the same goal
1576
+ - goal_title: title of the goal this ticket belongs to
1577
+ """
1578
+ from sqlalchemy.orm import Session, selectinload
1579
+
1580
+ from app.database_sync import sync_engine
1581
+ from app.models.ticket import Ticket
1582
+ from app.state_machine import TicketState
1583
+
1584
+ # Use the shared sync_engine instead of creating a new one each time
1585
+ # This prevents connection pool exhaustion
1586
+ with Session(sync_engine) as db:
1587
+ # Get the current ticket with its goal and dependencies
1588
+ ticket = (
1589
+ db.query(Ticket)
1590
+ .options(selectinload(Ticket.blocked_by), selectinload(Ticket.goal))
1591
+ .filter(Ticket.id == ticket_id)
1592
+ .first()
1593
+ )
1594
+
1595
+ if not ticket or not ticket.goal_id:
1596
+ return None
1597
+
1598
+ context = {
1599
+ "goal_title": ticket.goal.title if ticket.goal else None,
1600
+ "dependencies": [],
1601
+ "completed_tickets": [],
1602
+ }
1603
+
1604
+ # Add dependency information
1605
+ if ticket.blocked_by:
1606
+ context["dependencies"].append(
1607
+ {"title": ticket.blocked_by.title, "state": ticket.blocked_by.state}
1608
+ )
1609
+
1610
+ # Get completed tickets in the same goal (for context)
1611
+ completed_tickets = (
1612
+ db.query(Ticket)
1613
+ .filter(
1614
+ Ticket.goal_id == ticket.goal_id,
1615
+ Ticket.state == TicketState.DONE.value,
1616
+ Ticket.id != ticket_id,
1617
+ )
1618
+ .order_by(Ticket.created_at.asc())
1619
+ .limit(5)
1620
+ .all()
1621
+ )
1622
+
1623
+ for comp_ticket in completed_tickets:
1624
+ context["completed_tickets"].append(
1625
+ {"title": comp_ticket.title, "description": comp_ticket.description}
1626
+ )
1627
+
1628
+ return (
1629
+ context
1630
+ if (context["dependencies"] or context["completed_tickets"])
1631
+ else None
1632
+ )
1633
+
1634
+
1635
+ def execute_ticket_task(job_id: str) -> dict:
1636
+ """
1637
+ Execute task for a ticket using Claude Code CLI (headless) or Cursor CLI (interactive).
1638
+
1639
+ Execution Modes:
1640
+ - Claude CLI (headless): Runs automatically, transitions based on result.
1641
+ - Cursor CLI (interactive): Prepares workspace + prompt, then hands off to user.
1642
+
1643
+ State Transitions:
1644
+ - Headless success with diff → verifying
1645
+ - Headless success with NO diff → blocked (reason: no changes produced)
1646
+ - Headless failure → blocked
1647
+ - Interactive (Cursor) → needs_human immediately
1648
+
1649
+ YOLO Mode:
1650
+ If yolo_mode is enabled in config AND the repo is in the allowlist,
1651
+ Claude CLI runs with --dangerously-skip-permissions. Otherwise it runs
1652
+ in permissioned mode (may require user approval for certain operations).
1653
+ """
1654
+ # Enable real-time log streaming for this job
1655
+ set_current_job(job_id)
1656
+
1657
+ try:
1658
+ return _execute_ticket_task_impl(job_id)
1659
+ except Exception as e:
1660
+ # Catch-all: if _execute_ticket_task_impl crashes with an unhandled
1661
+ # exception, properly fail the job and block the ticket instead of
1662
+ # leaving them in a zombie RUNNING/EXECUTING state.
1663
+ import logging
1664
+
1665
+ logger = logging.getLogger(__name__)
1666
+ logger.error(
1667
+ f"execute_ticket_task crashed for job {job_id}: {e}",
1668
+ exc_info=True,
1669
+ )
1670
+ try:
1671
+ update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
1672
+ except Exception:
1673
+ pass
1674
+ try:
1675
+ # Try to find the ticket_id from the job to transition it
1676
+ result = get_job_with_ticket(job_id)
1677
+ if result:
1678
+ _, ticket = result
1679
+ transition_ticket_sync(
1680
+ ticket.id,
1681
+ TicketState.BLOCKED,
1682
+ reason=f"Execution crashed: {e}",
1683
+ actor_id="execute_worker",
1684
+ )
1685
+ except Exception:
1686
+ pass
1687
+ return {
1688
+ "job_id": job_id,
1689
+ "status": "failed",
1690
+ "error": f"Unexpected error: {e}",
1691
+ }
1692
+ finally:
1693
+ # Signal streaming finished and clean up
1694
+ stream_finished(job_id)
1695
+ set_current_job(None)
1696
+
1697
+
1698
+ def _execute_ticket_task_impl(job_id: str) -> dict:
1699
+ """Implementation of execute_ticket_task (separated for streaming wrapper)."""
1700
+ # Get job and ticket info
1701
+ result = get_job_with_ticket(job_id)
1702
+ if not result:
1703
+ return {
1704
+ "job_id": job_id,
1705
+ "status": "failed",
1706
+ "error": "Job or ticket not found",
1707
+ }
1708
+
1709
+ job, ticket = result
1710
+ goal_id = ticket.goal_id
1711
+ ticket_id = ticket.id
1712
+
1713
+ # Check if ticket is already in a terminal/blocked state - skip execution if so
1714
+ # This prevents re-execution of jobs for already-blocked tickets
1715
+ if ticket.state in [
1716
+ TicketState.BLOCKED.value,
1717
+ TicketState.DONE.value,
1718
+ TicketState.ABANDONED.value,
1719
+ ]:
1720
+ import logging
1721
+
1722
+ logging.getLogger(__name__).info(
1723
+ f"Skipping execution for job {job_id}: ticket {ticket_id} is already in {ticket.state} state"
1724
+ )
1725
+ update_job_finished(job_id, JobStatus.FAILED, exit_code=0)
1726
+ return {
1727
+ "job_id": job_id,
1728
+ "status": "skipped",
1729
+ "reason": f"Ticket already in {ticket.state} state",
1730
+ "ticket_id": ticket_id,
1731
+ }
1732
+
1733
+ # Ensure workspace exists
1734
+ worktree_path, workspace_error = ensure_workspace_for_ticket(ticket_id, goal_id)
1735
+
1736
+ # Get log path (use worktree if available, fallback otherwise)
1737
+ log_path, log_path_relative = get_log_path_for_job(job_id, worktree_path)
1738
+
1739
+ write_log(log_path, "Starting execute task...")
1740
+
1741
+ # Workspace is required for execution
1742
+ if workspace_error or not worktree_path:
1743
+ write_log(
1744
+ log_path,
1745
+ f"ERROR: Could not create workspace: {workspace_error or 'Unknown error'}",
1746
+ )
1747
+ write_log(log_path, "Execution requires a valid git worktree. Failing job.")
1748
+ update_job_started(job_id, log_path_relative)
1749
+ update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
1750
+ transition_ticket_sync(
1751
+ ticket_id,
1752
+ TicketState.BLOCKED,
1753
+ reason=f"Execution failed: workspace creation error - {workspace_error}",
1754
+ actor_id="execute_worker",
1755
+ )
1756
+ return {"job_id": job_id, "status": "failed", "error": workspace_error}
1757
+
1758
+ write_log(log_path, f"Workspace ready at: {worktree_path}")
1759
+
1760
+ # Mark as running
1761
+ if not update_job_started(job_id, log_path_relative):
1762
+ write_log(log_path, "Job was canceled or not found, aborting.")
1763
+ return {"job_id": job_id, "status": "canceled"}
1764
+
1765
+ # Check for cancellation BEFORE transitioning to EXECUTING state
1766
+ # This prevents a race where the ticket transitions to EXECUTING but the job
1767
+ # is immediately cancelled, leaving the ticket stuck in EXECUTING with no runner
1768
+ if check_canceled(job_id):
1769
+ write_log(log_path, "Job canceled before execution started, aborting.")
1770
+ return {"job_id": job_id, "status": "canceled"}
1771
+
1772
+ # Transition ticket to EXECUTING state BEFORE any execution work begins
1773
+ # This is critical - the ticket MUST be in EXECUTING state while running
1774
+ # This handles transitions from PLANNED, DONE (changes requested), or NEEDS_HUMAN
1775
+ from app.state_machine import validate_transition
1776
+
1777
+ with get_sync_db() as db:
1778
+ current_ticket = db.query(Ticket).filter(Ticket.id == ticket_id).first()
1779
+ if not current_ticket:
1780
+ write_log(log_path, "ERROR: Ticket not found in database")
1781
+ update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
1782
+ return {"job_id": job_id, "status": "failed", "error": "Ticket not found"}
1783
+
1784
+ current_state = TicketState(current_ticket.state)
1785
+ write_log(log_path, f"Current ticket state: '{current_state.value}'")
1786
+
1787
+ if current_state == TicketState.EXECUTING:
1788
+ write_log(log_path, "Ticket already in 'executing' state")
1789
+ elif validate_transition(current_state, TicketState.EXECUTING):
1790
+ write_log(
1791
+ log_path,
1792
+ f"Transitioning ticket from '{current_state.value}' to 'executing'",
1793
+ )
1794
+ # Important: Do transition INSIDE the same db context to ensure atomicity
1795
+ current_ticket.state = TicketState.EXECUTING.value
1796
+
1797
+ # Create transition event
1798
+ event = TicketEvent(
1799
+ ticket_id=ticket_id,
1800
+ event_type=EventType.TRANSITIONED.value,
1801
+ from_state=current_state.value,
1802
+ to_state=TicketState.EXECUTING.value,
1803
+ actor_type=ActorType.EXECUTOR.value,
1804
+ actor_id="execute_worker",
1805
+ reason=f"Execution started (job {job_id})",
1806
+ payload_json=json.dumps({"job_id": job_id}),
1807
+ )
1808
+ db.add(event)
1809
+ db.commit()
1810
+ write_log(log_path, "Successfully transitioned to 'executing' state")
1811
+ else:
1812
+ # Invalid state transition — abort rather than bypass state machine
1813
+ write_log(
1814
+ log_path,
1815
+ f"ABORTING: Cannot transition from '{current_state.value}' to 'executing' (invalid transition)",
1816
+ )
1817
+ with get_sync_db() as db:
1818
+ job = db.query(Job).filter(Job.id == job_id).first()
1819
+ if job:
1820
+ job.status = JobStatus.FAILED.value
1821
+ job.finished_at = datetime.now()
1822
+ db.commit()
1823
+ return
1824
+
1825
+ # Load configuration from DB (board config is the single source of truth)
1826
+ board_config = None
1827
+ board_repo_root = None
1828
+ if ticket.board_id:
1829
+ with get_sync_db() as db:
1830
+ from app.models.board import Board
1831
+
1832
+ board = db.query(Board).filter(Board.id == ticket.board_id).first()
1833
+ if board:
1834
+ if board.config:
1835
+ board_config = board.config
1836
+ if board.repo_root:
1837
+ board_repo_root = board.repo_root
1838
+
1839
+ config = DraftConfig.from_board_config(board_config)
1840
+ execute_config = config.execute_config
1841
+ planner_config = config.planner_config
1842
+
1843
+ # Use board's repo_root as the authoritative main repo path.
1844
+ # Falls back to GIT_REPO_PATH env var / default only when no board repo_root.
1845
+ if board_repo_root:
1846
+ main_repo_path = Path(board_repo_root)
1847
+ else:
1848
+ main_repo_path = WorkspaceService.get_repo_path()
1849
+
1850
+ # =========================================================================
1851
+ # WORKTREE SAFETY VALIDATION (enforced, not assumed)
1852
+ # =========================================================================
1853
+ write_log(log_path, "Validating worktree safety...")
1854
+ worktree_validator = WorktreeValidator(main_repo_path)
1855
+ validation_result = worktree_validator.validate(worktree_path)
1856
+
1857
+ if not validation_result.valid:
1858
+ write_log(log_path, f"SAFETY CHECK FAILED: {validation_result.error}")
1859
+ write_log(log_path, f"Reason: {validation_result.message}")
1860
+ write_log(log_path, "Refusing to execute in unsafe location.")
1861
+ update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
1862
+ transition_ticket_sync(
1863
+ ticket_id,
1864
+ TicketState.BLOCKED,
1865
+ reason=f"Safety check failed: {validation_result.message}",
1866
+ payload={
1867
+ "validation_error": validation_result.error,
1868
+ "worktree_path": str(worktree_path),
1869
+ "main_repo_path": str(main_repo_path),
1870
+ },
1871
+ actor_id="execute_worker",
1872
+ )
1873
+ return {
1874
+ "job_id": job_id,
1875
+ "status": "failed",
1876
+ "error": f"Safety check failed: {validation_result.message}",
1877
+ "validation_error": validation_result.error,
1878
+ }
1879
+
1880
+ write_log(log_path, f"Worktree validated: branch={validation_result.branch}")
1881
+
1882
+ # =========================================================================
1883
+ # YOLO MODE CHECK (refuse if enabled but allowlist empty)
1884
+ # =========================================================================
1885
+ yolo_status = execute_config.check_yolo_status(
1886
+ str(worktree_path.resolve()),
1887
+ repo_root=str(main_repo_path),
1888
+ )
1889
+ model_info = (
1890
+ f", model={execute_config.executor_model}"
1891
+ if execute_config.executor_model
1892
+ else ""
1893
+ )
1894
+ write_log(
1895
+ log_path,
1896
+ f"Execute config: timeout={execute_config.timeout}s, preferred_executor={execute_config.preferred_executor}{model_info}",
1897
+ )
1898
+
1899
+ if yolo_status == YoloStatus.REFUSED:
1900
+ refusal_reason = execute_config.get_yolo_refusal_reason(
1901
+ repo_root=str(main_repo_path)
1902
+ )
1903
+ write_log(log_path, f"YOLO MODE REFUSED: {refusal_reason}")
1904
+ write_log(log_path, "Transitioning to needs_human for manual approval.")
1905
+ update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
1906
+ transition_ticket_sync(
1907
+ ticket_id,
1908
+ TicketState.NEEDS_HUMAN,
1909
+ reason=f"YOLO mode refused: {refusal_reason}",
1910
+ payload={
1911
+ "yolo_refused": True,
1912
+ "refusal_reason": refusal_reason,
1913
+ "worktree": str(worktree_path),
1914
+ },
1915
+ actor_id="execute_worker",
1916
+ )
1917
+ return {
1918
+ "job_id": job_id,
1919
+ "status": "yolo_refused",
1920
+ "worktree": str(worktree_path),
1921
+ "reason": refusal_reason,
1922
+ }
1923
+
1924
+ yolo_enabled = yolo_status == YoloStatus.ALLOWED
1925
+ write_log(log_path, f"YOLO mode: {yolo_status.value}")
1926
+
1927
+ # Detect available executor CLI
1928
+ try:
1929
+ executor_info = ExecutorService.detect_executor(
1930
+ preferred=execute_config.preferred_executor,
1931
+ agent_path=planner_config.agent_path,
1932
+ )
1933
+ write_log(
1934
+ log_path,
1935
+ f"Found executor: {executor_info.executor_type.value} ({executor_info.mode.value}) at {executor_info.path}",
1936
+ )
1937
+ except ExecutorNotFoundError as e:
1938
+ write_log(log_path, f"ERROR: {e.message}")
1939
+ write_log(
1940
+ log_path,
1941
+ "No code executor CLI found. Please install Claude Code CLI or Cursor CLI.",
1942
+ )
1943
+ update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
1944
+ transition_ticket_sync(
1945
+ ticket_id,
1946
+ TicketState.BLOCKED,
1947
+ reason=f"Execution failed: {e.message}",
1948
+ actor_id="execute_worker",
1949
+ )
1950
+ return {"job_id": job_id, "status": "failed", "error": e.message}
1951
+
1952
+ # Check for feedback from previous revision (if this is a re-run after changes requested)
1953
+ feedback_bundle = get_feedback_bundle_for_ticket(ticket_id)
1954
+ if feedback_bundle:
1955
+ write_log(
1956
+ log_path,
1957
+ f"Found feedback from revision #{feedback_bundle.get('revision_number', '?')}",
1958
+ )
1959
+ write_log(
1960
+ log_path, f" - Summary: {feedback_bundle.get('summary', '')[:100]}..."
1961
+ )
1962
+ write_log(
1963
+ log_path,
1964
+ f" - Comments to address: {len(feedback_bundle.get('comments', []))}",
1965
+ )
1966
+ else:
1967
+ write_log(log_path, "No previous revision feedback found (fresh execution)")
1968
+
1969
+ # Check for queued follow-up prompt (from instant follow-up queue)
1970
+ from app.services.queued_message_service import queued_message_service
1971
+
1972
+ followup_prompt = queued_message_service.get_followup_prompt(ticket_id)
1973
+ additional_context = None
1974
+ if followup_prompt:
1975
+ write_log(log_path, f"Found queued follow-up: {followup_prompt[:100]}...")
1976
+ additional_context = f"\n\n--- FOLLOW-UP REQUEST ---\n{followup_prompt}\n\nPlease address the above follow-up request while continuing work on this ticket."
1977
+
1978
+ # Get related tickets context for better prompt
1979
+ related_tickets_context = _get_related_tickets_context_sync(ticket_id)
1980
+ if related_tickets_context:
1981
+ write_log(
1982
+ log_path,
1983
+ f"Found context: {len(related_tickets_context.get('completed_tickets', []))} completed tickets, {len(related_tickets_context.get('dependencies', []))} dependencies",
1984
+ )
1985
+
1986
+ # Build prompt bundle
1987
+ write_log(log_path, "Building prompt bundle...")
1988
+ prompt_builder = PromptBundleBuilder(
1989
+ worktree_path, job_id, repo_root=main_repo_path
1990
+ )
1991
+ verify_commands = (
1992
+ config.verify_config.commands if config.verify_config.commands else None
1993
+ )
1994
+ prompt_file = prompt_builder.build_prompt(
1995
+ ticket_title=ticket.title,
1996
+ ticket_description=ticket.description,
1997
+ feedback_bundle=feedback_bundle,
1998
+ additional_context=additional_context,
1999
+ related_tickets_context=related_tickets_context,
2000
+ verify_commands=verify_commands,
2001
+ )
2002
+ write_log(log_path, f"Prompt bundle created at: {prompt_file}")
2003
+
2004
+ # Get evidence directory
2005
+ evidence_dir = prompt_builder.get_evidence_dir()
2006
+ evidence_records: list[str] = []
2007
+
2008
+ # Check for cancellation before execution
2009
+ if check_canceled(job_id):
2010
+ write_log(log_path, "Job canceled, stopping execution.")
2011
+ return {"job_id": job_id, "status": "canceled"}
2012
+
2013
+ # =========================================================================
2014
+ # INTERACTIVE EXECUTOR (Cursor) - Hand off to user immediately
2015
+ # =========================================================================
2016
+ if executor_info.is_interactive():
2017
+ write_log(
2018
+ log_path, f"Executor {executor_info.executor_type.value} is INTERACTIVE."
2019
+ )
2020
+ write_log(log_path, "Workspace and prompt bundle are ready.")
2021
+ write_log(log_path, "Transitioning to 'needs_human' for manual completion.")
2022
+ write_log(log_path, "")
2023
+ write_log(log_path, "=== INSTRUCTIONS FOR HUMAN ===")
2024
+ write_log(log_path, f"1. Open the worktree in your editor: {worktree_path}")
2025
+ write_log(log_path, f"2. Read the prompt: {prompt_file}")
2026
+ write_log(log_path, "3. Implement the requested changes")
2027
+ write_log(log_path, "4. Commit your changes")
2028
+ write_log(log_path, "5. Mark the ticket as ready for verification")
2029
+ write_log(log_path, "==============================")
2030
+
2031
+ update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
2032
+ transition_ticket_sync(
2033
+ ticket_id,
2034
+ TicketState.NEEDS_HUMAN,
2035
+ reason=f"Interactive executor ({executor_info.executor_type.value}): workspace ready, awaiting human completion",
2036
+ payload={
2037
+ "executor": executor_info.executor_type.value,
2038
+ "mode": executor_info.mode.value,
2039
+ "worktree": str(worktree_path),
2040
+ "prompt_file": str(prompt_file),
2041
+ },
2042
+ actor_id="execute_worker",
2043
+ )
2044
+ return {
2045
+ "job_id": job_id,
2046
+ "status": "needs_human",
2047
+ "worktree": str(worktree_path),
2048
+ "executor": executor_info.executor_type.value,
2049
+ "mode": executor_info.mode.value,
2050
+ "prompt_file": str(prompt_file),
2051
+ }
2052
+
2053
+ # =========================================================================
2054
+ # HEADLESS EXECUTOR (Claude) - Run automatically
2055
+ # =========================================================================
2056
+ write_log(
2057
+ log_path, f"Running headless executor: {executor_info.executor_type.value}..."
2058
+ )
2059
+ executor_evidence_id = str(uuid.uuid4())
2060
+
2061
+ # Check for existing session to continue (session continuity)
2062
+ from app.services.agent_session_service import get_session_service
2063
+
2064
+ session_service = get_session_service(worktree_path)
2065
+ existing_session = session_service.get_session(ticket_id)
2066
+ session_flag = None
2067
+ if existing_session:
2068
+ session_flag = session_service.get_continue_flag(
2069
+ ticket_id, executor_info.executor_type.value
2070
+ )
2071
+ if session_flag:
2072
+ write_log(
2073
+ log_path,
2074
+ f"Continuing from session: {existing_session.session_id} (execution #{existing_session.execution_count + 1})",
2075
+ )
2076
+
2077
+ # Get the command with YOLO mode and model selection
2078
+ # Returns (command, stdin_content) tuple - prompt piped via stdin to avoid ARG_MAX
2079
+ executor_command, executor_stdin = executor_info.get_apply_command(
2080
+ prompt_file,
2081
+ worktree_path,
2082
+ yolo_mode=yolo_enabled,
2083
+ model=execute_config.executor_model,
2084
+ )
2085
+
2086
+ # Add session continuation flag if available
2087
+ if session_flag:
2088
+ executor_command = executor_command + session_flag.split()
2089
+
2090
+ # Log command (without full prompt content)
2091
+ if yolo_enabled:
2092
+ write_log(
2093
+ log_path,
2094
+ f"Command: {executor_command[0]} --print --dangerously-skip-permissions <prompt>",
2095
+ )
2096
+ else:
2097
+ write_log(log_path, f"Command: {executor_command[0]} --print <prompt>")
2098
+ write_log(
2099
+ log_path,
2100
+ "NOTE: Running in permissioned mode. Some operations may require approval.",
2101
+ )
2102
+
2103
+ # Track execution timing for metadata
2104
+ executor_start_time = time.time()
2105
+
2106
+ # Enable log normalization for cursor-agent (outputs JSON streaming format)
2107
+ should_normalize = executor_info.executor_type == ExecutorType.CURSOR_AGENT
2108
+
2109
+ # Use retry wrapper for improved reliability (handles transient failures)
2110
+ executor_exit_code, executor_stdout_path, executor_stderr_path = (
2111
+ run_executor_cli_with_retry(
2112
+ command=executor_command,
2113
+ cwd=worktree_path,
2114
+ evidence_dir=evidence_dir,
2115
+ evidence_id=executor_evidence_id,
2116
+ repo_root=main_repo_path,
2117
+ timeout=execute_config.timeout,
2118
+ job_id=job_id, # Enable real-time streaming
2119
+ normalize_logs=should_normalize, # Parse cursor-agent JSON for nice display
2120
+ stdin_content=executor_stdin, # Pipe prompt via stdin (ARG_MAX safety)
2121
+ log_path=log_path, # Enable retry logging
2122
+ )
2123
+ )
2124
+
2125
+ # Calculate execution duration
2126
+ executor_duration_ms = int((time.time() - executor_start_time) * 1000)
2127
+
2128
+ # Create EXECUTOR_META evidence with structured metadata
2129
+ executor_meta_id = str(uuid.uuid4())
2130
+ executor_meta = {
2131
+ "exit_code": executor_exit_code,
2132
+ "duration_ms": executor_duration_ms,
2133
+ "executor_type": executor_info.executor_type.value,
2134
+ "mode": executor_info.mode.value,
2135
+ "command": f"{executor_command[0]} --print {'--dangerously-skip-permissions ' if yolo_enabled else ''}<prompt>",
2136
+ "yolo_enabled": yolo_enabled,
2137
+ "timeout_configured": execute_config.timeout,
2138
+ }
2139
+ executor_meta_path = evidence_dir / f"{executor_meta_id}.meta.json"
2140
+ executor_meta_path.write_text(json.dumps(executor_meta, indent=2))
2141
+ # Store relative path for secure DB storage
2142
+ executor_meta_relpath = str(executor_meta_path)
2143
+ create_evidence_record(
2144
+ ticket_id=ticket_id,
2145
+ job_id=job_id,
2146
+ command="executor_metadata",
2147
+ exit_code=executor_exit_code,
2148
+ stdout_path=executor_meta_relpath,
2149
+ stderr_path="",
2150
+ evidence_id=executor_meta_id,
2151
+ kind=EvidenceKind.EXECUTOR_META,
2152
+ )
2153
+ evidence_records.append(executor_meta_id)
2154
+
2155
+ # Create evidence record for executor output (typed)
2156
+ create_evidence_record(
2157
+ ticket_id=ticket_id,
2158
+ job_id=job_id,
2159
+ command=f"{executor_command[0]} --print {'--dangerously-skip-permissions ' if yolo_enabled else ''}<prompt>",
2160
+ exit_code=executor_exit_code,
2161
+ stdout_path=executor_stdout_path,
2162
+ stderr_path=executor_stderr_path,
2163
+ evidence_id=executor_evidence_id,
2164
+ kind=EvidenceKind.EXECUTOR_STDOUT,
2165
+ )
2166
+ evidence_records.append(executor_evidence_id)
2167
+
2168
+ # Extract and save session ID for continuity
2169
+ try:
2170
+ stdout_content = (main_repo_path / executor_stdout_path).read_text()
2171
+ new_session_id = session_service.extract_session_id_from_output(stdout_content)
2172
+ if new_session_id:
2173
+ session_service.save_session(
2174
+ session_id=new_session_id,
2175
+ ticket_id=ticket_id,
2176
+ agent_type=executor_info.executor_type.value,
2177
+ )
2178
+ write_log(
2179
+ log_path,
2180
+ f"Saved session ID for future continuity: {new_session_id[:16]}...",
2181
+ )
2182
+ except Exception as e:
2183
+ logger.debug(f"Could not extract session ID: {e}")
2184
+
2185
+ write_log(log_path, f"Executor completed in {executor_duration_ms}ms")
2186
+ if executor_exit_code == 0:
2187
+ write_log(log_path, "Executor CLI completed successfully (exit code: 0)")
2188
+ else:
2189
+ write_log(log_path, f"Executor CLI FAILED (exit code: {executor_exit_code})")
2190
+ # Read stderr for more details
2191
+ try:
2192
+ stderr_content = Path(executor_stderr_path).read_text()[:500]
2193
+ if stderr_content:
2194
+ write_log(log_path, f"Executor stderr: {stderr_content}")
2195
+ except Exception:
2196
+ pass
2197
+
2198
+ # Capture git diff regardless of exit code (to see what changes were made)
2199
+ write_log(log_path, "Capturing git diff...")
2200
+ diff_stat_evidence_id = str(uuid.uuid4())
2201
+ diff_patch_evidence_id = str(uuid.uuid4())
2202
+
2203
+ diff_exit_code, diff_stat_path, diff_patch_path, diff_stat, has_changes = (
2204
+ capture_git_diff(
2205
+ cwd=worktree_path,
2206
+ evidence_dir=evidence_dir,
2207
+ evidence_id=diff_stat_evidence_id, # Used for both files with different extensions
2208
+ repo_root=main_repo_path,
2209
+ )
2210
+ )
2211
+
2212
+ # Create typed evidence records for git diff
2213
+ create_evidence_record(
2214
+ ticket_id=ticket_id,
2215
+ job_id=job_id,
2216
+ command="git diff --stat",
2217
+ exit_code=diff_exit_code,
2218
+ stdout_path=diff_stat_path,
2219
+ stderr_path="", # stderr captured in patch record
2220
+ evidence_id=diff_stat_evidence_id,
2221
+ kind=EvidenceKind.GIT_DIFF_STAT,
2222
+ )
2223
+ evidence_records.append(diff_stat_evidence_id)
2224
+
2225
+ create_evidence_record(
2226
+ ticket_id=ticket_id,
2227
+ job_id=job_id,
2228
+ command="git diff",
2229
+ exit_code=diff_exit_code,
2230
+ stdout_path=diff_patch_path,
2231
+ stderr_path="",
2232
+ evidence_id=diff_patch_evidence_id,
2233
+ kind=EvidenceKind.GIT_DIFF_PATCH,
2234
+ )
2235
+ evidence_records.append(diff_patch_evidence_id)
2236
+
2237
+ write_log(log_path, f"Git diff summary:\n{diff_stat}")
2238
+ write_log(log_path, f"Has changes: {has_changes}")
2239
+
2240
+ # Check for cancellation before state transition
2241
+ if check_canceled(job_id):
2242
+ write_log(log_path, "Job canceled, stopping execution.")
2243
+ return {"job_id": job_id, "status": "canceled"}
2244
+
2245
+ # =========================================================================
2246
+ # STATE TRANSITIONS
2247
+ # =========================================================================
2248
+
2249
+ # Case 1: Executor failed
2250
+ if executor_exit_code != 0:
2251
+ write_log(log_path, f"Execution FAILED with exit code {executor_exit_code}")
2252
+ write_log(log_path, "Transitioning ticket to 'blocked'")
2253
+ transition_ticket_sync(
2254
+ ticket_id,
2255
+ TicketState.BLOCKED,
2256
+ reason=f"Execution failed: {executor_info.executor_type.value} CLI exited with code {executor_exit_code}",
2257
+ payload={
2258
+ "executor": executor_info.executor_type.value,
2259
+ "exit_code": executor_exit_code,
2260
+ "evidence_ids": evidence_records,
2261
+ "diff_summary": diff_stat,
2262
+ "yolo_mode": yolo_enabled,
2263
+ },
2264
+ actor_id="execute_worker",
2265
+ )
2266
+ update_job_finished(job_id, JobStatus.FAILED, exit_code=executor_exit_code)
2267
+ return {
2268
+ "job_id": job_id,
2269
+ "status": "failed",
2270
+ "worktree": str(worktree_path),
2271
+ "executor": executor_info.executor_type.value,
2272
+ "exit_code": executor_exit_code,
2273
+ "evidence_ids": evidence_records,
2274
+ "diff_summary": diff_stat,
2275
+ }
2276
+
2277
+ # Case 2: Executor succeeded but NO CHANGES produced
2278
+ if not has_changes:
2279
+ write_log(log_path, "Execution completed but NO CHANGES were produced.")
2280
+ write_log(log_path, "Analyzing why no code changes were made...")
2281
+
2282
+ # Read executor stdout for analysis
2283
+ try:
2284
+ executor_stdout_content = (
2285
+ main_repo_path / executor_stdout_path
2286
+ ).read_text()
2287
+ except Exception as e:
2288
+ write_log(log_path, f"Warning: Could not read executor output: {e}")
2289
+ executor_stdout_content = ""
2290
+
2291
+ # Analyze why no changes were produced using LLM
2292
+ analysis = analyze_no_changes_reason(
2293
+ ticket_title=ticket.title,
2294
+ ticket_description=ticket.description,
2295
+ executor_stdout=executor_stdout_content,
2296
+ planner_config=planner_config,
2297
+ )
2298
+
2299
+ write_log(log_path, f"Analysis result: {analysis.reason}")
2300
+ write_log(log_path, f" - Needs code changes: {analysis.needs_code_changes}")
2301
+ write_log(
2302
+ log_path, f" - Requires manual work: {analysis.requires_manual_work}"
2303
+ )
2304
+
2305
+ followup_ticket_id = None
2306
+
2307
+ # Handle based on analysis result
2308
+ if analysis.requires_manual_work and analysis.manual_work_description:
2309
+ # Create a [Manual Work] follow-up ticket
2310
+ write_log(log_path, "Creating [Manual Work] follow-up ticket...")
2311
+ followup_ticket_id = create_manual_work_followup_sync(
2312
+ parent_ticket_id=ticket_id,
2313
+ parent_ticket_title=ticket.title,
2314
+ manual_work_description=analysis.manual_work_description,
2315
+ goal_id=goal_id,
2316
+ board_id=ticket.board_id,
2317
+ )
2318
+ if followup_ticket_id:
2319
+ write_log(log_path, f"Created follow-up ticket: {followup_ticket_id}")
2320
+ else:
2321
+ write_log(log_path, "Warning: Failed to create follow-up ticket")
2322
+
2323
+ # Block the original ticket with reference to manual work
2324
+ reason = f"Requires manual work: {analysis.reason}"
2325
+ payload = {
2326
+ "executor": executor_info.executor_type.value,
2327
+ "evidence_ids": evidence_records,
2328
+ "diff_summary": diff_stat,
2329
+ "no_changes": True,
2330
+ "yolo_mode": yolo_enabled,
2331
+ "requires_manual_work": True,
2332
+ "manual_work_followup_id": followup_ticket_id,
2333
+ "analysis_reason": analysis.reason,
2334
+ }
2335
+
2336
+ elif not analysis.needs_code_changes:
2337
+ # No changes needed - mark as blocked with skip_followup flag
2338
+ write_log(log_path, "No code changes needed. Blocking without follow-up.")
2339
+ reason = f"No changes required: {analysis.reason}"
2340
+ payload = {
2341
+ "executor": executor_info.executor_type.value,
2342
+ "evidence_ids": evidence_records,
2343
+ "diff_summary": diff_stat,
2344
+ "no_changes": True,
2345
+ "yolo_mode": yolo_enabled,
2346
+ "no_changes_needed": True,
2347
+ "skip_followup": True, # Signal to planner to not create follow-ups
2348
+ "analysis_reason": analysis.reason,
2349
+ }
2350
+
2351
+ else:
2352
+ # Needs investigation - use original behavior (planner may create follow-up)
2353
+ write_log(log_path, "Needs investigation. Blocking for review.")
2354
+ reason = f"Execution completed but no code changes were produced: {analysis.reason}"
2355
+ payload = {
2356
+ "executor": executor_info.executor_type.value,
2357
+ "evidence_ids": evidence_records,
2358
+ "diff_summary": diff_stat,
2359
+ "no_changes": True,
2360
+ "yolo_mode": yolo_enabled,
2361
+ "needs_investigation": True,
2362
+ "analysis_reason": analysis.reason,
2363
+ }
2364
+
2365
+ write_log(
2366
+ log_path, f"Transitioning ticket to 'blocked' (reason: {reason[:100]}...)"
2367
+ )
2368
+ transition_ticket_sync(
2369
+ ticket_id,
2370
+ TicketState.BLOCKED,
2371
+ reason=reason,
2372
+ payload=payload,
2373
+ actor_id="execute_worker",
2374
+ )
2375
+ update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
2376
+
2377
+ result_payload = {
2378
+ "job_id": job_id,
2379
+ "status": "no_changes",
2380
+ "worktree": str(worktree_path),
2381
+ "executor": executor_info.executor_type.value,
2382
+ "evidence_ids": evidence_records,
2383
+ "diff_summary": diff_stat,
2384
+ "analysis": {
2385
+ "reason": analysis.reason,
2386
+ "needs_code_changes": analysis.needs_code_changes,
2387
+ "requires_manual_work": analysis.requires_manual_work,
2388
+ },
2389
+ }
2390
+ if followup_ticket_id:
2391
+ result_payload["manual_work_followup_id"] = followup_ticket_id
2392
+
2393
+ return result_payload
2394
+
2395
+ # Case 3: Executor succeeded with changes → verifying
2396
+ write_log(log_path, "Execution completed successfully with changes!")
2397
+
2398
+ # Create revision for this execution
2399
+ revision_id = create_revision_for_job(
2400
+ ticket_id=ticket_id,
2401
+ job_id=job_id,
2402
+ diff_stat_evidence_id=diff_stat_evidence_id,
2403
+ diff_patch_evidence_id=diff_patch_evidence_id,
2404
+ )
2405
+ if revision_id:
2406
+ write_log(log_path, f"Created revision {revision_id}")
2407
+ else:
2408
+ write_log(log_path, "WARNING: Failed to create revision record")
2409
+
2410
+ write_log(log_path, "Transitioning ticket to 'verifying'")
2411
+ transition_ticket_sync(
2412
+ ticket_id,
2413
+ TicketState.VERIFYING,
2414
+ reason=f"Execution completed by {executor_info.executor_type.value} CLI with changes",
2415
+ payload={
2416
+ "executor": executor_info.executor_type.value,
2417
+ "evidence_ids": evidence_records,
2418
+ "diff_summary": diff_stat,
2419
+ "has_changes": True,
2420
+ "yolo_mode": yolo_enabled,
2421
+ "revision_id": revision_id,
2422
+ },
2423
+ actor_id="execute_worker",
2424
+ )
2425
+ update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
2426
+ return {
2427
+ "job_id": job_id,
2428
+ "status": "succeeded",
2429
+ "worktree": str(worktree_path),
2430
+ "executor": executor_info.executor_type.value,
2431
+ "evidence_ids": evidence_records,
2432
+ "diff_summary": diff_stat,
2433
+ "has_changes": True,
2434
+ "revision_id": revision_id,
2435
+ }
2436
+
2437
+
2438
+ def verify_ticket_task(job_id: str) -> dict:
2439
+ """Verify task wrapper for Celery."""
2440
+ try:
2441
+ return _verify_ticket_task_impl(job_id)
2442
+ except Exception as e:
2443
+ import logging
2444
+
2445
+ logging.getLogger(__name__).error(
2446
+ f"verify_ticket_task crashed for job {job_id}: {e}",
2447
+ exc_info=True,
2448
+ )
2449
+ try:
2450
+ update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
2451
+ except Exception:
2452
+ pass
2453
+ try:
2454
+ result = get_job_with_ticket(job_id)
2455
+ if result:
2456
+ _, ticket = result
2457
+ transition_ticket_sync(
2458
+ ticket.id,
2459
+ TicketState.BLOCKED,
2460
+ reason=f"Verification crashed: {e}",
2461
+ actor_id="verify_worker",
2462
+ )
2463
+ except Exception:
2464
+ pass
2465
+ return {"job_id": job_id, "status": "failed", "error": str(e)}
2466
+
2467
+
2468
+ def _verify_ticket_task_impl(job_id: str) -> dict:
2469
+ """
2470
+ Verify task for a ticket.
2471
+
2472
+ This task:
2473
+ 1. Ensures a worktree exists for the ticket
2474
+ 2. Loads verification commands from draft.yaml
2475
+ 3. Runs each command in the isolated worktree directory
2476
+ 4. Creates Evidence records with captured stdout/stderr
2477
+ 5. Transitions ticket based on verification outcome
2478
+ """
2479
+ # Get job and ticket info
2480
+ result = get_job_with_ticket(job_id)
2481
+ if not result:
2482
+ return {
2483
+ "job_id": job_id,
2484
+ "status": "failed",
2485
+ "error": "Job or ticket not found",
2486
+ }
2487
+
2488
+ job, ticket = result
2489
+ goal_id = ticket.goal_id
2490
+ ticket_id = ticket.id
2491
+
2492
+ # Check if ticket is already in a terminal/blocked state - skip verification if so
2493
+ if ticket.state in [
2494
+ TicketState.BLOCKED.value,
2495
+ TicketState.DONE.value,
2496
+ TicketState.ABANDONED.value,
2497
+ ]:
2498
+ import logging
2499
+
2500
+ logging.getLogger(__name__).info(
2501
+ f"Skipping verification for job {job_id}: ticket {ticket_id} is already in {ticket.state} state"
2502
+ )
2503
+ update_job_finished(job_id, JobStatus.FAILED, exit_code=0)
2504
+ return {
2505
+ "job_id": job_id,
2506
+ "status": "skipped",
2507
+ "reason": f"Ticket already in {ticket.state} state",
2508
+ "ticket_id": ticket_id,
2509
+ }
2510
+
2511
+ # Ensure workspace exists
2512
+ worktree_path, workspace_error = ensure_workspace_for_ticket(ticket_id, goal_id)
2513
+
2514
+ # Get log path (use worktree if available, fallback otherwise)
2515
+ log_path, log_path_relative = get_log_path_for_job(job_id, worktree_path)
2516
+
2517
+ write_log(log_path, "Starting verify task...")
2518
+
2519
+ if workspace_error:
2520
+ write_log(log_path, f"WARNING: Could not create workspace: {workspace_error}")
2521
+ write_log(log_path, "Continuing with fallback execution...")
2522
+ else:
2523
+ write_log(log_path, f"Workspace ready at: {worktree_path}")
2524
+
2525
+ # Mark as running
2526
+ if not update_job_started(job_id, log_path_relative):
2527
+ write_log(log_path, "Job was canceled or not found, aborting.")
2528
+ return {"job_id": job_id, "status": "canceled"}
2529
+
2530
+ # Check for cancellation
2531
+ if check_canceled(job_id):
2532
+ write_log(log_path, "Job canceled, stopping execution.")
2533
+ return {"job_id": job_id, "status": "canceled"}
2534
+
2535
+ # Load configuration from DB (board config is the single source of truth)
2536
+ board_config = None
2537
+ if ticket.board_id:
2538
+ with get_sync_db() as db:
2539
+ from app.models.board import Board
2540
+
2541
+ board = db.query(Board).filter(Board.id == ticket.board_id).first()
2542
+ if board and board.config:
2543
+ board_config = board.config
2544
+
2545
+ config = DraftConfig.from_board_config(board_config)
2546
+ verify_config = config.verify_config
2547
+ verify_commands = verify_config.commands
2548
+
2549
+ # Get repo root for relative path computation
2550
+ repo_root = WorkspaceService.get_repo_path()
2551
+
2552
+ write_log(log_path, f"Loaded {len(verify_commands)} verification command(s)")
2553
+ write_log(
2554
+ log_path,
2555
+ "On success: transition to 'needs_human' (requires user approval to move to done)",
2556
+ )
2557
+
2558
+ if not verify_commands:
2559
+ write_log(
2560
+ log_path, "No verification commands configured, skipping verification."
2561
+ )
2562
+ # No commands = success, always transition to needs_human for review
2563
+ write_log(log_path, "Transitioning ticket to 'needs_human' for review")
2564
+ transition_ticket_sync(
2565
+ ticket_id,
2566
+ TicketState.NEEDS_HUMAN,
2567
+ reason="Verification passed (no commands configured), awaiting human approval",
2568
+ )
2569
+ update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
2570
+ return {
2571
+ "job_id": job_id,
2572
+ "status": "succeeded",
2573
+ "worktree": str(worktree_path) if worktree_path else None,
2574
+ }
2575
+
2576
+ # Get evidence directory (with validation against repo_root)
2577
+ evidence_dir = get_evidence_dir(worktree_path, job_id, repo_root=repo_root)
2578
+
2579
+ # Run verification commands with timing
2580
+ verify_start_time = time.time()
2581
+ all_succeeded = True
2582
+ failed_commands: list[dict] = []
2583
+ command_results: list[dict] = []
2584
+ evidence_records: list[str] = []
2585
+
2586
+ for i, command in enumerate(verify_commands):
2587
+ # Check for cancellation before each command
2588
+ if check_canceled(job_id):
2589
+ write_log(log_path, "Job canceled, stopping execution.")
2590
+ return {"job_id": job_id, "status": "canceled"}
2591
+
2592
+ write_log(
2593
+ log_path, f"Running command {i + 1}/{len(verify_commands)}: {command}"
2594
+ )
2595
+
2596
+ # Generate evidence ID
2597
+ evidence_id = str(uuid.uuid4())
2598
+ cmd_start_time = time.time()
2599
+
2600
+ # Run the command (returns relative paths for secure DB storage)
2601
+ exit_code, stdout_path, stderr_path = run_verification_command(
2602
+ command=command,
2603
+ cwd=worktree_path,
2604
+ evidence_dir=evidence_dir,
2605
+ evidence_id=evidence_id,
2606
+ repo_root=repo_root,
2607
+ timeout=300,
2608
+ extra_allowed_commands=verify_config.extra_allowed_commands,
2609
+ )
2610
+
2611
+ cmd_duration_ms = int((time.time() - cmd_start_time) * 1000)
2612
+
2613
+ # Track command result for metadata
2614
+ command_results.append(
2615
+ {
2616
+ "command": command,
2617
+ "exit_code": exit_code,
2618
+ "duration_ms": cmd_duration_ms,
2619
+ "evidence_id": evidence_id,
2620
+ }
2621
+ )
2622
+
2623
+ # Create evidence record (typed as verification output)
2624
+ create_evidence_record(
2625
+ ticket_id=ticket_id,
2626
+ job_id=job_id,
2627
+ command=command,
2628
+ exit_code=exit_code,
2629
+ stdout_path=stdout_path,
2630
+ stderr_path=stderr_path,
2631
+ evidence_id=evidence_id,
2632
+ kind=EvidenceKind.VERIFY_STDOUT,
2633
+ )
2634
+ evidence_records.append(evidence_id)
2635
+
2636
+ if exit_code == 0:
2637
+ write_log(
2638
+ log_path, f"Command succeeded (exit code: 0, {cmd_duration_ms}ms)"
2639
+ )
2640
+ else:
2641
+ write_log(
2642
+ log_path,
2643
+ f"Command FAILED (exit code: {exit_code}, {cmd_duration_ms}ms)",
2644
+ )
2645
+ all_succeeded = False
2646
+ failed_commands.append(
2647
+ {
2648
+ "command": command,
2649
+ "exit_code": exit_code,
2650
+ "evidence_id": evidence_id,
2651
+ }
2652
+ )
2653
+ # Stop on first failure
2654
+ write_log(log_path, "Stopping verification due to failure.")
2655
+ break
2656
+
2657
+ # Calculate total verification duration
2658
+ verify_duration_ms = int((time.time() - verify_start_time) * 1000)
2659
+
2660
+ # Create VERIFY_META evidence with structured metadata
2661
+ verify_meta_id = str(uuid.uuid4())
2662
+ verify_meta = {
2663
+ "total_duration_ms": verify_duration_ms,
2664
+ "commands_configured": verify_commands,
2665
+ "commands_run": len(command_results),
2666
+ "all_succeeded": all_succeeded,
2667
+ "results": command_results,
2668
+ }
2669
+ verify_meta_path = evidence_dir / f"{verify_meta_id}.meta.json"
2670
+ verify_meta_path.write_text(json.dumps(verify_meta, indent=2))
2671
+ # Store relative path for secure DB storage
2672
+ verify_meta_relpath = str(verify_meta_path)
2673
+ create_evidence_record(
2674
+ ticket_id=ticket_id,
2675
+ job_id=job_id,
2676
+ command="verify_metadata",
2677
+ exit_code=0 if all_succeeded else 1,
2678
+ stdout_path=verify_meta_relpath,
2679
+ stderr_path="",
2680
+ evidence_id=verify_meta_id,
2681
+ kind=EvidenceKind.VERIFY_META,
2682
+ )
2683
+ evidence_records.append(verify_meta_id)
2684
+
2685
+ write_log(log_path, f"Total verification time: {verify_duration_ms}ms")
2686
+
2687
+ # Transition ticket based on outcome
2688
+ if all_succeeded:
2689
+ write_log(log_path, "All verification commands passed!")
2690
+
2691
+ # Check if autonomy mode allows auto-approval (skip NEEDS_HUMAN)
2692
+ auto_approved = False
2693
+ try:
2694
+ from app.services.autonomy_service import AutonomyService
2695
+
2696
+ with get_sync_db() as autonomy_db:
2697
+ ticket_for_check = (
2698
+ autonomy_db.query(Ticket).filter(Ticket.id == ticket_id).first()
2699
+ )
2700
+ if ticket_for_check:
2701
+ autonomy_svc = AutonomyService()
2702
+ check = autonomy_svc.can_auto_approve_revision_sync(
2703
+ autonomy_db, ticket_for_check
2704
+ )
2705
+ if check.approved:
2706
+ write_log(
2707
+ log_path,
2708
+ f"Autonomy: auto-approving revision ({check.reason})",
2709
+ )
2710
+ # Transition directly to DONE, skipping NEEDS_HUMAN
2711
+ transition_ticket_sync(
2712
+ ticket_id,
2713
+ TicketState.DONE,
2714
+ reason=f"Auto-approved: verification passed ({len(verify_commands)} command(s)), {check.reason}",
2715
+ payload={
2716
+ "evidence_ids": evidence_records,
2717
+ "duration_ms": verify_duration_ms,
2718
+ "auto_approved": True,
2719
+ },
2720
+ actor_id="autonomy_service",
2721
+ )
2722
+ # Record audit event
2723
+ autonomy_svc.record_auto_action_sync(
2724
+ autonomy_db,
2725
+ ticket_for_check,
2726
+ action_type="approve_revision",
2727
+ details={
2728
+ "reason": check.reason,
2729
+ "evidence_ids": evidence_records,
2730
+ },
2731
+ from_state=TicketState.VERIFYING.value,
2732
+ to_state=TicketState.DONE.value,
2733
+ )
2734
+ autonomy_db.commit()
2735
+ auto_approved = True
2736
+ else:
2737
+ write_log(
2738
+ log_path, f"Autonomy: not auto-approving ({check.reason})"
2739
+ )
2740
+ except Exception as e:
2741
+ write_log(
2742
+ log_path, f"Autonomy check failed (falling back to NEEDS_HUMAN): {e}"
2743
+ )
2744
+ logger.warning(f"Autonomy check failed for ticket {ticket_id}: {e}")
2745
+
2746
+ if not auto_approved:
2747
+ # Default flow: transition to needs_human for review
2748
+ write_log(log_path, "Transitioning ticket to 'needs_human' for review")
2749
+ transition_ticket_sync(
2750
+ ticket_id,
2751
+ TicketState.NEEDS_HUMAN,
2752
+ reason=f"Verification passed: {len(verify_commands)} command(s) succeeded, awaiting human approval",
2753
+ payload={
2754
+ "evidence_ids": evidence_records,
2755
+ "duration_ms": verify_duration_ms,
2756
+ },
2757
+ )
2758
+
2759
+ update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
2760
+ return {
2761
+ "job_id": job_id,
2762
+ "status": "succeeded",
2763
+ "worktree": str(worktree_path) if worktree_path else None,
2764
+ "evidence_ids": evidence_records,
2765
+ "auto_approved": auto_approved,
2766
+ }
2767
+ else:
2768
+ # Verification failed
2769
+ failure_summary = "; ".join(
2770
+ [
2771
+ f"'{fc['command']}' failed with exit code {fc['exit_code']}"
2772
+ for fc in failed_commands
2773
+ ]
2774
+ )
2775
+ write_log(log_path, f"Verification FAILED: {failure_summary}")
2776
+ write_log(log_path, "Transitioning ticket to 'blocked'")
2777
+
2778
+ transition_ticket_sync(
2779
+ ticket_id,
2780
+ TicketState.BLOCKED,
2781
+ reason=f"Verification failed: {failure_summary}",
2782
+ payload={
2783
+ "evidence_ids": evidence_records,
2784
+ "failed_commands": failed_commands,
2785
+ },
2786
+ )
2787
+
2788
+ # Use the exit code of the first failed command
2789
+ final_exit_code = failed_commands[0]["exit_code"] if failed_commands else 1
2790
+ update_job_finished(job_id, JobStatus.FAILED, exit_code=final_exit_code)
2791
+
2792
+ return {
2793
+ "job_id": job_id,
2794
+ "status": "failed",
2795
+ "worktree": str(worktree_path) if worktree_path else None,
2796
+ "evidence_ids": evidence_records,
2797
+ "failed_commands": failed_commands,
2798
+ }
2799
+
2800
+
2801
+ def job_watchdog_task() -> dict:
2802
+ """
2803
+ Periodic task to monitor and recover stuck jobs.
2804
+
2805
+ This task runs every minute via Celery Beat and checks for:
2806
+ 1. RUNNING jobs with stale heartbeat (no update in 2 minutes)
2807
+ 2. RUNNING jobs that exceeded their timeout
2808
+ 3. QUEUED jobs stuck in queue for over 10 minutes
2809
+
2810
+ For each stuck job, it marks the job as FAILED and transitions
2811
+ the associated ticket to BLOCKED.
2812
+ """
2813
+ from app.services.job_watchdog_service import run_job_watchdog
2814
+
2815
+ result = run_job_watchdog()
2816
+
2817
+ return {
2818
+ "stale_jobs_recovered": result.stale_jobs_recovered,
2819
+ "timed_out_jobs_recovered": result.timed_out_jobs_recovered,
2820
+ "stuck_queued_jobs_failed": result.stuck_queued_jobs_failed,
2821
+ "tickets_blocked": result.tickets_blocked,
2822
+ "details": result.details,
2823
+ }
2824
+
2825
+
2826
+ def planner_tick_task() -> dict:
2827
+ """
2828
+ Periodic task to run a planner tick and pick up next tickets.
2829
+
2830
+ This task runs every 5 seconds via Celery Beat and:
2831
+ 1. Checks if any PLANNED tickets are ready to execute
2832
+ 2. If no ticket is currently EXECUTING/VERIFYING, queues the next one
2833
+ 3. Handles follow-ups for BLOCKED tickets (if LLM configured)
2834
+
2835
+ This ensures tickets automatically flow through the queue without
2836
+ requiring the /planner/start HTTP request to stay connected.
2837
+ """
2838
+ from app.services.planner_tick_sync import PlannerLockError, run_planner_tick_sync
2839
+
2840
+ try:
2841
+ result = run_planner_tick_sync()
2842
+ return {
2843
+ "status": "success",
2844
+ "executed": result.get("executed", 0),
2845
+ "followups_created": result.get("followups_created", 0),
2846
+ "reflections_added": result.get("reflections_added", 0),
2847
+ }
2848
+ except PlannerLockError:
2849
+ # Lock conflict - another tick is running, this is fine
2850
+ return {"status": "skipped", "reason": "Another tick in progress"}
2851
+ except Exception as e:
2852
+ import logging
2853
+
2854
+ logging.getLogger(__name__).error(f"Planner tick failed: {e}")
2855
+ return {"status": "error", "error": str(e)[:200]}
2856
+
2857
+
2858
+ def resume_ticket_task(job_id: str) -> dict:
2859
+ """Resume task wrapper for Celery."""
2860
+ return _resume_ticket_task_impl(job_id)
2861
+
2862
+
2863
+ def _resume_ticket_task_impl(job_id: str) -> dict:
2864
+ """
2865
+ Resume a ticket after human completion (interactive executor flow).
2866
+
2867
+ This task is used when a ticket was transitioned to 'needs_human' by an
2868
+ interactive executor (like Cursor). The human has made their changes, and
2869
+ now wants to continue the workflow.
2870
+
2871
+ This task:
2872
+ 1. Validates the ticket is in 'needs_human' state
2873
+ 2. Captures the git diff as evidence
2874
+ 3. Checks if there are any changes
2875
+ 4. Transitions to 'verifying' if changes exist, or 'blocked' if no changes
2876
+ """
2877
+ # Get job and ticket info
2878
+ result = get_job_with_ticket(job_id)
2879
+ if not result:
2880
+ return {
2881
+ "job_id": job_id,
2882
+ "status": "failed",
2883
+ "error": "Job or ticket not found",
2884
+ }
2885
+
2886
+ job, ticket = result
2887
+ goal_id = ticket.goal_id
2888
+ ticket_id = ticket.id
2889
+
2890
+ # Ensure workspace exists
2891
+ worktree_path, workspace_error = ensure_workspace_for_ticket(ticket_id, goal_id)
2892
+
2893
+ # Get log path (use worktree if available, fallback otherwise)
2894
+ log_path, log_path_relative = get_log_path_for_job(job_id, worktree_path)
2895
+
2896
+ write_log(log_path, "Starting resume task...")
2897
+
2898
+ # Workspace is required
2899
+ if workspace_error or not worktree_path:
2900
+ write_log(
2901
+ log_path,
2902
+ f"ERROR: Could not find workspace: {workspace_error or 'Unknown error'}",
2903
+ )
2904
+ update_job_started(job_id, log_path_relative)
2905
+ update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
2906
+ return {"job_id": job_id, "status": "failed", "error": workspace_error}
2907
+
2908
+ write_log(log_path, f"Workspace found at: {worktree_path}")
2909
+
2910
+ # Mark as running
2911
+ if not update_job_started(job_id, log_path_relative):
2912
+ write_log(log_path, "Job was canceled or not found, aborting.")
2913
+ return {"job_id": job_id, "status": "canceled"}
2914
+
2915
+ # Validate ticket is in needs_human state
2916
+ if ticket.state != TicketState.NEEDS_HUMAN.value:
2917
+ write_log(
2918
+ log_path,
2919
+ f"ERROR: Ticket is in '{ticket.state}' state, expected 'needs_human'",
2920
+ )
2921
+ write_log(
2922
+ log_path, "Resume can only be called on tickets in 'needs_human' state."
2923
+ )
2924
+ update_job_finished(job_id, JobStatus.FAILED, exit_code=1)
2925
+ return {
2926
+ "job_id": job_id,
2927
+ "status": "failed",
2928
+ "error": f"Ticket must be in 'needs_human' state to resume, got '{ticket.state}'",
2929
+ }
2930
+
2931
+ # Get repo root for evidence storage and relative path computation
2932
+ repo_root = WorkspaceService.get_repo_path()
2933
+
2934
+ # Get evidence directory (central location)
2935
+ evidence_dir = central_evidence_dir(job_id)
2936
+ evidence_records: list[str] = []
2937
+
2938
+ # Capture git diff
2939
+ write_log(log_path, "Capturing git diff...")
2940
+ diff_stat_evidence_id = str(uuid.uuid4())
2941
+ diff_patch_evidence_id = str(uuid.uuid4())
2942
+
2943
+ diff_exit_code, diff_stat_path, diff_patch_path, diff_stat, has_changes = (
2944
+ capture_git_diff(
2945
+ cwd=worktree_path,
2946
+ evidence_dir=evidence_dir,
2947
+ evidence_id=diff_stat_evidence_id,
2948
+ repo_root=repo_root,
2949
+ )
2950
+ )
2951
+
2952
+ # Create typed evidence records for git diff
2953
+ create_evidence_record(
2954
+ ticket_id=ticket_id,
2955
+ job_id=job_id,
2956
+ command="git diff --stat",
2957
+ exit_code=diff_exit_code,
2958
+ stdout_path=diff_stat_path,
2959
+ stderr_path="",
2960
+ evidence_id=diff_stat_evidence_id,
2961
+ kind=EvidenceKind.GIT_DIFF_STAT,
2962
+ )
2963
+ evidence_records.append(diff_stat_evidence_id)
2964
+
2965
+ create_evidence_record(
2966
+ ticket_id=ticket_id,
2967
+ job_id=job_id,
2968
+ command="git diff",
2969
+ exit_code=diff_exit_code,
2970
+ stdout_path=diff_patch_path,
2971
+ stderr_path="",
2972
+ evidence_id=diff_patch_evidence_id,
2973
+ kind=EvidenceKind.GIT_DIFF_PATCH,
2974
+ )
2975
+ evidence_records.append(diff_patch_evidence_id)
2976
+
2977
+ write_log(log_path, f"Git diff summary:\n{diff_stat}")
2978
+ write_log(log_path, f"Has changes: {has_changes}")
2979
+
2980
+ # Determine outcome
2981
+ if not has_changes:
2982
+ write_log(log_path, "No changes detected in worktree.")
2983
+ write_log(log_path, "Transitioning to 'blocked' (reason: no changes)")
2984
+ transition_ticket_sync(
2985
+ ticket_id,
2986
+ TicketState.BLOCKED,
2987
+ reason="Resume completed but no code changes were found in worktree",
2988
+ payload={
2989
+ "evidence_ids": evidence_records,
2990
+ "diff_summary": diff_stat,
2991
+ "no_changes": True,
2992
+ },
2993
+ actor_id="resume_worker",
2994
+ )
2995
+ update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
2996
+ return {
2997
+ "job_id": job_id,
2998
+ "status": "no_changes",
2999
+ "worktree": str(worktree_path),
3000
+ "evidence_ids": evidence_records,
3001
+ }
3002
+
3003
+ # Changes exist - transition to verifying
3004
+ write_log(log_path, "Changes detected! Transitioning to 'verifying'")
3005
+ transition_ticket_sync(
3006
+ ticket_id,
3007
+ TicketState.VERIFYING,
3008
+ reason="Human completed changes, ready for verification",
3009
+ payload={
3010
+ "evidence_ids": evidence_records,
3011
+ "diff_summary": diff_stat,
3012
+ "resumed_from_interactive": True,
3013
+ },
3014
+ actor_id="resume_worker",
3015
+ )
3016
+ update_job_finished(job_id, JobStatus.SUCCEEDED, exit_code=0)
3017
+
3018
+ write_log(log_path, "Resume completed successfully!")
3019
+ return {
3020
+ "job_id": job_id,
3021
+ "status": "succeeded",
3022
+ "worktree": str(worktree_path),
3023
+ "evidence_ids": evidence_records,
3024
+ "diff_summary": diff_stat,
3025
+ }
3026
+
3027
+
3028
+ def poll_pr_statuses():
3029
+ """
3030
+ Periodic task to poll GitHub PR statuses for tickets.
3031
+
3032
+ This task runs every 5 minutes and:
3033
+ 1. Finds tickets with open PRs
3034
+ 2. Checks PR status on GitHub
3035
+ 3. Auto-transitions tickets if PR is merged
3036
+ """
3037
+ import subprocess
3038
+ from datetime import datetime
3039
+ from pathlib import Path
3040
+
3041
+ from app.models.workspace import Workspace
3042
+ from app.services.git_host import get_git_host_provider
3043
+ from app.state_machine import TicketState
3044
+
3045
+ # First, collect ticket info without holding DB connection during network calls
3046
+ tickets_to_check = []
3047
+
3048
+ with get_sync_db() as db:
3049
+ # Find tickets with open PRs
3050
+ tickets_with_prs = (
3051
+ db.query(Ticket)
3052
+ .filter(
3053
+ Ticket.pr_number.isnot(None),
3054
+ Ticket.pr_state.in_(["OPEN", "CLOSED"]),
3055
+ )
3056
+ .all()
3057
+ )
3058
+
3059
+ if not tickets_with_prs:
3060
+ return {"message": "No PRs to poll", "checked": 0}
3061
+
3062
+ for ticket in tickets_with_prs:
3063
+ # Get workspace for repo path
3064
+ workspace = (
3065
+ db.query(Workspace).filter(Workspace.ticket_id == ticket.id).first()
3066
+ )
3067
+
3068
+ if workspace and workspace.worktree_path:
3069
+ repo_path = Path(workspace.worktree_path)
3070
+ if repo_path.exists():
3071
+ tickets_to_check.append(
3072
+ {
3073
+ "ticket_id": ticket.id,
3074
+ "pr_number": ticket.pr_number,
3075
+ "pr_state": ticket.pr_state,
3076
+ "pr_merged_at": ticket.pr_merged_at,
3077
+ "repo_path": str(repo_path),
3078
+ }
3079
+ )
3080
+
3081
+ if not tickets_to_check:
3082
+ return {"message": "No valid tickets to poll", "checked": 0}
3083
+
3084
+ git_host = get_git_host_provider()
3085
+
3086
+ # Check if git host CLI is available
3087
+ if not git_host.is_available():
3088
+ return {
3089
+ "message": f"{git_host.name} CLI not available, skipping poll",
3090
+ "checked": 0,
3091
+ }
3092
+
3093
+ updated_count = 0
3094
+ merged_count = 0
3095
+ errors = []
3096
+
3097
+ # Poll GitHub outside of DB session to avoid blocking
3098
+ for ticket_info in tickets_to_check:
3099
+ try:
3100
+ repo_path = Path(ticket_info["repo_path"])
3101
+ pr_number = ticket_info["pr_number"]
3102
+
3103
+ # Use synchronous subprocess call with timeout instead of asyncio.run()
3104
+ result = subprocess.run(
3105
+ ["gh", "pr", "view", str(pr_number), "--json", "state,merged"],
3106
+ cwd=repo_path,
3107
+ capture_output=True,
3108
+ text=True,
3109
+ timeout=30, # 30 second timeout per PR check
3110
+ )
3111
+
3112
+ if result.returncode != 0:
3113
+ continue
3114
+
3115
+ pr_details = json.loads(result.stdout)
3116
+ ticket_info["new_state"] = pr_details.get("state", "OPEN")
3117
+ ticket_info["merged"] = pr_details.get("merged", False)
3118
+
3119
+ except subprocess.TimeoutExpired:
3120
+ errors.append(f"Timeout checking PR for ticket {ticket_info['ticket_id']}")
3121
+ continue
3122
+ except Exception as e:
3123
+ errors.append(
3124
+ f"Error polling PR for ticket {ticket_info['ticket_id']}: {e}"
3125
+ )
3126
+ continue
3127
+
3128
+ # Now update DB with results (quick operation)
3129
+ with get_sync_db() as db:
3130
+ for ticket_info in tickets_to_check:
3131
+ if "new_state" not in ticket_info:
3132
+ continue # Skip if we didn't get PR details
3133
+
3134
+ try:
3135
+ ticket = (
3136
+ db.query(Ticket)
3137
+ .filter(Ticket.id == ticket_info["ticket_id"])
3138
+ .first()
3139
+ )
3140
+ if not ticket:
3141
+ continue
3142
+
3143
+ old_state = ticket.pr_state
3144
+ ticket.pr_state = ticket_info["new_state"]
3145
+
3146
+ if ticket_info.get("merged") and not ticket.pr_merged_at:
3147
+ ticket.pr_merged_at = datetime.now()
3148
+ merged_count += 1
3149
+
3150
+ # Auto-transition ticket if PR was merged
3151
+ if ticket_info.get("merged") and old_state != "MERGED":
3152
+ from app.state_machine import validate_transition
3153
+
3154
+ ticket.pr_state = "MERGED"
3155
+
3156
+ if validate_transition(ticket.state, TicketState.DONE.value):
3157
+ ticket.state = TicketState.DONE.value
3158
+
3159
+ # Create event
3160
+ event = TicketEvent(
3161
+ ticket_id=ticket.id,
3162
+ event_type=EventType.TRANSITIONED.value,
3163
+ from_state=old_state or "REVIEW",
3164
+ to_state=TicketState.DONE.value,
3165
+ actor_type=ActorType.SYSTEM.value,
3166
+ actor_id="poll_pr_statuses",
3167
+ reason=f"PR #{ticket.pr_number} was merged",
3168
+ )
3169
+ db.add(event)
3170
+ else:
3171
+ import logging
3172
+
3173
+ logging.getLogger(__name__).warning(
3174
+ f"Cannot transition ticket {ticket.id} from "
3175
+ f"{ticket.state} to DONE on PR merge"
3176
+ )
3177
+
3178
+ updated_count += 1
3179
+
3180
+ except Exception as e:
3181
+ errors.append(f"Error updating ticket {ticket_info['ticket_id']}: {e}")
3182
+ continue
3183
+
3184
+ return {
3185
+ "message": "PR polling completed",
3186
+ "checked": len(tickets_to_check),
3187
+ "updated": updated_count,
3188
+ "merged": merged_count,
3189
+ "errors": errors[:10] if errors else [], # Limit error list
3190
+ }