@mamdouh-aboammar/agentic-workflow 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/.claude-plugin/plugin.json +10 -0
  2. package/.codex-plugin/plugin.json +13 -0
  3. package/.skills.json +19 -0
  4. package/AGENTS.md +1344 -0
  5. package/CLAUDE.md +178 -0
  6. package/GEMINI.md +102 -0
  7. package/LICENSE +21 -0
  8. package/README.md +350 -0
  9. package/SKILL.md +132 -0
  10. package/bin/agentic-hooks.sh +79 -0
  11. package/bin/cli.js +1060 -0
  12. package/core/__init__.py +52 -0
  13. package/core/ai_evaluator.py +117 -0
  14. package/core/autopilot_engine.py +368 -0
  15. package/core/clean_code_guard.py +188 -0
  16. package/core/engine_py/__init__.py +29 -0
  17. package/core/engine_py/agent_worker.py +136 -0
  18. package/core/engine_py/decider.py +150 -0
  19. package/core/engine_py/energy.py +45 -0
  20. package/core/engine_py/event_bus.py +63 -0
  21. package/core/engine_py/executor.py +186 -0
  22. package/core/engine_py/models.py +193 -0
  23. package/core/engine_py/queue.py +314 -0
  24. package/core/engine_py/runner.py +116 -0
  25. package/core/engine_py/system_workers.py +70 -0
  26. package/core/engine_py/toon_adapter.py +586 -0
  27. package/core/engine_py/verification_controller.py +208 -0
  28. package/core/engine_py/worker.py +167 -0
  29. package/core/engine_spec/event_schema.json +65 -0
  30. package/core/engine_spec/example_workflow.yaml +73 -0
  31. package/core/engine_spec/workflow_schema.json +127 -0
  32. package/core/hooks/__init__.py +29 -0
  33. package/core/hooks/adapters/__init__.py +25 -0
  34. package/core/hooks/adapters/claude_adapter.py +83 -0
  35. package/core/hooks/adapters/cli_agent_adapter.py +82 -0
  36. package/core/hooks/adapters/codex_adapter.py +78 -0
  37. package/core/hooks/adapters/cursor_adapter.py +73 -0
  38. package/core/hooks/adapters/gemini_adapter.py +93 -0
  39. package/core/hooks/adapters/homebrew_adapter.py +69 -0
  40. package/core/hooks/adapters/mcp_proxy.py +133 -0
  41. package/core/hooks/adapters/shell_adapter.py +65 -0
  42. package/core/hooks/dispatcher.py +118 -0
  43. package/core/hooks/policy_engine.py +375 -0
  44. package/core/hooks/session_end.py +141 -0
  45. package/core/hooks/types.py +147 -0
  46. package/core/integrations/__init__.py +28 -0
  47. package/core/integrations/installer.py +225 -0
  48. package/core/integrations/lifecycle_director.py +175 -0
  49. package/core/integrations/registry.py +105 -0
  50. package/core/multi_agent_system.py +164 -0
  51. package/core/skills_indexer.py +742 -0
  52. package/core/system/__init__.py +25 -0
  53. package/core/system/announcements.py +72 -0
  54. package/core/system/dependencies.py +69 -0
  55. package/core/system/doctor.py +171 -0
  56. package/core/system/health.py +144 -0
  57. package/core/system/installer.py +137 -0
  58. package/core/system/notifications.py +97 -0
  59. package/core/system/refresher.py +110 -0
  60. package/core/system/updater.py +167 -0
  61. package/core/system/version_tracker.py +65 -0
  62. package/docs/architecture_plan.md +7 -0
  63. package/docs/guides/failure-recovery.md +714 -0
  64. package/docs/implementation_summary.md +10 -0
  65. package/docs/protocols/autopilot-execution.md +148 -0
  66. package/docs/protocols/code-change-protocol.md +49 -0
  67. package/docs/protocols/context-preservation-detail.md +114 -0
  68. package/docs/protocols/quality-gates.md +110 -0
  69. package/docs/protocols/ulw-mode.md +60 -0
  70. package/docs/research_findings.md +10 -0
  71. package/docs/solutions/autonomous-autopilot-engine-architecture.md +38 -0
  72. package/install.sh +111 -0
  73. package/marketplace.json +37 -0
  74. package/package.json +81 -0
  75. package/skills/agentic-workflow/SKILL.md +132 -0
  76. package/skills/agentic-workflow/skill-spec.json +100 -0
  77. package/soul.md +445 -0
  78. package/src/engine_ts/decider.ts +186 -0
  79. package/src/engine_ts/event-bus.ts +57 -0
  80. package/src/engine_ts/executor.ts +262 -0
  81. package/src/engine_ts/index.ts +12 -0
  82. package/src/engine_ts/queue.ts +93 -0
  83. package/src/engine_ts/runner.ts +108 -0
  84. package/src/engine_ts/skills-indexer.ts +264 -0
  85. package/src/engine_ts/toon-adapter.ts +91 -0
  86. package/src/engine_ts/types.ts +134 -0
  87. package/src/engine_ts/verification-controller.ts +204 -0
  88. package/src/engine_ts/worker.ts +280 -0
  89. package/src/hooks/adapters/claude-adapter.ts +54 -0
  90. package/src/hooks/adapters/cli-agent-adapter.ts +46 -0
  91. package/src/hooks/adapters/codex-adapter.ts +69 -0
  92. package/src/hooks/adapters/cursor-adapter.ts +60 -0
  93. package/src/hooks/adapters/gemini-adapter.ts +71 -0
  94. package/src/hooks/adapters/homebrew-adapter.ts +36 -0
  95. package/src/hooks/adapters/mcp-proxy.ts +66 -0
  96. package/src/hooks/adapters/shell-adapter.ts +42 -0
  97. package/src/hooks/dispatcher.ts +113 -0
  98. package/src/hooks/index.ts +16 -0
  99. package/src/hooks/policy-engine.ts +376 -0
  100. package/src/hooks/session-end.ts +125 -0
  101. package/src/hooks/types.ts +61 -0
  102. package/src/index.d.ts +34 -0
  103. package/src/index.ts +23 -0
  104. package/src/integrations/index.ts +7 -0
  105. package/src/integrations/installer.ts +208 -0
  106. package/src/integrations/lifecycle-director.ts +139 -0
  107. package/src/integrations/registry.ts +82 -0
  108. package/src/system/announcements.ts +143 -0
  109. package/src/system/dependencies.ts +176 -0
  110. package/src/system/doctor.ts +374 -0
  111. package/src/system/health.ts +270 -0
  112. package/src/system/index.ts +14 -0
  113. package/src/system/installer.ts +262 -0
  114. package/src/system/notifications.ts +180 -0
  115. package/src/system/refresher.ts +207 -0
  116. package/src/system/types.ts +268 -0
  117. package/src/system/updater.ts +219 -0
  118. package/src/system/version-tracker.ts +137 -0
@@ -0,0 +1,186 @@
1
+ """
2
+ executor.py — Main Engine Orchestrator & Single-Writer SOT Coordinator.
3
+
4
+ Implements Absolute Criterion 2:
5
+ - Single-File SOT (state.yaml) updated exclusively by this Orchestrator
6
+ - Atomic write with filesystem lock to eliminate race conditions
7
+ - Decider evaluation loop driving tasks into the queue
8
+ - Event publishing for all lifecycle transitions
9
+ """
10
+
11
+ import os
12
+ import time
13
+ import uuid
14
+ import asyncio
15
+ from typing import Optional, Dict, Any
16
+
17
+ try:
18
+ import yaml
19
+ except ImportError:
20
+ yaml = None
21
+
22
+ import json
23
+
24
+ from core.engine_py.models import (
25
+ WorkflowDefinition, WorkflowInstance, WorkflowStatus, TaskStatus, EngineEvent
26
+ )
27
+ from core.engine_py.decider import AgenticDecider
28
+ from core.engine_py.queue import TaskQueue
29
+ from core.engine_py.event_bus import AsyncEventBus
30
+ from core.engine_py.energy import EnergyBudget
31
+
32
+
33
+ class AgenticExecutor:
34
+ """Orchestrates workflow lifecycle, state transitions, and atomic SOT persistence."""
35
+
36
+ def __init__(
37
+ self,
38
+ workflow_def: WorkflowDefinition,
39
+ queue: TaskQueue,
40
+ event_bus: AsyncEventBus,
41
+ project_dir: str = ".",
42
+ sot_filename: str = "state.yaml",
43
+ energy_budget: Optional[EnergyBudget] = None
44
+ ):
45
+ self.workflow_def = workflow_def
46
+ self.queue = queue
47
+ self.event_bus = event_bus
48
+ self.project_dir = os.path.abspath(project_dir)
49
+ self.sot_path = os.path.join(self.project_dir, sot_filename)
50
+ self.decider = AgenticDecider()
51
+ self.energy = energy_budget or EnergyBudget()
52
+
53
+ self.trace_id = f"auto_{int(time.time())}_{uuid.uuid4().hex[:6]}"
54
+ self.instance = WorkflowInstance(
55
+ workflow_id=f"wf_{uuid.uuid4().hex[:8]}",
56
+ workflow_def=workflow_def,
57
+ trace_id=self.trace_id,
58
+ variables=dict(workflow_def.input_parameters)
59
+ )
60
+ self._write_sot()
61
+
62
+ def _write_sot(self) -> None:
63
+ """Atomically persists Single Source of Truth state to state.yaml."""
64
+ current_stage = self.instance.get_current_stage()
65
+ sot_data = {
66
+ "workflow": {
67
+ "title": self.workflow_def.name,
68
+ "version": self.workflow_def.version,
69
+ "workflow_id": self.instance.workflow_id,
70
+ "trace_id": self.instance.trace_id,
71
+ "status": self.instance.status.value,
72
+ "current_stage": current_stage.id if current_stage else "COMPLETED",
73
+ "current_stage_name": current_stage.name if current_stage else "All Stages Finished",
74
+ "total_stages": len(self.workflow_def.stages),
75
+ "autopilot": {"enabled": self.instance.autopilot_enabled, "status": self.instance.status.value},
76
+ "energy_budget": {
77
+ "remaining_pct": round(self.energy.energy_percentage, 1),
78
+ "refuel_count": self.energy.refuel_count,
79
+ "consumed_tokens": self.energy.consumed_tokens
80
+ }
81
+ },
82
+ "tasks": {
83
+ tid: {
84
+ "task_id": t.task_id,
85
+ "stage_id": t.stage_id,
86
+ "type": t.task_def.type,
87
+ "role": t.task_def.role.value,
88
+ "status": t.status.value,
89
+ "attempt": t.attempt,
90
+ "pacs_score": t.pacs_score,
91
+ "gate_verdict": t.gate_verdict,
92
+ "deliverable": t.task_def.deliverable_path,
93
+ "error": t.error_message
94
+ }
95
+ for tid, t in self.instance.tasks.items()
96
+ },
97
+ "outputs": self.instance.outputs
98
+ }
99
+
100
+ temp_path = f"{self.sot_path}.tmp"
101
+ with open(temp_path, "w", encoding="utf-8") as f:
102
+ if yaml is not None:
103
+ yaml.dump(sot_data, f, sort_keys=False)
104
+ else:
105
+ json.dump(sot_data, f, indent=2)
106
+ os.replace(temp_path, self.sot_path)
107
+
108
+ async def step(self) -> bool:
109
+ """Executes one evaluation cycle of the state decider."""
110
+ # 1. Reclaim any expired leases in queue
111
+ self.queue.reclaim_expired()
112
+
113
+ # 2. Evaluate state
114
+ res = self.decider.evaluate(self.instance)
115
+
116
+ # 3. Schedule newly ready tasks into queue
117
+ for task in res.tasks_to_schedule:
118
+ self.queue.push(task)
119
+ await self.event_bus.publish(EngineEvent(
120
+ event_type="task.scheduled",
121
+ workflow_id=self.instance.workflow_id,
122
+ stage_id=task.stage_id,
123
+ task_id=task.task_id,
124
+ trace_id=self.instance.trace_id,
125
+ payload={"type": task.task_def.type, "role": task.task_def.role.value}
126
+ ))
127
+
128
+ for task in res.tasks_to_retry:
129
+ self.queue.push(task)
130
+ await self.event_bus.publish(EngineEvent(
131
+ event_type="task.retrying",
132
+ workflow_id=self.instance.workflow_id,
133
+ stage_id=task.stage_id,
134
+ task_id=task.task_id,
135
+ trace_id=self.instance.trace_id,
136
+ payload={"attempt": task.attempt}
137
+ ))
138
+
139
+ if res.stage_transitioned:
140
+ await self.event_bus.publish(EngineEvent(
141
+ event_type="stage.transitioned",
142
+ workflow_id=self.instance.workflow_id,
143
+ stage_id=res.new_stage_id,
144
+ trace_id=self.instance.trace_id
145
+ ))
146
+
147
+ # 4. Atomic SOT update
148
+ self._write_sot()
149
+
150
+ if res.is_terminal:
151
+ event_name = "workflow.completed" if res.workflow_status == WorkflowStatus.COMPLETED else "workflow.failed"
152
+ await self.event_bus.publish(EngineEvent(
153
+ event_type=event_name,
154
+ workflow_id=self.instance.workflow_id,
155
+ trace_id=self.instance.trace_id,
156
+ payload={"status": res.workflow_status.value, "error": res.error_message}
157
+ ))
158
+ return False
159
+
160
+ return True
161
+
162
+ async def run_until_complete(self, workers: list, poll_interval: float = 0.05, max_iterations: int = 100) -> bool:
163
+ """Runs the orchestrator loop alongside active workers until terminal state."""
164
+ await self.event_bus.publish(EngineEvent(
165
+ event_type="workflow.started",
166
+ workflow_id=self.instance.workflow_id,
167
+ trace_id=self.instance.trace_id,
168
+ payload={"name": self.workflow_def.name}
169
+ ))
170
+
171
+ iterations = 0
172
+ while iterations < max_iterations:
173
+ iterations += 1
174
+
175
+ # Let workers process available tasks
176
+ for w in workers:
177
+ await w.run_once()
178
+
179
+ # Advance orchestrator decider step
180
+ active = await self.step()
181
+ if not active:
182
+ break
183
+
184
+ await asyncio.sleep(poll_interval)
185
+
186
+ return self.instance.status == WorkflowStatus.COMPLETED
@@ -0,0 +1,193 @@
1
+ """
2
+ models.py — Data structures and state definitions for AgenticWorkflow Engine.
3
+
4
+ Defines:
5
+ - Workflow & Task definitions
6
+ - Workflow & Task execution instances
7
+ - State enumerations (TaskStatus, WorkflowStatus, BackoffType)
8
+ - Standard EngineEvent envelope
9
+ """
10
+
11
+ import time
12
+ import uuid
13
+ from enum import Enum
14
+ from dataclasses import dataclass, field, asdict
15
+ from typing import Dict, List, Optional, Any, Union
16
+
17
+
18
+ class TaskStatus(str, Enum):
19
+ SCHEDULED = "SCHEDULED"
20
+ POLLED = "POLLED"
21
+ IN_PROGRESS = "IN_PROGRESS"
22
+ GATE_EVALUATING = "GATE_EVALUATING"
23
+ DIAGNOSING = "DIAGNOSING"
24
+ COMPLETED = "COMPLETED"
25
+ FAILED = "FAILED"
26
+ TIMED_OUT = "TIMED_OUT"
27
+ SKIPPED = "SKIPPED"
28
+ CANCELLED = "CANCELLED"
29
+
30
+
31
+ class WorkflowStatus(str, Enum):
32
+ RUNNING = "RUNNING"
33
+ PAUSED = "PAUSED"
34
+ COMPLETED = "COMPLETED"
35
+ FAILED = "FAILED"
36
+ CANCELLED = "CANCELLED"
37
+
38
+
39
+ class BackoffType(str, Enum):
40
+ FIXED = "FIXED"
41
+ LINEAR = "LINEAR"
42
+ EXPONENTIAL = "EXPONENTIAL"
43
+
44
+
45
+ class AgentRole(str, Enum):
46
+ ORCHESTRATOR = "orchestrator"
47
+ RESEARCHER = "researcher"
48
+ ARCHITECT = "architect"
49
+ ENGINEER = "engineer"
50
+ REVIEWER = "reviewer"
51
+ FACT_CHECKER = "fact_checker"
52
+ CLEAN_CODE_GUARD = "clean_code_guard"
53
+ RECOVERY = "recovery"
54
+
55
+
56
+ @dataclass
57
+ class RetryPolicy:
58
+ max_retries: int = 3
59
+ delay_seconds: float = 2.0
60
+ backoff_rate: BackoffType = BackoffType.EXPONENTIAL
61
+
62
+ def calculate_delay(self, attempt: int) -> float:
63
+ if self.backoff_rate == BackoffType.FIXED:
64
+ return self.delay_seconds
65
+ elif self.backoff_rate == BackoffType.LINEAR:
66
+ return self.delay_seconds * attempt
67
+ else: # EXPONENTIAL
68
+ return self.delay_seconds * (2 ** (attempt - 1))
69
+
70
+
71
+ @dataclass
72
+ class CircuitBreakerConfig:
73
+ failure_threshold: int = 2
74
+ cooldown_seconds: float = 30.0
75
+
76
+
77
+ @dataclass
78
+ class TaskDefinition:
79
+ id: str
80
+ type: str # system.code, system.wait, agent.task, agent.human, agent.review, etc.
81
+ name: str = ""
82
+ description: str = ""
83
+ role: AgentRole = AgentRole.ENGINEER
84
+ deliverable_path: Optional[str] = None
85
+ criteria: List[str] = field(default_factory=list)
86
+ input_parameters: Dict[str, Any] = field(default_factory=dict)
87
+ retry_policy: RetryPolicy = field(default_factory=RetryPolicy)
88
+ timeout_seconds: int = 300
89
+ circuit_breaker: CircuitBreakerConfig = field(default_factory=CircuitBreakerConfig)
90
+ compensation_task: Optional[str] = None
91
+ branches: Dict[str, Any] = field(default_factory=dict)
92
+
93
+
94
+ @dataclass
95
+ class StageDefinition:
96
+ id: str
97
+ name: str
98
+ stage_type: str = "custom" # research, planning, implementation, verification
99
+ tasks: List[TaskDefinition] = field(default_factory=list)
100
+
101
+
102
+ @dataclass
103
+ class WorkflowDefinition:
104
+ name: str
105
+ version: str = "1.0.0"
106
+ description: str = ""
107
+ stages: List[StageDefinition] = field(default_factory=list)
108
+ input_parameters: Dict[str, Any] = field(default_factory=dict)
109
+ timeout_seconds: int = 3600
110
+ failure_workflow: Optional[str] = None
111
+
112
+
113
+ @dataclass
114
+ class TaskInstance:
115
+ task_id: str
116
+ workflow_id: str
117
+ stage_id: str
118
+ task_def: TaskDefinition
119
+ status: TaskStatus = TaskStatus.SCHEDULED
120
+ attempt: int = 1
121
+ input_data: Dict[str, Any] = field(default_factory=dict)
122
+ output_data: Dict[str, Any] = field(default_factory=dict)
123
+ worker_id: Optional[str] = None
124
+ scheduled_at: float = field(default_factory=time.time)
125
+ started_at: Optional[float] = None
126
+ completed_at: Optional[float] = None
127
+ lease_expires_at: Optional[float] = None
128
+ pacs_score: Optional[int] = None
129
+ gate_verdict: Optional[str] = None
130
+ error_message: Optional[str] = None
131
+ trace_id: str = ""
132
+
133
+ def to_dict(self) -> Dict[str, Any]:
134
+ d = asdict(self)
135
+ d["status"] = self.status.value
136
+ d["task_def"]["role"] = self.task_def.role.value
137
+ d["task_def"]["retry_policy"]["backoff_rate"] = self.task_def.retry_policy.backoff_rate.value
138
+ return d
139
+
140
+
141
+ @dataclass
142
+ class WorkflowInstance:
143
+ workflow_id: str
144
+ workflow_def: WorkflowDefinition
145
+ trace_id: str
146
+ status: WorkflowStatus = WorkflowStatus.RUNNING
147
+ current_stage_index: int = 0
148
+ tasks: Dict[str, TaskInstance] = field(default_factory=dict)
149
+ variables: Dict[str, Any] = field(default_factory=dict)
150
+ outputs: Dict[str, Any] = field(default_factory=dict)
151
+ started_at: float = field(default_factory=time.time)
152
+ completed_at: Optional[float] = None
153
+ autopilot_enabled: bool = True
154
+ error_message: Optional[str] = None
155
+
156
+ def get_current_stage(self) -> Optional[StageDefinition]:
157
+ if 0 <= self.current_stage_index < len(self.workflow_def.stages):
158
+ return self.workflow_def.stages[self.current_stage_index]
159
+ return None
160
+
161
+ def to_dict(self) -> Dict[str, Any]:
162
+ return {
163
+ "workflow_id": self.workflow_id,
164
+ "name": self.workflow_def.name,
165
+ "version": self.workflow_def.version,
166
+ "trace_id": self.trace_id,
167
+ "status": self.status.value,
168
+ "current_stage_index": self.current_stage_index,
169
+ "current_stage_id": self.get_current_stage().id if self.get_current_stage() else None,
170
+ "autopilot_enabled": self.autopilot_enabled,
171
+ "tasks": {tid: t.to_dict() for tid, t in self.tasks.items()},
172
+ "variables": self.variables,
173
+ "outputs": self.outputs,
174
+ "started_at": self.started_at,
175
+ "completed_at": self.completed_at,
176
+ "error_message": self.error_message
177
+ }
178
+
179
+
180
+ @dataclass
181
+ class EngineEvent:
182
+ event_id: str = field(default_factory=lambda: f"evt_{uuid.uuid4().hex[:12]}")
183
+ event_type: str = "workflow.started"
184
+ timestamp: float = field(default_factory=lambda: time.time() * 1000)
185
+ trace_id: str = ""
186
+ workflow_id: str = ""
187
+ stage_id: Optional[str] = None
188
+ task_id: Optional[str] = None
189
+ worker_id: Optional[str] = None
190
+ payload: Dict[str, Any] = field(default_factory=dict)
191
+
192
+ def to_dict(self) -> Dict[str, Any]:
193
+ return asdict(self)
@@ -0,0 +1,314 @@
1
+ """
2
+ queue.py — Pluggable Task Queuing System (QueueDAO).
3
+
4
+ Implements:
5
+ - Abstract TaskQueue base interface
6
+ - InMemoryTaskQueue for testing & sub-millisecond local execution
7
+ - SQLiteTaskQueue for durable, ACID, zero-infra persistence with lease renewal
8
+ """
9
+
10
+ import time
11
+ import json
12
+ import sqlite3
13
+ import threading
14
+ from contextlib import contextmanager
15
+ from abc import ABC, abstractmethod
16
+ from typing import Dict, List, Optional, Any
17
+ from core.engine_py.models import TaskInstance, TaskStatus, TaskDefinition, AgentRole, RetryPolicy, BackoffType, CircuitBreakerConfig
18
+
19
+
20
+ class TaskQueue(ABC):
21
+ """Abstract interface for task scheduling, leasing, and completion."""
22
+
23
+ @abstractmethod
24
+ def push(self, task: TaskInstance) -> None:
25
+ """Enqueues a task for worker execution."""
26
+ pass
27
+
28
+ @abstractmethod
29
+ def poll(self, task_types: List[str], worker_id: str, lease_seconds: float = 60.0) -> Optional[TaskInstance]:
30
+ """Polls for a matching task and acquires an execution lease."""
31
+ pass
32
+
33
+ @abstractmethod
34
+ def ack(self, task_id: str) -> None:
35
+ """Acknowledges successful task completion, removing it from queue."""
36
+ pass
37
+
38
+ @abstractmethod
39
+ def nack(self, task_id: str, requeue: bool = True) -> None:
40
+ """Negative acknowledgment; resets task to QUEUED or marks failed."""
41
+ pass
42
+
43
+ @abstractmethod
44
+ def heartbeat(self, task_id: str, worker_id: str, extend_seconds: float = 60.0) -> bool:
45
+ """Renews the active worker lease to prevent timeout reclamation."""
46
+ pass
47
+
48
+ @abstractmethod
49
+ def reclaim_expired(self) -> List[str]:
50
+ """Reclaims tasks whose leases have expired and returns their IDs."""
51
+ pass
52
+
53
+ @abstractmethod
54
+ def size(self) -> int:
55
+ """Returns count of active tasks in queue."""
56
+ pass
57
+
58
+
59
+ class InMemoryTaskQueue(TaskQueue):
60
+ """Thread-safe in-memory task queue."""
61
+
62
+ def __init__(self):
63
+ self._lock = threading.Lock()
64
+ self._tasks: Dict[str, TaskInstance] = {}
65
+ self._queue: List[str] = [] # task_ids waiting to be polled
66
+
67
+ def push(self, task: TaskInstance) -> None:
68
+ with self._lock:
69
+ self._tasks[task.task_id] = task
70
+ if task.task_id not in self._queue:
71
+ self._queue.append(task.task_id)
72
+
73
+ def poll(self, task_types: List[str], worker_id: str, lease_seconds: float = 60.0) -> Optional[TaskInstance]:
74
+ with self._lock:
75
+ for idx, task_id in enumerate(self._queue):
76
+ task = self._tasks.get(task_id)
77
+ if task and (not task_types or task.task_def.type in task_types):
78
+ self._queue.pop(idx)
79
+ task.status = TaskStatus.POLLED
80
+ task.worker_id = worker_id
81
+ task.lease_expires_at = time.time() + lease_seconds
82
+ return task
83
+ return None
84
+
85
+ def ack(self, task_id: str) -> None:
86
+ with self._lock:
87
+ self._tasks.pop(task_id, None)
88
+ if task_id in self._queue:
89
+ self._queue.remove(task_id)
90
+
91
+ def nack(self, task_id: str, requeue: bool = True) -> None:
92
+ with self._lock:
93
+ task = self._tasks.get(task_id)
94
+ if task:
95
+ task.worker_id = None
96
+ task.lease_expires_at = None
97
+ if requeue and task_id not in self._queue:
98
+ task.status = TaskStatus.SCHEDULED
99
+ self._queue.append(task_id)
100
+ elif not requeue:
101
+ task.status = TaskStatus.FAILED
102
+ self._tasks.pop(task_id, None)
103
+
104
+ def heartbeat(self, task_id: str, worker_id: str, extend_seconds: float = 60.0) -> bool:
105
+ with self._lock:
106
+ task = self._tasks.get(task_id)
107
+ if task and task.worker_id == worker_id:
108
+ task.lease_expires_at = time.time() + extend_seconds
109
+ return True
110
+ return False
111
+
112
+ def reclaim_expired(self) -> List[str]:
113
+ now = time.time()
114
+ reclaimed = []
115
+ with self._lock:
116
+ for task_id, task in list(self._tasks.items()):
117
+ if task.worker_id and task.lease_expires_at and task.lease_expires_at < now:
118
+ task.worker_id = None
119
+ task.lease_expires_at = None
120
+ task.status = TaskStatus.SCHEDULED
121
+ if task_id not in self._queue:
122
+ self._queue.append(task_id)
123
+ reclaimed.append(task_id)
124
+ return reclaimed
125
+
126
+ def size(self) -> int:
127
+ with self._lock:
128
+ return len(self._queue)
129
+
130
+
131
+ class SQLiteTaskQueue(TaskQueue):
132
+ """Durable, ACID SQLite-backed queue with row locking and lease renewal."""
133
+
134
+ def __init__(self, db_path: str = ".queue.db"):
135
+ self.db_path = db_path
136
+ self._lock = threading.Lock()
137
+ self._init_db()
138
+
139
+ @contextmanager
140
+ def _connection(self):
141
+ conn = sqlite3.connect(self.db_path, timeout=30.0)
142
+ conn.row_factory = sqlite3.Row
143
+ try:
144
+ yield conn
145
+ finally:
146
+ conn.close()
147
+
148
+ def _init_db(self) -> None:
149
+ with self._lock, self._connection() as conn:
150
+ conn.execute("PRAGMA journal_mode=WAL;")
151
+ conn.execute("""
152
+ CREATE TABLE IF NOT EXISTS tasks (
153
+ task_id TEXT PRIMARY KEY,
154
+ task_type TEXT NOT NULL,
155
+ workflow_id TEXT NOT NULL,
156
+ stage_id TEXT NOT NULL,
157
+ status TEXT NOT NULL,
158
+ worker_id TEXT,
159
+ lease_expires_at REAL,
160
+ task_json TEXT NOT NULL,
161
+ created_at REAL NOT NULL
162
+ );
163
+ """)
164
+ conn.execute("CREATE INDEX IF NOT EXISTS idx_tasks_poll ON tasks(status, task_type);")
165
+ conn.commit()
166
+
167
+ def push(self, task: TaskInstance) -> None:
168
+ with self._lock, self._connection() as conn:
169
+ conn.execute("""
170
+ INSERT OR REPLACE INTO tasks
171
+ (task_id, task_type, workflow_id, stage_id, status, worker_id, lease_expires_at, task_json, created_at)
172
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
173
+ """, (
174
+ task.task_id,
175
+ task.task_def.type,
176
+ task.workflow_id,
177
+ task.stage_id,
178
+ "QUEUED",
179
+ None,
180
+ None,
181
+ json.dumps(task.to_dict()),
182
+ time.time()
183
+ ))
184
+ conn.commit()
185
+
186
+ def poll(self, task_types: List[str], worker_id: str, lease_seconds: float = 60.0) -> Optional[TaskInstance]:
187
+ with self._lock, self._connection() as conn:
188
+ query = "SELECT task_id, task_json FROM tasks WHERE status = 'QUEUED'"
189
+ params: List[Any] = []
190
+ if task_types:
191
+ placeholders = ",".join("?" for _ in task_types)
192
+ query += f" AND task_type IN ({placeholders})"
193
+ params.extend(task_types)
194
+ query += " ORDER BY created_at ASC LIMIT 1"
195
+
196
+ cursor = conn.execute(query, params)
197
+ row = cursor.fetchone()
198
+ if not row:
199
+ return None
200
+
201
+ task_id = row["task_id"]
202
+ now = time.time()
203
+ expires_at = now + lease_seconds
204
+
205
+ conn.execute("""
206
+ UPDATE tasks
207
+ SET status = 'ASSIGNED', worker_id = ?, lease_expires_at = ?
208
+ WHERE task_id = ?
209
+ """, (worker_id, expires_at, task_id))
210
+ conn.commit()
211
+
212
+ raw_dict = json.loads(row["task_json"])
213
+ task = self._deserialize_task(raw_dict)
214
+ task.status = TaskStatus.POLLED
215
+ task.worker_id = worker_id
216
+ task.lease_expires_at = expires_at
217
+ return task
218
+
219
+ def ack(self, task_id: str) -> None:
220
+ with self._lock, self._connection() as conn:
221
+ conn.execute("DELETE FROM tasks WHERE task_id = ?", (task_id,))
222
+ conn.commit()
223
+
224
+ def nack(self, task_id: str, requeue: bool = True) -> None:
225
+ with self._lock, self._connection() as conn:
226
+ if requeue:
227
+ conn.execute("""
228
+ UPDATE tasks
229
+ SET status = 'QUEUED', worker_id = NULL, lease_expires_at = NULL
230
+ WHERE task_id = ?
231
+ """, (task_id,))
232
+ else:
233
+ conn.execute("DELETE FROM tasks WHERE task_id = ?", (task_id,))
234
+ conn.commit()
235
+
236
+ def heartbeat(self, task_id: str, worker_id: str, extend_seconds: float = 60.0) -> bool:
237
+ with self._lock, self._connection() as conn:
238
+ cursor = conn.execute("""
239
+ UPDATE tasks
240
+ SET lease_expires_at = ?
241
+ WHERE task_id = ? AND worker_id = ?
242
+ """, (time.time() + extend_seconds, task_id, worker_id))
243
+ conn.commit()
244
+ return cursor.rowcount > 0
245
+
246
+ def reclaim_expired(self) -> List[str]:
247
+ now = time.time()
248
+ with self._lock, self._connection() as conn:
249
+ cursor = conn.execute("""
250
+ SELECT task_id FROM tasks
251
+ WHERE status = 'ASSIGNED' AND lease_expires_at < ?
252
+ """, (now,))
253
+ expired_ids = [row["task_id"] for row in cursor.fetchall()]
254
+ if expired_ids:
255
+ placeholders = ",".join("?" for _ in expired_ids)
256
+ conn.execute(f"""
257
+ UPDATE tasks
258
+ SET status = 'QUEUED', worker_id = NULL, lease_expires_at = NULL
259
+ WHERE task_id IN ({placeholders})
260
+ """, expired_ids)
261
+ conn.commit()
262
+ return expired_ids
263
+
264
+ def size(self) -> int:
265
+ with self._lock, self._connection() as conn:
266
+ cursor = conn.execute("SELECT COUNT(*) as count FROM tasks WHERE status = 'QUEUED'")
267
+ row = cursor.fetchone()
268
+ return row["count"] if row else 0
269
+
270
+ def _deserialize_task(self, d: Dict[str, Any]) -> TaskInstance:
271
+ td = d["task_def"]
272
+ rp = td.get("retry_policy", {})
273
+ cb = td.get("circuit_breaker", {})
274
+ task_def = TaskDefinition(
275
+ id=td["id"],
276
+ type=td["type"],
277
+ name=td.get("name", ""),
278
+ description=td.get("description", ""),
279
+ role=AgentRole(td.get("role", "engineer")),
280
+ deliverable_path=td.get("deliverable_path"),
281
+ criteria=td.get("criteria", []),
282
+ input_parameters=td.get("input_parameters", {}),
283
+ retry_policy=RetryPolicy(
284
+ max_retries=rp.get("max_retries", 3),
285
+ delay_seconds=rp.get("delay_seconds", 2.0),
286
+ backoff_rate=BackoffType(rp.get("backoff_rate", "EXPONENTIAL"))
287
+ ),
288
+ timeout_seconds=td.get("timeout_seconds", 300),
289
+ circuit_breaker=CircuitBreakerConfig(
290
+ failure_threshold=cb.get("failure_threshold", 2),
291
+ cooldown_seconds=cb.get("cooldown_seconds", 30.0)
292
+ ),
293
+ compensation_task=td.get("compensation_task"),
294
+ branches=td.get("branches", {})
295
+ )
296
+ return TaskInstance(
297
+ task_id=d["task_id"],
298
+ workflow_id=d["workflow_id"],
299
+ stage_id=d["stage_id"],
300
+ task_def=task_def,
301
+ status=TaskStatus(d.get("status", "SCHEDULED")),
302
+ attempt=d.get("attempt", 1),
303
+ input_data=d.get("input_data", {}),
304
+ output_data=d.get("output_data", {}),
305
+ worker_id=d.get("worker_id"),
306
+ scheduled_at=d.get("scheduled_at", time.time()),
307
+ started_at=d.get("started_at"),
308
+ completed_at=d.get("completed_at"),
309
+ lease_expires_at=d.get("lease_expires_at"),
310
+ pacs_score=d.get("pacs_score"),
311
+ gate_verdict=d.get("gate_verdict"),
312
+ error_message=d.get("error_message"),
313
+ trace_id=d.get("trace_id", "")
314
+ )