@litmers/cursorflow-orchestrator 0.1.20 → 0.1.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/commands/cursorflow-clean.md +19 -0
- package/commands/cursorflow-runs.md +59 -0
- package/commands/cursorflow-stop.md +55 -0
- package/dist/cli/clean.js +171 -0
- package/dist/cli/clean.js.map +1 -1
- package/dist/cli/index.js +7 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/init.js +1 -1
- package/dist/cli/init.js.map +1 -1
- package/dist/cli/logs.js +83 -42
- package/dist/cli/logs.js.map +1 -1
- package/dist/cli/monitor.d.ts +7 -0
- package/dist/cli/monitor.js +1007 -189
- package/dist/cli/monitor.js.map +1 -1
- package/dist/cli/prepare.js +87 -3
- package/dist/cli/prepare.js.map +1 -1
- package/dist/cli/resume.js +188 -236
- package/dist/cli/resume.js.map +1 -1
- package/dist/cli/run.js +125 -3
- package/dist/cli/run.js.map +1 -1
- package/dist/cli/runs.d.ts +5 -0
- package/dist/cli/runs.js +214 -0
- package/dist/cli/runs.js.map +1 -0
- package/dist/cli/setup-commands.js +0 -0
- package/dist/cli/signal.js +1 -1
- package/dist/cli/signal.js.map +1 -1
- package/dist/cli/stop.d.ts +5 -0
- package/dist/cli/stop.js +215 -0
- package/dist/cli/stop.js.map +1 -0
- package/dist/cli/tasks.d.ts +10 -0
- package/dist/cli/tasks.js +165 -0
- package/dist/cli/tasks.js.map +1 -0
- package/dist/core/auto-recovery.d.ts +212 -0
- package/dist/core/auto-recovery.js +737 -0
- package/dist/core/auto-recovery.js.map +1 -0
- package/dist/core/failure-policy.d.ts +156 -0
- package/dist/core/failure-policy.js +488 -0
- package/dist/core/failure-policy.js.map +1 -0
- package/dist/core/orchestrator.d.ts +15 -2
- package/dist/core/orchestrator.js +397 -15
- package/dist/core/orchestrator.js.map +1 -1
- package/dist/core/reviewer.d.ts +2 -0
- package/dist/core/reviewer.js +2 -0
- package/dist/core/reviewer.js.map +1 -1
- package/dist/core/runner.d.ts +33 -10
- package/dist/core/runner.js +321 -146
- package/dist/core/runner.js.map +1 -1
- package/dist/services/logging/buffer.d.ts +67 -0
- package/dist/services/logging/buffer.js +309 -0
- package/dist/services/logging/buffer.js.map +1 -0
- package/dist/services/logging/console.d.ts +89 -0
- package/dist/services/logging/console.js +169 -0
- package/dist/services/logging/console.js.map +1 -0
- package/dist/services/logging/file-writer.d.ts +71 -0
- package/dist/services/logging/file-writer.js +516 -0
- package/dist/services/logging/file-writer.js.map +1 -0
- package/dist/services/logging/formatter.d.ts +39 -0
- package/dist/services/logging/formatter.js +227 -0
- package/dist/services/logging/formatter.js.map +1 -0
- package/dist/services/logging/index.d.ts +11 -0
- package/dist/services/logging/index.js +30 -0
- package/dist/services/logging/index.js.map +1 -0
- package/dist/services/logging/parser.d.ts +31 -0
- package/dist/services/logging/parser.js +222 -0
- package/dist/services/logging/parser.js.map +1 -0
- package/dist/services/process/index.d.ts +59 -0
- package/dist/services/process/index.js +257 -0
- package/dist/services/process/index.js.map +1 -0
- package/dist/types/agent.d.ts +20 -0
- package/dist/types/agent.js +6 -0
- package/dist/types/agent.js.map +1 -0
- package/dist/types/config.d.ts +65 -0
- package/dist/types/config.js +6 -0
- package/dist/types/config.js.map +1 -0
- package/dist/types/events.d.ts +125 -0
- package/dist/types/events.js +6 -0
- package/dist/types/events.js.map +1 -0
- package/dist/types/index.d.ts +12 -0
- package/dist/types/index.js +37 -0
- package/dist/types/index.js.map +1 -0
- package/dist/types/lane.d.ts +43 -0
- package/dist/types/lane.js +6 -0
- package/dist/types/lane.js.map +1 -0
- package/dist/types/logging.d.ts +71 -0
- package/dist/types/logging.js +16 -0
- package/dist/types/logging.js.map +1 -0
- package/dist/types/review.d.ts +17 -0
- package/dist/types/review.js +6 -0
- package/dist/types/review.js.map +1 -0
- package/dist/types/run.d.ts +32 -0
- package/dist/types/run.js +6 -0
- package/dist/types/run.js.map +1 -0
- package/dist/types/task.d.ts +71 -0
- package/dist/types/task.js +6 -0
- package/dist/types/task.js.map +1 -0
- package/dist/ui/components.d.ts +134 -0
- package/dist/ui/components.js +389 -0
- package/dist/ui/components.js.map +1 -0
- package/dist/ui/log-viewer.d.ts +49 -0
- package/dist/ui/log-viewer.js +449 -0
- package/dist/ui/log-viewer.js.map +1 -0
- package/dist/utils/checkpoint.d.ts +87 -0
- package/dist/utils/checkpoint.js +317 -0
- package/dist/utils/checkpoint.js.map +1 -0
- package/dist/utils/config.d.ts +4 -0
- package/dist/utils/config.js +11 -2
- package/dist/utils/config.js.map +1 -1
- package/dist/utils/cursor-agent.js.map +1 -1
- package/dist/utils/dependency.d.ts +74 -0
- package/dist/utils/dependency.js +420 -0
- package/dist/utils/dependency.js.map +1 -0
- package/dist/utils/doctor.js +10 -5
- package/dist/utils/doctor.js.map +1 -1
- package/dist/utils/enhanced-logger.d.ts +10 -33
- package/dist/utils/enhanced-logger.js +94 -9
- package/dist/utils/enhanced-logger.js.map +1 -1
- package/dist/utils/git.d.ts +121 -0
- package/dist/utils/git.js +322 -2
- package/dist/utils/git.js.map +1 -1
- package/dist/utils/health.d.ts +91 -0
- package/dist/utils/health.js +556 -0
- package/dist/utils/health.js.map +1 -0
- package/dist/utils/lock.d.ts +95 -0
- package/dist/utils/lock.js +332 -0
- package/dist/utils/lock.js.map +1 -0
- package/dist/utils/log-buffer.d.ts +17 -0
- package/dist/utils/log-buffer.js +14 -0
- package/dist/utils/log-buffer.js.map +1 -0
- package/dist/utils/log-constants.d.ts +23 -0
- package/dist/utils/log-constants.js +28 -0
- package/dist/utils/log-constants.js.map +1 -0
- package/dist/utils/log-formatter.d.ts +9 -0
- package/dist/utils/log-formatter.js +113 -70
- package/dist/utils/log-formatter.js.map +1 -1
- package/dist/utils/log-service.d.ts +19 -0
- package/dist/utils/log-service.js +47 -0
- package/dist/utils/log-service.js.map +1 -0
- package/dist/utils/logger.d.ts +46 -27
- package/dist/utils/logger.js +82 -60
- package/dist/utils/logger.js.map +1 -1
- package/dist/utils/process-manager.d.ts +21 -0
- package/dist/utils/process-manager.js +138 -0
- package/dist/utils/process-manager.js.map +1 -0
- package/dist/utils/retry.d.ts +121 -0
- package/dist/utils/retry.js +374 -0
- package/dist/utils/retry.js.map +1 -0
- package/dist/utils/run-service.d.ts +88 -0
- package/dist/utils/run-service.js +412 -0
- package/dist/utils/run-service.js.map +1 -0
- package/dist/utils/state.d.ts +58 -2
- package/dist/utils/state.js +306 -3
- package/dist/utils/state.js.map +1 -1
- package/dist/utils/task-service.d.ts +82 -0
- package/dist/utils/task-service.js +348 -0
- package/dist/utils/task-service.js.map +1 -0
- package/dist/utils/types.d.ts +2 -272
- package/dist/utils/types.js +16 -0
- package/dist/utils/types.js.map +1 -1
- package/package.json +38 -23
- package/scripts/ai-security-check.js +0 -1
- package/scripts/local-security-gate.sh +0 -0
- package/scripts/monitor-lanes.sh +94 -0
- package/scripts/patches/test-cursor-agent.js +0 -1
- package/scripts/release.sh +0 -0
- package/scripts/setup-security.sh +0 -0
- package/scripts/stream-logs.sh +72 -0
- package/scripts/verify-and-fix.sh +0 -0
- package/src/cli/clean.ts +180 -0
- package/src/cli/index.ts +7 -0
- package/src/cli/init.ts +1 -1
- package/src/cli/logs.ts +79 -42
- package/src/cli/monitor.ts +1815 -899
- package/src/cli/prepare.ts +97 -3
- package/src/cli/resume.ts +220 -277
- package/src/cli/run.ts +154 -3
- package/src/cli/runs.ts +212 -0
- package/src/cli/setup-commands.ts +0 -0
- package/src/cli/signal.ts +1 -1
- package/src/cli/stop.ts +209 -0
- package/src/cli/tasks.ts +154 -0
- package/src/core/auto-recovery.ts +909 -0
- package/src/core/failure-policy.ts +592 -0
- package/src/core/orchestrator.ts +1136 -675
- package/src/core/reviewer.ts +4 -0
- package/src/core/runner.ts +1443 -1217
- package/src/services/logging/buffer.ts +326 -0
- package/src/services/logging/console.ts +193 -0
- package/src/services/logging/file-writer.ts +526 -0
- package/src/services/logging/formatter.ts +268 -0
- package/src/services/logging/index.ts +16 -0
- package/src/services/logging/parser.ts +232 -0
- package/src/services/process/index.ts +261 -0
- package/src/types/agent.ts +24 -0
- package/src/types/config.ts +79 -0
- package/src/types/events.ts +156 -0
- package/src/types/index.ts +29 -0
- package/src/types/lane.ts +56 -0
- package/src/types/logging.ts +96 -0
- package/src/types/review.ts +20 -0
- package/src/types/run.ts +37 -0
- package/src/types/task.ts +79 -0
- package/src/ui/components.ts +430 -0
- package/src/ui/log-viewer.ts +485 -0
- package/src/utils/checkpoint.ts +374 -0
- package/src/utils/config.ts +11 -2
- package/src/utils/cursor-agent.ts +1 -1
- package/src/utils/dependency.ts +482 -0
- package/src/utils/doctor.ts +11 -5
- package/src/utils/enhanced-logger.ts +108 -49
- package/src/utils/git.ts +871 -499
- package/src/utils/health.ts +596 -0
- package/src/utils/lock.ts +346 -0
- package/src/utils/log-buffer.ts +28 -0
- package/src/utils/log-constants.ts +26 -0
- package/src/utils/log-formatter.ts +120 -37
- package/src/utils/log-service.ts +49 -0
- package/src/utils/logger.ts +100 -51
- package/src/utils/process-manager.ts +100 -0
- package/src/utils/retry.ts +413 -0
- package/src/utils/run-service.ts +433 -0
- package/src/utils/state.ts +369 -3
- package/src/utils/task-service.ts +370 -0
- package/src/utils/types.ts +2 -315
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"failure-policy.js","sourceRoot":"","sources":["../../src/core/failure-policy.ts"],"names":[],"mappings":";AAAA;;;;;;;GAOG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAkIH,oCAsIC;AAKD,wCA0JC;AAKD,gCAiBC;AAKD,8CAUC;AAKD,8BA0DC;AAcD,kDAYC;AAaD,4CAQC;AAED,kCAYC;AAtkBD,wDAA0C;AAC1C,0CAAiE;AAEjE,IAAY,WAoBX;AApBD,WAAY,WAAW;IACrB,wCAAyB,CAAA;IACzB,sDAAuC,CAAA;IACvC,oDAAqC,CAAA;IACrC,sDAAuC,CAAA;IACvC,oDAAqC,CAAA;IACrC,oDAAqC,CAAA;IACrC,8CAA+B,CAAA;IAC/B,sDAAuC,CAAA;IACvC,gDAAiC,CAAA;IACjC,oDAAqC,CAAA;IACrC,sDAAuC,CAAA;IACvC,wDAAyC,CAAA;IACzC,0CAA2B,CAAA;IAC3B,sCAAuB,CAAA;IACvB,sDAAuC,CAAA;IACvC,gDAAiC,CAAA;IACjC,8CAA+B,CAAA;IAC/B,oDAAqC,CAAA;IACrC,8CAA+B,CAAA;AACjC,CAAC,EApBW,WAAW,2BAAX,WAAW,QAoBtB;AAED,IAAY,cAcX;AAdD,WAAY,cAAc;IACxB,qDAAmC,CAAA;IACnC,qDAAmC,CAAA;IACnC,2CAAyB,CAAA;IACzB,+CAA6B,CAAA;IAC7B,+EAA6D,CAAA;IAC7D,uDAAqC,CAAA;IACrC,2CAAyB,CAAA;IACzB,iDAA+B,CAAA;IAC/B,mDAAiC,CAAA;IACjC,yCAAuB,CAAA;IACvB,yDAAuC,CAAA;IACvC,2CAAyB,CAAA;IACzB,+BAAa,CAAA;AACf,CAAC,EAdW,cAAc,8BAAd,cAAc,QAczB;AA6BY,QAAA,oBAAoB,GAAyB;IACxD,aAAa,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,EAAQ,4CAA4C;IAChF,iBAAiB,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,EAAG,8BAA8B;IAClE,aAAa,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,EAAO,0BAA0B;IAC9D,oBAAoB,EAAE,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,+BAA+B;IACrE,qBAAqB,EAAE;QACrB,0BAA0B;QAC1B,cAAc;QACd,eAAe;QACf,eAAe;QACf,WAAW;QACX,YAAY;QACZ,cAAc;QACd,WAAW;QACX,UAAU;QACV,WAAW;KACZ;IACD,WAAW,EAAE,CAAC;CACf,CAAC;AAiCF;;;;;;;;GAQG;AACH,SAAgB,YAAY,CAAC,OAAqB,EAAE,SAA+B,4BAAoB;IACrG,MAAM,EACJ,UAAU,EACV,UAAU,EACV,cAAc,EACd,UAAU,EACV,YAAY,GAAG,CAAC,EAChB,eAAe,EACf,aAAa,GAAG,CAAC,CAAC,EAAE,uBAAuB;IAC3C,mBAAmB,GAAG,CAAC,GACxB,GAAG,OAAO,CAAC;IAEZ,0CAA0C;IAC1C,MAAM,eAAe,GAAG,UAAU,IAAI,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;IACjG,MAAM,oBAAoB,GAAG,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC;IAElG,yBAAyB;IACzB,IAAI,eAAe,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;QAC7E,OAAO;YACL,IAAI,EAAE,WAAW,CAAC,aAAa;YAC/B,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,cAAc,CAAC,gBAAgB,CAAC,CAAC,CAAC,cAAc,CAAC,UAAU;YACvG,OAAO,EAAE,oCAAoC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,GAAG,KAAK,CAAC,UAAU;YAC/F,WAAW,EAAE,YAAY,GAAG,MAAM,CAAC,WAAW;YAC9C,OAAO,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,EAAE,YAAY,EAAE;SACxE,CAAC;IACJ,CAAC;IAED,gEAAgE;IAChE,IAAI,aAAa,KAAK,CAAC,IAAI,UAAU,GAAG,oBAAoB,EAAE,CAAC;QAC7D,OAAO;YACL,IAAI,EAAE,WAAW,CAAC,iBAAiB;YACnC,MAAM,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC,CAAC,cAAc,CAAC,gBAAgB;YACzF,OAAO,EAAE,8BAA8B,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,wBAAwB;YAC5F,WAAW,EAAE,IAAI;YACjB,OAAO,EAAE,EAAE,UAAU,EAAE,aAAa,EAAE,UAAU,EAAE;SACnD,CAAC;IACJ,CAAC;IAED,kDAAkD;IAClD,IAAI,cAAc,IAAI,cAAc,GAAG,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAChE,OAAO;YACL,IAAI,EAAE,WAAW,CAAC,iBAAiB;YACnC,MAAM,EAAE,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;gBACnD,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,eAAe,CAAC,CAAC;oBACnD,cAAc,CAAC,gBAAgB;YACvC,OAAO,EAAE,mBAAmB,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,UAAU;YACxE,WAAW,EAAE,IAAI;YACjB,OAAO,EAAE,EAAE,cAAc,EAAE,UAAU,EAAE;SACxC,CAAC;IACJ,CAAC;IAED,oDAAoD;IACpD,IAAI,UAAU,KAAK,CAAC,IAAI,UAAU,GAAG,oBAAoB,EAAE,CAAC;QAC1D,OAAO;YACL,IAAI,EAAE,WAAW,CAAC,UAAU;YAC5B,MAAM,EAAE,cAAc,CAAC,eAAe;YACtC,OAAO,EAAE,iBAAiB,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,IAAI,CAAC,+BAA+B;YACtF,WAAW,EAAE,IAAI;YACjB,OAAO,EAAE,EAAE,UAAU,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE;SACnD,CAAC;IACJ,CAAC;IAED,mDAAmD;IACnD,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;QACrB,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,iCAAiC;QAErE,IAAI,UAAU,GAAG,YAAY,EAAE,CAAC;YAC9B,OAAO;gBACL,IAAI,EAAE,WAAW,CAAC,UAAU;gBAC5B,MAAM,EAAE,cAAc,CAAC,eAAe;gBACtC,OAAO,EAAE,8DAA8D;gBACvE,WAAW,EAAE,IAAI;gBACjB,OAAO,EAAE,EAAE,UAAU,EAAE,mBAAmB,EAAE,KAAK,EAAE,CAAC,EAAE;aACvD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,kDAAkD;IAClD,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;QACrB,MAAM,oBAAoB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,wCAAwC;QAEpF,IAAI,UAAU,GAAG,oBAAoB,EAAE,CAAC;YACtC,IAAI,YAAY,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;gBACtC,OAAO;oBACL,IAAI,EAAE,WAAW,CAAC,UAAU;oBAC5B,MAAM,EAAE,cAAc,CAAC,gBAAgB;oBACvC,OAAO,EAAE,sEAAsE;oBAC/E,WAAW,EAAE,IAAI;oBACjB,OAAO,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE;iBACjF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO;oBACL,IAAI,EAAE,WAAW,CAAC,UAAU;oBAC5B,MAAM,EAAE,cAAc,CAAC,UAAU;oBACjC,OAAO,EAAE,qBAAqB,YAAY,mCAAmC;oBAC7E,WAAW,EAAE,KAAK;oBAClB,OAAO,EAAE,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE;iBACpC,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACpB,MAAM,kBAAkB,GAAG,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,gCAAgC;QAExF,IAAI,UAAU,GAAG,kBAAkB,EAAE,CAAC;YACpC,IAAI,YAAY,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;gBACtC,OAAO;oBACL,IAAI,EAAE,WAAW,CAAC,UAAU;oBAC5B,MAAM,EAAE,cAAc,CAAC,eAAe;oBACtC,OAAO,EAAE,sDAAsD;oBAC/D,WAAW,EAAE,IAAI;oBACjB,OAAO,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE;iBACzD,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO;oBACL,IAAI,EAAE,WAAW,CAAC,UAAU;oBAC5B,MAAM,EAAE,cAAc,CAAC,UAAU;oBACjC,OAAO,EAAE,gEAAgE;oBACzE,WAAW,EAAE,KAAK;oBAClB,OAAO,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE;iBACtC,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,OAAO;QACL,IAAI,EAAE,WAAW,CAAC,UAAU;QAC5B,MAAM,EAAE,cAAc,CAAC,IAAI;QAC3B,OAAO,EAAE,sBAAsB;QAC/B,WAAW,EAAE,IAAI;KAClB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAgB,cAAc,CAAC,KAAgC,EAAE,OAAwB;IACvF,MAAM,GAAG,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAExC,qCAAqC;IACrC,IAAI,OAAO,EAAE,kBAAkB,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,IAAA,yBAAiB,EAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAC9D,IAAI,OAAO,CAAC,QAAQ,EAAE,KAAK,oBAAY,CAAC,IAAI,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;YAC7C,OAAO;gBACL,IAAI,EAAE,WAAW,CAAC,iBAAiB;gBACnC,MAAM,EAAE,cAAc,CAAC,cAAc;gBACrC,OAAO,EAAE,kCAAkC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG;gBACzE,WAAW,EAAE,IAAI;gBACjB,gBAAgB,EAAE,QAAQ;aAC3B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC;QAC1D,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC;QACtD,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9D,OAAO;YACL,IAAI,EAAE,WAAW,CAAC,aAAa;YAC/B,MAAM,EAAE,CAAC,OAAO,EAAE,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,YAAY;YAChG,OAAO,EAAE,4BAA4B;YACrC,WAAW,EAAE,IAAI;YACjB,gBAAgB,EAAE,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,IAAI,CAAC,CAAC;SAC/D,CAAC;IACJ,CAAC;IAED,+BAA+B;IAC/B,IAAI,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QAClE,OAAO;YACL,IAAI,EAAE,WAAW,CAAC,iBAAiB;YACnC,MAAM,EAAE,CAAC,OAAO,EAAE,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,YAAY;YAChG,OAAO,EAAE,8EAA8E;YACvF,WAAW,EAAE,IAAI;YACjB,gBAAgB,EAAE,KAAK;SACxB,CAAC;IACJ,CAAC;IAED,2BAA2B;IAC3B,IAAI,GAAG,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC;QACjE,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QACvD,OAAO;YACL,IAAI,EAAE,WAAW,CAAC,gBAAgB;YAClC,MAAM,EAAE,cAAc,CAAC,aAAa;YACpC,OAAO,EAAE,6DAA6D;YACtE,WAAW,EAAE,KAAK;SACnB,CAAC;IACJ,CAAC;IAED,iBAAiB;IACjB,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;QACnD,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAC7D,OAAO;YACL,IAAI,EAAE,WAAW,CAAC,gBAAgB;YAClC,MAAM,EAAE,cAAc,CAAC,cAAc;YACrC,OAAO,EAAE,iDAAiD;YAC1D,WAAW,EAAE,IAAI;YACjB,gBAAgB,EAAE,KAAK,EAAE,WAAW;SACrC,CAAC;IACJ,CAAC;IAED,aAAa;IACb,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACzD,OAAO;YACL,IAAI,EAAE,WAAW,CAAC,aAAa;YAC/B,MAAM,EAAE,CAAC,OAAO,EAAE,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,YAAY;YAChG,OAAO,EAAE,sBAAsB;YAC/B,WAAW,EAAE,IAAI;SAClB,CAAC;IACJ,CAAC;IAED,+CAA+C;IAC/C,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;QACvG,OAAO;YACL,IAAI,EAAE,WAAW,CAAC,cAAc;YAChC,MAAM,EAAE,cAAc,CAAC,iBAAiB;YACxC,OAAO,EAAE,uDAAuD;YAChE,WAAW,EAAE,IAAI;SAClB,CAAC;IACJ,CAAC;IAED,+CAA+C;IAC/C,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,kBAAkB,CAAC;QAC5D,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC;QAClE,OAAO;YACL,IAAI,EAAE,WAAW,CAAC,iBAAiB;YACnC,MAAM,EAAE,cAAc,CAAC,iBAAiB;YACxC,OAAO,EAAE,iDAAiD;YAC1D,WAAW,EAAE,IAAI;SAClB,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QAC7E,OAAO;YACL,IAAI,EAAE,WAAW,CAAC,SAAS;YAC3B,MAAM,EAAE,CAAC,OAAO,EAAE,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,CAAC,SAAS;YAC7F,OAAO,EAAE,uBAAuB;YAChC,WAAW,EAAE,IAAI;SAClB,CAAC;IACJ,CAAC;IAED,qCAAqC;IACrC,IAAI,OAAO,EAAE,QAAQ,KAAK,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,4BAA4B,CAAC,EAAE,CAAC;QAC1E,OAAO;YACL,IAAI,EAAE,WAAW,CAAC,gBAAgB;YAClC,MAAM,EAAE,cAAc,CAAC,IAAI,EAAE,wCAAwC;YACrE,OAAO,EAAE,+CAA+C;YACxD,WAAW,EAAE,KAAK;SACnB,CAAC;IACJ,CAAC;IAED,yBAAyB;IACzB,IAAI,GAAG,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;QAC5E,MAAM,mBAAmB,GAAG,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACpD,OAAO;YACL,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC,WAAW,CAAC,kBAAkB,CAAC,CAAC,CAAC,WAAW,CAAC,iBAAiB;YAC1F,MAAM,EAAE,cAAc,CAAC,UAAU;YACjC,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC,CAAC,+BAA+B;YAC7F,WAAW,EAAE,KAAK;SACnB,CAAC;IACJ,CAAC;IAED,sBAAsB;IACtB,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;QAC3G,OAAO;YACL,IAAI,EAAE,WAAW,CAAC,gBAAgB;YAClC,MAAM,EAAE,cAAc,CAAC,4BAA4B;YACnD,OAAO,EAAE,iCAAiC;YAC1C,WAAW,EAAE,KAAK;SACnB,CAAC;IACJ,CAAC;IAED,gCAAgC;IAChC,IAAI,OAAO,EAAE,UAAU,KAAK,SAAS,IAAI,OAAO,CAAC,UAAU,IAAI,CAAC,EAAE,CAAC;QACjE,OAAO,YAAY,CAAC;YAClB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,CAAC;YACnC,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,eAAe,EAAE,OAAO,CAAC,eAAe;SACzC,CAAC,CAAC;IACL,CAAC;IAED,uBAAuB;IACvB,OAAO;QACL,IAAI,EAAE,WAAW,CAAC,aAAa;QAC/B,MAAM,EAAE,cAAc,CAAC,UAAU;QACjC,OAAO,EAAE,KAAK,IAAI,4BAA4B,OAAO,EAAE,QAAQ,EAAE;QACjE,WAAW,EAAE,KAAK;KACnB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAgB,UAAU,CAAC,QAAgB,EAAE,QAAyB,EAAE,iBAAsB,MAAM;IAClG,MAAM,KAAK,GAAG,IAAI,QAAQ,GAAG,CAAC;IAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,KAAK,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,QAAQ,CAAC,MAAM,EAAE,CAAC;IACpG,MAAM,UAAU,GAAG,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAEjH,MAAM,OAAO,GAAG,GAAG,KAAK,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,CAAC,OAAO,GAAG,WAAW,GAAG,UAAU,EAAE,CAAC;IAE5F,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;QACzB,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;SAAM,CAAC;QACN,cAAc,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;IAED,yBAAyB;IACzB,IAAI,QAAQ,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7C,cAAc,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,iBAAiB,CAAC,QAAyB,EAAE,UAAkB;IAC7E,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;QAC9B,OAAO,QAAQ,CAAC,gBAAgB,CAAC;IACnC,CAAC;IAED,sBAAsB;IACtB,MAAM,SAAS,GAAG,IAAI,CAAC;IACvB,MAAM,QAAQ,GAAG,KAAK,CAAC;IAEvB,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,QAAQ,CAAC,CAAC;AACjE,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,SAAS,CAC7B,QAAgB,EAChB,EAAoB,EACpB,OAAoD,EACpD,UAII,EAAE;IAEN,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC;IAC3C,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC;IAC5C,IAAI,OAAO,GAAG,CAAC,CAAC;IAEhB,mCAAmC;IACnC,MAAM,OAAO,GAAG,OAAO,CAAC,kBAAkB;QACxC,CAAC,CAAC,IAAA,yBAAiB,EAAC,OAAO,CAAC,kBAAkB,CAAC;QAC/C,CAAC,CAAC,IAAI,CAAC;IAET,OAAO,IAAI,EAAE,CAAC;QACZ,wBAAwB;QACxB,IAAI,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,IAAI,QAAQ,mCAAmC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9F,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;YAC5D,SAAS;QACX,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,EAAE,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QAE/B,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;YACd,IAAI,OAAO;gBAAE,OAAO,CAAC,aAAa,EAAE,CAAC;YACrC,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,IAAI,OAAO;YAAE,OAAO,CAAC,aAAa,EAAE,CAAC;QAErC,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE;YAC5C,UAAU,EAAE,OAAO;YACnB,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;SAC/C,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,cAAc,CAAC,UAAU;YAC7C,QAAQ,CAAC,MAAM,KAAK,cAAc,CAAC,cAAc,CAAC;YACnD,OAAO,GAAG,UAAU,EAAE,CAAC;YACzB,OAAO,EAAE,CAAC;YACV,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAE/B,MAAM,KAAK,GAAG,iBAAiB,CAAC,QAAQ,EAAE,OAAO,CAAC,IAAI,WAAW,CAAC;YAClE,MAAM,CAAC,IAAI,CAAC,WAAW,OAAO,IAAI,UAAU,wBAAwB,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;YAEpG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YACzD,SAAS;QACX,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;AACH,CAAC;AAcD,SAAgB,mBAAmB,CACjC,QAAgB,EAChB,QAAyB,EACzB,OAAuB;IAEvB,OAAO;QACL,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,QAAQ;QACR,QAAQ;QACR,OAAO;QACP,QAAQ,EAAE,KAAK;KAChB,CAAC;AACJ,CAAC;AAaD,SAAgB,gBAAgB;IAC9B,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,MAAM,EAAE,EAAiC;QACzC,QAAQ,EAAE,EAAoC;QAC9C,cAAc,EAAE,CAAC;QACjB,cAAc,EAAE,CAAC;KAClB,CAAC;AACJ,CAAC;AAED,SAAgB,WAAW,CAAC,KAAmB,EAAE,QAAyB;IACxE,KAAK,CAAC,aAAa,EAAE,CAAC;IACtB,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACrE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAE7E,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;QACzB,KAAK,CAAC,cAAc,EAAE,CAAC;IACzB,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,cAAc,EAAE,CAAC;IACzB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -1,16 +1,25 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Orchestrator - Parallel lane execution with dependency management
|
|
3
3
|
*
|
|
4
|
-
*
|
|
4
|
+
* Features:
|
|
5
|
+
* - Multi-layer stall detection
|
|
6
|
+
* - Cyclic dependency detection
|
|
7
|
+
* - Enhanced recovery strategies
|
|
8
|
+
* - Health checks before start
|
|
5
9
|
*/
|
|
6
10
|
import { ChildProcess } from 'child_process';
|
|
7
11
|
import { WebhookConfig, EnhancedLogConfig } from '../utils/types';
|
|
8
12
|
import { EnhancedLogManager } from '../utils/enhanced-logger';
|
|
13
|
+
import { StallDetectionConfig } from './failure-policy';
|
|
14
|
+
import { AutoRecoveryConfig } from './auto-recovery';
|
|
9
15
|
export interface LaneInfo {
|
|
10
16
|
name: string;
|
|
11
17
|
path: string;
|
|
12
18
|
dependsOn: string[];
|
|
13
19
|
startIndex?: number;
|
|
20
|
+
restartCount?: number;
|
|
21
|
+
lastStateUpdate?: number;
|
|
22
|
+
taskStartTime?: number;
|
|
14
23
|
}
|
|
15
24
|
export interface SpawnLaneResult {
|
|
16
25
|
child: ChildProcess;
|
|
@@ -20,7 +29,7 @@ export interface SpawnLaneResult {
|
|
|
20
29
|
/**
|
|
21
30
|
* Spawn a lane process
|
|
22
31
|
*/
|
|
23
|
-
export declare function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex, pipelineBranch, worktreeDir, enhancedLogConfig, noGit, }: {
|
|
32
|
+
export declare function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex, pipelineBranch, worktreeDir, enhancedLogConfig, noGit, onActivity, }: {
|
|
24
33
|
laneName: string;
|
|
25
34
|
tasksFile: string;
|
|
26
35
|
laneRunDir: string;
|
|
@@ -30,6 +39,7 @@ export declare function spawnLane({ laneName, tasksFile, laneRunDir, executor, s
|
|
|
30
39
|
worktreeDir?: string;
|
|
31
40
|
enhancedLogConfig?: Partial<EnhancedLogConfig>;
|
|
32
41
|
noGit?: boolean;
|
|
42
|
+
onActivity?: () => void;
|
|
33
43
|
}): SpawnLaneResult;
|
|
34
44
|
/**
|
|
35
45
|
* Wait for child process to exit
|
|
@@ -55,6 +65,9 @@ export declare function orchestrate(tasksDir: string, options?: {
|
|
|
55
65
|
autoResolveDependencies?: boolean;
|
|
56
66
|
enhancedLogging?: Partial<EnhancedLogConfig>;
|
|
57
67
|
noGit?: boolean;
|
|
68
|
+
skipPreflight?: boolean;
|
|
69
|
+
stallConfig?: Partial<StallDetectionConfig>;
|
|
70
|
+
autoRecoveryConfig?: Partial<AutoRecoveryConfig>;
|
|
58
71
|
}): Promise<{
|
|
59
72
|
lanes: LaneInfo[];
|
|
60
73
|
exitCodes: Record<string, number>;
|
|
@@ -2,7 +2,11 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* Orchestrator - Parallel lane execution with dependency management
|
|
4
4
|
*
|
|
5
|
-
*
|
|
5
|
+
* Features:
|
|
6
|
+
* - Multi-layer stall detection
|
|
7
|
+
* - Cyclic dependency detection
|
|
8
|
+
* - Enhanced recovery strategies
|
|
9
|
+
* - Health checks before start
|
|
6
10
|
*/
|
|
7
11
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
8
12
|
if (k2 === undefined) k2 = k;
|
|
@@ -56,10 +60,23 @@ const child_process_2 = require("child_process");
|
|
|
56
60
|
const path_1 = require("../utils/path");
|
|
57
61
|
const enhanced_logger_1 = require("../utils/enhanced-logger");
|
|
58
62
|
const log_formatter_1 = require("../utils/log-formatter");
|
|
63
|
+
const failure_policy_1 = require("./failure-policy");
|
|
64
|
+
const auto_recovery_1 = require("./auto-recovery");
|
|
65
|
+
const dependency_1 = require("../utils/dependency");
|
|
66
|
+
const health_1 = require("../utils/health");
|
|
67
|
+
const checkpoint_1 = require("../utils/checkpoint");
|
|
68
|
+
const lock_1 = require("../utils/lock");
|
|
69
|
+
/** Default stall detection configuration - 2 minute idle timeout for recovery */
|
|
70
|
+
const DEFAULT_ORCHESTRATOR_STALL_CONFIG = {
|
|
71
|
+
...failure_policy_1.DEFAULT_STALL_CONFIG,
|
|
72
|
+
idleTimeoutMs: 2 * 60 * 1000, // 2 minutes (idle detection for continue signal)
|
|
73
|
+
progressTimeoutMs: 10 * 60 * 1000, // 10 minutes (only triggers if no activity at all)
|
|
74
|
+
maxRestarts: 2,
|
|
75
|
+
};
|
|
59
76
|
/**
|
|
60
77
|
* Spawn a lane process
|
|
61
78
|
*/
|
|
62
|
-
function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0, pipelineBranch, worktreeDir, enhancedLogConfig, noGit = false, }) {
|
|
79
|
+
function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0, pipelineBranch, worktreeDir, enhancedLogConfig, noGit = false, onActivity, }) {
|
|
63
80
|
fs.mkdirSync(laneRunDir, { recursive: true });
|
|
64
81
|
// Use extension-less resolve to handle both .ts (dev) and .js (dist)
|
|
65
82
|
const runnerPath = require.resolve('./runner');
|
|
@@ -91,6 +108,8 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
|
|
|
91
108
|
if (logConfig.enabled) {
|
|
92
109
|
// Create callback for clean console output
|
|
93
110
|
const onParsedMessage = (msg) => {
|
|
111
|
+
if (onActivity)
|
|
112
|
+
onActivity();
|
|
94
113
|
const formatted = (0, log_formatter_1.formatMessageForConsole)(msg, {
|
|
95
114
|
laneLabel: `[${laneName}]`,
|
|
96
115
|
includeTimestamp: true
|
|
@@ -118,12 +137,23 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
|
|
|
118
137
|
lineBuffer = lines.pop() || '';
|
|
119
138
|
for (const line of lines) {
|
|
120
139
|
const trimmed = line.trim();
|
|
121
|
-
//
|
|
122
|
-
if
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
140
|
+
// Show if it's a timestamped log line (starts with [YYYY-MM-DD... or [HH:MM:SS])
|
|
141
|
+
// or if it's NOT a noisy JSON line
|
|
142
|
+
const hasTimestamp = /^\[\d{4}-\d{2}-\d{2}T|\^\[\d{2}:\d{2}:\d{2}\]/.test(trimmed);
|
|
143
|
+
const isJson = trimmed.startsWith('{') || trimmed.includes('{"type"');
|
|
144
|
+
if (trimmed && !isJson) {
|
|
145
|
+
if (onActivity)
|
|
146
|
+
onActivity();
|
|
147
|
+
// If line alreedy has timestamp format, just add lane prefix
|
|
148
|
+
if (hasTimestamp) {
|
|
149
|
+
// Insert lane name after first timestamp
|
|
150
|
+
const formatted = trimmed.replace(/^(\[[^\]]+\])/, `$1 ${logger.COLORS.magenta}[${laneName}]${logger.COLORS.reset}`);
|
|
151
|
+
process.stdout.write(formatted + '\n');
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
// Add full prefix: timestamp + lane
|
|
155
|
+
process.stdout.write(`${logger.COLORS.gray}[${new Date().toLocaleTimeString('en-US', { hour12: false })}]${logger.COLORS.reset} ${logger.COLORS.magenta}[${laneName}]${logger.COLORS.reset} ${line}\n`);
|
|
156
|
+
}
|
|
127
157
|
}
|
|
128
158
|
}
|
|
129
159
|
});
|
|
@@ -141,11 +171,14 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
|
|
|
141
171
|
trimmed.startsWith('Switched to a new branch') ||
|
|
142
172
|
trimmed.startsWith('HEAD is now at') ||
|
|
143
173
|
trimmed.includes('actual output');
|
|
174
|
+
const ts = new Date().toLocaleTimeString('en-US', { hour12: false });
|
|
144
175
|
if (isStatus) {
|
|
145
|
-
process.stdout.write(`${logger.COLORS.gray}[${
|
|
176
|
+
process.stdout.write(`${logger.COLORS.gray}[${ts}]${logger.COLORS.reset} ${logger.COLORS.magenta}[${laneName}]${logger.COLORS.reset} ${trimmed}\n`);
|
|
146
177
|
}
|
|
147
178
|
else {
|
|
148
|
-
|
|
179
|
+
if (onActivity)
|
|
180
|
+
onActivity();
|
|
181
|
+
process.stderr.write(`${logger.COLORS.gray}[${ts}]${logger.COLORS.reset} ${logger.COLORS.magenta}[${laneName}]${logger.COLORS.reset} ${logger.COLORS.red}❌ ERR ${trimmed}${logger.COLORS.reset}\n`);
|
|
149
182
|
}
|
|
150
183
|
}
|
|
151
184
|
}
|
|
@@ -266,7 +299,7 @@ async function resolveAllDependencies(blockedLanes, allLanes, laneRunDirs, pipel
|
|
|
266
299
|
const worktreeDir = state?.worktreeDir || (0, path_1.safeJoin)(runRoot, 'resolution-worktree');
|
|
267
300
|
if (!fs.existsSync(worktreeDir)) {
|
|
268
301
|
logger.info(`Creating resolution worktree at ${worktreeDir}`);
|
|
269
|
-
git.createWorktree(worktreeDir, pipelineBranch, { baseBranch:
|
|
302
|
+
git.createWorktree(worktreeDir, pipelineBranch, { baseBranch: git.getCurrentBranch() });
|
|
270
303
|
}
|
|
271
304
|
// 3. Resolve on pipeline branch
|
|
272
305
|
logger.info(`Resolving dependencies on ${pipelineBranch}`);
|
|
@@ -336,6 +369,50 @@ async function orchestrate(tasksDir, options = {}) {
|
|
|
336
369
|
if (lanes.length === 0) {
|
|
337
370
|
throw new Error(`No lane task files found in ${tasksDir}`);
|
|
338
371
|
}
|
|
372
|
+
// Run preflight checks
|
|
373
|
+
if (!options.skipPreflight) {
|
|
374
|
+
logger.section('🔍 Preflight Checks');
|
|
375
|
+
const preflight = await (0, health_1.preflightCheck)({
|
|
376
|
+
requireRemote: !options.noGit,
|
|
377
|
+
requireAuth: true,
|
|
378
|
+
});
|
|
379
|
+
if (!preflight.canProceed) {
|
|
380
|
+
(0, health_1.printPreflightReport)(preflight);
|
|
381
|
+
throw new Error('Preflight check failed. Please fix the blockers above.');
|
|
382
|
+
}
|
|
383
|
+
// Auto-repair if there are warnings
|
|
384
|
+
if (preflight.warnings.length > 0) {
|
|
385
|
+
logger.info('Attempting auto-repair...');
|
|
386
|
+
const repair = await (0, health_1.autoRepair)();
|
|
387
|
+
if (repair.repaired.length > 0) {
|
|
388
|
+
for (const r of repair.repaired) {
|
|
389
|
+
logger.success(`✓ ${r}`);
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
logger.success('✓ Preflight checks passed');
|
|
394
|
+
}
|
|
395
|
+
// Validate dependencies and detect cycles
|
|
396
|
+
logger.section('📊 Dependency Analysis');
|
|
397
|
+
const depInfos = lanes.map(l => ({
|
|
398
|
+
name: l.name,
|
|
399
|
+
dependsOn: l.dependsOn,
|
|
400
|
+
}));
|
|
401
|
+
const depValidation = (0, dependency_1.validateDependencies)(depInfos);
|
|
402
|
+
if (!depValidation.valid) {
|
|
403
|
+
logger.error('❌ Dependency validation failed:');
|
|
404
|
+
for (const err of depValidation.errors) {
|
|
405
|
+
logger.error(` • ${err}`);
|
|
406
|
+
}
|
|
407
|
+
throw new Error('Invalid dependency configuration');
|
|
408
|
+
}
|
|
409
|
+
if (depValidation.warnings.length > 0) {
|
|
410
|
+
for (const warn of depValidation.warnings) {
|
|
411
|
+
logger.warn(`⚠️ ${warn}`);
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
// Print dependency graph
|
|
415
|
+
(0, dependency_1.printDependencyGraph)(depInfos);
|
|
339
416
|
const config = (0, config_1.loadConfig)();
|
|
340
417
|
const logsDir = (0, config_1.getLogsDir)(config);
|
|
341
418
|
const runId = `run-${Date.now()}`;
|
|
@@ -344,8 +421,30 @@ async function orchestrate(tasksDir, options = {}) {
|
|
|
344
421
|
? (path.isAbsolute(options.runDir) ? options.runDir : path.resolve(process.cwd(), options.runDir)) // nosemgrep
|
|
345
422
|
: (0, path_1.safeJoin)(logsDir, 'runs', runId);
|
|
346
423
|
fs.mkdirSync(runRoot, { recursive: true });
|
|
424
|
+
// Clean stale locks before starting
|
|
425
|
+
try {
|
|
426
|
+
const lockDir = (0, lock_1.getLockDir)(git.getRepoRoot());
|
|
427
|
+
const cleaned = (0, lock_1.cleanStaleLocks)(lockDir);
|
|
428
|
+
if (cleaned > 0) {
|
|
429
|
+
logger.info(`Cleaned ${cleaned} stale lock(s)`);
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
catch {
|
|
433
|
+
// Ignore lock cleanup errors
|
|
434
|
+
}
|
|
347
435
|
const randomSuffix = Math.random().toString(36).substring(2, 7);
|
|
348
436
|
const pipelineBranch = `cursorflow/run-${Date.now().toString(36)}-${randomSuffix}`;
|
|
437
|
+
// Stall detection configuration
|
|
438
|
+
const stallConfig = {
|
|
439
|
+
...DEFAULT_ORCHESTRATOR_STALL_CONFIG,
|
|
440
|
+
...options.stallConfig,
|
|
441
|
+
};
|
|
442
|
+
// Initialize auto-recovery manager
|
|
443
|
+
const autoRecoveryManager = (0, auto_recovery_1.getAutoRecoveryManager)({
|
|
444
|
+
...auto_recovery_1.DEFAULT_AUTO_RECOVERY_CONFIG,
|
|
445
|
+
idleTimeoutMs: stallConfig.idleTimeoutMs, // Sync with stall config
|
|
446
|
+
...options.autoRecoveryConfig,
|
|
447
|
+
});
|
|
349
448
|
// Initialize event system
|
|
350
449
|
events_1.events.setRunId(runId);
|
|
351
450
|
if (options.webhooks) {
|
|
@@ -366,6 +465,7 @@ async function orchestrate(tasksDir, options = {}) {
|
|
|
366
465
|
// Track start index for each lane (initially 0)
|
|
367
466
|
for (const lane of lanes) {
|
|
368
467
|
lane.startIndex = 0;
|
|
468
|
+
lane.restartCount = 0;
|
|
369
469
|
}
|
|
370
470
|
const laneRunDirs = {};
|
|
371
471
|
const laneWorktreeDirs = {};
|
|
@@ -453,7 +553,14 @@ async function orchestrate(tasksDir, options = {}) {
|
|
|
453
553
|
for (const lane of readyToStart) {
|
|
454
554
|
if (running.size >= maxConcurrent)
|
|
455
555
|
break;
|
|
556
|
+
const laneStatePath = (0, path_1.safeJoin)(laneRunDirs[lane.name], 'state.json');
|
|
557
|
+
// Validate and repair state before starting
|
|
558
|
+
const validation = (0, state_1.validateLaneState)(laneStatePath, { autoRepair: true });
|
|
559
|
+
if (!validation.valid && !validation.repaired) {
|
|
560
|
+
logger.warn(`[${lane.name}] State validation issues: ${validation.issues.join(', ')}`);
|
|
561
|
+
}
|
|
456
562
|
logger.info(`Lane started: ${lane.name}${lane.startIndex ? ` (resuming from ${lane.startIndex})` : ''}`);
|
|
563
|
+
let lastOutput = '';
|
|
457
564
|
const spawnResult = spawnLane({
|
|
458
565
|
laneName: lane.name,
|
|
459
566
|
tasksFile: lane.path,
|
|
@@ -464,23 +571,282 @@ async function orchestrate(tasksDir, options = {}) {
|
|
|
464
571
|
worktreeDir: laneWorktreeDirs[lane.name],
|
|
465
572
|
enhancedLogConfig: options.enhancedLogging,
|
|
466
573
|
noGit: options.noGit,
|
|
574
|
+
onActivity: () => {
|
|
575
|
+
const info = running.get(lane.name);
|
|
576
|
+
if (info) {
|
|
577
|
+
const now = Date.now();
|
|
578
|
+
info.lastActivity = now;
|
|
579
|
+
// Also reset progress tracking when there's activity (THNK/TOOL events)
|
|
580
|
+
// This prevents STALL_NO_PROGRESS from firing when agent is actively working
|
|
581
|
+
info.lastStateUpdate = now;
|
|
582
|
+
info.stallPhase = 0; // Reset stall phase since agent is responding
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
});
|
|
586
|
+
// Track last output and bytes received for long operation and stall detection
|
|
587
|
+
if (spawnResult.child.stdout) {
|
|
588
|
+
spawnResult.child.stdout.on('data', (data) => {
|
|
589
|
+
const info = running.get(lane.name);
|
|
590
|
+
if (info) {
|
|
591
|
+
info.lastOutput = data.toString().trim().split('\n').pop() || '';
|
|
592
|
+
info.bytesReceived += data.length;
|
|
593
|
+
// Update auto-recovery manager
|
|
594
|
+
autoRecoveryManager.recordActivity(lane.name, data.length, info.lastOutput);
|
|
595
|
+
}
|
|
596
|
+
});
|
|
597
|
+
}
|
|
598
|
+
const now = Date.now();
|
|
599
|
+
running.set(lane.name, {
|
|
600
|
+
...spawnResult,
|
|
601
|
+
lastActivity: now,
|
|
602
|
+
lastStateUpdate: now,
|
|
603
|
+
stallPhase: 0,
|
|
604
|
+
taskStartTime: now,
|
|
605
|
+
lastOutput: '',
|
|
606
|
+
statePath: laneStatePath,
|
|
607
|
+
bytesReceived: 0,
|
|
608
|
+
lastBytesCheck: 0,
|
|
609
|
+
continueSignalsSent: 0,
|
|
467
610
|
});
|
|
468
|
-
|
|
611
|
+
// Register lane with auto-recovery manager
|
|
612
|
+
autoRecoveryManager.registerLane(lane.name);
|
|
613
|
+
// Update lane tracking
|
|
614
|
+
lane.taskStartTime = now;
|
|
469
615
|
events_1.events.emit('lane.started', {
|
|
470
616
|
laneName: lane.name,
|
|
471
617
|
pid: spawnResult.child.pid,
|
|
472
618
|
logPath: spawnResult.logPath,
|
|
473
619
|
});
|
|
474
620
|
}
|
|
475
|
-
// 3. Wait for any running lane to finish
|
|
621
|
+
// 3. Wait for any running lane to finish OR check for stalls
|
|
476
622
|
if (running.size > 0) {
|
|
623
|
+
// Polling timeout for stall detection
|
|
624
|
+
let pollTimeout;
|
|
625
|
+
const pollPromise = new Promise(resolve => {
|
|
626
|
+
pollTimeout = setTimeout(() => resolve({ name: '__poll__', code: 0 }), 10000);
|
|
627
|
+
});
|
|
477
628
|
const promises = Array.from(running.entries()).map(async ([name, { child }]) => {
|
|
478
629
|
const code = await waitChild(child);
|
|
479
630
|
return { name, code };
|
|
480
631
|
});
|
|
481
|
-
const
|
|
632
|
+
const result = await Promise.race([...promises, pollPromise]);
|
|
633
|
+
if (pollTimeout)
|
|
634
|
+
clearTimeout(pollTimeout);
|
|
635
|
+
if (result.name === '__poll__') {
|
|
636
|
+
// Periodic stall check with multi-layer detection and escalating recovery
|
|
637
|
+
for (const [laneName, info] of running.entries()) {
|
|
638
|
+
const now = Date.now();
|
|
639
|
+
const idleTime = now - info.lastActivity;
|
|
640
|
+
const lane = lanes.find(l => l.name === laneName);
|
|
641
|
+
// Check state file for progress updates
|
|
642
|
+
let progressTime = 0;
|
|
643
|
+
try {
|
|
644
|
+
const stateStat = fs.statSync(info.statePath);
|
|
645
|
+
const stateUpdateTime = stateStat.mtimeMs;
|
|
646
|
+
if (stateUpdateTime > info.lastStateUpdate) {
|
|
647
|
+
info.lastStateUpdate = stateUpdateTime;
|
|
648
|
+
}
|
|
649
|
+
progressTime = now - info.lastStateUpdate;
|
|
650
|
+
}
|
|
651
|
+
catch {
|
|
652
|
+
// State file might not exist yet
|
|
653
|
+
}
|
|
654
|
+
// Calculate bytes received since last check
|
|
655
|
+
const bytesDelta = info.bytesReceived - info.lastBytesCheck;
|
|
656
|
+
info.lastBytesCheck = info.bytesReceived;
|
|
657
|
+
// Use multi-layer stall analysis with enhanced context
|
|
658
|
+
const analysis = (0, failure_policy_1.analyzeStall)({
|
|
659
|
+
stallPhase: info.stallPhase,
|
|
660
|
+
idleTimeMs: idleTime,
|
|
661
|
+
progressTimeMs: progressTime,
|
|
662
|
+
lastOutput: info.lastOutput,
|
|
663
|
+
restartCount: lane.restartCount || 0,
|
|
664
|
+
taskStartTimeMs: info.taskStartTime,
|
|
665
|
+
bytesReceived: bytesDelta, // Bytes since last check
|
|
666
|
+
continueSignalsSent: info.continueSignalsSent,
|
|
667
|
+
}, stallConfig);
|
|
668
|
+
// Only act if action is not NONE
|
|
669
|
+
if (analysis.action !== failure_policy_1.RecoveryAction.NONE) {
|
|
670
|
+
(0, failure_policy_1.logFailure)(laneName, analysis);
|
|
671
|
+
info.logManager?.log('error', analysis.message);
|
|
672
|
+
if (analysis.action === failure_policy_1.RecoveryAction.CONTINUE_SIGNAL) {
|
|
673
|
+
const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
|
|
674
|
+
try {
|
|
675
|
+
fs.writeFileSync(interventionPath, 'continue');
|
|
676
|
+
info.stallPhase = 1;
|
|
677
|
+
info.lastActivity = now;
|
|
678
|
+
info.continueSignalsSent++;
|
|
679
|
+
logger.info(`[${laneName}] Sent continue signal (#${info.continueSignalsSent})`);
|
|
680
|
+
events_1.events.emit('recovery.continue_signal', {
|
|
681
|
+
laneName,
|
|
682
|
+
idleSeconds: Math.round(idleTime / 1000),
|
|
683
|
+
signalCount: info.continueSignalsSent,
|
|
684
|
+
});
|
|
685
|
+
}
|
|
686
|
+
catch (e) {
|
|
687
|
+
logger.error(`Failed to write intervention file for ${laneName}: ${e}`);
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
else if (analysis.action === failure_policy_1.RecoveryAction.STRONGER_PROMPT) {
|
|
691
|
+
const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
|
|
692
|
+
const strongerPrompt = `[SYSTEM INTERVENTION] You seem to be stuck. Please continue with your current task immediately. If you're waiting for something, explain what you need and proceed with what you can do now. If you've completed the task, summarize your work and finish.`;
|
|
693
|
+
try {
|
|
694
|
+
fs.writeFileSync(interventionPath, strongerPrompt);
|
|
695
|
+
info.stallPhase = 2;
|
|
696
|
+
info.lastActivity = now;
|
|
697
|
+
logger.warn(`[${laneName}] Sent stronger prompt after continue signal failed`);
|
|
698
|
+
events_1.events.emit('recovery.stronger_prompt', { laneName });
|
|
699
|
+
}
|
|
700
|
+
catch (e) {
|
|
701
|
+
logger.error(`Failed to write intervention file for ${laneName}: ${e}`);
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
else if (analysis.action === failure_policy_1.RecoveryAction.KILL_AND_RESTART ||
|
|
705
|
+
analysis.action === failure_policy_1.RecoveryAction.RESTART_LANE ||
|
|
706
|
+
analysis.action === failure_policy_1.RecoveryAction.RESTART_LANE_FROM_CHECKPOINT) {
|
|
707
|
+
lane.restartCount = (lane.restartCount || 0) + 1;
|
|
708
|
+
info.stallPhase = 3;
|
|
709
|
+
// Try to get checkpoint info
|
|
710
|
+
const checkpoint = (0, checkpoint_1.getLatestCheckpoint)(laneRunDirs[laneName]);
|
|
711
|
+
if (checkpoint) {
|
|
712
|
+
logger.info(`[${laneName}] Checkpoint available: ${checkpoint.id} (task ${checkpoint.taskIndex})`);
|
|
713
|
+
}
|
|
714
|
+
// Kill the process
|
|
715
|
+
try {
|
|
716
|
+
info.child.kill('SIGKILL');
|
|
717
|
+
}
|
|
718
|
+
catch {
|
|
719
|
+
// Process might already be dead
|
|
720
|
+
}
|
|
721
|
+
logger.warn(`[${laneName}] Killing and restarting lane (restart #${lane.restartCount})`);
|
|
722
|
+
events_1.events.emit('recovery.restart', {
|
|
723
|
+
laneName,
|
|
724
|
+
restartCount: lane.restartCount,
|
|
725
|
+
maxRestarts: stallConfig.maxRestarts,
|
|
726
|
+
});
|
|
727
|
+
}
|
|
728
|
+
else if (analysis.action === failure_policy_1.RecoveryAction.RUN_DOCTOR) {
|
|
729
|
+
info.stallPhase = 4;
|
|
730
|
+
// Run diagnostics
|
|
731
|
+
logger.error(`[${laneName}] Running diagnostics due to persistent failures...`);
|
|
732
|
+
// Import health check dynamically to avoid circular dependency
|
|
733
|
+
const { checkAgentHealth, checkAuthHealth } = await Promise.resolve().then(() => __importStar(require('../utils/health')));
|
|
734
|
+
const [agentHealth, authHealth] = await Promise.all([
|
|
735
|
+
checkAgentHealth(),
|
|
736
|
+
checkAuthHealth(),
|
|
737
|
+
]);
|
|
738
|
+
const issues = [];
|
|
739
|
+
if (!agentHealth.ok)
|
|
740
|
+
issues.push(`Agent: ${agentHealth.message}`);
|
|
741
|
+
if (!authHealth.ok)
|
|
742
|
+
issues.push(`Auth: ${authHealth.message}`);
|
|
743
|
+
if (issues.length > 0) {
|
|
744
|
+
logger.error(`[${laneName}] Diagnostic issues found:\n ${issues.join('\n ')}`);
|
|
745
|
+
}
|
|
746
|
+
else {
|
|
747
|
+
logger.warn(`[${laneName}] No obvious issues found. The problem may be with the AI model or network.`);
|
|
748
|
+
}
|
|
749
|
+
// Save diagnostic to file
|
|
750
|
+
const diagnosticPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'diagnostic.json');
|
|
751
|
+
fs.writeFileSync(diagnosticPath, JSON.stringify({
|
|
752
|
+
timestamp: Date.now(),
|
|
753
|
+
agentHealthy: agentHealth.ok,
|
|
754
|
+
authHealthy: authHealth.ok,
|
|
755
|
+
issues,
|
|
756
|
+
analysis,
|
|
757
|
+
}, null, 2));
|
|
758
|
+
// Kill the process
|
|
759
|
+
try {
|
|
760
|
+
info.child.kill('SIGKILL');
|
|
761
|
+
}
|
|
762
|
+
catch {
|
|
763
|
+
// Process might already be dead
|
|
764
|
+
}
|
|
765
|
+
logger.error(`[${laneName}] Aborting lane after diagnostic. Check ${diagnosticPath} for details.`);
|
|
766
|
+
// Save POF for failed recovery
|
|
767
|
+
const recoveryState = autoRecoveryManager.getState(laneName);
|
|
768
|
+
if (recoveryState) {
|
|
769
|
+
try {
|
|
770
|
+
const laneStatePath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'state.json');
|
|
771
|
+
const laneState = (0, state_1.loadState)(laneStatePath);
|
|
772
|
+
const pofDir = (0, path_1.safeJoin)(runRoot, '..', '..', 'pof');
|
|
773
|
+
const diagnosticInfo = {
|
|
774
|
+
timestamp: Date.now(),
|
|
775
|
+
agentHealthy: agentHealth.ok,
|
|
776
|
+
authHealthy: authHealth.ok,
|
|
777
|
+
systemHealthy: true,
|
|
778
|
+
suggestedAction: issues.length > 0 ? 'Fix the issues above and retry' : 'Try with a different model',
|
|
779
|
+
details: issues.join('\n') || 'No obvious issues found',
|
|
780
|
+
};
|
|
781
|
+
const pofEntry = (0, auto_recovery_1.createPOFFromRecoveryState)(runId, runRoot, laneName, recoveryState, laneState, diagnosticInfo);
|
|
782
|
+
(0, auto_recovery_1.savePOF)(runId, pofDir, pofEntry);
|
|
783
|
+
}
|
|
784
|
+
catch (pofError) {
|
|
785
|
+
logger.warn(`[${laneName}] Failed to save POF: ${pofError.message}`);
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
events_1.events.emit('recovery.diagnosed', {
|
|
789
|
+
laneName,
|
|
790
|
+
diagnostic: { agentHealthy: agentHealth.ok, authHealthy: authHealth.ok, issues },
|
|
791
|
+
});
|
|
792
|
+
}
|
|
793
|
+
else if (analysis.action === failure_policy_1.RecoveryAction.ABORT_LANE) {
|
|
794
|
+
info.stallPhase = 5;
|
|
795
|
+
try {
|
|
796
|
+
info.child.kill('SIGKILL');
|
|
797
|
+
}
|
|
798
|
+
catch {
|
|
799
|
+
// Process might already be dead
|
|
800
|
+
}
|
|
801
|
+
logger.error(`[${laneName}] Aborting lane due to repeated stalls`);
|
|
802
|
+
// Save POF for failed recovery
|
|
803
|
+
const recoveryState = autoRecoveryManager.getState(laneName);
|
|
804
|
+
if (recoveryState) {
|
|
805
|
+
try {
|
|
806
|
+
const laneStatePath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'state.json');
|
|
807
|
+
const laneState = (0, state_1.loadState)(laneStatePath);
|
|
808
|
+
const pofDir = (0, path_1.safeJoin)(runRoot, '..', '..', 'pof');
|
|
809
|
+
const pofEntry = (0, auto_recovery_1.createPOFFromRecoveryState)(runId, runRoot, laneName, recoveryState, laneState, recoveryState.diagnosticInfo);
|
|
810
|
+
(0, auto_recovery_1.savePOF)(runId, pofDir, pofEntry);
|
|
811
|
+
}
|
|
812
|
+
catch (pofError) {
|
|
813
|
+
logger.warn(`[${laneName}] Failed to save POF: ${pofError.message}`);
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
else if (analysis.action === failure_policy_1.RecoveryAction.SEND_GIT_GUIDANCE) {
|
|
818
|
+
// Send guidance message to agent for git issues
|
|
819
|
+
const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
|
|
820
|
+
// Determine which guidance to send based on the failure type
|
|
821
|
+
let guidance;
|
|
822
|
+
if (analysis.type === failure_policy_1.FailureType.GIT_PUSH_REJECTED) {
|
|
823
|
+
guidance = (0, auto_recovery_1.getGitPushFailureGuidance)();
|
|
824
|
+
}
|
|
825
|
+
else if (analysis.type === failure_policy_1.FailureType.MERGE_CONFLICT) {
|
|
826
|
+
guidance = (0, auto_recovery_1.getMergeConflictGuidance)();
|
|
827
|
+
}
|
|
828
|
+
else {
|
|
829
|
+
guidance = (0, auto_recovery_1.getGitErrorGuidance)(analysis.message);
|
|
830
|
+
}
|
|
831
|
+
try {
|
|
832
|
+
fs.writeFileSync(interventionPath, guidance);
|
|
833
|
+
info.lastActivity = now;
|
|
834
|
+
logger.info(`[${laneName}] Sent git issue guidance to agent`);
|
|
835
|
+
}
|
|
836
|
+
catch (e) {
|
|
837
|
+
logger.error(`[${laneName}] Failed to send guidance: ${e.message}`);
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
continue;
|
|
843
|
+
}
|
|
844
|
+
const finished = result;
|
|
845
|
+
const info = running.get(finished.name);
|
|
482
846
|
running.delete(finished.name);
|
|
483
847
|
exitCodes[finished.name] = finished.code;
|
|
848
|
+
// Unregister from auto-recovery manager
|
|
849
|
+
autoRecoveryManager.unregisterLane(finished.name);
|
|
484
850
|
if (finished.code === 0) {
|
|
485
851
|
completedLanes.add(finished.name);
|
|
486
852
|
events_1.events.emit('lane.completed', {
|
|
@@ -510,11 +876,27 @@ async function orchestrate(tasksDir, options = {}) {
|
|
|
510
876
|
}
|
|
511
877
|
}
|
|
512
878
|
else {
|
|
879
|
+
// Check if it was a restart request
|
|
880
|
+
if (info.stallPhase === 2) {
|
|
881
|
+
logger.info(`🔄 Lane ${finished.name} is being restarted due to stall...`);
|
|
882
|
+
// Update startIndex from current state to resume from the same task
|
|
883
|
+
const statePath = (0, path_1.safeJoin)(laneRunDirs[finished.name], 'state.json');
|
|
884
|
+
const state = (0, state_1.loadState)(statePath);
|
|
885
|
+
if (state) {
|
|
886
|
+
const lane = lanes.find(l => l.name === finished.name);
|
|
887
|
+
if (lane) {
|
|
888
|
+
lane.startIndex = state.currentTaskIndex;
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
// Note: we don't add to failedLanes or completedLanes,
|
|
892
|
+
// so it will be eligible to start again in the next iteration.
|
|
893
|
+
continue;
|
|
894
|
+
}
|
|
513
895
|
failedLanes.add(finished.name);
|
|
514
896
|
events_1.events.emit('lane.failed', {
|
|
515
897
|
laneName: finished.name,
|
|
516
898
|
exitCode: finished.code,
|
|
517
|
-
error: 'Process exited with non-zero code',
|
|
899
|
+
error: info.stallPhase === 3 ? 'Stopped due to repeated stall' : 'Process exited with non-zero code',
|
|
518
900
|
});
|
|
519
901
|
}
|
|
520
902
|
printLaneStatus(lanes, laneRunDirs);
|