@nocobase/app-plugin-agents 0.1.0-beta.11 → 0.1.0-beta.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/README.md +10 -2
- package/dist/client/api/client.d.ts +3 -2
- package/dist/client/api/client.d.ts.map +1 -1
- package/dist/client/api/client.js +3 -0
- package/dist/client/api/client.js.map +1 -1
- package/dist/client/api/runners.d.ts +2 -2
- package/dist/client/api/runners.d.ts.map +1 -1
- package/dist/client/api/runners.js.map +1 -1
- package/dist/client/components/variables/variable-dialogs.d.ts.map +1 -1
- package/dist/client/components/variables/variable-dialogs.js +7 -7
- package/dist/client/components/variables/variable-dialogs.js.map +1 -1
- package/dist/client/config.d.ts +2 -0
- package/dist/client/config.d.ts.map +1 -1
- package/dist/client/config.js +2 -0
- package/dist/client/config.js.map +1 -1
- package/dist/client/lib/skill-settings-context.d.ts +9 -0
- package/dist/client/lib/skill-settings-context.d.ts.map +1 -0
- package/dist/client/lib/skill-settings-context.js +4 -0
- package/dist/client/lib/skill-settings-context.js.map +1 -0
- package/dist/client/lib/skill-settings.d.ts +12 -0
- package/dist/client/lib/skill-settings.d.ts.map +1 -0
- package/dist/client/lib/skill-settings.js +37 -0
- package/dist/client/lib/skill-settings.js.map +1 -0
- package/dist/client/locales/en-US.d.ts +1 -0
- package/dist/client/locales/en-US.d.ts.map +1 -1
- package/dist/client/locales/en-US.js +1 -0
- package/dist/client/locales/en-US.js.map +1 -1
- package/dist/client/locales/skills.en-US.d.ts +6 -0
- package/dist/client/locales/skills.en-US.d.ts.map +1 -1
- package/dist/client/locales/skills.en-US.js +6 -0
- package/dist/client/locales/skills.en-US.js.map +1 -1
- package/dist/client/locales/skills.zh-CN.d.ts.map +1 -1
- package/dist/client/locales/skills.zh-CN.js +6 -0
- package/dist/client/locales/skills.zh-CN.js.map +1 -1
- package/dist/client/locales/zh-CN.d.ts.map +1 -1
- package/dist/client/locales/zh-CN.js +1 -0
- package/dist/client/locales/zh-CN.js.map +1 -1
- package/dist/client/pages/agents/agent-dialog.d.ts.map +1 -1
- package/dist/client/pages/agents/agent-dialog.js +4 -4
- package/dist/client/pages/agents/agent-dialog.js.map +1 -1
- package/dist/client/pages/runtimes/connect.js.map +1 -1
- package/dist/client/pages/runtimes/index.js.map +1 -1
- package/dist/client/pages/runtimes/runner-cells.d.ts +3 -3
- package/dist/client/pages/runtimes/runner-cells.d.ts.map +1 -1
- package/dist/client/pages/runtimes/runner-sheet.d.ts +2 -2
- package/dist/client/pages/runtimes/runner-sheet.d.ts.map +1 -1
- package/dist/client/pages/runtimes/runner-sheet.js +15 -1
- package/dist/client/pages/runtimes/runner-sheet.js.map +1 -1
- package/dist/client/pages/runtimes/workspace-diagnostics.d.ts +2 -2
- package/dist/client/pages/runtimes/workspace-diagnostics.d.ts.map +1 -1
- package/dist/client/pages/runtimes/workspace-diagnostics.js.map +1 -1
- package/dist/client/pages/skills/detail/index.d.ts.map +1 -1
- package/dist/client/pages/skills/detail/index.js +11 -12
- package/dist/client/pages/skills/detail/index.js.map +1 -1
- package/dist/package.json +1 -1
- package/dist/server/cli/surface.d.ts.map +1 -1
- package/dist/server/cli/surface.js +4 -1
- package/dist/server/cli/surface.js.map +1 -1
- package/dist/server/composition.d.ts +2 -0
- package/dist/server/composition.d.ts.map +1 -1
- package/dist/server/composition.js +10 -3
- package/dist/server/composition.js.map +1 -1
- package/dist/server/core/agents/agent.service.d.ts.map +1 -1
- package/dist/server/core/agents/agent.service.js +5 -1
- package/dist/server/core/agents/agent.service.js.map +1 -1
- package/dist/server/core/agents/agent.store.d.ts +2 -0
- package/dist/server/core/agents/agent.store.d.ts.map +1 -1
- package/dist/server/core/agents/agent.store.js +10 -0
- package/dist/server/core/agents/agent.store.js.map +1 -1
- package/dist/server/core/agents/index.d.ts +1 -1
- package/dist/server/core/agents/index.d.ts.map +1 -1
- package/dist/server/core/agents/index.js +1 -1
- package/dist/server/core/agents/index.js.map +1 -1
- package/dist/server/core/brief/brief.js +2 -2
- package/dist/server/core/brief/brief.js.map +1 -1
- package/dist/server/core/callers/index.d.ts +9 -0
- package/dist/server/core/callers/index.d.ts.map +1 -1
- package/dist/server/core/callers/index.js +15 -0
- package/dist/server/core/callers/index.js.map +1 -1
- package/dist/server/core/conversations/chat-settings.d.ts.map +1 -1
- package/dist/server/core/conversations/chat-settings.js +12 -2
- package/dist/server/core/conversations/chat-settings.js.map +1 -1
- package/dist/server/core/conversations/conversation.service.d.ts.map +1 -1
- package/dist/server/core/conversations/conversation.service.js +3 -0
- package/dist/server/core/conversations/conversation.service.js.map +1 -1
- package/dist/server/core/reports/model-usage.d.ts.map +1 -1
- package/dist/server/core/reports/model-usage.js +42 -44
- package/dist/server/core/reports/model-usage.js.map +1 -1
- package/dist/server/core/reports/prices.d.ts.map +1 -1
- package/dist/server/core/reports/prices.js +29 -5
- package/dist/server/core/reports/prices.js.map +1 -1
- package/dist/server/core/reports/report.service.d.ts +4 -2
- package/dist/server/core/reports/report.service.d.ts.map +1 -1
- package/dist/server/core/reports/report.service.js +147 -68
- package/dist/server/core/reports/report.service.js.map +1 -1
- package/dist/server/core/runs/claim.d.ts +2 -0
- package/dist/server/core/runs/claim.d.ts.map +1 -1
- package/dist/server/core/runs/claim.js +76 -54
- package/dist/server/core/runs/claim.js.map +1 -1
- package/dist/server/core/runs/execution.d.ts +1 -1
- package/dist/server/core/runs/execution.d.ts.map +1 -1
- package/dist/server/core/runs/execution.js +7 -2
- package/dist/server/core/runs/execution.js.map +1 -1
- package/dist/server/core/runs/index.d.ts +2 -2
- package/dist/server/core/runs/index.d.ts.map +1 -1
- package/dist/server/core/runs/index.js +2 -2
- package/dist/server/core/runs/index.js.map +1 -1
- package/dist/server/core/runs/lifecycle.d.ts.map +1 -1
- package/dist/server/core/runs/lifecycle.js +47 -17
- package/dist/server/core/runs/lifecycle.js.map +1 -1
- package/dist/server/core/runs/policy.d.ts +9 -2
- package/dist/server/core/runs/policy.d.ts.map +1 -1
- package/dist/server/core/runs/policy.js +17 -8
- package/dist/server/core/runs/policy.js.map +1 -1
- package/dist/server/core/runs/run.service.d.ts +7 -1
- package/dist/server/core/runs/run.service.d.ts.map +1 -1
- package/dist/server/core/runs/run.service.js +5 -2
- package/dist/server/core/runs/run.service.js.map +1 -1
- package/dist/server/core/runs/run.store.d.ts +5 -0
- package/dist/server/core/runs/run.store.d.ts.map +1 -1
- package/dist/server/core/runs/run.store.js +35 -11
- package/dist/server/core/runs/run.store.js.map +1 -1
- package/dist/server/core/runs/runner-view.d.ts +5 -0
- package/dist/server/core/runs/runner-view.d.ts.map +1 -1
- package/dist/server/core/runs/runner-view.js +5 -1
- package/dist/server/core/runs/runner-view.js.map +1 -1
- package/dist/server/core/runs/sweeper.d.ts.map +1 -1
- package/dist/server/core/runs/sweeper.js +12 -8
- package/dist/server/core/runs/sweeper.js.map +1 -1
- package/dist/server/core/runs/transitions.d.ts +4 -1
- package/dist/server/core/runs/transitions.d.ts.map +1 -1
- package/dist/server/core/runs/transitions.js +31 -11
- package/dist/server/core/runs/transitions.js.map +1 -1
- package/dist/server/core/runs/workload.d.ts +17 -6
- package/dist/server/core/runs/workload.d.ts.map +1 -1
- package/dist/server/core/runs/workload.js +182 -126
- package/dist/server/core/runs/workload.js.map +1 -1
- package/dist/server/core/skills/skill.service.d.ts.map +1 -1
- package/dist/server/core/skills/skill.service.js +44 -8
- package/dist/server/core/skills/skill.service.js.map +1 -1
- package/dist/server/core/transaction-read.d.ts +4 -0
- package/dist/server/core/transaction-read.d.ts.map +1 -0
- package/dist/server/core/transaction-read.js +30 -0
- package/dist/server/core/transaction-read.js.map +1 -0
- package/dist/server/core/variables/index.d.ts +1 -1
- package/dist/server/core/variables/index.d.ts.map +1 -1
- package/dist/server/core/variables/index.js.map +1 -1
- package/dist/server/core/variables/variable.service.d.ts +11 -0
- package/dist/server/core/variables/variable.service.d.ts.map +1 -1
- package/dist/server/core/variables/variable.service.js +63 -20
- package/dist/server/core/variables/variable.service.js.map +1 -1
- package/dist/server/kernel/scopes.d.ts +6 -0
- package/dist/server/kernel/scopes.d.ts.map +1 -1
- package/dist/server/kernel/scopes.js.map +1 -1
- package/dist/server/online/services.d.ts.map +1 -1
- package/dist/server/online/services.js +101 -24
- package/dist/server/online/services.js.map +1 -1
- package/dist/server/providers/agents.d.ts +9 -1
- package/dist/server/providers/agents.d.ts.map +1 -1
- package/dist/server/providers/agents.js +3 -0
- package/dist/server/providers/agents.js.map +1 -1
- package/dist/server/providers/runners.d.ts +3 -1
- package/dist/server/providers/runners.d.ts.map +1 -1
- package/dist/server/providers/runners.js +12 -3
- package/dist/server/providers/runners.js.map +1 -1
- package/dist/server/routes/admin.d.ts +2 -0
- package/dist/server/routes/admin.d.ts.map +1 -1
- package/dist/server/routes/admin.js +12 -0
- package/dist/server/routes/admin.js.map +1 -1
- package/dist/server/routes/chat.d.ts.map +1 -1
- package/dist/server/routes/chat.js +27 -6
- package/dist/server/routes/chat.js.map +1 -1
- package/dist/server/routes/index.d.ts.map +1 -1
- package/dist/server/routes/index.js +3 -0
- package/dist/server/routes/index.js.map +1 -1
- package/dist/server/routes/runners/admin.d.ts.map +1 -1
- package/dist/server/routes/runners/admin.js +55 -9
- package/dist/server/routes/runners/admin.js.map +1 -1
- package/dist/server/routes/runners/install.d.ts.map +1 -1
- package/dist/server/routes/runners/install.js +7 -2
- package/dist/server/routes/runners/install.js.map +1 -1
- package/dist/server/routes/schemas.d.ts +7 -1
- package/dist/server/routes/schemas.d.ts.map +1 -1
- package/dist/server/routes/schemas.js +187 -64
- package/dist/server/routes/schemas.js.map +1 -1
- package/dist/server/runners/runner.service.d.ts +12 -2
- package/dist/server/runners/runner.service.d.ts.map +1 -1
- package/dist/server/runners/runner.service.js +119 -55
- package/dist/server/runners/runner.service.js.map +1 -1
- package/dist/server/runners/runner.store.d.ts +2 -2
- package/dist/server/runners/runner.store.d.ts.map +1 -1
- package/dist/server/runners/runner.store.js +8 -5
- package/dist/server/runners/runner.store.js.map +1 -1
- package/dist/server/runners/slots.d.ts +5 -2
- package/dist/server/runners/slots.d.ts.map +1 -1
- package/dist/server/runners/slots.js +3 -4
- package/dist/server/runners/slots.js.map +1 -1
- package/dist/shared/conversations.d.ts +2 -0
- package/dist/shared/conversations.d.ts.map +1 -1
- package/dist/shared/conversations.js.map +1 -1
- package/dist/shared/runners.d.ts +11 -2
- package/dist/shared/runners.d.ts.map +1 -1
- package/dist/shared/runners.js.map +1 -1
- package/dist/shared/runs.d.ts +8 -0
- package/dist/shared/runs.d.ts.map +1 -1
- package/dist/shared/runs.js +22 -0
- package/dist/shared/runs.js.map +1 -1
- package/dist/shared/skills.d.ts +3 -1
- package/dist/shared/skills.d.ts.map +1 -1
- package/dist/shared/skills.js.map +1 -1
- package/package.json +14 -14
- package/skills/nocobase-app-plugin-agents/references/runners.md +6 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sweeper.js","sourceRoot":"","sources":["../../../../server/core/runs/sweeper.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,OAAO,EACL,OAAO,EACP,WAAW,GAEZ,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAKxB,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EACL,MAAM,EACN,UAAU,EACV,aAAa,EACb,QAAQ,EACR,SAAS,EAET,KAAK,GACN,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,UAAU,EAAuB,MAAM,kBAAkB,CAAC;AAqB9E,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;AAE3C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,EAAM,EACN,IAAwC,EACxC,GAAc;IAEd,IAAI,CAAC,GAAG,CAAC,YAAY;QAAE,OAAO;IAC9B,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC;QAC9C,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,CAAC,CAAC,GAAG,CAAC;YACJ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC;SACrC,CAAC;KACL,CAAC,CAAC;IACH,IAAI,QAAQ,GAAG,CAAC;QAAE,OAAO;IACzB,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC;QAClD,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,CAAC,CAAC,GAAG,CAAC;YACJ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC;YAC5B,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC;SACrC,CAAC;QACJ,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IACH,IAAI,CAAC,MAAM;QAAE,OAAO;IACpB,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC;IACtE,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,OAAO;IAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI;QAC7B,MAAM,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC;YACjC,MAAM,EAAE;gBACN,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;gBACnB,KAAK,EAAE,GAAG,CAAC,EAAE;gBACb,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,YAAY,EAAE,IAAI;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,IAAI;gBAC1B,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,CAAC;gBAC3C,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,IAAI,CAAC;gBAC7C,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,CAAC;gBAC3C,SAAS,EAAE,GAAG;aACf;SACF,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAiB;IAC7C,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IAE3B,OAAO;QACL,KAAK,CAAC,KAAK;YACT,MAAM,IAAI,CAAC,qBAAqB,EAAE,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;YAExD,MAAM,IAAI,GAAG,KAAK,EAAE,GAAc,EAAE,MAAqB,EAAE,EAAE;gBAC3D,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC1B,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;oBACvD,IACE,CAAC,OAAO;wBACR,OAAO,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM;wBAC7B,OAAO,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ;
|
|
1
|
+
{"version":3,"file":"sweeper.js","sourceRoot":"","sources":["../../../../server/core/runs/sweeper.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,OAAO,EACL,OAAO,EACP,WAAW,GAEZ,MAAM,0BAA0B,CAAC;AAElC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAKxB,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EACL,MAAM,EACN,UAAU,EACV,aAAa,EACb,QAAQ,EACR,SAAS,EAET,KAAK,GACN,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,UAAU,EAAuB,MAAM,kBAAkB,CAAC;AAqB9E,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;AAE3C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,EAAM,EACN,IAAwC,EACxC,GAAc;IAEd,IAAI,CAAC,GAAG,CAAC,YAAY;QAAE,OAAO;IAC9B,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC;QAC9C,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,CAAC,CAAC,GAAG,CAAC;YACJ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC;SACrC,CAAC;KACL,CAAC,CAAC;IACH,IAAI,QAAQ,GAAG,CAAC;QAAE,OAAO;IACzB,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC;QAClD,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,CAAC,CAAC,GAAG,CAAC;YACJ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC;YAC5B,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC;SACrC,CAAC;QACJ,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,KAAK,EAAE,CAAC;KACT,CAAC,CAAC;IACH,IAAI,CAAC,MAAM;QAAE,OAAO;IACpB,MAAM,MAAM,GAAG,aAAa,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC;IACtE,IAAI,CAAC,MAAM,CAAC,OAAO;QAAE,OAAO;IAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI;QAC7B,MAAM,SAAS,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC;YACjC,MAAM,EAAE;gBACN,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE;gBACnB,KAAK,EAAE,GAAG,CAAC,EAAE;gBACb,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,YAAY,EAAE,IAAI;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,IAAI;gBAC1B,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,CAAC;gBAC3C,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,IAAI,CAAC;gBAC7C,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,CAAC;gBAC3C,SAAS,EAAE,GAAG;aACf;SACF,CAAC,CAAC;AACP,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAiB;IAC7C,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;IAE3B,OAAO;QACL,KAAK,CAAC,KAAK;YACT,MAAM,IAAI,CAAC,qBAAqB,EAAE,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;YAExD,MAAM,IAAI,GAAG,KAAK,EAAE,GAAc,EAAE,MAAqB,EAAE,EAAE;gBAC3D,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC1B,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;oBACvD,IACE,CAAC,OAAO;wBACR,OAAO,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM;wBAC7B,OAAO,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ;wBACjC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC;wBAE/C,OAAO;oBACT,MAAM,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;oBACxC,MAAM,OAAO,GAAG,MAAM,UAAU,CAC9B,IAAI,EACJ,IAAI,EACJ,OAAO,EACP,MAAM,EACN,SAAS,EACT,SAAS,CACV,CAAC;oBACF,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ;wBAAE,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;yBACjD,IAAI,OAAO,CAAC,GAAG;wBAAE,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;gBAC3C,CAAC,CAAC,CAAC;YACL,CAAC,CAAC;YACF,MAAM,GAAG,GAAG,KAAK,EACf,GAAc,EACd,OAAwC,EACxC,EAAE;gBACF,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;oBAC1B,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;oBACvD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM;wBAAE,OAAO;oBACtD,IAAI,OAAO,CAAC,MAAM,KAAK,QAAQ;wBAC7B,MAAM,YAAY,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;oBAC1C,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;oBAC5D,IAAI,KAAK,EAAE,MAAM,KAAK,QAAQ;wBAAE,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;oBACnD,IAAI,KAAK,EAAE,MAAM,KAAK,WAAW;wBAAE,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;gBAC3D,CAAC,CAAC,CAAC;YACL,CAAC,CAAC;YAEF,MAAM,IAAI,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC;gBACzC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;aAC9D,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CACpB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACvB,IAAI,EACJ,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CACxD,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,EAAE,CAClB,CACF,CACF,CAAC;YACF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;YAC3C,MAAM,aAAa,GAAG,KAAK,EAAE,OAAe,EAAE,EAAE;gBAC9C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC3B,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;oBAC7C,QAAQ,CAAC,GAAG,CACV,OAAO,EACP,aAAa,CAAC,KAAK,EAAE,UAAU,IAAI,IAAI,CAAC,CAAC,aAAa,CACvD,CAAC;gBACJ,CAAC;gBACD,OAAO,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;YAChC,CAAC,CAAC;YACF,MAAM,IAAI,GAAG,CAAC,EAAiB,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAE9D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IACE,GAAG,CAAC,iBAAiB;oBACrB,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,aAAa,EACnE,CAAC;oBACD,MAAM,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;oBAC7D,SAAS;gBACX,CAAC;gBACD,IAAI,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC7C,MAAM,IAAI,CACR,GAAG,EACH,GAAG,CAAC,MAAM,KAAK,YAAY;wBACzB,CAAC,CAAC,0BAA0B;wBAC5B,CAAC,CAAC,eAAe,CACpB,CAAC;oBACF,SAAS;gBACX,CAAC;gBACD,IACE,GAAG,CAAC,MAAM,KAAK,YAAY;oBAC3B,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,cAAc,EAChE,CAAC;oBACD,MAAM,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;oBAChC,SAAS;gBACX,CAAC;gBACD,IACE,GAAG,CAAC,MAAM,KAAK,SAAS;oBACxB,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,GAAG,CAAC,OAAO,EAAE,EACxC,CAAC;oBACD,MAAM,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,CAAC;oBAChC,SAAS;gBACX,CAAC;gBACD,IACE,GAAG,CAAC,MAAM,KAAK,SAAS;oBACxB,IAAI,CAAC,GAAG,CAAC,cAAc,IAAI,GAAG,CAAC,SAAS,CAAC;wBACvC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBAExD,MAAM,GAAG,CAAC,GAAG,EAAE;wBACb,MAAM,EAAE,QAAQ;wBAChB,MAAM,EAAE,aAAa;wBACrB,MAAM,EAAE,wCAAwC;qBACjD,CAAC,CAAC;YACP,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC;gBAC3C,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,CAAC,CAAC,GAAG,CAAC;oBACJ,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC;oBAC/B,CAAC;yBACE,QAAQ,CAAC,OAAO,CAAC;yBACjB,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CACd,KAAK,CAAC,EAAE,CAAC;wBACP,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC;wBACpC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,QAAQ,EAAE;wBACnC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,QAAQ,EAAE;qBACnC,CAAC,CACH;iBACJ,CAAC;aACL,CAAC,CAAC;YACH,KAAK,MAAM,GAAG,IAAI,MAAM;gBACtB,MAAM,GAAG,CAAC,GAAG,EAAE;oBACb,MAAM,EAAE,WAAW;oBACnB,MAAM,EAAE,WAAW;oBACnB,MAAM,EAAE,yBAAyB;iBAClC,CAAC,CAAC;YAEL,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC3C,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;oBAC1B,IAAI,MAAM,GAAkB,IAAI,CAAC;oBACjC,OAAO,IAAI,EAAE,CAAC;wBACZ,IAAI,IAAI,GAAG,IAAI,CAAC,KAAK;6BAClB,UAAU,CAAC,QAAQ,CAAC;6BACpB,MAAM,CAAC,IAAI,CAAC;6BACZ,KAAK,CAAC,QAAQ,EAAE,GAAG,EAAE,QAAQ,CAAC;6BAC9B,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;wBAC3C,IAAI,MAAM,KAAK,IAAI;4BAAE,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;wBAC1D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC;wBAC1D,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM;4BACtB,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC;gCAC5B,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gCAC3D,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE;6BACjC,CAAC;4BACJ,CAAC,CAAC,EAAE,CAAC;wBACP,IAAI,CAAC,KAAK,CAAC,MAAM;4BAAE,MAAM;wBACzB,KAAK,MAAM,SAAS,IAAI,KAAK;4BAC3B,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;gCAC1B,MAAM,iBAAiB,CACrB,IAAI,CAAC,IAAI,EACT,SAAS,CAAC,OAAO,EACjB,KAAK,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAC1B,CAAC;gCACF,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;gCAC7D,IACE,CAAC,OAAO;oCACR,OAAO,CAAC,MAAM,KAAK,QAAQ;oCAC3B,OAAO,CAAC,QAAQ,KAAK,IAAI;oCACzB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC;oCACrD,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,IAAI,CAAC;wCAClC,IAAI,CAAC,SAAS,CAAC,cAAc,IAAI,IAAI,CAAC;oCACxC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;oCAEzD,OAAO;gCACT,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,aAAc,CAC1C,IAAI,CAAC,IAAI,EACT,KAAK,CAAC,OAAO,CAAC,EACd,KAAK,CAAC,GAAG,EAAE,CACZ,CAAC;gCACF,IAAI,CAAC,OAAO;oCAAE,OAAO;gCACrB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;oCACjD,MAAM,EAAE,WAAW;oCACnB,GAAG,OAAO;iCACX,CAAC,CAAC;gCACH,IAAI,KAAK;oCAAE,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;4BACnC,CAAC,CAAC,CAAC;wBACL,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;oBAC5B,CAAC;gBACH,CAAC;gBACD,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC;gBAC3C,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;oBAAE,SAAS;gBAC5B,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC;gBAChD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC;oBAC1C,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,CAAC,CAAC,GAAG,CAAC;wBACJ,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC;wBAC/B,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;wBACxC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;wBAClC,CAAC,CAAC,EAAE,CAAC;4BACH,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE;4BAC7B,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;yBACrC,CAAC;qBACH,CAAC;iBACL,CAAC,CAAC;gBACH,KAAK,MAAM,GAAG,IAAI,KAAK;oBACrB,MAAM,GAAG,CAAC,GAAG,EAAE;wBACb,MAAM,EAAE,QAAQ;wBAChB,MAAM,EAAE,eAAe;wBACvB,MAAM,EAAE,kCAAkC;qBAC3C,CAAC,CAAC;YACP,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -11,6 +11,8 @@ import { type RunRecord } from './run.store.js';
|
|
|
11
11
|
export interface TransitionDeps {
|
|
12
12
|
readonly clock: Clock;
|
|
13
13
|
readonly subjects: SubjectRegistry;
|
|
14
|
+
/** The first back-off of a failed attempt (`retryDelayMs`); 30 seconds when left out. */
|
|
15
|
+
readonly retryBaseDelayMs?: number;
|
|
14
16
|
}
|
|
15
17
|
export interface FinishOutcome {
|
|
16
18
|
readonly status: 'completed' | 'failed' | 'cancelled';
|
|
@@ -19,6 +21,7 @@ export interface FinishOutcome {
|
|
|
19
21
|
readonly summary?: string;
|
|
20
22
|
readonly sessionId?: string;
|
|
21
23
|
readonly cancelledById?: string;
|
|
24
|
+
readonly failureSource?: 'sweeper';
|
|
22
25
|
}
|
|
23
26
|
/** Revokes every live token of the run. */
|
|
24
27
|
export declare function revokeTokens(tx: Tx, runId: string, now: string): Promise<void>;
|
|
@@ -37,5 +40,5 @@ export interface RequeueOutcome {
|
|
|
37
40
|
* Takes a held run back from its runner after a retryable failure: it goes back to the queue while attempts remain,
|
|
38
41
|
* and fails with `reason` otherwise.
|
|
39
42
|
*/
|
|
40
|
-
export declare function requeueRun(tx: Tx, deps: TransitionDeps, run: RunRecord, reason: FailureReason, detail?: string): Promise<RequeueOutcome>;
|
|
43
|
+
export declare function requeueRun(tx: Tx, deps: TransitionDeps, run: RunRecord, reason: FailureReason, detail?: string, source?: 'runner' | 'sweeper'): Promise<RequeueOutcome>;
|
|
41
44
|
//# sourceMappingURL=transitions.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transitions.d.ts","sourceRoot":"","sources":["../../../../server/core/runs/transitions.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"transitions.d.ts","sourceRoot":"","sources":["../../../../server/core/runs/transitions.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAOzE,OAAO,EAAS,KAAK,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,oBAAoB,CAAC;AAC7C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAOlD,OAAO,EAML,KAAK,SAAS,EACf,MAAM,gBAAgB,CAAC;AAExB,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IACnC,yFAAyF;IACzF,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;IACtD,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,aAAa,CAAC,EAAE,SAAS,CAAC;CACpC;AAED,2CAA2C;AAC3C,wBAAsB,YAAY,CAChC,EAAE,EAAE,EAAE,EACN,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,IAAI,CAAC,CAMf;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,EAAE,EAAE,EAAE,EACN,IAAI,EAAE,cAAc,EACpB,GAAG,EAAE,SAAS,EACd,OAAO,EAAE,aAAa,EACtB,SAAS,UAAQ,EACjB,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAC/B,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAsE3B;AA2CD,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,0EAA0E;IAC1E,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,GAAG,EAAE,SAAS,GAAG,IAAI,CAAC;CAChC;AAED;;;GAGG;AACH,wBAAsB,UAAU,CAC9B,EAAE,EAAE,EAAE,EACN,IAAI,EAAE,cAAc,EACpB,GAAG,EAAE,SAAS,EACd,MAAM,EAAE,aAAa,EACrB,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,GAAE,QAAQ,GAAG,SAAoB,GACtC,OAAO,CAAC,cAAc,CAAC,CAuFzB"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { isPreStartInterruption, PRE_START_INTERRUPTION_LIMIT, consecutivePreStartInterruptions, } from '../../../shared/runs.js';
|
|
1
2
|
import { later } from '../../kernel/clock.js';
|
|
2
3
|
import { retryDelayMs, WORKSPACE_WAIT_LIMIT_MS, workspaceBusyDelayMs, } from './policy.js';
|
|
3
4
|
import { endExecution } from './execution.js';
|
|
4
|
-
import {
|
|
5
|
+
import { findRunRecord, runsRepo, tokensRepo, toExecutions, toRun, } from './run.store.js';
|
|
5
6
|
/** Revokes every live token of the run. */
|
|
6
7
|
export async function revokeTokens(tx, runId, now) {
|
|
7
8
|
await tokensRepo(tx.conn).updateMany({
|
|
@@ -42,10 +43,10 @@ export async function finishRun(tx, deps, run, outcome, versioned = false, after
|
|
|
42
43
|
filter: (f) => f.and([
|
|
43
44
|
f.string('id').eq(run.id),
|
|
44
45
|
f.string('status').eq(run.status),
|
|
46
|
+
f.number('attempt').eq(Number(run.attempt)),
|
|
45
47
|
...(versioned ? [f.number('revision').eq(run.revision)] : []),
|
|
46
48
|
...(run.status === 'queued'
|
|
47
49
|
? [
|
|
48
|
-
f.number('attempt').eq(Number(run.attempt)),
|
|
49
50
|
run.queueEnteredAt
|
|
50
51
|
? f.and([
|
|
51
52
|
f
|
|
@@ -73,7 +74,7 @@ export async function finishRun(tx, deps, run, outcome, versioned = false, after
|
|
|
73
74
|
if (result.updatedCount !== 1)
|
|
74
75
|
return null;
|
|
75
76
|
await afterGuard?.();
|
|
76
|
-
await endExecution(tx.conn, run, now, outcome.reason ?? null);
|
|
77
|
+
await endExecution(tx.conn, run, now, outcome.reason ?? null, outcome.failureSource);
|
|
77
78
|
await revokeTokens(tx, run.id, now);
|
|
78
79
|
const ended = (await findRunRecord(tx.conn, run.id));
|
|
79
80
|
tx.emit({ type: 'run.changed', runId: run.id, status: ended.status });
|
|
@@ -83,7 +84,7 @@ export async function finishRun(tx, deps, run, outcome, versioned = false, after
|
|
|
83
84
|
return ended;
|
|
84
85
|
}
|
|
85
86
|
/**
|
|
86
|
-
* Workspace waits and
|
|
87
|
+
* Workspace waits, runner upgrades and pre-start environment interruptions do not consume maxAttempts. Include the open current attempt
|
|
87
88
|
* when reporting workspace contention; only consecutive workspace waits set the contention limit.
|
|
88
89
|
*/
|
|
89
90
|
function workspaceWaits(run, reporting) {
|
|
@@ -91,7 +92,8 @@ function workspaceWaits(run, reporting) {
|
|
|
91
92
|
const current = executions.find((item) => item.attempt === Number(run.attempt) && item.finishedAt === null);
|
|
92
93
|
const ended = executions.filter((item) => item !== current);
|
|
93
94
|
let total = ended.filter((item) => item.failureReason === 'workspaceBusy' ||
|
|
94
|
-
item.failureReason === 'runnerUpgrade'
|
|
95
|
+
item.failureReason === 'runnerUpgrade' ||
|
|
96
|
+
isPreStartInterruption(item)).length;
|
|
95
97
|
let trailing = 0;
|
|
96
98
|
let since = null;
|
|
97
99
|
if (reporting) {
|
|
@@ -112,10 +114,27 @@ function workspaceWaits(run, reporting) {
|
|
|
112
114
|
* Takes a held run back from its runner after a retryable failure: it goes back to the queue while attempts remain,
|
|
113
115
|
* and fails with `reason` otherwise.
|
|
114
116
|
*/
|
|
115
|
-
export async function requeueRun(tx, deps, run, reason, detail) {
|
|
117
|
+
export async function requeueRun(tx, deps, run, reason, detail, source = 'runner') {
|
|
116
118
|
const now = deps.clock.now();
|
|
117
119
|
const waits = workspaceWaits(run, reason === 'workspaceBusy');
|
|
118
|
-
const
|
|
120
|
+
const preStartInterruption = source === 'sweeper' &&
|
|
121
|
+
run.status === 'dispatched' &&
|
|
122
|
+
run.startedAt === null &&
|
|
123
|
+
(reason === 'runnerOfflineBeforeStart' || reason === 'startTimeout');
|
|
124
|
+
const failureSource = preStartInterruption ? 'sweeper' : undefined;
|
|
125
|
+
if (preStartInterruption &&
|
|
126
|
+
consecutivePreStartInterruptions(toExecutions(run.executionHistory)) + 1 >=
|
|
127
|
+
PRE_START_INTERRUPTION_LIMIT) {
|
|
128
|
+
const ended = await finishRun(tx, deps, run, {
|
|
129
|
+
status: 'failed',
|
|
130
|
+
reason: 'startTimeout',
|
|
131
|
+
failureSource,
|
|
132
|
+
detail: 'The runtime did not receive the run in 10 consecutive pre-start attempts.',
|
|
133
|
+
});
|
|
134
|
+
return { status: ended?.status ?? run.status, run: ended };
|
|
135
|
+
}
|
|
136
|
+
const interruption = reason === 'runnerUpgrade' || preStartInterruption;
|
|
137
|
+
const exhausted = interruption
|
|
119
138
|
? false
|
|
120
139
|
: reason === 'workspaceBusy'
|
|
121
140
|
? waits.since !== null &&
|
|
@@ -130,16 +149,17 @@ export async function requeueRun(tx, deps, run, reason, detail) {
|
|
|
130
149
|
return { status: ended?.status ?? run.status, run: ended };
|
|
131
150
|
}
|
|
132
151
|
const nowText = now.toISOString();
|
|
133
|
-
const delay =
|
|
152
|
+
const delay = interruption
|
|
134
153
|
? 0
|
|
135
154
|
: reason === 'workspaceBusy'
|
|
136
155
|
? workspaceBusyDelayMs(waits.trailing)
|
|
137
|
-
: retryDelayMs(reason, Number(run.attempt) - waits.total);
|
|
156
|
+
: retryDelayMs(reason, Number(run.attempt) - waits.total, deps.retryBaseDelayMs);
|
|
138
157
|
const availableAt = delay > 0 ? later(now, delay) : null;
|
|
139
158
|
const result = await runsRepo(tx.conn).updateMany({
|
|
140
159
|
filter: (f) => f.and([
|
|
141
160
|
f.string('id').eq(run.id),
|
|
142
|
-
f.
|
|
161
|
+
f.string('status').eq(run.status),
|
|
162
|
+
f.number('attempt').eq(Number(run.attempt)),
|
|
143
163
|
run.runnerId === null
|
|
144
164
|
? f.string('runnerId').eq(null)
|
|
145
165
|
: f.string('runnerId').eq(run.runnerId),
|
|
@@ -164,7 +184,7 @@ export async function requeueRun(tx, deps, run, reason, detail) {
|
|
|
164
184
|
});
|
|
165
185
|
if (result.updatedCount !== 1)
|
|
166
186
|
return { status: run.status, run: null };
|
|
167
|
-
await endExecution(tx.conn, run, nowText, reason);
|
|
187
|
+
await endExecution(tx.conn, run, nowText, reason, failureSource);
|
|
168
188
|
await revokeTokens(tx, run.id, nowText);
|
|
169
189
|
tx.emit({ type: 'run.changed', runId: run.id, status: 'queued' });
|
|
170
190
|
tx.emit({ type: 'run.requeued', runId: run.id, availableAt });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transitions.js","sourceRoot":"","sources":["../../../../server/core/runs/transitions.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"transitions.js","sourceRoot":"","sources":["../../../../server/core/runs/transitions.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,sBAAsB,EACtB,4BAA4B,EAC5B,gCAAgC,GACjC,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,KAAK,EAAc,MAAM,uBAAuB,CAAC;AAG1D,OAAO,EACL,YAAY,EACZ,uBAAuB,EACvB,oBAAoB,GACrB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EACL,aAAa,EACb,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,KAAK,GAEN,MAAM,gBAAgB,CAAC;AAmBxB,2CAA2C;AAC3C,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,EAAM,EACN,KAAa,EACb,GAAW;IAEX,MAAM,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC;QACnC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QACnE,MAAM,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE;KAC3B,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,EAAM,EACN,IAAoB,EACpB,GAAc,EACd,OAAsB,EACtB,SAAS,GAAG,KAAK,EACjB,UAAgC;IAEhC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,MAAM,GAA4B;QACtC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,UAAU,EAAE,GAAG;QACf,cAAc,EAAE,IAAI;QACpB,WAAW,EAAE,IAAI;QACjB,SAAS,EAAE,GAAG;KACf,CAAC;IACF,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;QACnC,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC;QAC5B,MAAM,CAAC,aAAa,GAAG,IAAI,CAAC;IAC9B,CAAC;SAAM,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QACxC,MAAM,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;QACtC,MAAM,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC;IAClE,CAAC;SAAM,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS;QACrC,MAAM,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACzD,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS;QAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IACpE,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS;QAAE,MAAM,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IAC1E,IAAI,OAAO,CAAC,aAAa,KAAK,SAAS;QACrC,MAAM,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAC/C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC;QAChD,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,CAAC,CAAC,GAAG,CAAC;YACJ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC;YACjC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC3C,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7D,GAAG,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ;gBACzB,CAAC,CAAC;oBACE,GAAG,CAAC,cAAc;wBAChB,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;4BACJ,CAAC;iCACE,IAAI,CAAC,gBAAgB,CAAC;iCACtB,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;4BAC1C,CAAC;iCACE,IAAI,CAAC,gBAAgB,CAAC;iCACtB,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;yBAC1C,CAAC;wBACJ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,KAAK,EAAE;oBACpC,GAAG,CAAC,WAAW;wBACb,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;4BACJ,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;4BAC1D,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;yBAC1D,CAAC;wBACJ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE;iBAClC;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,QAAQ,KAAK,IAAI;gBACnB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC;gBAC/B,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC1C,CAAC;QACJ,MAAM;KACP,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,YAAY,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3C,MAAM,UAAU,EAAE,EAAE,CAAC;IACrB,MAAM,YAAY,CAChB,EAAE,CAAC,IAAI,EACP,GAAG,EACH,GAAG,EACH,OAAO,CAAC,MAAM,IAAI,IAAI,EACtB,OAAO,CAAC,aAAa,CACtB,CAAC;IACF,MAAM,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,CAAC,MAAM,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,CAAE,CAAC;IACtD,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IACtE,MAAM,IAAI,CAAC,QAAQ;SAChB,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC;QACvB,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IAC5C,OAAO,KAAK,CAAC;AACf,CAAC;AAWD;;;GAGG;AACH,SAAS,cAAc,CAAC,GAAc,EAAE,SAAkB;IACxD,MAAM,UAAU,GAAG,YAAY,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAC7B,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,CAC3E,CAAC;IACF,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;IAC5D,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,CACtB,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,CAAC,aAAa,KAAK,eAAe;QACtC,IAAI,CAAC,aAAa,KAAK,eAAe;QACtC,sBAAsB,CAAC,IAAI,CAAC,CAC/B,CAAC,MAAM,CAAC;IACT,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,KAAK,GAAkB,IAAI,CAAC;IAChC,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,IAAI,CAAC,CAAC;QACX,QAAQ,GAAG,CAAC,CAAC;QACb,KAAK,GAAG,OAAO,EAAE,YAAY,IAAI,GAAG,CAAC,YAAY,IAAI,IAAI,CAAC;QAC1D,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,CAAC,aAAa,KAAK,eAAe;gBAAE,MAAM;YAClD,QAAQ,IAAI,CAAC,CAAC;YACd,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;AACpC,CAAC;AASD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,EAAM,EACN,IAAoB,EACpB,GAAc,EACd,MAAqB,EACrB,MAAe,EACf,SAA+B,QAAQ;IAEvC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,MAAM,KAAK,eAAe,CAAC,CAAC;IAC9D,MAAM,oBAAoB,GACxB,MAAM,KAAK,SAAS;QACpB,GAAG,CAAC,MAAM,KAAK,YAAY;QAC3B,GAAG,CAAC,SAAS,KAAK,IAAI;QACtB,CAAC,MAAM,KAAK,0BAA0B,IAAI,MAAM,KAAK,cAAc,CAAC,CAAC;IACvE,MAAM,aAAa,GAAG,oBAAoB,CAAC,CAAC,CAAE,SAAmB,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9E,IACE,oBAAoB;QACpB,gCAAgC,CAAC,YAAY,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,GAAG,CAAC;YACtE,4BAA4B,EAC9B,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;YAC3C,MAAM,EAAE,QAAQ;YAChB,MAAM,EAAE,cAAc;YACtB,aAAa;YACb,MAAM,EACJ,2EAA2E;SAC9E,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;IAC7D,CAAC;IACD,MAAM,YAAY,GAAG,MAAM,KAAK,eAAe,IAAI,oBAAoB,CAAC;IACxE,MAAM,SAAS,GAAG,YAAY;QAC5B,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,MAAM,KAAK,eAAe;YAC1B,CAAC,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI;gBACpB,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,uBAAuB;YACpE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;IACnE,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;YAC3C,MAAM,EAAE,QAAQ;YAChB,MAAM;YACN,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;SAC5C,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,IAAI,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;IAC7D,CAAC;IACD,MAAM,OAAO,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IAClC,MAAM,KAAK,GAAG,YAAY;QACxB,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,MAAM,KAAK,eAAe;YAC1B,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,QAAQ,CAAC;YACtC,CAAC,CAAC,YAAY,CACV,MAAM,EACN,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,KAAK,EACjC,IAAI,CAAC,gBAAgB,CACtB,CAAC;IACR,MAAM,WAAW,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACzD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC;QAChD,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CACZ,CAAC,CAAC,GAAG,CAAC;YACJ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC;YACjC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC3C,GAAG,CAAC,QAAQ,KAAK,IAAI;gBACnB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC;gBAC/B,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC1C,CAAC;QACJ,MAAM,EAAE;YACN,MAAM,EAAE,QAAQ;YAChB,cAAc,EAAE,OAAO;YACvB,iBAAiB,EAAE,IAAI;YACvB,iBAAiB,EAAE,IAAI;YACvB,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC;YAChC,WAAW;YACX,cAAc,EAAE,IAAI;YACpB,YAAY,EAAE,IAAI;YAClB,SAAS,EAAE,IAAI;YACf,YAAY,EAAE,KAAK;YACnB,aAAa,EAAE,CAAC;YAChB,aAAa,EAAE,MAAM;YACrB,aAAa,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC;YACpE,SAAS,EAAE,OAAO;SACnB;KACF,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,YAAY,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;IACxE,MAAM,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;IACjE,MAAM,YAAY,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACxC,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;IAClE,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;IAC9D,OAAO;QACL,MAAM,EAAE,QAAQ;QAChB,OAAO,EAAE,WAAW,IAAI,OAAO;QAC/B,GAAG,EAAE,CAAC,MAAM,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI;KACpD,CAAC;AACJ,CAAC"}
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The open runs on one kind of subject as a board of work shows them (`RunService.workload`): the runs held, with the
|
|
3
|
-
* runner and the newest thing each reported, and the runs queued, with their place in the claim order and
|
|
4
|
-
*
|
|
3
|
+
* runner and the newest thing each reported, and the runs queued, with their place in the claim order and why each
|
|
4
|
+
* waits by the claim's own rules (`claim.ts`), without claiming anything.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
* (for the agents' and runners' counts and the busy subjects), the runners, the active jobs,
|
|
8
|
-
* runs' inputs, and the newest event of each held run
|
|
6
|
+
* The list is a fixed number of reads whatever its length: the open runs of the kind, every queued run (for the claim
|
|
7
|
+
* order), every held run (for the agents' and runners' counts and the busy subjects), the runners, the active jobs,
|
|
8
|
+
* the agents, the open runs' inputs, and the newest event of each held run (two reads for all of them). A queued row
|
|
9
|
+
* explains itself from those alone (`explainWait`); the claim's per-run diagnosis of its candidate runners, which
|
|
10
|
+
* reads more, is one run's detail (`RunService.queueWait`, `readQueueWait`).
|
|
9
11
|
*/
|
|
10
12
|
import type { AgentTool, RunnerFeature } from '@nocobase/agent-protocol';
|
|
11
13
|
import type { DatabaseConnection } from '@nocobase/db';
|
|
@@ -44,6 +46,15 @@ export interface WorkloadRunners {
|
|
|
44
46
|
all(conn: DatabaseConnection): Promise<Runner[]>;
|
|
45
47
|
jobsByRunner(conn: DatabaseConnection): Promise<ReadonlyMap<string, number>>;
|
|
46
48
|
}
|
|
47
|
-
|
|
49
|
+
/**
|
|
50
|
+
* The open runs on one kind of subject, in a fixed number of reads whatever their number. A queued run says why it
|
|
51
|
+
* waits by the claim's rules alone (`explainWait`); `readQueueWait` gives one run's exact reason.
|
|
52
|
+
*/
|
|
53
|
+
export declare function readWorkload(conn: DatabaseConnection, clock: Clock, runnerSource: WorkloadRunners, query: WorkloadQuery): Promise<Workload>;
|
|
54
|
+
/**
|
|
55
|
+
* Exactly why one queued run waits: the claim's rules refined by its read-only diagnosis of the run's candidate
|
|
56
|
+
* runners (`ClaimService.diagnose`). Null when the run does not exist or is not queued.
|
|
57
|
+
*/
|
|
58
|
+
export declare function readQueueWait(conn: DatabaseConnection, clock: Clock, runnerSource: WorkloadRunners, runId: string, diagnose?: import('./claim.js').ClaimService['diagnose']): Promise<RunWait | null>;
|
|
48
59
|
export {};
|
|
49
60
|
//# sourceMappingURL=workload.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"workload.d.ts","sourceRoot":"","sources":["../../../../server/core/runs/workload.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"workload.d.ts","sourceRoot":"","sources":["../../../../server/core/runs/workload.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAEvD,OAAO,EAAc,KAAK,KAAK,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EAIL,KAAK,MAAM,EACZ,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAKL,KAAK,OAAO,EACZ,KAAK,QAAQ,EACb,KAAK,aAAa,EAEnB,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAInD,OAAO,EAML,KAAK,SAAS,EACf,MAAM,gBAAgB,CAAC;AAexB,uEAAuE;AACvE,wBAAgB,UAAU,CACxB,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,UAAU,GAAG,WAAW,GAAG,IAAI,CAAC,EACnD,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,UAAU,GAAG,WAAW,GAAG,IAAI,CAAC,GAClD,MAAM,CAMR;AAED,4EAA4E;AAC5E,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC;IACnB,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAC7B,sBAAsB;IACtB,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,yDAAyD;IACzD,QAAQ,CAAC,UAAU,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjD,wGAAwG;IACxG,QAAQ,CAAC,cAAc,CAAC,EAAE,WAAW,CACnC,MAAM,EACN,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,CAC7C,CAAC;IACF,gCAAgC;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,sEAAsE;IACtE,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC;CAClC;AAED,KAAK,SAAS,GAAG,IAAI,CACnB,SAAS,EACT,aAAa,GAAG,aAAa,GAAG,eAAe,GAAG,eAAe,CAClE,GACC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,mBAAmB,CAAC,CAAC,GAAG;IAC9C,QAAQ,CAAC,QAAQ,EAAE,SAAS,aAAa,EAAE,CAAC;CAC7C,CAAC;AAeJ;;;GAGG;AACH,wBAAgB,WAAW,CACzB,GAAG,EAAE,SAAS,EACd,OAAO,EAAE,WAAW,GACnB,IAAI,CAAC,OAAO,EAAE,UAAU,GAAG,eAAe,CAAC,CAuG7C;AAuCD,yGAAyG;AACzG,MAAM,WAAW,eAAe;IAC9B,GAAG,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACjD,YAAY,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC9E;AAgQD;;;GAGG;AACH,wBAAsB,YAAY,CAChC,IAAI,EAAE,kBAAkB,EACxB,KAAK,EAAE,KAAK,EACZ,YAAY,EAAE,eAAe,EAC7B,KAAK,EAAE,aAAa,GACnB,OAAO,CAAC,QAAQ,CAAC,CAwGnB;AAED;;;GAGG;AACH,wBAAsB,aAAa,CACjC,IAAI,EAAE,kBAAkB,EACxB,KAAK,EAAE,KAAK,EACZ,YAAY,EAAE,eAAe,EAC7B,KAAK,EAAE,MAAM,EACb,QAAQ,CAAC,EAAE,OAAO,YAAY,EAAE,YAAY,CAAC,UAAU,CAAC,GACvD,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,CAgBzB"}
|
|
@@ -152,19 +152,9 @@ function cut(text) {
|
|
|
152
152
|
? `${line.slice(0, RUN_ACTIVITY_TEXT_MAX - 1)}…`
|
|
153
153
|
: line;
|
|
154
154
|
}
|
|
155
|
-
|
|
155
|
+
/** The queue as the claim sees it, in a fixed number of reads. */
|
|
156
|
+
async function readQueueState(conn, clock, runnerSource) {
|
|
156
157
|
const now = clock.now();
|
|
157
|
-
const limit = Math.min(Math.max(query.limit ?? DEFAULT_LIMIT, 1), MAX_LIMIT);
|
|
158
|
-
const open = await runsRepo(conn).findMany({
|
|
159
|
-
filter: (f) => f.and([
|
|
160
|
-
f.string('subjectKind').eq(query.subjectKind),
|
|
161
|
-
f.or(['queued', ...ACTIVE].map((status) => f.string('status').eq(status))),
|
|
162
|
-
]),
|
|
163
|
-
sort: (sort) => [sort.field('createdAt').asc(), sort.field('id').asc()],
|
|
164
|
-
limit: limit + 1,
|
|
165
|
-
});
|
|
166
|
-
const truncated = open.length > limit;
|
|
167
|
-
const mine = open.slice(0, limit);
|
|
168
158
|
const queued = (await runsRepo(conn).findMany({
|
|
169
159
|
filter: { status: 'queued' },
|
|
170
160
|
limit: QUEUE_SCAN,
|
|
@@ -209,6 +199,155 @@ export async function readWorkload(conn, clock, runnerSource, query, diagnose) {
|
|
|
209
199
|
agentQueued.set(run.agentId, count);
|
|
210
200
|
agentPosition.set(run.id, count);
|
|
211
201
|
}
|
|
202
|
+
return {
|
|
203
|
+
now,
|
|
204
|
+
queued,
|
|
205
|
+
runners,
|
|
206
|
+
online,
|
|
207
|
+
agents,
|
|
208
|
+
agentById,
|
|
209
|
+
runnerUsed,
|
|
210
|
+
runnerToolUsed,
|
|
211
|
+
agentActive,
|
|
212
|
+
busyKeys,
|
|
213
|
+
position,
|
|
214
|
+
agentPosition,
|
|
215
|
+
agentQueued,
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
/** Why a queued run waits by the claim's rules alone, without reading anything more (`explainWait`). */
|
|
219
|
+
function quickWait(record, requires, state) {
|
|
220
|
+
return explainWait({ ...record, requires }, {
|
|
221
|
+
now: state.now,
|
|
222
|
+
agent: state.agentById.get(record.agentId) ?? null,
|
|
223
|
+
runners: state.online,
|
|
224
|
+
runnerUsed: state.runnerUsed,
|
|
225
|
+
runnerToolUsed: state.runnerToolUsed,
|
|
226
|
+
agentActive: state.agentActive.get(record.agentId) ?? 0,
|
|
227
|
+
sameWorkActive: state.busyKeys.has(keyOf(record)),
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
const DIAGNOSTIC_CODES = {
|
|
231
|
+
ready: 'next',
|
|
232
|
+
busy: 'runnersBusy',
|
|
233
|
+
updatePending: 'updatePending',
|
|
234
|
+
pruning: 'pruning',
|
|
235
|
+
offline: 'noRunnerOnline',
|
|
236
|
+
incompatible: 'incompatible',
|
|
237
|
+
restricted: 'restricted',
|
|
238
|
+
concurrencyFull: 'concurrencyFull',
|
|
239
|
+
sameWorkActive: 'sameWorkActive',
|
|
240
|
+
delayed: 'delayed',
|
|
241
|
+
toolUnavailable: 'toolUnavailable',
|
|
242
|
+
toolNotInstalled: 'toolNotInstalled',
|
|
243
|
+
noSharedRunner: 'noSharedRunner',
|
|
244
|
+
unknown: 'unknown',
|
|
245
|
+
};
|
|
246
|
+
/** The quick explanation refined by the claim's own read-only diagnosis of the run's candidates. */
|
|
247
|
+
function refineWait(record, previous, diagnostic, state) {
|
|
248
|
+
// Keep the established identity and protected-variable explanation, including its precise parameters.
|
|
249
|
+
const detailed = [
|
|
250
|
+
'secretsNotAllowed',
|
|
251
|
+
'delayed',
|
|
252
|
+
'agentArchived',
|
|
253
|
+
'setupRetrying',
|
|
254
|
+
].includes(previous.reason);
|
|
255
|
+
const entry = diagnostic?.entries.find((entry) => entry.reason === 'ready') ??
|
|
256
|
+
diagnostic?.entries[0];
|
|
257
|
+
if (!entry || detailed)
|
|
258
|
+
return previous;
|
|
259
|
+
const agent = state.agentById.get(record.agentId);
|
|
260
|
+
const active = state.agentActive.get(record.agentId) ?? 0;
|
|
261
|
+
const loads = entry.runners.filter((runner) => runner.reason === undefined ||
|
|
262
|
+
runner.reason === 'ready' ||
|
|
263
|
+
runner.reason === 'busy');
|
|
264
|
+
const toolFull = entry.reason === 'busy' &&
|
|
265
|
+
loads.some((runner) => (runner.used ?? 0) < (runner.limit ?? 0)) &&
|
|
266
|
+
loads.every((runner) => (runner.toolUsed ?? 0) >= (runner.toolLimit ?? 0) ||
|
|
267
|
+
runner.reportedFree === 0);
|
|
268
|
+
return {
|
|
269
|
+
...previous,
|
|
270
|
+
reason: entry.reason === 'ready' && Number(record.claimFailures) > 0
|
|
271
|
+
? 'setupRetrying'
|
|
272
|
+
: toolFull
|
|
273
|
+
? 'toolSlotsFull'
|
|
274
|
+
: DIAGNOSTIC_CODES[entry.reason],
|
|
275
|
+
params: {
|
|
276
|
+
blockers: (entry.blockers ?? [])
|
|
277
|
+
.filter((reason) => reason !== entry.reason)
|
|
278
|
+
.map((reason) => DIAGNOSTIC_CODES[reason]),
|
|
279
|
+
tool: entry.tool,
|
|
280
|
+
...(entry.reason === 'concurrencyFull'
|
|
281
|
+
? { active, limit: agent.maxConcurrentRuns }
|
|
282
|
+
: entry.reason === 'busy'
|
|
283
|
+
? {
|
|
284
|
+
used: loads.reduce((n, runner) => n + (toolFull ? (runner.toolUsed ?? 0) : (runner.used ?? 0)), 0),
|
|
285
|
+
limit: loads.reduce((n, runner) => n +
|
|
286
|
+
(toolFull ? (runner.toolLimit ?? 0) : (runner.limit ?? 0)), 0),
|
|
287
|
+
runners: loads.length,
|
|
288
|
+
reportedFree: loads.map((runner) => String(runner.reportedTotalFree ?? '—')),
|
|
289
|
+
detail: loads
|
|
290
|
+
.map((runner) => `${entry.tool} ${runner.toolUsed}/${runner.toolLimit}${runner.reportedTotalFree === 0 ? '; global free 0' : ''}`)
|
|
291
|
+
.join('; '),
|
|
292
|
+
}
|
|
293
|
+
: {}),
|
|
294
|
+
},
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
/** The newest activity of each of `runIds` in two reads: the newest sequence of each, then those events. */
|
|
298
|
+
async function newestActivities(conn, runIds) {
|
|
299
|
+
const activity = new Map();
|
|
300
|
+
if (runIds.length === 0)
|
|
301
|
+
return activity;
|
|
302
|
+
const newest = await eventsRepo(conn).groupBy({
|
|
303
|
+
by: ['runId'],
|
|
304
|
+
aggregate: (aggregate) => ({ seq: aggregate.max('seq') }),
|
|
305
|
+
filter: (f) => f.and([
|
|
306
|
+
f.or(runIds.map((id) => f.string('runId').eq(id))),
|
|
307
|
+
f.or(ACTIVITY_TYPES.map((type) => f.string('type').eq(type))),
|
|
308
|
+
]),
|
|
309
|
+
});
|
|
310
|
+
const pairs = newest.flatMap((group) => group.seq === null || group.seq === undefined
|
|
311
|
+
? []
|
|
312
|
+
: [{ runId: String(group.runId), seq: Number(group.seq) }]);
|
|
313
|
+
if (pairs.length === 0)
|
|
314
|
+
return activity;
|
|
315
|
+
const events = await eventsRepo(conn).findMany({
|
|
316
|
+
filter: (f) => f.and([
|
|
317
|
+
f.or(pairs.map((pair) => f.and([
|
|
318
|
+
f.string('runId').eq(pair.runId),
|
|
319
|
+
f.number('seq').eq(pair.seq),
|
|
320
|
+
]))),
|
|
321
|
+
f.or(ACTIVITY_TYPES.map((type) => f.string('type').eq(type))),
|
|
322
|
+
]),
|
|
323
|
+
});
|
|
324
|
+
for (const event of events)
|
|
325
|
+
activity.set(event.runId, {
|
|
326
|
+
at: iso(event.at) ?? event.at,
|
|
327
|
+
type: event.type,
|
|
328
|
+
tool: event.tool,
|
|
329
|
+
text: cut(event.content ?? event.output),
|
|
330
|
+
});
|
|
331
|
+
return activity;
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* The open runs on one kind of subject, in a fixed number of reads whatever their number. A queued run says why it
|
|
335
|
+
* waits by the claim's rules alone (`explainWait`); `readQueueWait` gives one run's exact reason.
|
|
336
|
+
*/
|
|
337
|
+
export async function readWorkload(conn, clock, runnerSource, query) {
|
|
338
|
+
const limit = Math.min(Math.max(query.limit ?? DEFAULT_LIMIT, 1), MAX_LIMIT);
|
|
339
|
+
const open = await runsRepo(conn).findMany({
|
|
340
|
+
filter: (f) => f.and([
|
|
341
|
+
f.string('subjectKind').eq(query.subjectKind),
|
|
342
|
+
f.or(['queued', ...ACTIVE].map((status) => f.string('status').eq(status))),
|
|
343
|
+
]),
|
|
344
|
+
sort: (sort) => [sort.field('createdAt').asc(), sort.field('id').asc()],
|
|
345
|
+
limit: limit + 1,
|
|
346
|
+
});
|
|
347
|
+
const truncated = open.length > limit;
|
|
348
|
+
const mine = open.slice(0, limit);
|
|
349
|
+
const state = await readQueueState(conn, clock, runnerSource);
|
|
350
|
+
const { queued, runners, online, agents, runnerUsed, agentActive } = state;
|
|
212
351
|
// The newest input's trigger, per open run.
|
|
213
352
|
const ids = mine.map((run) => run.id);
|
|
214
353
|
const triggers = new Map();
|
|
@@ -225,129 +364,25 @@ export async function readWorkload(conn, clock, runnerSource, query, diagnose) {
|
|
|
225
364
|
triggers.set(input.runId, trigger);
|
|
226
365
|
}
|
|
227
366
|
}
|
|
228
|
-
// The newest activity of each held run
|
|
229
|
-
const activity =
|
|
230
|
-
for (const run of mine) {
|
|
231
|
-
if (!ACTIVE.includes(run.status))
|
|
232
|
-
continue;
|
|
233
|
-
const event = await eventsRepo(conn).findOne({
|
|
234
|
-
filter: (f) => f.and([
|
|
235
|
-
f.string('runId').eq(run.id),
|
|
236
|
-
f.or(ACTIVITY_TYPES.map((type) => f.string('type').eq(type))),
|
|
237
|
-
]),
|
|
238
|
-
sort: (sort) => sort.field('seq').desc(),
|
|
239
|
-
});
|
|
240
|
-
if (event)
|
|
241
|
-
activity.set(run.id, {
|
|
242
|
-
at: iso(event.at) ?? event.at,
|
|
243
|
-
type: event.type,
|
|
244
|
-
tool: event.tool,
|
|
245
|
-
text: cut(event.content ?? event.output),
|
|
246
|
-
});
|
|
247
|
-
}
|
|
367
|
+
// The newest activity of each held run.
|
|
368
|
+
const activity = await newestActivities(conn, mine.filter((run) => ACTIVE.includes(run.status)).map((run) => run.id));
|
|
248
369
|
const runnerName = new Map(runners.map((runner) => [runner.id, runner.name]));
|
|
249
|
-
const runs =
|
|
370
|
+
const runs = mine.map((record) => {
|
|
250
371
|
const run = toRun(record);
|
|
251
|
-
const isQueued = record.status === 'queued';
|
|
252
|
-
const candidateAgent = agentById.get(run.agentId);
|
|
253
|
-
const diagnostic = isQueued && candidateAgent?.status === 'enabled' && diagnose
|
|
254
|
-
? await diagnose(conn, run, candidateAgent)
|
|
255
|
-
: null;
|
|
256
|
-
const previous = explainWait({ ...record, requires: run.requires }, {
|
|
257
|
-
now,
|
|
258
|
-
agent: candidateAgent ?? null,
|
|
259
|
-
runners: online,
|
|
260
|
-
runnerUsed,
|
|
261
|
-
runnerToolUsed,
|
|
262
|
-
agentActive: agentActive.get(run.agentId) ?? 0,
|
|
263
|
-
sameWorkActive: busyKeys.has(keyOf(record)),
|
|
264
|
-
});
|
|
265
|
-
// Keep the established identity and protected-variable explanation, including its precise parameters.
|
|
266
|
-
const detailed = [
|
|
267
|
-
'secretsNotAllowed',
|
|
268
|
-
'delayed',
|
|
269
|
-
'agentArchived',
|
|
270
|
-
'setupRetrying',
|
|
271
|
-
].includes(previous.reason);
|
|
272
|
-
const entry = diagnostic?.entries.find((entry) => entry.reason === 'ready') ??
|
|
273
|
-
diagnostic?.entries[0];
|
|
274
|
-
const codes = {
|
|
275
|
-
ready: 'next',
|
|
276
|
-
busy: 'runnersBusy',
|
|
277
|
-
updatePending: 'updatePending',
|
|
278
|
-
pruning: 'pruning',
|
|
279
|
-
offline: 'noRunnerOnline',
|
|
280
|
-
incompatible: 'incompatible',
|
|
281
|
-
restricted: 'restricted',
|
|
282
|
-
concurrencyFull: 'concurrencyFull',
|
|
283
|
-
sameWorkActive: 'sameWorkActive',
|
|
284
|
-
delayed: 'delayed',
|
|
285
|
-
toolUnavailable: 'toolUnavailable',
|
|
286
|
-
toolNotInstalled: 'toolNotInstalled',
|
|
287
|
-
noSharedRunner: 'noSharedRunner',
|
|
288
|
-
unknown: 'unknown',
|
|
289
|
-
};
|
|
290
|
-
const loads = entry?.runners.filter((runner) => runner.reason === undefined ||
|
|
291
|
-
runner.reason === 'ready' ||
|
|
292
|
-
runner.reason === 'busy') ?? [];
|
|
293
|
-
const toolFull = entry?.reason === 'busy' &&
|
|
294
|
-
loads.some((runner) => (runner.used ?? 0) < (runner.limit ?? 0)) &&
|
|
295
|
-
loads.every((runner) => (runner.toolUsed ?? 0) >= (runner.toolLimit ?? 0) ||
|
|
296
|
-
runner.reportedFree === 0);
|
|
297
372
|
return {
|
|
298
373
|
...run,
|
|
299
374
|
trigger: triggers.get(run.id) ?? null,
|
|
300
|
-
runnerName: run.runnerId
|
|
301
|
-
? (runnerName.get(run.runnerId) ?? null)
|
|
302
|
-
: null,
|
|
375
|
+
runnerName: run.runnerId ? (runnerName.get(run.runnerId) ?? null) : null,
|
|
303
376
|
lastActivity: activity.get(run.id) ?? null,
|
|
304
|
-
wait:
|
|
377
|
+
wait: record.status === 'queued'
|
|
305
378
|
? {
|
|
306
|
-
...
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
reason: entry.reason === 'ready' &&
|
|
310
|
-
Number(record.claimFailures) > 0
|
|
311
|
-
? 'setupRetrying'
|
|
312
|
-
: toolFull
|
|
313
|
-
? 'toolSlotsFull'
|
|
314
|
-
: codes[entry.reason],
|
|
315
|
-
params: {
|
|
316
|
-
blockers: (entry.blockers ?? [])
|
|
317
|
-
.filter((reason) => reason !== entry.reason)
|
|
318
|
-
.map((reason) => codes[reason]),
|
|
319
|
-
tool: entry.tool,
|
|
320
|
-
...(entry.reason === 'concurrencyFull'
|
|
321
|
-
? {
|
|
322
|
-
active: agentActive.get(run.agentId) ?? 0,
|
|
323
|
-
limit: candidateAgent.maxConcurrentRuns,
|
|
324
|
-
}
|
|
325
|
-
: entry.reason === 'busy'
|
|
326
|
-
? {
|
|
327
|
-
used: loads.reduce((n, runner) => n +
|
|
328
|
-
(toolFull
|
|
329
|
-
? (runner.toolUsed ?? 0)
|
|
330
|
-
: (runner.used ?? 0)), 0),
|
|
331
|
-
limit: loads.reduce((n, runner) => n +
|
|
332
|
-
(toolFull
|
|
333
|
-
? (runner.toolLimit ?? 0)
|
|
334
|
-
: (runner.limit ?? 0)), 0),
|
|
335
|
-
runners: loads.length,
|
|
336
|
-
reportedFree: loads.map((runner) => String(runner.reportedTotalFree ?? '—')),
|
|
337
|
-
detail: loads
|
|
338
|
-
.map((runner) => `${entry.tool} ${runner.toolUsed}/${runner.toolLimit}${runner.reportedTotalFree === 0 ? '; global free 0' : ''}`)
|
|
339
|
-
.join('; '),
|
|
340
|
-
}
|
|
341
|
-
: {}),
|
|
342
|
-
},
|
|
343
|
-
}
|
|
344
|
-
: {}),
|
|
345
|
-
position: position.get(run.id) ?? null,
|
|
346
|
-
agentPosition: agentPosition.get(run.id) ?? 1,
|
|
379
|
+
...quickWait(record, run.requires, state),
|
|
380
|
+
position: state.position.get(run.id) ?? null,
|
|
381
|
+
agentPosition: state.agentPosition.get(run.id) ?? 1,
|
|
347
382
|
}
|
|
348
383
|
: null,
|
|
349
384
|
};
|
|
350
|
-
})
|
|
385
|
+
});
|
|
351
386
|
// Held runs first, the longest-running first; then queued ones in claim order.
|
|
352
387
|
const order = new Map(queued.map((run, index) => [run.id, index]));
|
|
353
388
|
runs.sort((a, b) => {
|
|
@@ -368,7 +403,7 @@ export async function readWorkload(conn, clock, runnerSource, query, diagnose) {
|
|
|
368
403
|
loads.push({
|
|
369
404
|
agentId: agent.id,
|
|
370
405
|
active: agentActive.get(agent.id) ?? 0,
|
|
371
|
-
queued: agentQueued.get(agent.id) ?? 0,
|
|
406
|
+
queued: state.agentQueued.get(agent.id) ?? 0,
|
|
372
407
|
maxConcurrentRuns: agent.maxConcurrentRuns,
|
|
373
408
|
online: agent.status === 'enabled' &&
|
|
374
409
|
online.some((runner) => agent.modelEntries.some((entry) => runsEntry(runner, entry))),
|
|
@@ -387,4 +422,25 @@ export async function readWorkload(conn, clock, runnerSource, query, diagnose) {
|
|
|
387
422
|
},
|
|
388
423
|
};
|
|
389
424
|
}
|
|
425
|
+
/**
|
|
426
|
+
* Exactly why one queued run waits: the claim's rules refined by its read-only diagnosis of the run's candidate
|
|
427
|
+
* runners (`ClaimService.diagnose`). Null when the run does not exist or is not queued.
|
|
428
|
+
*/
|
|
429
|
+
export async function readQueueWait(conn, clock, runnerSource, runId, diagnose) {
|
|
430
|
+
const record = await runsRepo(conn).findOne({ filter: { id: runId } });
|
|
431
|
+
if (!record || record.status !== 'queued')
|
|
432
|
+
return null;
|
|
433
|
+
const state = await readQueueState(conn, clock, runnerSource);
|
|
434
|
+
const run = toRun(record);
|
|
435
|
+
const agent = state.agentById.get(run.agentId);
|
|
436
|
+
const previous = quickWait(record, run.requires, state);
|
|
437
|
+
const diagnostic = agent?.status === 'enabled' && diagnose
|
|
438
|
+
? await diagnose(conn, run, agent)
|
|
439
|
+
: null;
|
|
440
|
+
return {
|
|
441
|
+
...refineWait(record, previous, diagnostic, state),
|
|
442
|
+
position: state.position.get(run.id) ?? null,
|
|
443
|
+
agentPosition: state.agentPosition.get(run.id) ?? 1,
|
|
444
|
+
};
|
|
445
|
+
}
|
|
390
446
|
//# sourceMappingURL=workload.js.map
|