getbotons 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +505 -2
- package/installer/admin.mjs +6 -0
- package/installer/botons.mjs +430 -0
- package/installer/brain-clis.mjs +28 -0
- package/installer/linger.mjs +39 -0
- package/installer/output.mjs +27 -0
- package/installer/revision.mjs +30 -0
- package/installer/shell-path.mjs +66 -0
- package/installer/source-lock.json +2891 -0
- package/package.json +26 -4
- package/payload/LICENSE +21 -0
- package/payload/SECURITY.md +33 -0
- package/payload/installer/admin.mjs +6 -0
- package/payload/installer/botons.mjs +430 -0
- package/payload/installer/brain-clis.mjs +28 -0
- package/payload/installer/linger.mjs +39 -0
- package/payload/installer/output.mjs +27 -0
- package/payload/installer/revision.mjs +30 -0
- package/payload/installer/shell-path.mjs +66 -0
- package/payload/installer/source-lock.json +2891 -0
- package/payload/npm-shrinkwrap.json +51 -0
- package/payload/package.json +13 -0
- package/payload/scripts/backup.ts +47 -0
- package/payload/scripts/botons-backup.service +15 -0
- package/payload/scripts/botons-backup.timer +11 -0
- package/payload/scripts/botons.service +36 -0
- package/payload/scripts/check-browser.ts +90 -0
- package/payload/scripts/doctor.ts +790 -0
- package/payload/scripts/install-backup-timer.ts +52 -0
- package/payload/scripts/install-service.ts +206 -0
- package/payload/scripts/login.ts +17 -0
- package/payload/scripts/migrate-names.ts +47 -0
- package/payload/scripts/pair.ts +163 -0
- package/payload/scripts/restore.ts +108 -0
- package/payload/server/bin/botons +4 -0
- package/payload/server/bin/legacy/minion +4 -0
- package/payload/server/bin/legacy/tots +4 -0
- package/payload/server/src/acting.ts +16 -0
- package/payload/server/src/api-keys.ts +544 -0
- package/payload/server/src/api.ts +1484 -0
- package/payload/server/src/app.ts +546 -0
- package/payload/server/src/approvals.ts +873 -0
- package/payload/server/src/ask-rules.ts +181 -0
- package/payload/server/src/audit.ts +158 -0
- package/payload/server/src/auth.ts +497 -0
- package/payload/server/src/backup.ts +382 -0
- package/payload/server/src/brain-login.ts +75 -0
- package/payload/server/src/browser-cert.ts +67 -0
- package/payload/server/src/browser-handoff.ts +119 -0
- package/payload/server/src/browser-handover.ts +175 -0
- package/payload/server/src/browser-policy.ts +210 -0
- package/payload/server/src/browser-proxy.ts +580 -0
- package/payload/server/src/browser-request.ts +335 -0
- package/payload/server/src/browser-screenshot.ts +56 -0
- package/payload/server/src/browser-send.ts +155 -0
- package/payload/server/src/browser.ts +2164 -0
- package/payload/server/src/callback.ts +218 -0
- package/payload/server/src/capabilities.ts +90 -0
- package/payload/server/src/cards.ts +243 -0
- package/payload/server/src/cli/approve-mcp.ts +119 -0
- package/payload/server/src/cli/botons.ts +629 -0
- package/payload/server/src/cli/browser-mcp.ts +168 -0
- package/payload/server/src/cli/workspace-file.ts +78 -0
- package/payload/server/src/cli-homes.ts +38 -0
- package/payload/server/src/cli-install.ts +104 -0
- package/payload/server/src/cli-updates.ts +112 -0
- package/payload/server/src/codex-config-repair.ts +80 -0
- package/payload/server/src/config.ts +473 -0
- package/payload/server/src/connector-access.ts +158 -0
- package/payload/server/src/connector-launch.ts +203 -0
- package/payload/server/src/connectors.ts +947 -0
- package/payload/server/src/disk-usage.ts +62 -0
- package/payload/server/src/engine.ts +2328 -0
- package/payload/server/src/env-init.ts +5 -0
- package/payload/server/src/events.ts +46 -0
- package/payload/server/src/files.ts +528 -0
- package/payload/server/src/github-access.ts +52 -0
- package/payload/server/src/health.ts +117 -0
- package/payload/server/src/held-dir.ts +96 -0
- package/payload/server/src/hooks.ts +141 -0
- package/payload/server/src/http.ts +207 -0
- package/payload/server/src/index.ts +139 -0
- package/payload/server/src/install-type.ts +11 -0
- package/payload/server/src/instructions.ts +204 -0
- package/payload/server/src/isolation.ts +234 -0
- package/payload/server/src/legacy-upgrade.ts +668 -0
- package/payload/server/src/limits.ts +44 -0
- package/payload/server/src/lock.ts +75 -0
- package/payload/server/src/log.ts +137 -0
- package/payload/server/src/managed-clis.ts +65 -0
- package/payload/server/src/mcp-client.ts +229 -0
- package/payload/server/src/notify.ts +193 -0
- package/payload/server/src/oauth-connections.ts +650 -0
- package/payload/server/src/oauth.ts +311 -0
- package/payload/server/src/outbound.ts +142 -0
- package/payload/server/src/portable.ts +1669 -0
- package/payload/server/src/prompt.ts +676 -0
- package/payload/server/src/providers/api-brain.ts +612 -0
- package/payload/server/src/providers/api-history.ts +131 -0
- package/payload/server/src/providers/api-tools.ts +481 -0
- package/payload/server/src/providers/api-wire.ts +690 -0
- package/payload/server/src/providers/claude-cli.ts +620 -0
- package/payload/server/src/providers/cli-check.ts +265 -0
- package/payload/server/src/providers/cli-turn.ts +20 -0
- package/payload/server/src/providers/codex-cli.ts +552 -0
- package/payload/server/src/providers/codex-workspace.ts +61 -0
- package/payload/server/src/providers/failures.ts +143 -0
- package/payload/server/src/providers/index.ts +131 -0
- package/payload/server/src/providers/mcp-stdio.ts +147 -0
- package/payload/server/src/providers/mock.ts +94 -0
- package/payload/server/src/providers/process.ts +268 -0
- package/payload/server/src/providers/readiness.ts +43 -0
- package/payload/server/src/providers/sse.ts +73 -0
- package/payload/server/src/providers/types.ts +173 -0
- package/payload/server/src/rates.ts +128 -0
- package/payload/server/src/reactions.ts +266 -0
- package/payload/server/src/recent-logs.ts +489 -0
- package/payload/server/src/rehearsal.ts +593 -0
- package/payload/server/src/rehearsals.ts +303 -0
- package/payload/server/src/replay.ts +165 -0
- package/payload/server/src/sandbox.ts +767 -0
- package/payload/server/src/schedule.ts +214 -0
- package/payload/server/src/scheduler.ts +24 -0
- package/payload/server/src/schema.ts +882 -0
- package/payload/server/src/secrets.ts +46 -0
- package/payload/server/src/setup.ts +356 -0
- package/payload/server/src/skill-packages.ts +363 -0
- package/payload/server/src/skills.ts +527 -0
- package/payload/server/src/spend.ts +255 -0
- package/payload/server/src/ssrf.ts +126 -0
- package/payload/server/src/starter-gate.ts +168 -0
- package/payload/server/src/starters.ts +610 -0
- package/payload/server/src/storage.ts +32 -0
- package/payload/server/src/store.ts +3087 -0
- package/payload/server/src/tailnet.ts +132 -0
- package/payload/server/src/templates.ts +454 -0
- package/payload/server/src/trash.ts +772 -0
- package/payload/server/src/util.ts +97 -0
- package/payload/server/src/webhooks.ts +216 -0
- package/payload/server/src/work-feed.ts +473 -0
- package/payload/shared/app-version.d.mts +1 -0
- package/payload/shared/app-version.mjs +3 -0
- package/payload/shared/boton.ts +323 -0
- package/payload/shared/brand.ts +107 -0
- package/payload/shared/cards.ts +427 -0
- package/payload/shared/effort.ts +91 -0
- package/payload/shared/files.ts +104 -0
- package/payload/shared/handoff.ts +97 -0
- package/payload/shared/install-commands.ts +7 -0
- package/payload/shared/legacy-names.d.mts +35 -0
- package/payload/shared/legacy-names.mjs +244 -0
- package/payload/shared/live-view.ts +94 -0
- package/payload/shared/look-critter.ts +189 -0
- package/payload/shared/look-tater.ts +127 -0
- package/payload/shared/look.ts +200 -0
- package/payload/shared/reactions.ts +43 -0
- package/payload/shared/seeded.ts +34 -0
- package/payload/shared/services.ts +90 -0
- package/payload/shared/types.ts +3016 -0
- package/payload/shared/util.ts +68 -0
- package/payload/shared/versions.ts +44 -0
- package/payload/shared/wake.ts +164 -0
- package/payload/web/dist/apple-touch-icon.png +0 -0
- package/payload/web/dist/assets/Icons-nytVAa9g.js +1 -0
- package/payload/web/dist/assets/NewBotDialog-DXr2XhDP.js +1 -0
- package/payload/web/dist/assets/PairScreen-CMcJO7cQ.js +1 -0
- package/payload/web/dist/assets/Panel-Bvqz3I3m.js +4 -0
- package/payload/web/dist/assets/ReactionPicker-B0SKsjiy.js +1 -0
- package/payload/web/dist/assets/SettingRows-CvLomtuo.js +4 -0
- package/payload/web/dist/assets/SettingsDialog-CQ41LV-P.js +8 -0
- package/payload/web/dist/assets/Setup-C-3P7Mtn.js +1 -0
- package/payload/web/dist/assets/Skills-DXMH69kH.js +1 -0
- package/payload/web/dist/assets/StarterSetup-Ce2V1_Cs.js +5 -0
- package/payload/web/dist/assets/Steps-CKtB3GAM.js +1 -0
- package/payload/web/dist/assets/emoji-catalog-DfpPz8qw.js +540 -0
- package/payload/web/dist/assets/index-BEU7FDxV.css +1 -0
- package/payload/web/dist/assets/index-Dhwn-rf0.js +3 -0
- package/payload/web/dist/assets/react-C1iE8mnY.js +9 -0
- package/payload/web/dist/assets/reactions-rw0YeCmA.js +1 -0
- package/payload/web/dist/assets/rolldown-runtime-XKyaHtl9.js +1 -0
- package/payload/web/dist/assets/util-Cx5feSJu.js +1 -0
- package/payload/web/dist/assets/vendor-B3qdcjGo.js +14 -0
- package/payload/web/dist/favicon.svg +1 -0
- package/payload/web/dist/icon-192.png +0 -0
- package/payload/web/dist/icon-512.png +0 -0
- package/payload/web/dist/icon.svg +1 -0
- package/payload/web/dist/index.html +67 -0
- package/payload/web/dist/manifest.webmanifest +12 -0
- package/shared/legacy-names.d.mts +35 -0
- package/shared/legacy-names.mjs +244 -0
|
@@ -0,0 +1,2328 @@
|
|
|
1
|
+
import { cliHomesEnv } from './cli-homes.ts';
|
|
2
|
+
import { withLegacyTurnNames } from '../../shared/legacy-names.mjs';
|
|
3
|
+
import { createHash } from 'node:crypto';
|
|
4
|
+
import { mkdirSync } from 'node:fs';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import type {
|
|
7
|
+
AccessLevel,
|
|
8
|
+
ApprovalOrigin,
|
|
9
|
+
AskResult,
|
|
10
|
+
Bot,
|
|
11
|
+
BotStatus,
|
|
12
|
+
CancelRunsResult,
|
|
13
|
+
FileRef,
|
|
14
|
+
Message,
|
|
15
|
+
MessageMeta,
|
|
16
|
+
NotifyPolicy,
|
|
17
|
+
NotifyResult,
|
|
18
|
+
BrainRef,
|
|
19
|
+
ProviderHandoff,
|
|
20
|
+
ProviderSwitchInput,
|
|
21
|
+
ProviderSwitchPreview,
|
|
22
|
+
SwitchSummaryOffer,
|
|
23
|
+
BrainReadiness,
|
|
24
|
+
Routine,
|
|
25
|
+
RoutineRun,
|
|
26
|
+
RunTrigger,
|
|
27
|
+
Settings,
|
|
28
|
+
ShareResult,
|
|
29
|
+
StopResult,
|
|
30
|
+
TokenUsage,
|
|
31
|
+
TurnDraft,
|
|
32
|
+
TurnOutcome,
|
|
33
|
+
TurnStep,
|
|
34
|
+
UpdateBotInput,
|
|
35
|
+
} from '../../shared/types.ts';
|
|
36
|
+
import { effortProblem, type ModelEfforts } from '../../shared/effort.ts';
|
|
37
|
+
import { BROWSER_CONNECTOR, BROWSER_SERVER, durationText, isPastOneOff, totalTokens } from '../../shared/util.ts';
|
|
38
|
+
import type { Approvals } from './approvals.ts';
|
|
39
|
+
import type { Browser } from './browser.ts';
|
|
40
|
+
import { splitRules } from './ask-rules.ts';
|
|
41
|
+
import type { Auth } from './auth.ts';
|
|
42
|
+
import type { Config } from './config.ts';
|
|
43
|
+
import { type Connectors, OFF_REASON, type ResolvedConnectors } from './connectors.ts';
|
|
44
|
+
import type { LaunchPolicy } from './connector-launch.ts';
|
|
45
|
+
import type { EventHub } from './events.ts';
|
|
46
|
+
import { fileRef, resolveInWorkspace } from './files.ts';
|
|
47
|
+
import { snapshotInstructions } from './instructions.ts';
|
|
48
|
+
import { privatePaths } from './isolation.ts';
|
|
49
|
+
import { RunLimiter } from './limits.ts';
|
|
50
|
+
import { type LogFields, log } from './log.ts';
|
|
51
|
+
import type { Notifier, PushRequest } from './notify.ts';
|
|
52
|
+
import {
|
|
53
|
+
ASK_HOP_LIMIT,
|
|
54
|
+
attachmentNote,
|
|
55
|
+
chatInput,
|
|
56
|
+
reactionFollowUpPrompt,
|
|
57
|
+
chatTrigger,
|
|
58
|
+
contextBlock,
|
|
59
|
+
parseQuiet,
|
|
60
|
+
routinePrompt,
|
|
61
|
+
sessionKey,
|
|
62
|
+
sessionStart,
|
|
63
|
+
type TurnBrain,
|
|
64
|
+
skillInvocation,
|
|
65
|
+
summaryRequest,
|
|
66
|
+
systemPrompt,
|
|
67
|
+
type Trigger,
|
|
68
|
+
} from './prompt.ts';
|
|
69
|
+
import type { ProviderRegistry } from './providers/index.ts';
|
|
70
|
+
import { DEFAULT_COOLDOWN_MS, MIN_COOLDOWN_MS } from './providers/failures.ts';
|
|
71
|
+
import { type McpStartup, type Provider, ProviderError, type TurnResult, type TurnUsage } from './providers/types.ts';
|
|
72
|
+
import type { Skills } from './skills.ts';
|
|
73
|
+
import type { StarterGate } from './starter-gate.ts';
|
|
74
|
+
import { excerptOf, type Feedback, ownersOwn, type Reactions } from './reactions.ts';
|
|
75
|
+
import { type WorkFeed, workEventPrompt } from './work-feed.ts';
|
|
76
|
+
import { MESSAGE_LIMIT, PIN_LIMIT, REPLAY_BUDGET, type Replayable, WEBHOOK_HISTORY_BUDGET, WEBHOOK_HISTORY_MAX } from './replay.ts';
|
|
77
|
+
import { type BotRecord, HttpError, runChip, type Store, type TurnStamp } from './store.ts';
|
|
78
|
+
|
|
79
|
+
/** A switch preview as the server checks it: all but the replay's estimate, which no confirmation binds. */
|
|
80
|
+
type SwitchFacts = Omit<ProviderSwitchPreview, 'replayNow'>;
|
|
81
|
+
|
|
82
|
+
type PendingHandoff = NonNullable<ReturnType<Store['pendingHandoff']>>;
|
|
83
|
+
/** What the summary offer goes by of the Boton's provider now (see Engine.#switchBrains). */
|
|
84
|
+
type OldBrain = Pick<BrainReadiness, 'state' | 'problem'>;
|
|
85
|
+
import { clipText, newId, nowIso, orNull } from './util.ts';
|
|
86
|
+
|
|
87
|
+
type Work = { kind: 'chat' } | { kind: 'routine'; runId: string };
|
|
88
|
+
|
|
89
|
+
/** A Boton's job, from when it starts waiting for a slot until it ends. */
|
|
90
|
+
interface Job {
|
|
91
|
+
wait: AbortController;
|
|
92
|
+
/** The queued run this job waits for a slot to start. */
|
|
93
|
+
runId?: string;
|
|
94
|
+
turn?: AbortController;
|
|
95
|
+
run?: { runId: string; routineId: string; routineName: string; trigger: RunTrigger };
|
|
96
|
+
/**
|
|
97
|
+
* Why this job was ended (Stop, a pause, Cancel), kept for a turn that has no agent process yet (its connectors are
|
|
98
|
+
* still being set up): #invoke ends it before one starts. The first reason stays.
|
|
99
|
+
*/
|
|
100
|
+
aborted?: Error;
|
|
101
|
+
/** Its turn handed the agent CLI its prompt, so a run it ends may have partly run. */
|
|
102
|
+
spawned?: boolean;
|
|
103
|
+
/**
|
|
104
|
+
* What its turn read, set once with the read and before anything awaits: the Boton (its stamp), the handoff note
|
|
105
|
+
* pending then, and the session it runs on (a routine's own, or the chat's). A switch while it sets up or runs (even
|
|
106
|
+
* away and back) keeps its session and its note's ack out, and ends the saved session it is on (see updateBot).
|
|
107
|
+
*/
|
|
108
|
+
read?: Readonly<{ stamp: TurnStamp; handoff: PendingHandoff | null; own: string | null; fresh: boolean }>;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
type Chain = { threadId: string; hop: number };
|
|
112
|
+
|
|
113
|
+
export const ROUTINE_QUEUE_MAX = 3;
|
|
114
|
+
const QUEUE_FULL_RETRY_S = 60;
|
|
115
|
+
const CHIP_EVERY_MS = 3_600_000;
|
|
116
|
+
const ASK_MAX = 20_000;
|
|
117
|
+
export const DRAFT_EVERY_MS = 150;
|
|
118
|
+
const NOTIFY_PUSHES_PER_HOUR = 10;
|
|
119
|
+
const CUT_OFF_LIMIT = 3;
|
|
120
|
+
/** Each list in a handoff note holds at most this many. */
|
|
121
|
+
const HANDOFF_ITEMS = 5;
|
|
122
|
+
/** The summary a move asks the old session for: its turn's time limit, and how much of what it writes is kept. */
|
|
123
|
+
export const SWITCH_SUMMARY_MS = 120_000;
|
|
124
|
+
export const SWITCH_SUMMARY_CHARS = 2000;
|
|
125
|
+
const RETRY_STEPS = [1, 5, 15];
|
|
126
|
+
const DRAIN_GRACE_MS = 10_000;
|
|
127
|
+
/** How long a Cancel waits for the run's agent process to exit (process.ts kills its group within 3 s). */
|
|
128
|
+
const CANCEL_WAIT_MS = 10_000;
|
|
129
|
+
|
|
130
|
+
/** Abort reasons: a shutdown leaves the turn's work for the next server process; Stop or a pause ends it; Cancel ends one run. */
|
|
131
|
+
const INTERRUPTED = new Error('Interrupted by a restart');
|
|
132
|
+
const STOPPED = new Error('Stopped');
|
|
133
|
+
/** Why a move's summary was stopped by a pause (see #halt). */
|
|
134
|
+
const PAUSED = new Error('Paused');
|
|
135
|
+
/** A move's summary refused once it had a slot: what it was asked for changed while it waited, and no turn ran. */
|
|
136
|
+
const NOT_STARTED = new Error('Not started');
|
|
137
|
+
const CANCELLED = new Error('Cancelled');
|
|
138
|
+
|
|
139
|
+
/** What ended the job's turn, even one ended before it had an agent process to stop. */
|
|
140
|
+
const abortReason = (job: Job): unknown => job.aborted ?? job.turn?.signal.reason;
|
|
141
|
+
|
|
142
|
+
/** A turn that ended [quiet] on a reaction, with every reacted-to message of the owner's answered. */
|
|
143
|
+
const QUIET_ONLY = /^\[quiet\]$/i;
|
|
144
|
+
/** Ahead of the messages a turn reacted around without answering. Turn text: it starts no new session. */
|
|
145
|
+
const unansweredNote = (owner: string) =>
|
|
146
|
+
`(Note from the Botons server: your last turn reacted to some of ${owner}'s messages and ended [quiet], so the ones below that got no reaction have no answer yet. Answer them in words: a question, request, decision or error always gets words.)`;
|
|
147
|
+
|
|
148
|
+
export const INTERRUPTED_NOTE =
|
|
149
|
+
'(Note from the Botons server: your previous attempt at some or all of the messages below was cut off, by a server restart or a usage limit, so it may have partly run. Check what was already done before you repeat anything with side effects.)';
|
|
150
|
+
|
|
151
|
+
const pairKey = (a: string, b: string) => (a < b ? `${a}|${b}` : `${b}|${a}`);
|
|
152
|
+
|
|
153
|
+
function threadOf(messages: Message[]): Chain | null {
|
|
154
|
+
let best: Chain | null = null;
|
|
155
|
+
for (const m of messages) {
|
|
156
|
+
if (!m.meta?.fromBotId || !m.meta.threadId) continue;
|
|
157
|
+
const hop = m.meta.hop ?? 1;
|
|
158
|
+
if (!best || hop > best.hop) best = { threadId: m.meta.threadId, hop };
|
|
159
|
+
}
|
|
160
|
+
return best;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
interface TurnInfo {
|
|
164
|
+
/** Its row in the turns table; "" until it starts. */
|
|
165
|
+
id: string;
|
|
166
|
+
model: string;
|
|
167
|
+
/** The level it asked for; "": the provider's default. */
|
|
168
|
+
effort: string;
|
|
169
|
+
usage: TurnUsage | null;
|
|
170
|
+
steps: number;
|
|
171
|
+
/** Its time limit, in ms: set as it starts. */
|
|
172
|
+
limitMs: number;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* The level a turn asks for: its routine's own, else the Boton's (`effort`), with what the model's own list says
|
|
177
|
+
* (`known`, see shared/effort.ts). It asks for none ("", the CLI's default) for a level the provider or that list
|
|
178
|
+
* doesn't have, which the store refuses to save but which a Boton's change or a new model list can leave behind; the
|
|
179
|
+
* level it would have asked for is `dropped`.
|
|
180
|
+
*/
|
|
181
|
+
function turnEffort(bot: Pick<BotRecord, 'provider' | 'effort'>, routine: Pick<Routine, 'effort'> | null, known: ModelEfforts | null): { effort: string; dropped: string } {
|
|
182
|
+
const effort = routine?.effort || bot.effort;
|
|
183
|
+
return effort && effortProblem(bot.provider, effort, known) ? { effort: '', dropped: effort } : { effort, dropped: '' };
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const newTurnInfo = (): TurnInfo => ({ id: '', model: '', effort: '', usage: null, steps: 0, limitMs: 0 });
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* The turn's agent process ran past its time limit before anything else ended it (runJsonl, and the mock, keep only
|
|
190
|
+
* the first of a timeout and a stop), so a stop, cancel or shutdown that came while it was exiting doesn't change why.
|
|
191
|
+
*/
|
|
192
|
+
const timedOut = (err: unknown): boolean => err instanceof ProviderError && !!err.exit?.timedOut;
|
|
193
|
+
|
|
194
|
+
/** Text that is, or may yet become, a quiet routine reply ("[quiet] …"): a routine run doesn't show it as a draft. */
|
|
195
|
+
function mayBeQuiet(text: string): boolean {
|
|
196
|
+
const t = text.replace(/^[\s*_`]+/, '').toLowerCase();
|
|
197
|
+
return t.length < 7 ? '[quiet]'.startsWith(t) : t.startsWith('[quiet]');
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* A webhook run's payload (or a webhook routine's test run's) comes from outside. Sandboxed (outside, with
|
|
202
|
+
* Workspace access): its token can't change routines or memories or ask other Botons, and it gets no shared
|
|
203
|
+
* folder, so the payload can't reach later turns or other Botons.
|
|
204
|
+
*/
|
|
205
|
+
function turnScope(bot: BotRecord, trigger: Trigger): { access: AccessLevel; outside: boolean; sandboxed: boolean } {
|
|
206
|
+
if (trigger.kind === 'chat') return { access: bot.access, outside: false, sandboxed: false };
|
|
207
|
+
const outside = fromOutside(trigger.routine, trigger.run.trigger);
|
|
208
|
+
const access = bot.access === 'host' && outside && trigger.routine.webhookWorkspace ? 'workspace' : bot.access;
|
|
209
|
+
return { access, outside, sandboxed: outside && access === 'workspace' };
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
const fromOutside = (routine: Pick<Routine, 'webhook'>, trigger: RunTrigger): boolean => trigger === 'webhook' || (trigger === 'test' && !!routine.webhook);
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* The session a routine's run is on. A webhook run never carries on the chat's session, whatever the routine's
|
|
216
|
+
* `session` says, since a session holds all the chat it has seen: it gets a fresh session that ends with the run
|
|
217
|
+
* ('fresh'), given only the chat's latest `webhookHistory` messages (the owner's choice, none by default), so no run
|
|
218
|
+
* sees more of the chat than that. A routine's own session holds none of the chat, so one without that history
|
|
219
|
+
* carries on there.
|
|
220
|
+
*/
|
|
221
|
+
export function runSession(routine: Pick<Routine, 'session' | 'webhookHistory' | 'webhook'>, trigger: RunTrigger): 'chat' | 'own' | 'fresh' {
|
|
222
|
+
if (!fromOutside(routine, trigger)) return routine.session;
|
|
223
|
+
return routine.session === 'own' && routine.webhookHistory === 0 ? 'own' : 'fresh';
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** The first `n` of a lazy list: read no further. */
|
|
227
|
+
function* take(list: Iterable<Replayable>, n: number): Generator<Replayable> {
|
|
228
|
+
if (n <= 0) return;
|
|
229
|
+
let i = 0;
|
|
230
|
+
for (const x of list) {
|
|
231
|
+
yield x;
|
|
232
|
+
if (++i >= n) return;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
class Semaphore {
|
|
237
|
+
#free: number;
|
|
238
|
+
#waiting: Array<() => void> = [];
|
|
239
|
+
constructor(n: number) {
|
|
240
|
+
this.#free = n;
|
|
241
|
+
}
|
|
242
|
+
acquire(signal: AbortSignal): Promise<boolean> {
|
|
243
|
+
if (this.#free > 0) {
|
|
244
|
+
this.#free--;
|
|
245
|
+
return Promise.resolve(true);
|
|
246
|
+
}
|
|
247
|
+
return new Promise<boolean>((resolve) => {
|
|
248
|
+
const granted = () => {
|
|
249
|
+
signal.removeEventListener('abort', onAbort);
|
|
250
|
+
resolve(true);
|
|
251
|
+
};
|
|
252
|
+
const onAbort = () => {
|
|
253
|
+
const i = this.#waiting.indexOf(granted);
|
|
254
|
+
if (i >= 0) this.#waiting.splice(i, 1);
|
|
255
|
+
resolve(false);
|
|
256
|
+
};
|
|
257
|
+
this.#waiting.push(granted);
|
|
258
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
release(): void {
|
|
262
|
+
const next = this.#waiting.shift();
|
|
263
|
+
if (next) next();
|
|
264
|
+
else this.#free++;
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** One serial queue per Boton (chat and routine runs share its conversation); a semaphore caps agent processes. */
|
|
269
|
+
export class Engine {
|
|
270
|
+
store!: Store;
|
|
271
|
+
auth!: Auth;
|
|
272
|
+
notifier!: Notifier;
|
|
273
|
+
connectors!: Connectors;
|
|
274
|
+
skills!: Skills;
|
|
275
|
+
approvals!: Approvals;
|
|
276
|
+
reactions!: Reactions;
|
|
277
|
+
browser?: Browser;
|
|
278
|
+
workFeed!: WorkFeed;
|
|
279
|
+
/** Notes what a starter's test runs ran on (starter-gate.ts). */
|
|
280
|
+
starterGate?: StarterGate;
|
|
281
|
+
#cfg: Config;
|
|
282
|
+
#hub: EventHub;
|
|
283
|
+
#providers: ProviderRegistry;
|
|
284
|
+
#status = new Map<string, BotStatus>();
|
|
285
|
+
#chains = new Map<string, Promise<void>>();
|
|
286
|
+
#queued = new Map<string, number>();
|
|
287
|
+
/** Per Boton, the owner's messages a turn ended [quiet] on after reacting to others only: they still want words. */
|
|
288
|
+
#unanswered = new Map<string, Set<string>>();
|
|
289
|
+
#jobs = new Map<string, Job>();
|
|
290
|
+
/** A move waiting for the old session's summary holds its Boton (see #moveWithSummary). */
|
|
291
|
+
#summaries = new Map<string, { hold: symbol; controller: AbortController }>();
|
|
292
|
+
/** While a turn's `botons` token is valid. `bot` as the turn read it, so what it posts is marked stale like its reply. */
|
|
293
|
+
#turns = new Map<string, { bot: BotRecord; origin: ApprovalOrigin; access: AccessLevel; chain: Chain | null; connectors: string[] }>();
|
|
294
|
+
#slots: Semaphore;
|
|
295
|
+
#draining = false;
|
|
296
|
+
#maintenance = false;
|
|
297
|
+
#closing = new AbortController();
|
|
298
|
+
#interruptedCount = 0;
|
|
299
|
+
#notifyPushes = new Map<string, number[]>();
|
|
300
|
+
#limiter: RunLimiter;
|
|
301
|
+
#askLimiter: RunLimiter;
|
|
302
|
+
#chipAt = new Map<string, number>();
|
|
303
|
+
#cooldowns = new Map<string, { until: Date; reason: string }>();
|
|
304
|
+
#noticed = new Map<string, number>();
|
|
305
|
+
#timers = new Map<string, NodeJS.Timeout>();
|
|
306
|
+
|
|
307
|
+
constructor(cfg: Config, hub: EventHub, providers: ProviderRegistry) {
|
|
308
|
+
this.#cfg = cfg;
|
|
309
|
+
this.#hub = hub;
|
|
310
|
+
this.#providers = providers;
|
|
311
|
+
this.#slots = new Semaphore(cfg.maxConcurrentTurns);
|
|
312
|
+
this.#limiter = new RunLimiter(cfg.routineRunsPerHour, cfg.runsPerHour);
|
|
313
|
+
this.#askLimiter = new RunLimiter(cfg.asksPerPairPerHour, cfg.asksPerHour);
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
statusOf = (botId: string): BotStatus => this.#status.get(botId) ?? { state: 'idle', activity: null, since: null };
|
|
317
|
+
|
|
318
|
+
turnOrigin = (botId: string): ApprovalOrigin | null => this.#turns.get(botId)?.origin ?? null;
|
|
319
|
+
|
|
320
|
+
turnAccess = (botId: string): AccessLevel | null => this.#turns.get(botId)?.access ?? null;
|
|
321
|
+
/** The turns running now that were given connector `id`: turning it off doesn't reach into those turns. */
|
|
322
|
+
turnsWith = (id: string): { botId: string; turnId: string }[] => [...this.#turns].flatMap(([botId, t]) => (t.connectors.includes(id) && t.origin.turnId ? [{ botId, turnId: t.origin.turnId }] : []));
|
|
323
|
+
|
|
324
|
+
queueChat(botId: string): void {
|
|
325
|
+
if (this.store.hasBot(botId) && !this.store.pausedReason(botId)) this.#enqueue(botId, { kind: 'chat' });
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
#setStatus(botId: string, state: BotStatus['state'], activity: string | null = null) {
|
|
329
|
+
const prev = this.statusOf(botId);
|
|
330
|
+
if (prev.state === state && prev.activity === activity) return;
|
|
331
|
+
const status: BotStatus = { state, activity, since: prev.state === state && prev.since ? prev.since : nowIso() };
|
|
332
|
+
if (state === 'working' && prev.state === 'working' && prev.draft) status.draft = prev.draft;
|
|
333
|
+
if (state === 'idle') this.#status.delete(botId);
|
|
334
|
+
else this.#status.set(botId, status);
|
|
335
|
+
this.#hub.publish({ type: 'bot.status', botId, status: this.statusOf(botId) });
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/** `end()` drops the draft without an event: the turn's message (same turnId) replaces it on clients. */
|
|
339
|
+
#drafts(botId: string, turnId: string, routine: boolean): { push: (text: string) => void; end: () => void } {
|
|
340
|
+
let pending: string | null = null;
|
|
341
|
+
let last = 0;
|
|
342
|
+
let timer: NodeJS.Timeout | undefined;
|
|
343
|
+
let ended = false;
|
|
344
|
+
const publish = () => {
|
|
345
|
+
timer = undefined;
|
|
346
|
+
const s = this.#status.get(botId);
|
|
347
|
+
if (pending === null || ended || s?.state !== 'working') return;
|
|
348
|
+
const draft: TurnDraft = { turnId, text: pending, at: nowIso() };
|
|
349
|
+
pending = null;
|
|
350
|
+
last = Date.now();
|
|
351
|
+
this.#status.set(botId, { ...s, draft });
|
|
352
|
+
this.#hub.publish({ type: 'message.draft', botId, draft });
|
|
353
|
+
};
|
|
354
|
+
return {
|
|
355
|
+
push: (text) => {
|
|
356
|
+
if (ended || !text.trim() || (routine && mayBeQuiet(text))) return;
|
|
357
|
+
pending = text;
|
|
358
|
+
if (timer) return;
|
|
359
|
+
const wait = last + DRAFT_EVERY_MS - Date.now();
|
|
360
|
+
if (wait <= 0) publish();
|
|
361
|
+
else timer = setTimeout(publish, wait);
|
|
362
|
+
},
|
|
363
|
+
end: () => {
|
|
364
|
+
ended = true;
|
|
365
|
+
clearTimeout(timer);
|
|
366
|
+
const s = this.#status.get(botId);
|
|
367
|
+
if (s?.draft?.turnId === turnId) this.#status.set(botId, { ...s, draft: null });
|
|
368
|
+
},
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
#chipDue(key: string): boolean {
|
|
373
|
+
const now = Date.now();
|
|
374
|
+
if (now - (this.#chipAt.get(key) ?? 0) < CHIP_EVERY_MS) return false;
|
|
375
|
+
this.#chipAt.set(key, now);
|
|
376
|
+
return true;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// ------------------------------------------------------------------ inputs
|
|
380
|
+
|
|
381
|
+
sendMessage(botId: string, text: string, attachments: FileRef[] = []): Message {
|
|
382
|
+
this.#refuseWhileDraining();
|
|
383
|
+
const t = text.trim();
|
|
384
|
+
if (!t && !attachments.length) throw new HttpError(400, 'text is required');
|
|
385
|
+
if (t.length > 50_000) throw new HttpError(400, 'Message is too long');
|
|
386
|
+
this.store.getBot(botId);
|
|
387
|
+
const message = this.store.addMessage(botId, 'user', t, attachments.length ? { attachments } : null, true);
|
|
388
|
+
if (!this.store.pausedReason(botId)) this.#enqueue(botId, { kind: 'chat' });
|
|
389
|
+
return message;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/** Nothing waits for the answer (it comes back as a message, see #answerAsks), so two Botons asking each other can't deadlock. */
|
|
393
|
+
ask(fromId: string, toId: string, text: unknown): AskResult {
|
|
394
|
+
this.#refuseWhileDraining();
|
|
395
|
+
if (typeof text !== 'string' || !text.trim()) throw new HttpError(400, 'text is required');
|
|
396
|
+
const t = text.trim();
|
|
397
|
+
if (t.length > ASK_MAX) throw new HttpError(400, `text is too long (max ${ASK_MAX} characters)`);
|
|
398
|
+
const from = this.store.getBot(fromId);
|
|
399
|
+
if (toId === fromId) throw new HttpError(400, "You can't ask yourself. Do the work in this turn instead.");
|
|
400
|
+
if (!this.store.hasBot(toId)) throw new HttpError(404, `No Boton with id ${toId}. Run \`botons team\` to see who is on the team.`);
|
|
401
|
+
const to = this.store.getBot(toId);
|
|
402
|
+
const running = this.#turns.get(fromId);
|
|
403
|
+
const hop = (running?.chain?.hop ?? 0) + 1;
|
|
404
|
+
const threadId = running?.chain?.threadId ?? newId('thr');
|
|
405
|
+
const about: MessageMeta = { toBotId: to.id, toBotName: to.name, askText: t, threadId, hop };
|
|
406
|
+
const chip = (kind: 'ask.refused' | 'ask.sent', label: string) => this.store.addMessage(from.id, 'event', label, this.#postMeta(running?.bot ?? from, { kind, ...about }));
|
|
407
|
+
const fields: LogFields = { botId: from.id, toBotId: to.id, threadId, hop };
|
|
408
|
+
|
|
409
|
+
if (hop > ASK_HOP_LIMIT) {
|
|
410
|
+
chip('ask.refused', `Didn't ask ${to.name}: ${ASK_HOP_LIMIT} asks in a row is the limit`);
|
|
411
|
+
log.info('ask refused: hop limit', fields);
|
|
412
|
+
const owner = this.store.getSettings().ownerName;
|
|
413
|
+
throw new HttpError(
|
|
414
|
+
409,
|
|
415
|
+
`${to.name} was not asked: this thread between Botons already has ${ASK_HOP_LIMIT} asks in a row, the limit. Put what you have in your reply instead; ${owner} can see it in your chat.`,
|
|
416
|
+
);
|
|
417
|
+
}
|
|
418
|
+
const pair = pairKey(from.id, to.id);
|
|
419
|
+
const hit = this.#askLimiter.check(pair);
|
|
420
|
+
if (hit) {
|
|
421
|
+
const { s, wait } = retryAfter(hit.retryAfterMs);
|
|
422
|
+
const between = hit.limit === 'routine' ? `${from.name} and ${to.name}` : 'all Botons';
|
|
423
|
+
if (this.#chipDue(pair)) chip('ask.refused', `Didn't ask ${to.name}: too many asks (limit: ${hit.max} an hour between ${between})`);
|
|
424
|
+
log.warn('ask refused: rate limit', { ...fields, limit: hit.limit === 'routine' ? 'pair' : 'total', max: hit.max });
|
|
425
|
+
throw new HttpError(429, `${to.name} was not asked: ${between} have sent ${hit.max} asks in the last hour, the limit. Try again in ${wait}, or put it in your reply.`, {
|
|
426
|
+
'retry-after': String(s),
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
this.#askLimiter.record(pair);
|
|
430
|
+
const message = this.store.addMessage(to.id, 'user', t, { kind: 'ask', fromBotId: from.id, fromBotName: from.name, threadId, hop }, true);
|
|
431
|
+
chip('ask.sent', `Asked ${to.name}`);
|
|
432
|
+
log.info('ask sent', fields);
|
|
433
|
+
if (!this.store.pausedReason(to.id)) this.#enqueue(to.id, { kind: 'chat' });
|
|
434
|
+
return { message, threadId, hop };
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/** Only asks are answered, never replies, so two Botons can't loop. */
|
|
438
|
+
#answerAsks(bot: BotRecord, answered: Message[], text: string) {
|
|
439
|
+
const asks = new Map<string, Message>();
|
|
440
|
+
for (const m of answered) {
|
|
441
|
+
const asker = m.meta?.kind === 'ask' ? m.meta.fromBotId : undefined;
|
|
442
|
+
if (!asker) continue;
|
|
443
|
+
const prev = asks.get(asker);
|
|
444
|
+
if (!prev || (m.meta?.hop ?? 1) >= (prev.meta?.hop ?? 1)) asks.set(asker, m);
|
|
445
|
+
}
|
|
446
|
+
for (const [asker, ask] of asks) {
|
|
447
|
+
if (!this.store.hasBot(asker)) continue;
|
|
448
|
+
const reply = text.length > ASK_MAX ? `${text.slice(0, ASK_MAX)}…` : text;
|
|
449
|
+
this.store.addMessage(asker, 'user', reply, { kind: 'ask.reply', fromBotId: bot.id, fromBotName: bot.name, threadId: ask.meta?.threadId, hop: ask.meta?.hop, askId: ask.id }, true);
|
|
450
|
+
log.info('ask answered', { botId: asker, fromBotId: bot.id, threadId: ask.meta?.threadId, hop: ask.meta?.hop });
|
|
451
|
+
if (!this.store.pausedReason(asker)) this.#enqueue(asker, { kind: 'chat' });
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
triggerRoutine(routineId: string, trigger: RunTrigger, payload: string | null = null, headers: Record<string, string> | null = null): RoutineRun {
|
|
456
|
+
this.#refuseWhileDraining();
|
|
457
|
+
const routine = this.store.getRoutine(routineId);
|
|
458
|
+
const limited = trigger !== 'schedule';
|
|
459
|
+
const paused = this.store.pausedReason(routine.botId);
|
|
460
|
+
if (limited && paused) throw new HttpError(409, `${paused}. Resume to run routines.`);
|
|
461
|
+
if (limited) this.#checkLimits(routine, trigger);
|
|
462
|
+
const run = this.store.createRun(routine, trigger, payload, headers);
|
|
463
|
+
if (limited) this.#limiter.record(routine.id);
|
|
464
|
+
log.info('run queued', { botId: routine.botId, routineId: routine.id, runId: run.id, trigger });
|
|
465
|
+
this.#enqueue(routine.botId, { kind: 'routine', runId: run.id });
|
|
466
|
+
return run;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/** The owner's retry of a failed run: a new run with its trigger, payload and headers, under the limits a manual run has. */
|
|
470
|
+
retryRun(runId: string): RoutineRun {
|
|
471
|
+
this.#refuseWhileDraining();
|
|
472
|
+
const failed = this.store.getRun(runId);
|
|
473
|
+
const routine = this.store.getRoutine(failed.routineId);
|
|
474
|
+
const paused = this.store.pausedReason(routine.botId);
|
|
475
|
+
if (paused) throw new HttpError(409, `${paused}. Resume to run routines.`);
|
|
476
|
+
if (failed.status !== 'failed') throw new HttpError(409, `Only a failed run can be retried; this one is ${failed.status}`);
|
|
477
|
+
if (failed.meta?.retriedAs) throw new HttpError(409, `This run was already retried as ${failed.meta.retriedAs}`);
|
|
478
|
+
this.#checkLimits(routine, failed.trigger);
|
|
479
|
+
const run = this.store.createRetryRun(failed);
|
|
480
|
+
this.#limiter.record(routine.id);
|
|
481
|
+
log.info('run queued', { botId: routine.botId, routineId: routine.id, runId: run.id, trigger: run.trigger, retryOf: failed.id });
|
|
482
|
+
this.#enqueue(routine.botId, { kind: 'routine', runId: run.id });
|
|
483
|
+
return run;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
fireScheduled(routine: Routine): RoutineRun | null {
|
|
487
|
+
// A one-off has no next slot (firing it retired the routine): its run is queued, never skipped.
|
|
488
|
+
if (routine.schedule?.startsWith('once ')) return this.triggerRoutine(routine.id, 'schedule');
|
|
489
|
+
const skip = (reason: string) => {
|
|
490
|
+
const run = this.store.recordSkippedRun(routine, 'schedule', reason);
|
|
491
|
+
log.info('scheduled run skipped', { botId: routine.botId, routineId: routine.id, runId: run.id, trigger: 'schedule', reason });
|
|
492
|
+
return null;
|
|
493
|
+
};
|
|
494
|
+
const paused = this.store.pausedReason(routine.botId);
|
|
495
|
+
if (paused) return skip(paused);
|
|
496
|
+
const bot = this.store.getBot(routine.botId);
|
|
497
|
+
const until = this.#coolingUntil(bot.provider);
|
|
498
|
+
if (until) return skip(`${this.#providerName(bot.provider)} usage limit until ${when(until, bot.timezone)}`);
|
|
499
|
+
if (this.store.activeRunCount(routine.id) > 0) return skip('The previous run was still queued or running');
|
|
500
|
+
return this.triggerRoutine(routine.id, 'schedule');
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
#checkLimits(routine: Routine, trigger: RunTrigger) {
|
|
504
|
+
let refusal: { message: string; chip: string; retryAfterS: number };
|
|
505
|
+
if (this.store.queuedRunCount(routine.id) >= ROUTINE_QUEUE_MAX) {
|
|
506
|
+
// Waiting out a usage limit, the queue moves at the reset.
|
|
507
|
+
const reset = this.#coolingUntil(this.#providerOf(routine.botId));
|
|
508
|
+
refusal = {
|
|
509
|
+
message: `Routine “${routine.name}” already has ${ROUTINE_QUEUE_MAX} runs waiting to start. Try again once they have run.`,
|
|
510
|
+
chip: `${ROUTINE_QUEUE_MAX} already waiting`,
|
|
511
|
+
retryAfterS: reset ? Math.max(QUEUE_FULL_RETRY_S, Math.ceil((reset.getTime() - Date.now()) / 1000)) : QUEUE_FULL_RETRY_S,
|
|
512
|
+
};
|
|
513
|
+
} else {
|
|
514
|
+
const hit = this.#limiter.check(routine.id);
|
|
515
|
+
if (!hit) return;
|
|
516
|
+
const { s, wait } = retryAfter(hit.retryAfterMs);
|
|
517
|
+
refusal =
|
|
518
|
+
hit.limit === 'routine'
|
|
519
|
+
? { message: `Routine “${routine.name}” has started ${hit.max} webhook and manual runs in the last hour, its limit. Try again in ${wait}.`, chip: `${hit.max} an hour for this routine`, retryAfterS: s }
|
|
520
|
+
: { message: `Botons has started ${hit.max} webhook and manual runs in the last hour, its limit. Try again in ${wait}.`, chip: `${hit.max} an hour for all routines`, retryAfterS: s };
|
|
521
|
+
}
|
|
522
|
+
if (this.#chipDue(routine.id)) {
|
|
523
|
+
this.store.addMessage(routine.botId, 'event', `Refused ${trigger} runs (limit: ${refusal.chip})`, {
|
|
524
|
+
kind: 'routine.rate_limited',
|
|
525
|
+
routineId: routine.id,
|
|
526
|
+
routineName: routine.name,
|
|
527
|
+
trigger,
|
|
528
|
+
});
|
|
529
|
+
}
|
|
530
|
+
throw new HttpError(429, refusal.message, { 'retry-after': String(refusal.retryAfterS) });
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
noteCutOffTurns(turns: { id: string; botId: string; steps: number }[]): void {
|
|
534
|
+
for (const t of turns) {
|
|
535
|
+
if (!this.store.hasBot(t.botId)) continue;
|
|
536
|
+
const chip = this.store.addMessage(t.botId, 'event', 'Cut off by a restart', { kind: 'turn.interrupted', turnId: t.id, steps: t.steps });
|
|
537
|
+
this.store.linkTurnMessage(t.id, chip.id);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
/** Before resume(): messages a turn that crashes the server would cut off forever get an error instead. */
|
|
542
|
+
giveUpCutOffChats(): void {
|
|
543
|
+
for (const [botId, messages] of this.store.countCutOffChats(CUT_OFF_LIMIT)) {
|
|
544
|
+
const bot = orNull(() => this.store.getBot(botId));
|
|
545
|
+
if (!bot) continue;
|
|
546
|
+
const ids = messages.map((m) => m.id);
|
|
547
|
+
this.store.markProcessed(ids);
|
|
548
|
+
const why = `The last ${CUT_OFF_LIMIT} attempts to answer this were cut off by a restart or crash, so it was not tried again. Some of it may have run. Send it again to retry.`;
|
|
549
|
+
this.#post(bot, 'error', why);
|
|
550
|
+
this.#answerAsks(bot, messages, `(${bot.name} could not answer: ${why})`);
|
|
551
|
+
log.warn('gave up on chat messages cut off at every start', { botId, messages: ids.length, limit: CUT_OFF_LIMIT });
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
resume(): void {
|
|
556
|
+
for (const c of this.store.cooldowns()) {
|
|
557
|
+
if (c.until.getTime() <= Date.now()) this.store.clearCooldown(c.provider);
|
|
558
|
+
else this.#startCooldown(c.provider, c.until, c.reason);
|
|
559
|
+
}
|
|
560
|
+
this.#resumeWork(() => true);
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
#resumeWork(which: (botId: string) => boolean): void {
|
|
564
|
+
const ok = (botId: string) => which(botId) && !this.store.pausedReason(botId);
|
|
565
|
+
const work: { at: string; botId: string; work: Work }[] = [];
|
|
566
|
+
for (const c of this.store.pendingChats()) {
|
|
567
|
+
if (ok(c.botId) && !this.#coolingUntil(this.#providerOf(c.botId))) work.push({ at: c.since, botId: c.botId, work: { kind: 'chat' } });
|
|
568
|
+
}
|
|
569
|
+
for (const r of this.store.queuedRuns()) {
|
|
570
|
+
if (!ok(r.botId)) continue;
|
|
571
|
+
if (r.retryAt && new Date(r.retryAt).getTime() > Date.now()) this.#wakeRun(r);
|
|
572
|
+
else work.push({ at: r.startedAt, botId: r.botId, work: { kind: 'routine', runId: r.id } });
|
|
573
|
+
}
|
|
574
|
+
work.sort((a, b) => a.at.localeCompare(b.at));
|
|
575
|
+
for (const w of work) this.#enqueue(w.botId, w.work);
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
#providerOf(botId: string): string {
|
|
579
|
+
return orNull(() => this.store.getBot(botId).provider) ?? '';
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
#providerName(id: string): string {
|
|
583
|
+
return (this.#providers.get(id)?.label ?? id).replace(/ \(.*\)$/, '');
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
// ------------------------------------------------------------ usage limits
|
|
587
|
+
|
|
588
|
+
usageLimit(provider: string): { until: string; reason: string } | null {
|
|
589
|
+
const c = this.#cooldowns.get(provider);
|
|
590
|
+
return c && c.until.getTime() > Date.now() ? { until: c.until.toISOString(), reason: c.reason } : null;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
#coolingUntil(provider: string): Date | null {
|
|
594
|
+
const c = this.#cooldowns.get(provider);
|
|
595
|
+
return c && c.until.getTime() > Date.now() ? c.until : null;
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
#startCooldown(provider: string, until: Date, reason: string) {
|
|
599
|
+
const prev = this.#cooldowns.get(provider);
|
|
600
|
+
if (prev && prev.until.getTime() >= until.getTime()) return;
|
|
601
|
+
this.#cooldowns.set(provider, { until, reason });
|
|
602
|
+
this.store.setCooldown(provider, until, reason);
|
|
603
|
+
log.warn('usage limit hit: holding its turns until the reset', { provider, until: until.toISOString(), reason: reason.slice(0, 200) });
|
|
604
|
+
this.#wakeAt(`cooldown:${provider}`, until, () => {
|
|
605
|
+
this.#cooldowns.delete(provider);
|
|
606
|
+
this.store.clearCooldown(provider);
|
|
607
|
+
log.info('usage limit has reset: resuming its Botons', { provider });
|
|
608
|
+
this.#resumeWork((botId) => this.#providerOf(botId) === provider);
|
|
609
|
+
});
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
#noticeUsageLimit(bot: BotRecord, until: Date) {
|
|
613
|
+
if (this.#noticed.get(bot.id) === until.getTime() || !this.store.hasBot(bot.id)) return;
|
|
614
|
+
this.#noticed.set(bot.id, until.getTime());
|
|
615
|
+
this.store.addMessage(bot.id, 'event', `${this.#providerName(bot.provider)} usage limit reached; continuing ${when(until, bot.timezone)}`, {
|
|
616
|
+
kind: 'usage.limit',
|
|
617
|
+
retryAt: until.toISOString(),
|
|
618
|
+
});
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
/** Run `fn` at `at` (in steps, for waits longer than a timer can). Skipped once the engine is draining. */
|
|
622
|
+
#wakeAt(key: string, at: Date, fn: () => void) {
|
|
623
|
+
clearTimeout(this.#timers.get(key));
|
|
624
|
+
const t = setTimeout(
|
|
625
|
+
() => {
|
|
626
|
+
this.#timers.delete(key);
|
|
627
|
+
if (this.#draining) return;
|
|
628
|
+
if (Date.now() < at.getTime()) return this.#wakeAt(key, at, fn);
|
|
629
|
+
try {
|
|
630
|
+
fn();
|
|
631
|
+
} catch (err) {
|
|
632
|
+
log.error('wake-up failed', { wakeUp: key, err });
|
|
633
|
+
}
|
|
634
|
+
},
|
|
635
|
+
Math.min(Math.max(0, at.getTime() - Date.now()) + 5, 2 ** 31 - 1),
|
|
636
|
+
);
|
|
637
|
+
t.unref();
|
|
638
|
+
this.#timers.set(key, t);
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
#wakeRun(run: RoutineRun) {
|
|
642
|
+
if (!run.retryAt) return;
|
|
643
|
+
this.#wakeAt(`run:${run.id}`, new Date(run.retryAt), () => this.#enqueue(run.botId, { kind: 'routine', runId: run.id }));
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
// ------------------------------------------------------- stopping, pausing
|
|
647
|
+
|
|
648
|
+
get draining(): boolean {
|
|
649
|
+
return this.#draining;
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
#refuseWhileDraining() {
|
|
653
|
+
if (this.#draining) throw new HttpError(503, 'Botons is restarting. Try again in a minute.');
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
/** Hold persisted queued work, let active work finish, then resume even if maintenance fails. */
|
|
657
|
+
async maintain<T>(run: () => Promise<T>, budgetMs = 60_000): Promise<T> {
|
|
658
|
+
if (this.#maintenance || this.#draining) throw new HttpError(409, 'Botons is already updating or restarting.');
|
|
659
|
+
this.#maintenance = true;
|
|
660
|
+
try {
|
|
661
|
+
if (!(await this.#settled(budgetMs))) throw new HttpError(409, 'Running turns are still finishing. Try Update now again when they finish.');
|
|
662
|
+
if (this.#draining) throw new HttpError(409, 'Botons is restarting. Try again after it starts.');
|
|
663
|
+
return await run();
|
|
664
|
+
} finally {
|
|
665
|
+
this.#maintenance = false;
|
|
666
|
+
if (!this.#draining) this.#resumeWork(() => true);
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
async drain(budgetMs: number): Promise<{ interrupted: number }> {
|
|
671
|
+
this.#draining = true;
|
|
672
|
+
// Retries and usage-limit resets are saved with their runs and cooldowns; the next process sets them again.
|
|
673
|
+
for (const t of this.#timers.values()) clearTimeout(t);
|
|
674
|
+
this.#timers.clear();
|
|
675
|
+
if (!(await this.#settled(budgetMs))) {
|
|
676
|
+
this.interruptAll();
|
|
677
|
+
if (!(await this.#settled(DRAIN_GRACE_MS))) log.warn('some agent processes did not exit in time');
|
|
678
|
+
}
|
|
679
|
+
return { interrupted: this.#interruptedCount };
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
interruptAll(): void {
|
|
683
|
+
for (const { controller } of this.#summaries.values()) controller.abort(INTERRUPTED);
|
|
684
|
+
for (const { turn } of this.#jobs.values()) {
|
|
685
|
+
if (!turn || turn.signal.aborted) continue;
|
|
686
|
+
turn.abort(INTERRUPTED);
|
|
687
|
+
this.#interruptedCount++;
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
async #settled(ms: number): Promise<boolean> {
|
|
692
|
+
let timer: NodeJS.Timeout | undefined;
|
|
693
|
+
const timeout = new Promise<false>((resolve) => (timer = setTimeout(() => resolve(false), ms)));
|
|
694
|
+
try {
|
|
695
|
+
return await Promise.race([Promise.allSettled([...this.#chains.values()]).then(() => true as const), timeout]);
|
|
696
|
+
} finally {
|
|
697
|
+
clearTimeout(timer);
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
stopAll(): void {
|
|
702
|
+
for (const { controller } of this.#summaries.values()) controller.abort(STOPPED);
|
|
703
|
+
for (const job of this.#jobs.values()) job.turn?.abort();
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
stop(botId: string, opts: { clearQueue?: boolean } = {}): StopResult {
|
|
707
|
+
const stopped = this.#jobs.has(botId) || this.#summaries.has(botId);
|
|
708
|
+
this.#halt(botId);
|
|
709
|
+
let cancelledRuns = 0;
|
|
710
|
+
if (opts.clearQueue) {
|
|
711
|
+
cancelledRuns = this.store.cancelQueuedRuns(botId, 'Cancelled');
|
|
712
|
+
this.store.dropPendingChat(botId);
|
|
713
|
+
}
|
|
714
|
+
return { stopped, cancelledRuns, queuedRuns: this.store.queuedRuns(botId).length };
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
cancelQueuedRuns(botId: string): CancelRunsResult {
|
|
718
|
+
this.store.getBot(botId);
|
|
719
|
+
return { cancelledRuns: this.store.cancelQueuedRuns(botId, 'Cancelled') };
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
/**
|
|
723
|
+
* The owner ends one run, and nothing else: a queued one (held to try again included) never starts; a running one
|
|
724
|
+
* has only its own turn's agent process stopped, so the Boton's chat and its other runs carry on. Resolves with the
|
|
725
|
+
* run as it ended: one that finished before the stop reached it keeps its own outcome.
|
|
726
|
+
*/
|
|
727
|
+
async cancelRun(runId: string): Promise<RoutineRun> {
|
|
728
|
+
const run = this.store.getRun(runId);
|
|
729
|
+
if (run.status === 'queued') {
|
|
730
|
+
clearTimeout(this.#timers.get(`run:${run.id}`));
|
|
731
|
+
this.#timers.delete(`run:${run.id}`);
|
|
732
|
+
// Held to try again: an earlier attempt ran, so it may have partly run.
|
|
733
|
+
const started = run.attempts > 0;
|
|
734
|
+
const ended = this.store.updateRun(run.id, { status: 'failed', error: 'Cancelled', cancelled: { started } });
|
|
735
|
+
// A job waiting for a slot to start it gives up the wait (and, no longer queued, the run); any other finds it gone.
|
|
736
|
+
const job = this.#jobs.get(run.botId);
|
|
737
|
+
if (job && !job.run && job.runId === run.id) job.wait.abort();
|
|
738
|
+
if (this.store.hasBot(run.botId)) this.#postStopped(run.botId, ended, undefined, 'Cancelled');
|
|
739
|
+
log.info('run cancelled', { botId: run.botId, routineId: run.routineId, runId: run.id, trigger: run.trigger, was: 'queued', started });
|
|
740
|
+
return ended;
|
|
741
|
+
}
|
|
742
|
+
if (run.status !== 'running') throw new HttpError(409, `This run already ended (${run.status})`);
|
|
743
|
+
const job = this.#jobs.get(run.botId);
|
|
744
|
+
if (job?.run?.runId !== run.id) throw new HttpError(409, 'This run is not running on this server; it ends as interrupted at the next start');
|
|
745
|
+
this.#abortTurn(job, CANCELLED);
|
|
746
|
+
log.info('run cancelled', { botId: run.botId, routineId: run.routineId, runId: run.id, trigger: run.trigger, was: 'running' });
|
|
747
|
+
const ended = await this.waitForRun(run, CANCEL_WAIT_MS);
|
|
748
|
+
if (ended.status === 'running') throw new HttpError(504, 'The run was told to stop, but its agent process has not exited yet');
|
|
749
|
+
return ended;
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
pauseBot(botId: string, paused: boolean): Bot {
|
|
753
|
+
const before = this.store.getBot(botId);
|
|
754
|
+
if (before.paused === paused) return this.store.publicBot(before);
|
|
755
|
+
const bot = this.store.setBotPaused(botId, paused);
|
|
756
|
+
this.store.addMessage(botId, 'event', paused ? 'Paused' : 'Resumed', { kind: paused ? 'bot.paused' : 'bot.resumed' });
|
|
757
|
+
if (paused) this.#halt(botId, PAUSED);
|
|
758
|
+
else this.#resumeWork((id) => id === botId);
|
|
759
|
+
return bot;
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
/**
|
|
763
|
+
* What moving the Boton to `input.provider` keeps and loses, as the update would do it. The provider is part of
|
|
764
|
+
* every session's key (sessionKey), so the chat's session and each routine's own end with the move; the model is
|
|
765
|
+
* not, so a model change alone keeps them, and needs no preview.
|
|
766
|
+
*/
|
|
767
|
+
async switchPreview(botId: string, input: ProviderSwitchInput, connections?: string[]): Promise<ProviderSwitchPreview> {
|
|
768
|
+
this.#checkSwitchInput(input);
|
|
769
|
+
const from = this.store.getBot(botId).provider;
|
|
770
|
+
const { ready: readiness, old } = await this.#switchBrains(from, input.provider);
|
|
771
|
+
// Moved by another update while the providers were checked: what the old one's check found is not this one's.
|
|
772
|
+
if (this.store.getBot(botId).provider !== from) throw new HttpError(409, 'The Boton moved while this was checked: review it again');
|
|
773
|
+
const facts = this.#switchPreview(botId, input, connections, readiness, old);
|
|
774
|
+
return { ...facts, replayNow: this.#replayNow(botId, facts) };
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
/** What the new session's first turn would be given of the chat now, as the turn builds it: an estimate for the preview. */
|
|
778
|
+
#replayNow(botId: string, facts: SwitchFacts): ProviderSwitchPreview['replayNow'] {
|
|
779
|
+
const bot = this.store.getBot(botId);
|
|
780
|
+
const handoff = facts.from.provider !== facts.to.provider ? this.#handoff(bot, facts) : null;
|
|
781
|
+
const summary = handoff?.summary ? this.store.handoffSummary(bot.id, handoff.summary.messageId) : null;
|
|
782
|
+
const src = this.store.replaySource(bot.id, bot.contextSince, summary ? [summary.id] : []);
|
|
783
|
+
const { plan } = sessionStart({ handoff, summary, ...src }, this.store.getSettings(), bot.timezone);
|
|
784
|
+
return { used: plan.used, messages: plan.messages, routines: plan.routines, pinned: plan.pinned, excerpts: plan.excerpts, olderLeftOut: plan.olderLeftOut, since: plan.since };
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
#checkSwitchInput(input: ProviderSwitchInput) {
|
|
788
|
+
if (typeof input?.provider !== 'string' || !this.#providers.ids().has(input.provider)) throw new HttpError(400, 'provider must be one of this server\'s providers');
|
|
789
|
+
if (input.model !== undefined && typeof input.model !== 'string') throw new HttpError(400, 'model must be a string');
|
|
790
|
+
if (input.effort !== undefined && typeof input.effort !== 'string') throw new HttpError(400, 'effort must be a level name, or "" for the provider\'s default');
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
/** Both sides of a switch come from one registry snapshot, so no intervening check can split its facts. */
|
|
794
|
+
async #switchBrains(from: string, to: string): Promise<{ ready: Pick<ProviderSwitchPreview['readiness'], 'state' | 'reason'>; old: OldBrain }> {
|
|
795
|
+
const infos = await this.#providers.infos();
|
|
796
|
+
const destination = infos.find((p) => p.id === to);
|
|
797
|
+
const state = !destination ? 'unknown' : !destination.available ? 'unavailable' : destination.checkedAt && destination.loggedIn === true ? 'ready' : 'unknown';
|
|
798
|
+
const source = infos.find((p) => p.id === from)?.readiness;
|
|
799
|
+
return {
|
|
800
|
+
ready: { state, reason: destination?.reason ?? null },
|
|
801
|
+
old: source ? { state: source.state, problem: source.problem } : { state: 'unknown', problem: 'not-checked' },
|
|
802
|
+
};
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
/**
|
|
806
|
+
* Synchronous, so the update can check a confirmation and apply the move with nothing in between. `old`: what the
|
|
807
|
+
* last check found of the Boton's provider now. `hold`: the move with a summary that holds the Boton, which
|
|
808
|
+
* doesn't count as busy for itself.
|
|
809
|
+
*/
|
|
810
|
+
#switchPreview(
|
|
811
|
+
botId: string,
|
|
812
|
+
input: ProviderSwitchInput,
|
|
813
|
+
connections: string[] | undefined,
|
|
814
|
+
ready: Pick<ProviderSwitchPreview['readiness'], 'state' | 'reason'>,
|
|
815
|
+
old: OldBrain,
|
|
816
|
+
hold?: symbol,
|
|
817
|
+
): SwitchFacts {
|
|
818
|
+
this.#checkSwitchInput(input);
|
|
819
|
+
const bot = this.store.getBot(botId);
|
|
820
|
+
const provider = input.provider;
|
|
821
|
+
const moved = provider !== bot.provider;
|
|
822
|
+
// As the update cleans them.
|
|
823
|
+
const model = input.model !== undefined ? input.model.trim() : bot.model;
|
|
824
|
+
const effortReset = input.effort === undefined && !!bot.effort && !!effortProblem(provider, bot.effort, this.store.effortCaps(provider, model));
|
|
825
|
+
const effort = input.effort !== undefined ? input.effort.trim() : effortReset ? '' : bot.effort;
|
|
826
|
+
const routines = this.store.listRoutines(botId);
|
|
827
|
+
const routineModels = moved ? routines.flatMap((r) => (r.model ? [{ id: r.id, name: r.name, model: r.model }] : [])) : [];
|
|
828
|
+
// A moved Boton's routines run its model (their own was the old provider's), as resetRoutineEfforts then checks.
|
|
829
|
+
const routineEfforts = routines.flatMap((r) =>
|
|
830
|
+
r.effort && effortProblem(provider, r.effort, this.store.effortCaps(provider, (moved ? '' : r.model) || model)) ? [{ id: r.id, name: r.name, effort: r.effort }] : [],
|
|
831
|
+
);
|
|
832
|
+
// A turn that has read the Boton (setting up its connectors, or running) is on the old provider: the session it
|
|
833
|
+
// is starting or carrying on is one the move ends, even before it is saved. One still waiting for a slot reads the
|
|
834
|
+
// Boton when it starts, so it runs on the new provider.
|
|
835
|
+
const underway = this.#underway(bot);
|
|
836
|
+
const working = !!underway;
|
|
837
|
+
const busyOwn = underway?.own ? routines.find((r) => r.id === underway.own) : undefined;
|
|
838
|
+
const key = sessionKey(bot.provider, systemPrompt(bot, this.store.getSettings(), this.#cfg));
|
|
839
|
+
const savedChat = !!bot.sessionId && bot.sessionKey === key;
|
|
840
|
+
const saved = this.store.routineSessions(botId).flatMap((r) => (r.sessionKey === key ? [{ id: r.id, name: r.name }] : []));
|
|
841
|
+
const sessions = moved
|
|
842
|
+
? {
|
|
843
|
+
chat: savedChat || (working && !busyOwn),
|
|
844
|
+
routines: busyOwn && !saved.some((r) => r.id === busyOwn.id) ? [...saved, { id: busyOwn.id, name: busyOwn.name }] : saved,
|
|
845
|
+
}
|
|
846
|
+
: { chat: false, routines: [] };
|
|
847
|
+
const catalog = this.connectors.catalog().connectors;
|
|
848
|
+
const lost = moved
|
|
849
|
+
? (connections ?? bot.connections).flatMap((id) => {
|
|
850
|
+
const c = catalog.find((x) => x.id === id);
|
|
851
|
+
return c && c.providers.includes(bot.provider) && !c.providers.includes(provider) ? [{ id, name: c.name }] : [];
|
|
852
|
+
})
|
|
853
|
+
: [];
|
|
854
|
+
const required = moved && (sessions.chat || sessions.routines.length > 0);
|
|
855
|
+
const { runs } = this.store.activeRuns(botId, new Date(), 1000);
|
|
856
|
+
const noSummary = this.#noSummary(bot, required, savedChat, old, hold);
|
|
857
|
+
const shown: Omit<SwitchFacts, 'confirmation' | 'summary'> & { summary: Omit<SwitchSummaryOffer, 'confirmation'> } = {
|
|
858
|
+
from: { ...this.#brainRef(bot.provider, bot.model), effort: bot.effort },
|
|
859
|
+
to: { ...this.#brainRef(provider, model), effort },
|
|
860
|
+
required,
|
|
861
|
+
sessions,
|
|
862
|
+
working: moved && working,
|
|
863
|
+
queuedRuns: runs.filter((r) => r.status === 'queued').length,
|
|
864
|
+
replay: { budget: REPLAY_BUDGET, messageLimit: MESSAGE_LIMIT, pinned: this.store.pinnedMessages(botId).length, pinLimit: PIN_LIMIT },
|
|
865
|
+
kept: { memories: this.store.listMemories(botId).length, routines: routines.length },
|
|
866
|
+
effortReset,
|
|
867
|
+
routineModels,
|
|
868
|
+
routineEfforts,
|
|
869
|
+
connectors: lost,
|
|
870
|
+
readiness: { ...ready, usageLimitUntil: this.usageLimit(provider)?.until ?? null },
|
|
871
|
+
oldUsageLimitUntil: moved ? (this.usageLimit(bot.provider)?.until ?? null) : null,
|
|
872
|
+
summary: { available: !noSummary, reason: noSummary, timeoutMs: SWITCH_SUMMARY_MS, maxChars: SWITCH_SUMMARY_CHARS },
|
|
873
|
+
};
|
|
874
|
+
// Stands for everything the owner is shown, for this Boton only (no clock in it, so the same facts give the same
|
|
875
|
+
// confirmation): any change to it refuses the update until the owner reviews it again. Moving with the summary
|
|
876
|
+
// has its own, so the owner's choice of it is bound too: one can't stand for the other.
|
|
877
|
+
const digest = (kind: string) => `sw_${createHash('sha256').update(JSON.stringify([kind, botId, shown])).digest('base64url').slice(0, 32)}`;
|
|
878
|
+
const confirmation = required ? digest('botons-switch') : null;
|
|
879
|
+
return { ...shown, confirmation, summary: { ...shown.summary, confirmation: noSummary ? null : digest('botons-switch-summary') } };
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
/**
|
|
883
|
+
* Why the old session can't be asked for a summary now (null: it can). Only a saved chat session on the provider
|
|
884
|
+
* the Boton is on, with nothing else running for it, on a provider that is there, usable and not out of usage.
|
|
885
|
+
*/
|
|
886
|
+
#noSummary(bot: BotRecord, required: boolean, savedChat: boolean, old: OldBrain, hold?: symbol): string | null {
|
|
887
|
+
const from = this.#providerName(bot.provider);
|
|
888
|
+
if (!required) return 'No session ends with this move';
|
|
889
|
+
// A summary turn must be given no tools at all, not only a sandbox around them: only a CLI that can do that asks.
|
|
890
|
+
const provider = this.#providers.get(bot.provider);
|
|
891
|
+
if (provider && !provider.toolFree) {
|
|
892
|
+
return `${from} can't be asked for a summary: its CLI can't run a turn with no tools (shell, file edits and others stay on)`;
|
|
893
|
+
}
|
|
894
|
+
if (!savedChat) return `${bot.name} has no chat session on ${from} to ask`;
|
|
895
|
+
const held = this.#summaries.get(bot.id);
|
|
896
|
+
if (held && held.hold !== hold) return `A summary for a move is already being written`;
|
|
897
|
+
if (!held && (this.#chains.has(bot.id) || this.#jobs.has(bot.id))) return `${bot.name} is busy: a summary can be asked for once it is idle`;
|
|
898
|
+
if (this.#maintenance) return 'Botons is updating its CLI';
|
|
899
|
+
if (this.#draining) return 'Botons is restarting';
|
|
900
|
+
if (this.store.pausedReason(bot.id)) return `${bot.name} is paused`;
|
|
901
|
+
if (!provider) return `${from} is not on this server`;
|
|
902
|
+
const limit = this.#coolingUntil(bot.provider);
|
|
903
|
+
if (limit) return `${from} is out of usage until ${when(limit, bot.timezone)}`;
|
|
904
|
+
if (old.state !== 'ready') return `${from} isn't confirmed ready on this server (${old.state}${old.problem && old.problem !== old.state ? `: ${old.problem}` : ''}); check it in Settings`;
|
|
905
|
+
return null;
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
/**
|
|
909
|
+
* The session of the turn that read the Boton as it is now (its provider, since its latest switch), if one is
|
|
910
|
+
* underway. One that read it before a later switch (moved away and back, say) was that switch's: the session it
|
|
911
|
+
* was on is already ended.
|
|
912
|
+
*/
|
|
913
|
+
#underway(bot: BotRecord): { own: string | null; fresh: boolean } | null {
|
|
914
|
+
const read = this.#jobs.get(bot.id)?.read;
|
|
915
|
+
return read && read.stamp.provider === bot.provider && read.stamp.generation === this.store.switchGeneration(bot.id) ? { own: read.own, fresh: read.fresh } : null;
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
#brainRef(provider: string, model: string) {
|
|
919
|
+
return { provider, label: this.#providerName(provider), model };
|
|
920
|
+
}
|
|
921
|
+
|
|
922
|
+
/**
|
|
923
|
+
* A switch that would end a session needs `switchConfirmation` from a preview of exactly this switch (see
|
|
924
|
+
* switchPreview): the owner saw what is kept and lost, and nothing it showed changed since. The provider checks are
|
|
925
|
+
* the only wait: from reading the Boton to saving the move nothing awaits, so what is checked is what is applied.
|
|
926
|
+
* With `switchSummary`, the old session writes its summary first (see #moveWithSummary).
|
|
927
|
+
*/
|
|
928
|
+
async updateBot(botId: string, input: UpdateBotInput, providers: Set<string>, switchConfirmation?: unknown, switchSummary?: unknown): Promise<Bot> {
|
|
929
|
+
if (switchSummary !== undefined && typeof switchSummary !== 'boolean') throw new HttpError(400, 'switchSummary must be true or false');
|
|
930
|
+
const target = typeof input?.provider === 'string' && providers.has(input.provider) ? input.provider : null;
|
|
931
|
+
const was = this.store.getBot(botId).provider;
|
|
932
|
+
const brains = target && target !== was ? await this.#switchBrains(was, target) : null;
|
|
933
|
+
const ready = brains?.ready ?? null;
|
|
934
|
+
const old = brains?.old ?? null;
|
|
935
|
+
const current = this.store.getBot(botId);
|
|
936
|
+
const before = current.provider;
|
|
937
|
+
if (input.connections !== undefined) input = { ...input, connections: this.connectors.check(input.connections, current.connections) };
|
|
938
|
+
if (input.skills !== undefined) input = { ...input, skills: this.skills.checkAllowlist(input.skills, current.skills) };
|
|
939
|
+
const switching = input.provider !== undefined && input.provider !== before && providers.has(input.provider);
|
|
940
|
+
if (switchSummary === true && !switching) throw new HttpError(400, 'switchSummary goes with a move to another provider');
|
|
941
|
+
let handoff: ProviderHandoff | null = null;
|
|
942
|
+
if (switching) {
|
|
943
|
+
// Moved elsewhere by another update while the providers were checked: what was reviewed is not this.
|
|
944
|
+
if (!ready || !old || was !== before) throw new HttpError(409, `What moving ${current.name} keeps and loses changed since you reviewed it: review it again`);
|
|
945
|
+
// A move asked for with a summary holds the Boton until it is done: no other move goes under it.
|
|
946
|
+
if (this.#summaries.has(botId)) throw new HttpError(409, `${current.name}'s session is writing a summary for a move: wait for it, then review the move again`);
|
|
947
|
+
const move = { provider: input.provider!, model: input.model, effort: input.effort };
|
|
948
|
+
const preview = this.#switchPreview(botId, move, input.connections, ready, old);
|
|
949
|
+
if (switchSummary === true) {
|
|
950
|
+
if (!preview.summary.available) throw new HttpError(409, `${preview.summary.reason}, so its session can't write a summary now: review the move again`);
|
|
951
|
+
if (switchConfirmation !== preview.summary.confirmation) throw new HttpError(409, `What moving ${current.name} to ${preview.to.label} keeps and loses changed since you reviewed it: review it again`);
|
|
952
|
+
return this.#moveWithSummary(current, input, providers, move, preview);
|
|
953
|
+
}
|
|
954
|
+
if (preview.required && switchConfirmation !== preview.confirmation) {
|
|
955
|
+
throw new HttpError(
|
|
956
|
+
409,
|
|
957
|
+
switchConfirmation === undefined
|
|
958
|
+
? `Moving ${current.name} to ${preview.to.label} ends its session on ${preview.from.label}: review what is kept and lost (POST /api/bots/${botId}/switch-preview), then send its confirmation as switchConfirmation`
|
|
959
|
+
: `What moving ${current.name} to ${preview.to.label} keeps and loses changed since you reviewed it: review it again`,
|
|
960
|
+
);
|
|
961
|
+
}
|
|
962
|
+
handoff = this.#handoff(current, preview);
|
|
963
|
+
}
|
|
964
|
+
// The turn underway carries on its session on the old provider past what is saved, and that context can't come
|
|
965
|
+
// with the move: the saved session it is on ends with it, so no later move back resumes it. Read with the
|
|
966
|
+
// preview, so it is the session the owner was shown ending.
|
|
967
|
+
const underway = switching ? this.#underway(current) : null;
|
|
968
|
+
// The move, its event (whose seq is the new switch generation) and the session it ends together, or none.
|
|
969
|
+
const bot = this.store.atomically(() => {
|
|
970
|
+
const b = this.store.updateBot(botId, input, providers);
|
|
971
|
+
if (b.provider !== before && handoff) {
|
|
972
|
+
this.#recordSwitch(b, this.#brainRef(current.provider, current.model), handoff);
|
|
973
|
+
if (underway?.own) this.store.dropRoutineSession(underway.own);
|
|
974
|
+
else if (underway && !underway.fresh) this.store.dropSession(botId);
|
|
975
|
+
}
|
|
976
|
+
return b;
|
|
977
|
+
});
|
|
978
|
+
this.#afterUpdate(current, bot, input);
|
|
979
|
+
return bot;
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
#afterUpdate(current: BotRecord, bot: Bot, input: UpdateBotInput): void {
|
|
983
|
+
const botId = bot.id;
|
|
984
|
+
if (bot.provider !== current.provider) {
|
|
985
|
+
// A routine's own model is one of the old provider's: the new CLI would be asked for it on every run.
|
|
986
|
+
const reset = this.store.resetRoutineModels(botId);
|
|
987
|
+
if (reset.length) {
|
|
988
|
+
const names = reset.map((r) => `“${r.name}” (was ${r.model})`).join(', ');
|
|
989
|
+
const one = reset.length === 1 ? reset[0] : null;
|
|
990
|
+
this.store.addMessage(botId, 'event', `Moved to ${this.#providerName(bot.provider)}, so ${reset.length === 1 ? 'routine' : 'routines'} ${names} now use${reset.length === 1 ? 's' : ''} the Boton's model`, {
|
|
991
|
+
kind: 'routine.model_reset',
|
|
992
|
+
...(one ? { routineId: one.id, routineName: one.name } : {}),
|
|
993
|
+
});
|
|
994
|
+
log.info("routine models reset: the Boton moved to another provider", { botId, provider: bot.provider, routines: reset.map((r) => r.id) });
|
|
995
|
+
}
|
|
996
|
+
this.store.releaseUsageLimitHolds(botId);
|
|
997
|
+
this.#resumeWork((id) => id === botId);
|
|
998
|
+
}
|
|
999
|
+
this.#effortNotes(current, bot, input);
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
/**
|
|
1003
|
+
* The move, once the old session has written its summary for the new one. The Boton is held meanwhile: its chat
|
|
1004
|
+
* and runs wait behind it (in its queue), and no other move or summary starts, so the summary turn is the only one
|
|
1005
|
+
* on the session. Everything the confirmation stands for (the providers' readiness and limits, the session, a reset,
|
|
1006
|
+
* another move, a pause, a restart) is checked again before the turn starts, once it has a slot, and again after it,
|
|
1007
|
+
* with the abort, right before the write: any change refuses the move, and the summary goes unsaved. Saved, the
|
|
1008
|
+
* summary and the move are one write, so the new session never starts without the summary that was asked for. A
|
|
1009
|
+
* summary that can't be written (no reply, a failure, its time limit, Stop, a pause) refuses the move.
|
|
1010
|
+
*/
|
|
1011
|
+
async #moveWithSummary(
|
|
1012
|
+
current: BotRecord,
|
|
1013
|
+
input: UpdateBotInput,
|
|
1014
|
+
providers: Set<string>,
|
|
1015
|
+
move: ProviderSwitchInput,
|
|
1016
|
+
preview: SwitchFacts,
|
|
1017
|
+
): Promise<Bot> {
|
|
1018
|
+
const botId = current.id;
|
|
1019
|
+
const hold = Symbol('switch summary');
|
|
1020
|
+
const controller = new AbortController();
|
|
1021
|
+
const { signal } = controller;
|
|
1022
|
+
let release!: () => void;
|
|
1023
|
+
const gate = new Promise<void>((resolve) => (release = resolve));
|
|
1024
|
+
// Free to take: the offer needs nothing queued or running for the Boton, and nothing awaited since it was read.
|
|
1025
|
+
this.#summaries.set(botId, { hold, controller });
|
|
1026
|
+
this.#chains.set(botId, gate);
|
|
1027
|
+
const generation = this.store.switchGeneration(botId);
|
|
1028
|
+
// The move as the owner confirmed it, if it still is (synchronous): the Boton, its session, and the same preview.
|
|
1029
|
+
const unchanged = (r: Pick<ProviderSwitchPreview['readiness'], 'state' | 'reason'>, o: OldBrain) => {
|
|
1030
|
+
if (!this.store.hasBot(botId)) return null;
|
|
1031
|
+
const now = this.store.getBot(botId);
|
|
1032
|
+
const same =
|
|
1033
|
+
now.provider === current.provider &&
|
|
1034
|
+
now.contextSince === current.contextSince &&
|
|
1035
|
+
now.sessionId === current.sessionId &&
|
|
1036
|
+
now.sessionKey === current.sessionKey &&
|
|
1037
|
+
this.store.switchGeneration(botId) === generation;
|
|
1038
|
+
const again = same ? this.#switchPreview(botId, move, input.connections, r, o, hold) : null;
|
|
1039
|
+
return again && again.summary.confirmation === preview.summary.confirmation ? { now, again } : null;
|
|
1040
|
+
};
|
|
1041
|
+
const aborted = () => (signal.reason === INTERRUPTED ? 'Botons is restarting' : signal.reason === PAUSED ? 'it was paused' : 'it was stopped');
|
|
1042
|
+
try {
|
|
1043
|
+
let written: { text: string; turnId: string };
|
|
1044
|
+
try {
|
|
1045
|
+
// Refresh both brains after acquiring the slot; a changed preview must spend no turn.
|
|
1046
|
+
written = await this.#writeSummary(current, preview, signal, async () => {
|
|
1047
|
+
const { ready: r, old: o } = await this.#switchBrains(current.provider, move.provider);
|
|
1048
|
+
return () => !!unchanged(r, o);
|
|
1049
|
+
});
|
|
1050
|
+
} catch (err) {
|
|
1051
|
+
if (err === NOT_STARTED && !signal.aborted) {
|
|
1052
|
+
throw new HttpError(409, `What moving ${current.name} to ${preview.to.label} keeps and loses changed while its summary waited to start, so nothing ran and nothing moved: review it again`);
|
|
1053
|
+
}
|
|
1054
|
+
const why = signal.aborted ? aborted() : errorText(err);
|
|
1055
|
+
throw new HttpError(409, `${current.name}'s session on ${preview.from.label} didn't write a summary (${why}), so nothing moved: move it without one, or try again`);
|
|
1056
|
+
}
|
|
1057
|
+
const { ready: r, old: o } = await this.#switchBrains(current.provider, move.provider);
|
|
1058
|
+
// Nothing awaits from here to the write: what is checked is what is saved.
|
|
1059
|
+
if (signal.aborted) {
|
|
1060
|
+
log.info('switch summary not saved: stopped once it was written', { botId, turnId: written.turnId, why: aborted() });
|
|
1061
|
+
throw new HttpError(409, `Moving ${current.name} was stopped (${aborted()}) once its summary was written, so nothing moved and the summary wasn't kept (its turn on ${preview.from.label} still ran)`);
|
|
1062
|
+
}
|
|
1063
|
+
const still = unchanged(r, o);
|
|
1064
|
+
if (!still) {
|
|
1065
|
+
log.info('switch summary not saved: the move changed while it was written', { botId, turnId: written.turnId });
|
|
1066
|
+
throw new HttpError(
|
|
1067
|
+
409,
|
|
1068
|
+
`What moving ${current.name} to ${preview.to.label} keeps and loses changed while its summary was being written, so nothing moved and the summary wasn't kept (its turn on ${preview.from.label} still ran): review it again`,
|
|
1069
|
+
);
|
|
1070
|
+
}
|
|
1071
|
+
const { now, again } = still;
|
|
1072
|
+
const handoff = this.#handoff(now, again);
|
|
1073
|
+
const { bot, summary } = this.store.atomically(() => {
|
|
1074
|
+
const summary = this.store.addMessage(botId, 'bot', written.text, { kind: 'provider.summary', provider: now.provider, providerLabel: preview.from.label, turnId: written.turnId });
|
|
1075
|
+
const bot = this.store.updateBot(botId, input, providers);
|
|
1076
|
+
this.#recordSwitch(bot, this.#brainRef(now.provider, now.model), { ...handoff, summary: { messageId: summary.id } });
|
|
1077
|
+
return { bot, summary };
|
|
1078
|
+
});
|
|
1079
|
+
this.store.linkTurnMessage(written.turnId, summary.id);
|
|
1080
|
+
this.#afterUpdate(now, bot, input);
|
|
1081
|
+
return bot;
|
|
1082
|
+
} finally {
|
|
1083
|
+
this.#summaries.delete(botId);
|
|
1084
|
+
release();
|
|
1085
|
+
if (this.#chains.get(botId) === gate) this.#chains.delete(botId);
|
|
1086
|
+
this.#setStatus(botId, (this.#queued.get(botId) ?? 0) > 0 ? 'queued' : 'idle');
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
/**
|
|
1091
|
+
* One turn on the Boton's chat session asking for its summary (summaryRequest): no tools or connectors at all
|
|
1092
|
+
* (TurnRequest.toolFree, only on a provider that can run one), no token for the `botons` CLI, its own time limit,
|
|
1093
|
+
* and never tried again. `stillAsked` is checked once it has a slot, before anything runs. It counts as a chat turn on the old provider, usage and all. The reply is the old agent's, bounded (clipText), never trusted.
|
|
1094
|
+
*/
|
|
1095
|
+
async #writeSummary(bot: BotRecord, preview: SwitchFacts, signal: AbortSignal, refreshAsked: () => Promise<() => boolean>): Promise<{ text: string; turnId: string }> {
|
|
1096
|
+
const provider = this.#providers.get(bot.provider);
|
|
1097
|
+
if (!provider) throw new Error(`${preview.from.label} is not on this server`);
|
|
1098
|
+
if (!provider.toolFree) throw new Error(`${preview.from.label} can't run a turn with no tools`);
|
|
1099
|
+
if (!(await this.#slots.acquire(signal))) throw new Error('Stopped');
|
|
1100
|
+
// The slot is held from here: released once, in the finally, whether the turn runs or not.
|
|
1101
|
+
try {
|
|
1102
|
+
if (signal.aborted) throw new Error('Stopped');
|
|
1103
|
+
const stillAsked = await refreshAsked();
|
|
1104
|
+
if (signal.aborted) throw new Error('Stopped');
|
|
1105
|
+
if (!stillAsked()) throw NOT_STARTED;
|
|
1106
|
+
const settings = this.store.getSettings();
|
|
1107
|
+
const sys = systemPrompt(bot, settings, this.#cfg);
|
|
1108
|
+
if (!bot.sessionId || bot.sessionKey !== sessionKey(provider.id, sys)) throw new Error(`no chat session on ${preview.from.label}`);
|
|
1109
|
+
const model = bot.model;
|
|
1110
|
+
const effort = turnEffort(bot, null, this.store.effortCaps(bot.provider, model)).effort;
|
|
1111
|
+
const turnId = this.store.startTurn({ botId: bot.id, trigger: 'chat', routineId: null, runId: null, provider: provider.id, requestedModel: model });
|
|
1112
|
+
const fields: LogFields = { botId: bot.id, turnId, provider: provider.id, model: model || undefined, ...(effort ? { effort } : {}) };
|
|
1113
|
+
const label = 'Writing a summary for the move';
|
|
1114
|
+
this.#setStatus(bot.id, 'working', label);
|
|
1115
|
+
log.info('switch summary started', fields);
|
|
1116
|
+
const started = Date.now();
|
|
1117
|
+
let result: TurnResult;
|
|
1118
|
+
try {
|
|
1119
|
+
result = await provider.runTurn({
|
|
1120
|
+
cwd: bot.workspace,
|
|
1121
|
+
sessionId: bot.sessionId,
|
|
1122
|
+
systemPrompt: sys,
|
|
1123
|
+
prompt: summaryRequest(preview.from, preview.to, settings.ownerName),
|
|
1124
|
+
model,
|
|
1125
|
+
...(effort ? { effort } : {}),
|
|
1126
|
+
access: 'workspace',
|
|
1127
|
+
privatePaths: privatePaths({
|
|
1128
|
+
env: { ...process.env, ...cliHomesEnv() },
|
|
1129
|
+
workspace: bot.workspace,
|
|
1130
|
+
cli: provider.cli ?? null,
|
|
1131
|
+
botonsHome: this.#cfg.home,
|
|
1132
|
+
workspacesDir: this.#cfg.workspacesDir,
|
|
1133
|
+
extra: this.#cfg.privatePaths,
|
|
1134
|
+
shared: [],
|
|
1135
|
+
}),
|
|
1136
|
+
sharedDirs: [],
|
|
1137
|
+
protectInstructions: true,
|
|
1138
|
+
groupDir: this.#cfg.agentsDir,
|
|
1139
|
+
// No BOTONS_API or BOTON_TOKEN: nothing in this turn may act as the Boton.
|
|
1140
|
+
env: {},
|
|
1141
|
+
toolFree: true,
|
|
1142
|
+
spend: { botId: bot.id, turnId, runId: null, routineId: null },
|
|
1143
|
+
signal,
|
|
1144
|
+
timeoutMs: SWITCH_SUMMARY_MS,
|
|
1145
|
+
onActivity: (a) => this.#setStatus(bot.id, 'working', `${label}: ${a}`),
|
|
1146
|
+
});
|
|
1147
|
+
} catch (err) {
|
|
1148
|
+
const outcome = timedOut(err) ? 'timeout' : signal.aborted ? (signal.reason === INTERRUPTED ? 'interrupted' : 'stopped') : 'error';
|
|
1149
|
+
const usage = err instanceof ProviderError ? (err.usage ?? null) : null;
|
|
1150
|
+
this.store.finishTurn(turnId, { outcome, errorKind: outcome === 'error' && err instanceof ProviderError ? err.kind : null, durationMs: Date.now() - started, usage, model: usage?.model, replayedTranscript: false });
|
|
1151
|
+
if (err instanceof ProviderError && err.kind === 'usage_limit' && !signal.aborted) this.#limitHit(bot, err);
|
|
1152
|
+
if (outcome === 'error') this.#providers.recordTurn(provider.id, { ok: false, error: errorText(err) });
|
|
1153
|
+
log.warn('switch summary failed', { ...fields, outcome, error: errorText(err) });
|
|
1154
|
+
if (outcome === 'timeout') throw new Error(`it ran past its time limit (${durationText(SWITCH_SUMMARY_MS)})`);
|
|
1155
|
+
throw err;
|
|
1156
|
+
}
|
|
1157
|
+
this.store.finishTurn(turnId, { outcome: 'ok', durationMs: Date.now() - started, usage: result.usage ?? null, model: result.usage?.model, replayedTranscript: false });
|
|
1158
|
+
this.#providers.recordTurn(provider.id, { ok: true });
|
|
1159
|
+
const { text, cut } = clipText(result.text, SWITCH_SUMMARY_CHARS);
|
|
1160
|
+
log.info('switch summary written', { ...fields, chars: text.length, cut, ...usageFields(result.usage ?? null) });
|
|
1161
|
+
if (!text) throw new Error('it replied with nothing');
|
|
1162
|
+
return { text, turnId };
|
|
1163
|
+
} finally {
|
|
1164
|
+
this.#slots.release();
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
/**
|
|
1169
|
+
* The note for the new provider's first turn, taken now from what Botons stores. A switch whose note no turn has
|
|
1170
|
+
* had yet hands on what that one was moving away from: the provider whose session and context were last live.
|
|
1171
|
+
*/
|
|
1172
|
+
#handoff(bot: BotRecord, preview: SwitchFacts): ProviderHandoff {
|
|
1173
|
+
const now = new Date();
|
|
1174
|
+
// Back to where it was before a switch no turn ran on, with the session saved there to resume: that one was the
|
|
1175
|
+
// move, not this. A session that move ended (a turn was on it) is not there to go back to, so its note stands.
|
|
1176
|
+
const pending = this.store.pendingHandoff(bot.id, bot.contextSince);
|
|
1177
|
+
const underway = this.#underway(bot);
|
|
1178
|
+
const resumes = !!bot.sessionId && !(underway && !underway.own && !underway.fresh);
|
|
1179
|
+
const earlier = pending && (pending.meta.handoff.from.provider !== preview.to.provider || !resumes) ? pending : null;
|
|
1180
|
+
const { runs, dueSoon } = this.store.activeRuns(bot.id, new Date(now.getTime() + 86_400_000), HANDOFF_ITEMS);
|
|
1181
|
+
return {
|
|
1182
|
+
from: earlier ? earlier.meta.handoff.from : { provider: preview.from.provider, label: preview.from.label, model: preview.from.model },
|
|
1183
|
+
to: { provider: preview.to.provider, label: preview.to.label, model: preview.to.model },
|
|
1184
|
+
at: now.toISOString(),
|
|
1185
|
+
sessionEnded: preview.sessions.chat || !!earlier?.meta.handoff.sessionEnded,
|
|
1186
|
+
...(preview.working || earlier?.meta.handoff.working ? { working: true } : {}),
|
|
1187
|
+
kept: preview.kept,
|
|
1188
|
+
waiting: this.approvals
|
|
1189
|
+
.pendingFor(bot.id)
|
|
1190
|
+
.slice(-HANDOFF_ITEMS)
|
|
1191
|
+
.map((a) => ({ id: a.id, title: a.title })),
|
|
1192
|
+
runs,
|
|
1193
|
+
dueSoon,
|
|
1194
|
+
decisions: this.approvals.recentDecisions(bot.id, new Date(now.getTime() - 7 * 86_400_000), HANDOFF_ITEMS),
|
|
1195
|
+
// The summary an earlier move's session wrote, which no turn has had yet (a move with one replaces it).
|
|
1196
|
+
...(earlier?.meta.handoff.summary ? { summary: earlier.meta.handoff.summary } : {}),
|
|
1197
|
+
};
|
|
1198
|
+
}
|
|
1199
|
+
|
|
1200
|
+
/** `from`: what this move left, which the note's `from` is not when it hands on an earlier move's. */
|
|
1201
|
+
#recordSwitch(bot: Bot, from: BrainRef, handoff: ProviderHandoff): void {
|
|
1202
|
+
const earlier = this.store.pendingHandoff(bot.id, this.store.contextSince(bot.id));
|
|
1203
|
+
if (earlier) this.store.endHandoff(earlier.id);
|
|
1204
|
+
const name = (b: ProviderHandoff['from']) => `${b.label} (${b.model || 'its default model'})`;
|
|
1205
|
+
const tail = handoff.sessionEnded ? `: its session on ${handoff.from.label} ended, so the next turn starts a new one with a handoff note` : ': the next turn starts with a handoff note';
|
|
1206
|
+
this.store.addMessage(bot.id, 'event', `Moved from ${name(from)} to ${name(handoff.to)}${tail}`, { kind: 'provider.switched', provider: bot.provider, handoff });
|
|
1207
|
+
log.info('provider switched: the next chat turn starts with a handoff note', { botId: bot.id, provider: bot.provider });
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
/**
|
|
1211
|
+
* What an update did to the levels turns ask for, said in the chat: a level the Boton's new provider or model
|
|
1212
|
+
* doesn't have, which goes back to the default (the Boton's own, unless this update set it: the owner's choice, said
|
|
1213
|
+
* nowhere), and routines whose own model's list leaves out the Boton's new level, whose runs then ask for none.
|
|
1214
|
+
*/
|
|
1215
|
+
#effortNotes(before: BotRecord, bot: Bot, input: UpdateBotInput): void {
|
|
1216
|
+
const name = this.#providerName(bot.provider);
|
|
1217
|
+
const gap = (effort: string, model: string) =>
|
|
1218
|
+
effortProblem(bot.provider, effort) ? `${name} has no effort level “${effort}”` : `${name}'s model list has no effort level “${effort}” for ${model || 'its default model'}`;
|
|
1219
|
+
const moved = bot.provider !== before.provider || bot.model !== before.model;
|
|
1220
|
+
if (moved) {
|
|
1221
|
+
for (const r of this.store.resetRoutineEfforts(bot.id)) {
|
|
1222
|
+
this.store.addMessage(bot.id, 'event', `${gap(r.effort!, r.model || bot.model)}, so routine “${r.name}” (was ${r.effort}) now uses the Boton's effort`, {
|
|
1223
|
+
kind: 'effort.reset',
|
|
1224
|
+
routineId: r.id,
|
|
1225
|
+
routineName: r.name,
|
|
1226
|
+
});
|
|
1227
|
+
}
|
|
1228
|
+
}
|
|
1229
|
+
if (moved && input.effort === undefined && before.effort && !bot.effort) {
|
|
1230
|
+
this.store.addMessage(bot.id, 'event', `${gap(before.effort, bot.model)}, so this Boton now asks for none: the CLI uses its default`, { kind: 'effort.reset' });
|
|
1231
|
+
}
|
|
1232
|
+
if (bot.effort && (bot.effort !== before.effort || moved)) {
|
|
1233
|
+
for (const r of this.store.listRoutines(bot.id)) {
|
|
1234
|
+
if (r.effort || !r.model || !effortProblem(bot.provider, bot.effort, this.store.effortCaps(bot.provider, r.model))) continue;
|
|
1235
|
+
this.store.addMessage(bot.id, 'event', `${gap(bot.effort, r.model)}, so routine “${r.name}” asks for none on its runs: give it an effort of its own`, {
|
|
1236
|
+
kind: 'effort.reset',
|
|
1237
|
+
routineId: r.id,
|
|
1238
|
+
routineName: r.name,
|
|
1239
|
+
});
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
pauseAll(paused: boolean): Settings {
|
|
1245
|
+
const settings = this.store.setPausedAll(paused);
|
|
1246
|
+
// A move's summary too, waiting for a slot or writing: stopped for good, so a resume doesn't bring it back.
|
|
1247
|
+
if (paused) for (const botId of new Set([...this.#jobs.keys(), ...this.#summaries.keys()])) this.#halt(botId, PAUSED);
|
|
1248
|
+
else this.#resumeWork(() => true);
|
|
1249
|
+
return settings;
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1252
|
+
/** `why`: what a move's summary being written is told (the Boton's jobs are stopped either way). */
|
|
1253
|
+
#halt(botId: string, why: Error = STOPPED) {
|
|
1254
|
+
this.#summaries.get(botId)?.controller.abort(why);
|
|
1255
|
+
const job = this.#jobs.get(botId);
|
|
1256
|
+
job?.wait.abort();
|
|
1257
|
+
if (job) this.#abortTurn(job, STOPPED);
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
/** Ends the job's turn: its agent process, or (its connectors still being set up) the one it was about to start. */
|
|
1261
|
+
#abortTurn(job: Job, reason: Error) {
|
|
1262
|
+
job.aborted ??= job.turn?.signal.aborted ? job.turn.signal.reason : reason;
|
|
1263
|
+
job.turn?.abort(job.aborted);
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
/** Never rejects: a run whose row is gone resolves as failed. */
|
|
1267
|
+
waitForRun(run: RoutineRun, timeoutMs: number, signal?: AbortSignal): Promise<RoutineRun> {
|
|
1268
|
+
return new Promise((resolve) => {
|
|
1269
|
+
let last = run;
|
|
1270
|
+
const current = (reason: string) => this.store.findRun(run.id) ?? goneRun(last, reason);
|
|
1271
|
+
const done = (r: RoutineRun) => {
|
|
1272
|
+
clearTimeout(timer);
|
|
1273
|
+
unsubscribe();
|
|
1274
|
+
signal?.removeEventListener('abort', onAbort);
|
|
1275
|
+
this.#closing.signal.removeEventListener('abort', onAbort);
|
|
1276
|
+
resolve(r);
|
|
1277
|
+
};
|
|
1278
|
+
const onAbort = () => done(current('Run was deleted'));
|
|
1279
|
+
this.#closing.signal.addEventListener('abort', onAbort, { once: true });
|
|
1280
|
+
const unsubscribe = this.#hub.subscribe((ev) => {
|
|
1281
|
+
if (ev.type === 'run.upserted' && ev.run.id === run.id) {
|
|
1282
|
+
last = ev.run;
|
|
1283
|
+
if (isFinished(ev.run)) done(ev.run);
|
|
1284
|
+
} else if (ev.type === 'routine.deleted' && ev.routineId === run.routineId) {
|
|
1285
|
+
done(current('Routine was deleted'));
|
|
1286
|
+
} else if (ev.type === 'bot.deleted' && ev.botId === run.botId) {
|
|
1287
|
+
done(current('Boton was deleted'));
|
|
1288
|
+
}
|
|
1289
|
+
});
|
|
1290
|
+
const timer = setTimeout(onAbort, timeoutMs);
|
|
1291
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
1292
|
+
const now = current('Run was deleted');
|
|
1293
|
+
if (isFinished(now) || signal?.aborted || this.#closing.signal.aborted) done(now);
|
|
1294
|
+
});
|
|
1295
|
+
}
|
|
1296
|
+
|
|
1297
|
+
get closing(): AbortSignal {
|
|
1298
|
+
return this.#closing.signal;
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
endWaits(): void {
|
|
1302
|
+
this.#closing.abort();
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
// ------------------------------------------------------------------- queue
|
|
1306
|
+
|
|
1307
|
+
#enqueue(botId: string, work: Work) {
|
|
1308
|
+
this.#queued.set(botId, (this.#queued.get(botId) ?? 0) + 1);
|
|
1309
|
+
if (this.statusOf(botId).state === 'idle') this.#setStatus(botId, 'queued');
|
|
1310
|
+
const prev = this.#chains.get(botId) ?? Promise.resolve();
|
|
1311
|
+
const next = prev
|
|
1312
|
+
.then(() => this.#execute(botId, work))
|
|
1313
|
+
.catch((err) => log.error('job failed', { botId, err }));
|
|
1314
|
+
this.#chains.set(botId, next);
|
|
1315
|
+
void next.then(() => {
|
|
1316
|
+
if (this.#chains.get(botId) === next) this.#chains.delete(botId);
|
|
1317
|
+
});
|
|
1318
|
+
}
|
|
1319
|
+
|
|
1320
|
+
async #execute(botId: string, work: Work) {
|
|
1321
|
+
this.#queued.set(botId, (this.#queued.get(botId) ?? 1) - 1);
|
|
1322
|
+
try {
|
|
1323
|
+
// Shutting down: the message stays pending and the run queued, for the next process.
|
|
1324
|
+
if (this.#draining || this.#maintenance) return;
|
|
1325
|
+
if (!this.store.hasBot(botId)) {
|
|
1326
|
+
if (work.kind === 'routine') this.#failRunQuietly(work.runId, 'Boton was deleted');
|
|
1327
|
+
return;
|
|
1328
|
+
}
|
|
1329
|
+
// Paused: resuming queues it again.
|
|
1330
|
+
if (!this.#hasWork(botId, work) || this.store.pausedReason(botId)) return;
|
|
1331
|
+
const job: Job = { wait: new AbortController(), ...(work.kind === 'routine' ? { runId: work.runId } : {}) };
|
|
1332
|
+
this.#jobs.set(botId, job);
|
|
1333
|
+
let slot = false;
|
|
1334
|
+
try {
|
|
1335
|
+
slot = await this.#slots.acquire(job.wait.signal);
|
|
1336
|
+
if (this.#draining || this.#maintenance || this.store.pausedReason(botId)) return;
|
|
1337
|
+
// Only stop(), pausing and cancelling its run give up the wait.
|
|
1338
|
+
if (!slot) return this.#stoppedBeforeStart(botId, work);
|
|
1339
|
+
if (work.kind === 'chat') await this.#chatTurn(botId, job);
|
|
1340
|
+
else await this.#routineTurn(botId, work.runId, job);
|
|
1341
|
+
} finally {
|
|
1342
|
+
if (slot) this.#slots.release();
|
|
1343
|
+
this.#jobs.delete(botId);
|
|
1344
|
+
}
|
|
1345
|
+
} finally {
|
|
1346
|
+
this.#setStatus(botId, (this.#queued.get(botId) ?? 0) > 0 ? 'queued' : 'idle');
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
#hasWork(botId: string, work: Work): boolean {
|
|
1351
|
+
if (work.kind === 'chat') return this.store.hasPendingChat(botId);
|
|
1352
|
+
const run = this.store.findRun(work.runId);
|
|
1353
|
+
if (run?.status !== 'queued') return false;
|
|
1354
|
+
if (run.retryAt && new Date(run.retryAt).getTime() > Date.now()) {
|
|
1355
|
+
this.#wakeRun(run);
|
|
1356
|
+
return false;
|
|
1357
|
+
}
|
|
1358
|
+
return true;
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
#stoppedBeforeStart(botId: string, work: Work) {
|
|
1362
|
+
if (work.kind === 'chat') {
|
|
1363
|
+
if (this.store.dropPendingChat(botId)) this.#postStopped(botId);
|
|
1364
|
+
return;
|
|
1365
|
+
}
|
|
1366
|
+
const run = this.store.findRun(work.runId);
|
|
1367
|
+
if (run?.status !== 'queued') return;
|
|
1368
|
+
this.#failRunQuietly(run.id, 'Stopped', { cancelled: { started: run.attempts > 0 } });
|
|
1369
|
+
this.#postStopped(botId, run);
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
#chipCutOff(botId: string, turn: TurnInfo, text: string) {
|
|
1373
|
+
if (!turn.id || !turn.steps) return;
|
|
1374
|
+
try {
|
|
1375
|
+
if (!this.store.hasBot(botId)) return;
|
|
1376
|
+
this.#linkTurn(turn, this.store.addMessage(botId, 'event', text, { kind: 'turn.interrupted', ...this.#turnMeta(turn) }));
|
|
1377
|
+
} catch (err) {
|
|
1378
|
+
log.debug('could not post the chip of a cut-off turn', { botId, err });
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
#postStopped(botId: string, run?: RoutineRun, turn?: TurnInfo, text = 'Stopped'): Message {
|
|
1383
|
+
const name = run ? this.store.findRoutineName(run.routineId) : null;
|
|
1384
|
+
return this.store.addMessage(botId, 'event', text, {
|
|
1385
|
+
kind: 'turn.stopped',
|
|
1386
|
+
...(run ? { routineId: run.routineId, ...(name ? { routineName: name } : {}), runId: run.id } : {}),
|
|
1387
|
+
...(turn ? this.#turnMeta(turn) : {}),
|
|
1388
|
+
});
|
|
1389
|
+
}
|
|
1390
|
+
|
|
1391
|
+
#failRunQuietly(runId: string, error: string, how?: { cancelled?: { started: boolean }; timedOut?: { limitMs: number } }) {
|
|
1392
|
+
try {
|
|
1393
|
+
this.store.updateRun(runId, { status: 'failed', error, ...how });
|
|
1394
|
+
} catch {}
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
// ------------------------------------------------------------------- turns
|
|
1398
|
+
|
|
1399
|
+
/** The turn has read the Boton: what it runs on from now, whatever the Boton changes to meanwhile. Synchronous. */
|
|
1400
|
+
#readBy(job: Job, bot: BotRecord, routine?: { routine: Routine; trigger: RunTrigger }) {
|
|
1401
|
+
const session = routine ? runSession(routine.routine, routine.trigger) : 'chat';
|
|
1402
|
+
job.read = Object.freeze({
|
|
1403
|
+
stamp: Object.freeze({ contextSince: bot.contextSince, provider: bot.provider, generation: this.store.switchGeneration(bot.id) }),
|
|
1404
|
+
handoff: this.store.pendingHandoff(bot.id, bot.contextSince),
|
|
1405
|
+
own: session === 'own' ? routine!.routine.id : null,
|
|
1406
|
+
fresh: session === 'fresh',
|
|
1407
|
+
});
|
|
1408
|
+
}
|
|
1409
|
+
|
|
1410
|
+
async #chatTurn(botId: string, job: Job) {
|
|
1411
|
+
const bot = orNull(() => this.store.getBot(botId));
|
|
1412
|
+
if (!bot) return;
|
|
1413
|
+
this.#readBy(job, bot);
|
|
1414
|
+
const until = this.#coolingUntil(bot.provider);
|
|
1415
|
+
if (until) return this.#noticeUsageLimit(bot, until);
|
|
1416
|
+
// Marks them in flight: if the process dies before the reply is saved, the next one re-runs them with a note.
|
|
1417
|
+
const { messages: pending, interrupted, more } = this.store.startChatTurn(botId);
|
|
1418
|
+
if (!pending.length) return;
|
|
1419
|
+
// Messages for another audience wait for a turn of their own, once this one's are answered.
|
|
1420
|
+
const next = () => {
|
|
1421
|
+
if (more && !this.store.pausedReason(botId)) this.#enqueue(botId, { kind: 'chat' });
|
|
1422
|
+
};
|
|
1423
|
+
const ids = pending.map((m) => m.id);
|
|
1424
|
+
const started = Date.now();
|
|
1425
|
+
const startedAt = nowIso();
|
|
1426
|
+
const turn = newTurnInfo();
|
|
1427
|
+
// The owner's reactions not yet given (to a turn for the owner, never a colleague's ask), and the owner's own
|
|
1428
|
+
// messages it may react to.
|
|
1429
|
+
const feedback = pending.some((m) => m.meta?.kind === 'ask') ? [] : this.reactions.feedback(botId);
|
|
1430
|
+
const reactable = pending.filter(ownersOwn).map((m) => m.id);
|
|
1431
|
+
try {
|
|
1432
|
+
const owner = this.store.getSettings().ownerName;
|
|
1433
|
+
const found = await attachmentsNow(bot.workspace, pending);
|
|
1434
|
+
const text = chatInput(
|
|
1435
|
+
pending.map((m) => {
|
|
1436
|
+
// What the Boton reads of a decision comes from the approval request as saved.
|
|
1437
|
+
if (m.meta?.kind === 'approval.decided') return { ...m, text: this.approvals.decisionPrompt(m, owner) };
|
|
1438
|
+
if (m.meta?.kind === 'work.event') return { ...m, text: workEventPrompt(m.meta.work, owner) };
|
|
1439
|
+
if (m.meta?.kind === 'connector.granted') return { ...m, text: grantedPrompt(m.meta.connectorId ?? '') };
|
|
1440
|
+
if (m.meta?.kind === 'reaction.followup') return { ...m, text: this.#followUpText(m, owner) };
|
|
1441
|
+
if (m.meta?.kind === 'card.answered') return { ...m, text: `(${owner}'s answer to your card ${(m.meta.cardId ?? '').replace(/[^A-Za-z0-9_-]/g, '')}, from the chat:)\n${m.text}` };
|
|
1442
|
+
// Only the owner's messages run a /<skill>, not another Boton's.
|
|
1443
|
+
const call = m.meta?.fromBotId || m.meta?.kind ? null : this.skills.invoked(bot, m.text);
|
|
1444
|
+
const typed = call ? { ...m, text: skillInvocation(m.text, call.skill, call.input, owner) } : m;
|
|
1445
|
+
return { ...typed, text: withAttachments(typed, bot.workspace, found, owner) };
|
|
1446
|
+
}),
|
|
1447
|
+
owner,
|
|
1448
|
+
);
|
|
1449
|
+
const images = pending.flatMap((m) => m.meta?.attachments ?? []).flatMap((a) => {
|
|
1450
|
+
const at = found.get(a.path);
|
|
1451
|
+
return at && isAttachableImage(a) ? [at] : [];
|
|
1452
|
+
});
|
|
1453
|
+
const askers = [...new Set(pending.flatMap((m) => (m.meta?.kind === 'ask' && m.meta.fromBotName ? [m.meta.fromBotName] : [])))];
|
|
1454
|
+
const left = this.#unanswered.get(botId);
|
|
1455
|
+
this.#unanswered.delete(botId);
|
|
1456
|
+
const unanswered = !!left && pending.some((m) => left.has(m.id));
|
|
1457
|
+
const body = `${interrupted ? `${INTERRUPTED_NOTE}\n\n` : ''}${unanswered ? `${unansweredNote(owner)}\n\n` : ''}${text}`;
|
|
1458
|
+
const result = await this.#invoke(bot, chatTrigger(pending), body, turn, job, { ids, images, askers, chain: threadOf(pending), reactable, feedback });
|
|
1459
|
+
this.reactions.markTold(feedback);
|
|
1460
|
+
const note = this.#quietReply(result.text, pending.some(ownersOwn), turn);
|
|
1461
|
+
if (note !== null) {
|
|
1462
|
+
this.store.atomically(() => {
|
|
1463
|
+
// Linked and marked together: a quiet turn whose first message is marked has exactly its own marked.
|
|
1464
|
+
this.store.linkTurnMessage(turn.id, ids[0]!);
|
|
1465
|
+
this.store.markProcessed(ids);
|
|
1466
|
+
// Nothing was posted, so the wake-ups it answered and what it did in the chat leave the transcript too (the
|
|
1467
|
+
// API and history keep them).
|
|
1468
|
+
this.store.markQuiet(turn.id, ids);
|
|
1469
|
+
});
|
|
1470
|
+
this.#answerAsks(bot, pending, note || '(nothing to report)');
|
|
1471
|
+
return next();
|
|
1472
|
+
}
|
|
1473
|
+
// Reacted instead of replying (exactly [quiet], nothing more), to the owner's own messages only: each message it
|
|
1474
|
+
// reacted to in this turn is answered with nothing posted. One it didn't react to waits for another turn, which
|
|
1475
|
+
// ends: each such turn answers at least one message, and one that reacts to none posts its reply as usual.
|
|
1476
|
+
const quiet = !result.text.trim() || QUIET_ONLY.test(result.text.trim());
|
|
1477
|
+
const reacted = quiet && pending.every(ownersOwn) ? this.reactions.botonReactedTo(botId, ids, startedAt) : new Set<string>();
|
|
1478
|
+
if (reacted.size) {
|
|
1479
|
+
const waiting = ids.filter((id) => !reacted.has(id));
|
|
1480
|
+
this.store.atomically(() => {
|
|
1481
|
+
this.store.markProcessed([...reacted]);
|
|
1482
|
+
this.store.requeueChat(waiting);
|
|
1483
|
+
});
|
|
1484
|
+
if (waiting.length) {
|
|
1485
|
+
this.#unanswered.set(botId, new Set(waiting));
|
|
1486
|
+
this.#enqueue(botId, { kind: 'chat' });
|
|
1487
|
+
} else next();
|
|
1488
|
+
return;
|
|
1489
|
+
}
|
|
1490
|
+
const reply = result.text.trim() || '(no reply)';
|
|
1491
|
+
const posted = this.store.finishChatTurn(ids, botId, reply, this.#postMeta(bot, this.#replyMeta(bot, started, turn)));
|
|
1492
|
+
this.store.linkTurnMessage(turn.id, posted.id);
|
|
1493
|
+
this.#answerAsks(bot, pending, reply);
|
|
1494
|
+
next();
|
|
1495
|
+
} catch (err) {
|
|
1496
|
+
// Ran past its time limit first: an error like any other, whatever came while it was exiting.
|
|
1497
|
+
const reason = timedOut(err) ? null : abortReason(job);
|
|
1498
|
+
// Cut off by a shutdown: the messages stay in flight for the next process.
|
|
1499
|
+
if (reason === INTERRUPTED) return void this.#chipCutOff(botId, turn, 'Cut off by a restart');
|
|
1500
|
+
// The usage limit: they stay in flight too (so the retry gets the note) until it resets.
|
|
1501
|
+
if (err instanceof ProviderError && err.kind === 'usage_limit' && reason !== STOPPED) {
|
|
1502
|
+
this.#noticeUsageLimit(bot, this.#limitHit(bot, err));
|
|
1503
|
+
this.#chipCutOff(botId, turn, 'Cut off by the usage limit');
|
|
1504
|
+
return;
|
|
1505
|
+
}
|
|
1506
|
+
this.store.markProcessed(ids);
|
|
1507
|
+
if (!this.store.hasBot(botId)) return;
|
|
1508
|
+
next();
|
|
1509
|
+
if (reason === STOPPED) return void this.#linkTurn(turn, this.#postStopped(botId, undefined, turn));
|
|
1510
|
+
const msg = timedOut(err) ? `Stopped: the turn ran past its time limit (${durationText(turn.limitMs)}). It may have partly run.` : errorText(err);
|
|
1511
|
+
this.#linkTurn(turn, this.#post(bot, 'error', msg, this.#turnMeta(turn)));
|
|
1512
|
+
// A Boton waiting on an answer hears that none is coming (not when the owner stopped the turn: they meant to).
|
|
1513
|
+
this.#answerAsks(bot, pending, `(${bot.name} could not answer: ${msg})`);
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
|
|
1517
|
+
async #routineTurn(botId: string, runId: string, job: Job) {
|
|
1518
|
+
const started = Date.now();
|
|
1519
|
+
const queued = this.store.findRun(runId);
|
|
1520
|
+
// Gone: its routine or Boton was deleted while it was queued, and the delete already failed it.
|
|
1521
|
+
if (!queued || queued.status !== 'queued') return;
|
|
1522
|
+
const routine = orNull(() => this.store.getRoutine(queued.routineId));
|
|
1523
|
+
if (!routine) return this.#failRunQuietly(runId, 'Routine was deleted');
|
|
1524
|
+
const bot = orNull(() => this.store.getBot(botId));
|
|
1525
|
+
if (!bot) return this.#failRunQuietly(runId, 'Boton was deleted');
|
|
1526
|
+
this.#readBy(job, bot, { routine, trigger: queued.trigger });
|
|
1527
|
+
// A network retry is dropped once its routine is paused or (a scheduled one) its next slot has come; one held for a usage limit runs at the reset.
|
|
1528
|
+
if (queued.attempts > 0 && this.store.runHeldFor(runId) !== 'usage_limit') {
|
|
1529
|
+
const paused = routinePaused(routine, queued);
|
|
1530
|
+
const slotDue = queued.trigger === 'schedule' && !!routine.nextRunAt && new Date(routine.nextRunAt).getTime() <= Date.now();
|
|
1531
|
+
if (paused || slotDue) return this.#failRun(bot, routine, queued, queued.lastError ?? 'Failed', paused ? 'the routine was paused' : 'its next scheduled run is due');
|
|
1532
|
+
}
|
|
1533
|
+
const until = this.#coolingUntil(bot.provider);
|
|
1534
|
+
if (until) {
|
|
1535
|
+
this.store.holdRun(runId, until, `${this.#providerName(bot.provider)} usage limit until ${when(until, bot.timezone)}`, 'usage_limit');
|
|
1536
|
+
this.#noticeUsageLimit(bot, until);
|
|
1537
|
+
return;
|
|
1538
|
+
}
|
|
1539
|
+
const run = this.store.startRun(runId);
|
|
1540
|
+
job.run = { runId, routineId: routine.id, routineName: routine.name, trigger: run.trigger };
|
|
1541
|
+
const turn = newTurnInfo();
|
|
1542
|
+
try {
|
|
1543
|
+
const trigger: Trigger = { kind: 'routine', routine, run };
|
|
1544
|
+
const { access, sandboxed } = turnScope(bot, trigger);
|
|
1545
|
+
const skill = routine.skill ? (this.skills.enabledFor(bot, routine.skill) ?? { missing: routine.skill }) : undefined;
|
|
1546
|
+
// The run record has the whole payload, which RoutineRun only previews.
|
|
1547
|
+
const prompt = routinePrompt(routine, this.store.findRunRecord(runId)!, { owner: this.store.getSettings().ownerName, sandboxed, held: access !== bot.access, skill, limitMs: this.#limitFor(routine), session: runSession(routine, run.trigger) });
|
|
1548
|
+
const result = await this.#invoke(bot, trigger, prompt, turn, job);
|
|
1549
|
+
const note = this.#quietReply(result.text, false, turn);
|
|
1550
|
+
if (note !== null) {
|
|
1551
|
+
if (this.store.findRun(runId)) this.store.updateRun(runId, { status: 'succeeded', quiet: true, reply: note ? note.slice(0, 8000) : null });
|
|
1552
|
+
// What it did in the chat goes with it.
|
|
1553
|
+
if (this.store.hasBot(botId)) this.store.markQuiet(turn.id, []);
|
|
1554
|
+
return;
|
|
1555
|
+
}
|
|
1556
|
+
const text = result.text.trim() || '(no reply)';
|
|
1557
|
+
// The routine (and this run) may have been deleted during the turn, by the Boton even: the reply still belongs in the chat.
|
|
1558
|
+
if (this.store.hasBot(botId)) {
|
|
1559
|
+
this.#postRunChip(bot, routine, run);
|
|
1560
|
+
this.#linkTurn(turn, this.#post(bot, 'bot', text, { ...this.#replyMeta(bot, started, turn), routineId: routine.id, routineName: routine.name, runId }));
|
|
1561
|
+
}
|
|
1562
|
+
if (this.store.findRun(runId)) {
|
|
1563
|
+
this.store.updateRun(runId, { status: 'succeeded', reply: text.slice(0, 8000) });
|
|
1564
|
+
if (this.#policyOf(routine) === 'always' && run.trigger !== 'test') {
|
|
1565
|
+
this.#pushForRun(runId, {
|
|
1566
|
+
bot,
|
|
1567
|
+
reason: 'result',
|
|
1568
|
+
detail: `“${routine.name}”: ${text}`,
|
|
1569
|
+
generic: 'A routine has a new result.',
|
|
1570
|
+
priority: 'default',
|
|
1571
|
+
});
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
} catch (err) {
|
|
1575
|
+
// Ran past its time limit before a stop, cancel or shutdown reached it: it ends as timed out.
|
|
1576
|
+
const reason = timedOut(err) ? null : abortReason(job);
|
|
1577
|
+
// Cut off by a shutdown: the run stays running, and the next process marks it interrupted.
|
|
1578
|
+
if (reason === INTERRUPTED) return;
|
|
1579
|
+
// Deleted mid-turn: it already ended as failed.
|
|
1580
|
+
if (!this.store.findRun(runId)) return;
|
|
1581
|
+
// The owner ended it: never retried by itself, and no push about a failure they caused.
|
|
1582
|
+
if (reason === STOPPED || reason === CANCELLED) {
|
|
1583
|
+
const why = reason === CANCELLED ? 'Cancelled' : 'Stopped';
|
|
1584
|
+
// An agent process got this attempt's prompt, or an earlier attempt (held to try again) ran.
|
|
1585
|
+
this.#failRunQuietly(runId, why, { cancelled: { started: !!job.spawned || run.attempts > 1 } });
|
|
1586
|
+
if (this.store.hasBot(botId)) {
|
|
1587
|
+
this.#postRunChip(bot, routine, run);
|
|
1588
|
+
this.#linkTurn(turn, this.#postStopped(botId, this.store.getRun(runId), turn, why));
|
|
1589
|
+
}
|
|
1590
|
+
return;
|
|
1591
|
+
}
|
|
1592
|
+
// Never tried again by itself (not even a scheduled run): it may have partly run, and would likely run as long again.
|
|
1593
|
+
if (timedOut(err)) {
|
|
1594
|
+
log.warn('run stopped at its time limit', { botId, routineId: routine.id, runId, trigger: run.trigger, attempt: run.attempts, timeLimitMs: turn.limitMs });
|
|
1595
|
+
return this.#failRun(bot, routine, run, `Ran past its time limit (${durationText(turn.limitMs)}) and was stopped`, undefined, turn, turn.limitMs);
|
|
1596
|
+
}
|
|
1597
|
+
const msg = errorText(err);
|
|
1598
|
+
if (err instanceof ProviderError && err.kind === 'usage_limit') {
|
|
1599
|
+
const reset = this.#limitHit(bot, err);
|
|
1600
|
+
this.store.holdRun(runId, reset, msg, 'usage_limit');
|
|
1601
|
+
this.#noticeUsageLimit(bot, reset);
|
|
1602
|
+
return;
|
|
1603
|
+
}
|
|
1604
|
+
const delay = this.#retryDelay(run, err);
|
|
1605
|
+
if (delay !== null) {
|
|
1606
|
+
const held = this.store.holdRun(runId, new Date(Date.now() + delay), msg, 'retry');
|
|
1607
|
+
log.warn('run failed on a network error: trying again later', { botId, routineId: routine.id, runId, trigger: run.trigger, attempt: held.attempts, retryAt: held.retryAt, error: msg.slice(0, 200) });
|
|
1608
|
+
this.#wakeRun(held);
|
|
1609
|
+
return;
|
|
1610
|
+
}
|
|
1611
|
+
const unknownModel = err instanceof ProviderError && err.kind === 'other' && !!routine.model && routine.model !== bot.model;
|
|
1612
|
+
const hint = unknownModel ? ` (This routine asks for the model “${routine.model}”, not the Boton's: if ${this.#providerName(bot.provider)} doesn't have it, change or clear the routine's model.)` : '';
|
|
1613
|
+
this.#failRun(bot, routine, run, `${msg}${hint}`, undefined, turn);
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
|
|
1617
|
+
/**
|
|
1618
|
+
* What every turn's local connectors never see (connector-launch.ts), whatever the Boton's access: the data folder,
|
|
1619
|
+
* every workspace and the shared folder by any path, and this Boton's private paths where they fall in their view.
|
|
1620
|
+
*/
|
|
1621
|
+
#connectorLaunch(bot: BotRecord, provider: Provider): LaunchPolicy {
|
|
1622
|
+
const workspaces = this.store.listBots().map((b) => b.workspace);
|
|
1623
|
+
const apart = [this.#cfg.home, this.#cfg.workspacesDir, this.#cfg.sharedDir, bot.workspace, ...workspaces];
|
|
1624
|
+
const hidden = privatePaths({
|
|
1625
|
+
env: { ...process.env, ...cliHomesEnv() },
|
|
1626
|
+
workspace: bot.workspace,
|
|
1627
|
+
cli: provider.cli ?? null,
|
|
1628
|
+
botonsHome: this.#cfg.home,
|
|
1629
|
+
workspacesDir: this.#cfg.workspacesDir,
|
|
1630
|
+
extra: this.#cfg.privatePaths,
|
|
1631
|
+
shared: [this.#cfg.sharedDir],
|
|
1632
|
+
});
|
|
1633
|
+
return { apart, hidden, dir: this.connectors.launchDir };
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
/** A turn's time limit: a routine's own, never past the server's (which may have been lowered since it was set). */
|
|
1637
|
+
#limitFor(routine: Routine | null): number {
|
|
1638
|
+
return Math.min(routine?.timeLimitMs ?? this.#cfg.turnTimeoutMs, this.#cfg.turnTimeoutMs);
|
|
1639
|
+
}
|
|
1640
|
+
|
|
1641
|
+
#postRunChip(bot: BotRecord, routine: Routine, run: RoutineRun) {
|
|
1642
|
+
this.#post(bot, 'event', ...runChip(run, routine.name));
|
|
1643
|
+
}
|
|
1644
|
+
|
|
1645
|
+
/** `timedOutAt`: the time limit it ran past, in ms. */
|
|
1646
|
+
#failRun(bot: BotRecord, routine: Routine, run: RoutineRun, msg: string, why?: string, turn?: TurnInfo, timedOutAt?: number) {
|
|
1647
|
+
this.#failRunQuietly(run.id, msg, timedOutAt ? { timedOut: { limitMs: timedOutAt } } : undefined);
|
|
1648
|
+
const tries = run.attempts > 1 ? ` after ${run.attempts} attempts` : '';
|
|
1649
|
+
const raise =
|
|
1650
|
+
timedOutAt === undefined
|
|
1651
|
+
? ''
|
|
1652
|
+
: timedOutAt < this.#cfg.turnTimeoutMs
|
|
1653
|
+
? `. It may have partly run, so it isn't tried again by itself. If it needs longer, raise its time limit (up to ${durationText(this.#cfg.turnTimeoutMs)}).`
|
|
1654
|
+
: `. It may have partly run, so it isn't tried again by itself. ${durationText(timedOutAt)} is the longest this server lets a turn run (BOTONS_TURN_TIMEOUT_MS).`;
|
|
1655
|
+
const note = why ? ` (not tried again: ${why})` : raise;
|
|
1656
|
+
if (this.store.hasBot(bot.id)) {
|
|
1657
|
+
this.#postRunChip(bot, routine, run);
|
|
1658
|
+
const meta: MessageMeta = { routineId: routine.id, routineName: routine.name, runId: run.id, ...(turn ? this.#turnMeta(turn) : {}) };
|
|
1659
|
+
const error = this.#post(bot, 'error', `Routine “${routine.name}” failed${tries}: ${msg}${note}`, meta);
|
|
1660
|
+
if (turn) this.#linkTurn(turn, error);
|
|
1661
|
+
}
|
|
1662
|
+
if (this.#policyOf(routine) !== 'never' && run.trigger !== 'test') {
|
|
1663
|
+
this.#pushForRun(run.id, {
|
|
1664
|
+
bot,
|
|
1665
|
+
reason: 'failure',
|
|
1666
|
+
detail: `“${routine.name}” failed${tries}: ${msg}`,
|
|
1667
|
+
generic: 'A routine failed.',
|
|
1668
|
+
priority: 'high',
|
|
1669
|
+
});
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
|
|
1673
|
+
#limitHit(bot: BotRecord, err: ProviderError): Date {
|
|
1674
|
+
let reset = err.retryAt ?? new Date(Date.now() + DEFAULT_COOLDOWN_MS);
|
|
1675
|
+
// A reset that has already come would re-run the turn at once: wait a minute instead.
|
|
1676
|
+
if (reset.getTime() <= Date.now()) reset = new Date(Date.now() + MIN_COOLDOWN_MS);
|
|
1677
|
+
this.#startCooldown(bot.provider, reset, err.message);
|
|
1678
|
+
return this.#coolingUntil(bot.provider) ?? reset;
|
|
1679
|
+
}
|
|
1680
|
+
|
|
1681
|
+
#retryDelay(run: RoutineRun, err: unknown): number | null {
|
|
1682
|
+
if (!(err instanceof ProviderError) || err.kind !== 'transient') return null;
|
|
1683
|
+
if (byHand(run) || run.attempts > this.#cfg.runRetries) return null;
|
|
1684
|
+
const delay = this.#cfg.runRetryDelayMs * RETRY_STEPS[Math.min(run.attempts, RETRY_STEPS.length) - 1];
|
|
1685
|
+
const routine = orNull(() => this.store.getRoutine(run.routineId));
|
|
1686
|
+
if (!routine || routinePaused(routine, run)) return null;
|
|
1687
|
+
if (run.trigger === 'schedule' && routine.nextRunAt && new Date(routine.nextRunAt).getTime() <= Date.now() + delay) return null;
|
|
1688
|
+
return delay;
|
|
1689
|
+
}
|
|
1690
|
+
|
|
1691
|
+
pushInterrupted(runs: RoutineRun[]): void {
|
|
1692
|
+
for (const run of runs) {
|
|
1693
|
+
const routine = orNull(() => this.store.getRoutine(run.routineId));
|
|
1694
|
+
const bot = routine && orNull(() => this.store.getBot(run.botId));
|
|
1695
|
+
if (!routine || !bot || routine.notify === 'never' || run.trigger === 'test') continue;
|
|
1696
|
+
this.#pushForRun(run.id, {
|
|
1697
|
+
bot,
|
|
1698
|
+
reason: 'failure',
|
|
1699
|
+
detail: `“${routine.name}” was interrupted by a restart`,
|
|
1700
|
+
generic: 'A routine failed.',
|
|
1701
|
+
priority: 'high',
|
|
1702
|
+
});
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
|
|
1706
|
+
/** The routine's notify policy now (the Boton may have changed it during the run), else as the run started. */
|
|
1707
|
+
#policyOf(routine: Routine): NotifyPolicy {
|
|
1708
|
+
return orNull(() => this.store.getRoutine(routine.id).notify) ?? routine.notify;
|
|
1709
|
+
}
|
|
1710
|
+
|
|
1711
|
+
#pushForRun(runId: string, req: PushRequest): void {
|
|
1712
|
+
void this.notifier
|
|
1713
|
+
.push(req)
|
|
1714
|
+
.then((deliveries) => {
|
|
1715
|
+
for (const d of deliveries) this.store.addRunDelivery(runId, d);
|
|
1716
|
+
})
|
|
1717
|
+
.catch((err) => log.warn('could not record a push for a run', { runId, err }));
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
async notify(botId: string, text: unknown, priority: unknown = 'default'): Promise<NotifyResult> {
|
|
1721
|
+
const bot = this.store.getBot(botId);
|
|
1722
|
+
if (typeof text !== 'string' || !text.trim()) throw new HttpError(400, 'text is required');
|
|
1723
|
+
const t = text.trim();
|
|
1724
|
+
if (t.length > 1000) throw new HttpError(400, 'text is too long (max 1000 characters)');
|
|
1725
|
+
if (priority !== 'default' && priority !== 'high') throw new HttpError(400, 'priority must be "default" or "high"');
|
|
1726
|
+
const current = this.#jobs.get(botId)?.run;
|
|
1727
|
+
const chip: MessageMeta = { kind: 'notify.sent', notifyText: t, priority, ...runMeta(current) };
|
|
1728
|
+
const message = this.store.addMessage(botId, 'event', 'Sent a notification', this.#postMeta(this.#turns.get(botId)?.bot ?? bot, chip));
|
|
1729
|
+
|
|
1730
|
+
let skipped: string | null = null;
|
|
1731
|
+
const policy = current ? orNull(() => this.store.getRoutine(current.routineId).notify) : null;
|
|
1732
|
+
const now = Date.now();
|
|
1733
|
+
const recent = (this.#notifyPushes.get(botId) ?? []).filter((at) => now - at < 3_600_000);
|
|
1734
|
+
if (!this.notifier.enabled) skipped = 'push notifications are not set up';
|
|
1735
|
+
else if (current?.trigger === 'test') skipped = `this is a test run of routine “${current.routineName}”, which pushes nothing`;
|
|
1736
|
+
else if (policy === 'never') skipped = `routine “${current!.routineName}” is set to never notify`;
|
|
1737
|
+
else if (policy === 'failures') skipped = `routine “${current!.routineName}” only notifies about failures`;
|
|
1738
|
+
else if (recent.length >= NOTIFY_PUSHES_PER_HOUR) skipped = `already sent ${NOTIFY_PUSHES_PER_HOUR} pushes in the last hour`;
|
|
1739
|
+
if (skipped) return { message, deliveries: [], skipped };
|
|
1740
|
+
|
|
1741
|
+
this.#notifyPushes.set(botId, [...recent, now]);
|
|
1742
|
+
const deliveries = await this.notifier.push({ bot, reason: 'notify', detail: t, generic: 'New notification.', priority });
|
|
1743
|
+
if (current) for (const d of deliveries) this.store.addRunDelivery(current.runId, d);
|
|
1744
|
+
return { message, deliveries, skipped: null };
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1747
|
+
async shareFile(botId: string, path: unknown, title: unknown): Promise<ShareResult> {
|
|
1748
|
+
const bot = this.store.getBot(botId);
|
|
1749
|
+
if (typeof path !== 'string' || !path.trim()) throw new HttpError(400, 'path is required');
|
|
1750
|
+
if (title !== undefined && title !== null && typeof title !== 'string') throw new HttpError(400, 'title must be a string');
|
|
1751
|
+
const t = (title ?? '').replace(/\s+/g, ' ').trim();
|
|
1752
|
+
if (t.length > 200) throw new HttpError(400, 'title is too long (max 200 characters)');
|
|
1753
|
+
const found = await resolveInWorkspace(bot.workspace, path.trim());
|
|
1754
|
+
if (found.stat.isDirectory()) throw new HttpError(400, `${path.trim()} is a folder: share a file in it`);
|
|
1755
|
+
if (!found.stat.isFile()) throw new HttpError(400, `${path.trim()} is not a regular file`);
|
|
1756
|
+
const file = fileRef(found);
|
|
1757
|
+
const chip: MessageMeta = { kind: 'file.shared', file: { ...file, ...(t ? { title: t } : {}) }, ...runMeta(this.#jobs.get(botId)?.run) };
|
|
1758
|
+
const message = this.store.addMessage(botId, 'event', `Shared ${file.path}`, this.#postMeta(this.#turns.get(botId)?.bot ?? bot, chip));
|
|
1759
|
+
return { message, file };
|
|
1760
|
+
}
|
|
1761
|
+
|
|
1762
|
+
/** A follow-up reaction's request, from the message as saved (its start) and the reaction the server recorded. */
|
|
1763
|
+
#followUpText(m: Message, owner: string): string {
|
|
1764
|
+
const target = m.meta?.reactionTo ? this.store.findMessage(m.meta.reactionTo) : null;
|
|
1765
|
+
return reactionFollowUpPrompt(m.meta?.emoji ?? '', m.meta?.reactionTo ?? '', target && target.botId === m.botId ? excerptOf(target.text) : null, owner);
|
|
1766
|
+
}
|
|
1767
|
+
|
|
1768
|
+
/** All automatic replies share the same suppression rule; owner replies use reactions instead. */
|
|
1769
|
+
#quietReply(text: string, owner: boolean, turn: TurnInfo): string | null {
|
|
1770
|
+
const note = owner ? null : parseQuiet(text);
|
|
1771
|
+
if (note !== null) {
|
|
1772
|
+
this.store.saveQuietTurnNote(turn.id, turn.steps++, note);
|
|
1773
|
+
}
|
|
1774
|
+
return note;
|
|
1775
|
+
}
|
|
1776
|
+
|
|
1777
|
+
#post(bot: BotRecord, role: 'bot' | 'error' | 'event', text: string, meta: MessageMeta | null = null): Message {
|
|
1778
|
+
return this.store.addMessage(bot.id, role, text, this.#postMeta(bot, meta));
|
|
1779
|
+
}
|
|
1780
|
+
|
|
1781
|
+
/** A message answering a conversation reset since `bot` was read is marked stale, out of the new transcript. */
|
|
1782
|
+
#postMeta(bot: BotRecord, meta: MessageMeta | null): MessageMeta | null {
|
|
1783
|
+
const stale = this.store.contextSince(bot.id) !== bot.contextSince;
|
|
1784
|
+
return stale ? { ...meta, staleContext: true } : meta;
|
|
1785
|
+
}
|
|
1786
|
+
|
|
1787
|
+
#context(
|
|
1788
|
+
bot: BotRecord,
|
|
1789
|
+
trigger: Trigger,
|
|
1790
|
+
settings: Settings,
|
|
1791
|
+
mcp: ResolvedConnectors & { offForWebhook: boolean },
|
|
1792
|
+
lastTurn: { id: string; reason: string }[],
|
|
1793
|
+
sandboxed: boolean,
|
|
1794
|
+
askFirst: boolean,
|
|
1795
|
+
chat: { reactable?: string[]; feedback?: Feedback[] } = {},
|
|
1796
|
+
): string {
|
|
1797
|
+
const team = settings.primaryBotId === bot.id ? this.store.team(bot.id) : undefined;
|
|
1798
|
+
// Chat-side context, so never for a run on its own session or a webhook run's fresh one.
|
|
1799
|
+
const results = trigger.kind === 'chat' || runSession(trigger.routine, trigger.run.trigger) === 'chat' ? this.store.ownSessionResults(bot.id) : [];
|
|
1800
|
+
const routine = trigger.kind === 'routine' ? trigger.routine : null;
|
|
1801
|
+
const model = routine?.model || bot.model;
|
|
1802
|
+
const known = this.store.effortCaps(bot.provider, model);
|
|
1803
|
+
const { effort, dropped } = turnEffort(bot, routine, known);
|
|
1804
|
+
const brain: TurnBrain = {
|
|
1805
|
+
provider: this.#providerName(bot.provider),
|
|
1806
|
+
model,
|
|
1807
|
+
effort,
|
|
1808
|
+
...(dropped ? { dropped } : {}),
|
|
1809
|
+
...(!effort && known?.defaultEffort ? { listedDefault: known.defaultEffort } : {}),
|
|
1810
|
+
...(routine?.model ? { routineModel: true } : {}),
|
|
1811
|
+
...(routine?.effort && !dropped ? { routineEffort: true } : {}),
|
|
1812
|
+
};
|
|
1813
|
+
return contextBlock(bot, settings, trigger, this.store.listRoutines(bot.id), this.store.listMemories(bot.id), {
|
|
1814
|
+
brain,
|
|
1815
|
+
team,
|
|
1816
|
+
sharedDir: sandboxed ? undefined : this.#cfg.sharedDir,
|
|
1817
|
+
connectors: mcp.servers.map((s) => s.name),
|
|
1818
|
+
connectorsOff: mcp.offForWebhook,
|
|
1819
|
+
connectorsLeftOut: mcp.skipped,
|
|
1820
|
+
connectorsLastTurn: lastTurn,
|
|
1821
|
+
connectorsRemoved: this.connectors
|
|
1822
|
+
.notices(bot)
|
|
1823
|
+
.filter((n) => n.kind === 'removed')
|
|
1824
|
+
.map((n) => n.name),
|
|
1825
|
+
requestable: sandboxed || mcp.offForWebhook ? 0 : this.connectors.forBoton(bot).filter((c) => c.state === 'available').length,
|
|
1826
|
+
// The browser comes as one of its connectors, so only when it got the connector's server this turn.
|
|
1827
|
+
browser: mcp.ids[BROWSER_SERVER] === BROWSER_CONNECTOR && this.browser ? { sites: this.browser.sites(bot.id) } : null,
|
|
1828
|
+
skills: this.skills.index(bot),
|
|
1829
|
+
results,
|
|
1830
|
+
reactable: chat.reactable,
|
|
1831
|
+
feedback: chat.feedback,
|
|
1832
|
+
approvals: {
|
|
1833
|
+
pending: this.approvals.pendingFor(bot.id),
|
|
1834
|
+
alwaysAllow: bot.alwaysAllow,
|
|
1835
|
+
askFirst: askFirst ? splitRules(bot.askFirst, bot.alwaysAllow, sandboxed).ask : [],
|
|
1836
|
+
},
|
|
1837
|
+
});
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
/**
|
|
1841
|
+
* What the CLI showed of the connectors it was given: the ones it couldn't start go to the owner like the ones the
|
|
1842
|
+
* turn went without; the ones it said nothing about are logged as unconfirmed, never as working.
|
|
1843
|
+
*/
|
|
1844
|
+
#noteStartup(botId: string, mcp: ResolvedConnectors, startup: McpStartup[] | undefined, fields: LogFields) {
|
|
1845
|
+
if (!mcp.servers.length) return;
|
|
1846
|
+
const shown = new Map((startup ?? []).map((s) => [s.name, s]));
|
|
1847
|
+
const started = mcp.servers.filter((s) => shown.get(s.name)?.started).map((s) => s.name);
|
|
1848
|
+
const notStarted = mcp.servers.flatMap((s) => {
|
|
1849
|
+
const r = shown.get(s.name);
|
|
1850
|
+
return r && !r.started ? [{ name: s.name, reason: r.reason ?? 'it did not start' }] : [];
|
|
1851
|
+
});
|
|
1852
|
+
const unconfirmed = mcp.servers.filter((s) => !shown.has(s.name)).map((s) => s.name);
|
|
1853
|
+
const tools = Object.fromEntries(started.flatMap((name) => (shown.get(name)?.tools ? [[name, shown.get(name)!.tools!.length]] : [])));
|
|
1854
|
+
(notStarted.length ? log.warn : log.info)('connectors at the end of the turn', { ...fields, started, ...(Object.keys(tools).length ? { tools } : {}), notStarted, unconfirmed });
|
|
1855
|
+
try {
|
|
1856
|
+
if (startup?.length) this.connectors.recordStartup(botId, mcp.ids, startup);
|
|
1857
|
+
} catch (err) {
|
|
1858
|
+
// The Boton was deleted mid-turn.
|
|
1859
|
+
log.debug('could not record what the CLI showed of its connectors', { ...fields, err });
|
|
1860
|
+
}
|
|
1861
|
+
}
|
|
1862
|
+
|
|
1863
|
+
#replyMeta(bot: BotRecord, started: number, turn: TurnInfo): MessageMeta {
|
|
1864
|
+
return { provider: bot.provider, model: turn.model, ...(turn.effort ? { effort: turn.effort } : {}), durationMs: Date.now() - started, ...this.#turnMeta(turn) };
|
|
1865
|
+
}
|
|
1866
|
+
|
|
1867
|
+
#turnMeta(turn: TurnInfo): MessageMeta | null {
|
|
1868
|
+
if (!turn.id) return null;
|
|
1869
|
+
const u = turn.usage;
|
|
1870
|
+
const usage = u && totalTokens(u) > 0 ? tokenUsage(u) : null;
|
|
1871
|
+
return { turnId: turn.id, ...(usage ? { usage } : {}), ...(turn.steps ? { steps: turn.steps } : {}) };
|
|
1872
|
+
}
|
|
1873
|
+
|
|
1874
|
+
#linkTurn(turn: TurnInfo, message: Message) {
|
|
1875
|
+
if (turn.id) this.store.linkTurnMessage(turn.id, message.id);
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1878
|
+
/** Leaves out of `mcp` each connector the owner has turned off since it was resolved, and returns those. */
|
|
1879
|
+
#leaveOutTurnedOff(mcp: ResolvedConnectors): { id: string; reason: string }[] {
|
|
1880
|
+
const off = Object.entries(mcp.ids).filter(([, id]) => this.connectors.isOff(id));
|
|
1881
|
+
for (const [name] of off) {
|
|
1882
|
+
mcp.servers = mcp.servers.filter((s) => s.name !== name);
|
|
1883
|
+
delete mcp.ids[name];
|
|
1884
|
+
}
|
|
1885
|
+
const skipped = off.map(([, id]) => ({ id, reason: OFF_REASON }));
|
|
1886
|
+
mcp.skipped.push(...skipped);
|
|
1887
|
+
return skipped;
|
|
1888
|
+
}
|
|
1889
|
+
|
|
1890
|
+
async #invoke(
|
|
1891
|
+
bot: BotRecord,
|
|
1892
|
+
trigger: Trigger,
|
|
1893
|
+
body: string,
|
|
1894
|
+
turn: TurnInfo,
|
|
1895
|
+
job: Job,
|
|
1896
|
+
chat: { ids?: string[]; images?: string[]; askers?: string[]; chain?: Chain | null; reactable?: string[]; feedback?: Feedback[] } = {},
|
|
1897
|
+
): Promise<TurnResult> {
|
|
1898
|
+
const r = trigger.kind === 'routine' ? trigger : null;
|
|
1899
|
+
const about = r ? { trigger: r.run.trigger, routineId: r.routine.id, runId: r.run.id } : { trigger: 'chat' as const, ...(chat.chain ? { threadId: chat.chain.threadId } : {}) };
|
|
1900
|
+
const { access, outside, sandboxed } = turnScope(bot, trigger);
|
|
1901
|
+
const session = r ? runSession(r.routine, r.run.trigger) : 'chat';
|
|
1902
|
+
const own = session === 'own' ? r!.routine : null;
|
|
1903
|
+
const fresh = session === 'fresh';
|
|
1904
|
+
const requestedModel = r?.routine.model || bot.model;
|
|
1905
|
+
turn.model = requestedModel;
|
|
1906
|
+
const requestedEffort = turnEffort(bot, r?.routine ?? null, this.store.effortCaps(bot.provider, requestedModel)).effort;
|
|
1907
|
+
turn.effort = requestedEffort;
|
|
1908
|
+
const limitMs = this.#limitFor(r?.routine ?? null);
|
|
1909
|
+
turn.limitMs = limitMs;
|
|
1910
|
+
const settings = this.store.getSettings();
|
|
1911
|
+
const provider = this.#providers.get(bot.provider);
|
|
1912
|
+
// No connectors with a payload from outside: their tools run without asking, so it could have the Boton read mail or post as the owner.
|
|
1913
|
+
// A token can't be swapped in mid-turn, so it must last the whole turn (or half its lifetime, if shorter).
|
|
1914
|
+
// A starter's test run: what it runs on, before and after waiting on its connectors (see StarterGate.noteTest).
|
|
1915
|
+
const gate = r?.run.trigger === 'test' && this.store.starterPending(r.routine.id) ? this.starterGate : undefined;
|
|
1916
|
+
const gateBefore = gate?.parts(bot, r!.routine);
|
|
1917
|
+
const mcp: ResolvedConnectors & { offForWebhook: boolean } = outside
|
|
1918
|
+
? { servers: [], skipped: [], ids: {}, offForWebhook: bot.connections.length > 0 }
|
|
1919
|
+
: { ...(await this.connectors.resolve(bot.connections, provider, { forMs: limitMs + 60_000, launch: provider ? this.#connectorLaunch(bot, provider) : undefined })), offForWebhook: false };
|
|
1920
|
+
// Its launch files last from here to the end of the turn, however it ends (a setup step that throws included).
|
|
1921
|
+
try {
|
|
1922
|
+
// Resolving waits (an OAuth token, say), and the owner may turn one off meanwhile: left out here, in the same tick as
|
|
1923
|
+
// the context and notices below, so neither lists one this turn won't get. Checked again right before each CLI starts.
|
|
1924
|
+
this.#leaveOutTurnedOff(mcp);
|
|
1925
|
+
if (gate && gateBefore) {
|
|
1926
|
+
try {
|
|
1927
|
+
gate.noteTest(r!.run, gateBefore, gate.parts(bot, r!.routine));
|
|
1928
|
+
} catch (err) {
|
|
1929
|
+
// Not noted, so it can't count for turning the routine on; the run itself goes ahead.
|
|
1930
|
+
log.warn('could not note what a test run runs on', { botId: bot.id, routineId: r!.routine.id, runId: r!.run.id, err });
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
// Connectors this turn gets that the last one went without (a CLI that couldn't start one, say): its notice says why.
|
|
1934
|
+
const given = new Set(Object.values(mcp.ids));
|
|
1935
|
+
const lastTurn = outside ? [] : this.connectors.notices(bot).flatMap((n) => (n.kind === 'skipped' && given.has(n.connectorId) ? [{ id: n.connectorId, reason: n.reason }] : []));
|
|
1936
|
+
// What it went without, and why, is the owner's to see in the Boton's settings: never only in the log.
|
|
1937
|
+
if (!outside) this.connectors.recordTurn(bot.id, mcp);
|
|
1938
|
+
const promptNow = () => {
|
|
1939
|
+
const givenNow = new Set(Object.values(mcp.ids));
|
|
1940
|
+
const recovering = lastTurn.filter((n) => givenNow.has(n.id));
|
|
1941
|
+
return `${this.#context(bot, trigger, settings, mcp, recovering, sandboxed, !!provider?.askFirst, chat)}\n\n${body}`;
|
|
1942
|
+
};
|
|
1943
|
+
let prompt = promptNow();
|
|
1944
|
+
const row = { botId: bot.id, trigger: about.trigger, routineId: r?.routine.id ?? null, runId: r?.run.id ?? null, provider: bot.provider, requestedModel };
|
|
1945
|
+
if (!provider) {
|
|
1946
|
+
log.error('turn not started: its provider is not on this server', { botId: bot.id, ...about, provider: bot.provider });
|
|
1947
|
+
turn.id = this.store.startTurn(row);
|
|
1948
|
+
this.store.finishTurn(turn.id, { outcome: 'error', durationMs: 0, replayedTranscript: false });
|
|
1949
|
+
throw new Error(`Provider "${bot.provider}" is not available on this server.`);
|
|
1950
|
+
}
|
|
1951
|
+
mkdirSync(bot.workspace, { recursive: true });
|
|
1952
|
+
|
|
1953
|
+
const sys = systemPrompt(bot, settings, this.#cfg);
|
|
1954
|
+
const key = sessionKey(provider.id, sys);
|
|
1955
|
+
const saved = fresh ? { sessionId: null, sessionKey: null } : own ? this.store.routineSession(own.id) : { sessionId: bot.sessionId, sessionKey: bot.sessionKey };
|
|
1956
|
+
const existing = saved.sessionKey === key ? saved.sessionId : null;
|
|
1957
|
+
// The first chat-session turn on the provider a switch moved it to, whose new session starts with the note.
|
|
1958
|
+
// Pending when the turn read the Boton, not now: a switch since is not this turn's.
|
|
1959
|
+
const stamp = job.read!.stamp;
|
|
1960
|
+
const pendingHandoff = own || fresh ? null : job.read!.handoff;
|
|
1961
|
+
const handoff = pendingHandoff?.meta.handoff.to.provider === provider.id ? pendingHandoff : null;
|
|
1962
|
+
turn.id = this.store.startTurn(row);
|
|
1963
|
+
const started = Date.now();
|
|
1964
|
+
// Never the prompt or the reply: only what the turn was and how it went.
|
|
1965
|
+
const fields: LogFields = {
|
|
1966
|
+
botId: bot.id,
|
|
1967
|
+
...about,
|
|
1968
|
+
turnId: turn.id,
|
|
1969
|
+
provider: provider.id,
|
|
1970
|
+
model: requestedModel || undefined,
|
|
1971
|
+
...(requestedEffort ? { effort: requestedEffort } : {}),
|
|
1972
|
+
...(own ? { routineSession: 'own' } : fresh ? { routineSession: 'fresh', webhookHistory: r!.routine.webhookHistory } : {}),
|
|
1973
|
+
...(limitMs !== this.#cfg.turnTimeoutMs ? { timeLimitMs: limitMs } : {}),
|
|
1974
|
+
};
|
|
1975
|
+
if (mcp.skipped.length) log.info('connectors left out of the turn', { ...fields, skipped: mcp.skipped });
|
|
1976
|
+
if (mcp.offForWebhook) log.info('connectors left out of the turn: a webhook started it', { ...fields, connectors: bot.connections });
|
|
1977
|
+
if (saved.sessionId && !existing) log.info('session reset: the system prompt or provider changed, so the turn starts a new session', { ...fields, sessionId: saved.sessionId });
|
|
1978
|
+
|
|
1979
|
+
const controller = new AbortController();
|
|
1980
|
+
job.turn = controller;
|
|
1981
|
+
// Stopped, paused or cancelled while its connectors were being set up: no agent process starts.
|
|
1982
|
+
if (job.aborted) controller.abort(job.aborted);
|
|
1983
|
+
const label = r ? `Running routine “${r.routine.name}”` : chat.askers?.length ? `Answering ${chat.askers.join(' and ')}` : undefined;
|
|
1984
|
+
this.#setStatus(bot.id, 'working', label ?? 'Thinking');
|
|
1985
|
+
const turnToken = this.auth.mintBotonToken(bot.id, { sandboxed });
|
|
1986
|
+
// Its managed browser, when the browser connector is among the servers it got: driven only with this turn's token.
|
|
1987
|
+
const browserTurn = !!this.browser && mcp.ids[BROWSER_SERVER] === BROWSER_CONNECTOR;
|
|
1988
|
+
if (browserTurn) this.browser!.beginTurn(bot.id, turnToken.id);
|
|
1989
|
+
const running = {
|
|
1990
|
+
bot,
|
|
1991
|
+
access,
|
|
1992
|
+
chain: chat.chain ?? null,
|
|
1993
|
+
connectors: Object.values(mcp.ids),
|
|
1994
|
+
origin: {
|
|
1995
|
+
trigger: about.trigger,
|
|
1996
|
+
turnId: turn.id,
|
|
1997
|
+
routineId: r?.routine.id ?? null,
|
|
1998
|
+
routineName: r?.routine.name ?? null,
|
|
1999
|
+
runId: r?.run.id ?? null,
|
|
2000
|
+
sandboxed,
|
|
2001
|
+
askedBy: chat.askers ?? [],
|
|
2002
|
+
},
|
|
2003
|
+
};
|
|
2004
|
+
this.#turns.set(bot.id, running);
|
|
2005
|
+
const sharedDirs = sandboxed ? [] : [this.#cfg.sharedDir];
|
|
2006
|
+
let replayed = false;
|
|
2007
|
+
// The time limit runs from the first agent process: waiting in the queue or on connectors doesn't count, and a retry
|
|
2008
|
+
// in a new session (the first found its session gone) gets what is left, not a limit of its own.
|
|
2009
|
+
let deadline: number | null = null;
|
|
2010
|
+
const drafts = this.#drafts(bot.id, turn.id, trigger.kind === 'routine' || (trigger.kind === 'chat' && !chat?.reactable?.length));
|
|
2011
|
+
const onStep = (s: TurnStep) => {
|
|
2012
|
+
try {
|
|
2013
|
+
this.store.addTurnStep(turn.id, turn.steps, s);
|
|
2014
|
+
turn.steps++;
|
|
2015
|
+
} catch (err) {
|
|
2016
|
+
// The Boton was deleted mid-turn, and its turns with it.
|
|
2017
|
+
log.debug('could not save a step', { ...fields, err });
|
|
2018
|
+
}
|
|
2019
|
+
};
|
|
2020
|
+
|
|
2021
|
+
const attempt = (sessionId: string | null) => {
|
|
2022
|
+
// A retry in a new session after the turn's time ran out: it timed out, and no new agent process starts.
|
|
2023
|
+
if (deadline !== null && Date.now() >= deadline) {
|
|
2024
|
+
const err = new ProviderError(`${provider.label} timed out.`, false);
|
|
2025
|
+
err.exit = { exitCode: null, signal: null, timedOut: true, stderrTail: '' };
|
|
2026
|
+
return Promise.reject(err);
|
|
2027
|
+
}
|
|
2028
|
+
if (controller.signal.aborted) return Promise.reject(controller.signal.reason);
|
|
2029
|
+
// Turned off since the prompt was built (a sandboxed turn waits on its instruction files first, a retry on the
|
|
2030
|
+
// first attempt): left out after all, and the prompt and notices say so. From here to the CLI starting nothing
|
|
2031
|
+
// waits, and from then on the turn is listed as having the rest: turning one off says it keeps it until it ends.
|
|
2032
|
+
const turnedOff = this.#leaveOutTurnedOff(mcp);
|
|
2033
|
+
if (turnedOff.length) {
|
|
2034
|
+
this.connectors.recordTurn(bot.id, { skipped: turnedOff, ids: {} });
|
|
2035
|
+
running.connectors = Object.values(mcp.ids);
|
|
2036
|
+
prompt = promptNow();
|
|
2037
|
+
}
|
|
2038
|
+
let input = prompt;
|
|
2039
|
+
replayed = false;
|
|
2040
|
+
// A webhook run's fresh session: the chat's latest messages the owner gave its routine, nothing more (no pins, no
|
|
2041
|
+
// routine results, no handoff), inside a budget of their own.
|
|
2042
|
+
if (fresh && r!.routine.webhookHistory > 0) {
|
|
2043
|
+
const src = this.store.replaySource(bot.id, bot.contextSince, []);
|
|
2044
|
+
const start = sessionStart({ chat: take(src.chat, Math.min(r!.routine.webhookHistory, WEBHOOK_HISTORY_MAX)), routines: [], pins: [], budget: WEBHOOK_HISTORY_BUDGET }, settings, bot.timezone);
|
|
2045
|
+
input = [start.text, prompt].filter(Boolean).join('\n\n');
|
|
2046
|
+
replayed = start.plan.messages > 0;
|
|
2047
|
+
log.info('webhook run: the chat it was given was sized to its budget', { ...fields, replay: start.plan });
|
|
2048
|
+
}
|
|
2049
|
+
// A routine's own session starts with its context and instruction only, not the chat.
|
|
2050
|
+
else if (!sessionId && !own && !fresh) {
|
|
2051
|
+
// The summary the old session wrote for the move goes in the note, so not in the replay as well.
|
|
2052
|
+
const summary = handoff?.meta.handoff.summary ? this.store.handoffSummary(bot.id, handoff.meta.handoff.summary.messageId) : null;
|
|
2053
|
+
const src = this.store.replaySource(bot.id, bot.contextSince, [...(chat.ids ?? []), ...(summary ? [summary.id] : [])]);
|
|
2054
|
+
const start = sessionStart({ handoff: handoff?.meta.handoff ?? null, summary, ...src }, settings, bot.timezone);
|
|
2055
|
+
input = [start.text, prompt].filter(Boolean).join('\n\n');
|
|
2056
|
+
replayed = start.plan.messages + start.plan.routines + start.plan.pinned > 0;
|
|
2057
|
+
// Sizes only, never the text: what the new session was given.
|
|
2058
|
+
log.info('new session: the replay was sized to its budget', { ...fields, replay: start.plan });
|
|
2059
|
+
}
|
|
2060
|
+
job.spawned = true;
|
|
2061
|
+
deadline ??= Date.now() + limitMs;
|
|
2062
|
+
const api = withLegacyTurnNames({ BOTONS_API: `http://127.0.0.1:${this.#cfg.port}`, BOTON_BOT_ID: bot.id, BOTON_TOKEN: turnToken.token });
|
|
2063
|
+
return provider.runTurn({
|
|
2064
|
+
cwd: bot.workspace,
|
|
2065
|
+
sessionId,
|
|
2066
|
+
systemPrompt: sys,
|
|
2067
|
+
prompt: input,
|
|
2068
|
+
model: requestedModel,
|
|
2069
|
+
...(requestedEffort ? { effort: requestedEffort } : {}),
|
|
2070
|
+
access,
|
|
2071
|
+
images: chat.images ?? [],
|
|
2072
|
+
privatePaths:
|
|
2073
|
+
access === 'workspace'
|
|
2074
|
+
? privatePaths({
|
|
2075
|
+
env: { ...process.env, ...cliHomesEnv() },
|
|
2076
|
+
workspace: bot.workspace,
|
|
2077
|
+
cli: provider.cli ?? null,
|
|
2078
|
+
botonsHome: this.#cfg.home,
|
|
2079
|
+
workspacesDir: this.#cfg.workspacesDir,
|
|
2080
|
+
extra: this.#cfg.privatePaths,
|
|
2081
|
+
shared: sharedDirs,
|
|
2082
|
+
})
|
|
2083
|
+
: [],
|
|
2084
|
+
sharedDirs,
|
|
2085
|
+
protectInstructions: sandboxed,
|
|
2086
|
+
groupDir: this.#cfg.agentsDir,
|
|
2087
|
+
env: withLegacyTurnNames({
|
|
2088
|
+
BOTONS_API: `http://127.0.0.1:${this.#cfg.port}`,
|
|
2089
|
+
BOTON_BOT_ID: bot.id,
|
|
2090
|
+
// The `botons` CLI runs on the server's own Node, which may not be on PATH (under systemd, say).
|
|
2091
|
+
BOTONS_NODE: process.execPath,
|
|
2092
|
+
BOTON_TOKEN: turnToken.token,
|
|
2093
|
+
// The only folder `botons skill --file` reads from (the CLI runs outside a Workspace Boton's sandbox).
|
|
2094
|
+
BOTON_WORKSPACE: bot.workspace,
|
|
2095
|
+
// server/bin/legacy: the command's deprecated names, for one release (shared/legacy-names.mjs).
|
|
2096
|
+
PATH: `${this.#cfg.botonBinDir}:${join(this.#cfg.botonBinDir, 'legacy')}:${process.env.PATH ?? ''}`,
|
|
2097
|
+
}),
|
|
2098
|
+
// The browser's server reaches the Botons API as this turn (Codex reads the token from its environment, never argv).
|
|
2099
|
+
mcpServers: mcp.servers.map((x) => (browserTurn && x.name === BROWSER_SERVER && x.kind === 'stdio' ? { ...x, env: { ...x.env, ...api } } : x)),
|
|
2100
|
+
privateDir: this.connectors.privateDir,
|
|
2101
|
+
// An API key brain's spending caps count this turn (its own row) and, for a routine, its run.
|
|
2102
|
+
spend: { botId: bot.id, turnId: turn.id, runId: r?.run.id ?? null, routineId: r?.routine.id ?? null },
|
|
2103
|
+
botonCli: join(this.#cfg.botonBinDir, 'botons'),
|
|
2104
|
+
// A sandboxed turn asks about the always-allowed rules too.
|
|
2105
|
+
...(provider.askFirst && bot.askFirst.length
|
|
2106
|
+
? { askFirst: { ...splitRules(bot.askFirst, bot.alwaysAllow, sandboxed), command: join(this.#cfg.botonBinDir, 'botons'), args: ['mcp'] } }
|
|
2107
|
+
: {}),
|
|
2108
|
+
signal: controller.signal,
|
|
2109
|
+
timeoutMs: Math.max(1, deadline - Date.now()),
|
|
2110
|
+
onActivity: (a) => this.#setStatus(bot.id, 'working', label ? `${label}: ${a}` : a),
|
|
2111
|
+
onText: drafts.push,
|
|
2112
|
+
onStep,
|
|
2113
|
+
onToolResult: (obs) => {
|
|
2114
|
+
const connectorId = mcp.ids[obs.server];
|
|
2115
|
+
if (connectorId) this.workFeed?.observe(bot.id, connectorId, obs);
|
|
2116
|
+
},
|
|
2117
|
+
});
|
|
2118
|
+
};
|
|
2119
|
+
|
|
2120
|
+
let sessionLost: true | undefined;
|
|
2121
|
+
// What the attempt that found its session gone used: the turn's row counts it with the retry's.
|
|
2122
|
+
let lostUsage: TurnUsage | null = null;
|
|
2123
|
+
const finish = (outcome: Exclude<TurnOutcome, 'running'>, used: TurnUsage | null, errorKind: string | null = null) => {
|
|
2124
|
+
const durationMs = Date.now() - started;
|
|
2125
|
+
const usage = addUsage(used, lostUsage);
|
|
2126
|
+
turn.usage = usage;
|
|
2127
|
+
turn.model = usage?.model ?? requestedModel;
|
|
2128
|
+
this.store.finishTurn(turn.id, { outcome, errorKind, durationMs, usage, model: usage?.model, replayedTranscript: replayed });
|
|
2129
|
+
return { durationMs, usage };
|
|
2130
|
+
};
|
|
2131
|
+
// Given to the CLI, not yet known to have started: #noteStartup logs what the CLI showed of them.
|
|
2132
|
+
log.info('turn started', { ...fields, session: existing ? 'resume' : 'new', ...(mcp.servers.length ? { connectorsGiven: mcp.servers.map((s) => s.name) } : {}) });
|
|
2133
|
+
let result: TurnResult;
|
|
2134
|
+
// A sandboxed turn must leave the instruction files later turns load as they were (see instructions.ts).
|
|
2135
|
+
const putBack = sandboxed
|
|
2136
|
+
? await snapshotInstructions(bot.workspace).catch((err) => {
|
|
2137
|
+
log.warn('the instruction files could not be noted before a sandboxed turn, so they will not be put back after it (the CLI sandbox still keeps it from writing them)', { ...fields, err });
|
|
2138
|
+
return null;
|
|
2139
|
+
})
|
|
2140
|
+
: null;
|
|
2141
|
+
try {
|
|
2142
|
+
try {
|
|
2143
|
+
result = await attempt(existing);
|
|
2144
|
+
} catch (err) {
|
|
2145
|
+
// Only a session the provider says is gone, before the turn did anything: the retry runs the whole prompt again.
|
|
2146
|
+
if (!(existing && err instanceof ProviderError && err.sessionLost) || controller.signal.aborted) throw err;
|
|
2147
|
+
sessionLost = true;
|
|
2148
|
+
lostUsage = err.usage ?? null;
|
|
2149
|
+
log.warn('session lost: starting a new one', { ...fields, sessionId: existing, sessionLost });
|
|
2150
|
+
result = await attempt(null);
|
|
2151
|
+
} finally {
|
|
2152
|
+
turnToken.revoke();
|
|
2153
|
+
this.#turns.delete(bot.id);
|
|
2154
|
+
if (browserTurn) await this.browser!.endTurn(bot.id, turnToken.id).catch((err) => log.warn('the browser did not close cleanly', { ...fields, err }));
|
|
2155
|
+
drafts.end();
|
|
2156
|
+
if (putBack) await this.#putBackInstructions(bot, putBack, fields);
|
|
2157
|
+
}
|
|
2158
|
+
} catch (err) {
|
|
2159
|
+
if (err instanceof ProviderError) this.#noteStartup(bot.id, mcp, err.mcpStartup, fields);
|
|
2160
|
+
const signal = controller.signal;
|
|
2161
|
+
const outcome = timedOut(err) ? 'timeout' : signal.aborted ? (signal.reason === INTERRUPTED ? 'interrupted' : 'stopped') : 'error';
|
|
2162
|
+
const { durationMs, usage } = finish(outcome, err instanceof ProviderError ? (err.usage ?? null) : null, outcome === 'error' && err instanceof ProviderError ? err.kind : null);
|
|
2163
|
+
const exit = err instanceof ProviderError ? err.exit : undefined;
|
|
2164
|
+
const end: LogFields = { ...fields, ...usageFields(usage), outcome, durationMs, sessionLost, error: errorText(err) };
|
|
2165
|
+
if (exit) Object.assign(end, { exitCode: exit.exitCode, signal: exit.signal ?? undefined, timedOut: exit.timedOut, stderr: exit.stderrTail || undefined });
|
|
2166
|
+
if (outcome === 'stopped' || outcome === 'interrupted') log.info('turn ended', end);
|
|
2167
|
+
else {
|
|
2168
|
+
this.#providers.recordTurn(provider.id, { ok: false, error: errorText(err) });
|
|
2169
|
+
log.error('turn ended', end);
|
|
2170
|
+
}
|
|
2171
|
+
throw err;
|
|
2172
|
+
}
|
|
2173
|
+
this.#noteStartup(bot.id, mcp, result.mcpStartup, fields);
|
|
2174
|
+
const { durationMs, usage } = finish('ok', result.usage ?? null);
|
|
2175
|
+
this.#providers.recordTurn(provider.id, { ok: true });
|
|
2176
|
+
log.info('turn ended', { ...fields, outcome: 'ok', durationMs, sessionLost, ...usageFields(usage) });
|
|
2177
|
+
if (result.warnings?.length) {
|
|
2178
|
+
log.warn(`turn completed after ${result.warnings.length} ${provider.id} error(s)`, { ...fields, lastWarning: result.warnings.at(-1) });
|
|
2179
|
+
}
|
|
2180
|
+
// Started fresh during the turn (or deleted): keep the reset, drop this session.
|
|
2181
|
+
// Moved to another provider during the turn (or reset): the turn's reply stands, but its session is not kept, so
|
|
2182
|
+
// the next turn starts on the Boton's provider as the move said.
|
|
2183
|
+
// Moved away and back counts too (the generation): the move ended this session, whatever the provider is now.
|
|
2184
|
+
const at: TurnStamp = { ...stamp, provider: provider.id };
|
|
2185
|
+
// A fresh session ends with its run: nothing it was told carries on.
|
|
2186
|
+
const kept = fresh || this.store.atomically(() => {
|
|
2187
|
+
const saved = own ? this.store.setRoutineSession(own.id, result.sessionId, key, at) : this.store.setSession(bot.id, result.sessionId, key, at);
|
|
2188
|
+
// The new provider's chat session has its turn now (with the note, or carrying on one that was its own before).
|
|
2189
|
+
if (saved && handoff) this.store.endHandoff(handoff.id, at.generation);
|
|
2190
|
+
return saved;
|
|
2191
|
+
});
|
|
2192
|
+
if (!kept && this.store.hasBot(bot.id)) {
|
|
2193
|
+
const moved = this.#providerOf(bot.id) !== provider.id || this.store.switchGeneration(bot.id) !== at.generation;
|
|
2194
|
+
log.info(
|
|
2195
|
+
moved
|
|
2196
|
+
? 'moved to another provider during the turn; not keeping its session'
|
|
2197
|
+
: own
|
|
2198
|
+
? "the routine's session was reset or changed during the turn; not keeping it"
|
|
2199
|
+
: 'conversation was reset during the turn; not keeping its session',
|
|
2200
|
+
{ ...fields, sessionId: result.sessionId },
|
|
2201
|
+
);
|
|
2202
|
+
}
|
|
2203
|
+
return result;
|
|
2204
|
+
} finally {
|
|
2205
|
+
this.connectors.release(mcp);
|
|
2206
|
+
}
|
|
2207
|
+
}
|
|
2208
|
+
|
|
2209
|
+
async #putBackInstructions(bot: BotRecord, putBack: () => Promise<string[]>, fields: LogFields) {
|
|
2210
|
+
let undone: string[];
|
|
2211
|
+
try {
|
|
2212
|
+
undone = await putBack();
|
|
2213
|
+
} catch (err) {
|
|
2214
|
+
log.warn('could not check the instruction files after a sandboxed turn', { ...fields, err });
|
|
2215
|
+
return;
|
|
2216
|
+
}
|
|
2217
|
+
if (!undone.length) return;
|
|
2218
|
+
log.warn('a sandboxed turn changed instruction files; put them back', { ...fields, files: undone });
|
|
2219
|
+
if (!this.store.hasBot(bot.id)) return;
|
|
2220
|
+
const list = undone.join(', ');
|
|
2221
|
+
this.store.addMessage(bot.id, 'event', `Undid this webhook run's changes to ${list}: later turns would have followed ${undone.length > 1 ? 'them' : 'it'} as instructions`, {
|
|
2222
|
+
kind: 'instructions.restored',
|
|
2223
|
+
});
|
|
2224
|
+
}
|
|
2225
|
+
}
|
|
2226
|
+
|
|
2227
|
+
/** The follow-up turn for a connector granted on request, from the server. */
|
|
2228
|
+
function grantedPrompt(connectorId: string): string {
|
|
2229
|
+
const id = connectorId.replace(/[^a-z0-9:_-]/g, '');
|
|
2230
|
+
return `(From the Botons server: the connector ${id} you asked for was added to your connectors, so its tools are yours from this turn on. Carry on with what you needed it for, then say what you did.)`;
|
|
2231
|
+
}
|
|
2232
|
+
|
|
2233
|
+
/** Runs started by hand (Run now, or a test): never retried, and run whether or not the routine is paused. */
|
|
2234
|
+
const byHand = (run: Pick<RoutineRun, 'trigger'>) => run.trigger === 'manual' || run.trigger === 'test';
|
|
2235
|
+
|
|
2236
|
+
/** A one-off that has fired is disabled too, but retired, not paused: its run still gets its retries. */
|
|
2237
|
+
function routinePaused(routine: Routine, run: RoutineRun): boolean {
|
|
2238
|
+
return !routine.enabled && !byHand(run) && !isPastOneOff(routine.schedule);
|
|
2239
|
+
}
|
|
2240
|
+
|
|
2241
|
+
function isFinished(run: RoutineRun): boolean {
|
|
2242
|
+
return run.status === 'succeeded' || run.status === 'failed' || run.status === 'skipped';
|
|
2243
|
+
}
|
|
2244
|
+
|
|
2245
|
+
function goneRun(last: RoutineRun, reason: string): RoutineRun {
|
|
2246
|
+
if (isFinished(last)) return last;
|
|
2247
|
+
return { ...last, status: 'failed', error: reason, finishedAt: nowIso() };
|
|
2248
|
+
}
|
|
2249
|
+
|
|
2250
|
+
function when(at: Date, tz: string): string {
|
|
2251
|
+
return at.toLocaleString('en-US', { timeZone: tz, weekday: 'short', hour: 'numeric', minute: '2-digit' });
|
|
2252
|
+
}
|
|
2253
|
+
|
|
2254
|
+
function retryAfter(ms: number): { s: number; wait: string } {
|
|
2255
|
+
const s = Math.max(1, Math.ceil(ms / 1000));
|
|
2256
|
+
return { s, wait: s < 120 ? `${s} seconds` : `${Math.ceil(s / 60)} minutes` };
|
|
2257
|
+
}
|
|
2258
|
+
|
|
2259
|
+
function runMeta(run: Job['run']) {
|
|
2260
|
+
return run ? { routineId: run.routineId, routineName: run.routineName, runId: run.runId } : {};
|
|
2261
|
+
}
|
|
2262
|
+
|
|
2263
|
+
function isAttachableImage(a: FileRef): boolean {
|
|
2264
|
+
// Codex splits --image values on commas.
|
|
2265
|
+
return /^image\/(png|jpeg|gif|webp)$/.test(a.type) && !a.path.includes(',');
|
|
2266
|
+
}
|
|
2267
|
+
|
|
2268
|
+
/** Checked again as the turn starts: the Boton may have replaced a file with a symlink leading outside (Codex's --image would read it). */
|
|
2269
|
+
async function attachmentsNow(workspace: string, messages: Message[]): Promise<Map<string, string | null>> {
|
|
2270
|
+
const found = new Map<string, string | null>();
|
|
2271
|
+
for (const a of messages.flatMap((m) => m.meta?.attachments ?? [])) {
|
|
2272
|
+
if (found.has(a.path)) continue;
|
|
2273
|
+
const at = await resolveInWorkspace(workspace, a.path).then(
|
|
2274
|
+
(r) => (r.stat.isFile() ? join(workspace, r.rel) : null),
|
|
2275
|
+
() => null,
|
|
2276
|
+
);
|
|
2277
|
+
found.set(a.path, at);
|
|
2278
|
+
}
|
|
2279
|
+
return found;
|
|
2280
|
+
}
|
|
2281
|
+
|
|
2282
|
+
function withAttachments(m: Message, workspace: string, found: Map<string, string | null>, owner: string): string {
|
|
2283
|
+
const files = m.meta?.attachments ?? [];
|
|
2284
|
+
if (!files.length) return m.text;
|
|
2285
|
+
const note = attachmentNote(
|
|
2286
|
+
files.map((f) => {
|
|
2287
|
+
const at = found.get(f.path);
|
|
2288
|
+
return { ...f, path: at ?? join(workspace, f.path), missing: !at };
|
|
2289
|
+
}),
|
|
2290
|
+
owner,
|
|
2291
|
+
);
|
|
2292
|
+
return m.text ? `${m.text}\n\n${note}` : note;
|
|
2293
|
+
}
|
|
2294
|
+
|
|
2295
|
+
/** The cost is known only if both attempts' are. */
|
|
2296
|
+
function addUsage(u: TurnUsage | null, earlier: TurnUsage | null): TurnUsage | null {
|
|
2297
|
+
if (!earlier || totalTokens(earlier) === 0) return u ?? earlier;
|
|
2298
|
+
if (!u) return earlier;
|
|
2299
|
+
const sum: TurnUsage = {
|
|
2300
|
+
...u,
|
|
2301
|
+
inputTokens: u.inputTokens + earlier.inputTokens,
|
|
2302
|
+
outputTokens: u.outputTokens + earlier.outputTokens,
|
|
2303
|
+
cacheReadTokens: u.cacheReadTokens + earlier.cacheReadTokens,
|
|
2304
|
+
cacheWriteTokens: u.cacheWriteTokens + earlier.cacheWriteTokens,
|
|
2305
|
+
};
|
|
2306
|
+
if (u.costUsd !== undefined && earlier.costUsd !== undefined) sum.costUsd = u.costUsd + earlier.costUsd;
|
|
2307
|
+
else delete sum.costUsd;
|
|
2308
|
+
if (u.costHeldUsd || earlier.costHeldUsd) sum.costHeldUsd = (u.costHeldUsd ?? 0) + (earlier.costHeldUsd ?? 0);
|
|
2309
|
+
if (earlier.numTurns !== undefined) sum.numTurns = (u.numTurns ?? 0) + earlier.numTurns;
|
|
2310
|
+
return sum;
|
|
2311
|
+
}
|
|
2312
|
+
|
|
2313
|
+
function tokenUsage(u: TurnUsage): TokenUsage {
|
|
2314
|
+
const out: TokenUsage = { inputTokens: u.inputTokens, outputTokens: u.outputTokens, cacheReadTokens: u.cacheReadTokens, cacheWriteTokens: u.cacheWriteTokens };
|
|
2315
|
+
if (u.costUsd !== undefined) out.costUsd = u.costUsd;
|
|
2316
|
+
if (u.costBasis) out.costBasis = u.costBasis;
|
|
2317
|
+
if (u.costHeldUsd) out.costHeldUsd = u.costHeldUsd;
|
|
2318
|
+
return out;
|
|
2319
|
+
}
|
|
2320
|
+
|
|
2321
|
+
function usageFields(u: TurnUsage | null): LogFields {
|
|
2322
|
+
return u ? { modelRan: u.model, inputTokens: u.inputTokens, outputTokens: u.outputTokens, cacheReadTokens: u.cacheReadTokens, cacheWriteTokens: u.cacheWriteTokens } : {};
|
|
2323
|
+
}
|
|
2324
|
+
|
|
2325
|
+
function errorText(err: unknown): string {
|
|
2326
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2327
|
+
return msg.length > 2000 ? `${msg.slice(0, 2000)}…` : msg;
|
|
2328
|
+
}
|