@sema-agent/server 1.241.0 → 1.243.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/budget.d.ts +19 -0
- package/dist/budget.d.ts.map +1 -1
- package/dist/budget.js +32 -0
- package/dist/budget.js.map +1 -1
- package/dist/capabilities/center-prompts.d.ts +1 -0
- package/dist/capabilities/center-prompts.d.ts.map +1 -1
- package/dist/capabilities/center-prompts.js +4 -0
- package/dist/capabilities/center-prompts.js.map +1 -1
- package/dist/config-lkg.d.ts.map +1 -1
- package/dist/config-lkg.js +1 -6
- package/dist/config-lkg.js.map +1 -1
- package/dist/env-facts.d.ts +9 -0
- package/dist/env-facts.d.ts.map +1 -1
- package/dist/env-facts.js +22 -0
- package/dist/env-facts.js.map +1 -1
- package/dist/http/server.d.ts +23 -1
- package/dist/http/server.d.ts.map +1 -1
- package/dist/http/server.js +114 -2
- package/dist/http/server.js.map +1 -1
- package/dist/main.js +195 -17
- package/dist/main.js.map +1 -1
- package/dist/plugins/remote-env-e2b.d.ts.map +1 -1
- package/dist/plugins/remote-env-e2b.js +3 -1
- package/dist/plugins/remote-env-e2b.js.map +1 -1
- package/dist/plugins/remote-env-k8s.d.ts.map +1 -1
- package/dist/plugins/remote-env-k8s.js.map +1 -1
- package/dist/sema-registry.d.ts +4 -1
- package/dist/sema-registry.d.ts.map +1 -1
- package/dist/sema-registry.js +48 -21
- package/dist/sema-registry.js.map +1 -1
- package/dist/task-settings.d.ts +4 -1
- package/dist/task-settings.d.ts.map +1 -1
- package/dist/task-settings.js +45 -1
- package/dist/task-settings.js.map +1 -1
- package/package.json +2 -2
package/dist/main.js
CHANGED
|
@@ -4,12 +4,13 @@ import { createHash } from "node:crypto";
|
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
5
|
import { join } from "node:path";
|
|
6
6
|
import { loadRemoteExec } from "@sema-agent/registry-core/node";
|
|
7
|
+
import { skillContentHash } from "@sema-agent/registry-core";
|
|
7
8
|
import { Runner, TtlSessionStore, uuidv7, FileRosterStore, defaultTaskRegistry, combinePolicies, createDurableQuestionPolicy, createAllowDenyPolicy, tightenTaskSpec, isThinkingLevel, QUESTION_AWAITS_RESUME, FileWorkflowRunStore, InMemoryWorkflowRunStore, workflowsCapability, NodeLspManager, createFileWorkflowScriptStore, expandTiers, NodeExecutionEnv, CenterPromptSource, FilePromptArtifactStore, FilePromptSourceStateStore, MemoryPromptArtifactStore, MemoryPromptSourceStateStore, verifyPromptArtifact } from "@sema-agent/core";
|
|
8
9
|
import { PgMemoryEngineBackend, ensurePgMemoryEngineSchema } from "./plugins/memory-engine-pg.js";
|
|
9
10
|
import { TiDBMemoryEngineBackend, ensureTiDBMemoryEngineSchema } from "./plugins/memory-engine-tidb.js";
|
|
10
11
|
import { PgMemoryHistoryStore, ensurePgMemoryHistorySchema, PgMemorySyncStore, ensurePgMemorySyncSchema } from "./plugins/memory-sync-store-pg.js";
|
|
11
12
|
import { TiDBMemoryHistoryStore, ensureTiDBMemoryHistorySchema, TiDBMemorySyncStore, ensureTiDBMemorySyncSchema } from "./plugins/memory-sync-store-tidb.js";
|
|
12
|
-
import { resolveTaskModel, modelSupportsImages } from "./model-select.js";
|
|
13
|
+
import { resolveTaskModel, modelSupportsImages, matchCatalogModel } from "./model-select.js";
|
|
13
14
|
import { derivePkgSourceEnv, customPkgSourceFromEnv } from "./sandbox-pkg-source.js";
|
|
14
15
|
import { createSessionTitler } from "./session-titler.js";
|
|
15
16
|
import { SessionWatchRegistry, posIntEnv } from "./session-watch.js";
|
|
@@ -20,7 +21,7 @@ import { createSendUserFileIssuer } from "./plugins/send-user-file.js";
|
|
|
20
21
|
import { withLedgerRecording } from "./plugins/send-file-ledger.js";
|
|
21
22
|
import { basename, resolve } from "node:path";
|
|
22
23
|
import { stat as fsStat, readFile as fsReadFile } from "node:fs/promises";
|
|
23
|
-
import { buildEnvFacts, egressForRemoteExec, ensureScratchpadDir, purgeScratchpadDir, sweepStaleScratchpads } from "./env-facts.js";
|
|
24
|
+
import { buildEnvFacts, egressForRemoteExec, ensureScratchpadDir, purgeScratchpadDir, sweepStaleScratchpads, resumeFactsForLane } from "./env-facts.js";
|
|
24
25
|
import { normalizeSuggestNextPrompts, normalizeResilience, normalizeAttachments, normalizeResumeAtMode, resolveTaskLimits, taskAgentsSpecFragment, retainBackgroundProcessesFromBody, toolNameListFromBody, promptProfileFromBody } from "./spec-fields.js";
|
|
25
26
|
import { createBrain, brainSummary } from "./brain.js";
|
|
26
27
|
import { loadConfig, logConfigDiagnostics } from "./config.js";
|
|
@@ -64,22 +65,22 @@ import { createLeaderEndpoint } from "./leader/endpoint.js";
|
|
|
64
65
|
import { buildLeaderRunner } from "./leader/wire.js";
|
|
65
66
|
import { WorkflowNotifyGate, JournalingWorkflowRunStore, FileWorkflowNotifyJournalStore, } from "./orchestration/workflow-notify-journal.js";
|
|
66
67
|
import { FileWorkflowCompletionInbox, InMemoryWorkflowCompletionInbox, resolveCompletionRoute, resolveServedSession, } from "./orchestration/workflow-completion-inbox.js";
|
|
67
|
-
import { buildPricing, buildTracer, cappedCeiling, ModelUsageTracker, PromptManifestTracker } from "./budget.js";
|
|
68
|
+
import { buildPricing, buildTracer, buildSideQueryAccountant, cappedCeiling, ModelUsageTracker, PromptManifestTracker } from "./budget.js";
|
|
68
69
|
import { CostQuota } from "./observability/cost-quota.js";
|
|
69
70
|
import { startOtlpExporter } from "./observability/otel-exporter.js";
|
|
70
71
|
import { HEARTBEAT_MS } from "./runs.js";
|
|
71
72
|
import { ElicitationCoordinator } from "./elicitation.js";
|
|
72
73
|
import { QuestionCoordinator } from "./question.js";
|
|
73
74
|
import { ToolApprovalCoordinator } from "./tool-approval.js";
|
|
74
|
-
import { applyEffective, mutateInPlace, logEffectiveDiff, applyCenterSkills, resolveMcpServers, mcpForScenario, restartReasons, fetchPromptArtifact } from "./sema-registry.js";
|
|
75
|
+
import { applyEffective, mutateInPlace, logEffectiveDiff, applyCenterSkills, resolveMcpServers, mcpForScenario, restartReasons, modelPlaneChanged, planeHasActiveTiers, fetchPromptArtifact } from "./sema-registry.js";
|
|
75
76
|
import { ensureSealedKeyStore, reportExecutionPublicKey } from "./sealed-key.js";
|
|
76
77
|
import { createConfigProvider, raceBootFetch, BOOT_FETCH_DEFERRED } from "./config-provider.js";
|
|
77
|
-
import { validatePromptsDomain, centerPromptProvider, applyCatalogToSource, CORE_ENGINE_VERSION, withPromptArtifactBackfill } from "./capabilities/center-prompts.js";
|
|
78
|
+
import { validatePromptsDomain, centerPromptProvider, centerIdentityAssembled, applyCatalogToSource, CORE_ENGINE_VERSION, withPromptArtifactBackfill } from "./capabilities/center-prompts.js";
|
|
78
79
|
import { defaultLkgPath, defaultSkillCacheDir, saveLkg, loadLkg } from "./config-lkg.js";
|
|
79
80
|
import { TiDBRosterStore, PgRosterStore, ensureTiDBRosterSchema, ensurePgRosterSchema } from "./plugins/roster-store-sql.js";
|
|
80
81
|
import { createPrincipalCapsClient, gateExecutionLane, scopedTokenNeedsWorker, applyObserverEnvOptIn } from "./runtime-caps-resolver.js";
|
|
81
82
|
import { applyRuntimeGovernance, stripDelegationTools } from "./runtime-governance.js";
|
|
82
|
-
import { parseTaskSettings, applyTaskSettings, coercePermissionMode, withPermissionMode, effectiveThinking } from "./task-settings.js";
|
|
83
|
+
import { parseTaskSettings, applyTaskSettings, coercePermissionMode, withPermissionMode, effectiveThinking, acceptAppendSystemPrompt, hasConstitutionAnchors, providerDropsAppend, MAX_SETTINGS_OUTPUT_STYLE_CHARS } from "./task-settings.js";
|
|
83
84
|
import { buildTaskHooks, composeHooks } from "./hooks/hook-runner.js";
|
|
84
85
|
import { createHookLlm } from "./hooks/hook-llm.js";
|
|
85
86
|
import { selfOrchestrationFromBody, enableForkFromBody, workflowModelAllowlistFor, normalizeRetainSubagentSessions } from "./task-workflow.js";
|
|
@@ -130,6 +131,9 @@ async function main() {
|
|
|
130
131
|
let effective;
|
|
131
132
|
let latestEffective;
|
|
132
133
|
let lastRejectedCandidate;
|
|
134
|
+
let appliedPlaneEff;
|
|
135
|
+
let planeDeferredNoHandoff;
|
|
136
|
+
let bootSkillStale = false;
|
|
133
137
|
let pendingRestart;
|
|
134
138
|
const cc = config.configCenter;
|
|
135
139
|
const localRoot = config.configLocalDir ?? process.env.AGENT_DATA_DIR ?? join(homedir(), ".ai-agent");
|
|
@@ -239,19 +243,45 @@ async function main() {
|
|
|
239
243
|
let bootConfigPending;
|
|
240
244
|
const lkgEnabled = configProvider?.kind === "remote" && process.env.CONFIG_LKG_DISABLED !== "true";
|
|
241
245
|
const lkgPath = process.env.CONFIG_LKG_PATH ?? defaultLkgPath(cc?.worker);
|
|
246
|
+
const lkgSurvivesRestart = lkgEnabled && process.env.CONFIG_LKG_DURABLE === "true";
|
|
242
247
|
let lkgBooted = false;
|
|
243
248
|
let bootLkgCandidate;
|
|
249
|
+
let lastPrewarmBodiesVerified = false;
|
|
244
250
|
const persistLkgDurable = async (effectiveObj, etag) => {
|
|
251
|
+
lastPrewarmBodiesVerified = false;
|
|
252
|
+
if (configProvider?.kind === "local")
|
|
253
|
+
return true;
|
|
245
254
|
if (!lkgEnabled)
|
|
246
|
-
return;
|
|
255
|
+
return false;
|
|
247
256
|
try {
|
|
248
257
|
if (effectiveObj.skills && config.configCenter) {
|
|
249
258
|
await applyCenterSkills([], effectiveObj.skills, config.configCenter.baseUrl, config.configCenter.token, logger, undefined, defaultSkillCacheDir());
|
|
250
259
|
}
|
|
260
|
+
const enabledSkills = (effectiveObj.skills?.skills ?? []).filter((m) => m.enabled !== false);
|
|
261
|
+
const cacheDir = defaultSkillCacheDir();
|
|
262
|
+
for (const m of enabledSkills) {
|
|
263
|
+
const hex = m.contentHash.replace(/^sha256:/, "").toLowerCase();
|
|
264
|
+
if (!/^[0-9a-f]{64}$/.test(hex))
|
|
265
|
+
return false;
|
|
266
|
+
try {
|
|
267
|
+
const text = await fsReadFile(join(cacheDir, hex), "utf8");
|
|
268
|
+
if (`sha256:${createHash("sha256").update(text, "utf8").digest("hex")}` !== `sha256:${hex}`) {
|
|
269
|
+
logger.warn("config_lkg_skill_cache_incomplete", { skill: m.name, hash: m.contentHash, note: "cached body fails its hash — LKG NOT replaced, handoff NOT confirmed (the previous good LKG keeps serving offline boots)" });
|
|
270
|
+
return false;
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
catch {
|
|
274
|
+
logger.warn("config_lkg_skill_cache_incomplete", { skill: m.name, hash: m.contentHash, note: "enabled skill body not in the cache — LKG NOT replaced, handoff NOT confirmed" });
|
|
275
|
+
return false;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
lastPrewarmBodiesVerified = true;
|
|
251
279
|
await saveLkg(lkgPath, cc?.worker, effectiveObj, etag);
|
|
280
|
+
return lkgSurvivesRestart;
|
|
252
281
|
}
|
|
253
282
|
catch (err) {
|
|
254
283
|
logger.warn("config_lkg_save_failed", { path: lkgPath, err: String(err), note: "durable persist failed — restart signal proceeds anyway (a dead disk + dead center double-fault can re-loop; fix the disk)" });
|
|
284
|
+
return false;
|
|
255
285
|
}
|
|
256
286
|
};
|
|
257
287
|
if (configProvider) {
|
|
@@ -273,6 +303,7 @@ async function main() {
|
|
|
273
303
|
await adoptCenterPrompts(lkgR.effective, "boot-lkg");
|
|
274
304
|
markRosterLanded(lkgR.effective);
|
|
275
305
|
effective = lkgR.effective;
|
|
306
|
+
appliedPlaneEff = lkgR.effective;
|
|
276
307
|
latestEffective = lkgR.effective;
|
|
277
308
|
if (lkgR.etag !== undefined)
|
|
278
309
|
ccEtag = lkgR.etag;
|
|
@@ -323,6 +354,7 @@ async function main() {
|
|
|
323
354
|
await adoptCenterPrompts(r.effective, "boot");
|
|
324
355
|
markRosterLanded(r.effective);
|
|
325
356
|
effective = r.effective;
|
|
357
|
+
appliedPlaneEff = r.effective;
|
|
326
358
|
if (bootClean)
|
|
327
359
|
latestEffective = r.effective;
|
|
328
360
|
if (bootClean && !lkgBooted)
|
|
@@ -568,6 +600,7 @@ async function main() {
|
|
|
568
600
|
const fleetUsage = config.configCenter && process.env.FLEET_ADVERTISE_ADDRESS ? new FleetUsageAccumulator() : undefined;
|
|
569
601
|
const fleetLease = buildFleetLeaseFromEnv(config, { logger, metrics });
|
|
570
602
|
const tracer = buildTracer(metrics, costQuota, modelUsageTracker, fleetUsage, fleetLease, (m) => config.modelQuotaWeights[m] ?? 1, promptManifestTracker);
|
|
603
|
+
const sideQueryAccounting = buildSideQueryAccountant(metrics, costQuota, fleetUsage, fleetLease, (m) => config.modelQuotaWeights[m] ?? 1);
|
|
571
604
|
const toolResultStore = backend?.toolResult ? backend.toolResult() : undefined;
|
|
572
605
|
const sessionPolicyStore = backend ? backend.sessionPolicy() : undefined;
|
|
573
606
|
const fileSnapshotStore = backend ? backend.fileSnapshot() : undefined;
|
|
@@ -1023,6 +1056,7 @@ async function main() {
|
|
|
1023
1056
|
},
|
|
1024
1057
|
};
|
|
1025
1058
|
const runner = new Runner(runnerDeps);
|
|
1059
|
+
const runnerTierFrozen = Object.keys(config.tiers).length > 0;
|
|
1026
1060
|
const { hookModelFor, hookLlm } = createHookLlm({ config, getKeyResolver: () => keyResolver, metrics });
|
|
1027
1061
|
const hookAgentRunner = new Runner({ ...runnerDeps, sessionStore: new TtlSessionStore({ defaultTtlDays: 1 / 24 }) });
|
|
1028
1062
|
const hookAgent = async ({ prompt, model, timeoutMs }) => {
|
|
@@ -1131,7 +1165,28 @@ async function main() {
|
|
|
1131
1165
|
skills = await applyCenterSkills(skills, effective.skills, config.configCenter.baseUrl, config.configCenter.token, logger, undefined, process.env.CONFIG_LKG_DISABLED !== "true" ? defaultSkillCacheDir() : undefined);
|
|
1132
1166
|
}
|
|
1133
1167
|
if (bootLkgCandidate && lkgEnabled) {
|
|
1134
|
-
await
|
|
1168
|
+
const bootPublished = await persistLkgDurable(bootLkgCandidate.effective, bootLkgCandidate.etag).catch((err) => {
|
|
1169
|
+
logger.warn("config_lkg_save_failed", { path: lkgPath, err: String(err), note: "boot LKG persist failed — retained for the per-tick retry" });
|
|
1170
|
+
return false;
|
|
1171
|
+
});
|
|
1172
|
+
{
|
|
1173
|
+
const enabledManifest = (bootLkgCandidate.effective.skills?.skills ?? []).filter((m) => m.enabled !== false);
|
|
1174
|
+
const canonScenarios = (a) => [...(a ?? [])].sort().join("\u0000");
|
|
1175
|
+
const liveSkillMiss = enabledManifest.some((m) => !skills.some((sk) => sk.spec.name === m.name &&
|
|
1176
|
+
skillContentHash(sk.spec.content) === m.contentHash &&
|
|
1177
|
+
sk.spec.description === m.description &&
|
|
1178
|
+
canonScenarios(sk.scenarios) === canonScenarios(m.scenarios)));
|
|
1179
|
+
if (liveSkillMiss)
|
|
1180
|
+
bootSkillStale = true;
|
|
1181
|
+
if (liveSkillMiss && (bootPublished || (!lkgSurvivesRestart && lastPrewarmBodiesVerified))) {
|
|
1182
|
+
pendingRestart = { restartRequired: true, reasons: ["skills"], version: bootLkgCandidate.effective.version, since: Date.now() };
|
|
1183
|
+
logger.warn("config_boot_skill_stale_restart", { version: bootLkgCandidate.effective.version, durableHandoff: bootPublished, note: bootPublished ? "live skill application incomplete but the durable LKG carries the full candidate — skills restart published (the next boot reads the complete LKG)" : "live skill application incomplete on a non-durable deployment but every enabled body is verified in the cache — skills restart published (transient failure proven repaired; the restarted process re-pulls live)" });
|
|
1184
|
+
}
|
|
1185
|
+
else if (liveSkillMiss || (!bootPublished && lkgSurvivesRestart)) {
|
|
1186
|
+
planeDeferredNoHandoff = { version: bootLkgCandidate.effective.version, since: Date.now(), blocked: liveSkillMiss ? ["skills"] : [], candidate: bootLkgCandidate.effective, ...(bootLkgCandidate.etag !== undefined ? { candidateEtag: bootLkgCandidate.etag } : {}) };
|
|
1187
|
+
logger.warn("config_lkg_boot_publication_retained", { version: bootLkgCandidate.effective.version, liveSkillMiss, durableDeclared: lkgSurvivesRestart, note: (lkgSurvivesRestart ? "boot LKG publication incomplete on a durable-declared deployment — candidate retained; the refresh loop retries after each confirming 304" : "live skill application incomplete on a non-durable deployment and the body cache could not be verified — skills debt retained WITHOUT a restart signal (codex R37: signaling now could restart-storm; each confirming 304 re-verifies and publishes once the bodies prove fetchable)") + (liveSkillMiss && lkgSurvivesRestart ? "; this process runs skill-stale, promotion will publish a skills restart" : "") });
|
|
1188
|
+
}
|
|
1189
|
+
}
|
|
1135
1190
|
}
|
|
1136
1191
|
if (effective?.mcp)
|
|
1137
1192
|
config.mcpServers = resolveMcpServers(effective.mcp, logger);
|
|
@@ -1243,7 +1298,29 @@ async function main() {
|
|
|
1243
1298
|
refreshInFlight = true;
|
|
1244
1299
|
try {
|
|
1245
1300
|
await retryPendingCatalog("refresh-retry");
|
|
1301
|
+
if (planeDeferredNoHandoff) {
|
|
1302
|
+
metrics.inc("models_tiers_plane_deferred_stuck_total");
|
|
1303
|
+
logger.warn("models_tiers_deferred_no_handoff", { version: planeDeferredNoHandoff.version, since: planeDeferredNoHandoff.since, note: "candidate still retained without a next-boot handoff — mount restart-surviving LKG storage + CONFIG_LKG_DURABLE=true, or restart manually after ensuring the candidate is re-pulled at boot" });
|
|
1304
|
+
}
|
|
1246
1305
|
const r = prefetched !== undefined ? prefetched : await configProvider.fetchEffective(ccEtag);
|
|
1306
|
+
if (r === null && planeDeferredNoHandoff) {
|
|
1307
|
+
const skillsDebt = (planeDeferredNoHandoff.blocked ?? []).includes("skills");
|
|
1308
|
+
const promoted = lkgSurvivesRestart || skillsDebt ? await persistLkgDurable(planeDeferredNoHandoff.candidate, planeDeferredNoHandoff.candidateEtag) : false;
|
|
1309
|
+
const skillsProofPromotion = !promoted && !lkgSurvivesRestart && skillsDebt && lastPrewarmBodiesVerified && planeDeferredNoHandoff.planeDeferred !== true;
|
|
1310
|
+
if (skillsProofPromotion) {
|
|
1311
|
+
pendingRestart = { restartRequired: true, reasons: ["skills"], version: planeDeferredNoHandoff.version, since: Date.now() };
|
|
1312
|
+
logger.info("config_boot_skill_debt_promoted", { version: planeDeferredNoHandoff.version, note: "skill bodies verified on a source-confirmed-current candidate — the withheld skills restart is now published (non-durable lane, repair proven)" });
|
|
1313
|
+
planeDeferredNoHandoff = undefined;
|
|
1314
|
+
}
|
|
1315
|
+
else if (promoted) {
|
|
1316
|
+
const promotedReasons = [...new Set([...(planeDeferredNoHandoff.blocked ?? []), ...(planeDeferredNoHandoff.planeDeferred ? ["models-tiers"] : [])])];
|
|
1317
|
+
if (promotedReasons.length > 0) {
|
|
1318
|
+
pendingRestart = { restartRequired: true, reasons: promotedReasons, version: planeDeferredNoHandoff.version, since: Date.now() };
|
|
1319
|
+
}
|
|
1320
|
+
logger.info("models_tiers_deferred_promoted", { version: planeDeferredNoHandoff.version, reasons: promotedReasons, note: promotedReasons.length > 0 ? "handoff persistence recovered on a source-confirmed-current candidate — the withheld restart signal is now published" : "boot publication retry completed — LKG now durable, no restart needed (nothing was withheld)" });
|
|
1321
|
+
planeDeferredNoHandoff = undefined;
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1247
1324
|
if (r) {
|
|
1248
1325
|
const promptsRaw = r.effective.prompts;
|
|
1249
1326
|
const promptsGate = promptsRaw !== undefined ? validatePromptsDomain(promptsRaw) : { ok: true };
|
|
@@ -1274,13 +1351,50 @@ async function main() {
|
|
|
1274
1351
|
ccEtag = r.etag;
|
|
1275
1352
|
}
|
|
1276
1353
|
else {
|
|
1277
|
-
|
|
1354
|
+
const planeDeferred = (runnerTierFrozen || planeHasActiveTiers(r.effective)) && modelPlaneChanged(appliedPlaneEff, r.effective);
|
|
1355
|
+
applyEffective(config, r.effective, logger, { teamsOnly: true, sealedKeys, ...(planeDeferred ? { deferModelPlane: true } : {}) });
|
|
1356
|
+
if (planeDeferred) {
|
|
1357
|
+
logger.warn("models_tiers_plane_deferred", { version: r.effective.version, note: "tier-frozen Runner: the changed model plane (models/roles/tiers/default) is NOT hot-applied — admission stays on the Runner's generation; restart applies the new plane (models-tiers restart signal rides /health)" });
|
|
1358
|
+
}
|
|
1359
|
+
else {
|
|
1360
|
+
appliedPlaneEff = r.effective;
|
|
1361
|
+
planeDeferredNoHandoff = undefined;
|
|
1362
|
+
}
|
|
1278
1363
|
await adoptCenterPrompts(r.effective, "refresh");
|
|
1279
|
-
|
|
1364
|
+
if (!planeDeferred)
|
|
1365
|
+
markRosterLanded(r.effective);
|
|
1280
1366
|
mutateInPlace(pricing, buildPricing(config.models));
|
|
1281
1367
|
keyResolver = buildKeyResolver(config.modelApiKeyEnv, process.env, config.modelApiKeys);
|
|
1282
|
-
await persistLkgDurable(r.effective, r.etag);
|
|
1368
|
+
const lkgPersisted = await persistLkgDurable(r.effective, r.etag);
|
|
1283
1369
|
const reasons = restartReasons(effective, r.effective);
|
|
1370
|
+
const skillsChangeDetected = reasons.includes("skills");
|
|
1371
|
+
if (bootSkillStale && !skillsChangeDetected)
|
|
1372
|
+
reasons.push("skills");
|
|
1373
|
+
if (planeDeferred) {
|
|
1374
|
+
if (lkgPersisted) {
|
|
1375
|
+
if (!reasons.includes("models-tiers"))
|
|
1376
|
+
reasons.push("models-tiers");
|
|
1377
|
+
planeDeferredNoHandoff = undefined;
|
|
1378
|
+
}
|
|
1379
|
+
else {
|
|
1380
|
+
const blocked = reasons.splice(0, reasons.length);
|
|
1381
|
+
planeDeferredNoHandoff = { version: r.effective.version, since: planeDeferredNoHandoff?.since ?? Date.now(), ...(blocked.length > 0 ? { blocked } : {}), planeDeferred: true, candidate: r.effective, ...(r.etag !== undefined ? { candidateEtag: r.etag } : {}) };
|
|
1382
|
+
logger.warn("models_tiers_deferred_no_handoff", { version: r.effective.version, blockedReasons: blocked, note: "plane deferred but no NEXT-BOOT handoff (LKG disabled/unwritable, or storage not declared restart-surviving — set CONFIG_LKG_DURABLE=true) — the WHOLE restart signal is withheld (it would loop); the old generation keeps serving" });
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
else if (lkgSurvivesRestart && !lkgPersisted) {
|
|
1386
|
+
const blocked = reasons.splice(0, reasons.length);
|
|
1387
|
+
planeDeferredNoHandoff = { version: r.effective.version, since: planeDeferredNoHandoff?.since ?? Date.now(), blocked, candidate: r.effective, ...(r.etag !== undefined ? { candidateEtag: r.etag } : {}) };
|
|
1388
|
+
logger.warn("restart_candidate_publication_incomplete", { version: r.effective.version, blockedReasons: blocked, note: "durable LKG declared but the candidate's publication is incomplete (skill bodies unverified) — restart signal withheld; retried every tick until the cache completes" });
|
|
1389
|
+
}
|
|
1390
|
+
else if (!planeDeferred && !lkgSurvivesRestart && bootSkillStale && !skillsChangeDetected && !lastPrewarmBodiesVerified) {
|
|
1391
|
+
const i = reasons.indexOf("skills");
|
|
1392
|
+
if (i >= 0) {
|
|
1393
|
+
reasons.splice(i, 1);
|
|
1394
|
+
planeDeferredNoHandoff = { version: r.effective.version, since: planeDeferredNoHandoff?.since ?? Date.now(), blocked: ["skills"], candidate: r.effective, ...(r.etag !== undefined ? { candidateEtag: r.etag } : {}) };
|
|
1395
|
+
logger.warn("config_boot_skill_debt_withheld", { version: r.effective.version, note: "boot skill debt still unproven on a non-durable deployment (body cache does not verify) — skills restart withheld this tick; re-verified each tick and published once the bodies prove fetchable" });
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1284
1398
|
if (reasons.length === 0) {
|
|
1285
1399
|
pendingRestart = undefined;
|
|
1286
1400
|
}
|
|
@@ -1338,16 +1452,41 @@ async function main() {
|
|
|
1338
1452
|
ccEtag = r.etag;
|
|
1339
1453
|
return;
|
|
1340
1454
|
}
|
|
1341
|
-
|
|
1455
|
+
const planeDeferredLate = (runnerTierFrozen || planeHasActiveTiers(r.effective)) && modelPlaneChanged(appliedPlaneEff, r.effective);
|
|
1456
|
+
applyEffective(config, r.effective, logger, { teamsOnly: true, sealedKeys, ...(planeDeferredLate ? { deferModelPlane: true } : {}) });
|
|
1457
|
+
if (planeDeferredLate)
|
|
1458
|
+
logger.warn("models_tiers_plane_deferred", { version: r.effective.version, note: "tier-frozen Runner (env tiers): the late-boot center model plane is NOT hot-applied — restart applies it" });
|
|
1459
|
+
else {
|
|
1460
|
+
appliedPlaneEff = r.effective;
|
|
1461
|
+
planeDeferredNoHandoff = undefined;
|
|
1462
|
+
}
|
|
1342
1463
|
await adoptCenterPrompts(r.effective, "boot-deferred");
|
|
1343
|
-
|
|
1464
|
+
if (!planeDeferredLate)
|
|
1465
|
+
markRosterLanded(r.effective);
|
|
1344
1466
|
mutateInPlace(pricing, buildPricing(config.models));
|
|
1345
1467
|
keyResolver = buildKeyResolver(config.modelApiKeyEnv, process.env, config.modelApiKeys);
|
|
1346
1468
|
effective = r.effective;
|
|
1347
1469
|
latestEffective = r.effective;
|
|
1348
1470
|
ccEtag = r.etag;
|
|
1349
|
-
await persistLkgDurable(r.effective, r.etag);
|
|
1471
|
+
const lkgPersistedLate = await persistLkgDurable(r.effective, r.etag);
|
|
1350
1472
|
const reasons = restartReasons(undefined, r.effective);
|
|
1473
|
+
if (planeDeferredLate) {
|
|
1474
|
+
if (lkgPersistedLate) {
|
|
1475
|
+
if (!reasons.includes("models-tiers"))
|
|
1476
|
+
reasons.push("models-tiers");
|
|
1477
|
+
planeDeferredNoHandoff = undefined;
|
|
1478
|
+
}
|
|
1479
|
+
else {
|
|
1480
|
+
const blocked = reasons.splice(0, reasons.length);
|
|
1481
|
+
planeDeferredNoHandoff = { version: r.effective.version, since: planeDeferredNoHandoff?.since ?? Date.now(), ...(blocked.length > 0 ? { blocked } : {}), planeDeferred: true, candidate: r.effective, ...(r.etag !== undefined ? { candidateEtag: r.etag } : {}) };
|
|
1482
|
+
logger.warn("models_tiers_deferred_no_handoff", { version: r.effective.version, blockedReasons: blocked, note: "late-boot plane deferred but no NEXT-BOOT handoff (see CONFIG_LKG_DURABLE) — the WHOLE restart signal is withheld (it would loop); old generation keeps serving" });
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
else if (planeDeferredLate === false && lkgSurvivesRestart && !lkgPersistedLate) {
|
|
1486
|
+
const blocked = reasons.splice(0, reasons.length);
|
|
1487
|
+
planeDeferredNoHandoff = { version: r.effective.version, since: planeDeferredNoHandoff?.since ?? Date.now(), blocked, candidate: r.effective, ...(r.etag !== undefined ? { candidateEtag: r.etag } : {}) };
|
|
1488
|
+
logger.warn("restart_candidate_publication_incomplete", { version: r.effective.version, blockedReasons: blocked, note: "late-boot candidate publication incomplete on a durable-declared deployment — restart signal withheld; retried every tick" });
|
|
1489
|
+
}
|
|
1351
1490
|
if (reasons.length > 0)
|
|
1352
1491
|
pendingRestart = { restartRequired: true, reasons, version: r.effective.version, since: Date.now() };
|
|
1353
1492
|
logger.info("config_loaded_deferred", { source: configProvider.kind, version: r.effective.version, models: (r.effective.models?.models ?? []).filter((m) => m.enabled !== false).length, ...(reasons.length > 0 ? { restartRequired: true, restartReasons: reasons } : {}) });
|
|
@@ -1534,6 +1673,7 @@ async function main() {
|
|
|
1534
1673
|
rateLimiter,
|
|
1535
1674
|
costQuota,
|
|
1536
1675
|
...(fleetLease ? { fleetLease } : {}),
|
|
1676
|
+
sideQueryAccounting,
|
|
1537
1677
|
...(outcomeSink ? { outcomeSink } : {}),
|
|
1538
1678
|
...(memoryExportBackend ? { memoryExport: (scope) => exportMemoryScope(memoryExportBackend, scope) } : {}),
|
|
1539
1679
|
...(memoryExportBackend && memorySyncCursors
|
|
@@ -1545,17 +1685,29 @@ async function main() {
|
|
|
1545
1685
|
pricingConfigured: Object.values(pricing).some((p) => Object.values(p).some((v) => typeof v === "number" && v > 0)),
|
|
1546
1686
|
},
|
|
1547
1687
|
restartState: () => pendingRestart,
|
|
1688
|
+
planeDeferredState: () => planeDeferredNoHandoff,
|
|
1548
1689
|
drainState,
|
|
1549
1690
|
storeDegraded: storeBackendDegraded,
|
|
1550
|
-
resolveSpec: async (body, _req, auth) => {
|
|
1691
|
+
resolveSpec: async (body, _req, auth, opts) => {
|
|
1551
1692
|
const requested = gateScenarioRequest(await principalCaps?.scenarioRuling(auth?.principal), body.scenario, config.defaultScenario);
|
|
1552
1693
|
gateExecutionLane(await principalCaps?.executionRuling(auth?.principal), config.remoteExec?.provider ?? "in-process");
|
|
1553
1694
|
const scenarioName = scenarios[requested] ? requested : "default";
|
|
1554
1695
|
if (scenarioName === "autonomous")
|
|
1555
1696
|
logger.warn("scenario_autonomous_deprecated", { alias: "code", note: "renamed by [891]; alias keeps finalVerification pinned — pass scenario:'code' (+ explicit finalVerification if wanted)" });
|
|
1556
1697
|
const cap = selectScenario(scenarios, scenarioName)(body, auth?.principal);
|
|
1698
|
+
const centerDecls = centerPrompts?.declarations;
|
|
1699
|
+
const appendLessPack = centerDecls
|
|
1700
|
+
? centerIdentityAssembled(centerDecls, scenarioName, hasConstitutionAnchors)
|
|
1701
|
+
: providerDropsAppend(cap.promptProvider, typeof body.systemPrompt === "string" ? body.systemPrompt : undefined);
|
|
1702
|
+
const acceptedAppend = acceptAppendSystemPrompt(body.appendSystemPrompt, (detail) => logger.warn("append_system_prompt_dropped", { detail, sessionId: auth?.sessionId ?? null }), appendLessPack);
|
|
1557
1703
|
const objective = typeof body.objective === "string" ? body.objective : "";
|
|
1558
1704
|
const parsedSettings = parseTaskSettings(body.settings);
|
|
1705
|
+
if (opts?.leg === "fresh" && appendLessPack) {
|
|
1706
|
+
const riderPresent = typeof body.appendSystemPrompt === "string" && body.appendSystemPrompt.length > 0;
|
|
1707
|
+
if (riderPresent || parsedSettings.settings?.outputStyle) {
|
|
1708
|
+
throw new HttpError(400, `${riderPresent ? "appendSystemPrompt" : "settings.outputStyle"} is not supported when the effective prompt is already assembled (constitution anchors in systemPrompt, or a center assembled-identity pack for this scenario): the assembler's pass-through path cannot mount the append slot — fold it into the assembled prompt instead`);
|
|
1709
|
+
}
|
|
1710
|
+
}
|
|
1559
1711
|
if (parsedSettings.deferred.length > 0) {
|
|
1560
1712
|
logger.warn("task_settings_deferred", { fields: parsedSettings.deferred, sessionId: auth?.sessionId ?? null });
|
|
1561
1713
|
}
|
|
@@ -1649,6 +1801,7 @@ async function main() {
|
|
|
1649
1801
|
const spec = {
|
|
1650
1802
|
objective: picked.cleanedObjective,
|
|
1651
1803
|
systemPrompt: typeof body.systemPrompt === "string" ? body.systemPrompt : undefined,
|
|
1804
|
+
appendSystemPrompt: acceptedAppend,
|
|
1652
1805
|
sessionId: auth?.sessionId,
|
|
1653
1806
|
principal: auth?.principal,
|
|
1654
1807
|
images: body.images,
|
|
@@ -1681,6 +1834,20 @@ async function main() {
|
|
|
1681
1834
|
attachments: normalizeAttachments(body.attachments),
|
|
1682
1835
|
...(taskHooks ? { hooks: composeHooks(deploymentHooks, taskHooks) } : {}),
|
|
1683
1836
|
model: picked.model,
|
|
1837
|
+
...((() => {
|
|
1838
|
+
if (typeof body.compactionModel !== "string" || body.compactionModel.length === 0)
|
|
1839
|
+
return {};
|
|
1840
|
+
const cm = matchCatalogModel(body.compactionModel, wireCatalog);
|
|
1841
|
+
if (cm === undefined) {
|
|
1842
|
+
if (opts?.leg === "fresh") {
|
|
1843
|
+
throw new HttpError(400, `unknown compactionModel "${body.compactionModel.slice(0, 120)}" — not in the configured catalog (name, tier word, or id; a catalog refresh may have removed it mid-request)`);
|
|
1844
|
+
}
|
|
1845
|
+
metrics.inc("task_model_unknown_fallback_total");
|
|
1846
|
+
logger.warn("compaction_model_unknown_fallback", { requested: body.compactionModel.slice(0, 120), sessionId: auth?.sessionId ?? null });
|
|
1847
|
+
return {};
|
|
1848
|
+
}
|
|
1849
|
+
return { compactionModel: cm };
|
|
1850
|
+
})()),
|
|
1684
1851
|
thinking: effectiveThinking(body.reasoningEffort, parsedSettings.settings?.ultracode === true),
|
|
1685
1852
|
getApiKeyAndHeaders: keyResolver,
|
|
1686
1853
|
...(() => {
|
|
@@ -1721,7 +1888,7 @@ async function main() {
|
|
|
1721
1888
|
})(cap.tools),
|
|
1722
1889
|
skills: mergeUserSkills(cap.skills, body.skills, logger),
|
|
1723
1890
|
mcp: resolveRequestMcp(mcpForScenario(config.mcpServers, scenarioName), body.mcpServers, config, logger),
|
|
1724
|
-
promptProvider:
|
|
1891
|
+
promptProvider: centerDecls ? centerPromptProvider(centerDecls, scenarioName) : cap.promptProvider,
|
|
1725
1892
|
toolPolicy: durableEnabled
|
|
1726
1893
|
? combinePolicies(createDurableQuestionPolicy(), createDurableAskPolicy({
|
|
1727
1894
|
requireApproval: config.approvalRequire, deny: config.approvalDeny, autoBudget: config.approvalAutoBudget, neverAuto: config.approvalNeverAuto,
|
|
@@ -1771,7 +1938,17 @@ async function main() {
|
|
|
1771
1938
|
const governedBase = applyRuntimeGovernance(spec, { autonomy: config.autonomy, commandPolicy: config.commandPolicy });
|
|
1772
1939
|
let governed = governedBase;
|
|
1773
1940
|
const bodyMode = coercePermissionMode(body.permissionMode);
|
|
1774
|
-
|
|
1941
|
+
let effectiveSettings = bodyMode ? withPermissionMode(parsedSettings.settings, bodyMode) : parsedSettings.settings;
|
|
1942
|
+
if (appendLessPack && effectiveSettings?.outputStyle) {
|
|
1943
|
+
logger.warn("output_style_dropped", { detail: "the effective prompt pack cannot mount the append slot (already-assembled systemPrompt or center assembled-identity declaration)", sessionId: auth?.sessionId ?? null });
|
|
1944
|
+
const { outputStyle: _dropped, ...rest } = effectiveSettings;
|
|
1945
|
+
effectiveSettings = Object.keys(rest).length > 0 ? rest : undefined;
|
|
1946
|
+
}
|
|
1947
|
+
if (effectiveSettings?.outputStyle && acceptedAppend && acceptedAppend.length + 2 + effectiveSettings.outputStyle.length > MAX_SETTINGS_OUTPUT_STYLE_CHARS) {
|
|
1948
|
+
logger.warn("output_style_dropped", { detail: `combined append carriers exceed the ${MAX_SETTINGS_OUTPUT_STYLE_CHARS} cap (rider ${acceptedAppend.length} + style ${effectiveSettings.outputStyle.length}) — style dropped (pre-cap stored body on a resume leg?)`, sessionId: auth?.sessionId ?? null });
|
|
1949
|
+
const { outputStyle: _dropped2, ...rest2 } = effectiveSettings;
|
|
1950
|
+
effectiveSettings = Object.keys(rest2).length > 0 ? rest2 : undefined;
|
|
1951
|
+
}
|
|
1775
1952
|
const hostSemanticsLane = config.remoteExec === undefined || config.remoteExec.provider === "host";
|
|
1776
1953
|
const scratchpadDir = hostSemanticsLane && auth?.sessionId
|
|
1777
1954
|
? await ensureScratchpadDir(config.localDataRoot ?? localRoot, auth.sessionId)
|
|
@@ -1861,6 +2038,7 @@ async function main() {
|
|
|
1861
2038
|
...(pkgSourceLane ? { pkgSource: config.sandboxPkgSource } : {}),
|
|
1862
2039
|
egress: egressForRemoteExec(config.remoteExec),
|
|
1863
2040
|
...(factScratchpadDir ? { scratchpadDir: factScratchpadDir } : {}),
|
|
2041
|
+
resumeFacts: resumeFactsForLane(config.remoteExec?.provider, config.remoteExec?.provider === "k8s" ? { k8sSnapshot: Boolean(config.remoteExec.s3Snapshot) } : undefined),
|
|
1864
2042
|
});
|
|
1865
2043
|
if (facts)
|
|
1866
2044
|
governed = { ...governed, envFacts: facts };
|