@lobu/cli 7.1.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/context.d.ts +7 -0
- package/dist/commands/context.d.ts.map +1 -1
- package/dist/commands/context.js +19 -2
- package/dist/commands/context.js.map +1 -1
- package/dist/connectors/chrome.ts +351 -0
- package/dist/connectors/chrome_bookmarks.ts +79 -0
- package/dist/connectors/chrome_downloads.ts +80 -0
- package/dist/connectors/chrome_history.ts +80 -0
- package/dist/connectors/index.ts +14 -8
- package/dist/db/migrations/20260518020000_runs_heartbeat_inflight_narrow.sql +36 -0
- package/dist/db/migrations/20260518040000_agent_transcript_snapshot.sql +54 -0
- package/dist/db/migrations/20260518050000_runs_denormalize_agent_conversation.sql +36 -0
- package/dist/db/migrations/20260518060000_revert_runs_denormalize.sql +29 -0
- package/dist/db/migrations/20260518070000_runs_heartbeat_inflight_widen.sql +33 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +37 -1
- package/dist/index.js.map +1 -1
- package/dist/internal/context.d.ts +4 -1
- package/dist/internal/context.d.ts.map +1 -1
- package/dist/internal/context.js +43 -3
- package/dist/internal/context.js.map +1 -1
- package/dist/internal/index.d.ts +1 -1
- package/dist/internal/index.d.ts.map +1 -1
- package/dist/internal/index.js +1 -1
- package/dist/internal/index.js.map +1 -1
- package/dist/server.bundle.mjs +1520 -568
- package/dist/start-local.bundle.mjs +1556 -570
- package/package.json +6 -6
- package/dist/connectors/browser/evaluate.ts +0 -120
- package/dist/connectors/browser/fill_form.ts +0 -107
- package/dist/connectors/browser/page_text.ts +0 -108
- package/dist/connectors/chrome_tabs.ts +0 -74
|
@@ -1173,7 +1173,7 @@ function createLogger(serviceName) {
|
|
|
1173
1173
|
format: USE_JSON_FORMAT ? jsonFormat : humanFormat
|
|
1174
1174
|
})
|
|
1175
1175
|
];
|
|
1176
|
-
const
|
|
1176
|
+
const logger100 = winston.createLogger({
|
|
1177
1177
|
level,
|
|
1178
1178
|
format: winston.format.combine(
|
|
1179
1179
|
winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }),
|
|
@@ -1187,12 +1187,12 @@ function createLogger(serviceName) {
|
|
|
1187
1187
|
if (isProduction || process.env.SENTRY_DSN) {
|
|
1188
1188
|
try {
|
|
1189
1189
|
const transport = new SentryTransport();
|
|
1190
|
-
|
|
1190
|
+
logger100.add(transport);
|
|
1191
1191
|
} catch {
|
|
1192
1192
|
}
|
|
1193
1193
|
}
|
|
1194
1194
|
});
|
|
1195
|
-
return
|
|
1195
|
+
return logger100;
|
|
1196
1196
|
}
|
|
1197
1197
|
var USE_WINSTON_LOGGER, USE_JSON_FORMAT, SentryTransport;
|
|
1198
1198
|
var init_logger2 = __esm({
|
|
@@ -1346,6 +1346,7 @@ var init_capabilities = __esm({
|
|
|
1346
1346
|
"browser.scripting",
|
|
1347
1347
|
"browser.history",
|
|
1348
1348
|
"browser.bookmarks",
|
|
1349
|
+
"browser.downloads",
|
|
1349
1350
|
"browser.debugger"
|
|
1350
1351
|
// browser.cookies intentionally absent in v1 — high-trust, not approved
|
|
1351
1352
|
];
|
|
@@ -2199,7 +2200,8 @@ function generateWorkerToken(userId, conversationId, deploymentName, options) {
|
|
|
2199
2200
|
platform: options.platform,
|
|
2200
2201
|
sessionKey: options.sessionKey,
|
|
2201
2202
|
traceId: options.traceId,
|
|
2202
|
-
jti: randomUUID()
|
|
2203
|
+
jti: randomUUID(),
|
|
2204
|
+
runId: options.runId
|
|
2203
2205
|
};
|
|
2204
2206
|
return encrypt(JSON.stringify(payload));
|
|
2205
2207
|
}
|
|
@@ -2223,6 +2225,12 @@ function verifyWorkerToken(token) {
|
|
|
2223
2225
|
);
|
|
2224
2226
|
return null;
|
|
2225
2227
|
}
|
|
2228
|
+
if (data.runId !== void 0) {
|
|
2229
|
+
if (typeof data.runId !== "number" || !Number.isInteger(data.runId) || data.runId <= 0) {
|
|
2230
|
+
logger10.error("Worker token rejected: runId must be a positive integer");
|
|
2231
|
+
return null;
|
|
2232
|
+
}
|
|
2233
|
+
}
|
|
2226
2234
|
const ttl = parsePositiveIntEnv("WORKER_TOKEN_TTL_MS", 2 * 60 * 60 * 1e3);
|
|
2227
2235
|
const skewMs = parsePositiveIntEnv(
|
|
2228
2236
|
"WORKER_TOKEN_CLOCK_SKEW_MS",
|
|
@@ -6143,12 +6151,32 @@ function sessionMatchesMetadataOwner(session, ownerPlatform, ownerUserId) {
|
|
|
6143
6151
|
}
|
|
6144
6152
|
return ownerPlatform === session.platform || session.platform === "external";
|
|
6145
6153
|
}
|
|
6154
|
+
async function resolveAuthorizedOrgId(store, agentId, ownerPlatform, ownerUserId) {
|
|
6155
|
+
if (!store || !ownerPlatform || !ownerUserId) return void 0;
|
|
6156
|
+
const orgs = await store.findAgentOrganizations(
|
|
6157
|
+
ownerPlatform,
|
|
6158
|
+
ownerUserId,
|
|
6159
|
+
agentId
|
|
6160
|
+
);
|
|
6161
|
+
if (orgs.length !== 1) return void 0;
|
|
6162
|
+
return orgs[0];
|
|
6163
|
+
}
|
|
6146
6164
|
async function verifyOwnedAgentAccess(session, agentId, config) {
|
|
6147
6165
|
if (session.isAdmin) {
|
|
6148
6166
|
return { authorized: true };
|
|
6149
6167
|
}
|
|
6150
6168
|
if (session.agentId) {
|
|
6151
|
-
|
|
6169
|
+
if (session.agentId !== agentId) {
|
|
6170
|
+
return { authorized: false };
|
|
6171
|
+
}
|
|
6172
|
+
const lookupUserId2 = resolveSettingsLookupUserId(session);
|
|
6173
|
+
const organizationId2 = await resolveAuthorizedOrgId(
|
|
6174
|
+
config.userAgentsStore,
|
|
6175
|
+
agentId,
|
|
6176
|
+
session.platform,
|
|
6177
|
+
lookupUserId2 || void 0
|
|
6178
|
+
);
|
|
6179
|
+
return { authorized: true, organizationId: organizationId2 };
|
|
6152
6180
|
}
|
|
6153
6181
|
const lookupUserId = resolveSettingsLookupUserId(session);
|
|
6154
6182
|
if (config.userAgentsStore) {
|
|
@@ -6158,10 +6186,17 @@ async function verifyOwnedAgentAccess(session, agentId, config) {
|
|
|
6158
6186
|
agentId
|
|
6159
6187
|
);
|
|
6160
6188
|
if (owns) {
|
|
6189
|
+
const organizationId2 = await resolveAuthorizedOrgId(
|
|
6190
|
+
config.userAgentsStore,
|
|
6191
|
+
agentId,
|
|
6192
|
+
session.platform,
|
|
6193
|
+
lookupUserId
|
|
6194
|
+
);
|
|
6161
6195
|
return {
|
|
6162
6196
|
authorized: true,
|
|
6163
6197
|
ownerPlatform: session.platform,
|
|
6164
|
-
ownerUserId: lookupUserId
|
|
6198
|
+
ownerUserId: lookupUserId,
|
|
6199
|
+
organizationId: organizationId2
|
|
6165
6200
|
};
|
|
6166
6201
|
}
|
|
6167
6202
|
}
|
|
@@ -6180,10 +6215,17 @@ async function verifyOwnedAgentAccess(session, agentId, config) {
|
|
|
6180
6215
|
config.userAgentsStore.addAgent(session.platform, lookupUserId, agentId).catch(() => {
|
|
6181
6216
|
});
|
|
6182
6217
|
}
|
|
6218
|
+
const organizationId = metadata.organizationId ?? await resolveAuthorizedOrgId(
|
|
6219
|
+
config.userAgentsStore,
|
|
6220
|
+
agentId,
|
|
6221
|
+
metadata.owner.platform,
|
|
6222
|
+
metadata.owner.userId
|
|
6223
|
+
);
|
|
6183
6224
|
return {
|
|
6184
6225
|
authorized: true,
|
|
6185
6226
|
ownerPlatform: metadata.owner.platform,
|
|
6186
|
-
ownerUserId: metadata.owner.userId
|
|
6227
|
+
ownerUserId: metadata.owner.userId,
|
|
6228
|
+
organizationId
|
|
6187
6229
|
};
|
|
6188
6230
|
}
|
|
6189
6231
|
function createTokenVerifier(config) {
|
|
@@ -6193,6 +6235,12 @@ function createTokenVerifier(config) {
|
|
|
6193
6235
|
return result.authorized ? payload : null;
|
|
6194
6236
|
};
|
|
6195
6237
|
}
|
|
6238
|
+
function createOwnershipResolver(config) {
|
|
6239
|
+
return async (payload, agentId) => {
|
|
6240
|
+
if (!payload) return { authorized: false };
|
|
6241
|
+
return verifyOwnedAgentAccess(payload, agentId, config);
|
|
6242
|
+
};
|
|
6243
|
+
}
|
|
6196
6244
|
var init_agent_ownership = __esm({
|
|
6197
6245
|
"src/gateway/routes/shared/agent-ownership.ts"() {
|
|
6198
6246
|
"use strict";
|
|
@@ -6587,41 +6635,46 @@ function createAgentApi(config) {
|
|
|
6587
6635
|
if (requestSignal?.aborted) {
|
|
6588
6636
|
return;
|
|
6589
6637
|
}
|
|
6590
|
-
|
|
6591
|
-
|
|
6592
|
-
event: "connected",
|
|
6593
|
-
data: JSON.stringify({
|
|
6594
|
-
agentId: session.agentId || sessionKey3,
|
|
6595
|
-
timestamp: Date.now()
|
|
6596
|
-
})
|
|
6597
|
-
});
|
|
6598
|
-
for (const entry of sseManager.getRecentEvents(sseKey)) {
|
|
6599
|
-
await stream2.writeSSE({
|
|
6600
|
-
event: entry.event,
|
|
6601
|
-
data: JSON.stringify(entry.data)
|
|
6602
|
-
});
|
|
6603
|
-
}
|
|
6604
|
-
const heartbeatInterval = setInterval(async () => {
|
|
6605
|
-
try {
|
|
6606
|
-
await stream2.writeSSE({
|
|
6607
|
-
event: "ping",
|
|
6608
|
-
data: JSON.stringify({ timestamp: Date.now() })
|
|
6609
|
-
});
|
|
6610
|
-
} catch {
|
|
6611
|
-
clearInterval(heartbeatInterval);
|
|
6612
|
-
}
|
|
6613
|
-
}, 3e4);
|
|
6638
|
+
let heartbeatInterval;
|
|
6639
|
+
let connectionAdded = false;
|
|
6614
6640
|
let cleanedUp = false;
|
|
6615
6641
|
const cleanup = () => {
|
|
6616
6642
|
if (cleanedUp) return;
|
|
6617
6643
|
cleanedUp = true;
|
|
6618
|
-
clearInterval(heartbeatInterval);
|
|
6619
|
-
|
|
6644
|
+
if (heartbeatInterval) clearInterval(heartbeatInterval);
|
|
6645
|
+
if (connectionAdded) {
|
|
6646
|
+
sseManager.removeConnection(sseKey, stream2);
|
|
6647
|
+
}
|
|
6620
6648
|
logger24.info(`SSE connection closed for session ${sseKey}`);
|
|
6621
6649
|
};
|
|
6622
6650
|
stream2.onAbort(cleanup);
|
|
6623
6651
|
const detachAbortBridge = bindRequestAbortToStream(requestSignal, stream2);
|
|
6652
|
+
sseManager.addConnection(sseKey, stream2);
|
|
6653
|
+
connectionAdded = true;
|
|
6624
6654
|
try {
|
|
6655
|
+
await stream2.writeSSE({
|
|
6656
|
+
event: "connected",
|
|
6657
|
+
data: JSON.stringify({
|
|
6658
|
+
agentId: session.agentId || sessionKey3,
|
|
6659
|
+
timestamp: Date.now()
|
|
6660
|
+
})
|
|
6661
|
+
});
|
|
6662
|
+
for (const entry of sseManager.getRecentEvents(sseKey)) {
|
|
6663
|
+
await stream2.writeSSE({
|
|
6664
|
+
event: entry.event,
|
|
6665
|
+
data: JSON.stringify(entry.data)
|
|
6666
|
+
});
|
|
6667
|
+
}
|
|
6668
|
+
heartbeatInterval = setInterval(async () => {
|
|
6669
|
+
try {
|
|
6670
|
+
await stream2.writeSSE({
|
|
6671
|
+
event: "ping",
|
|
6672
|
+
data: JSON.stringify({ timestamp: Date.now() })
|
|
6673
|
+
});
|
|
6674
|
+
} catch {
|
|
6675
|
+
if (heartbeatInterval) clearInterval(heartbeatInterval);
|
|
6676
|
+
}
|
|
6677
|
+
}, 3e4);
|
|
6625
6678
|
while (!stream2.aborted && !stream2.closed) {
|
|
6626
6679
|
await stream2.sleep(1e3);
|
|
6627
6680
|
}
|
|
@@ -7520,6 +7573,20 @@ var init_agent_config = __esm({
|
|
|
7520
7573
|
import { readdir, readFile, stat } from "node:fs/promises";
|
|
7521
7574
|
import { join as join3, resolve } from "node:path";
|
|
7522
7575
|
import { Hono as Hono7 } from "hono";
|
|
7576
|
+
async function readLatestSnapshotJsonl(agentId, organizationId) {
|
|
7577
|
+
if (!organizationId) return null;
|
|
7578
|
+
const sql = getDb();
|
|
7579
|
+
const snapshotRows = await sql`
|
|
7580
|
+
SELECT snapshot_jsonl
|
|
7581
|
+
FROM public.agent_transcript_snapshot
|
|
7582
|
+
WHERE organization_id = ${organizationId}
|
|
7583
|
+
AND agent_id = ${agentId}
|
|
7584
|
+
AND terminal_status = 'completed'
|
|
7585
|
+
ORDER BY run_id DESC
|
|
7586
|
+
LIMIT 1
|
|
7587
|
+
`;
|
|
7588
|
+
return snapshotRows[0]?.snapshot_jsonl ?? null;
|
|
7589
|
+
}
|
|
7523
7590
|
function isSafeAgentId(id) {
|
|
7524
7591
|
return SAFE_AGENT_ID.test(id);
|
|
7525
7592
|
}
|
|
@@ -7616,17 +7683,23 @@ function entryToMessage(entry) {
|
|
|
7616
7683
|
}
|
|
7617
7684
|
return null;
|
|
7618
7685
|
}
|
|
7619
|
-
async function readSessionMessages(agentId, cursorParam, limit) {
|
|
7620
|
-
|
|
7621
|
-
if (
|
|
7622
|
-
|
|
7623
|
-
|
|
7624
|
-
|
|
7625
|
-
|
|
7626
|
-
|
|
7627
|
-
|
|
7686
|
+
async function readSessionMessages(agentId, cursorParam, limit, organizationId) {
|
|
7687
|
+
let content = null;
|
|
7688
|
+
if (process.env.LOBU_SESSION_STORE !== "file") {
|
|
7689
|
+
content = await readLatestSnapshotJsonl(agentId, organizationId);
|
|
7690
|
+
}
|
|
7691
|
+
if (content === null) {
|
|
7692
|
+
const sessionPath = await findSessionFile(agentId);
|
|
7693
|
+
if (!sessionPath) {
|
|
7694
|
+
return {
|
|
7695
|
+
messages: [],
|
|
7696
|
+
nextCursor: null,
|
|
7697
|
+
hasMore: false,
|
|
7698
|
+
sessionId: "none"
|
|
7699
|
+
};
|
|
7700
|
+
}
|
|
7701
|
+
content = await readFile(sessionPath, "utf-8");
|
|
7628
7702
|
}
|
|
7629
|
-
const content = await readFile(sessionPath, "utf-8");
|
|
7630
7703
|
const { entries, sessionId } = parseSessionEntries(content);
|
|
7631
7704
|
const allMessages = [];
|
|
7632
7705
|
for (const entry of entries) {
|
|
@@ -7648,19 +7721,25 @@ async function readSessionMessages(agentId, cursorParam, limit) {
|
|
|
7648
7721
|
sessionId: sessionId || "unknown"
|
|
7649
7722
|
};
|
|
7650
7723
|
}
|
|
7651
|
-
async function readSessionStats(agentId) {
|
|
7652
|
-
|
|
7653
|
-
if (
|
|
7654
|
-
|
|
7655
|
-
|
|
7656
|
-
|
|
7657
|
-
|
|
7658
|
-
|
|
7659
|
-
|
|
7660
|
-
|
|
7661
|
-
|
|
7724
|
+
async function readSessionStats(agentId, organizationId) {
|
|
7725
|
+
let content = null;
|
|
7726
|
+
if (process.env.LOBU_SESSION_STORE !== "file") {
|
|
7727
|
+
content = await readLatestSnapshotJsonl(agentId, organizationId);
|
|
7728
|
+
}
|
|
7729
|
+
if (content === null) {
|
|
7730
|
+
const sessionPath = await findSessionFile(agentId);
|
|
7731
|
+
if (!sessionPath) {
|
|
7732
|
+
return {
|
|
7733
|
+
sessionId: "none",
|
|
7734
|
+
messageCount: 0,
|
|
7735
|
+
userMessages: 0,
|
|
7736
|
+
assistantMessages: 0,
|
|
7737
|
+
totalInputTokens: 0,
|
|
7738
|
+
totalOutputTokens: 0
|
|
7739
|
+
};
|
|
7740
|
+
}
|
|
7741
|
+
content = await readFile(sessionPath, "utf-8");
|
|
7662
7742
|
}
|
|
7663
|
-
const content = await readFile(sessionPath, "utf-8");
|
|
7664
7743
|
const { entries, sessionId } = parseSessionEntries(content);
|
|
7665
7744
|
let messageCount = 0;
|
|
7666
7745
|
let userMessages = 0;
|
|
@@ -7696,17 +7775,18 @@ async function readSessionStats(agentId) {
|
|
|
7696
7775
|
function createAgentHistoryRoutes(deps) {
|
|
7697
7776
|
const app2 = new Hono7();
|
|
7698
7777
|
const { connectionManager } = deps;
|
|
7699
|
-
const
|
|
7778
|
+
const resolveOwnership = createOwnershipResolver({
|
|
7700
7779
|
userAgentsStore: deps.userAgentsStore,
|
|
7701
7780
|
agentMetadataStore: deps.agentConfigStore
|
|
7702
7781
|
});
|
|
7703
|
-
async function
|
|
7782
|
+
async function getAuthorizedAgentScope(c) {
|
|
7704
7783
|
const session = await verifySettingsSession(c);
|
|
7705
7784
|
if (!session) return null;
|
|
7706
7785
|
const agentId = c.req.param("agentId") || session.agentId || null;
|
|
7707
7786
|
if (!agentId || !isSafeAgentId(agentId)) return null;
|
|
7708
|
-
const
|
|
7709
|
-
|
|
7787
|
+
const result = await resolveOwnership(session, agentId);
|
|
7788
|
+
if (!result.authorized) return null;
|
|
7789
|
+
return { agentId, organizationId: result.organizationId };
|
|
7710
7790
|
}
|
|
7711
7791
|
async function resolveActiveAgent(agentId) {
|
|
7712
7792
|
if (connectionManager.getDeploymentsForAgent(agentId).length > 0) {
|
|
@@ -7739,10 +7819,21 @@ function createAgentHistoryRoutes(deps) {
|
|
|
7739
7819
|
}
|
|
7740
7820
|
}
|
|
7741
7821
|
app2.get("/status", async (c) => {
|
|
7742
|
-
const
|
|
7743
|
-
if (!
|
|
7744
|
-
const { connected, resolvedAgentId } = await resolveActiveAgent(
|
|
7745
|
-
|
|
7822
|
+
const scope = await getAuthorizedAgentScope(c);
|
|
7823
|
+
if (!scope) return errorResponse(c, "Unauthorized", 401);
|
|
7824
|
+
const { connected, resolvedAgentId } = await resolveActiveAgent(
|
|
7825
|
+
scope.agentId
|
|
7826
|
+
);
|
|
7827
|
+
let hasSessionFile = false;
|
|
7828
|
+
if (process.env.LOBU_SESSION_STORE !== "file") {
|
|
7829
|
+
hasSessionFile = await readLatestSnapshotJsonl(
|
|
7830
|
+
resolvedAgentId,
|
|
7831
|
+
scope.organizationId
|
|
7832
|
+
) !== null;
|
|
7833
|
+
}
|
|
7834
|
+
if (!hasSessionFile) {
|
|
7835
|
+
hasSessionFile = !!await findSessionFile(resolvedAgentId);
|
|
7836
|
+
}
|
|
7746
7837
|
return c.json({
|
|
7747
7838
|
connected: connected || hasSessionFile,
|
|
7748
7839
|
hasHttpServer: !!connectionManager.getHttpUrl(resolvedAgentId),
|
|
@@ -7750,14 +7841,14 @@ function createAgentHistoryRoutes(deps) {
|
|
|
7750
7841
|
});
|
|
7751
7842
|
});
|
|
7752
7843
|
app2.get("/session/messages", async (c) => {
|
|
7753
|
-
const
|
|
7754
|
-
if (!
|
|
7844
|
+
const scope = await getAuthorizedAgentScope(c);
|
|
7845
|
+
if (!scope) return errorResponse(c, "Unauthorized", 401);
|
|
7755
7846
|
const cursor = c.req.query("cursor") || "";
|
|
7756
7847
|
const limit = Math.min(parseInt(c.req.query("limit") || "50", 10), 200);
|
|
7757
7848
|
const result = await proxyOrFallback(
|
|
7758
|
-
agentId,
|
|
7849
|
+
scope.agentId,
|
|
7759
7850
|
`/session/messages?cursor=${cursor}&limit=${limit}`,
|
|
7760
|
-
(resolved) => readSessionMessages(resolved, cursor, limit)
|
|
7851
|
+
(resolved) => readSessionMessages(resolved, cursor, limit, scope.organizationId)
|
|
7761
7852
|
);
|
|
7762
7853
|
if (!result) {
|
|
7763
7854
|
return c.json(
|
|
@@ -7774,12 +7865,12 @@ function createAgentHistoryRoutes(deps) {
|
|
|
7774
7865
|
return c.json(result.data);
|
|
7775
7866
|
});
|
|
7776
7867
|
app2.get("/session/stats", async (c) => {
|
|
7777
|
-
const
|
|
7778
|
-
if (!
|
|
7868
|
+
const scope = await getAuthorizedAgentScope(c);
|
|
7869
|
+
if (!scope) return errorResponse(c, "Unauthorized", 401);
|
|
7779
7870
|
const result = await proxyOrFallback(
|
|
7780
|
-
agentId,
|
|
7871
|
+
scope.agentId,
|
|
7781
7872
|
"/session/stats",
|
|
7782
|
-
readSessionStats
|
|
7873
|
+
(resolved) => readSessionStats(resolved, scope.organizationId)
|
|
7783
7874
|
);
|
|
7784
7875
|
if (!result) {
|
|
7785
7876
|
return c.json({ error: "Agent offline", connected: false }, 503);
|
|
@@ -7793,6 +7884,7 @@ var init_agent_history = __esm({
|
|
|
7793
7884
|
"src/gateway/routes/public/agent-history.ts"() {
|
|
7794
7885
|
"use strict";
|
|
7795
7886
|
init_src();
|
|
7887
|
+
init_client();
|
|
7796
7888
|
init_helpers();
|
|
7797
7889
|
init_agent_ownership();
|
|
7798
7890
|
init_settings_auth();
|
|
@@ -9576,8 +9668,10 @@ async function postOAuthCompletionPrompt(params) {
|
|
|
9576
9668
|
{},
|
|
9577
9669
|
agentSettingsStore
|
|
9578
9670
|
);
|
|
9579
|
-
const
|
|
9671
|
+
const instance = connectionId ? chatInstanceManager2?.getInstance(connectionId) : void 0;
|
|
9672
|
+
const conversationState = instance?.conversationState;
|
|
9580
9673
|
const conversationHistory = connectionId && conversationState ? await conversationState.getHistory(connectionId, channelId).catch(() => []) : [];
|
|
9674
|
+
const organizationId = instance?.connection.organizationId ?? void 0;
|
|
9581
9675
|
const scopeSuffix = scope ? ` (granted scopes: ${scope})` : "";
|
|
9582
9676
|
const messageText = `[System] Authentication for "${mcpId}" completed successfully${scopeSuffix}. Retry the user's previous request that required ${mcpId} and report the result \u2014 do not ask for confirmation first.`;
|
|
9583
9677
|
const messageId = randomUUID5();
|
|
@@ -9589,6 +9683,7 @@ async function postOAuthCompletionPrompt(params) {
|
|
|
9589
9683
|
conversationId: conversationId || channelId,
|
|
9590
9684
|
teamId: teamId ?? platform,
|
|
9591
9685
|
agentId,
|
|
9686
|
+
organizationId,
|
|
9592
9687
|
messageId,
|
|
9593
9688
|
messageText,
|
|
9594
9689
|
channelId,
|
|
@@ -12994,7 +13089,6 @@ async function storePendingQuestion(questionId, organizationId, connectionId, ex
|
|
|
12994
13089
|
connection_id = EXCLUDED.connection_id,
|
|
12995
13090
|
expected_user_id = EXCLUDED.expected_user_id,
|
|
12996
13091
|
entry_payload = EXCLUDED.entry_payload,
|
|
12997
|
-
created_at = now(),
|
|
12998
13092
|
claimed_at = NULL
|
|
12999
13093
|
`;
|
|
13000
13094
|
}
|
|
@@ -13013,20 +13107,38 @@ async function claimPendingQuestion(questionId, organizationId, connectionId, ex
|
|
|
13013
13107
|
if (rows.length === 0) return null;
|
|
13014
13108
|
return rows[0].entry_payload ?? null;
|
|
13015
13109
|
}
|
|
13016
|
-
async function sweepStalePendingInteractions(maxAgeMs = 24 * 60 * 60 * 1e3) {
|
|
13110
|
+
async function sweepStalePendingInteractions(maxAgeMs = 24 * 60 * 60 * 1e3, limit = DEFAULT_SWEEP_LIMIT) {
|
|
13017
13111
|
const sql = getDb();
|
|
13018
13112
|
const cutoff = new Date(Date.now() - maxAgeMs);
|
|
13019
13113
|
const rows = await sql`
|
|
13020
13114
|
DELETE FROM pending_interactions
|
|
13021
|
-
WHERE
|
|
13115
|
+
WHERE id IN (
|
|
13116
|
+
SELECT id FROM pending_interactions
|
|
13117
|
+
WHERE created_at < ${cutoff}
|
|
13118
|
+
LIMIT ${limit}
|
|
13119
|
+
)
|
|
13022
13120
|
RETURNING id
|
|
13023
13121
|
`;
|
|
13024
13122
|
return rows.map((r) => r.id);
|
|
13025
13123
|
}
|
|
13124
|
+
async function deletePendingQuestion(questionId, organizationId, connectionId, expectedUserId) {
|
|
13125
|
+
const sql = getDb();
|
|
13126
|
+
const rows = await sql`
|
|
13127
|
+
DELETE FROM pending_interactions
|
|
13128
|
+
WHERE id = ${questionId}
|
|
13129
|
+
AND organization_id = ${organizationId}
|
|
13130
|
+
AND connection_id = ${connectionId}
|
|
13131
|
+
AND expected_user_id = ${expectedUserId}
|
|
13132
|
+
RETURNING id
|
|
13133
|
+
`;
|
|
13134
|
+
return rows.length > 0;
|
|
13135
|
+
}
|
|
13136
|
+
var DEFAULT_SWEEP_LIMIT;
|
|
13026
13137
|
var init_pending_interaction_store = __esm({
|
|
13027
13138
|
"src/gateway/connections/pending-interaction-store.ts"() {
|
|
13028
13139
|
"use strict";
|
|
13029
13140
|
init_client();
|
|
13141
|
+
DEFAULT_SWEEP_LIMIT = 1e3;
|
|
13030
13142
|
}
|
|
13031
13143
|
});
|
|
13032
13144
|
|
|
@@ -13131,34 +13243,14 @@ function registerInteractionBridge(interactionService, manager, connection, chat
|
|
|
13131
13243
|
const PENDING_SENT_SWEEP_INTERVAL_MS = 60 * 60 * 1e3;
|
|
13132
13244
|
const pendingSentMessages = /* @__PURE__ */ new Map();
|
|
13133
13245
|
const pendingSentSweepTimer = setInterval(() => {
|
|
13134
|
-
sweepPendingSent().catch((error) => {
|
|
13135
|
-
logger45.warn(
|
|
13136
|
-
{ connectionId, error: String(error) },
|
|
13137
|
-
"pendingSentMessages sweep failed"
|
|
13138
|
-
);
|
|
13139
|
-
});
|
|
13140
|
-
}, PENDING_SENT_SWEEP_INTERVAL_MS);
|
|
13141
|
-
pendingSentSweepTimer.unref?.();
|
|
13142
|
-
async function sweepPendingSent() {
|
|
13143
13246
|
const ttlCutoff = Date.now() - PENDING_SENT_TTL_MS;
|
|
13144
13247
|
for (const [id, entry] of pendingSentMessages) {
|
|
13145
13248
|
if (entry.registeredAt <= ttlCutoff) {
|
|
13146
13249
|
pendingSentMessages.delete(id);
|
|
13147
13250
|
}
|
|
13148
13251
|
}
|
|
13149
|
-
|
|
13150
|
-
|
|
13151
|
-
deletedIds = await sweepStalePendingInteractions();
|
|
13152
|
-
} catch (error) {
|
|
13153
|
-
logger45.debug(
|
|
13154
|
-
{ connectionId, error: String(error) },
|
|
13155
|
-
"sweepStalePendingInteractions failed during local sweep"
|
|
13156
|
-
);
|
|
13157
|
-
}
|
|
13158
|
-
for (const id of deletedIds) {
|
|
13159
|
-
pendingSentMessages.delete(id);
|
|
13160
|
-
}
|
|
13161
|
-
}
|
|
13252
|
+
}, PENDING_SENT_SWEEP_INTERVAL_MS);
|
|
13253
|
+
pendingSentSweepTimer.unref?.();
|
|
13162
13254
|
function rememberSentMessage(questionId, sent) {
|
|
13163
13255
|
if (!sent) return;
|
|
13164
13256
|
pendingSentMessages.set(questionId, {
|
|
@@ -13247,7 +13339,7 @@ ${event.options.map((o, i) => `${i + 1}. ${o}`).join("\n")}`;
|
|
|
13247
13339
|
);
|
|
13248
13340
|
if (!sent) {
|
|
13249
13341
|
try {
|
|
13250
|
-
await
|
|
13342
|
+
await deletePendingQuestion(
|
|
13251
13343
|
event.id,
|
|
13252
13344
|
organizationId,
|
|
13253
13345
|
connectionId,
|
|
@@ -14593,6 +14685,7 @@ var init_message_handler_bridge = __esm({
|
|
|
14593
14685
|
conversationId,
|
|
14594
14686
|
teamId: teamId || platform,
|
|
14595
14687
|
agentId,
|
|
14688
|
+
organizationId: this.connection.organizationId,
|
|
14596
14689
|
messageId,
|
|
14597
14690
|
messageText: value,
|
|
14598
14691
|
channelId,
|
|
@@ -14791,15 +14884,47 @@ var init_chat_instance_manager = __esm({
|
|
|
14791
14884
|
continue;
|
|
14792
14885
|
}
|
|
14793
14886
|
try {
|
|
14794
|
-
if (connection.status === "active") {
|
|
14887
|
+
if (connection.status === "active" || connection.status === "error") {
|
|
14795
14888
|
await this.startInstance(connection);
|
|
14889
|
+
if (connection.status === "error") {
|
|
14890
|
+
const recoveryOrgId = connection.organizationId;
|
|
14891
|
+
const clearError = () => this.connectionStore.updateConnection(connection.id, {
|
|
14892
|
+
status: "active",
|
|
14893
|
+
errorMessage: void 0
|
|
14894
|
+
});
|
|
14895
|
+
if (recoveryOrgId) {
|
|
14896
|
+
await orgContext.run(
|
|
14897
|
+
{ organizationId: recoveryOrgId },
|
|
14898
|
+
clearError
|
|
14899
|
+
);
|
|
14900
|
+
} else {
|
|
14901
|
+
await clearError();
|
|
14902
|
+
}
|
|
14903
|
+
logger47.info(
|
|
14904
|
+
{ id: connection.id },
|
|
14905
|
+
"Recovered previously-errored connection"
|
|
14906
|
+
);
|
|
14907
|
+
}
|
|
14796
14908
|
}
|
|
14797
14909
|
} catch (error) {
|
|
14798
14910
|
logger47.error({ id: connection.id, error: String(error) }, "Failed to load connection");
|
|
14799
|
-
|
|
14911
|
+
const errOrgId = connection.organizationId;
|
|
14912
|
+
const markErrored = () => this.connectionStore.updateConnection(connection.id, {
|
|
14800
14913
|
status: "error",
|
|
14801
14914
|
errorMessage: `Startup failed: ${error instanceof Error ? error.message : String(error)}`
|
|
14802
14915
|
});
|
|
14916
|
+
try {
|
|
14917
|
+
if (errOrgId) {
|
|
14918
|
+
await orgContext.run({ organizationId: errOrgId }, markErrored);
|
|
14919
|
+
} else {
|
|
14920
|
+
await markErrored();
|
|
14921
|
+
}
|
|
14922
|
+
} catch (markErr) {
|
|
14923
|
+
logger47.error(
|
|
14924
|
+
{ id: connection.id, error: String(markErr) },
|
|
14925
|
+
"Failed to mark connection as errored"
|
|
14926
|
+
);
|
|
14927
|
+
}
|
|
14803
14928
|
}
|
|
14804
14929
|
}
|
|
14805
14930
|
}
|
|
@@ -15054,8 +15179,7 @@ var init_chat_instance_manager = __esm({
|
|
|
15054
15179
|
return instance;
|
|
15055
15180
|
}
|
|
15056
15181
|
async startInstance(connection) {
|
|
15057
|
-
|
|
15058
|
-
if (!callerOrgId && connection.organizationId) {
|
|
15182
|
+
if (connection.organizationId) {
|
|
15059
15183
|
return orgContext.run(
|
|
15060
15184
|
{ organizationId: connection.organizationId },
|
|
15061
15185
|
() => this.startInstanceUnscoped(connection)
|
|
@@ -15693,6 +15817,7 @@ var init_chat_instance_manager = __esm({
|
|
|
15693
15817
|
channelId: options.channelId,
|
|
15694
15818
|
teamId: options.teamId,
|
|
15695
15819
|
agentId: options.agentId,
|
|
15820
|
+
organizationId: connection.organizationId,
|
|
15696
15821
|
botId: `${name}-platform`,
|
|
15697
15822
|
platform: name,
|
|
15698
15823
|
messageText: message,
|
|
@@ -16139,6 +16264,7 @@ var init_chat_response_bridge = __esm({
|
|
|
16139
16264
|
"src/gateway/connections/chat-response-bridge.ts"() {
|
|
16140
16265
|
"use strict";
|
|
16141
16266
|
init_src();
|
|
16267
|
+
init_client();
|
|
16142
16268
|
init_link_buttons();
|
|
16143
16269
|
init_platform_strategies();
|
|
16144
16270
|
logger49 = createLogger("chat-response-bridge");
|
|
@@ -16276,6 +16402,31 @@ var init_chat_response_bridge = __esm({
|
|
|
16276
16402
|
"No session file to delete on reset"
|
|
16277
16403
|
);
|
|
16278
16404
|
}
|
|
16405
|
+
if (process.env.LOBU_SESSION_STORE !== "file") {
|
|
16406
|
+
try {
|
|
16407
|
+
const sql = getDb();
|
|
16408
|
+
const deleted = await sql`
|
|
16409
|
+
DELETE FROM public.agent_transcript_snapshot s
|
|
16410
|
+
USING public.agents a
|
|
16411
|
+
WHERE s.agent_id = ${agentId}
|
|
16412
|
+
AND s.conversation_id = ${payload.conversationId}
|
|
16413
|
+
AND a.id = s.agent_id
|
|
16414
|
+
AND a.organization_id = s.organization_id
|
|
16415
|
+
RETURNING s.id
|
|
16416
|
+
`;
|
|
16417
|
+
if (deleted.length > 0) {
|
|
16418
|
+
logger49.info(
|
|
16419
|
+
{ agentId, conversationId: payload.conversationId, count: deleted.length },
|
|
16420
|
+
"Purged agent_transcript_snapshot rows for session reset"
|
|
16421
|
+
);
|
|
16422
|
+
}
|
|
16423
|
+
} catch (error) {
|
|
16424
|
+
logger49.warn(
|
|
16425
|
+
{ agentId, conversationId: payload.conversationId, error: String(error) },
|
|
16426
|
+
"Failed to purge transcript snapshots on session reset (next boot may rehydrate stale history)"
|
|
16427
|
+
);
|
|
16428
|
+
}
|
|
16429
|
+
}
|
|
16279
16430
|
}
|
|
16280
16431
|
}
|
|
16281
16432
|
logger49.info(
|
|
@@ -20465,7 +20616,8 @@ var init_runs_queue = __esm({
|
|
|
20465
20616
|
const runAtSql = delayMs > 0 ? `now() + ${Number(delayMs) / 1e3}::float * interval '1 second'` : "now()";
|
|
20466
20617
|
const expiresAtSql = expireInSeconds && expireInSeconds > 0 ? `now() + ${Number(expireInSeconds)}::int * interval '1 second'` : "NULL";
|
|
20467
20618
|
const sql = getDb();
|
|
20468
|
-
const actionInput =
|
|
20619
|
+
const actionInput = sql.json(data ?? {});
|
|
20620
|
+
const organizationIdFromPayload = typeof data?.organizationId === "string" && data.organizationId.length > 0 ? data.organizationId : null;
|
|
20469
20621
|
const id = await sql.begin(async (tx) => {
|
|
20470
20622
|
const result = await tx.unsafe(
|
|
20471
20623
|
`INSERT INTO public.runs (
|
|
@@ -20480,9 +20632,10 @@ var init_runs_queue = __esm({
|
|
|
20480
20632
|
run_at,
|
|
20481
20633
|
priority,
|
|
20482
20634
|
expires_at,
|
|
20483
|
-
retry_delay_seconds
|
|
20635
|
+
retry_delay_seconds,
|
|
20636
|
+
organization_id
|
|
20484
20637
|
) VALUES (
|
|
20485
|
-
$1, $2, $3, $4
|
|
20638
|
+
$1, $2, $3, $4, $5, $6, 0, 'pending', ${runAtSql}, $7, ${expiresAtSql}, $8, $9
|
|
20486
20639
|
)
|
|
20487
20640
|
ON CONFLICT (idempotency_key)
|
|
20488
20641
|
WHERE idempotency_key IS NOT NULL
|
|
@@ -20497,7 +20650,8 @@ var init_runs_queue = __esm({
|
|
|
20497
20650
|
idempotencyKey,
|
|
20498
20651
|
maxAttempts,
|
|
20499
20652
|
priority,
|
|
20500
|
-
retryDelaySeconds
|
|
20653
|
+
retryDelaySeconds,
|
|
20654
|
+
organizationIdFromPayload
|
|
20501
20655
|
]
|
|
20502
20656
|
);
|
|
20503
20657
|
if (result.length === 0 && idempotencyKey) {
|
|
@@ -21411,6 +21565,34 @@ var init_user_agents_store = __esm({
|
|
|
21411
21565
|
const agents = await this.listAgents(platform, userId, organizationId);
|
|
21412
21566
|
return agents.includes(agentId);
|
|
21413
21567
|
}
|
|
21568
|
+
/**
|
|
21569
|
+
* Resolve the orgs in which `(platform, userId)` owns `agentId`.
|
|
21570
|
+
*
|
|
21571
|
+
* Reads `agent_users` directly, which IS the per-org owner mapping —
|
|
21572
|
+
* unlike `agents.{owner_platform, owner_user_id}` (used by the prior
|
|
21573
|
+
* `resolveAuthorizedOrgId` in agent-ownership.ts) those columns are
|
|
21574
|
+
* legacy and unique on `(owner_platform, owner_user_id, id)` only by
|
|
21575
|
+
* convention; they can return the wrong org when the same human owns
|
|
21576
|
+
* the same agentId across two orgs. The authoritative mapping for
|
|
21577
|
+
* "this user is allowed to read this agent's snapshot in org X" lives
|
|
21578
|
+
* here. Codex round 2 finding B on PR #865.
|
|
21579
|
+
*
|
|
21580
|
+
* Returns an empty array if the user owns no instance of `agentId`.
|
|
21581
|
+
* Typically returns 1 element; >1 means the same user has the same
|
|
21582
|
+
* agentId in multiple orgs (rare but legal).
|
|
21583
|
+
*/
|
|
21584
|
+
async findAgentOrganizations(platform, userId, agentId) {
|
|
21585
|
+
const sql = getDb();
|
|
21586
|
+
const rows = await sql`
|
|
21587
|
+
SELECT organization_id
|
|
21588
|
+
FROM agent_users
|
|
21589
|
+
WHERE platform = ${platform}
|
|
21590
|
+
AND user_id = ${userId}
|
|
21591
|
+
AND agent_id = ${agentId}
|
|
21592
|
+
ORDER BY organization_id
|
|
21593
|
+
`;
|
|
21594
|
+
return rows.map((r) => r.organization_id);
|
|
21595
|
+
}
|
|
21414
21596
|
};
|
|
21415
21597
|
}
|
|
21416
21598
|
});
|
|
@@ -22270,10 +22452,183 @@ var init_job_router = __esm({
|
|
|
22270
22452
|
}
|
|
22271
22453
|
});
|
|
22272
22454
|
|
|
22273
|
-
// src/gateway/gateway/
|
|
22455
|
+
// src/gateway/gateway/transcript-routes.ts
|
|
22274
22456
|
import { Hono as Hono18 } from "hono";
|
|
22457
|
+
function authenticate(c) {
|
|
22458
|
+
const authHeader = c.req.header("authorization");
|
|
22459
|
+
if (!authHeader?.startsWith("Bearer ")) return null;
|
|
22460
|
+
const token = authHeader.substring(7);
|
|
22461
|
+
return verifyWorkerToken(token);
|
|
22462
|
+
}
|
|
22463
|
+
async function isRunOwnedByJwtScope(runId, organizationId, agentId, conversationId) {
|
|
22464
|
+
const sql = getDb();
|
|
22465
|
+
const rows = await sql`
|
|
22466
|
+
SELECT 1 AS ok FROM public.runs
|
|
22467
|
+
WHERE id = ${runId}
|
|
22468
|
+
AND organization_id = ${organizationId}
|
|
22469
|
+
AND CASE jsonb_typeof(action_input)
|
|
22470
|
+
WHEN 'object' THEN action_input ->> 'agentId'
|
|
22471
|
+
WHEN 'string' THEN (action_input #>> '{}')::jsonb ->> 'agentId'
|
|
22472
|
+
ELSE NULL
|
|
22473
|
+
END = ${agentId}
|
|
22474
|
+
AND CASE jsonb_typeof(action_input)
|
|
22475
|
+
WHEN 'object' THEN action_input ->> 'conversationId'
|
|
22476
|
+
WHEN 'string' THEN (action_input #>> '{}')::jsonb ->> 'conversationId'
|
|
22477
|
+
ELSE NULL
|
|
22478
|
+
END = ${conversationId}
|
|
22479
|
+
LIMIT 1
|
|
22480
|
+
`;
|
|
22481
|
+
return rows.length > 0;
|
|
22482
|
+
}
|
|
22483
|
+
function createTranscriptRoutes() {
|
|
22484
|
+
const app2 = new Hono18();
|
|
22485
|
+
app2.get("/snapshot", async (c) => {
|
|
22486
|
+
const token = authenticate(c);
|
|
22487
|
+
if (!token) return c.json({ error: "Invalid token" }, 401);
|
|
22488
|
+
const { organizationId, agentId, conversationId } = token;
|
|
22489
|
+
if (!organizationId || !agentId || !conversationId) {
|
|
22490
|
+
return c.json({ error: "Token missing required scope" }, 400);
|
|
22491
|
+
}
|
|
22492
|
+
const sql = getDb();
|
|
22493
|
+
const rows = await sql`
|
|
22494
|
+
SELECT snapshot_jsonl
|
|
22495
|
+
FROM public.agent_transcript_snapshot
|
|
22496
|
+
WHERE organization_id = ${organizationId}
|
|
22497
|
+
AND agent_id = ${agentId}
|
|
22498
|
+
AND conversation_id = ${conversationId}
|
|
22499
|
+
AND terminal_status = 'completed'
|
|
22500
|
+
ORDER BY run_id DESC
|
|
22501
|
+
LIMIT 1
|
|
22502
|
+
`;
|
|
22503
|
+
const row = rows[0];
|
|
22504
|
+
if (!row) {
|
|
22505
|
+
return c.json({ error: "No snapshot found" }, 404);
|
|
22506
|
+
}
|
|
22507
|
+
return c.body(row.snapshot_jsonl, 200, {
|
|
22508
|
+
"content-type": "application/x-ndjson; charset=utf-8"
|
|
22509
|
+
});
|
|
22510
|
+
});
|
|
22511
|
+
app2.post("/snapshot", async (c) => {
|
|
22512
|
+
const token = authenticate(c);
|
|
22513
|
+
if (!token) return c.json({ error: "Invalid token" }, 401);
|
|
22514
|
+
const { organizationId, agentId, conversationId } = token;
|
|
22515
|
+
if (!organizationId || !agentId || !conversationId) {
|
|
22516
|
+
return c.json({ error: "Token missing required scope" }, 400);
|
|
22517
|
+
}
|
|
22518
|
+
let body2;
|
|
22519
|
+
try {
|
|
22520
|
+
body2 = await c.req.json();
|
|
22521
|
+
} catch {
|
|
22522
|
+
return c.json({ error: "Invalid JSON body" }, 400);
|
|
22523
|
+
}
|
|
22524
|
+
const terminalStatus = body2.terminalStatus;
|
|
22525
|
+
const snapshotJsonl = body2.snapshotJsonl;
|
|
22526
|
+
if (terminalStatus !== "completed" && terminalStatus !== "failed" && terminalStatus !== "timeout" && terminalStatus !== "cancelled") {
|
|
22527
|
+
return c.json({ error: "Invalid terminalStatus" }, 400);
|
|
22528
|
+
}
|
|
22529
|
+
if (typeof snapshotJsonl !== "string" || snapshotJsonl.length === 0) {
|
|
22530
|
+
return c.json({ error: "Missing snapshotJsonl" }, 400);
|
|
22531
|
+
}
|
|
22532
|
+
const byteSize = Buffer.byteLength(snapshotJsonl, "utf-8");
|
|
22533
|
+
if (byteSize > MAX_SNAPSHOT_BYTES) {
|
|
22534
|
+
logger72.warn(
|
|
22535
|
+
`Rejecting oversize snapshot (${byteSize} > ${MAX_SNAPSHOT_BYTES} bytes) for (${organizationId}, ${agentId}, ${conversationId})`
|
|
22536
|
+
);
|
|
22537
|
+
return c.json({ error: "Snapshot too large" }, 413);
|
|
22538
|
+
}
|
|
22539
|
+
const rawRunId = body2.runId;
|
|
22540
|
+
const runId = typeof rawRunId === "number" && Number.isFinite(rawRunId) && rawRunId > 0 ? rawRunId : null;
|
|
22541
|
+
if (runId === null) {
|
|
22542
|
+
return c.json({ error: "Missing or invalid runId" }, 400);
|
|
22543
|
+
}
|
|
22544
|
+
if (token.runId !== runId) {
|
|
22545
|
+
logger72.warn(
|
|
22546
|
+
`Token runId mismatch: token.runId=${token.runId ?? "<absent>"} body.runId=${runId}; rejecting snapshot`
|
|
22547
|
+
);
|
|
22548
|
+
return c.json({ error: "runId out of scope" }, 403);
|
|
22549
|
+
}
|
|
22550
|
+
if (!await isRunOwnedByJwtScope(
|
|
22551
|
+
runId,
|
|
22552
|
+
organizationId,
|
|
22553
|
+
agentId,
|
|
22554
|
+
conversationId
|
|
22555
|
+
)) {
|
|
22556
|
+
logger72.warn(
|
|
22557
|
+
`Run ${runId} does not belong to (${organizationId}, ${agentId}, ${conversationId}); rejecting snapshot`
|
|
22558
|
+
);
|
|
22559
|
+
return c.json({ error: "runId out of scope" }, 403);
|
|
22560
|
+
}
|
|
22561
|
+
const sql = getDb();
|
|
22562
|
+
try {
|
|
22563
|
+
const inserted = await sql`
|
|
22564
|
+
INSERT INTO public.agent_transcript_snapshot
|
|
22565
|
+
(organization_id, agent_id, conversation_id, run_id,
|
|
22566
|
+
snapshot_jsonl, byte_size, terminal_status)
|
|
22567
|
+
VALUES
|
|
22568
|
+
(${organizationId}, ${agentId}, ${conversationId}, ${runId},
|
|
22569
|
+
${snapshotJsonl}, ${byteSize}, ${terminalStatus})
|
|
22570
|
+
ON CONFLICT (organization_id, agent_id, conversation_id, run_id)
|
|
22571
|
+
DO NOTHING
|
|
22572
|
+
RETURNING id
|
|
22573
|
+
`;
|
|
22574
|
+
if (inserted.length === 0) {
|
|
22575
|
+
return c.json({ error: "Snapshot already exists for run" }, 409);
|
|
22576
|
+
}
|
|
22577
|
+
logger72.info(
|
|
22578
|
+
`Wrote snapshot id=${inserted[0].id} run_id=${runId} byte_size=${byteSize} status=${terminalStatus}`
|
|
22579
|
+
);
|
|
22580
|
+
return c.json({ id: inserted[0].id });
|
|
22581
|
+
} catch (err) {
|
|
22582
|
+
logger72.error(
|
|
22583
|
+
`Snapshot INSERT failed: ${err instanceof Error ? err.message : String(err)}`
|
|
22584
|
+
);
|
|
22585
|
+
return c.json({ error: "Internal error" }, 500);
|
|
22586
|
+
}
|
|
22587
|
+
});
|
|
22588
|
+
app2.delete("/snapshot", async (c) => {
|
|
22589
|
+
const token = authenticate(c);
|
|
22590
|
+
if (!token) return c.json({ error: "Invalid token" }, 401);
|
|
22591
|
+
const { organizationId, agentId, conversationId } = token;
|
|
22592
|
+
if (!organizationId || !agentId || !conversationId) {
|
|
22593
|
+
return c.json({ error: "Token missing required scope" }, 400);
|
|
22594
|
+
}
|
|
22595
|
+
const sql = getDb();
|
|
22596
|
+
try {
|
|
22597
|
+
const deleted = await sql`
|
|
22598
|
+
DELETE FROM public.agent_transcript_snapshot
|
|
22599
|
+
WHERE organization_id = ${organizationId}
|
|
22600
|
+
AND agent_id = ${agentId}
|
|
22601
|
+
AND conversation_id = ${conversationId}
|
|
22602
|
+
RETURNING id
|
|
22603
|
+
`;
|
|
22604
|
+
logger72.info(
|
|
22605
|
+
`Purged ${deleted.length} snapshot row(s) for (${organizationId}, ${agentId}, ${conversationId}) on session reset`
|
|
22606
|
+
);
|
|
22607
|
+
return c.json({ deleted: deleted.length });
|
|
22608
|
+
} catch (err) {
|
|
22609
|
+
logger72.error(
|
|
22610
|
+
`Snapshot DELETE failed: ${err instanceof Error ? err.message : String(err)}`
|
|
22611
|
+
);
|
|
22612
|
+
return c.json({ error: "Internal error" }, 500);
|
|
22613
|
+
}
|
|
22614
|
+
});
|
|
22615
|
+
return app2;
|
|
22616
|
+
}
|
|
22617
|
+
var logger72, MAX_SNAPSHOT_BYTES;
|
|
22618
|
+
var init_transcript_routes = __esm({
|
|
22619
|
+
"src/gateway/gateway/transcript-routes.ts"() {
|
|
22620
|
+
"use strict";
|
|
22621
|
+
init_src();
|
|
22622
|
+
init_client();
|
|
22623
|
+
logger72 = createLogger("worker-transcript");
|
|
22624
|
+
MAX_SNAPSHOT_BYTES = 4 * 1024 * 1024;
|
|
22625
|
+
}
|
|
22626
|
+
});
|
|
22627
|
+
|
|
22628
|
+
// src/gateway/gateway/index.ts
|
|
22629
|
+
import { Hono as Hono19 } from "hono";
|
|
22275
22630
|
import { stream } from "hono/streaming";
|
|
22276
|
-
var
|
|
22631
|
+
var logger73, WorkerGateway;
|
|
22277
22632
|
var init_gateway2 = __esm({
|
|
22278
22633
|
"src/gateway/gateway/index.ts"() {
|
|
22279
22634
|
"use strict";
|
|
@@ -22284,7 +22639,8 @@ var init_gateway2 = __esm({
|
|
|
22284
22639
|
init_model_selection();
|
|
22285
22640
|
init_connection_manager();
|
|
22286
22641
|
init_job_router();
|
|
22287
|
-
|
|
22642
|
+
init_transcript_routes();
|
|
22643
|
+
logger73 = createLogger("worker-gateway");
|
|
22288
22644
|
WorkerGateway = class {
|
|
22289
22645
|
app;
|
|
22290
22646
|
connectionManager;
|
|
@@ -22308,7 +22664,7 @@ var init_gateway2 = __esm({
|
|
|
22308
22664
|
this.providerCatalogService = providerCatalogService;
|
|
22309
22665
|
this.agentSettingsStore = agentSettingsStore;
|
|
22310
22666
|
this.secretStore = secretStore;
|
|
22311
|
-
this.app = new
|
|
22667
|
+
this.app = new Hono19();
|
|
22312
22668
|
this.setupRoutes();
|
|
22313
22669
|
}
|
|
22314
22670
|
/**
|
|
@@ -22333,7 +22689,8 @@ var init_gateway2 = __esm({
|
|
|
22333
22689
|
"/session-context",
|
|
22334
22690
|
(c) => this.handleSessionContextRequest(c)
|
|
22335
22691
|
);
|
|
22336
|
-
|
|
22692
|
+
this.app.route("/transcript", createTranscriptRoutes());
|
|
22693
|
+
logger73.debug("Worker gateway routes registered");
|
|
22337
22694
|
}
|
|
22338
22695
|
async enrichMcpStatus(mcpStatus, agentId, userId) {
|
|
22339
22696
|
const secretStore = this.secretStore;
|
|
@@ -22362,7 +22719,7 @@ var init_gateway2 = __esm({
|
|
|
22362
22719
|
mcp.id
|
|
22363
22720
|
);
|
|
22364
22721
|
} catch (error) {
|
|
22365
|
-
|
|
22722
|
+
logger73.warn("Failed to look up stored MCP credential", {
|
|
22366
22723
|
mcpId: mcp.id,
|
|
22367
22724
|
agentId,
|
|
22368
22725
|
userId,
|
|
@@ -22419,6 +22776,29 @@ var init_gateway2 = __esm({
|
|
|
22419
22776
|
});
|
|
22420
22777
|
}
|
|
22421
22778
|
};
|
|
22779
|
+
let connectionAdded = false;
|
|
22780
|
+
let cleanupRan = false;
|
|
22781
|
+
let aborted = false;
|
|
22782
|
+
const runCleanup = () => {
|
|
22783
|
+
if (cleanupRan) return;
|
|
22784
|
+
cleanupRan = true;
|
|
22785
|
+
aborted = true;
|
|
22786
|
+
if (!connectionAdded) {
|
|
22787
|
+
return;
|
|
22788
|
+
}
|
|
22789
|
+
const current = this.connectionManager.getConnection(deploymentName);
|
|
22790
|
+
if (current && current.writer !== sseWriter) {
|
|
22791
|
+
logger73.debug(
|
|
22792
|
+
`Ignoring stale disconnect for ${deploymentName} (replaced by newer SSE)`
|
|
22793
|
+
);
|
|
22794
|
+
return;
|
|
22795
|
+
}
|
|
22796
|
+
this.jobRouter.pauseWorker(deploymentName).catch((err) => {
|
|
22797
|
+
logger73.error(`Failed to pause worker ${deploymentName}:`, err);
|
|
22798
|
+
});
|
|
22799
|
+
this.connectionManager.removeConnection(deploymentName);
|
|
22800
|
+
};
|
|
22801
|
+
sseWriter.onClose(runCleanup);
|
|
22422
22802
|
const detachAbortBridge = bindRequestAbortToStream(
|
|
22423
22803
|
requestSignal,
|
|
22424
22804
|
streamWriter
|
|
@@ -22428,8 +22808,13 @@ var init_gateway2 = __esm({
|
|
|
22428
22808
|
c.header("Connection", "keep-alive");
|
|
22429
22809
|
c.header("X-Accel-Buffering", "no");
|
|
22430
22810
|
await this.jobRouter.pauseWorker(deploymentName);
|
|
22811
|
+
if (aborted || requestSignal?.aborted) {
|
|
22812
|
+
detachAbortBridge();
|
|
22813
|
+
runCleanup();
|
|
22814
|
+
return;
|
|
22815
|
+
}
|
|
22431
22816
|
if (this.connectionManager.isConnected(deploymentName)) {
|
|
22432
|
-
|
|
22817
|
+
logger73.info(
|
|
22433
22818
|
`Cleaning up stale connection for ${deploymentName} before new SSE`
|
|
22434
22819
|
);
|
|
22435
22820
|
this.connectionManager.removeConnection(deploymentName);
|
|
@@ -22442,21 +22827,14 @@ var init_gateway2 = __esm({
|
|
|
22442
22827
|
sseWriter,
|
|
22443
22828
|
httpPort
|
|
22444
22829
|
);
|
|
22830
|
+
connectionAdded = true;
|
|
22831
|
+
if (aborted || requestSignal?.aborted) {
|
|
22832
|
+
detachAbortBridge();
|
|
22833
|
+
runCleanup();
|
|
22834
|
+
return;
|
|
22835
|
+
}
|
|
22445
22836
|
await this.jobRouter.registerWorker(deploymentName);
|
|
22446
22837
|
await this.jobRouter.resumeWorker(deploymentName);
|
|
22447
|
-
sseWriter.onClose(() => {
|
|
22448
|
-
const current = this.connectionManager.getConnection(deploymentName);
|
|
22449
|
-
if (current && current.writer !== sseWriter) {
|
|
22450
|
-
logger72.debug(
|
|
22451
|
-
`Ignoring stale disconnect for ${deploymentName} (replaced by newer SSE)`
|
|
22452
|
-
);
|
|
22453
|
-
return;
|
|
22454
|
-
}
|
|
22455
|
-
this.jobRouter.pauseWorker(deploymentName).catch((err) => {
|
|
22456
|
-
logger72.error(`Failed to pause worker ${deploymentName}:`, err);
|
|
22457
|
-
});
|
|
22458
|
-
this.connectionManager.removeConnection(deploymentName);
|
|
22459
|
-
});
|
|
22460
22838
|
try {
|
|
22461
22839
|
while (!isClosed) {
|
|
22462
22840
|
await streamWriter.sleep(1e3);
|
|
@@ -22479,36 +22857,37 @@ var init_gateway2 = __esm({
|
|
|
22479
22857
|
try {
|
|
22480
22858
|
const body2 = await c.req.json();
|
|
22481
22859
|
const { jobId, ...responseData } = body2;
|
|
22482
|
-
const
|
|
22483
|
-
|
|
22860
|
+
const orgEnriched = auth.tokenData.organizationId && !responseData.organizationId ? { ...responseData, organizationId: auth.tokenData.organizationId } : responseData;
|
|
22861
|
+
const enrichedResponse = auth.tokenData.connectionId && (!orgEnriched.platformMetadata || typeof orgEnriched.platformMetadata === "object") ? {
|
|
22862
|
+
...orgEnriched,
|
|
22484
22863
|
platformMetadata: {
|
|
22485
|
-
...
|
|
22864
|
+
...orgEnriched.platformMetadata || {},
|
|
22486
22865
|
connectionId: auth.tokenData.connectionId
|
|
22487
22866
|
}
|
|
22488
|
-
} :
|
|
22867
|
+
} : orgEnriched;
|
|
22489
22868
|
if (jobId) {
|
|
22490
22869
|
this.jobRouter.acknowledgeJob(jobId);
|
|
22491
22870
|
}
|
|
22492
22871
|
if (enrichedResponse.received) {
|
|
22493
22872
|
if (enrichedResponse.heartbeat) {
|
|
22494
|
-
|
|
22873
|
+
logger73.debug(
|
|
22495
22874
|
`[WORKER-GATEWAY] Received heartbeat ACK from ${deploymentName}`
|
|
22496
22875
|
);
|
|
22497
22876
|
}
|
|
22498
22877
|
return c.json({ success: true });
|
|
22499
22878
|
}
|
|
22500
|
-
|
|
22879
|
+
logger73.info(
|
|
22501
22880
|
`[WORKER-GATEWAY] Received response with fields: ${Object.keys(enrichedResponse).join(", ")}`
|
|
22502
22881
|
);
|
|
22503
22882
|
if (enrichedResponse.delta) {
|
|
22504
|
-
|
|
22883
|
+
logger73.info(
|
|
22505
22884
|
`[WORKER-GATEWAY] Stream delta: deltaLength=${enrichedResponse.delta.length}`
|
|
22506
22885
|
);
|
|
22507
22886
|
}
|
|
22508
22887
|
await this.queue.send("thread_response", enrichedResponse);
|
|
22509
22888
|
return c.json({ success: true });
|
|
22510
22889
|
} catch (error) {
|
|
22511
|
-
|
|
22890
|
+
logger73.error(`Error handling worker response: ${error}`);
|
|
22512
22891
|
return c.json({ error: "Failed to process response" }, 500);
|
|
22513
22892
|
}
|
|
22514
22893
|
}
|
|
@@ -22597,7 +22976,7 @@ var init_gateway2 = __esm({
|
|
|
22597
22976
|
mcpInstructions[result.value.mcpId] = result.value.instructions;
|
|
22598
22977
|
}
|
|
22599
22978
|
} else {
|
|
22600
|
-
|
|
22979
|
+
logger73.error("MCP tool fetch rejected", {
|
|
22601
22980
|
reason: result.reason instanceof Error ? result.reason.message : String(result.reason)
|
|
22602
22981
|
});
|
|
22603
22982
|
}
|
|
@@ -22625,13 +23004,13 @@ var init_gateway2 = __esm({
|
|
|
22625
23004
|
}
|
|
22626
23005
|
}
|
|
22627
23006
|
} catch (error) {
|
|
22628
|
-
|
|
23007
|
+
logger73.error("Failed to fetch skills config for worker sync", {
|
|
22629
23008
|
error
|
|
22630
23009
|
});
|
|
22631
23010
|
}
|
|
22632
23011
|
}
|
|
22633
23012
|
const mergedSkillsInstructions = contextData.skillsInstructions || "";
|
|
22634
|
-
|
|
23013
|
+
logger73.info(
|
|
22635
23014
|
`Session context for ${userId}: ${Object.keys(mcpConfig.mcpServers || {}).length} MCPs, ${contextData.agentInstructions.length} chars agent instructions, ${contextData.platformInstructions.length} chars platform instructions, ${contextData.networkInstructions.length} chars network instructions, ${mergedSkillsInstructions.length} chars skills instructions, ${enrichedMcpStatus.length} MCP status entries, ${Object.keys(mcpTools).length} MCP tool lists, ${Object.keys(mcpInstructions).length} MCP instructions, ${skillsConfig.length} skills, provider: ${providerConfig.defaultProvider || "none"}`
|
|
22636
23015
|
);
|
|
22637
23016
|
return c.json({
|
|
@@ -22648,7 +23027,7 @@ var init_gateway2 = __esm({
|
|
|
22648
23027
|
skillsConfig
|
|
22649
23028
|
});
|
|
22650
23029
|
} catch (error) {
|
|
22651
|
-
|
|
23030
|
+
logger73.error("Failed to generate session context", { err: error });
|
|
22652
23031
|
return c.json({ error: "session_context_error" }, 500);
|
|
22653
23032
|
}
|
|
22654
23033
|
}
|
|
@@ -22660,11 +23039,11 @@ var init_gateway2 = __esm({
|
|
|
22660
23039
|
const token = authHeader.substring(7);
|
|
22661
23040
|
const tokenData = verifyWorkerToken(token);
|
|
22662
23041
|
if (!tokenData) {
|
|
22663
|
-
|
|
23042
|
+
logger73.warn("Invalid token");
|
|
22664
23043
|
return null;
|
|
22665
23044
|
}
|
|
22666
23045
|
if (tokenData.jti && await getRevokedTokenStore().isRevoked(tokenData.jti)) {
|
|
22667
|
-
|
|
23046
|
+
logger73.warn("Revoked worker token");
|
|
22668
23047
|
return null;
|
|
22669
23048
|
}
|
|
22670
23049
|
return { tokenData, token };
|
|
@@ -22780,12 +23159,12 @@ var init_gateway2 = __esm({
|
|
|
22780
23159
|
});
|
|
22781
23160
|
|
|
22782
23161
|
// src/gateway/infrastructure/queue/queue-producer.ts
|
|
22783
|
-
var
|
|
23162
|
+
var logger74, QueueProducer;
|
|
22784
23163
|
var init_queue_producer = __esm({
|
|
22785
23164
|
"src/gateway/infrastructure/queue/queue-producer.ts"() {
|
|
22786
23165
|
"use strict";
|
|
22787
23166
|
init_src();
|
|
22788
|
-
|
|
23167
|
+
logger74 = createLogger("queue-producer");
|
|
22789
23168
|
QueueProducer = class {
|
|
22790
23169
|
queue;
|
|
22791
23170
|
isInitialized = false;
|
|
@@ -22800,9 +23179,9 @@ var init_queue_producer = __esm({
|
|
|
22800
23179
|
try {
|
|
22801
23180
|
await this.queue.createQueue("messages");
|
|
22802
23181
|
this.isInitialized = true;
|
|
22803
|
-
|
|
23182
|
+
logger74.debug("Queue producer initialized");
|
|
22804
23183
|
} catch (error) {
|
|
22805
|
-
|
|
23184
|
+
logger74.error("Failed to initialize queue producer:", error);
|
|
22806
23185
|
throw error;
|
|
22807
23186
|
}
|
|
22808
23187
|
}
|
|
@@ -22811,7 +23190,7 @@ var init_queue_producer = __esm({
|
|
|
22811
23190
|
*/
|
|
22812
23191
|
async stop() {
|
|
22813
23192
|
this.isInitialized = false;
|
|
22814
|
-
|
|
23193
|
+
logger74.debug("Queue producer stopped");
|
|
22815
23194
|
}
|
|
22816
23195
|
/**
|
|
22817
23196
|
* Enqueue any message (direct or thread) to the single 'messages' queue
|
|
@@ -22832,12 +23211,12 @@ var init_queue_producer = __esm({
|
|
|
22832
23211
|
singletonKey: rawSingletonKey.replace(/:/g, "-")
|
|
22833
23212
|
// Prevent duplicates within canonical conversation identity
|
|
22834
23213
|
});
|
|
22835
|
-
|
|
23214
|
+
logger74.info(
|
|
22836
23215
|
`Enqueued message job ${jobId} for user ${payload.userId}, conversation ${payload.conversationId}`
|
|
22837
23216
|
);
|
|
22838
23217
|
return jobId || "job-sent";
|
|
22839
23218
|
} catch (error) {
|
|
22840
|
-
|
|
23219
|
+
logger74.error(
|
|
22841
23220
|
`Failed to enqueue message for user ${payload.userId}:`,
|
|
22842
23221
|
error
|
|
22843
23222
|
);
|
|
@@ -22886,12 +23265,12 @@ function assertConnectionId(connectionId, kind) {
|
|
|
22886
23265
|
);
|
|
22887
23266
|
}
|
|
22888
23267
|
}
|
|
22889
|
-
var
|
|
23268
|
+
var logger75, SAFE_LINK_BUTTON_SCHEMES, InteractionService;
|
|
22890
23269
|
var init_interactions2 = __esm({
|
|
22891
23270
|
"src/gateway/interactions.ts"() {
|
|
22892
23271
|
"use strict";
|
|
22893
23272
|
init_src();
|
|
22894
|
-
|
|
23273
|
+
logger75 = createLogger("interactions");
|
|
22895
23274
|
SAFE_LINK_BUTTON_SCHEMES = /* @__PURE__ */ new Set(["http:", "https:"]);
|
|
22896
23275
|
InteractionService = class extends EventEmitter {
|
|
22897
23276
|
beforeCreateHook;
|
|
@@ -22922,7 +23301,7 @@ var init_interactions2 = __esm({
|
|
|
22922
23301
|
question,
|
|
22923
23302
|
options
|
|
22924
23303
|
};
|
|
22925
|
-
|
|
23304
|
+
logger75.info(
|
|
22926
23305
|
`Posted question ${posted.id} for conversation ${conversationId}`
|
|
22927
23306
|
);
|
|
22928
23307
|
this.emit("question:created", posted);
|
|
@@ -22955,7 +23334,7 @@ var init_interactions2 = __esm({
|
|
|
22955
23334
|
args,
|
|
22956
23335
|
grantPattern
|
|
22957
23336
|
};
|
|
22958
|
-
|
|
23337
|
+
logger75.info(
|
|
22959
23338
|
`Posted tool approval ${posted.id} for ${mcpId}/${toolName} agent=${agentId}`
|
|
22960
23339
|
);
|
|
22961
23340
|
this.emit("tool:approval-needed", posted);
|
|
@@ -22984,7 +23363,7 @@ var init_interactions2 = __esm({
|
|
|
22984
23363
|
body: body2,
|
|
22985
23364
|
linkType
|
|
22986
23365
|
};
|
|
22987
|
-
|
|
23366
|
+
logger75.info(
|
|
22988
23367
|
`Posted link button ${posted.id} for conversation ${conversationId} (${linkType})`
|
|
22989
23368
|
);
|
|
22990
23369
|
this.emit("link-button:created", posted);
|
|
@@ -23025,7 +23404,7 @@ var init_interactions2 = __esm({
|
|
|
23025
23404
|
platform,
|
|
23026
23405
|
text
|
|
23027
23406
|
};
|
|
23028
|
-
|
|
23407
|
+
logger75.info(
|
|
23029
23408
|
`Posted status message ${posted.id} for conversation ${conversationId}`
|
|
23030
23409
|
);
|
|
23031
23410
|
this.emit("status-message:created", posted);
|
|
@@ -23045,7 +23424,7 @@ var init_interactions2 = __esm({
|
|
|
23045
23424
|
blocking: false,
|
|
23046
23425
|
prompts
|
|
23047
23426
|
};
|
|
23048
|
-
|
|
23427
|
+
logger75.info(
|
|
23049
23428
|
`Created suggestion ${suggestion.id} for conversation ${conversationId}`
|
|
23050
23429
|
);
|
|
23051
23430
|
this.emit("suggestion:created", suggestion);
|
|
@@ -23116,12 +23495,12 @@ function findMatchingRule(hostname, rules) {
|
|
|
23116
23495
|
function hashPolicy(organizationId, agentId, judgeName, policy) {
|
|
23117
23496
|
return crypto3.createHash("sha256").update(`${organizationId} ${agentId} ${judgeName} ${policy}`).digest("hex").slice(0, 16);
|
|
23118
23497
|
}
|
|
23119
|
-
var
|
|
23498
|
+
var logger76, PolicyStore;
|
|
23120
23499
|
var init_policy_store = __esm({
|
|
23121
23500
|
"src/gateway/permissions/policy-store.ts"() {
|
|
23122
23501
|
"use strict";
|
|
23123
23502
|
init_src();
|
|
23124
|
-
|
|
23503
|
+
logger76 = createLogger("policy-store");
|
|
23125
23504
|
PolicyStore = class _PolicyStore {
|
|
23126
23505
|
policies = /* @__PURE__ */ new Map();
|
|
23127
23506
|
static composeKey(organizationId, agentId) {
|
|
@@ -23130,7 +23509,7 @@ var init_policy_store = __esm({
|
|
|
23130
23509
|
set(organizationId, agentId, bundle) {
|
|
23131
23510
|
const prepared = prepareBundle(organizationId, agentId, bundle);
|
|
23132
23511
|
this.policies.set(_PolicyStore.composeKey(organizationId, agentId), prepared);
|
|
23133
|
-
|
|
23512
|
+
logger76.debug("Set egress policy bundle", {
|
|
23134
23513
|
organizationId,
|
|
23135
23514
|
agentId,
|
|
23136
23515
|
domains: prepared.judgedDomains.length,
|
|
@@ -23161,7 +23540,7 @@ var init_policy_store = __esm({
|
|
|
23161
23540
|
const judgeName = matched.judge ?? "default";
|
|
23162
23541
|
const judge = prepared.preparedJudges[judgeName];
|
|
23163
23542
|
if (!judge) {
|
|
23164
|
-
|
|
23543
|
+
logger76.warn(
|
|
23165
23544
|
"Judge rule matched but named policy not found \u2014 failing closed",
|
|
23166
23545
|
{ organizationId, agentId, hostname, judgeName }
|
|
23167
23546
|
);
|
|
@@ -23179,7 +23558,7 @@ var init_policy_store = __esm({
|
|
|
23179
23558
|
});
|
|
23180
23559
|
|
|
23181
23560
|
// src/gateway/proxy/secret-proxy.ts
|
|
23182
|
-
import { Hono as
|
|
23561
|
+
import { Hono as Hono20 } from "hono";
|
|
23183
23562
|
function defaultPlaceholderTtlSeconds() {
|
|
23184
23563
|
const raw = process.env.SECRET_PLACEHOLDER_TTL_MS;
|
|
23185
23564
|
if (raw) {
|
|
@@ -23194,8 +23573,8 @@ function lookupPlaceholderMapping(placeholder, expectedOrganizationId) {
|
|
|
23194
23573
|
const uuid = placeholder.slice(prefixIdx + PLACEHOLDER_PREFIX.length);
|
|
23195
23574
|
const mapping = placeholderCache.get(uuid);
|
|
23196
23575
|
if (!mapping) return null;
|
|
23197
|
-
if (expectedOrganizationId
|
|
23198
|
-
|
|
23576
|
+
if (expectedOrganizationId !== void 0 && mapping.organizationId !== expectedOrganizationId) {
|
|
23577
|
+
logger77.warn(
|
|
23199
23578
|
{
|
|
23200
23579
|
mappingAgentId: mapping.agentId,
|
|
23201
23580
|
mappingOrg: mapping.organizationId,
|
|
@@ -23205,6 +23584,15 @@ function lookupPlaceholderMapping(placeholder, expectedOrganizationId) {
|
|
|
23205
23584
|
);
|
|
23206
23585
|
return null;
|
|
23207
23586
|
}
|
|
23587
|
+
if (!mapping.organizationId) {
|
|
23588
|
+
logger77.warn(
|
|
23589
|
+
{
|
|
23590
|
+
mappingAgentId: mapping.agentId,
|
|
23591
|
+
expectedOrg: expectedOrganizationId
|
|
23592
|
+
},
|
|
23593
|
+
"Placeholder mapping accessed without organizationId \u2014 legacy row, schedule rotation"
|
|
23594
|
+
);
|
|
23595
|
+
}
|
|
23208
23596
|
return mapping;
|
|
23209
23597
|
}
|
|
23210
23598
|
function safeDecodePathSegment(value) {
|
|
@@ -23236,13 +23624,14 @@ function generatePlaceholder(agentId, envVarName, secretRef, deploymentName, opt
|
|
|
23236
23624
|
);
|
|
23237
23625
|
return `${PLACEHOLDER_PREFIX}${uuid}`;
|
|
23238
23626
|
}
|
|
23239
|
-
var
|
|
23627
|
+
var logger77, PLACEHOLDER_PREFIX, RESOLVE_FAILURE_THRESHOLD, RESOLVE_FAILURE_WINDOW_MS, ResolutionFailureLimiter, resolutionFailureLimiter, PlaceholderCache, placeholderCache, SecretProxy;
|
|
23240
23628
|
var init_secret_proxy = __esm({
|
|
23241
23629
|
"src/gateway/proxy/secret-proxy.ts"() {
|
|
23242
23630
|
"use strict";
|
|
23243
23631
|
init_src();
|
|
23632
|
+
init_org_context();
|
|
23244
23633
|
init_rate_limiter();
|
|
23245
|
-
|
|
23634
|
+
logger77 = createLogger("secret-proxy");
|
|
23246
23635
|
PLACEHOLDER_PREFIX = "lobu_secret_";
|
|
23247
23636
|
RESOLVE_FAILURE_THRESHOLD = 20;
|
|
23248
23637
|
RESOLVE_FAILURE_WINDOW_MS = 5 * 60 * 1e3;
|
|
@@ -23352,11 +23741,11 @@ var init_secret_proxy = __esm({
|
|
|
23352
23741
|
this.slugMap = /* @__PURE__ */ new Map();
|
|
23353
23742
|
for (const upstream of config.providerUpstreams ?? []) {
|
|
23354
23743
|
this.slugMap.set(upstream.slug, upstream.upstreamBaseUrl);
|
|
23355
|
-
|
|
23744
|
+
logger77.debug(
|
|
23356
23745
|
`Registered provider upstream: ${upstream.slug} -> ${upstream.upstreamBaseUrl}`
|
|
23357
23746
|
);
|
|
23358
23747
|
}
|
|
23359
|
-
this.app = new
|
|
23748
|
+
this.app = new Hono20();
|
|
23360
23749
|
this.setupRoutes();
|
|
23361
23750
|
}
|
|
23362
23751
|
setAuthProfilesManager(manager) {
|
|
@@ -23387,7 +23776,7 @@ var init_secret_proxy = __esm({
|
|
|
23387
23776
|
if (providerId) {
|
|
23388
23777
|
this.slugToProviderId.set(upstream.slug, providerId);
|
|
23389
23778
|
}
|
|
23390
|
-
|
|
23779
|
+
logger77.debug(
|
|
23391
23780
|
`Registered provider upstream: ${upstream.slug} -> ${upstream.upstreamBaseUrl}${providerId ? ` (providerId: ${providerId})` : ""}`
|
|
23392
23781
|
);
|
|
23393
23782
|
}
|
|
@@ -23409,7 +23798,7 @@ var init_secret_proxy = __esm({
|
|
|
23409
23798
|
try {
|
|
23410
23799
|
return await this.forward(c);
|
|
23411
23800
|
} catch (error) {
|
|
23412
|
-
|
|
23801
|
+
logger77.error("Secret proxy error:", error);
|
|
23413
23802
|
return c.json({ error: "Internal proxy error" }, 500);
|
|
23414
23803
|
}
|
|
23415
23804
|
}
|
|
@@ -23505,12 +23894,12 @@ var init_secret_proxy = __esm({
|
|
|
23505
23894
|
const { shouldLog, nowThrottled } = resolutionFailureLimiter.recordFailure(source);
|
|
23506
23895
|
if (shouldLog) {
|
|
23507
23896
|
if (nowThrottled) {
|
|
23508
|
-
|
|
23897
|
+
logger77.warn(
|
|
23509
23898
|
{ source },
|
|
23510
23899
|
"Throttling placeholder resolution for source after repeated failures"
|
|
23511
23900
|
);
|
|
23512
23901
|
} else {
|
|
23513
|
-
|
|
23902
|
+
logger77.warn({ source }, "Failed to resolve secret placeholder");
|
|
23514
23903
|
}
|
|
23515
23904
|
}
|
|
23516
23905
|
return "";
|
|
@@ -23561,9 +23950,13 @@ var init_secret_proxy = __esm({
|
|
|
23561
23950
|
const orgId = await this.agentOrgResolver(urlAgentId);
|
|
23562
23951
|
if (orgId) expectedOrganizationId = orgId;
|
|
23563
23952
|
} catch (err) {
|
|
23564
|
-
|
|
23953
|
+
logger77.error(
|
|
23565
23954
|
{ urlAgentId, err: String(err) },
|
|
23566
|
-
"agentOrgResolver failed \u2014
|
|
23955
|
+
"agentOrgResolver failed \u2014 rejecting request to preserve org isolation"
|
|
23956
|
+
);
|
|
23957
|
+
return c.json(
|
|
23958
|
+
{ error: "Service Unavailable: failed to resolve agent organization" },
|
|
23959
|
+
503
|
|
23567
23960
|
);
|
|
23568
23961
|
}
|
|
23569
23962
|
}
|
|
@@ -23574,26 +23967,26 @@ var init_secret_proxy = __esm({
|
|
|
23574
23967
|
expectedOrganizationId
|
|
23575
23968
|
);
|
|
23576
23969
|
if (!mapping) {
|
|
23577
|
-
|
|
23970
|
+
logger77.warn(
|
|
23578
23971
|
{ urlAgentId },
|
|
23579
23972
|
"Rejecting proxy request: placeholder did not resolve"
|
|
23580
23973
|
);
|
|
23581
23974
|
return c.json({ error: "Unauthorized" }, 401);
|
|
23582
23975
|
}
|
|
23583
23976
|
if (mapping.agentId !== urlAgentId) {
|
|
23584
|
-
|
|
23977
|
+
logger77.warn(
|
|
23585
23978
|
{ urlAgentId, mappingAgentId: mapping.agentId },
|
|
23586
23979
|
"Rejecting proxy request: placeholder agentId does not match URL"
|
|
23587
23980
|
);
|
|
23588
23981
|
return c.json({ error: "Forbidden" }, 403);
|
|
23589
23982
|
}
|
|
23590
23983
|
} else if (callerToken) {
|
|
23591
|
-
|
|
23984
|
+
logger77.debug(
|
|
23592
23985
|
{ urlAgentId },
|
|
23593
23986
|
"Proxy request authenticated by non-placeholder token; agentId binding skipped"
|
|
23594
23987
|
);
|
|
23595
23988
|
} else {
|
|
23596
|
-
|
|
23989
|
+
logger77.warn(
|
|
23597
23990
|
{ urlAgentId },
|
|
23598
23991
|
"Rejecting proxy request: names an agent but carries no auth header"
|
|
23599
23992
|
);
|
|
@@ -23616,17 +24009,23 @@ var init_secret_proxy = __esm({
|
|
|
23616
24009
|
if (urlAgentId && resolvedSlug && this.authProfilesManager) {
|
|
23617
24010
|
const providerId = this.slugToProviderId.get(resolvedSlug);
|
|
23618
24011
|
if (providerId) {
|
|
23619
|
-
const
|
|
23620
|
-
|
|
23621
|
-
|
|
23622
|
-
|
|
23623
|
-
|
|
24012
|
+
const runWithOrg = (fn) => expectedOrganizationId ? orgContext.run({ organizationId: expectedOrganizationId }, fn) : fn();
|
|
24013
|
+
const authProfilesManager = this.authProfilesManager;
|
|
24014
|
+
const profile = await runWithOrg(
|
|
24015
|
+
() => authProfilesManager.getBestProfile(
|
|
24016
|
+
urlAgentId,
|
|
24017
|
+
providerId,
|
|
24018
|
+
void 0,
|
|
24019
|
+
providerContext
|
|
24020
|
+
)
|
|
23624
24021
|
);
|
|
23625
24022
|
const userIdForRefresh = providerContext?.userId;
|
|
23626
|
-
const credential = profile && userIdForRefresh ? await
|
|
23627
|
-
|
|
23628
|
-
|
|
23629
|
-
|
|
24023
|
+
const credential = profile && userIdForRefresh ? await runWithOrg(
|
|
24024
|
+
() => authProfilesManager.ensureFreshCredential(profile, {
|
|
24025
|
+
userId: userIdForRefresh,
|
|
24026
|
+
agentId: urlAgentId
|
|
24027
|
+
})
|
|
24028
|
+
) : profile?.credential;
|
|
23630
24029
|
if (credential) {
|
|
23631
24030
|
headers.authorization = `Bearer ${credential}`;
|
|
23632
24031
|
} else if (this.systemKeyResolver) {
|
|
@@ -23634,7 +24033,7 @@ var init_secret_proxy = __esm({
|
|
|
23634
24033
|
if (systemKey) {
|
|
23635
24034
|
headers.authorization = `Bearer ${systemKey}`;
|
|
23636
24035
|
} else {
|
|
23637
|
-
|
|
24036
|
+
logger77.warn(
|
|
23638
24037
|
`No auth profile or system key for agent ${urlAgentId}, provider ${providerId}`
|
|
23639
24038
|
);
|
|
23640
24039
|
return c.json(
|
|
@@ -23649,7 +24048,7 @@ var init_secret_proxy = __esm({
|
|
|
23649
24048
|
);
|
|
23650
24049
|
}
|
|
23651
24050
|
} else {
|
|
23652
|
-
|
|
24051
|
+
logger77.warn(
|
|
23653
24052
|
`No auth profile for agent ${urlAgentId}, provider ${providerId}`
|
|
23654
24053
|
);
|
|
23655
24054
|
return c.json(
|
|
@@ -23664,7 +24063,7 @@ var init_secret_proxy = __esm({
|
|
|
23664
24063
|
);
|
|
23665
24064
|
}
|
|
23666
24065
|
} else {
|
|
23667
|
-
|
|
24066
|
+
logger77.warn(`No providerId mapping for slug "${resolvedSlug}"`);
|
|
23668
24067
|
}
|
|
23669
24068
|
} else {
|
|
23670
24069
|
const source = urlAgentId ?? getClientIp({
|
|
@@ -23692,10 +24091,10 @@ var init_secret_proxy = __esm({
|
|
|
23692
24091
|
}
|
|
23693
24092
|
}
|
|
23694
24093
|
}
|
|
23695
|
-
|
|
24094
|
+
logger77.info(`Forwarding to upstream: ${method} ${upstream}`);
|
|
23696
24095
|
const response = await fetch(upstream, { method, headers, body: body2 });
|
|
23697
24096
|
if (!response.ok) {
|
|
23698
|
-
|
|
24097
|
+
logger77.warn(
|
|
23699
24098
|
`Upstream returned ${response.status} for ${method} ${upstream}`
|
|
23700
24099
|
);
|
|
23701
24100
|
}
|
|
@@ -23732,14 +24131,14 @@ var init_secret_proxy = __esm({
|
|
|
23732
24131
|
});
|
|
23733
24132
|
|
|
23734
24133
|
// src/gateway/proxy/token-refresh-job.ts
|
|
23735
|
-
var
|
|
24134
|
+
var logger78, EXPIRY_BUFFER_MS, TokenRefreshJob;
|
|
23736
24135
|
var init_token_refresh_job = __esm({
|
|
23737
24136
|
"src/gateway/proxy/token-refresh-job.ts"() {
|
|
23738
24137
|
"use strict";
|
|
23739
24138
|
init_src();
|
|
23740
24139
|
init_client();
|
|
23741
24140
|
init_org_context();
|
|
23742
|
-
|
|
24141
|
+
logger78 = createLogger("token-refresh-job");
|
|
23743
24142
|
EXPIRY_BUFFER_MS = 5 * 60 * 1e3;
|
|
23744
24143
|
TokenRefreshJob = class {
|
|
23745
24144
|
constructor(authProfilesManager, refreshableProviders) {
|
|
@@ -23766,7 +24165,7 @@ var init_token_refresh_job = __esm({
|
|
|
23766
24165
|
() => this.maybeRefresh(userId, agentId)
|
|
23767
24166
|
);
|
|
23768
24167
|
} catch (err) {
|
|
23769
|
-
|
|
24168
|
+
logger78.warn(
|
|
23770
24169
|
{ userId, agentId, organizationId, err: String(err) },
|
|
23771
24170
|
"Token refresh failed for user/agent \u2014 continuing scan"
|
|
23772
24171
|
);
|
|
@@ -23789,7 +24188,7 @@ var init_token_refresh_job = __esm({
|
|
|
23789
24188
|
async refreshForUserAgent(userId, agentId) {
|
|
23790
24189
|
const organizationId = await this.lookupAgentOrg(agentId);
|
|
23791
24190
|
if (organizationId === null) {
|
|
23792
|
-
|
|
24191
|
+
logger78.warn(
|
|
23793
24192
|
{ userId, agentId },
|
|
23794
24193
|
"Skipping token refresh \u2014 agent has no org row (deleted?)"
|
|
23795
24194
|
);
|
|
@@ -23836,7 +24235,7 @@ var init_token_refresh_job = __esm({
|
|
|
23836
24235
|
const expiresAt = oauthProfile.metadata.expiresAt || 0;
|
|
23837
24236
|
const isExpiring = expiresAt <= Date.now() + EXPIRY_BUFFER_MS;
|
|
23838
24237
|
if (!isExpiring) continue;
|
|
23839
|
-
|
|
24238
|
+
logger78.info(
|
|
23840
24239
|
`Refreshing ${providerId} token for user ${userId} agent ${agentId} profile ${oauthProfile.id}`,
|
|
23841
24240
|
{ expiresAt: new Date(expiresAt).toISOString() }
|
|
23842
24241
|
);
|
|
@@ -23860,11 +24259,11 @@ var init_token_refresh_job = __esm({
|
|
|
23860
24259
|
},
|
|
23861
24260
|
makePrimary: false
|
|
23862
24261
|
});
|
|
23863
|
-
|
|
24262
|
+
logger78.info(
|
|
23864
24263
|
`Token refreshed for user ${userId} agent ${agentId} (${providerId})`
|
|
23865
24264
|
);
|
|
23866
24265
|
} catch (error) {
|
|
23867
|
-
|
|
24266
|
+
logger78.error(
|
|
23868
24267
|
`Failed to refresh ${providerId} token for user ${userId} agent ${agentId}`,
|
|
23869
24268
|
{
|
|
23870
24269
|
error,
|
|
@@ -24436,12 +24835,12 @@ async function loadBedrockModelsFromAws() {
|
|
|
24436
24835
|
function loadFallbackRegistryModels() {
|
|
24437
24836
|
return getModels("amazon-bedrock").map(normalizeRegistryModel).sort(sortModels);
|
|
24438
24837
|
}
|
|
24439
|
-
var
|
|
24838
|
+
var logger79, CACHE_TTL_MS3, BedrockModelCatalog;
|
|
24440
24839
|
var init_bedrock_model_catalog = __esm({
|
|
24441
24840
|
"src/gateway/services/bedrock-model-catalog.ts"() {
|
|
24442
24841
|
"use strict";
|
|
24443
24842
|
init_src();
|
|
24444
|
-
|
|
24843
|
+
logger79 = createLogger("bedrock-model-catalog");
|
|
24445
24844
|
CACHE_TTL_MS3 = 5 * 60 * 1e3;
|
|
24446
24845
|
BedrockModelCatalog = class {
|
|
24447
24846
|
cacheTtlMs;
|
|
@@ -24464,7 +24863,7 @@ var init_bedrock_model_catalog = __esm({
|
|
|
24464
24863
|
};
|
|
24465
24864
|
return models;
|
|
24466
24865
|
} catch (error) {
|
|
24467
|
-
|
|
24866
|
+
logger79.warn(
|
|
24468
24867
|
{
|
|
24469
24868
|
error: error instanceof Error ? error.message : String(error)
|
|
24470
24869
|
},
|
|
@@ -24498,7 +24897,7 @@ var init_bedrock_model_catalog = __esm({
|
|
|
24498
24897
|
// src/gateway/services/bedrock-openai-service.ts
|
|
24499
24898
|
import { getModel } from "@mariozechner/pi-ai";
|
|
24500
24899
|
import { streamBedrock } from "@mariozechner/pi-ai/dist/providers/amazon-bedrock.js";
|
|
24501
|
-
import { Hono as
|
|
24900
|
+
import { Hono as Hono21 } from "hono";
|
|
24502
24901
|
function parseDataUrl(url) {
|
|
24503
24902
|
if (!url) return null;
|
|
24504
24903
|
const match = url.match(/^data:([^;,]+);base64,(.+)$/);
|
|
@@ -24785,16 +25184,16 @@ function createSseStream(requestModel, stream2, includeUsage) {
|
|
|
24785
25184
|
}
|
|
24786
25185
|
});
|
|
24787
25186
|
}
|
|
24788
|
-
var
|
|
25187
|
+
var logger80, BedrockOpenAIService;
|
|
24789
25188
|
var init_bedrock_openai_service = __esm({
|
|
24790
25189
|
"src/gateway/services/bedrock-openai-service.ts"() {
|
|
24791
25190
|
"use strict";
|
|
24792
25191
|
init_src();
|
|
24793
25192
|
init_middleware();
|
|
24794
25193
|
init_bedrock_model_catalog();
|
|
24795
|
-
|
|
25194
|
+
logger80 = createLogger("bedrock-openai-service");
|
|
24796
25195
|
BedrockOpenAIService = class {
|
|
24797
|
-
app = new
|
|
25196
|
+
app = new Hono21();
|
|
24798
25197
|
modelCatalog;
|
|
24799
25198
|
modelResolver;
|
|
24800
25199
|
bedrockStreamer;
|
|
@@ -24883,7 +25282,7 @@ var init_bedrock_openai_service = __esm({
|
|
|
24883
25282
|
const context2 = buildBedrockContext(request2);
|
|
24884
25283
|
const stopSequences = Array.isArray(request2.stop) ? request2.stop : typeof request2.stop === "string" ? [request2.stop] : [];
|
|
24885
25284
|
const toolChoice = mapToolChoice(request2.tool_choice);
|
|
24886
|
-
|
|
25285
|
+
logger80.info(
|
|
24887
25286
|
{
|
|
24888
25287
|
modelId: request2.model,
|
|
24889
25288
|
region,
|
|
@@ -24968,12 +25367,12 @@ function buildRegistryMap(configAgents) {
|
|
|
24968
25367
|
}
|
|
24969
25368
|
return result;
|
|
24970
25369
|
}
|
|
24971
|
-
var
|
|
25370
|
+
var logger81, DeclaredAgentRegistry;
|
|
24972
25371
|
var init_declared_agent_registry = __esm({
|
|
24973
25372
|
"src/gateway/services/declared-agent-registry.ts"() {
|
|
24974
25373
|
"use strict";
|
|
24975
25374
|
init_src();
|
|
24976
|
-
|
|
25375
|
+
logger81 = createLogger("declared-agent-registry");
|
|
24977
25376
|
DeclaredAgentRegistry = class {
|
|
24978
25377
|
entries = /* @__PURE__ */ new Map();
|
|
24979
25378
|
has(agentId) {
|
|
@@ -24994,7 +25393,7 @@ var init_declared_agent_registry = __esm({
|
|
|
24994
25393
|
for (const [agentId, entry] of next) {
|
|
24995
25394
|
this.entries.set(agentId, entry);
|
|
24996
25395
|
}
|
|
24997
|
-
|
|
25396
|
+
logger81.debug(`Registry now holds ${this.entries.size} declared agent(s)`);
|
|
24998
25397
|
}
|
|
24999
25398
|
};
|
|
25000
25399
|
}
|
|
@@ -25036,12 +25435,12 @@ function hasImageGenerationAccess(profileProviderId, profile) {
|
|
|
25036
25435
|
if (!scopes) return true;
|
|
25037
25436
|
return scopes.has("api.model.image.request") || scopes.has("api.model.request") || scopes.has("model.image.request");
|
|
25038
25437
|
}
|
|
25039
|
-
var
|
|
25438
|
+
var logger82, IMAGE_CAPABLE_PROVIDERS, ImageGenerationService;
|
|
25040
25439
|
var init_image_generation_service = __esm({
|
|
25041
25440
|
"src/gateway/services/image-generation-service.ts"() {
|
|
25042
25441
|
"use strict";
|
|
25043
25442
|
init_src();
|
|
25044
|
-
|
|
25443
|
+
logger82 = createLogger("image-generation-service");
|
|
25045
25444
|
IMAGE_CAPABLE_PROVIDERS = [
|
|
25046
25445
|
{
|
|
25047
25446
|
profileProviderId: "chatgpt",
|
|
@@ -25076,7 +25475,7 @@ var init_image_generation_service = __esm({
|
|
|
25076
25475
|
);
|
|
25077
25476
|
if (!profile?.credential) continue;
|
|
25078
25477
|
if (!hasImageGenerationAccess(profileProviderId, profile)) {
|
|
25079
|
-
|
|
25478
|
+
logger82.info("Skipping provider without image-generation scope", {
|
|
25080
25479
|
agentId,
|
|
25081
25480
|
profileProviderId,
|
|
25082
25481
|
authType: profile.authType
|
|
@@ -25106,7 +25505,7 @@ var init_image_generation_service = __esm({
|
|
|
25106
25505
|
agentId
|
|
25107
25506
|
);
|
|
25108
25507
|
}
|
|
25109
|
-
|
|
25508
|
+
logger82.info("Generating image", {
|
|
25110
25509
|
agentId,
|
|
25111
25510
|
provider: config.provider,
|
|
25112
25511
|
profileProviderId: config.profileProviderId,
|
|
@@ -25125,7 +25524,7 @@ var init_image_generation_service = __esm({
|
|
|
25125
25524
|
};
|
|
25126
25525
|
} catch (error) {
|
|
25127
25526
|
const errorMessage3 = error instanceof Error ? error.message : String(error);
|
|
25128
|
-
|
|
25527
|
+
logger82.error("Image generation failed", {
|
|
25129
25528
|
agentId,
|
|
25130
25529
|
provider: config.provider,
|
|
25131
25530
|
profileProviderId: config.profileProviderId,
|
|
@@ -25139,7 +25538,7 @@ var init_image_generation_service = __esm({
|
|
|
25139
25538
|
}
|
|
25140
25539
|
noProviderError(message, agentId) {
|
|
25141
25540
|
const availableProviders = IMAGE_CAPABLE_PROVIDERS.map((p) => p.provider);
|
|
25142
|
-
|
|
25541
|
+
logger82.info(message, { agentId, availableProviders });
|
|
25143
25542
|
return { error: message, availableProviders };
|
|
25144
25543
|
}
|
|
25145
25544
|
async generateWithOpenAI(prompt, apiKey, options) {
|
|
@@ -25228,13 +25627,13 @@ var init_session = __esm({
|
|
|
25228
25627
|
});
|
|
25229
25628
|
|
|
25230
25629
|
// src/gateway/services/session-manager.ts
|
|
25231
|
-
var
|
|
25630
|
+
var logger83, StateAdapterSessionStore, SessionManager;
|
|
25232
25631
|
var init_session_manager = __esm({
|
|
25233
25632
|
"src/gateway/services/session-manager.ts"() {
|
|
25234
25633
|
"use strict";
|
|
25235
25634
|
init_src();
|
|
25236
25635
|
init_session();
|
|
25237
|
-
|
|
25636
|
+
logger83 = createLogger("session-manager");
|
|
25238
25637
|
StateAdapterSessionStore = class {
|
|
25239
25638
|
constructor(conversations) {
|
|
25240
25639
|
this.conversations = conversations;
|
|
@@ -25244,23 +25643,23 @@ var init_session_manager = __esm({
|
|
|
25244
25643
|
try {
|
|
25245
25644
|
return await this.conversations.getSession(sessionKey3);
|
|
25246
25645
|
} catch (error) {
|
|
25247
|
-
|
|
25646
|
+
logger83.error(`Failed to get session ${sessionKey3}:`, error);
|
|
25248
25647
|
return null;
|
|
25249
25648
|
}
|
|
25250
25649
|
}
|
|
25251
25650
|
async set(sessionKey3, session) {
|
|
25252
25651
|
await this.conversations.setSession(sessionKey3, session);
|
|
25253
|
-
|
|
25652
|
+
logger83.debug(`Stored session ${sessionKey3}`);
|
|
25254
25653
|
}
|
|
25255
25654
|
async delete(sessionKey3) {
|
|
25256
25655
|
await this.conversations.deleteSession(sessionKey3);
|
|
25257
|
-
|
|
25656
|
+
logger83.debug(`Deleted session ${sessionKey3}`);
|
|
25258
25657
|
}
|
|
25259
25658
|
async getByThread(channelId, threadTs) {
|
|
25260
25659
|
try {
|
|
25261
25660
|
return await this.conversations.getSessionByThread(channelId, threadTs);
|
|
25262
25661
|
} catch (error) {
|
|
25263
|
-
|
|
25662
|
+
logger83.error(
|
|
25264
25663
|
`Failed to get session by thread ${channelId}:${threadTs}:`,
|
|
25265
25664
|
error
|
|
25266
25665
|
);
|
|
@@ -25269,7 +25668,7 @@ var init_session_manager = __esm({
|
|
|
25269
25668
|
}
|
|
25270
25669
|
/** Optional cleanup - state adapter TTL handles this automatically */
|
|
25271
25670
|
async cleanup() {
|
|
25272
|
-
|
|
25671
|
+
logger83.debug("StateAdapter TTL handles automatic cleanup");
|
|
25273
25672
|
return 0;
|
|
25274
25673
|
}
|
|
25275
25674
|
};
|
|
@@ -25539,17 +25938,17 @@ data: ${JSON.stringify({ reason })}
|
|
|
25539
25938
|
});
|
|
25540
25939
|
|
|
25541
25940
|
// src/gateway/watchers/run-tracker.ts
|
|
25542
|
-
var
|
|
25941
|
+
var logger84, WatcherRunTracker;
|
|
25543
25942
|
var init_run_tracker = __esm({
|
|
25544
25943
|
"src/gateway/watchers/run-tracker.ts"() {
|
|
25545
25944
|
"use strict";
|
|
25546
25945
|
init_src();
|
|
25547
|
-
|
|
25946
|
+
logger84 = createLogger("watcher-run-tracker");
|
|
25548
25947
|
WatcherRunTracker = class {
|
|
25549
25948
|
pending = /* @__PURE__ */ new Map();
|
|
25550
25949
|
register(handle) {
|
|
25551
25950
|
if (this.pending.has(handle.messageId)) {
|
|
25552
|
-
|
|
25951
|
+
logger84.warn(
|
|
25553
25952
|
{ messageId: handle.messageId, runId: handle.runId },
|
|
25554
25953
|
"duplicate watcher run registration ignored"
|
|
25555
25954
|
);
|
|
@@ -25564,7 +25963,7 @@ var init_run_tracker = __esm({
|
|
|
25564
25963
|
try {
|
|
25565
25964
|
await handle.onResolve(result);
|
|
25566
25965
|
} catch (err) {
|
|
25567
|
-
|
|
25966
|
+
logger84.error(
|
|
25568
25967
|
{ err, messageId, runId: handle.runId },
|
|
25569
25968
|
"watcher run onResolve callback failed"
|
|
25570
25969
|
);
|
|
@@ -25626,12 +26025,12 @@ function resolveProviderRegistryFromRaw(raw) {
|
|
|
25626
26025
|
try {
|
|
25627
26026
|
rawParsed = JSON.parse(raw);
|
|
25628
26027
|
} catch {
|
|
25629
|
-
|
|
26028
|
+
logger85.error("Invalid providers JSON");
|
|
25630
26029
|
return null;
|
|
25631
26030
|
}
|
|
25632
26031
|
const substituted = raw.replace(/\$\{env:([^}]+)\}/g, (_match, varName) => {
|
|
25633
26032
|
if (isBlockedEnvSubstitution(varName)) {
|
|
25634
|
-
|
|
26033
|
+
logger85.warn(`Blocked env substitution for sensitive var: ${varName}`);
|
|
25635
26034
|
return "";
|
|
25636
26035
|
}
|
|
25637
26036
|
return process.env[varName] || "";
|
|
@@ -25640,21 +26039,21 @@ function resolveProviderRegistryFromRaw(raw) {
|
|
|
25640
26039
|
try {
|
|
25641
26040
|
parsed = JSON.parse(substituted);
|
|
25642
26041
|
} catch {
|
|
25643
|
-
|
|
26042
|
+
logger85.error("Invalid providers JSON after env substitution");
|
|
25644
26043
|
return null;
|
|
25645
26044
|
}
|
|
25646
26045
|
if (!Array.isArray(parsed.providers)) {
|
|
25647
|
-
|
|
26046
|
+
logger85.error("Invalid providers config: missing 'providers' array");
|
|
25648
26047
|
return null;
|
|
25649
26048
|
}
|
|
25650
26049
|
return { raw: rawParsed, resolved: parsed };
|
|
25651
26050
|
}
|
|
25652
|
-
var
|
|
26051
|
+
var logger85, ENV_SUBSTITUTION_BLOCKLIST, ProviderRegistryService;
|
|
25653
26052
|
var init_provider_registry_service = __esm({
|
|
25654
26053
|
"src/gateway/services/provider-registry-service.ts"() {
|
|
25655
26054
|
"use strict";
|
|
25656
26055
|
init_src();
|
|
25657
|
-
|
|
26056
|
+
logger85 = createLogger("provider-registry-service");
|
|
25658
26057
|
ENV_SUBSTITUTION_BLOCKLIST = /* @__PURE__ */ new Set([
|
|
25659
26058
|
"ENCRYPTION_KEY",
|
|
25660
26059
|
"DATABASE_URL",
|
|
@@ -25676,7 +26075,7 @@ var init_provider_registry_service = __esm({
|
|
|
25676
26075
|
const config = { providers: preloadedProviders };
|
|
25677
26076
|
this.loaded = config;
|
|
25678
26077
|
this.rawLoaded = config;
|
|
25679
|
-
|
|
26078
|
+
logger85.info(
|
|
25680
26079
|
`Loaded ${preloadedProviders.length} bundled provider(s) (injected)`
|
|
25681
26080
|
);
|
|
25682
26081
|
}
|
|
@@ -25713,7 +26112,7 @@ var init_provider_registry_service = __esm({
|
|
|
25713
26112
|
if (this.configUrl.startsWith("http://") || this.configUrl.startsWith("https://")) {
|
|
25714
26113
|
const response = await fetch(this.configUrl);
|
|
25715
26114
|
if (!response.ok) {
|
|
25716
|
-
|
|
26115
|
+
logger85.error(`Failed to fetch providers config: ${response.status}`);
|
|
25717
26116
|
return null;
|
|
25718
26117
|
}
|
|
25719
26118
|
raw = await response.text();
|
|
@@ -25724,10 +26123,10 @@ var init_provider_registry_service = __esm({
|
|
|
25724
26123
|
if (!resolved) return null;
|
|
25725
26124
|
this.rawLoaded = resolved.raw;
|
|
25726
26125
|
this.loaded = resolved.resolved;
|
|
25727
|
-
|
|
26126
|
+
logger85.info(`Loaded ${this.loaded.providers.length} bundled provider(s)`);
|
|
25728
26127
|
return this.loaded;
|
|
25729
26128
|
} catch (error) {
|
|
25730
|
-
|
|
26129
|
+
logger85.debug("Providers config not available", { error });
|
|
25731
26130
|
return null;
|
|
25732
26131
|
}
|
|
25733
26132
|
}
|
|
@@ -25739,12 +26138,12 @@ var init_provider_registry_service = __esm({
|
|
|
25739
26138
|
function displayName(provider2) {
|
|
25740
26139
|
return TTS_CAPABLE_PROVIDERS.find((p) => p.ttsProvider === provider2)?.displayName ?? provider2;
|
|
25741
26140
|
}
|
|
25742
|
-
var
|
|
26141
|
+
var logger86, TTS_CAPABLE_PROVIDERS, TranscriptionService;
|
|
25743
26142
|
var init_transcription_service = __esm({
|
|
25744
26143
|
"src/gateway/services/transcription-service.ts"() {
|
|
25745
26144
|
"use strict";
|
|
25746
26145
|
init_src();
|
|
25747
|
-
|
|
26146
|
+
logger86 = createLogger("transcription-service");
|
|
25748
26147
|
TTS_CAPABLE_PROVIDERS = [
|
|
25749
26148
|
{
|
|
25750
26149
|
profileProviderId: "chatgpt",
|
|
@@ -25785,7 +26184,7 @@ var init_transcription_service = __esm({
|
|
|
25785
26184
|
}
|
|
25786
26185
|
const attemptErrors = [];
|
|
25787
26186
|
for (const config of configs) {
|
|
25788
|
-
|
|
26187
|
+
logger86.info("Transcribing audio", {
|
|
25789
26188
|
agentId,
|
|
25790
26189
|
provider: config.provider,
|
|
25791
26190
|
profileProviderId: config.profileProviderId,
|
|
@@ -25798,7 +26197,7 @@ var init_transcription_service = __esm({
|
|
|
25798
26197
|
config,
|
|
25799
26198
|
mimeType
|
|
25800
26199
|
);
|
|
25801
|
-
|
|
26200
|
+
logger86.info("Transcription successful", {
|
|
25802
26201
|
agentId,
|
|
25803
26202
|
provider: config.provider,
|
|
25804
26203
|
profileProviderId: config.profileProviderId,
|
|
@@ -25807,7 +26206,7 @@ var init_transcription_service = __esm({
|
|
|
25807
26206
|
return { text, provider: config.provider };
|
|
25808
26207
|
} catch (error) {
|
|
25809
26208
|
const errorMessage3 = error instanceof Error ? error.message : String(error);
|
|
25810
|
-
|
|
26209
|
+
logger86.error("Transcription failed", {
|
|
25811
26210
|
agentId,
|
|
25812
26211
|
provider: config.provider,
|
|
25813
26212
|
profileProviderId: config.profileProviderId,
|
|
@@ -25875,7 +26274,7 @@ var init_transcription_service = __esm({
|
|
|
25875
26274
|
try {
|
|
25876
26275
|
providerConfigs = await this.providerConfigSource();
|
|
25877
26276
|
} catch (error) {
|
|
25878
|
-
|
|
26277
|
+
logger86.warn("Failed to load provider configs for STT", {
|
|
25879
26278
|
error: error instanceof Error ? error.message : String(error)
|
|
25880
26279
|
});
|
|
25881
26280
|
return [];
|
|
@@ -25887,7 +26286,7 @@ var init_transcription_service = __esm({
|
|
|
25887
26286
|
const sttEnabled = stt ? stt.enabled !== false : compat === "openai";
|
|
25888
26287
|
if (!sttEnabled) continue;
|
|
25889
26288
|
if (compat !== "openai") {
|
|
25890
|
-
|
|
26289
|
+
logger86.warn("Unsupported config-driven STT compatibility", {
|
|
25891
26290
|
providerId,
|
|
25892
26291
|
compat
|
|
25893
26292
|
});
|
|
@@ -25898,7 +26297,7 @@ var init_transcription_service = __esm({
|
|
|
25898
26297
|
stt?.baseUrl || entry.upstreamBaseUrl
|
|
25899
26298
|
);
|
|
25900
26299
|
if (!endpoint) {
|
|
25901
|
-
|
|
26300
|
+
logger86.warn("Invalid STT endpoint configuration", {
|
|
25902
26301
|
providerId,
|
|
25903
26302
|
transcriptionPath: stt?.transcriptionPath,
|
|
25904
26303
|
baseUrl: stt?.baseUrl || entry.upstreamBaseUrl
|
|
@@ -25937,7 +26336,7 @@ var init_transcription_service = __esm({
|
|
|
25937
26336
|
if (!config) {
|
|
25938
26337
|
return this.noProviderError("No audio provider configured", agentId);
|
|
25939
26338
|
}
|
|
25940
|
-
|
|
26339
|
+
logger86.info("Synthesizing audio", {
|
|
25941
26340
|
agentId,
|
|
25942
26341
|
provider: config.provider,
|
|
25943
26342
|
textLength: text.length,
|
|
@@ -25945,7 +26344,7 @@ var init_transcription_service = __esm({
|
|
|
25945
26344
|
});
|
|
25946
26345
|
try {
|
|
25947
26346
|
const result = await this.synthesizeWithProvider(text, config, options);
|
|
25948
|
-
|
|
26347
|
+
logger86.info("Synthesis successful", {
|
|
25949
26348
|
agentId,
|
|
25950
26349
|
provider: config.provider,
|
|
25951
26350
|
audioSize: result.audioBuffer.length
|
|
@@ -25953,7 +26352,7 @@ var init_transcription_service = __esm({
|
|
|
25953
26352
|
return { ...result, provider: config.provider };
|
|
25954
26353
|
} catch (error) {
|
|
25955
26354
|
const errorMessage3 = error instanceof Error ? error.message : String(error);
|
|
25956
|
-
|
|
26355
|
+
logger86.error("Synthesis failed", {
|
|
25957
26356
|
agentId,
|
|
25958
26357
|
provider: config.provider,
|
|
25959
26358
|
error: errorMessage3
|
|
@@ -25966,7 +26365,7 @@ var init_transcription_service = __esm({
|
|
|
25966
26365
|
}
|
|
25967
26366
|
noProviderError(message, agentId) {
|
|
25968
26367
|
const availableProviders = TTS_CAPABLE_PROVIDERS.map((p) => p.ttsProvider);
|
|
25969
|
-
|
|
26368
|
+
logger86.info(message, { agentId, availableProviders });
|
|
25970
26369
|
return { error: message, availableProviders };
|
|
25971
26370
|
}
|
|
25972
26371
|
// ==========================================================================
|
|
@@ -26219,7 +26618,7 @@ var init_transcription_service = __esm({
|
|
|
26219
26618
|
});
|
|
26220
26619
|
|
|
26221
26620
|
// src/gateway/services/core-services.ts
|
|
26222
|
-
var
|
|
26621
|
+
var logger87, CoreServices;
|
|
26223
26622
|
var init_core_services = __esm({
|
|
26224
26623
|
"src/gateway/services/core-services.ts"() {
|
|
26225
26624
|
"use strict";
|
|
@@ -26275,7 +26674,7 @@ var init_core_services = __esm({
|
|
|
26275
26674
|
init_provider_config_resolver();
|
|
26276
26675
|
init_provider_registry_service();
|
|
26277
26676
|
init_transcription_service();
|
|
26278
|
-
|
|
26677
|
+
logger87 = createLogger("core-services");
|
|
26279
26678
|
CoreServices = class {
|
|
26280
26679
|
constructor(config, options) {
|
|
26281
26680
|
this.config = config;
|
|
@@ -26381,20 +26780,20 @@ var init_core_services = __esm({
|
|
|
26381
26780
|
* Initialize all core services in dependency order
|
|
26382
26781
|
*/
|
|
26383
26782
|
async initialize() {
|
|
26384
|
-
|
|
26783
|
+
logger87.debug("Initializing core services...");
|
|
26385
26784
|
await this.initializeQueue();
|
|
26386
|
-
|
|
26785
|
+
logger87.debug("Queue initialized");
|
|
26387
26786
|
await this.initializeSessionServices();
|
|
26388
|
-
|
|
26787
|
+
logger87.debug("Session services initialized");
|
|
26389
26788
|
await this.initializeClaudeServices();
|
|
26390
|
-
|
|
26789
|
+
logger87.debug("Auth & provider services initialized");
|
|
26391
26790
|
await this.initializeMcpServices();
|
|
26392
|
-
|
|
26791
|
+
logger87.debug("MCP services initialized");
|
|
26393
26792
|
await this.initializeQueueProducer();
|
|
26394
|
-
|
|
26793
|
+
logger87.debug("Queue producer initialized");
|
|
26395
26794
|
this.initializeCommandRegistry();
|
|
26396
|
-
|
|
26397
|
-
|
|
26795
|
+
logger87.debug("Command registry initialized");
|
|
26796
|
+
logger87.info("Core services initialized successfully");
|
|
26398
26797
|
}
|
|
26399
26798
|
/** Public entry point for the scheduler-registered ephemeral-table sweep.
|
|
26400
26799
|
* Deletes expired rows from oauth_states, rate_limits, grants, and
|
|
@@ -26411,13 +26810,13 @@ var init_core_services = __esm({
|
|
|
26411
26810
|
]);
|
|
26412
26811
|
const pendingInteractions = pendingIds.length;
|
|
26413
26812
|
if (oauthStates + rate + grants + completedRuns + pendingInteractions > 0) {
|
|
26414
|
-
|
|
26813
|
+
logger87.debug(
|
|
26415
26814
|
{ oauthStates, rate, grants, completedRuns, pendingInteractions },
|
|
26416
26815
|
"Ephemeral table sweeper deleted expired rows"
|
|
26417
26816
|
);
|
|
26418
26817
|
}
|
|
26419
26818
|
} catch (error) {
|
|
26420
|
-
|
|
26819
|
+
logger87.warn(
|
|
26421
26820
|
{ error: error instanceof Error ? error.message : String(error) },
|
|
26422
26821
|
"Ephemeral table sweeper failed"
|
|
26423
26822
|
);
|
|
@@ -26429,7 +26828,7 @@ var init_core_services = __esm({
|
|
|
26429
26828
|
async initializeQueue() {
|
|
26430
26829
|
this.queue = new RunsQueue();
|
|
26431
26830
|
await this.queue.start();
|
|
26432
|
-
|
|
26831
|
+
logger87.debug("Queue connection established (runs-table substrate)");
|
|
26433
26832
|
}
|
|
26434
26833
|
async initializeQueueProducer() {
|
|
26435
26834
|
if (!this.queue) {
|
|
@@ -26437,7 +26836,7 @@ var init_core_services = __esm({
|
|
|
26437
26836
|
}
|
|
26438
26837
|
this.queueProducer = new QueueProducer(this.queue);
|
|
26439
26838
|
await this.queueProducer.start();
|
|
26440
|
-
|
|
26839
|
+
logger87.debug("Queue producer initialized");
|
|
26441
26840
|
}
|
|
26442
26841
|
// ============================================================================
|
|
26443
26842
|
// 2. Session Services Initialization
|
|
@@ -26452,17 +26851,17 @@ var init_core_services = __esm({
|
|
|
26452
26851
|
new ConversationStateStore(stateAdapter)
|
|
26453
26852
|
);
|
|
26454
26853
|
this.sessionManager = new SessionManager(sessionStore);
|
|
26455
|
-
|
|
26854
|
+
logger87.debug("Session manager initialized");
|
|
26456
26855
|
this.interactionService = new InteractionService();
|
|
26457
|
-
|
|
26856
|
+
logger87.debug("Interaction service initialized");
|
|
26458
26857
|
this.sseManager = new SseManager();
|
|
26459
|
-
|
|
26858
|
+
logger87.debug("SSE manager initialized");
|
|
26460
26859
|
this.watcherRunTracker = new WatcherRunTracker();
|
|
26461
|
-
|
|
26860
|
+
logger87.debug("Watcher run tracker initialized");
|
|
26462
26861
|
this.grantStore = new GrantStore();
|
|
26463
|
-
|
|
26862
|
+
logger87.debug("Grant store initialized");
|
|
26464
26863
|
this.policyStore = new PolicyStore();
|
|
26465
|
-
|
|
26864
|
+
logger87.debug("Policy store initialized");
|
|
26466
26865
|
const defaultSecretStore = new PostgresSecretStore();
|
|
26467
26866
|
this.secretStore = this.options?.secretStore ?? new SecretStoreRegistry(
|
|
26468
26867
|
defaultSecretStore,
|
|
@@ -26475,7 +26874,7 @@ var init_core_services = __esm({
|
|
|
26475
26874
|
}
|
|
26476
26875
|
}
|
|
26477
26876
|
);
|
|
26478
|
-
|
|
26877
|
+
logger87.debug("Secret store initialized");
|
|
26479
26878
|
this.channelBindingService = new ChannelBindingService();
|
|
26480
26879
|
this.userAgentsStore = new UserAgentsStore();
|
|
26481
26880
|
if (!this.configStore || !this.connectionStore || !this.accessStore) {
|
|
@@ -26488,7 +26887,7 @@ var init_core_services = __esm({
|
|
|
26488
26887
|
inMemoryStore,
|
|
26489
26888
|
this.config.agents
|
|
26490
26889
|
);
|
|
26491
|
-
|
|
26890
|
+
logger87.debug(
|
|
26492
26891
|
`Agent sub-stores initialized (in-memory, ${this.config.agents.length} agent(s) from config)`
|
|
26493
26892
|
);
|
|
26494
26893
|
} else {
|
|
@@ -26497,11 +26896,11 @@ var init_core_services = __esm({
|
|
|
26497
26896
|
);
|
|
26498
26897
|
}
|
|
26499
26898
|
} else {
|
|
26500
|
-
|
|
26899
|
+
logger87.debug("Using host-provided agent sub-stores (embedded mode)");
|
|
26501
26900
|
}
|
|
26502
26901
|
this.agentSettingsStore = new AgentSettingsStore(this.configStore);
|
|
26503
26902
|
this.agentMetadataStore = new AgentMetadataStore(this.configStore);
|
|
26504
|
-
|
|
26903
|
+
logger87.debug(
|
|
26505
26904
|
"Agent settings, channel binding, user agents & metadata stores initialized"
|
|
26506
26905
|
);
|
|
26507
26906
|
const externalAuthKv = /* @__PURE__ */ new Map();
|
|
@@ -26527,7 +26926,7 @@ var init_core_services = __esm({
|
|
|
26527
26926
|
}
|
|
26528
26927
|
}) ?? void 0;
|
|
26529
26928
|
if (this.externalAuthClient) {
|
|
26530
|
-
|
|
26929
|
+
logger87.debug("External OAuth client initialized");
|
|
26531
26930
|
}
|
|
26532
26931
|
}
|
|
26533
26932
|
// ============================================================================
|
|
@@ -26588,7 +26987,7 @@ var init_core_services = __esm({
|
|
|
26588
26987
|
}
|
|
26589
26988
|
}
|
|
26590
26989
|
}
|
|
26591
|
-
|
|
26990
|
+
logger87.debug(
|
|
26592
26991
|
"Auth profile, model preference, transcription, and image generation services initialized"
|
|
26593
26992
|
);
|
|
26594
26993
|
this.secretProxy = new SecretProxy(
|
|
@@ -26597,7 +26996,7 @@ var init_core_services = __esm({
|
|
|
26597
26996
|
},
|
|
26598
26997
|
this.secretStore
|
|
26599
26998
|
);
|
|
26600
|
-
|
|
26999
|
+
logger87.debug(
|
|
26601
27000
|
`Secret proxy initialized (upstream: ${this.config.anthropicProxy.anthropicBaseUrl || "https://api.anthropic.com"})`
|
|
26602
27001
|
);
|
|
26603
27002
|
if (!this.authProfilesManager) {
|
|
@@ -26608,24 +27007,24 @@ var init_core_services = __esm({
|
|
|
26608
27007
|
this.tokenRefreshJob = new TokenRefreshJob(this.authProfilesManager, [
|
|
26609
27008
|
{ providerId: "claude", oauthClient: new OAuthClient(CLAUDE_PROVIDER) }
|
|
26610
27009
|
]);
|
|
26611
|
-
|
|
27010
|
+
logger87.debug("Token refresh job constructed");
|
|
26612
27011
|
this.oauthStateStore = createOAuthStateStore("claude");
|
|
26613
27012
|
const claudeOAuthModule = new ClaudeOAuthModule(
|
|
26614
27013
|
this.authProfilesManager,
|
|
26615
27014
|
this.modelPreferenceStore
|
|
26616
27015
|
);
|
|
26617
27016
|
moduleRegistry.register(claudeOAuthModule);
|
|
26618
|
-
|
|
27017
|
+
logger87.debug(
|
|
26619
27018
|
`Claude OAuth module registered (system token: ${claudeOAuthModule.hasSystemKey() ? "available" : "not available"})`
|
|
26620
27019
|
);
|
|
26621
27020
|
const chatgptOAuthModule = new ChatGPTOAuthModule(this.authProfilesManager);
|
|
26622
27021
|
moduleRegistry.register(chatgptOAuthModule);
|
|
26623
|
-
|
|
27022
|
+
logger87.debug(
|
|
26624
27023
|
`ChatGPT OAuth module registered (system token: ${chatgptOAuthModule.hasSystemKey() ? "available" : "not available"})`
|
|
26625
27024
|
);
|
|
26626
27025
|
const geminiCliModule = new GeminiCliModule(this.authProfilesManager);
|
|
26627
27026
|
moduleRegistry.register(geminiCliModule);
|
|
26628
|
-
|
|
27027
|
+
logger87.debug("Gemini CLI module registered (acpx sub-agent shell-out)");
|
|
26629
27028
|
const bedrockModelCatalog = new BedrockModelCatalog();
|
|
26630
27029
|
const bedrockProviderModule = new BedrockProviderModule(
|
|
26631
27030
|
this.authProfilesManager,
|
|
@@ -26635,23 +27034,23 @@ var init_core_services = __esm({
|
|
|
26635
27034
|
this.bedrockOpenAIService = new BedrockOpenAIService({
|
|
26636
27035
|
modelCatalog: bedrockModelCatalog
|
|
26637
27036
|
});
|
|
26638
|
-
|
|
27037
|
+
logger87.debug("Bedrock provider module registered");
|
|
26639
27038
|
const injectedProviders = this.options?.providerRegistry;
|
|
26640
27039
|
if (injectedProviders) {
|
|
26641
27040
|
this.providerRegistryService = new ProviderRegistryService(
|
|
26642
27041
|
void 0,
|
|
26643
27042
|
injectedProviders
|
|
26644
27043
|
);
|
|
26645
|
-
|
|
27044
|
+
logger87.debug(
|
|
26646
27045
|
`Provider registry initialized from injected providers (${injectedProviders.length})`
|
|
26647
27046
|
);
|
|
26648
27047
|
} else {
|
|
26649
27048
|
const registryPath = resolveProviderRegistryPath();
|
|
26650
27049
|
this.providerRegistryService = new ProviderRegistryService(registryPath);
|
|
26651
27050
|
if (registryPath) {
|
|
26652
|
-
|
|
27051
|
+
logger87.info(`Provider registry path: ${registryPath}`);
|
|
26653
27052
|
} else {
|
|
26654
|
-
|
|
27053
|
+
logger87.warn(
|
|
26655
27054
|
"No providers registry found (set LOBU_PROVIDER_REGISTRY_PATH or run from a dir with config/providers.json) \u2014 config-driven providers will be unavailable"
|
|
26656
27055
|
);
|
|
26657
27056
|
}
|
|
@@ -26663,7 +27062,7 @@ var init_core_services = __esm({
|
|
|
26663
27062
|
() => this.providerConfigResolver ? this.providerConfigResolver.getProviderConfigs() : Promise.resolve({})
|
|
26664
27063
|
);
|
|
26665
27064
|
const configProviders = await this.providerConfigResolver.getProviderConfigs();
|
|
26666
|
-
|
|
27065
|
+
logger87.info(
|
|
26667
27066
|
`Provider registry loaded ${Object.keys(configProviders).length} config-driven provider(s)`
|
|
26668
27067
|
);
|
|
26669
27068
|
const registeredIds = new Set(
|
|
@@ -26671,7 +27070,7 @@ var init_core_services = __esm({
|
|
|
26671
27070
|
);
|
|
26672
27071
|
for (const [id, entry] of Object.entries(configProviders)) {
|
|
26673
27072
|
if (registeredIds.has(id)) {
|
|
26674
|
-
|
|
27073
|
+
logger87.info(
|
|
26675
27074
|
`Skipping config-driven provider "${id}" \u2014 already registered`
|
|
26676
27075
|
);
|
|
26677
27076
|
continue;
|
|
@@ -26693,7 +27092,7 @@ var init_core_services = __esm({
|
|
|
26693
27092
|
});
|
|
26694
27093
|
moduleRegistry.register(module);
|
|
26695
27094
|
registeredIds.add(id);
|
|
26696
|
-
|
|
27095
|
+
logger87.debug(
|
|
26697
27096
|
`Registered config-driven provider: ${id} (system key: ${module.hasSystemKey() ? "available" : "not available"})`
|
|
26698
27097
|
);
|
|
26699
27098
|
}
|
|
@@ -26702,7 +27101,7 @@ var init_core_services = __esm({
|
|
|
26702
27101
|
this.authProfilesManager,
|
|
26703
27102
|
this.declaredAgentRegistry
|
|
26704
27103
|
);
|
|
26705
|
-
|
|
27104
|
+
logger87.debug("Provider catalog service initialized");
|
|
26706
27105
|
if (this.secretProxy) {
|
|
26707
27106
|
this.secretProxy.setAuthProfilesManager(this.authProfilesManager);
|
|
26708
27107
|
this.secretProxy.setAgentOrgResolver(
|
|
@@ -26725,7 +27124,7 @@ var init_core_services = __esm({
|
|
|
26725
27124
|
}
|
|
26726
27125
|
return testEnv[mod.getCredentialEnvVarName()] || void 0;
|
|
26727
27126
|
});
|
|
26728
|
-
|
|
27127
|
+
logger87.debug("Provider upstreams registered with secret proxy");
|
|
26729
27128
|
}
|
|
26730
27129
|
}
|
|
26731
27130
|
// ============================================================================
|
|
@@ -26757,7 +27156,7 @@ var init_core_services = __esm({
|
|
|
26757
27156
|
this.mcpConfigService,
|
|
26758
27157
|
this.agentSettingsStore
|
|
26759
27158
|
);
|
|
26760
|
-
|
|
27159
|
+
logger87.debug("Instruction service initialized");
|
|
26761
27160
|
if (!this.secretStore) {
|
|
26762
27161
|
throw new Error("Secret store must be initialized before MCP proxy");
|
|
26763
27162
|
}
|
|
@@ -26808,7 +27207,7 @@ var init_core_services = __esm({
|
|
|
26808
27207
|
payload.message
|
|
26809
27208
|
);
|
|
26810
27209
|
};
|
|
26811
|
-
|
|
27210
|
+
logger87.debug("MCP proxy initialized");
|
|
26812
27211
|
this.workerGateway = new WorkerGateway(
|
|
26813
27212
|
this.queue,
|
|
26814
27213
|
this.config.mcp.publicGatewayUrl,
|
|
@@ -26819,10 +27218,10 @@ var init_core_services = __esm({
|
|
|
26819
27218
|
this.agentSettingsStore,
|
|
26820
27219
|
this.secretStore
|
|
26821
27220
|
);
|
|
26822
|
-
|
|
27221
|
+
logger87.debug("Worker gateway initialized");
|
|
26823
27222
|
await moduleRegistry.registerAvailableModules();
|
|
26824
27223
|
await moduleRegistry.initAll();
|
|
26825
|
-
|
|
27224
|
+
logger87.debug("Modules initialized");
|
|
26826
27225
|
}
|
|
26827
27226
|
// ============================================================================
|
|
26828
27227
|
// 7. Command Registry Initialization
|
|
@@ -26837,7 +27236,7 @@ var init_core_services = __esm({
|
|
|
26837
27236
|
registerBuiltInCommands(this.commandRegistry, {
|
|
26838
27237
|
agentSettingsStore: this.agentSettingsStore
|
|
26839
27238
|
});
|
|
26840
|
-
|
|
27239
|
+
logger87.debug("Command registry initialized with built-in commands");
|
|
26841
27240
|
}
|
|
26842
27241
|
// ============================================================================
|
|
26843
27242
|
// SDK-Embedded Helpers
|
|
@@ -26865,18 +27264,18 @@ var init_core_services = __esm({
|
|
|
26865
27264
|
// Shutdown
|
|
26866
27265
|
// ============================================================================
|
|
26867
27266
|
async shutdown() {
|
|
26868
|
-
|
|
27267
|
+
logger87.info("Shutting down core services...");
|
|
26869
27268
|
if (this.queueProducer) {
|
|
26870
27269
|
await this.queueProducer.stop();
|
|
26871
27270
|
}
|
|
26872
27271
|
if (this.workerGateway) {
|
|
26873
27272
|
this.workerGateway.shutdown();
|
|
26874
|
-
|
|
27273
|
+
logger87.info("Worker gateway shutdown complete");
|
|
26875
27274
|
}
|
|
26876
27275
|
if (this.queue) {
|
|
26877
27276
|
await this.queue.stop();
|
|
26878
27277
|
}
|
|
26879
|
-
|
|
27278
|
+
logger87.info("Core services shutdown complete");
|
|
26880
27279
|
}
|
|
26881
27280
|
// ============================================================================
|
|
26882
27281
|
// Service Accessors (implements ICoreServices interface)
|
|
@@ -27028,7 +27427,7 @@ var init_core_services = __esm({
|
|
|
27028
27427
|
});
|
|
27029
27428
|
|
|
27030
27429
|
// src/gateway/gateway-main.ts
|
|
27031
|
-
var
|
|
27430
|
+
var logger88, Gateway;
|
|
27032
27431
|
var init_gateway_main = __esm({
|
|
27033
27432
|
"src/gateway/gateway-main.ts"() {
|
|
27034
27433
|
"use strict";
|
|
@@ -27036,7 +27435,7 @@ var init_gateway_main = __esm({
|
|
|
27036
27435
|
init_platform2();
|
|
27037
27436
|
init_unified_thread_consumer();
|
|
27038
27437
|
init_core_services();
|
|
27039
|
-
|
|
27438
|
+
logger88 = createLogger("gateway");
|
|
27040
27439
|
Gateway = class {
|
|
27041
27440
|
constructor(config, options) {
|
|
27042
27441
|
this.config = config;
|
|
@@ -27073,7 +27472,7 @@ var init_gateway_main = __esm({
|
|
|
27073
27472
|
this.coreServices.getInstructionService()?.registerPlatformProvider(platform.name, provider2);
|
|
27074
27473
|
}
|
|
27075
27474
|
}
|
|
27076
|
-
|
|
27475
|
+
logger88.debug(`Platform registered: ${platform.name}`);
|
|
27077
27476
|
return this;
|
|
27078
27477
|
}
|
|
27079
27478
|
/**
|
|
@@ -27084,10 +27483,10 @@ var init_gateway_main = __esm({
|
|
|
27084
27483
|
* 4. Start all platforms
|
|
27085
27484
|
*/
|
|
27086
27485
|
async start() {
|
|
27087
|
-
|
|
27486
|
+
logger88.debug("Starting gateway...");
|
|
27088
27487
|
await this.coreServices.initialize();
|
|
27089
27488
|
for (const [name, platform] of this.platforms) {
|
|
27090
|
-
|
|
27489
|
+
logger88.debug(`Initializing platform: ${name}`);
|
|
27091
27490
|
await platform.initialize(this.coreServices);
|
|
27092
27491
|
}
|
|
27093
27492
|
const instructionService = this.coreServices.getInstructionService();
|
|
@@ -27102,7 +27501,7 @@ var init_gateway_main = __esm({
|
|
|
27102
27501
|
}
|
|
27103
27502
|
}
|
|
27104
27503
|
for (const [name, platform] of this.platforms) {
|
|
27105
|
-
|
|
27504
|
+
logger88.debug(`Starting platform: ${name}`);
|
|
27106
27505
|
await platform.start();
|
|
27107
27506
|
}
|
|
27108
27507
|
this.unifiedConsumer = new UnifiedThreadResponseConsumer(
|
|
@@ -27120,26 +27519,26 @@ var init_gateway_main = __esm({
|
|
|
27120
27519
|
* 3. Shutdown core services
|
|
27121
27520
|
*/
|
|
27122
27521
|
async stop() {
|
|
27123
|
-
|
|
27522
|
+
logger88.info("Stopping gateway...");
|
|
27124
27523
|
if (this.unifiedConsumer) {
|
|
27125
|
-
|
|
27524
|
+
logger88.info("Stopping unified thread response consumer");
|
|
27126
27525
|
try {
|
|
27127
27526
|
await this.unifiedConsumer.stop();
|
|
27128
27527
|
} catch (error) {
|
|
27129
|
-
|
|
27528
|
+
logger88.error("Failed to stop unified consumer:", error);
|
|
27130
27529
|
}
|
|
27131
27530
|
}
|
|
27132
27531
|
for (const [name, platform] of this.platforms) {
|
|
27133
|
-
|
|
27532
|
+
logger88.info(`Stopping platform: ${name}`);
|
|
27134
27533
|
try {
|
|
27135
27534
|
await platform.stop();
|
|
27136
27535
|
} catch (error) {
|
|
27137
|
-
|
|
27536
|
+
logger88.error(`Failed to stop platform ${name}:`, error);
|
|
27138
27537
|
}
|
|
27139
27538
|
}
|
|
27140
27539
|
await this.coreServices.shutdown();
|
|
27141
27540
|
this.isRunning = false;
|
|
27142
|
-
|
|
27541
|
+
logger88.info("\u2705 Gateway stopped");
|
|
27143
27542
|
}
|
|
27144
27543
|
/**
|
|
27145
27544
|
* Get gateway status
|
|
@@ -27204,7 +27603,7 @@ function isSecretEnvVar(name, providerModules) {
|
|
|
27204
27603
|
const upper = name.toUpperCase();
|
|
27205
27604
|
return upper.includes("_KEY") || upper.includes("_TOKEN") || upper.includes("_SECRET") || upper.includes("_PASSWORD");
|
|
27206
27605
|
}
|
|
27207
|
-
var
|
|
27606
|
+
var logger89, SECRET_PLACEHOLDER_TTL_SECONDS, GRANT_SYNC_CACHE_MAX, BaseDeploymentManager;
|
|
27208
27607
|
var init_base_deployment_manager = __esm({
|
|
27209
27608
|
"src/gateway/orchestration/base-deployment-manager.ts"() {
|
|
27210
27609
|
"use strict";
|
|
@@ -27212,7 +27611,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27212
27611
|
init_policy_store();
|
|
27213
27612
|
init_secret_proxy();
|
|
27214
27613
|
init_secrets();
|
|
27215
|
-
|
|
27614
|
+
logger89 = createLogger("orchestrator");
|
|
27216
27615
|
SECRET_PLACEHOLDER_TTL_SECONDS = (() => {
|
|
27217
27616
|
const raw = process.env.SECRET_PLACEHOLDER_TTL_MS;
|
|
27218
27617
|
if (raw) {
|
|
@@ -27321,7 +27720,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27321
27720
|
};
|
|
27322
27721
|
const deploymentName = generateDeploymentName(deploymentIdentity);
|
|
27323
27722
|
const canonicalConversationKey = buildCanonicalConversationKey(deploymentIdentity);
|
|
27324
|
-
|
|
27723
|
+
logger89.info(
|
|
27325
27724
|
`Worker deployment - conversationId: ${conversationId}, canonicalKey: ${canonicalConversationKey}, deploymentName: ${deploymentName}`
|
|
27326
27725
|
);
|
|
27327
27726
|
try {
|
|
@@ -27334,14 +27733,14 @@ var init_base_deployment_manager = __esm({
|
|
|
27334
27733
|
await this.scaleDeployment(deploymentName, 1);
|
|
27335
27734
|
return;
|
|
27336
27735
|
} catch (scaleErr) {
|
|
27337
|
-
|
|
27736
|
+
logger89.warn(
|
|
27338
27737
|
`scaleDeployment(${deploymentName}, 1) failed (${scaleErr instanceof Error ? scaleErr.message : String(scaleErr)}); re-spawning`
|
|
27339
27738
|
);
|
|
27340
27739
|
}
|
|
27341
27740
|
}
|
|
27342
27741
|
const maxDeployments = this.config.worker.maxDeployments;
|
|
27343
27742
|
if (maxDeployments > 0 && deployments.length >= maxDeployments) {
|
|
27344
|
-
|
|
27743
|
+
logger89.warn(
|
|
27345
27744
|
`\u26A0\uFE0F Maximum deployments limit reached (${deployments.length}/${maxDeployments}). Running cleanup before creating new deployment.`
|
|
27346
27745
|
);
|
|
27347
27746
|
await this.reconcileDeployments();
|
|
@@ -27405,7 +27804,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27405
27804
|
const organizationId = messageData.organizationId;
|
|
27406
27805
|
if (!this.policyStore || !agentId || !organizationId) {
|
|
27407
27806
|
if (!organizationId && agentId) {
|
|
27408
|
-
|
|
27807
|
+
logger89.warn(
|
|
27409
27808
|
{ agentId, deploymentName },
|
|
27410
27809
|
"Skipping egress policy sync \u2014 message has no organizationId"
|
|
27411
27810
|
);
|
|
@@ -27420,11 +27819,11 @@ var init_base_deployment_manager = __esm({
|
|
|
27420
27819
|
if (bundle) {
|
|
27421
27820
|
this.policyStore.set(organizationId, agentId, bundle);
|
|
27422
27821
|
if (deploymentName) {
|
|
27423
|
-
|
|
27822
|
+
logger89.info(
|
|
27424
27823
|
`Synced egress judge policy for ${deploymentName}: ${bundle.judgedDomains.length} rule(s), ${Object.keys(bundle.judges).length} judge(s)`
|
|
27425
27824
|
);
|
|
27426
27825
|
} else {
|
|
27427
|
-
|
|
27826
|
+
logger89.debug("Synced egress judge policy", {
|
|
27428
27827
|
organizationId,
|
|
27429
27828
|
agentId,
|
|
27430
27829
|
rules: bundle.judgedDomains.length,
|
|
@@ -27447,7 +27846,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27447
27846
|
for (const domain of messageData.networkConfig.allowedDomains) {
|
|
27448
27847
|
await this.grantStore.grant(agentId, domain, null, void 0, orgId);
|
|
27449
27848
|
}
|
|
27450
|
-
|
|
27849
|
+
logger89.info(
|
|
27451
27850
|
`Synced network config domains as grants for ${deploymentName}: ${messageData.networkConfig.allowedDomains.join(", ")}`
|
|
27452
27851
|
);
|
|
27453
27852
|
}
|
|
@@ -27455,7 +27854,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27455
27854
|
for (const pattern of messageData.preApprovedTools) {
|
|
27456
27855
|
await this.grantStore.grant(agentId, pattern, null, void 0, orgId);
|
|
27457
27856
|
}
|
|
27458
|
-
|
|
27857
|
+
logger89.info(
|
|
27459
27858
|
`Synced pre-approved tool patterns as grants for ${deploymentName}: ${messageData.preApprovedTools.join(", ")}`
|
|
27460
27859
|
);
|
|
27461
27860
|
}
|
|
@@ -27469,7 +27868,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27469
27868
|
for (const domain of NIX_DOMAINS) {
|
|
27470
27869
|
await this.grantStore.grant(agentId, domain, null, void 0, orgId);
|
|
27471
27870
|
}
|
|
27472
|
-
|
|
27871
|
+
logger89.info(
|
|
27473
27872
|
`Added Nix cache domains as grants for ${deploymentName}: ${NIX_DOMAINS.join(", ")}`
|
|
27474
27873
|
);
|
|
27475
27874
|
}
|
|
@@ -27608,7 +28007,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27608
28007
|
if (flakeUrl) envVars.NIX_FLAKE_URL = flakeUrl;
|
|
27609
28008
|
if (packages && packages.length > 0)
|
|
27610
28009
|
envVars.NIX_PACKAGES = packages.join(",");
|
|
27611
|
-
|
|
28010
|
+
logger89.debug(
|
|
27612
28011
|
`Nix config for ${deploymentName}: flakeUrl=${flakeUrl || "none"}, packages=${packages?.length || 0}`
|
|
27613
28012
|
);
|
|
27614
28013
|
}
|
|
@@ -27671,12 +28070,12 @@ var init_base_deployment_manager = __esm({
|
|
|
27671
28070
|
envVars[key] = placeholder;
|
|
27672
28071
|
hasSecrets = true;
|
|
27673
28072
|
} catch (error) {
|
|
27674
|
-
|
|
28073
|
+
logger89.warn(`Failed to generate placeholder for ${key}:`, error);
|
|
27675
28074
|
}
|
|
27676
28075
|
}
|
|
27677
28076
|
}
|
|
27678
28077
|
if (hasSecrets) {
|
|
27679
|
-
|
|
28078
|
+
logger89.info(
|
|
27680
28079
|
`\u{1F510} Generated secret placeholders for ${deploymentName}, routing through proxy`
|
|
27681
28080
|
);
|
|
27682
28081
|
}
|
|
@@ -27739,7 +28138,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27739
28138
|
providerContext
|
|
27740
28139
|
);
|
|
27741
28140
|
} catch (error) {
|
|
27742
|
-
|
|
28141
|
+
logger89.warn("Failed to build module environment variables:", error);
|
|
27743
28142
|
}
|
|
27744
28143
|
}
|
|
27745
28144
|
if (this.config.worker.env) {
|
|
@@ -27774,7 +28173,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27774
28173
|
}
|
|
27775
28174
|
}
|
|
27776
28175
|
if (primaryProvider) {
|
|
27777
|
-
|
|
28176
|
+
logger89.info(
|
|
27778
28177
|
{
|
|
27779
28178
|
agentId,
|
|
27780
28179
|
primaryProviderId: primaryProvider.providerId,
|
|
@@ -27820,7 +28219,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27820
28219
|
for (const domain of NPM_DOMAINS) {
|
|
27821
28220
|
await this.grantStore.grant(agentId, domain, null, void 0, orgId);
|
|
27822
28221
|
}
|
|
27823
|
-
|
|
28222
|
+
logger89.info(
|
|
27824
28223
|
`Added npm registry domains as grants for ${deploymentName}: ${NPM_DOMAINS.join(", ")}`
|
|
27825
28224
|
);
|
|
27826
28225
|
}
|
|
@@ -27839,12 +28238,12 @@ var init_base_deployment_manager = __esm({
|
|
|
27839
28238
|
`deployments/${deploymentName}/`
|
|
27840
28239
|
);
|
|
27841
28240
|
if (cleared > 0) {
|
|
27842
|
-
|
|
28241
|
+
logger89.debug(
|
|
27843
28242
|
`Cleared ${cleared} deployment secret(s) for ${deploymentName}`
|
|
27844
28243
|
);
|
|
27845
28244
|
}
|
|
27846
28245
|
} catch (error) {
|
|
27847
|
-
|
|
28246
|
+
logger89.warn(
|
|
27848
28247
|
`Failed to clear deployment secrets for ${deploymentName}:`,
|
|
27849
28248
|
error
|
|
27850
28249
|
);
|
|
@@ -27867,7 +28266,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27867
28266
|
async reconcileDeployments() {
|
|
27868
28267
|
try {
|
|
27869
28268
|
const maxDeployments = this.config.worker.maxDeployments;
|
|
27870
|
-
|
|
28269
|
+
logger89.debug("Running deployment cleanup...");
|
|
27871
28270
|
const activeDeployments = await this.listDeployments();
|
|
27872
28271
|
if (activeDeployments.length === 0) {
|
|
27873
28272
|
return;
|
|
@@ -27908,7 +28307,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27908
28307
|
if (results[j]?.status === "fulfilled") {
|
|
27909
28308
|
processedCount++;
|
|
27910
28309
|
} else {
|
|
27911
|
-
|
|
28310
|
+
logger89.error(
|
|
27912
28311
|
`\u274C Failed to delete deployment ${batch[j]}:`,
|
|
27913
28312
|
results[j].reason
|
|
27914
28313
|
);
|
|
@@ -27924,7 +28323,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27924
28323
|
if (results[j]?.status === "fulfilled") {
|
|
27925
28324
|
processedCount++;
|
|
27926
28325
|
} else {
|
|
27927
|
-
|
|
28326
|
+
logger89.error(
|
|
27928
28327
|
`\u274C Failed to scale down deployment ${batch[j]}:`,
|
|
27929
28328
|
results[j].reason
|
|
27930
28329
|
);
|
|
@@ -27932,12 +28331,12 @@ var init_base_deployment_manager = __esm({
|
|
|
27932
28331
|
}
|
|
27933
28332
|
}
|
|
27934
28333
|
if (processedCount > 0) {
|
|
27935
|
-
|
|
28334
|
+
logger89.info(
|
|
27936
28335
|
`\u2705 Cleanup completed: processed ${processedCount} deployment(s)`
|
|
27937
28336
|
);
|
|
27938
28337
|
}
|
|
27939
28338
|
} catch (error) {
|
|
27940
|
-
|
|
28339
|
+
logger89.error(
|
|
27941
28340
|
"Error during deployment reconciliation:",
|
|
27942
28341
|
error instanceof Error ? error.message : String(error)
|
|
27943
28342
|
);
|
|
@@ -28059,6 +28458,111 @@ function makeUnitName(deploymentName) {
|
|
|
28059
28458
|
const tag = Math.random().toString(36).slice(2, 8);
|
|
28060
28459
|
return `lobu-worker-${safe}-${tag}`;
|
|
28061
28460
|
}
|
|
28461
|
+
function getDefaultMaxReservedLocks() {
|
|
28462
|
+
const poolMax = Number.parseInt(process.env.DB_POOL_MAX || "20", 10);
|
|
28463
|
+
if (!Number.isFinite(poolMax) || poolMax <= 0) {
|
|
28464
|
+
return Math.max(1, 20 - POOL_HEADROOM);
|
|
28465
|
+
}
|
|
28466
|
+
return Math.max(1, poolMax - POOL_HEADROOM);
|
|
28467
|
+
}
|
|
28468
|
+
function getMaxReservedLocks() {
|
|
28469
|
+
const raw = process.env.LOBU_MAX_RESERVED_LOCKS;
|
|
28470
|
+
if (!raw) return getDefaultMaxReservedLocks();
|
|
28471
|
+
const n = Number.parseInt(raw, 10);
|
|
28472
|
+
if (!Number.isFinite(n) || n < 0) return getDefaultMaxReservedLocks();
|
|
28473
|
+
return n;
|
|
28474
|
+
}
|
|
28475
|
+
function getReservedLockCount() {
|
|
28476
|
+
return reservedLockCount;
|
|
28477
|
+
}
|
|
28478
|
+
async function acquireConversationLock(organizationId, agentId, conversationId) {
|
|
28479
|
+
if (process.env.LOBU_DISABLE_PREPARE === "1") {
|
|
28480
|
+
return { release: async () => {
|
|
28481
|
+
} };
|
|
28482
|
+
}
|
|
28483
|
+
const max = getMaxReservedLocks();
|
|
28484
|
+
if (reservedLockCount >= max) {
|
|
28485
|
+
logger90.warn(
|
|
28486
|
+
`Reserved-lock cap reached (${reservedLockCount}/${max}); deferring spawn for ${organizationId}/${agentId}/${conversationId}`
|
|
28487
|
+
);
|
|
28488
|
+
return null;
|
|
28489
|
+
}
|
|
28490
|
+
reservedLockCount += 1;
|
|
28491
|
+
if (!warnedNearCap && reservedLockCount >= Math.ceil(max * 0.8)) {
|
|
28492
|
+
logger90.warn(
|
|
28493
|
+
`Reserved-lock count near cap: ${reservedLockCount}/${max}. Tune via LOBU_MAX_RESERVED_LOCKS or scale pods.`
|
|
28494
|
+
);
|
|
28495
|
+
warnedNearCap = true;
|
|
28496
|
+
}
|
|
28497
|
+
let decremented = false;
|
|
28498
|
+
const decrementOnce = () => {
|
|
28499
|
+
if (decremented) return;
|
|
28500
|
+
decremented = true;
|
|
28501
|
+
reservedLockCount = Math.max(0, reservedLockCount - 1);
|
|
28502
|
+
if (warnedNearCap && reservedLockCount < Math.ceil(max * 0.8)) {
|
|
28503
|
+
warnedNearCap = false;
|
|
28504
|
+
}
|
|
28505
|
+
};
|
|
28506
|
+
const sql = getDb();
|
|
28507
|
+
let reserved;
|
|
28508
|
+
try {
|
|
28509
|
+
reserved = await sql.reserve();
|
|
28510
|
+
} catch (err) {
|
|
28511
|
+
decrementOnce();
|
|
28512
|
+
throw err;
|
|
28513
|
+
}
|
|
28514
|
+
const key2 = hashConvKey2(organizationId, agentId, conversationId);
|
|
28515
|
+
try {
|
|
28516
|
+
const rows = await reserved`SELECT pg_try_advisory_lock(${CONV_LOCK_KEY1}, ${key2}) AS acquired`;
|
|
28517
|
+
if (!rows[0]?.acquired) {
|
|
28518
|
+
reserved.release();
|
|
28519
|
+
decrementOnce();
|
|
28520
|
+
return null;
|
|
28521
|
+
}
|
|
28522
|
+
} catch (err) {
|
|
28523
|
+
reserved.release();
|
|
28524
|
+
decrementOnce();
|
|
28525
|
+
throw err;
|
|
28526
|
+
}
|
|
28527
|
+
return {
|
|
28528
|
+
async release() {
|
|
28529
|
+
const MAX_ATTEMPTS = 3;
|
|
28530
|
+
const BACKOFF_MS = 100;
|
|
28531
|
+
let lastErr = null;
|
|
28532
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
28533
|
+
try {
|
|
28534
|
+
await reserved`SELECT pg_advisory_unlock(${CONV_LOCK_KEY1}, ${key2})`;
|
|
28535
|
+
lastErr = null;
|
|
28536
|
+
break;
|
|
28537
|
+
} catch (err) {
|
|
28538
|
+
lastErr = err;
|
|
28539
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
28540
|
+
await new Promise((r) => setTimeout(r, BACKOFF_MS * attempt));
|
|
28541
|
+
}
|
|
28542
|
+
}
|
|
28543
|
+
}
|
|
28544
|
+
if (lastErr) {
|
|
28545
|
+
logger90.error(
|
|
28546
|
+
`Failed to release advisory lock after ${MAX_ATTEMPTS} attempts for ${organizationId}/${agentId}/${conversationId}: ${lastErr instanceof Error ? lastErr.message : String(lastErr)}`
|
|
28547
|
+
);
|
|
28548
|
+
}
|
|
28549
|
+
try {
|
|
28550
|
+
reserved.release();
|
|
28551
|
+
} catch {
|
|
28552
|
+
}
|
|
28553
|
+
decrementOnce();
|
|
28554
|
+
}
|
|
28555
|
+
};
|
|
28556
|
+
}
|
|
28557
|
+
function hashConvKey2(organizationId, agentId, conversationId) {
|
|
28558
|
+
const input = `${organizationId}:${agentId}:${conversationId}`;
|
|
28559
|
+
let hash = 2166136261;
|
|
28560
|
+
for (let i = 0; i < input.length; i++) {
|
|
28561
|
+
hash ^= input.charCodeAt(i);
|
|
28562
|
+
hash = hash + ((hash << 1) + (hash << 4) + (hash << 7) + (hash << 8) + (hash << 24)) | 0;
|
|
28563
|
+
}
|
|
28564
|
+
return hash;
|
|
28565
|
+
}
|
|
28062
28566
|
function buildEmbeddedWorkerPath(binPathEntries, existingPath) {
|
|
28063
28567
|
const segments = (existingPath || "").split(":").filter(Boolean);
|
|
28064
28568
|
for (const candidate of [...binPathEntries ?? []].reverse()) {
|
|
@@ -28115,15 +28619,20 @@ function nixPackageAttrRef(pkg) {
|
|
|
28115
28619
|
}
|
|
28116
28620
|
return `pkgs.${namespace}.${leaf}`;
|
|
28117
28621
|
}
|
|
28118
|
-
var
|
|
28622
|
+
var logger90, KILL_TIMEOUT_MS, cachedSystemdRun, CONV_LOCK_KEY1, POOL_HEADROOM, reservedLockCount, warnedNearCap, NIX_PACKAGE_NAMESPACES, NIX_LEAF_RE, NIX_ATTR_LEAF_RE, EmbeddedDeploymentManager;
|
|
28119
28623
|
var init_embedded_deployment = __esm({
|
|
28120
28624
|
"src/gateway/orchestration/impl/embedded-deployment.ts"() {
|
|
28121
28625
|
"use strict";
|
|
28122
28626
|
init_src();
|
|
28627
|
+
init_client();
|
|
28123
28628
|
init_base_deployment_manager();
|
|
28124
28629
|
init_deployment_utils();
|
|
28125
|
-
|
|
28630
|
+
logger90 = createLogger("orchestrator");
|
|
28126
28631
|
KILL_TIMEOUT_MS = 5e3;
|
|
28632
|
+
CONV_LOCK_KEY1 = 1819239029;
|
|
28633
|
+
POOL_HEADROOM = 5;
|
|
28634
|
+
reservedLockCount = 0;
|
|
28635
|
+
warnedNearCap = false;
|
|
28127
28636
|
NIX_PACKAGE_NAMESPACES = /* @__PURE__ */ new Set([
|
|
28128
28637
|
"python3Packages",
|
|
28129
28638
|
"python311Packages",
|
|
@@ -28175,7 +28684,7 @@ var init_embedded_deployment = __esm({
|
|
|
28175
28684
|
`Worker entry point not found: ${entryPoint}`
|
|
28176
28685
|
);
|
|
28177
28686
|
}
|
|
28178
|
-
|
|
28687
|
+
logger90.debug(`Worker entry point verified: ${entryPoint}`);
|
|
28179
28688
|
}
|
|
28180
28689
|
async spawnDeployment(deploymentName, username, userId, messageData) {
|
|
28181
28690
|
if (this.workers.has(deploymentName)) {
|
|
@@ -28196,111 +28705,164 @@ var init_embedded_deployment = __esm({
|
|
|
28196
28705
|
}
|
|
28197
28706
|
const workspaceDir = path5.resolve(`workspaces/${agentId}`);
|
|
28198
28707
|
fs2.mkdirSync(workspaceDir, { recursive: true, mode: 448 });
|
|
28199
|
-
const
|
|
28200
|
-
|
|
28201
|
-
|
|
28202
|
-
|
|
28203
|
-
|
|
28204
|
-
|
|
28205
|
-
|
|
28206
|
-
|
|
28207
|
-
|
|
28208
|
-
|
|
28209
|
-
|
|
28210
|
-
|
|
28211
|
-
|
|
28212
|
-
|
|
28213
|
-
|
|
28214
|
-
|
|
28215
|
-
|
|
28216
|
-
|
|
28217
|
-
|
|
28218
|
-
|
|
28219
|
-
|
|
28220
|
-
|
|
28221
|
-
|
|
28222
|
-
|
|
28223
|
-
|
|
28224
|
-
|
|
28225
|
-
|
|
28226
|
-
|
|
28227
|
-
|
|
28228
|
-
|
|
28229
|
-
|
|
28230
|
-
|
|
28231
|
-
|
|
28232
|
-
|
|
28233
|
-
|
|
28708
|
+
const snapshotModeEnabled = process.env.LOBU_SESSION_STORE !== "file";
|
|
28709
|
+
const conversationId = typeof messageData?.conversationId === "string" ? messageData.conversationId : null;
|
|
28710
|
+
const organizationId = typeof messageData?.organizationId === "string" ? messageData.organizationId : null;
|
|
28711
|
+
let convLock = null;
|
|
28712
|
+
if (snapshotModeEnabled && organizationId && conversationId) {
|
|
28713
|
+
try {
|
|
28714
|
+
convLock = await acquireConversationLock(
|
|
28715
|
+
organizationId,
|
|
28716
|
+
agentId,
|
|
28717
|
+
conversationId
|
|
28718
|
+
);
|
|
28719
|
+
} catch (err) {
|
|
28720
|
+
logger90.error(
|
|
28721
|
+
`Failed to acquire conversation lock: ${err instanceof Error ? err.message : String(err)}`
|
|
28722
|
+
);
|
|
28723
|
+
throw new OrchestratorError(
|
|
28724
|
+
"DEPLOYMENT_CREATE_FAILED" /* DEPLOYMENT_CREATE_FAILED */,
|
|
28725
|
+
"Could not acquire per-conversation lock"
|
|
28726
|
+
);
|
|
28727
|
+
}
|
|
28728
|
+
if (!convLock) {
|
|
28729
|
+
logger90.info(
|
|
28730
|
+
`Conversation lock busy for ${organizationId}/${agentId}/${conversationId}; deferring spawn`
|
|
28731
|
+
);
|
|
28732
|
+
throw new OrchestratorError(
|
|
28733
|
+
"DEPLOYMENT_CREATE_FAILED" /* DEPLOYMENT_CREATE_FAILED */,
|
|
28734
|
+
"Conversation lock busy on another pod"
|
|
28735
|
+
);
|
|
28736
|
+
}
|
|
28737
|
+
}
|
|
28738
|
+
let child;
|
|
28739
|
+
let commonEnvVars;
|
|
28740
|
+
try {
|
|
28741
|
+
commonEnvVars = await this.generateEnvironmentVariables(
|
|
28742
|
+
username,
|
|
28743
|
+
userId,
|
|
28744
|
+
deploymentName,
|
|
28745
|
+
messageData,
|
|
28746
|
+
true
|
|
28234
28747
|
);
|
|
28235
|
-
|
|
28236
|
-
|
|
28237
|
-
|
|
28238
|
-
|
|
28239
|
-
|
|
28240
|
-
|
|
28241
|
-
|
|
28242
|
-
const innerCommand = command;
|
|
28243
|
-
const innerArgs = spawnArgs;
|
|
28244
|
-
command = systemdRun;
|
|
28245
|
-
spawnArgs = [
|
|
28246
|
-
...buildSystemdRunArgs({ unitName, workspaceDir }),
|
|
28247
|
-
"--",
|
|
28248
|
-
innerCommand,
|
|
28249
|
-
...innerArgs
|
|
28250
|
-
];
|
|
28251
|
-
logger89.info(
|
|
28252
|
-
`Spawning embedded worker ${deploymentName} under systemd-run scope ${unitName}`
|
|
28748
|
+
commonEnvVars.WORKSPACE_DIR = workspaceDir;
|
|
28749
|
+
if (!snapshotModeEnabled) {
|
|
28750
|
+
commonEnvVars.LOBU_SESSION_STORE = "file";
|
|
28751
|
+
}
|
|
28752
|
+
const embeddedPath = buildEmbeddedWorkerPath(
|
|
28753
|
+
this.config.worker.binPathEntries,
|
|
28754
|
+
commonEnvVars.PATH || process.env.PATH
|
|
28253
28755
|
);
|
|
28756
|
+
if (embeddedPath) {
|
|
28757
|
+
commonEnvVars.PATH = embeddedPath;
|
|
28758
|
+
}
|
|
28759
|
+
const allowedDomains = messageData?.networkConfig?.allowedDomains ?? [];
|
|
28760
|
+
if (allowedDomains.length > 0) {
|
|
28761
|
+
commonEnvVars.JUST_BASH_ALLOWED_DOMAINS = JSON.stringify(allowedDomains);
|
|
28762
|
+
}
|
|
28763
|
+
const nixPackages = messageData?.nixConfig?.packages ?? [];
|
|
28764
|
+
const workerEntryPoint = this.getWorkerEntryPoint();
|
|
28765
|
+
const workerInvocation = buildWorkerInvocation(workerEntryPoint);
|
|
28766
|
+
let command;
|
|
28767
|
+
let spawnArgs;
|
|
28768
|
+
if (nixPackages.length > 0) {
|
|
28769
|
+
const packageRefs = nixPackages.map(nixPackageAttrRef);
|
|
28770
|
+
command = "nix-shell";
|
|
28771
|
+
spawnArgs = [
|
|
28772
|
+
"-E",
|
|
28773
|
+
`let pkgs = import <nixpkgs> {}; in pkgs.mkShell { buildInputs = [ ${packageRefs.join(" ")} ]; }`,
|
|
28774
|
+
"--run",
|
|
28775
|
+
buildShellCommand(workerInvocation.command, workerInvocation.args)
|
|
28776
|
+
];
|
|
28777
|
+
logger90.info(
|
|
28778
|
+
`Spawning embedded worker ${deploymentName} with nix packages: ${nixPackages.join(", ")}`
|
|
28779
|
+
);
|
|
28780
|
+
} else {
|
|
28781
|
+
command = workerInvocation.command;
|
|
28782
|
+
spawnArgs = workerInvocation.args;
|
|
28783
|
+
}
|
|
28784
|
+
const systemdRun = locateSystemdRun();
|
|
28785
|
+
if (systemdRun) {
|
|
28786
|
+
const unitName = makeUnitName(deploymentName);
|
|
28787
|
+
const innerCommand = command;
|
|
28788
|
+
const innerArgs = spawnArgs;
|
|
28789
|
+
command = systemdRun;
|
|
28790
|
+
spawnArgs = [
|
|
28791
|
+
...buildSystemdRunArgs({ unitName, workspaceDir }),
|
|
28792
|
+
"--",
|
|
28793
|
+
innerCommand,
|
|
28794
|
+
...innerArgs
|
|
28795
|
+
];
|
|
28796
|
+
logger90.info(
|
|
28797
|
+
`Spawning embedded worker ${deploymentName} under systemd-run scope ${unitName}`
|
|
28798
|
+
);
|
|
28799
|
+
}
|
|
28800
|
+
child = spawn(command, spawnArgs, {
|
|
28801
|
+
// Workers must not inherit gateway-only secrets or telemetry settings
|
|
28802
|
+
// (DATABASE_URL, SENTRY_DSN, OAuth secrets, etc.). Everything a worker
|
|
28803
|
+
// needs is assembled explicitly above, with optional operator-provided
|
|
28804
|
+
// values forwarded only via WORKER_ENV_*.
|
|
28805
|
+
env: commonEnvVars,
|
|
28806
|
+
cwd: workspaceDir,
|
|
28807
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
28808
|
+
});
|
|
28809
|
+
} catch (err) {
|
|
28810
|
+
if (convLock) {
|
|
28811
|
+
void convLock.release();
|
|
28812
|
+
}
|
|
28813
|
+
throw err;
|
|
28254
28814
|
}
|
|
28255
|
-
|
|
28256
|
-
|
|
28257
|
-
|
|
28258
|
-
|
|
28259
|
-
|
|
28260
|
-
|
|
28261
|
-
|
|
28262
|
-
|
|
28263
|
-
});
|
|
28815
|
+
let lockReleased = false;
|
|
28816
|
+
const releaseLockOnce = async () => {
|
|
28817
|
+
if (lockReleased) return;
|
|
28818
|
+
lockReleased = true;
|
|
28819
|
+
if (convLock) {
|
|
28820
|
+
await convLock.release();
|
|
28821
|
+
}
|
|
28822
|
+
};
|
|
28264
28823
|
child.once("error", (err) => {
|
|
28265
|
-
|
|
28824
|
+
logger90.error(
|
|
28266
28825
|
`Embedded worker ${deploymentName} spawn error: ${err.message}`
|
|
28267
28826
|
);
|
|
28268
28827
|
this.workers.delete(deploymentName);
|
|
28828
|
+
releaseLockOnce();
|
|
28269
28829
|
});
|
|
28270
28830
|
child.stdout?.on("data", (data) => {
|
|
28271
28831
|
for (const line of data.toString().trimEnd().split("\n")) {
|
|
28272
|
-
|
|
28832
|
+
logger90.info({ worker: deploymentName }, line);
|
|
28273
28833
|
}
|
|
28274
28834
|
});
|
|
28275
28835
|
child.stderr?.on("data", (data) => {
|
|
28276
28836
|
for (const line of data.toString().trimEnd().split("\n")) {
|
|
28277
|
-
|
|
28837
|
+
logger90.warn({ worker: deploymentName }, line);
|
|
28278
28838
|
}
|
|
28279
28839
|
});
|
|
28280
28840
|
child.once("exit", (code, signal) => {
|
|
28281
|
-
|
|
28282
|
-
|
|
28283
|
-
this.workers.delete(deploymentName);
|
|
28284
|
-
}
|
|
28841
|
+
this.workers.delete(deploymentName);
|
|
28842
|
+
releaseLockOnce();
|
|
28285
28843
|
if (signal) {
|
|
28286
|
-
|
|
28844
|
+
logger90.info(
|
|
28287
28845
|
`Embedded worker ${deploymentName} exited with signal ${signal}`
|
|
28288
28846
|
);
|
|
28289
28847
|
} else if (code !== 0) {
|
|
28290
|
-
|
|
28848
|
+
logger90.error(
|
|
28291
28849
|
`Embedded worker ${deploymentName} exited with code ${code}`
|
|
28292
28850
|
);
|
|
28293
28851
|
} else {
|
|
28294
|
-
|
|
28852
|
+
logger90.info(`Embedded worker ${deploymentName} exited cleanly`);
|
|
28295
28853
|
}
|
|
28296
28854
|
});
|
|
28297
28855
|
this.workers.set(deploymentName, {
|
|
28298
28856
|
process: child,
|
|
28299
28857
|
env: commonEnvVars,
|
|
28300
28858
|
lastActivity: /* @__PURE__ */ new Date(),
|
|
28301
|
-
workspaceDir
|
|
28859
|
+
workspaceDir,
|
|
28860
|
+
// Expose the idempotent release on the entry for introspection /
|
|
28861
|
+
// tests. The exit handler is the authoritative release site;
|
|
28862
|
+
// killWorker no longer touches this field.
|
|
28863
|
+
...convLock ? { releaseConvLock: releaseLockOnce } : {}
|
|
28302
28864
|
});
|
|
28303
|
-
|
|
28865
|
+
logger90.info(
|
|
28304
28866
|
`Started embedded worker subprocess for ${deploymentName} (pid=${child.pid})`
|
|
28305
28867
|
);
|
|
28306
28868
|
}
|
|
@@ -28308,7 +28870,7 @@ var init_embedded_deployment = __esm({
|
|
|
28308
28870
|
const entry = this.workers.get(deploymentName);
|
|
28309
28871
|
if (replicas === 0 && entry) {
|
|
28310
28872
|
await this.killWorker(entry, deploymentName);
|
|
28311
|
-
|
|
28873
|
+
logger90.info(`Stopped embedded worker ${deploymentName}`);
|
|
28312
28874
|
} else if (replicas === 1 && !entry) {
|
|
28313
28875
|
throw new Error(
|
|
28314
28876
|
`Embedded worker ${deploymentName} is not running \u2014 must re-create`
|
|
@@ -28319,7 +28881,7 @@ var init_embedded_deployment = __esm({
|
|
|
28319
28881
|
const entry = this.workers.get(deploymentName);
|
|
28320
28882
|
if (entry) {
|
|
28321
28883
|
await this.killWorker(entry, deploymentName);
|
|
28322
|
-
|
|
28884
|
+
logger90.info(`Stopped embedded worker: ${deploymentName}`);
|
|
28323
28885
|
}
|
|
28324
28886
|
}
|
|
28325
28887
|
async listDeployments() {
|
|
@@ -28347,7 +28909,14 @@ var init_embedded_deployment = __esm({
|
|
|
28347
28909
|
entry.lastActivity = /* @__PURE__ */ new Date();
|
|
28348
28910
|
}
|
|
28349
28911
|
}
|
|
28350
|
-
/** Send SIGTERM, then SIGKILL after timeout. Resolves on child exit.
|
|
28912
|
+
/** Send SIGTERM, then SIGKILL after timeout. Resolves on child exit.
|
|
28913
|
+
*
|
|
28914
|
+
* Does NOT release the conversation lock — the child's exit handler is
|
|
28915
|
+
* the authoritative release site, and the release call there is
|
|
28916
|
+
* idempotent. Releasing here before `await exited` (as a prior version
|
|
28917
|
+
* did) lets a sibling pod claim the conversation while this worker is
|
|
28918
|
+
* still flushing its cleanup() snapshot. Codex P1#3 on PR #865.
|
|
28919
|
+
*/
|
|
28351
28920
|
async killWorker(entry, deploymentName) {
|
|
28352
28921
|
const child = entry.process;
|
|
28353
28922
|
this.workers.delete(deploymentName);
|
|
@@ -28358,7 +28927,7 @@ var init_embedded_deployment = __esm({
|
|
|
28358
28927
|
child.kill("SIGTERM");
|
|
28359
28928
|
const killTimer = setTimeout(() => {
|
|
28360
28929
|
if (child.exitCode === null && child.signalCode === null) {
|
|
28361
|
-
|
|
28930
|
+
logger90.warn(
|
|
28362
28931
|
`Embedded worker ${deploymentName} did not exit after SIGTERM, sending SIGKILL`
|
|
28363
28932
|
);
|
|
28364
28933
|
child.kill("SIGKILL");
|
|
@@ -28376,14 +28945,14 @@ var init_embedded_deployment = __esm({
|
|
|
28376
28945
|
|
|
28377
28946
|
// src/gateway/orchestration/message-consumer.ts
|
|
28378
28947
|
import * as Sentry3 from "@sentry/node";
|
|
28379
|
-
var
|
|
28948
|
+
var logger91, MessageConsumer;
|
|
28380
28949
|
var init_message_consumer = __esm({
|
|
28381
28950
|
"src/gateway/orchestration/message-consumer.ts"() {
|
|
28382
28951
|
"use strict";
|
|
28383
28952
|
init_src();
|
|
28384
28953
|
init_queue();
|
|
28385
28954
|
init_base_deployment_manager();
|
|
28386
|
-
|
|
28955
|
+
logger91 = createLogger("orchestrator");
|
|
28387
28956
|
MessageConsumer = class {
|
|
28388
28957
|
queue;
|
|
28389
28958
|
deploymentManager;
|
|
@@ -28407,7 +28976,7 @@ var init_message_consumer = __esm({
|
|
|
28407
28976
|
await this.queue.start();
|
|
28408
28977
|
this.isRunning = true;
|
|
28409
28978
|
await this.queue.createQueue("messages");
|
|
28410
|
-
|
|
28979
|
+
logger91.debug("Created/verified messages queue");
|
|
28411
28980
|
await this.queue.work(
|
|
28412
28981
|
"messages",
|
|
28413
28982
|
async (job) => {
|
|
@@ -28425,7 +28994,7 @@ var init_message_consumer = __esm({
|
|
|
28425
28994
|
);
|
|
28426
28995
|
}
|
|
28427
28996
|
);
|
|
28428
|
-
|
|
28997
|
+
logger91.debug("Queue consumer started");
|
|
28429
28998
|
} catch (error) {
|
|
28430
28999
|
throw new OrchestratorError(
|
|
28431
29000
|
"QUEUE_JOB_PROCESSING_FAILED" /* QUEUE_JOB_PROCESSING_FAILED */,
|
|
@@ -28455,7 +29024,7 @@ var init_message_consumer = __esm({
|
|
|
28455
29024
|
"lobu.conversation_id": data?.conversationId || "unknown"
|
|
28456
29025
|
});
|
|
28457
29026
|
const childTraceparent = getTraceparent(queueSpan) || traceparent;
|
|
28458
|
-
|
|
29027
|
+
logger91.info(
|
|
28459
29028
|
{
|
|
28460
29029
|
traceparent,
|
|
28461
29030
|
traceId,
|
|
@@ -28466,6 +29035,10 @@ var init_message_consumer = __esm({
|
|
|
28466
29035
|
"Processing job with trace context"
|
|
28467
29036
|
);
|
|
28468
29037
|
try {
|
|
29038
|
+
const parsedRunId = Number(jobId);
|
|
29039
|
+
if (Number.isFinite(parsedRunId) && parsedRunId > 0) {
|
|
29040
|
+
data.runId = parsedRunId;
|
|
29041
|
+
}
|
|
28469
29042
|
const effectiveConversationId = data.conversationId;
|
|
28470
29043
|
if (!effectiveConversationId) {
|
|
28471
29044
|
throw new OrchestratorError(
|
|
@@ -28486,7 +29059,22 @@ var init_message_consumer = __esm({
|
|
|
28486
29059
|
channelId: data.channelId,
|
|
28487
29060
|
conversationId: effectiveConversationId
|
|
28488
29061
|
});
|
|
28489
|
-
|
|
29062
|
+
if (data.runId !== void 0) {
|
|
29063
|
+
data.runJobToken = generateWorkerToken(
|
|
29064
|
+
data.userId,
|
|
29065
|
+
effectiveConversationId,
|
|
29066
|
+
deploymentName,
|
|
29067
|
+
{
|
|
29068
|
+
channelId: data.channelId,
|
|
29069
|
+
teamId: data.teamId,
|
|
29070
|
+
agentId: data.agentId,
|
|
29071
|
+
organizationId: data.organizationId,
|
|
29072
|
+
platform: data.platform,
|
|
29073
|
+
runId: data.runId
|
|
29074
|
+
}
|
|
29075
|
+
);
|
|
29076
|
+
}
|
|
29077
|
+
logger91.info(
|
|
28490
29078
|
`Conversation routing - effectiveConversationId: ${effectiveConversationId}, canonicalKey: ${canonicalConversationKey}, deploymentName: ${deploymentName}`
|
|
28491
29079
|
);
|
|
28492
29080
|
await Sentry3.startSpan(
|
|
@@ -28503,7 +29091,7 @@ var init_message_consumer = __esm({
|
|
|
28503
29091
|
await this.sendToWorkerQueue(data, deploymentName);
|
|
28504
29092
|
}
|
|
28505
29093
|
);
|
|
28506
|
-
|
|
29094
|
+
logger91.info(
|
|
28507
29095
|
{ traceId, traceparent: childTraceparent, deploymentName },
|
|
28508
29096
|
"Enqueued message to thread queue"
|
|
28509
29097
|
);
|
|
@@ -28523,7 +29111,7 @@ var init_message_consumer = __esm({
|
|
|
28523
29111
|
},
|
|
28524
29112
|
level: "error"
|
|
28525
29113
|
});
|
|
28526
|
-
|
|
29114
|
+
logger91.error(
|
|
28527
29115
|
{
|
|
28528
29116
|
traceId,
|
|
28529
29117
|
error: bgError instanceof Error ? bgError.message : String(bgError),
|
|
@@ -28536,13 +29124,13 @@ var init_message_consumer = __esm({
|
|
|
28536
29124
|
);
|
|
28537
29125
|
this.trackFailedDeployment(deploymentName, data, bgError).catch(
|
|
28538
29126
|
(trackError) => {
|
|
28539
|
-
|
|
29127
|
+
logger91.error("Failed to track deployment failure:", trackError);
|
|
28540
29128
|
}
|
|
28541
29129
|
);
|
|
28542
29130
|
});
|
|
28543
29131
|
queueSpan?.setStatus({ code: SpanStatusCode.OK });
|
|
28544
29132
|
queueSpan?.end();
|
|
28545
|
-
|
|
29133
|
+
logger91.info({ traceId, jobId }, "Message job queued successfully");
|
|
28546
29134
|
} catch (error) {
|
|
28547
29135
|
queueSpan?.setStatus({
|
|
28548
29136
|
code: SpanStatusCode.ERROR,
|
|
@@ -28550,7 +29138,7 @@ var init_message_consumer = __esm({
|
|
|
28550
29138
|
});
|
|
28551
29139
|
queueSpan?.end();
|
|
28552
29140
|
Sentry3.captureException(error);
|
|
28553
|
-
|
|
29141
|
+
logger91.error({ traceId, jobId, error }, "Message job failed");
|
|
28554
29142
|
throw new OrchestratorError(
|
|
28555
29143
|
"QUEUE_JOB_PROCESSING_FAILED" /* QUEUE_JOB_PROCESSING_FAILED */,
|
|
28556
29144
|
`Failed to process message job: ${error instanceof Error ? error.message : String(error)}`,
|
|
@@ -28582,11 +29170,11 @@ var init_message_consumer = __esm({
|
|
|
28582
29170
|
true
|
|
28583
29171
|
);
|
|
28584
29172
|
}
|
|
28585
|
-
|
|
29173
|
+
logger91.info(
|
|
28586
29174
|
`\u2705 Sent message to thread queue ${threadQueueName} for conversation ${data.conversationId}, jobId: ${jobId}`
|
|
28587
29175
|
);
|
|
28588
29176
|
} catch (error) {
|
|
28589
|
-
|
|
29177
|
+
logger91.error(`\u274C [ERROR] sendToWorkerQueue failed:`, error);
|
|
28590
29178
|
throw new OrchestratorError(
|
|
28591
29179
|
"QUEUE_JOB_PROCESSING_FAILED" /* QUEUE_JOB_PROCESSING_FAILED */,
|
|
28592
29180
|
`Failed to send message to thread queue: ${error instanceof Error ? error.message : String(error)}`,
|
|
@@ -28632,7 +29220,7 @@ var init_message_consumer = __esm({
|
|
|
28632
29220
|
if (isNewThread) {
|
|
28633
29221
|
const acquired = this.acquireDeploymentLock(deploymentName);
|
|
28634
29222
|
if (!acquired) {
|
|
28635
|
-
|
|
29223
|
+
logger91.info(
|
|
28636
29224
|
{ traceId, deploymentName },
|
|
28637
29225
|
"Another handler is creating this deployment, waiting"
|
|
28638
29226
|
);
|
|
@@ -28640,7 +29228,7 @@ var init_message_consumer = __esm({
|
|
|
28640
29228
|
const rechecked = await this.deploymentManager.listDeployments();
|
|
28641
29229
|
if (rechecked.some((d) => d.deploymentName === deploymentName)) {
|
|
28642
29230
|
await this.deploymentManager.scaleDeployment(deploymentName, 1);
|
|
28643
|
-
|
|
29231
|
+
logger91.info(
|
|
28644
29232
|
{ traceId, deploymentName },
|
|
28645
29233
|
"Deployment created by other handler, scaled up"
|
|
28646
29234
|
);
|
|
@@ -28654,7 +29242,7 @@ var init_message_consumer = __esm({
|
|
|
28654
29242
|
try {
|
|
28655
29243
|
const recheckAfterLock = await this.deploymentManager.listDeployments();
|
|
28656
29244
|
if (recheckAfterLock.some((d) => d.deploymentName === deploymentName)) {
|
|
28657
|
-
|
|
29245
|
+
logger91.info(
|
|
28658
29246
|
{ traceId, deploymentName },
|
|
28659
29247
|
"Deployment already created by another handler after lock acquired"
|
|
28660
29248
|
);
|
|
@@ -28664,7 +29252,7 @@ var init_message_consumer = __esm({
|
|
|
28664
29252
|
);
|
|
28665
29253
|
return;
|
|
28666
29254
|
}
|
|
28667
|
-
|
|
29255
|
+
logger91.info(
|
|
28668
29256
|
{ traceId, traceparent, conversationId, deploymentName },
|
|
28669
29257
|
"New thread - creating deployment"
|
|
28670
29258
|
);
|
|
@@ -28674,24 +29262,24 @@ var init_message_consumer = __esm({
|
|
|
28674
29262
|
dataWithTrace,
|
|
28675
29263
|
recheckAfterLock
|
|
28676
29264
|
);
|
|
28677
|
-
|
|
29265
|
+
logger91.info({ traceId, deploymentName }, "Created deployment");
|
|
28678
29266
|
} finally {
|
|
28679
29267
|
this.releaseDeploymentLock(deploymentName);
|
|
28680
29268
|
}
|
|
28681
29269
|
} else {
|
|
28682
|
-
|
|
29270
|
+
logger91.info(
|
|
28683
29271
|
{ traceId, conversationId, deploymentName },
|
|
28684
29272
|
"Existing thread - ensuring worker exists"
|
|
28685
29273
|
);
|
|
28686
29274
|
await this.deploymentManager.syncNetworkConfigGrants(dataWithTrace);
|
|
28687
29275
|
try {
|
|
28688
29276
|
await this.deploymentManager.scaleDeployment(deploymentName, 1);
|
|
28689
|
-
|
|
29277
|
+
logger91.info(
|
|
28690
29278
|
{ traceId, deploymentName },
|
|
28691
29279
|
"Scaled existing worker to 1"
|
|
28692
29280
|
);
|
|
28693
29281
|
} catch {
|
|
28694
|
-
|
|
29282
|
+
logger91.info(
|
|
28695
29283
|
{ traceId, conversationId, deploymentName },
|
|
28696
29284
|
"Worker doesn't exist, creating it"
|
|
28697
29285
|
);
|
|
@@ -28700,11 +29288,11 @@ var init_message_consumer = __esm({
|
|
|
28700
29288
|
conversationId,
|
|
28701
29289
|
dataWithTrace
|
|
28702
29290
|
);
|
|
28703
|
-
|
|
29291
|
+
logger91.info({ traceId, deploymentName }, "Created worker");
|
|
28704
29292
|
}
|
|
28705
29293
|
}
|
|
28706
29294
|
await this.deploymentManager.updateDeploymentActivity(deploymentName);
|
|
28707
|
-
|
|
29295
|
+
logger91.info({ traceId, deploymentName }, "Worker is ready");
|
|
28708
29296
|
},
|
|
28709
29297
|
{
|
|
28710
29298
|
maxRetries: 3,
|
|
@@ -28712,7 +29300,7 @@ var init_message_consumer = __esm({
|
|
|
28712
29300
|
strategy: "linear",
|
|
28713
29301
|
jitter: true,
|
|
28714
29302
|
onRetry: (attempt, error) => {
|
|
28715
|
-
|
|
29303
|
+
logger91.warn(
|
|
28716
29304
|
{ traceId, deploymentName, attempt, maxAttempts: 3 },
|
|
28717
29305
|
`Retry attempt failed: ${error.message}`
|
|
28718
29306
|
);
|
|
@@ -28726,7 +29314,7 @@ var init_message_consumer = __esm({
|
|
|
28726
29314
|
*/
|
|
28727
29315
|
async trackFailedDeployment(deploymentName, data, error) {
|
|
28728
29316
|
try {
|
|
28729
|
-
|
|
29317
|
+
logger91.error(
|
|
28730
29318
|
{
|
|
28731
29319
|
deploymentName,
|
|
28732
29320
|
userId: data.userId,
|
|
@@ -28752,10 +29340,10 @@ var init_message_consumer = __esm({
|
|
|
28752
29340
|
processedMessageIds: [data.messageId]
|
|
28753
29341
|
});
|
|
28754
29342
|
} catch (notifyError) {
|
|
28755
|
-
|
|
29343
|
+
logger91.error("Failed to send error notification to user:", notifyError);
|
|
28756
29344
|
}
|
|
28757
29345
|
} catch (trackError) {
|
|
28758
|
-
|
|
29346
|
+
logger91.error("Failed to track deployment failure:", trackError);
|
|
28759
29347
|
}
|
|
28760
29348
|
}
|
|
28761
29349
|
/**
|
|
@@ -28769,7 +29357,7 @@ var init_message_consumer = __esm({
|
|
|
28769
29357
|
isRunning: this.isRunning
|
|
28770
29358
|
};
|
|
28771
29359
|
} catch (error) {
|
|
28772
|
-
|
|
29360
|
+
logger91.error("Failed to get queue stats:", error);
|
|
28773
29361
|
return {
|
|
28774
29362
|
isRunning: this.isRunning,
|
|
28775
29363
|
error: error instanceof Error ? error.message : String(error)
|
|
@@ -28781,7 +29369,7 @@ var init_message_consumer = __esm({
|
|
|
28781
29369
|
});
|
|
28782
29370
|
|
|
28783
29371
|
// src/gateway/orchestration/index.ts
|
|
28784
|
-
var
|
|
29372
|
+
var logger92, Orchestrator;
|
|
28785
29373
|
var init_orchestration = __esm({
|
|
28786
29374
|
"src/gateway/orchestration/index.ts"() {
|
|
28787
29375
|
"use strict";
|
|
@@ -28793,7 +29381,7 @@ var init_orchestration = __esm({
|
|
|
28793
29381
|
init_deployment_utils();
|
|
28794
29382
|
init_embedded_deployment();
|
|
28795
29383
|
init_message_consumer();
|
|
28796
|
-
|
|
29384
|
+
logger92 = createLogger("orchestrator");
|
|
28797
29385
|
Orchestrator = class {
|
|
28798
29386
|
config;
|
|
28799
29387
|
deploymentManager;
|
|
@@ -28833,7 +29421,7 @@ var init_orchestration = __esm({
|
|
|
28833
29421
|
}
|
|
28834
29422
|
const providerModules = getModelProviderModules();
|
|
28835
29423
|
this.deploymentManager.setProviderModules(providerModules);
|
|
28836
|
-
|
|
29424
|
+
logger92.debug(
|
|
28837
29425
|
`Provider modules injected into orchestrator (${providerModules.length})`
|
|
28838
29426
|
);
|
|
28839
29427
|
}
|
|
@@ -28846,9 +29434,9 @@ var init_orchestration = __esm({
|
|
|
28846
29434
|
await this.queueConsumer.start();
|
|
28847
29435
|
this.setupIdleCleanup();
|
|
28848
29436
|
this.isRunning = true;
|
|
28849
|
-
|
|
29437
|
+
logger92.debug("Orchestrator started");
|
|
28850
29438
|
} catch (error) {
|
|
28851
|
-
|
|
29439
|
+
logger92.error("\u274C Failed to start orchestrator:", error);
|
|
28852
29440
|
throw error;
|
|
28853
29441
|
}
|
|
28854
29442
|
}
|
|
@@ -28866,9 +29454,9 @@ var init_orchestration = __esm({
|
|
|
28866
29454
|
this.initialReconcileTimer = void 0;
|
|
28867
29455
|
}
|
|
28868
29456
|
if (this.activeReconciliation) {
|
|
28869
|
-
|
|
29457
|
+
logger92.info("Waiting for in-flight reconciliation to complete...");
|
|
28870
29458
|
const safeReconciliation = this.activeReconciliation.catch((error) => {
|
|
28871
|
-
|
|
29459
|
+
logger92.error(
|
|
28872
29460
|
"In-flight reconciliation failed during shutdown:",
|
|
28873
29461
|
error
|
|
28874
29462
|
);
|
|
@@ -28888,11 +29476,11 @@ var init_orchestration = __esm({
|
|
|
28888
29476
|
)
|
|
28889
29477
|
);
|
|
28890
29478
|
} catch (error) {
|
|
28891
|
-
|
|
29479
|
+
logger92.error("Error draining workers during shutdown:", error);
|
|
28892
29480
|
}
|
|
28893
|
-
|
|
29481
|
+
logger92.info("\u2705 Orchestrator stopped");
|
|
28894
29482
|
} catch (error) {
|
|
28895
|
-
|
|
29483
|
+
logger92.error("\u274C Error stopping orchestrator:", error);
|
|
28896
29484
|
}
|
|
28897
29485
|
}
|
|
28898
29486
|
setupIdleCleanup() {
|
|
@@ -28900,7 +29488,7 @@ var init_orchestration = __esm({
|
|
|
28900
29488
|
this.initialReconcileTimer = void 0;
|
|
28901
29489
|
if (this.shuttingDown) return;
|
|
28902
29490
|
const p = this.deploymentManager.reconcileDeployments().catch((error) => {
|
|
28903
|
-
|
|
29491
|
+
logger92.error("\u274C Initial deployment reconciliation failed:", error);
|
|
28904
29492
|
});
|
|
28905
29493
|
this.activeReconciliation = p;
|
|
28906
29494
|
p.finally(() => {
|
|
@@ -28910,7 +29498,7 @@ var init_orchestration = __esm({
|
|
|
28910
29498
|
this.cleanupInterval = setInterval(async () => {
|
|
28911
29499
|
if (this.shuttingDown) return;
|
|
28912
29500
|
if (this.isReconciling) {
|
|
28913
|
-
|
|
29501
|
+
logger92.debug(
|
|
28914
29502
|
"Skipping reconciliation interval: previous run still in progress"
|
|
28915
29503
|
);
|
|
28916
29504
|
return;
|
|
@@ -28921,7 +29509,7 @@ var init_orchestration = __esm({
|
|
|
28921
29509
|
this.activeReconciliation = p;
|
|
28922
29510
|
await p;
|
|
28923
29511
|
} catch (error) {
|
|
28924
|
-
|
|
29512
|
+
logger92.error(
|
|
28925
29513
|
"Error during deployment reconciliation:",
|
|
28926
29514
|
error instanceof Error ? error.message : String(error)
|
|
28927
29515
|
);
|
|
@@ -28960,20 +29548,20 @@ var init_orchestration = __esm({
|
|
|
28960
29548
|
function loadAllowedDomains() {
|
|
28961
29549
|
const allowedDomains = process.env.WORKER_ALLOWED_DOMAINS;
|
|
28962
29550
|
if (!allowedDomains) {
|
|
28963
|
-
|
|
29551
|
+
logger93.warn(
|
|
28964
29552
|
"\u26A0\uFE0F WORKER_ALLOWED_DOMAINS not set - workers will have NO internet access (complete isolation)"
|
|
28965
29553
|
);
|
|
28966
29554
|
return [];
|
|
28967
29555
|
}
|
|
28968
29556
|
const trimmed = allowedDomains.trim();
|
|
28969
29557
|
if (trimmed === "*") {
|
|
28970
|
-
|
|
29558
|
+
logger93.debug("WORKER_ALLOWED_DOMAINS=* - unrestricted internet access");
|
|
28971
29559
|
return ["*"];
|
|
28972
29560
|
}
|
|
28973
29561
|
const parsed = normalizeDomainPatterns(
|
|
28974
29562
|
trimmed.split(",").map((d) => d.trim()).filter((d) => d.length > 0)
|
|
28975
29563
|
) ?? [];
|
|
28976
|
-
|
|
29564
|
+
logger93.debug(
|
|
28977
29565
|
`Loaded ${parsed.length} allowed domains from WORKER_ALLOWED_DOMAINS`
|
|
28978
29566
|
);
|
|
28979
29567
|
return parsed;
|
|
@@ -28987,17 +29575,17 @@ function loadDisallowedDomains() {
|
|
|
28987
29575
|
const parsed = normalizeDomainPatterns(
|
|
28988
29576
|
disallowedDomains.split(",").map((d) => d.trim()).filter((d) => d.length > 0)
|
|
28989
29577
|
) ?? [];
|
|
28990
|
-
|
|
29578
|
+
logger93.debug(
|
|
28991
29579
|
`Loaded ${parsed.length} disallowed domains from WORKER_DISALLOWED_DOMAINS`
|
|
28992
29580
|
);
|
|
28993
29581
|
return parsed;
|
|
28994
29582
|
}
|
|
28995
|
-
var
|
|
29583
|
+
var logger93;
|
|
28996
29584
|
var init_network_allowlist = __esm({
|
|
28997
29585
|
"src/gateway/config/network-allowlist.ts"() {
|
|
28998
29586
|
"use strict";
|
|
28999
29587
|
init_src();
|
|
29000
|
-
|
|
29588
|
+
logger93 = createLogger("network-allowlist");
|
|
29001
29589
|
}
|
|
29002
29590
|
});
|
|
29003
29591
|
|
|
@@ -29031,7 +29619,7 @@ var init_policy_composer = __esm({
|
|
|
29031
29619
|
});
|
|
29032
29620
|
|
|
29033
29621
|
// src/gateway/proxy/egress-judge/judge.ts
|
|
29034
|
-
var
|
|
29622
|
+
var logger94;
|
|
29035
29623
|
var init_judge = __esm({
|
|
29036
29624
|
"src/gateway/proxy/egress-judge/judge.ts"() {
|
|
29037
29625
|
"use strict";
|
|
@@ -29040,7 +29628,7 @@ var init_judge = __esm({
|
|
|
29040
29628
|
init_cache();
|
|
29041
29629
|
init_circuit_breaker();
|
|
29042
29630
|
init_policy_composer();
|
|
29043
|
-
|
|
29631
|
+
logger94 = createLogger("egress-judge");
|
|
29044
29632
|
}
|
|
29045
29633
|
});
|
|
29046
29634
|
|
|
@@ -29077,7 +29665,7 @@ async function checkDomainAccess(hostname, agentId, organizationId, requestConte
|
|
|
29077
29665
|
organizationId
|
|
29078
29666
|
);
|
|
29079
29667
|
if (denied) {
|
|
29080
|
-
|
|
29668
|
+
logger95.debug(`Domain ${hostname} denied via grant (agent: ${agentId})`);
|
|
29081
29669
|
return { allowed: false, source: "grant" };
|
|
29082
29670
|
}
|
|
29083
29671
|
}
|
|
@@ -29090,7 +29678,7 @@ async function checkDomainAccess(hostname, agentId, organizationId, requestConte
|
|
|
29090
29678
|
organizationId
|
|
29091
29679
|
);
|
|
29092
29680
|
if (granted) {
|
|
29093
|
-
|
|
29681
|
+
logger95.debug(`Domain ${hostname} allowed via grant (agent: ${agentId})`);
|
|
29094
29682
|
return { allowed: true, source: "grant" };
|
|
29095
29683
|
}
|
|
29096
29684
|
}
|
|
@@ -29286,13 +29874,13 @@ function validateProxyAuth(req) {
|
|
|
29286
29874
|
}
|
|
29287
29875
|
const tokenData = verifyWorkerToken(creds.token);
|
|
29288
29876
|
if (!tokenData) {
|
|
29289
|
-
|
|
29877
|
+
logger95.warn(
|
|
29290
29878
|
`Proxy auth failed: invalid token (claimed deployment: ${creds.deploymentName})`
|
|
29291
29879
|
);
|
|
29292
29880
|
return null;
|
|
29293
29881
|
}
|
|
29294
29882
|
if (tokenData.jti && getRevokedTokenStore().isRevokedCached(tokenData.jti)) {
|
|
29295
|
-
|
|
29883
|
+
logger95.warn(
|
|
29296
29884
|
`Proxy auth failed: revoked jti (claimed deployment: ${creds.deploymentName})`
|
|
29297
29885
|
);
|
|
29298
29886
|
return null;
|
|
@@ -29302,7 +29890,7 @@ function validateProxyAuth(req) {
|
|
|
29302
29890
|
Buffer.from(creds.deploymentName)
|
|
29303
29891
|
);
|
|
29304
29892
|
if (!deploymentMatch) {
|
|
29305
|
-
|
|
29893
|
+
logger95.warn(
|
|
29306
29894
|
`Proxy auth failed: deployment mismatch (claimed: ${creds.deploymentName}, token: ${tokenData.deploymentName})`
|
|
29307
29895
|
);
|
|
29308
29896
|
return null;
|
|
@@ -29344,7 +29932,7 @@ function logAccessDecision(method, hostname, deploymentName, agentId, decision)
|
|
|
29344
29932
|
if (decision.allowed && decision.source === "global") {
|
|
29345
29933
|
return;
|
|
29346
29934
|
}
|
|
29347
|
-
|
|
29935
|
+
logger95.info("egress-decision", {
|
|
29348
29936
|
method,
|
|
29349
29937
|
hostname,
|
|
29350
29938
|
deploymentName,
|
|
@@ -29372,7 +29960,7 @@ async function handleConnect(req, clientSocket, head) {
|
|
|
29372
29960
|
const url = req.url || "";
|
|
29373
29961
|
const hostname = extractConnectHostname(url);
|
|
29374
29962
|
if (!hostname) {
|
|
29375
|
-
|
|
29963
|
+
logger95.warn(`Invalid CONNECT request: ${url}`);
|
|
29376
29964
|
clientSocket.write("HTTP/1.1 400 Bad Request\r\n\r\n");
|
|
29377
29965
|
clientSocket.end();
|
|
29378
29966
|
return;
|
|
@@ -29380,9 +29968,9 @@ async function handleConnect(req, clientSocket, head) {
|
|
|
29380
29968
|
let targetSocket = null;
|
|
29381
29969
|
clientSocket.on("error", (err) => {
|
|
29382
29970
|
if (err.code === "ECONNRESET") {
|
|
29383
|
-
|
|
29971
|
+
logger95.debug(`Client disconnected for ${hostname} (ECONNRESET)`);
|
|
29384
29972
|
} else {
|
|
29385
|
-
|
|
29973
|
+
logger95.debug(`Client connection error for ${hostname}: ${err.message}`);
|
|
29386
29974
|
}
|
|
29387
29975
|
try {
|
|
29388
29976
|
targetSocket?.end();
|
|
@@ -29397,7 +29985,7 @@ async function handleConnect(req, clientSocket, head) {
|
|
|
29397
29985
|
});
|
|
29398
29986
|
const auth = validateProxyAuth(req);
|
|
29399
29987
|
if (!auth) {
|
|
29400
|
-
|
|
29988
|
+
logger95.warn(`Proxy auth required for CONNECT to ${hostname}`);
|
|
29401
29989
|
try {
|
|
29402
29990
|
clientSocket.write(
|
|
29403
29991
|
'HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic realm="lobu-proxy"\r\n\r\n'
|
|
@@ -29422,7 +30010,7 @@ async function handleConnect(req, clientSocket, head) {
|
|
|
29422
30010
|
);
|
|
29423
30011
|
if (!decision.allowed) {
|
|
29424
30012
|
const reason = decision.judge?.reason ?? `Domain not allowed: ${hostname}`;
|
|
29425
|
-
|
|
30013
|
+
logger95.warn(
|
|
29426
30014
|
`Blocked CONNECT to ${hostname} (deployment: ${deploymentName}) - ${reason}`
|
|
29427
30015
|
);
|
|
29428
30016
|
try {
|
|
@@ -29440,7 +30028,7 @@ Content-Type: text/plain\r
|
|
|
29440
30028
|
}
|
|
29441
30029
|
const targetResolution = await resolveAndValidateTarget(hostname);
|
|
29442
30030
|
if (!targetResolution.ok) {
|
|
29443
|
-
|
|
30031
|
+
logger95.warn(
|
|
29444
30032
|
`Blocked CONNECT to ${hostname} (deployment: ${deploymentName}) - ${targetResolution.reason}`
|
|
29445
30033
|
);
|
|
29446
30034
|
try {
|
|
@@ -29462,17 +30050,17 @@ ${targetResolution.clientMessage}\r
|
|
|
29462
30050
|
clientSocket.end();
|
|
29463
30051
|
return;
|
|
29464
30052
|
}
|
|
29465
|
-
|
|
30053
|
+
logger95.debug(`Allowing CONNECT to ${hostname} via ${resolvedIp}`);
|
|
29466
30054
|
const [host, portStr] = url.split(":");
|
|
29467
30055
|
const port = portStr ? Number.parseInt(portStr, 10) : 443;
|
|
29468
30056
|
if (!host) {
|
|
29469
|
-
|
|
30057
|
+
logger95.warn(`Invalid CONNECT host: ${url}`);
|
|
29470
30058
|
clientSocket.write("HTTP/1.1 400 Bad Request\r\n\r\n");
|
|
29471
30059
|
clientSocket.end();
|
|
29472
30060
|
return;
|
|
29473
30061
|
}
|
|
29474
30062
|
if (!Number.isInteger(port) || port < 1 || port > 65535) {
|
|
29475
|
-
|
|
30063
|
+
logger95.warn(`Invalid CONNECT port: ${url}`);
|
|
29476
30064
|
clientSocket.write("HTTP/1.1 400 Bad Request\r\n\r\n");
|
|
29477
30065
|
clientSocket.end();
|
|
29478
30066
|
return;
|
|
@@ -29485,7 +30073,7 @@ ${targetResolution.clientMessage}\r
|
|
|
29485
30073
|
});
|
|
29486
30074
|
targetSocket = tunnelSocket;
|
|
29487
30075
|
tunnelSocket.on("error", (err) => {
|
|
29488
|
-
|
|
30076
|
+
logger95.debug(`Target connection error for ${hostname}: ${err.message}`);
|
|
29489
30077
|
try {
|
|
29490
30078
|
clientSocket.end();
|
|
29491
30079
|
} catch {
|
|
@@ -29516,7 +30104,7 @@ async function handleProxyRequest(req, res) {
|
|
|
29516
30104
|
const hostname = parsedUrl.hostname;
|
|
29517
30105
|
const auth = validateProxyAuth(req);
|
|
29518
30106
|
if (!auth) {
|
|
29519
|
-
|
|
30107
|
+
logger95.warn(`Proxy auth required for ${req.method} ${hostname}`);
|
|
29520
30108
|
res.writeHead(407, {
|
|
29521
30109
|
"Content-Type": "text/plain",
|
|
29522
30110
|
"Proxy-Authenticate": 'Basic realm="lobu-proxy"'
|
|
@@ -29538,7 +30126,7 @@ async function handleProxyRequest(req, res) {
|
|
|
29538
30126
|
);
|
|
29539
30127
|
if (!decision.allowed) {
|
|
29540
30128
|
const reason = decision.judge?.reason ?? `Domain not allowed: ${hostname}`;
|
|
29541
|
-
|
|
30129
|
+
logger95.warn(
|
|
29542
30130
|
`Blocked request to ${hostname} (deployment: ${deploymentName}) - ${reason}`
|
|
29543
30131
|
);
|
|
29544
30132
|
res.writeHead(403, escapeHeaderValue(reason), {
|
|
@@ -29552,7 +30140,7 @@ async function handleProxyRequest(req, res) {
|
|
|
29552
30140
|
}
|
|
29553
30141
|
const targetResolution = await resolveAndValidateTarget(hostname);
|
|
29554
30142
|
if (!targetResolution.ok) {
|
|
29555
|
-
|
|
30143
|
+
logger95.warn(
|
|
29556
30144
|
`Blocked request to ${hostname} (deployment: ${deploymentName}) - ${targetResolution.reason}`
|
|
29557
30145
|
);
|
|
29558
30146
|
res.writeHead(targetResolution.statusCode ?? 502, {
|
|
@@ -29568,7 +30156,7 @@ async function handleProxyRequest(req, res) {
|
|
|
29568
30156
|
res.end("Internal proxy error\n");
|
|
29569
30157
|
return;
|
|
29570
30158
|
}
|
|
29571
|
-
|
|
30159
|
+
logger95.debug(
|
|
29572
30160
|
`Proxying ${req.method} ${hostname}${parsedUrl.pathname} via ${resolvedIp}`
|
|
29573
30161
|
);
|
|
29574
30162
|
const forwardHeaders = { ...req.headers };
|
|
@@ -29585,7 +30173,7 @@ async function handleProxyRequest(req, res) {
|
|
|
29585
30173
|
proxyRes.pipe(res);
|
|
29586
30174
|
});
|
|
29587
30175
|
proxyReq.on("error", (err) => {
|
|
29588
|
-
|
|
30176
|
+
logger95.error(`Proxy request error for ${hostname}:`, err.message);
|
|
29589
30177
|
if (!res.headersSent) {
|
|
29590
30178
|
res.writeHead(502, { "Content-Type": "text/plain" });
|
|
29591
30179
|
res.end("Bad Gateway: Could not reach target server\n");
|
|
@@ -29600,7 +30188,7 @@ function startHttpProxy(port = 8118, host = "::") {
|
|
|
29600
30188
|
const global = getGlobalConfig();
|
|
29601
30189
|
const server = http.createServer((req, res) => {
|
|
29602
30190
|
handleProxyRequest(req, res).catch((err) => {
|
|
29603
|
-
|
|
30191
|
+
logger95.error("Error handling proxy request:", err);
|
|
29604
30192
|
if (!res.headersSent) {
|
|
29605
30193
|
res.writeHead(500, { "Content-Type": "text/plain" });
|
|
29606
30194
|
res.end("Internal proxy error\n");
|
|
@@ -29609,7 +30197,7 @@ function startHttpProxy(port = 8118, host = "::") {
|
|
|
29609
30197
|
});
|
|
29610
30198
|
server.on("connect", (req, clientSocket, head) => {
|
|
29611
30199
|
handleConnect(req, clientSocket, head).catch((err) => {
|
|
29612
|
-
|
|
30200
|
+
logger95.error("Error handling CONNECT:", err);
|
|
29613
30201
|
try {
|
|
29614
30202
|
clientSocket.write("HTTP/1.1 500 Internal Server Error\r\n\r\n");
|
|
29615
30203
|
clientSocket.end();
|
|
@@ -29618,13 +30206,13 @@ function startHttpProxy(port = 8118, host = "::") {
|
|
|
29618
30206
|
});
|
|
29619
30207
|
});
|
|
29620
30208
|
server.on("error", (err) => {
|
|
29621
|
-
|
|
30209
|
+
logger95.error("HTTP proxy server error:", err);
|
|
29622
30210
|
reject(err);
|
|
29623
30211
|
});
|
|
29624
30212
|
server.listen(port, host, () => {
|
|
29625
30213
|
server.removeAllListeners("error");
|
|
29626
30214
|
server.on("error", (err) => {
|
|
29627
|
-
|
|
30215
|
+
logger95.error("HTTP proxy server error:", err);
|
|
29628
30216
|
});
|
|
29629
30217
|
let mode;
|
|
29630
30218
|
if (isUnrestrictedMode(global.allowedDomains)) {
|
|
@@ -29634,7 +30222,7 @@ function startHttpProxy(port = 8118, host = "::") {
|
|
|
29634
30222
|
} else {
|
|
29635
30223
|
mode = "complete-isolation";
|
|
29636
30224
|
}
|
|
29637
|
-
|
|
30225
|
+
logger95.debug(
|
|
29638
30226
|
`HTTP proxy started on ${host}:${port} (mode=${mode}, allowed=${global.allowedDomains.length}, denied=${global.deniedDomains.length})`
|
|
29639
30227
|
);
|
|
29640
30228
|
resolve6(server);
|
|
@@ -29645,16 +30233,16 @@ function stopHttpProxy(server) {
|
|
|
29645
30233
|
return new Promise((resolve6, reject) => {
|
|
29646
30234
|
server.close((err) => {
|
|
29647
30235
|
if (err) {
|
|
29648
|
-
|
|
30236
|
+
logger95.error("Error stopping HTTP proxy:", err);
|
|
29649
30237
|
reject(err);
|
|
29650
30238
|
} else {
|
|
29651
|
-
|
|
30239
|
+
logger95.info("HTTP proxy stopped");
|
|
29652
30240
|
resolve6();
|
|
29653
30241
|
}
|
|
29654
30242
|
});
|
|
29655
30243
|
});
|
|
29656
30244
|
}
|
|
29657
|
-
var
|
|
30245
|
+
var logger95, blockedIpv4Ranges, blockedIpv6Ranges, blockedIpv4List, blockedIpv6List, globalConfig, proxyGrantStore, proxyPolicyStore, proxyEgressJudge, dnsLookupOverride;
|
|
29658
30246
|
var init_http_proxy = __esm({
|
|
29659
30247
|
"src/gateway/proxy/http-proxy.ts"() {
|
|
29660
30248
|
"use strict";
|
|
@@ -29662,7 +30250,7 @@ var init_http_proxy = __esm({
|
|
|
29662
30250
|
init_network_allowlist();
|
|
29663
30251
|
init_revoked_token_store();
|
|
29664
30252
|
init_judge();
|
|
29665
|
-
|
|
30253
|
+
logger95 = createLogger("http-proxy");
|
|
29666
30254
|
blockedIpv4Ranges = [
|
|
29667
30255
|
["0.0.0.0", 8],
|
|
29668
30256
|
["10.0.0.0", 8],
|
|
@@ -29711,26 +30299,26 @@ async function startFilteringProxy() {
|
|
|
29711
30299
|
const host = "127.0.0.1";
|
|
29712
30300
|
try {
|
|
29713
30301
|
proxyServer = await startHttpProxy(port, host);
|
|
29714
|
-
|
|
30302
|
+
logger96.debug(`HTTP proxy started on ${host}:${port}`);
|
|
29715
30303
|
} catch (error) {
|
|
29716
|
-
|
|
30304
|
+
logger96.error("Failed to start HTTP proxy:", error);
|
|
29717
30305
|
throw error;
|
|
29718
30306
|
}
|
|
29719
30307
|
}
|
|
29720
30308
|
async function stopFilteringProxy() {
|
|
29721
30309
|
if (proxyServer) {
|
|
29722
|
-
|
|
30310
|
+
logger96.info("Stopping HTTP proxy...");
|
|
29723
30311
|
await stopHttpProxy(proxyServer);
|
|
29724
30312
|
proxyServer = null;
|
|
29725
30313
|
}
|
|
29726
30314
|
}
|
|
29727
|
-
var
|
|
30315
|
+
var logger96, proxyServer;
|
|
29728
30316
|
var init_proxy_manager = __esm({
|
|
29729
30317
|
"src/gateway/proxy/proxy-manager.ts"() {
|
|
29730
30318
|
"use strict";
|
|
29731
30319
|
init_src();
|
|
29732
30320
|
init_http_proxy();
|
|
29733
|
-
|
|
30321
|
+
logger96 = createLogger("proxy-manager");
|
|
29734
30322
|
proxyServer = null;
|
|
29735
30323
|
process.on("SIGTERM", async () => {
|
|
29736
30324
|
await stopFilteringProxy();
|
|
@@ -43422,7 +44010,7 @@ async function createAuthRun(params) {
|
|
|
43422
44010
|
async function createConnectorOperationRun(params) {
|
|
43423
44011
|
const sql = getDb();
|
|
43424
44012
|
const approvalStatus = params.approvalMode === "queued" ? "pending" : "auto";
|
|
43425
|
-
const status = params.approvalMode === "
|
|
44013
|
+
const status = params.approvalMode === "inline" ? "running" : "pending";
|
|
43426
44014
|
const defRows = await sql`
|
|
43427
44015
|
SELECT version FROM connector_definitions
|
|
43428
44016
|
WHERE key = ${params.connectorKey}
|
|
@@ -46929,7 +47517,7 @@ var init_rate_limiter2 = __esm({
|
|
|
46929
47517
|
});
|
|
46930
47518
|
|
|
46931
47519
|
// src/auth/oauth/routes.ts
|
|
46932
|
-
import { Hono as
|
|
47520
|
+
import { Hono as Hono22 } from "hono";
|
|
46933
47521
|
async function parseRequestBody(c) {
|
|
46934
47522
|
const contentType = c.req.header("content-type") || "";
|
|
46935
47523
|
if (contentType.includes("application/x-www-form-urlencoded")) {
|
|
@@ -47101,7 +47689,7 @@ var init_routes = __esm({
|
|
|
47101
47689
|
init_scopes();
|
|
47102
47690
|
init_utils();
|
|
47103
47691
|
init_public_origin();
|
|
47104
|
-
oauthRoutes = new
|
|
47692
|
+
oauthRoutes = new Hono22();
|
|
47105
47693
|
oauthRoutes.get("/.well-known/oauth-protected-resource/:path{.+}", (c) => {
|
|
47106
47694
|
const provider2 = getProvider(c);
|
|
47107
47695
|
const metadata = provider2.getProtectedResourceMetadata();
|
|
@@ -47541,7 +48129,7 @@ var init_routes = __esm({
|
|
|
47541
48129
|
|
|
47542
48130
|
// src/auth/routes.ts
|
|
47543
48131
|
import { createHmac } from "node:crypto";
|
|
47544
|
-
import { Hono as
|
|
48132
|
+
import { Hono as Hono23 } from "hono";
|
|
47545
48133
|
function getAuthenticatedUser(c) {
|
|
47546
48134
|
const user = c.get("user");
|
|
47547
48135
|
if (!user) {
|
|
@@ -47648,7 +48236,7 @@ var init_routes2 = __esm({
|
|
|
47648
48236
|
init_middleware2();
|
|
47649
48237
|
init_clients();
|
|
47650
48238
|
init_tokens();
|
|
47651
|
-
credentialRoutes = new
|
|
48239
|
+
credentialRoutes = new Hono23();
|
|
47652
48240
|
credentialRoutes.get("/accounts", requireAuth, async (c) => {
|
|
47653
48241
|
const user = getAuthenticatedUser(c);
|
|
47654
48242
|
const sql = createDbClientFromEnv(c.env);
|
|
@@ -47860,7 +48448,7 @@ var init_routes2 = __esm({
|
|
|
47860
48448
|
|
|
47861
48449
|
// src/connect/routes.ts
|
|
47862
48450
|
import { createHash as createHash12, randomBytes as randomBytes12 } from "node:crypto";
|
|
47863
|
-
import { Hono as
|
|
48451
|
+
import { Hono as Hono24 } from "hono";
|
|
47864
48452
|
import { createMiddleware } from "hono/factory";
|
|
47865
48453
|
function buildPkceVerifier() {
|
|
47866
48454
|
return randomBytes12(32).toString("base64url");
|
|
@@ -48152,7 +48740,7 @@ var init_routes3 = __esm({
|
|
|
48152
48740
|
c.set("tokenRow", tokenRow);
|
|
48153
48741
|
return next();
|
|
48154
48742
|
});
|
|
48155
|
-
connectRoutes = new
|
|
48743
|
+
connectRoutes = new Hono24();
|
|
48156
48744
|
connectRoutes.get("/:token", async (c, next) => {
|
|
48157
48745
|
const accept = c.req.header("Accept") ?? "";
|
|
48158
48746
|
const token = c.req.param("token");
|
|
@@ -57835,6 +58423,86 @@ async function handleListAvailable(args, ctx) {
|
|
|
57835
58423
|
offset: result.offset
|
|
57836
58424
|
};
|
|
57837
58425
|
}
|
|
58426
|
+
async function waitForDeviceActionRun(runId, organizationId) {
|
|
58427
|
+
const sql = getDb();
|
|
58428
|
+
const QUEUE_BUDGET_MS = 6e4;
|
|
58429
|
+
const POST_CLAIM_BUDGET_MS = 95e3;
|
|
58430
|
+
const POLL_MS = 500;
|
|
58431
|
+
const queueDeadline = Date.now() + QUEUE_BUDGET_MS;
|
|
58432
|
+
let claimedAtMs = null;
|
|
58433
|
+
while (true) {
|
|
58434
|
+
const rows = await sql`
|
|
58435
|
+
SELECT status, action_output, error_message, claimed_at
|
|
58436
|
+
FROM runs
|
|
58437
|
+
WHERE id = ${runId} AND organization_id = ${organizationId}
|
|
58438
|
+
LIMIT 1
|
|
58439
|
+
`;
|
|
58440
|
+
const row = rows[0];
|
|
58441
|
+
if (!row) {
|
|
58442
|
+
return {
|
|
58443
|
+
status: "failed",
|
|
58444
|
+
error_message: `Run ${runId} disappeared from runs table while waiting.`
|
|
58445
|
+
};
|
|
58446
|
+
}
|
|
58447
|
+
if (row.status === "completed") {
|
|
58448
|
+
return {
|
|
58449
|
+
status: "completed",
|
|
58450
|
+
output: row.action_output ?? {}
|
|
58451
|
+
};
|
|
58452
|
+
}
|
|
58453
|
+
if (row.status === "failed" || row.status === "timeout") {
|
|
58454
|
+
return {
|
|
58455
|
+
status: row.status,
|
|
58456
|
+
error_message: row.error_message ?? `Run ${runId} ${row.status}`
|
|
58457
|
+
};
|
|
58458
|
+
}
|
|
58459
|
+
if (row.claimed_at && claimedAtMs == null) {
|
|
58460
|
+
claimedAtMs = row.claimed_at instanceof Date ? row.claimed_at.getTime() : new Date(row.claimed_at).getTime();
|
|
58461
|
+
}
|
|
58462
|
+
const now = Date.now();
|
|
58463
|
+
if (claimedAtMs != null) {
|
|
58464
|
+
if (now - claimedAtMs >= POST_CLAIM_BUDGET_MS) break;
|
|
58465
|
+
} else {
|
|
58466
|
+
if (now >= queueDeadline) break;
|
|
58467
|
+
}
|
|
58468
|
+
await new Promise((r) => setTimeout(r, POLL_MS));
|
|
58469
|
+
}
|
|
58470
|
+
const updated = await sql`
|
|
58471
|
+
UPDATE runs
|
|
58472
|
+
SET status = 'timeout',
|
|
58473
|
+
completed_at = current_timestamp,
|
|
58474
|
+
error_message = ${"waitForDeviceActionRun: device worker did not complete in time"}
|
|
58475
|
+
WHERE id = ${runId}
|
|
58476
|
+
AND organization_id = ${organizationId}
|
|
58477
|
+
AND status IN ('pending', 'running')
|
|
58478
|
+
RETURNING id
|
|
58479
|
+
`;
|
|
58480
|
+
if (updated.length === 0) {
|
|
58481
|
+
const finalRows = await sql`
|
|
58482
|
+
SELECT status, action_output, error_message
|
|
58483
|
+
FROM runs
|
|
58484
|
+
WHERE id = ${runId} AND organization_id = ${organizationId}
|
|
58485
|
+
LIMIT 1
|
|
58486
|
+
`;
|
|
58487
|
+
const final = finalRows[0];
|
|
58488
|
+
if (final?.status === "completed") {
|
|
58489
|
+
return {
|
|
58490
|
+
status: "completed",
|
|
58491
|
+
output: final.action_output ?? {}
|
|
58492
|
+
};
|
|
58493
|
+
}
|
|
58494
|
+
if (final?.status === "failed") {
|
|
58495
|
+
return {
|
|
58496
|
+
status: "failed",
|
|
58497
|
+
error_message: final.error_message ?? `Run ${runId} failed`
|
|
58498
|
+
};
|
|
58499
|
+
}
|
|
58500
|
+
}
|
|
58501
|
+
return {
|
|
58502
|
+
status: "timeout",
|
|
58503
|
+
error_message: claimedAtMs != null ? `Run ${runId} claimed but the device worker didn't finish within ${POST_CLAIM_BUDGET_MS}ms.` : `Run ${runId} was never claimed within ${QUEUE_BUDGET_MS}ms \u2014 the chrome-extension / device worker may be offline.`
|
|
58504
|
+
};
|
|
58505
|
+
}
|
|
57838
58506
|
async function handleExecute(args, env, ctx) {
|
|
57839
58507
|
const sql = getDb();
|
|
57840
58508
|
const resolved = await getOperationForConnection(
|
|
@@ -57857,13 +58525,23 @@ async function handleExecute(args, env, ctx) {
|
|
|
57857
58525
|
};
|
|
57858
58526
|
}
|
|
57859
58527
|
const shouldQueue = mode === "approval";
|
|
58528
|
+
const defRows = await sql`
|
|
58529
|
+
SELECT runtime FROM connector_definitions
|
|
58530
|
+
WHERE key = ${connection.connector_key}
|
|
58531
|
+
AND organization_id = ${ctx.organizationId}
|
|
58532
|
+
AND status = 'active'
|
|
58533
|
+
ORDER BY updated_at DESC, id DESC
|
|
58534
|
+
LIMIT 1
|
|
58535
|
+
`;
|
|
58536
|
+
const isDeviceBound = defRows[0]?.runtime != null;
|
|
58537
|
+
const approvalMode = shouldQueue ? "queued" : isDeviceBound ? "device" : "inline";
|
|
57860
58538
|
const runId = await createConnectorOperationRun({
|
|
57861
58539
|
organizationId: ctx.organizationId,
|
|
57862
58540
|
connectionId: connection.id,
|
|
57863
58541
|
connectorKey: connection.connector_key,
|
|
57864
58542
|
operationKey: operation.operation_key,
|
|
57865
58543
|
operationInput: input,
|
|
57866
|
-
approvalMode
|
|
58544
|
+
approvalMode,
|
|
57867
58545
|
requireCompiledCode: operation.backend === "local_action"
|
|
57868
58546
|
});
|
|
57869
58547
|
if (args.watcher_source) {
|
|
@@ -57938,6 +58616,31 @@ async function handleExecute(args, env, ctx) {
|
|
|
57938
58616
|
message: `Operation '${operation.name}' requires approval. Share the approval_url with the user to confirm.`
|
|
57939
58617
|
};
|
|
57940
58618
|
}
|
|
58619
|
+
if (approvalMode === "device") {
|
|
58620
|
+
const result2 = await waitForDeviceActionRun(runId, ctx.organizationId);
|
|
58621
|
+
if (result2.status === "completed") {
|
|
58622
|
+
return {
|
|
58623
|
+
action: "execute",
|
|
58624
|
+
run_id: runId,
|
|
58625
|
+
status: "completed",
|
|
58626
|
+
output: result2.output ?? {}
|
|
58627
|
+
};
|
|
58628
|
+
}
|
|
58629
|
+
if (result2.status === "timeout") {
|
|
58630
|
+
return {
|
|
58631
|
+
action: "execute",
|
|
58632
|
+
run_id: runId,
|
|
58633
|
+
status: "timeout",
|
|
58634
|
+
error_message: result2.error_message ?? "Device action run timed out."
|
|
58635
|
+
};
|
|
58636
|
+
}
|
|
58637
|
+
return {
|
|
58638
|
+
action: "execute",
|
|
58639
|
+
run_id: runId,
|
|
58640
|
+
status: "failed",
|
|
58641
|
+
error_message: result2.error_message ?? "Device action run failed."
|
|
58642
|
+
};
|
|
58643
|
+
}
|
|
57941
58644
|
const result = await executeOperationInline(
|
|
57942
58645
|
runId,
|
|
57943
58646
|
ctx.organizationId,
|
|
@@ -67830,7 +68533,7 @@ async function sseToJson(response) {
|
|
|
67830
68533
|
}
|
|
67831
68534
|
return response;
|
|
67832
68535
|
}
|
|
67833
|
-
function withSSEHeartbeat(response) {
|
|
68536
|
+
function withSSEHeartbeat(response, signal) {
|
|
67834
68537
|
if (!response.headers.get("content-type")?.includes("text/event-stream") || !response.body) {
|
|
67835
68538
|
return response;
|
|
67836
68539
|
}
|
|
@@ -67851,22 +68554,37 @@ function withSSEHeartbeat(response) {
|
|
|
67851
68554
|
if (intervalId) clearInterval(intervalId);
|
|
67852
68555
|
writer.abort(reason).catch(() => void 0);
|
|
67853
68556
|
};
|
|
68557
|
+
const adapter = {
|
|
68558
|
+
get aborted() {
|
|
68559
|
+
return terminated;
|
|
68560
|
+
},
|
|
68561
|
+
get closed() {
|
|
68562
|
+
return terminated;
|
|
68563
|
+
},
|
|
68564
|
+
abort() {
|
|
68565
|
+
abortWriter(new Error("Request aborted"));
|
|
68566
|
+
}
|
|
68567
|
+
};
|
|
67854
68568
|
intervalId = setInterval(() => {
|
|
67855
68569
|
writer.write(heartbeat2).catch(() => abortWriter(new Error("SSE heartbeat write failed")));
|
|
67856
68570
|
}, SSE_HEARTBEAT_INTERVAL_MS);
|
|
68571
|
+
const detachAbortBridge = bindRequestAbortToStream(signal, adapter);
|
|
67857
68572
|
response.body.pipeTo(
|
|
67858
68573
|
new WritableStream({
|
|
67859
68574
|
write(chunk) {
|
|
67860
68575
|
return writer.write(chunk);
|
|
67861
68576
|
},
|
|
67862
68577
|
close() {
|
|
68578
|
+
detachAbortBridge();
|
|
67863
68579
|
closeWriter();
|
|
67864
68580
|
},
|
|
67865
68581
|
abort(reason) {
|
|
68582
|
+
detachAbortBridge();
|
|
67866
68583
|
abortWriter(reason);
|
|
67867
68584
|
}
|
|
67868
68585
|
})
|
|
67869
68586
|
).catch(() => {
|
|
68587
|
+
detachAbortBridge();
|
|
67870
68588
|
abortWriter(new Error("Source SSE stream error"));
|
|
67871
68589
|
});
|
|
67872
68590
|
return new Response(readable, { status: response.status, headers: response.headers });
|
|
@@ -67876,7 +68594,7 @@ async function handleAndMaybeConvert(transport, req, wantsSSE) {
|
|
|
67876
68594
|
if (!wantsSSE && response.headers.get("content-type")?.includes("text/event-stream")) {
|
|
67877
68595
|
return sseToJson(response);
|
|
67878
68596
|
}
|
|
67879
|
-
return withSSEHeartbeat(response);
|
|
68597
|
+
return withSSEHeartbeat(response, req.signal);
|
|
67880
68598
|
}
|
|
67881
68599
|
async function resolveAuthWithInstructions(c, req) {
|
|
67882
68600
|
const authCtx = extractAuthContext(c);
|
|
@@ -68110,6 +68828,7 @@ var MCP_PROTOCOL_VERSION2, SESSION_MAX_AGE_MS, SESSION_CLEANUP_INTERVAL_MS, sess
|
|
|
68110
68828
|
var init_mcp_handler = __esm({
|
|
68111
68829
|
"src/mcp-handler.ts"() {
|
|
68112
68830
|
"use strict";
|
|
68831
|
+
init_sse_abort_bridge();
|
|
68113
68832
|
init_clients();
|
|
68114
68833
|
init_tool_access();
|
|
68115
68834
|
init_client();
|
|
@@ -68142,7 +68861,7 @@ var init_mcp_handler = __esm({
|
|
|
68142
68861
|
});
|
|
68143
68862
|
|
|
68144
68863
|
// src/lobu/client-routes.ts
|
|
68145
|
-
import { Hono as
|
|
68864
|
+
import { Hono as Hono25 } from "hono";
|
|
68146
68865
|
function isFirstPartyLobuClient(name, softwareId) {
|
|
68147
68866
|
const s = (softwareId ?? "").trim().toLowerCase();
|
|
68148
68867
|
if (s && LOBU_FIRST_PARTY_SOFTWARE_IDS.has(s)) return true;
|
|
@@ -68253,8 +68972,8 @@ var init_client_routes = __esm({
|
|
|
68253
68972
|
init_client();
|
|
68254
68973
|
init_mcp_handler();
|
|
68255
68974
|
init_org_context();
|
|
68256
|
-
routes = new
|
|
68257
|
-
platformSchemaRoutes = new
|
|
68975
|
+
routes = new Hono25();
|
|
68976
|
+
platformSchemaRoutes = new Hono25();
|
|
68258
68977
|
LOBU_FIRST_PARTY_SOFTWARE_IDS = /* @__PURE__ */ new Set([
|
|
68259
68978
|
"lobu-cli",
|
|
68260
68979
|
"lobu",
|
|
@@ -68543,7 +69262,7 @@ var init_client_routes = __esm({
|
|
|
68543
69262
|
// src/lobu/agent-routes.ts
|
|
68544
69263
|
import { readFile as readFile9 } from "node:fs/promises";
|
|
68545
69264
|
import { resolve as resolve5 } from "node:path";
|
|
68546
|
-
import { Hono as
|
|
69265
|
+
import { Hono as Hono26 } from "hono";
|
|
68547
69266
|
function toStringArray(value) {
|
|
68548
69267
|
if (Array.isArray(value)) return value.map(String);
|
|
68549
69268
|
if (typeof value === "string") {
|
|
@@ -68793,7 +69512,7 @@ var init_agent_routes = __esm({
|
|
|
68793
69512
|
init_gateway3();
|
|
68794
69513
|
init_postgres_stores();
|
|
68795
69514
|
init_org_context();
|
|
68796
|
-
routes2 = new
|
|
69515
|
+
routes2 = new Hono26();
|
|
68797
69516
|
configStore = createPostgresAgentConfigStore();
|
|
68798
69517
|
connectionStore = createPostgresAgentConnectionStore();
|
|
68799
69518
|
DEFAULT_PROVIDER_REGISTRY_CONFIG_PATH = resolve5(process.cwd(), "config/providers.json");
|
|
@@ -71520,6 +72239,192 @@ var init_join_public = __esm({
|
|
|
71520
72239
|
}
|
|
71521
72240
|
});
|
|
71522
72241
|
|
|
72242
|
+
// src/gateway/routes/internal/smoke.ts
|
|
72243
|
+
import { timingSafeEqual as timingSafeEqual3 } from "node:crypto";
|
|
72244
|
+
import { Hono as Hono27 } from "hono";
|
|
72245
|
+
function compareTokens(provided, expected) {
|
|
72246
|
+
if (provided.length !== expected.length) return false;
|
|
72247
|
+
try {
|
|
72248
|
+
return timingSafeEqual3(Buffer.from(provided), Buffer.from(expected));
|
|
72249
|
+
} catch {
|
|
72250
|
+
return false;
|
|
72251
|
+
}
|
|
72252
|
+
}
|
|
72253
|
+
function createSmokeRoutes() {
|
|
72254
|
+
const app2 = new Hono27();
|
|
72255
|
+
app2.post("/dispatch", async (c) => {
|
|
72256
|
+
const expected = process.env.SMOKE_TEST_TOKEN ?? "";
|
|
72257
|
+
const smokeAgentId = (process.env.SMOKE_TEST_AGENT_ID ?? "").trim();
|
|
72258
|
+
const smokeOrgId = (process.env.SMOKE_TEST_ORG_ID ?? "").trim();
|
|
72259
|
+
if (expected.length === 0 || smokeAgentId === "" || smokeOrgId === "") {
|
|
72260
|
+
return c.json(
|
|
72261
|
+
{
|
|
72262
|
+
error: "Smoke dispatch disabled (SMOKE_TEST_TOKEN/SMOKE_TEST_AGENT_ID/SMOKE_TEST_ORG_ID unset)"
|
|
72263
|
+
},
|
|
72264
|
+
503
|
|
72265
|
+
);
|
|
72266
|
+
}
|
|
72267
|
+
for (const h of FORWARDED_HEADERS) {
|
|
72268
|
+
if (c.req.header(h)) {
|
|
72269
|
+
logger97.warn(
|
|
72270
|
+
`Smoke dispatch refused: ${h} header present (request came through ingress)`
|
|
72271
|
+
);
|
|
72272
|
+
return c.json({ error: "Forwarded request refused" }, 403);
|
|
72273
|
+
}
|
|
72274
|
+
}
|
|
72275
|
+
const allowedHost = (process.env.SMOKE_TEST_ALLOWED_HOST ?? "").trim();
|
|
72276
|
+
if (allowedHost !== "") {
|
|
72277
|
+
const rawHost = (c.req.header("host") ?? "").toLowerCase();
|
|
72278
|
+
const hostPart = rawHost.split(":")[0] ?? "";
|
|
72279
|
+
const expected2 = allowedHost.toLowerCase();
|
|
72280
|
+
if (hostPart !== expected2 && !hostPart.startsWith(`${expected2}.`)) {
|
|
72281
|
+
logger97.warn(
|
|
72282
|
+
`Smoke dispatch refused: Host '${rawHost}' does not match SMOKE_TEST_ALLOWED_HOST '${allowedHost}'`
|
|
72283
|
+
);
|
|
72284
|
+
return c.json({ error: "Host header refused" }, 403);
|
|
72285
|
+
}
|
|
72286
|
+
}
|
|
72287
|
+
const auth = c.req.header("authorization") ?? "";
|
|
72288
|
+
if (!auth.startsWith("Bearer ")) {
|
|
72289
|
+
return c.json({ error: "Missing bearer token" }, 401);
|
|
72290
|
+
}
|
|
72291
|
+
const provided = auth.substring(7);
|
|
72292
|
+
if (!compareTokens(provided, expected)) {
|
|
72293
|
+
return c.json({ error: "Invalid smoke token" }, 401);
|
|
72294
|
+
}
|
|
72295
|
+
let body2;
|
|
72296
|
+
try {
|
|
72297
|
+
body2 = await c.req.json();
|
|
72298
|
+
} catch {
|
|
72299
|
+
return c.json({ error: "Invalid JSON body" }, 400);
|
|
72300
|
+
}
|
|
72301
|
+
const agentId = smokeAgentId;
|
|
72302
|
+
const organizationId = smokeOrgId;
|
|
72303
|
+
const conversationId = body2.conversationId?.trim();
|
|
72304
|
+
const messageText = body2.messageText?.trim() || "smoke-test ping";
|
|
72305
|
+
if (!conversationId) {
|
|
72306
|
+
return c.json({ error: "conversationId is required" }, 400);
|
|
72307
|
+
}
|
|
72308
|
+
if (!conversationId.startsWith("smoke-")) {
|
|
72309
|
+
return c.json(
|
|
72310
|
+
{
|
|
72311
|
+
error: "conversationId must start with 'smoke-' for safety"
|
|
72312
|
+
},
|
|
72313
|
+
400
|
|
72314
|
+
);
|
|
72315
|
+
}
|
|
72316
|
+
const idempotencyKey = `smoke:${conversationId}`;
|
|
72317
|
+
const messageId = `smoke-msg-${Date.now()}`;
|
|
72318
|
+
const payload = {
|
|
72319
|
+
platform: "smoke",
|
|
72320
|
+
userId: "smoke-user",
|
|
72321
|
+
botId: "smoke",
|
|
72322
|
+
conversationId,
|
|
72323
|
+
teamId: "smoke",
|
|
72324
|
+
agentId,
|
|
72325
|
+
organizationId,
|
|
72326
|
+
messageId,
|
|
72327
|
+
messageText,
|
|
72328
|
+
channelId: conversationId,
|
|
72329
|
+
platformMetadata: {
|
|
72330
|
+
agentId,
|
|
72331
|
+
chatId: conversationId,
|
|
72332
|
+
senderId: "smoke-user",
|
|
72333
|
+
responseChannel: conversationId,
|
|
72334
|
+
responseId: messageId,
|
|
72335
|
+
responseThreadId: conversationId
|
|
72336
|
+
},
|
|
72337
|
+
agentOptions: {}
|
|
72338
|
+
};
|
|
72339
|
+
const sql = getDb();
|
|
72340
|
+
try {
|
|
72341
|
+
const result = await sql`
|
|
72342
|
+
INSERT INTO public.runs (
|
|
72343
|
+
run_type,
|
|
72344
|
+
queue_name,
|
|
72345
|
+
action_input,
|
|
72346
|
+
idempotency_key,
|
|
72347
|
+
max_attempts,
|
|
72348
|
+
attempts,
|
|
72349
|
+
status,
|
|
72350
|
+
run_at,
|
|
72351
|
+
priority,
|
|
72352
|
+
retry_delay_seconds
|
|
72353
|
+
) VALUES (
|
|
72354
|
+
'chat_message',
|
|
72355
|
+
'messages',
|
|
72356
|
+
${sql.json(payload)},
|
|
72357
|
+
${idempotencyKey},
|
|
72358
|
+
1,
|
|
72359
|
+
0,
|
|
72360
|
+
'pending',
|
|
72361
|
+
now(),
|
|
72362
|
+
0,
|
|
72363
|
+
NULL
|
|
72364
|
+
)
|
|
72365
|
+
ON CONFLICT (idempotency_key)
|
|
72366
|
+
WHERE idempotency_key IS NOT NULL
|
|
72367
|
+
AND status IN ('pending', 'claimed', 'running')
|
|
72368
|
+
DO NOTHING
|
|
72369
|
+
RETURNING id
|
|
72370
|
+
`;
|
|
72371
|
+
let runId = null;
|
|
72372
|
+
if (result.length > 0 && result[0]) {
|
|
72373
|
+
runId = Number(result[0].id);
|
|
72374
|
+
} else {
|
|
72375
|
+
const existing = await sql`
|
|
72376
|
+
SELECT id FROM public.runs
|
|
72377
|
+
WHERE idempotency_key = ${idempotencyKey}
|
|
72378
|
+
AND status IN ('pending', 'claimed', 'running')
|
|
72379
|
+
ORDER BY id DESC
|
|
72380
|
+
LIMIT 1
|
|
72381
|
+
`;
|
|
72382
|
+
if (existing.length > 0 && existing[0]) {
|
|
72383
|
+
runId = Number(existing[0].id);
|
|
72384
|
+
}
|
|
72385
|
+
}
|
|
72386
|
+
if (runId === null) {
|
|
72387
|
+
return c.json({ error: "Failed to enqueue smoke run" }, 500);
|
|
72388
|
+
}
|
|
72389
|
+
try {
|
|
72390
|
+
await sql`SELECT pg_notify('runs_lobu:messages', 'chat_message')`;
|
|
72391
|
+
} catch (err) {
|
|
72392
|
+
logger97.warn(
|
|
72393
|
+
`pg_notify after smoke dispatch failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`
|
|
72394
|
+
);
|
|
72395
|
+
}
|
|
72396
|
+
logger97.info(
|
|
72397
|
+
`Smoke dispatch: runId=${runId} agentId=${agentId} org=${organizationId} conv=${conversationId}`
|
|
72398
|
+
);
|
|
72399
|
+
return c.json({ runId, idempotencyKey });
|
|
72400
|
+
} catch (err) {
|
|
72401
|
+
logger97.error(
|
|
72402
|
+
`Smoke dispatch INSERT failed: ${err instanceof Error ? err.message : String(err)}`
|
|
72403
|
+
);
|
|
72404
|
+
return c.json({ error: "Internal error" }, 500);
|
|
72405
|
+
}
|
|
72406
|
+
});
|
|
72407
|
+
return app2;
|
|
72408
|
+
}
|
|
72409
|
+
var logger97, FORWARDED_HEADERS;
|
|
72410
|
+
var init_smoke = __esm({
|
|
72411
|
+
"src/gateway/routes/internal/smoke.ts"() {
|
|
72412
|
+
"use strict";
|
|
72413
|
+
init_src();
|
|
72414
|
+
init_client();
|
|
72415
|
+
logger97 = createLogger("smoke-dispatch");
|
|
72416
|
+
FORWARDED_HEADERS = [
|
|
72417
|
+
"x-forwarded-for",
|
|
72418
|
+
"x-forwarded-host",
|
|
72419
|
+
"x-forwarded-proto",
|
|
72420
|
+
"x-forwarded-port",
|
|
72421
|
+
"x-forwarded-server",
|
|
72422
|
+
"x-real-ip",
|
|
72423
|
+
"forwarded"
|
|
72424
|
+
];
|
|
72425
|
+
}
|
|
72426
|
+
});
|
|
72427
|
+
|
|
71523
72428
|
// src/scheduled/check-due-feeds.ts
|
|
71524
72429
|
async function materializeDueFeeds(env, db) {
|
|
71525
72430
|
const sql = db ?? getDb();
|
|
@@ -71609,7 +72514,7 @@ async function createThreadForAgent(deps, args) {
|
|
|
71609
72514
|
isEphemeral: false
|
|
71610
72515
|
};
|
|
71611
72516
|
await sessionManager.setSession(session);
|
|
71612
|
-
|
|
72517
|
+
logger98.info(
|
|
71613
72518
|
`Created thread ${conversationId} for agent ${agentId}${reason ? ` (reason=${reason})` : ""}`
|
|
71614
72519
|
);
|
|
71615
72520
|
return { threadId: conversationId, token, expiresAt };
|
|
@@ -71649,12 +72554,12 @@ async function enqueueAgentMessage(deps, args) {
|
|
|
71649
72554
|
});
|
|
71650
72555
|
return { messageId, jobId };
|
|
71651
72556
|
}
|
|
71652
|
-
var
|
|
72557
|
+
var logger98, TOKEN_EXPIRATION_MS2;
|
|
71653
72558
|
var init_agent_threads = __esm({
|
|
71654
72559
|
"src/gateway/services/agent-threads.ts"() {
|
|
71655
72560
|
"use strict";
|
|
71656
72561
|
init_src();
|
|
71657
|
-
|
|
72562
|
+
logger98 = createLogger("agent-threads");
|
|
71658
72563
|
TOKEN_EXPIRATION_MS2 = 24 * 60 * 60 * 1e3;
|
|
71659
72564
|
}
|
|
71660
72565
|
});
|
|
@@ -73892,6 +74797,8 @@ async function postAuthSignal(c) {
|
|
|
73892
74797
|
async function completeActionRun(c) {
|
|
73893
74798
|
try {
|
|
73894
74799
|
const req = await c.req.json();
|
|
74800
|
+
const denied = await authorizeRunForWorker(c, req.run_id, req.worker_id);
|
|
74801
|
+
if (denied) return denied;
|
|
73895
74802
|
const sql = getDb();
|
|
73896
74803
|
const updatedRuns = await sql`
|
|
73897
74804
|
UPDATE runs
|
|
@@ -73900,8 +74807,17 @@ async function completeActionRun(c) {
|
|
|
73900
74807
|
action_output = ${req.action_output ? sql.json(req.action_output) : null},
|
|
73901
74808
|
error_message = ${req.error_message ?? null}
|
|
73902
74809
|
WHERE id = ${req.run_id}
|
|
74810
|
+
AND status = 'running'
|
|
74811
|
+
AND claimed_by = ${req.worker_id}
|
|
73903
74812
|
RETURNING organization_id, action_key
|
|
73904
74813
|
`;
|
|
74814
|
+
if (updatedRuns.length === 0) {
|
|
74815
|
+
logger_default.info(
|
|
74816
|
+
{ run_id: req.run_id, worker_id: req.worker_id, claimed_status: req.status },
|
|
74817
|
+
"[completeActionRun] no-op: run already in terminal state (likely gateway timeout)"
|
|
74818
|
+
);
|
|
74819
|
+
return c.json({ success: false, reason: "already_finalized" });
|
|
74820
|
+
}
|
|
73905
74821
|
const organizationId = updatedRuns[0]?.organization_id;
|
|
73906
74822
|
const actionKey = updatedRuns[0]?.action_key ?? "Action";
|
|
73907
74823
|
if (organizationId) {
|
|
@@ -74659,7 +75575,7 @@ __export(index_exports, {
|
|
|
74659
75575
|
import fs5 from "node:fs/promises";
|
|
74660
75576
|
import path9 from "node:path";
|
|
74661
75577
|
import { fileURLToPath as fileURLToPath6 } from "node:url";
|
|
74662
|
-
import { Hono as
|
|
75578
|
+
import { Hono as Hono28 } from "hono";
|
|
74663
75579
|
import { compress } from "hono/compress";
|
|
74664
75580
|
import { cors as cors2 } from "hono/cors";
|
|
74665
75581
|
import { pinoLogger } from "hono-pino";
|
|
@@ -74839,12 +75755,14 @@ var init_index = __esm({
|
|
|
74839
75755
|
init_logger();
|
|
74840
75756
|
init_openapi_generator();
|
|
74841
75757
|
init_public_origin();
|
|
75758
|
+
init_embedded_deployment();
|
|
74842
75759
|
init_scheduler_health();
|
|
74843
75760
|
init_rate_limiter2();
|
|
74844
75761
|
init_runtime_info();
|
|
74845
75762
|
init_workspace();
|
|
74846
75763
|
init_join_public();
|
|
74847
75764
|
init_multi_tenant();
|
|
75765
|
+
init_smoke();
|
|
74848
75766
|
init_worker_api();
|
|
74849
75767
|
LOCALHOST_HOSTNAMES2 = /* @__PURE__ */ new Set(["localhost", "127.0.0.1", "::1", "[::1]"]);
|
|
74850
75768
|
STATIC_TEXT_CONTENT_TYPES = {
|
|
@@ -74869,7 +75787,7 @@ var init_index = __esm({
|
|
|
74869
75787
|
".woff2": "font/woff2"
|
|
74870
75788
|
};
|
|
74871
75789
|
APP_ROOT2 = path9.resolve(fileURLToPath6(new URL(".", import.meta.url)), "..");
|
|
74872
|
-
app = new
|
|
75790
|
+
app = new Hono28();
|
|
74873
75791
|
app.use("/*", compress({ threshold: 1024 }));
|
|
74874
75792
|
app.use(
|
|
74875
75793
|
"/*",
|
|
@@ -74901,8 +75819,13 @@ var init_index = __esm({
|
|
|
74901
75819
|
if (contentType.startsWith("text/html")) {
|
|
74902
75820
|
const rawFrameAncestors = c.env.FRAME_ANCESTORS?.trim();
|
|
74903
75821
|
const frameAncestors = rawFrameAncestors ? rawFrameAncestors.split(/[\s,]+/).map((entry) => entry.trim()).filter((entry) => isValidFrameAncestor(entry)).join(" ") : "https://lobu.ai https://*.lobu.ai";
|
|
74904
|
-
const
|
|
74905
|
-
const
|
|
75822
|
+
const extraExtensionIds = (c.env.LOBU_OWLETTO_EXTENSION_IDS ?? "").split(",").map((s) => s.trim()).filter((s) => /^[a-p]{32}$/.test(s));
|
|
75823
|
+
const ownedExtensionIds = [
|
|
75824
|
+
// canonical, derived from apps/chrome/manifest.json's `key`
|
|
75825
|
+
"amnnhclgmbldmfcfamonoggjhfidemmm",
|
|
75826
|
+
...extraExtensionIds
|
|
75827
|
+
];
|
|
75828
|
+
const extensionAllowed = ownedExtensionIds.map((id) => ` chrome-extension://${id}`).join("");
|
|
74906
75829
|
c.header(
|
|
74907
75830
|
"Content-Security-Policy",
|
|
74908
75831
|
`frame-ancestors 'self' ${frameAncestors}${extensionAllowed}`
|
|
@@ -74974,6 +75897,17 @@ var init_index = __esm({
|
|
|
74974
75897
|
);
|
|
74975
75898
|
}
|
|
74976
75899
|
});
|
|
75900
|
+
app.get("/health/orchestrator", (c) => {
|
|
75901
|
+
const count = getReservedLockCount();
|
|
75902
|
+
const cap = getMaxReservedLocks();
|
|
75903
|
+
const nearCap = cap > 0 && count >= Math.ceil(cap * 0.8);
|
|
75904
|
+
return c.json({
|
|
75905
|
+
status: "ok",
|
|
75906
|
+
reserved_conversation_locks: count,
|
|
75907
|
+
reserved_conversation_locks_cap: cap,
|
|
75908
|
+
near_cap: nearCap
|
|
75909
|
+
});
|
|
75910
|
+
});
|
|
74977
75911
|
app.get("/health/scheduler", async (c) => {
|
|
74978
75912
|
try {
|
|
74979
75913
|
const health = await getSchedulerHealth(c.env);
|
|
@@ -75052,6 +75986,7 @@ var init_index = __esm({
|
|
|
75052
75986
|
</html>`);
|
|
75053
75987
|
});
|
|
75054
75988
|
app.get("/api/health", restHealth);
|
|
75989
|
+
app.route("/api/internal/smoke", createSmokeRoutes());
|
|
75055
75990
|
app.use("/api/workers/*", async (c, next) => {
|
|
75056
75991
|
const expected = c.env.WORKER_API_TOKEN;
|
|
75057
75992
|
const provided = c.req.header("Authorization")?.replace("Bearer ", "");
|
|
@@ -75067,7 +76002,13 @@ var init_index = __esm({
|
|
|
75067
76002
|
"/api/workers/poll",
|
|
75068
76003
|
"/api/workers/heartbeat",
|
|
75069
76004
|
"/api/workers/stream",
|
|
75070
|
-
"/api/workers/complete"
|
|
76005
|
+
"/api/workers/complete",
|
|
76006
|
+
// Action runs (run_type='action') finalize via /complete-action,
|
|
76007
|
+
// which persists action_output. The handler still goes through
|
|
76008
|
+
// authorizeRunForWorker so a user worker can only finalize runs
|
|
76009
|
+
// it claimed. Required for chrome-extension action tools to
|
|
76010
|
+
// return their observation back to the gateway.
|
|
76011
|
+
"/api/workers/complete-action"
|
|
75071
76012
|
]);
|
|
75072
76013
|
const requestPath = new URL(c.req.url).pathname;
|
|
75073
76014
|
const isAuthProfileSubpath = requestPath.startsWith("/api/workers/me/auth-profiles");
|
|
@@ -75548,56 +76489,75 @@ async function reapStaleRuns() {
|
|
|
75548
76489
|
try {
|
|
75549
76490
|
const errorMessage3 = "worker_heartbeat_lost";
|
|
75550
76491
|
const reaped = await reserved`
|
|
75551
|
-
|
|
75552
|
-
|
|
75553
|
-
|
|
75554
|
-
|
|
75555
|
-
|
|
75556
|
-
|
|
75557
|
-
|
|
75558
|
-
(
|
|
75559
|
-
|
|
75560
|
-
|
|
75561
|
-
|
|
75562
|
-
|
|
75563
|
-
|
|
75564
|
-
|
|
76492
|
+
WITH timed_out AS (
|
|
76493
|
+
UPDATE public.runs
|
|
76494
|
+
SET status = 'timeout',
|
|
76495
|
+
completed_at = current_timestamp,
|
|
76496
|
+
error_message = ${errorMessage3}
|
|
76497
|
+
WHERE run_type IN ('sync', 'action', 'embed_backfill', 'auth')
|
|
76498
|
+
AND status IN ('claimed', 'running')
|
|
76499
|
+
AND (
|
|
76500
|
+
(last_heartbeat_at IS NULL
|
|
76501
|
+
AND COALESCE(claimed_at, created_at)
|
|
76502
|
+
< current_timestamp - (${thresholdSeconds}::int * interval '1 second'))
|
|
76503
|
+
OR
|
|
76504
|
+
(last_heartbeat_at IS NOT NULL
|
|
76505
|
+
AND last_heartbeat_at
|
|
76506
|
+
< current_timestamp - (${thresholdSeconds}::int * interval '1 second'))
|
|
76507
|
+
)
|
|
76508
|
+
RETURNING id, run_type, feed_id, connection_id, connector_key, connector_version, organization_id
|
|
76509
|
+
),
|
|
76510
|
+
retries AS (
|
|
76511
|
+
INSERT INTO public.runs (
|
|
76512
|
+
organization_id, run_type, feed_id, connection_id,
|
|
76513
|
+
connector_key, connector_version, status, approval_status, created_at
|
|
75565
76514
|
)
|
|
75566
|
-
|
|
76515
|
+
SELECT
|
|
76516
|
+
t.organization_id, 'sync', t.feed_id, t.connection_id,
|
|
76517
|
+
t.connector_key, t.connector_version, 'pending', 'auto', current_timestamp
|
|
76518
|
+
FROM timed_out t
|
|
76519
|
+
WHERE t.run_type = 'sync'
|
|
76520
|
+
AND t.feed_id IS NOT NULL
|
|
76521
|
+
AND NOT EXISTS (
|
|
76522
|
+
-- Look for an unrelated active sync run on the same feed.
|
|
76523
|
+
-- Exclude timed_out.id because in PostgreSQL the sibling
|
|
76524
|
+
-- CTE UPDATE is not visible here (all CTEs see the same
|
|
76525
|
+
-- snapshot), so the row we just reaped still appears as
|
|
76526
|
+
-- running. Without this exclusion, every reap would
|
|
76527
|
+
-- dedupe against itself and no retries would ever land.
|
|
76528
|
+
SELECT 1 FROM public.runs r
|
|
76529
|
+
WHERE r.feed_id = t.feed_id
|
|
76530
|
+
AND r.run_type = 'sync'
|
|
76531
|
+
AND r.status IN ('pending', 'claimed', 'running')
|
|
76532
|
+
AND r.id NOT IN (SELECT id FROM timed_out)
|
|
76533
|
+
)
|
|
76534
|
+
RETURNING id, feed_id
|
|
76535
|
+
)
|
|
76536
|
+
SELECT
|
|
76537
|
+
(SELECT count(*)::int FROM timed_out) AS reaped,
|
|
76538
|
+
(SELECT count(*)::int FROM retries) AS retries_created,
|
|
76539
|
+
(SELECT count(*)::int FROM timed_out
|
|
76540
|
+
WHERE run_type = 'sync' AND feed_id IS NOT NULL) AS sync_eligible
|
|
75567
76541
|
`;
|
|
75568
|
-
|
|
76542
|
+
const reapedRow = reaped[0];
|
|
76543
|
+
const reapedCount = reapedRow?.reaped ?? 0;
|
|
76544
|
+
const retriesCreated = reapedRow?.retries_created ?? 0;
|
|
76545
|
+
const syncEligible = reapedRow?.sync_eligible ?? 0;
|
|
76546
|
+
if (reapedCount === 0) {
|
|
75569
76547
|
return { acquired: true, reaped: 0, retriesCreated: 0 };
|
|
75570
76548
|
}
|
|
75571
76549
|
logger_default.warn(
|
|
75572
|
-
{ reaped:
|
|
76550
|
+
{ reaped: reapedCount, retriesCreated, thresholdSeconds },
|
|
75573
76551
|
"[reaper] Marked stale connector runs as timeout (worker_heartbeat_lost)"
|
|
75574
76552
|
);
|
|
75575
|
-
|
|
75576
|
-
|
|
75577
|
-
|
|
75578
|
-
|
|
75579
|
-
|
|
75580
|
-
|
|
75581
|
-
organization_id, run_type, feed_id, connection_id,
|
|
75582
|
-
connector_key, connector_version, status, approval_status, created_at
|
|
75583
|
-
) VALUES (
|
|
75584
|
-
${row.organization_id}, 'sync', ${row.feed_id}, ${row.connection_id},
|
|
75585
|
-
${row.connector_key}, ${row.connector_version}, 'pending', 'auto', current_timestamp
|
|
75586
|
-
)
|
|
75587
|
-
`;
|
|
75588
|
-
retriesCreated += 1;
|
|
75589
|
-
} catch (err) {
|
|
75590
|
-
if (isUniqueViolation(err, "idx_runs_active_sync_per_feed")) {
|
|
75591
|
-
logger_default.info(
|
|
75592
|
-
{ feedId: row.feed_id },
|
|
75593
|
-
"[reaper] Skipped sync retry \u2014 another active sync run exists"
|
|
75594
|
-
);
|
|
75595
|
-
} else {
|
|
75596
|
-
logger_default.error({ err, runId: row.id }, "[reaper] Failed to insert sync retry");
|
|
75597
|
-
}
|
|
75598
|
-
}
|
|
76553
|
+
const skippedRetries = syncEligible - retriesCreated;
|
|
76554
|
+
if (skippedRetries > 0) {
|
|
76555
|
+
logger_default.info(
|
|
76556
|
+
{ count: skippedRetries },
|
|
76557
|
+
"[reaper] Skipped sync retries \u2014 another active sync run exists (ON CONFLICT DO NOTHING)"
|
|
76558
|
+
);
|
|
75599
76559
|
}
|
|
75600
|
-
return { acquired: true, reaped:
|
|
76560
|
+
return { acquired: true, reaped: reapedCount, retriesCreated };
|
|
75601
76561
|
} finally {
|
|
75602
76562
|
await reserved`SELECT pg_advisory_unlock(${REAPER_ADVISORY_LOCK_KEY})`;
|
|
75603
76563
|
}
|
|
@@ -75662,7 +76622,6 @@ var init_check_stalled_executions = __esm({
|
|
|
75662
76622
|
init_client();
|
|
75663
76623
|
init_connect_tokens();
|
|
75664
76624
|
init_logger();
|
|
75665
|
-
init_pg_errors();
|
|
75666
76625
|
init_automation();
|
|
75667
76626
|
REAPER_ADVISORY_LOCK_KEY = 1919841138;
|
|
75668
76627
|
DEFAULT_STALE_AFTER_SECONDS = 120;
|
|
@@ -76189,13 +77148,13 @@ import * as Sentry7 from "@sentry/node";
|
|
|
76189
77148
|
function cronSeedKey(name, tick) {
|
|
76190
77149
|
return `cron:${name}:${tick.toISOString()}`;
|
|
76191
77150
|
}
|
|
76192
|
-
var
|
|
77151
|
+
var logger99, TASK_QUEUE_NAME, TaskScheduler;
|
|
76193
77152
|
var init_task_scheduler = __esm({
|
|
76194
77153
|
"src/scheduled/task-scheduler.ts"() {
|
|
76195
77154
|
"use strict";
|
|
76196
77155
|
init_src();
|
|
76197
77156
|
init_cron();
|
|
76198
|
-
|
|
77157
|
+
logger99 = createLogger("task-scheduler");
|
|
76199
77158
|
TASK_QUEUE_NAME = "task";
|
|
76200
77159
|
TaskScheduler = class {
|
|
76201
77160
|
constructor(queue) {
|
|
@@ -76245,7 +77204,7 @@ var init_task_scheduler = __esm({
|
|
|
76245
77204
|
try {
|
|
76246
77205
|
await this.seedNextCronTick(reg);
|
|
76247
77206
|
} catch (err) {
|
|
76248
|
-
|
|
77207
|
+
logger99.error(
|
|
76249
77208
|
{ err, taskName: reg.name, cron: reg.cron },
|
|
76250
77209
|
"[task-scheduler] Failed to seed cron row at boot; will retry in background"
|
|
76251
77210
|
);
|
|
@@ -76254,7 +77213,7 @@ var init_task_scheduler = __esm({
|
|
|
76254
77213
|
}
|
|
76255
77214
|
await this.queue.work(TASK_QUEUE_NAME, (job) => this.dispatch(job));
|
|
76256
77215
|
const periodic = [...this.handlers.values()].filter((r) => r.cron).length;
|
|
76257
|
-
|
|
77216
|
+
logger99.info(
|
|
76258
77217
|
{ total: this.handlers.size, periodic },
|
|
76259
77218
|
"[task-scheduler] Started"
|
|
76260
77219
|
);
|
|
@@ -76266,12 +77225,12 @@ var init_task_scheduler = __esm({
|
|
|
76266
77225
|
setTimeout(() => {
|
|
76267
77226
|
if (!this.started) return;
|
|
76268
77227
|
this.seedNextCronTick(reg).then(
|
|
76269
|
-
() =>
|
|
77228
|
+
() => logger99.info(
|
|
76270
77229
|
{ taskName: reg.name, cron: reg.cron, attempt: i + 1 },
|
|
76271
77230
|
"[task-scheduler] Recovered cron seed in background"
|
|
76272
77231
|
)
|
|
76273
77232
|
).catch((err) => {
|
|
76274
|
-
|
|
77233
|
+
logger99.error(
|
|
76275
77234
|
{ err, taskName: reg.name, cron: reg.cron, attempt: i + 1 },
|
|
76276
77235
|
"[task-scheduler] Background cron seed retry failed"
|
|
76277
77236
|
);
|
|
@@ -76294,7 +77253,7 @@ var init_task_scheduler = __esm({
|
|
|
76294
77253
|
const data = job.data ?? { name: "", payload: {} };
|
|
76295
77254
|
const reg = this.handlers.get(data.name);
|
|
76296
77255
|
if (!reg) {
|
|
76297
|
-
|
|
77256
|
+
logger99.error(
|
|
76298
77257
|
{ taskName: data.name, runId: job.id },
|
|
76299
77258
|
"[task-scheduler] No handler registered for task; failing run"
|
|
76300
77259
|
);
|
|
@@ -76803,6 +77762,9 @@ function normalizeServerConfig(raw) {
|
|
|
76803
77762
|
if (typeof src.dataDir === "string" && src.dataDir.trim()) {
|
|
76804
77763
|
out.dataDir = src.dataDir.trim();
|
|
76805
77764
|
}
|
|
77765
|
+
if (src.lifecycle === "managed" || src.lifecycle === "external") {
|
|
77766
|
+
out.lifecycle = src.lifecycle;
|
|
77767
|
+
}
|
|
76806
77768
|
return Object.keys(out).length === 0 ? void 0 : out;
|
|
76807
77769
|
}
|
|
76808
77770
|
function applyUserServerConfigToEnv(configPath, contextOverride) {
|
|
@@ -76830,7 +77792,7 @@ import { pg_trgm } from "@electric-sql/pglite/contrib/pg_trgm";
|
|
|
76830
77792
|
import { vector } from "@electric-sql/pglite/vector";
|
|
76831
77793
|
import { PGLiteSocketServer } from "@electric-sql/pglite-socket";
|
|
76832
77794
|
import { getRequestListener } from "@hono/node-server";
|
|
76833
|
-
import { Hono as
|
|
77795
|
+
import { Hono as Hono29 } from "hono";
|
|
76834
77796
|
|
|
76835
77797
|
// src/db/embedded-schema-patches.ts
|
|
76836
77798
|
var EMBEDDED_SCHEMA_PATCHES = [
|
|
@@ -77626,6 +78588,30 @@ var EMBEDDED_SCHEMA_PATCHES = [
|
|
|
77626
78588
|
AND run_type IN ('sync', 'action', 'embed_backfill', 'auth')
|
|
77627
78589
|
`);
|
|
77628
78590
|
}
|
|
78591
|
+
},
|
|
78592
|
+
{
|
|
78593
|
+
id: "runs-heartbeat-inflight-narrow",
|
|
78594
|
+
apply: async (sql) => {
|
|
78595
|
+
await sql.unsafe(`DROP INDEX IF EXISTS public.idx_runs_heartbeat_inflight`);
|
|
78596
|
+
await sql.unsafe(`
|
|
78597
|
+
CREATE INDEX IF NOT EXISTS idx_runs_heartbeat_inflight
|
|
78598
|
+
ON public.runs (last_heartbeat_at)
|
|
78599
|
+
WHERE status IN ('claimed', 'running')
|
|
78600
|
+
AND run_type IN ('sync', 'auth')
|
|
78601
|
+
`);
|
|
78602
|
+
}
|
|
78603
|
+
},
|
|
78604
|
+
{
|
|
78605
|
+
id: "runs-heartbeat-inflight-widen",
|
|
78606
|
+
apply: async (sql) => {
|
|
78607
|
+
await sql.unsafe(`DROP INDEX IF EXISTS public.idx_runs_heartbeat_inflight`);
|
|
78608
|
+
await sql.unsafe(`
|
|
78609
|
+
CREATE INDEX IF NOT EXISTS idx_runs_heartbeat_inflight
|
|
78610
|
+
ON public.runs (last_heartbeat_at)
|
|
78611
|
+
WHERE status IN ('claimed', 'running')
|
|
78612
|
+
AND run_type IN ('sync', 'action', 'embed_backfill', 'auth')
|
|
78613
|
+
`);
|
|
78614
|
+
}
|
|
77629
78615
|
}
|
|
77630
78616
|
];
|
|
77631
78617
|
|
|
@@ -77733,7 +78719,7 @@ async function main() {
|
|
|
77733
78719
|
const stopScheduler = () => taskScheduler.stop();
|
|
77734
78720
|
const { startStaleRunReaper: startStaleRunReaper2 } = await Promise.resolve().then(() => (init_check_stalled_executions(), check_stalled_executions_exports));
|
|
77735
78721
|
const stopReaper = startStaleRunReaper2();
|
|
77736
|
-
const wrapper = new
|
|
78722
|
+
const wrapper = new Hono29();
|
|
77737
78723
|
wrapper.use("*", async (c, next) => {
|
|
77738
78724
|
const incoming = c.env?.incoming;
|
|
77739
78725
|
const peerRemoteAddress = incoming?.socket?.remoteAddress ?? null;
|