@lobu/cli 7.1.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/context.d.ts +7 -0
- package/dist/commands/context.d.ts.map +1 -1
- package/dist/commands/context.js +19 -2
- package/dist/commands/context.js.map +1 -1
- package/dist/connectors/chrome.ts +351 -0
- package/dist/connectors/chrome_bookmarks.ts +79 -0
- package/dist/connectors/chrome_downloads.ts +80 -0
- package/dist/connectors/chrome_history.ts +80 -0
- package/dist/connectors/index.ts +14 -8
- package/dist/db/migrations/20260518020000_runs_heartbeat_inflight_narrow.sql +36 -0
- package/dist/db/migrations/20260518040000_agent_transcript_snapshot.sql +54 -0
- package/dist/db/migrations/20260518050000_runs_denormalize_agent_conversation.sql +36 -0
- package/dist/db/migrations/20260518060000_revert_runs_denormalize.sql +29 -0
- package/dist/db/migrations/20260518070000_runs_heartbeat_inflight_widen.sql +33 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +37 -1
- package/dist/index.js.map +1 -1
- package/dist/internal/context.d.ts +4 -1
- package/dist/internal/context.d.ts.map +1 -1
- package/dist/internal/context.js +43 -3
- package/dist/internal/context.js.map +1 -1
- package/dist/internal/index.d.ts +1 -1
- package/dist/internal/index.d.ts.map +1 -1
- package/dist/internal/index.js +1 -1
- package/dist/internal/index.js.map +1 -1
- package/dist/server.bundle.mjs +1520 -568
- package/dist/start-local.bundle.mjs +1556 -570
- package/package.json +6 -6
- package/dist/connectors/browser/evaluate.ts +0 -120
- package/dist/connectors/browser/fill_form.ts +0 -107
- package/dist/connectors/browser/page_text.ts +0 -108
- package/dist/connectors/chrome_tabs.ts +0 -74
package/dist/server.bundle.mjs
CHANGED
|
@@ -782,7 +782,7 @@ function createLogger(serviceName) {
|
|
|
782
782
|
format: USE_JSON_FORMAT ? jsonFormat : humanFormat
|
|
783
783
|
})
|
|
784
784
|
];
|
|
785
|
-
const
|
|
785
|
+
const logger100 = winston.createLogger({
|
|
786
786
|
level,
|
|
787
787
|
format: winston.format.combine(
|
|
788
788
|
winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }),
|
|
@@ -796,12 +796,12 @@ function createLogger(serviceName) {
|
|
|
796
796
|
if (isProduction || process.env.SENTRY_DSN) {
|
|
797
797
|
try {
|
|
798
798
|
const transport = new SentryTransport();
|
|
799
|
-
|
|
799
|
+
logger100.add(transport);
|
|
800
800
|
} catch {
|
|
801
801
|
}
|
|
802
802
|
}
|
|
803
803
|
});
|
|
804
|
-
return
|
|
804
|
+
return logger100;
|
|
805
805
|
}
|
|
806
806
|
var USE_WINSTON_LOGGER, USE_JSON_FORMAT, SentryTransport;
|
|
807
807
|
var init_logger2 = __esm({
|
|
@@ -955,6 +955,7 @@ var init_capabilities = __esm({
|
|
|
955
955
|
"browser.scripting",
|
|
956
956
|
"browser.history",
|
|
957
957
|
"browser.bookmarks",
|
|
958
|
+
"browser.downloads",
|
|
958
959
|
"browser.debugger"
|
|
959
960
|
// browser.cookies intentionally absent in v1 — high-trust, not approved
|
|
960
961
|
];
|
|
@@ -1808,7 +1809,8 @@ function generateWorkerToken(userId, conversationId, deploymentName, options) {
|
|
|
1808
1809
|
platform: options.platform,
|
|
1809
1810
|
sessionKey: options.sessionKey,
|
|
1810
1811
|
traceId: options.traceId,
|
|
1811
|
-
jti: randomUUID()
|
|
1812
|
+
jti: randomUUID(),
|
|
1813
|
+
runId: options.runId
|
|
1812
1814
|
};
|
|
1813
1815
|
return encrypt(JSON.stringify(payload));
|
|
1814
1816
|
}
|
|
@@ -1832,6 +1834,12 @@ function verifyWorkerToken(token) {
|
|
|
1832
1834
|
);
|
|
1833
1835
|
return null;
|
|
1834
1836
|
}
|
|
1837
|
+
if (data.runId !== void 0) {
|
|
1838
|
+
if (typeof data.runId !== "number" || !Number.isInteger(data.runId) || data.runId <= 0) {
|
|
1839
|
+
logger10.error("Worker token rejected: runId must be a positive integer");
|
|
1840
|
+
return null;
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1835
1843
|
const ttl = parsePositiveIntEnv("WORKER_TOKEN_TTL_MS", 2 * 60 * 60 * 1e3);
|
|
1836
1844
|
const skewMs = parsePositiveIntEnv(
|
|
1837
1845
|
"WORKER_TOKEN_CLOCK_SKEW_MS",
|
|
@@ -5752,12 +5760,32 @@ function sessionMatchesMetadataOwner(session, ownerPlatform, ownerUserId) {
|
|
|
5752
5760
|
}
|
|
5753
5761
|
return ownerPlatform === session.platform || session.platform === "external";
|
|
5754
5762
|
}
|
|
5763
|
+
async function resolveAuthorizedOrgId(store, agentId, ownerPlatform, ownerUserId) {
|
|
5764
|
+
if (!store || !ownerPlatform || !ownerUserId) return void 0;
|
|
5765
|
+
const orgs = await store.findAgentOrganizations(
|
|
5766
|
+
ownerPlatform,
|
|
5767
|
+
ownerUserId,
|
|
5768
|
+
agentId
|
|
5769
|
+
);
|
|
5770
|
+
if (orgs.length !== 1) return void 0;
|
|
5771
|
+
return orgs[0];
|
|
5772
|
+
}
|
|
5755
5773
|
async function verifyOwnedAgentAccess(session, agentId, config) {
|
|
5756
5774
|
if (session.isAdmin) {
|
|
5757
5775
|
return { authorized: true };
|
|
5758
5776
|
}
|
|
5759
5777
|
if (session.agentId) {
|
|
5760
|
-
|
|
5778
|
+
if (session.agentId !== agentId) {
|
|
5779
|
+
return { authorized: false };
|
|
5780
|
+
}
|
|
5781
|
+
const lookupUserId2 = resolveSettingsLookupUserId(session);
|
|
5782
|
+
const organizationId2 = await resolveAuthorizedOrgId(
|
|
5783
|
+
config.userAgentsStore,
|
|
5784
|
+
agentId,
|
|
5785
|
+
session.platform,
|
|
5786
|
+
lookupUserId2 || void 0
|
|
5787
|
+
);
|
|
5788
|
+
return { authorized: true, organizationId: organizationId2 };
|
|
5761
5789
|
}
|
|
5762
5790
|
const lookupUserId = resolveSettingsLookupUserId(session);
|
|
5763
5791
|
if (config.userAgentsStore) {
|
|
@@ -5767,10 +5795,17 @@ async function verifyOwnedAgentAccess(session, agentId, config) {
|
|
|
5767
5795
|
agentId
|
|
5768
5796
|
);
|
|
5769
5797
|
if (owns) {
|
|
5798
|
+
const organizationId2 = await resolveAuthorizedOrgId(
|
|
5799
|
+
config.userAgentsStore,
|
|
5800
|
+
agentId,
|
|
5801
|
+
session.platform,
|
|
5802
|
+
lookupUserId
|
|
5803
|
+
);
|
|
5770
5804
|
return {
|
|
5771
5805
|
authorized: true,
|
|
5772
5806
|
ownerPlatform: session.platform,
|
|
5773
|
-
ownerUserId: lookupUserId
|
|
5807
|
+
ownerUserId: lookupUserId,
|
|
5808
|
+
organizationId: organizationId2
|
|
5774
5809
|
};
|
|
5775
5810
|
}
|
|
5776
5811
|
}
|
|
@@ -5789,10 +5824,17 @@ async function verifyOwnedAgentAccess(session, agentId, config) {
|
|
|
5789
5824
|
config.userAgentsStore.addAgent(session.platform, lookupUserId, agentId).catch(() => {
|
|
5790
5825
|
});
|
|
5791
5826
|
}
|
|
5827
|
+
const organizationId = metadata.organizationId ?? await resolveAuthorizedOrgId(
|
|
5828
|
+
config.userAgentsStore,
|
|
5829
|
+
agentId,
|
|
5830
|
+
metadata.owner.platform,
|
|
5831
|
+
metadata.owner.userId
|
|
5832
|
+
);
|
|
5792
5833
|
return {
|
|
5793
5834
|
authorized: true,
|
|
5794
5835
|
ownerPlatform: metadata.owner.platform,
|
|
5795
|
-
ownerUserId: metadata.owner.userId
|
|
5836
|
+
ownerUserId: metadata.owner.userId,
|
|
5837
|
+
organizationId
|
|
5796
5838
|
};
|
|
5797
5839
|
}
|
|
5798
5840
|
function createTokenVerifier(config) {
|
|
@@ -5802,6 +5844,12 @@ function createTokenVerifier(config) {
|
|
|
5802
5844
|
return result.authorized ? payload : null;
|
|
5803
5845
|
};
|
|
5804
5846
|
}
|
|
5847
|
+
function createOwnershipResolver(config) {
|
|
5848
|
+
return async (payload, agentId) => {
|
|
5849
|
+
if (!payload) return { authorized: false };
|
|
5850
|
+
return verifyOwnedAgentAccess(payload, agentId, config);
|
|
5851
|
+
};
|
|
5852
|
+
}
|
|
5805
5853
|
var init_agent_ownership = __esm({
|
|
5806
5854
|
"src/gateway/routes/shared/agent-ownership.ts"() {
|
|
5807
5855
|
"use strict";
|
|
@@ -6196,41 +6244,46 @@ function createAgentApi(config) {
|
|
|
6196
6244
|
if (requestSignal?.aborted) {
|
|
6197
6245
|
return;
|
|
6198
6246
|
}
|
|
6199
|
-
|
|
6200
|
-
|
|
6201
|
-
event: "connected",
|
|
6202
|
-
data: JSON.stringify({
|
|
6203
|
-
agentId: session.agentId || sessionKey3,
|
|
6204
|
-
timestamp: Date.now()
|
|
6205
|
-
})
|
|
6206
|
-
});
|
|
6207
|
-
for (const entry of sseManager.getRecentEvents(sseKey)) {
|
|
6208
|
-
await stream2.writeSSE({
|
|
6209
|
-
event: entry.event,
|
|
6210
|
-
data: JSON.stringify(entry.data)
|
|
6211
|
-
});
|
|
6212
|
-
}
|
|
6213
|
-
const heartbeatInterval = setInterval(async () => {
|
|
6214
|
-
try {
|
|
6215
|
-
await stream2.writeSSE({
|
|
6216
|
-
event: "ping",
|
|
6217
|
-
data: JSON.stringify({ timestamp: Date.now() })
|
|
6218
|
-
});
|
|
6219
|
-
} catch {
|
|
6220
|
-
clearInterval(heartbeatInterval);
|
|
6221
|
-
}
|
|
6222
|
-
}, 3e4);
|
|
6247
|
+
let heartbeatInterval;
|
|
6248
|
+
let connectionAdded = false;
|
|
6223
6249
|
let cleanedUp = false;
|
|
6224
6250
|
const cleanup = () => {
|
|
6225
6251
|
if (cleanedUp) return;
|
|
6226
6252
|
cleanedUp = true;
|
|
6227
|
-
clearInterval(heartbeatInterval);
|
|
6228
|
-
|
|
6253
|
+
if (heartbeatInterval) clearInterval(heartbeatInterval);
|
|
6254
|
+
if (connectionAdded) {
|
|
6255
|
+
sseManager.removeConnection(sseKey, stream2);
|
|
6256
|
+
}
|
|
6229
6257
|
logger24.info(`SSE connection closed for session ${sseKey}`);
|
|
6230
6258
|
};
|
|
6231
6259
|
stream2.onAbort(cleanup);
|
|
6232
6260
|
const detachAbortBridge = bindRequestAbortToStream(requestSignal, stream2);
|
|
6261
|
+
sseManager.addConnection(sseKey, stream2);
|
|
6262
|
+
connectionAdded = true;
|
|
6233
6263
|
try {
|
|
6264
|
+
await stream2.writeSSE({
|
|
6265
|
+
event: "connected",
|
|
6266
|
+
data: JSON.stringify({
|
|
6267
|
+
agentId: session.agentId || sessionKey3,
|
|
6268
|
+
timestamp: Date.now()
|
|
6269
|
+
})
|
|
6270
|
+
});
|
|
6271
|
+
for (const entry of sseManager.getRecentEvents(sseKey)) {
|
|
6272
|
+
await stream2.writeSSE({
|
|
6273
|
+
event: entry.event,
|
|
6274
|
+
data: JSON.stringify(entry.data)
|
|
6275
|
+
});
|
|
6276
|
+
}
|
|
6277
|
+
heartbeatInterval = setInterval(async () => {
|
|
6278
|
+
try {
|
|
6279
|
+
await stream2.writeSSE({
|
|
6280
|
+
event: "ping",
|
|
6281
|
+
data: JSON.stringify({ timestamp: Date.now() })
|
|
6282
|
+
});
|
|
6283
|
+
} catch {
|
|
6284
|
+
if (heartbeatInterval) clearInterval(heartbeatInterval);
|
|
6285
|
+
}
|
|
6286
|
+
}, 3e4);
|
|
6234
6287
|
while (!stream2.aborted && !stream2.closed) {
|
|
6235
6288
|
await stream2.sleep(1e3);
|
|
6236
6289
|
}
|
|
@@ -7129,6 +7182,20 @@ var init_agent_config = __esm({
|
|
|
7129
7182
|
import { readdir, readFile, stat } from "node:fs/promises";
|
|
7130
7183
|
import { join, resolve } from "node:path";
|
|
7131
7184
|
import { Hono as Hono7 } from "hono";
|
|
7185
|
+
async function readLatestSnapshotJsonl(agentId, organizationId) {
|
|
7186
|
+
if (!organizationId) return null;
|
|
7187
|
+
const sql = getDb();
|
|
7188
|
+
const snapshotRows = await sql`
|
|
7189
|
+
SELECT snapshot_jsonl
|
|
7190
|
+
FROM public.agent_transcript_snapshot
|
|
7191
|
+
WHERE organization_id = ${organizationId}
|
|
7192
|
+
AND agent_id = ${agentId}
|
|
7193
|
+
AND terminal_status = 'completed'
|
|
7194
|
+
ORDER BY run_id DESC
|
|
7195
|
+
LIMIT 1
|
|
7196
|
+
`;
|
|
7197
|
+
return snapshotRows[0]?.snapshot_jsonl ?? null;
|
|
7198
|
+
}
|
|
7132
7199
|
function isSafeAgentId(id) {
|
|
7133
7200
|
return SAFE_AGENT_ID.test(id);
|
|
7134
7201
|
}
|
|
@@ -7225,17 +7292,23 @@ function entryToMessage(entry) {
|
|
|
7225
7292
|
}
|
|
7226
7293
|
return null;
|
|
7227
7294
|
}
|
|
7228
|
-
async function readSessionMessages(agentId, cursorParam, limit) {
|
|
7229
|
-
|
|
7230
|
-
if (
|
|
7231
|
-
|
|
7232
|
-
|
|
7233
|
-
|
|
7234
|
-
|
|
7235
|
-
|
|
7236
|
-
|
|
7295
|
+
async function readSessionMessages(agentId, cursorParam, limit, organizationId) {
|
|
7296
|
+
let content = null;
|
|
7297
|
+
if (process.env.LOBU_SESSION_STORE !== "file") {
|
|
7298
|
+
content = await readLatestSnapshotJsonl(agentId, organizationId);
|
|
7299
|
+
}
|
|
7300
|
+
if (content === null) {
|
|
7301
|
+
const sessionPath = await findSessionFile(agentId);
|
|
7302
|
+
if (!sessionPath) {
|
|
7303
|
+
return {
|
|
7304
|
+
messages: [],
|
|
7305
|
+
nextCursor: null,
|
|
7306
|
+
hasMore: false,
|
|
7307
|
+
sessionId: "none"
|
|
7308
|
+
};
|
|
7309
|
+
}
|
|
7310
|
+
content = await readFile(sessionPath, "utf-8");
|
|
7237
7311
|
}
|
|
7238
|
-
const content = await readFile(sessionPath, "utf-8");
|
|
7239
7312
|
const { entries, sessionId } = parseSessionEntries(content);
|
|
7240
7313
|
const allMessages = [];
|
|
7241
7314
|
for (const entry of entries) {
|
|
@@ -7257,19 +7330,25 @@ async function readSessionMessages(agentId, cursorParam, limit) {
|
|
|
7257
7330
|
sessionId: sessionId || "unknown"
|
|
7258
7331
|
};
|
|
7259
7332
|
}
|
|
7260
|
-
async function readSessionStats(agentId) {
|
|
7261
|
-
|
|
7262
|
-
if (
|
|
7263
|
-
|
|
7264
|
-
|
|
7265
|
-
|
|
7266
|
-
|
|
7267
|
-
|
|
7268
|
-
|
|
7269
|
-
|
|
7270
|
-
|
|
7333
|
+
async function readSessionStats(agentId, organizationId) {
|
|
7334
|
+
let content = null;
|
|
7335
|
+
if (process.env.LOBU_SESSION_STORE !== "file") {
|
|
7336
|
+
content = await readLatestSnapshotJsonl(agentId, organizationId);
|
|
7337
|
+
}
|
|
7338
|
+
if (content === null) {
|
|
7339
|
+
const sessionPath = await findSessionFile(agentId);
|
|
7340
|
+
if (!sessionPath) {
|
|
7341
|
+
return {
|
|
7342
|
+
sessionId: "none",
|
|
7343
|
+
messageCount: 0,
|
|
7344
|
+
userMessages: 0,
|
|
7345
|
+
assistantMessages: 0,
|
|
7346
|
+
totalInputTokens: 0,
|
|
7347
|
+
totalOutputTokens: 0
|
|
7348
|
+
};
|
|
7349
|
+
}
|
|
7350
|
+
content = await readFile(sessionPath, "utf-8");
|
|
7271
7351
|
}
|
|
7272
|
-
const content = await readFile(sessionPath, "utf-8");
|
|
7273
7352
|
const { entries, sessionId } = parseSessionEntries(content);
|
|
7274
7353
|
let messageCount = 0;
|
|
7275
7354
|
let userMessages = 0;
|
|
@@ -7305,17 +7384,18 @@ async function readSessionStats(agentId) {
|
|
|
7305
7384
|
function createAgentHistoryRoutes(deps) {
|
|
7306
7385
|
const app3 = new Hono7();
|
|
7307
7386
|
const { connectionManager } = deps;
|
|
7308
|
-
const
|
|
7387
|
+
const resolveOwnership = createOwnershipResolver({
|
|
7309
7388
|
userAgentsStore: deps.userAgentsStore,
|
|
7310
7389
|
agentMetadataStore: deps.agentConfigStore
|
|
7311
7390
|
});
|
|
7312
|
-
async function
|
|
7391
|
+
async function getAuthorizedAgentScope(c) {
|
|
7313
7392
|
const session = await verifySettingsSession(c);
|
|
7314
7393
|
if (!session) return null;
|
|
7315
7394
|
const agentId = c.req.param("agentId") || session.agentId || null;
|
|
7316
7395
|
if (!agentId || !isSafeAgentId(agentId)) return null;
|
|
7317
|
-
const
|
|
7318
|
-
|
|
7396
|
+
const result = await resolveOwnership(session, agentId);
|
|
7397
|
+
if (!result.authorized) return null;
|
|
7398
|
+
return { agentId, organizationId: result.organizationId };
|
|
7319
7399
|
}
|
|
7320
7400
|
async function resolveActiveAgent(agentId) {
|
|
7321
7401
|
if (connectionManager.getDeploymentsForAgent(agentId).length > 0) {
|
|
@@ -7348,10 +7428,21 @@ function createAgentHistoryRoutes(deps) {
|
|
|
7348
7428
|
}
|
|
7349
7429
|
}
|
|
7350
7430
|
app3.get("/status", async (c) => {
|
|
7351
|
-
const
|
|
7352
|
-
if (!
|
|
7353
|
-
const { connected, resolvedAgentId } = await resolveActiveAgent(
|
|
7354
|
-
|
|
7431
|
+
const scope = await getAuthorizedAgentScope(c);
|
|
7432
|
+
if (!scope) return errorResponse(c, "Unauthorized", 401);
|
|
7433
|
+
const { connected, resolvedAgentId } = await resolveActiveAgent(
|
|
7434
|
+
scope.agentId
|
|
7435
|
+
);
|
|
7436
|
+
let hasSessionFile = false;
|
|
7437
|
+
if (process.env.LOBU_SESSION_STORE !== "file") {
|
|
7438
|
+
hasSessionFile = await readLatestSnapshotJsonl(
|
|
7439
|
+
resolvedAgentId,
|
|
7440
|
+
scope.organizationId
|
|
7441
|
+
) !== null;
|
|
7442
|
+
}
|
|
7443
|
+
if (!hasSessionFile) {
|
|
7444
|
+
hasSessionFile = !!await findSessionFile(resolvedAgentId);
|
|
7445
|
+
}
|
|
7355
7446
|
return c.json({
|
|
7356
7447
|
connected: connected || hasSessionFile,
|
|
7357
7448
|
hasHttpServer: !!connectionManager.getHttpUrl(resolvedAgentId),
|
|
@@ -7359,14 +7450,14 @@ function createAgentHistoryRoutes(deps) {
|
|
|
7359
7450
|
});
|
|
7360
7451
|
});
|
|
7361
7452
|
app3.get("/session/messages", async (c) => {
|
|
7362
|
-
const
|
|
7363
|
-
if (!
|
|
7453
|
+
const scope = await getAuthorizedAgentScope(c);
|
|
7454
|
+
if (!scope) return errorResponse(c, "Unauthorized", 401);
|
|
7364
7455
|
const cursor = c.req.query("cursor") || "";
|
|
7365
7456
|
const limit = Math.min(parseInt(c.req.query("limit") || "50", 10), 200);
|
|
7366
7457
|
const result = await proxyOrFallback(
|
|
7367
|
-
agentId,
|
|
7458
|
+
scope.agentId,
|
|
7368
7459
|
`/session/messages?cursor=${cursor}&limit=${limit}`,
|
|
7369
|
-
(resolved) => readSessionMessages(resolved, cursor, limit)
|
|
7460
|
+
(resolved) => readSessionMessages(resolved, cursor, limit, scope.organizationId)
|
|
7370
7461
|
);
|
|
7371
7462
|
if (!result) {
|
|
7372
7463
|
return c.json(
|
|
@@ -7383,12 +7474,12 @@ function createAgentHistoryRoutes(deps) {
|
|
|
7383
7474
|
return c.json(result.data);
|
|
7384
7475
|
});
|
|
7385
7476
|
app3.get("/session/stats", async (c) => {
|
|
7386
|
-
const
|
|
7387
|
-
if (!
|
|
7477
|
+
const scope = await getAuthorizedAgentScope(c);
|
|
7478
|
+
if (!scope) return errorResponse(c, "Unauthorized", 401);
|
|
7388
7479
|
const result = await proxyOrFallback(
|
|
7389
|
-
agentId,
|
|
7480
|
+
scope.agentId,
|
|
7390
7481
|
"/session/stats",
|
|
7391
|
-
readSessionStats
|
|
7482
|
+
(resolved) => readSessionStats(resolved, scope.organizationId)
|
|
7392
7483
|
);
|
|
7393
7484
|
if (!result) {
|
|
7394
7485
|
return c.json({ error: "Agent offline", connected: false }, 503);
|
|
@@ -7402,6 +7493,7 @@ var init_agent_history = __esm({
|
|
|
7402
7493
|
"src/gateway/routes/public/agent-history.ts"() {
|
|
7403
7494
|
"use strict";
|
|
7404
7495
|
init_src();
|
|
7496
|
+
init_client();
|
|
7405
7497
|
init_helpers();
|
|
7406
7498
|
init_agent_ownership();
|
|
7407
7499
|
init_settings_auth();
|
|
@@ -9185,8 +9277,10 @@ async function postOAuthCompletionPrompt(params) {
|
|
|
9185
9277
|
{},
|
|
9186
9278
|
agentSettingsStore
|
|
9187
9279
|
);
|
|
9188
|
-
const
|
|
9280
|
+
const instance = connectionId ? chatInstanceManager2?.getInstance(connectionId) : void 0;
|
|
9281
|
+
const conversationState = instance?.conversationState;
|
|
9189
9282
|
const conversationHistory = connectionId && conversationState ? await conversationState.getHistory(connectionId, channelId).catch(() => []) : [];
|
|
9283
|
+
const organizationId = instance?.connection.organizationId ?? void 0;
|
|
9190
9284
|
const scopeSuffix = scope ? ` (granted scopes: ${scope})` : "";
|
|
9191
9285
|
const messageText = `[System] Authentication for "${mcpId}" completed successfully${scopeSuffix}. Retry the user's previous request that required ${mcpId} and report the result \u2014 do not ask for confirmation first.`;
|
|
9192
9286
|
const messageId = randomUUID5();
|
|
@@ -9198,6 +9292,7 @@ async function postOAuthCompletionPrompt(params) {
|
|
|
9198
9292
|
conversationId: conversationId || channelId,
|
|
9199
9293
|
teamId: teamId ?? platform,
|
|
9200
9294
|
agentId,
|
|
9295
|
+
organizationId,
|
|
9201
9296
|
messageId,
|
|
9202
9297
|
messageText,
|
|
9203
9298
|
channelId,
|
|
@@ -12603,7 +12698,6 @@ async function storePendingQuestion(questionId, organizationId, connectionId, ex
|
|
|
12603
12698
|
connection_id = EXCLUDED.connection_id,
|
|
12604
12699
|
expected_user_id = EXCLUDED.expected_user_id,
|
|
12605
12700
|
entry_payload = EXCLUDED.entry_payload,
|
|
12606
|
-
created_at = now(),
|
|
12607
12701
|
claimed_at = NULL
|
|
12608
12702
|
`;
|
|
12609
12703
|
}
|
|
@@ -12622,20 +12716,38 @@ async function claimPendingQuestion(questionId, organizationId, connectionId, ex
|
|
|
12622
12716
|
if (rows.length === 0) return null;
|
|
12623
12717
|
return rows[0].entry_payload ?? null;
|
|
12624
12718
|
}
|
|
12625
|
-
async function sweepStalePendingInteractions(maxAgeMs = 24 * 60 * 60 * 1e3) {
|
|
12719
|
+
async function sweepStalePendingInteractions(maxAgeMs = 24 * 60 * 60 * 1e3, limit = DEFAULT_SWEEP_LIMIT) {
|
|
12626
12720
|
const sql = getDb();
|
|
12627
12721
|
const cutoff = new Date(Date.now() - maxAgeMs);
|
|
12628
12722
|
const rows = await sql`
|
|
12629
12723
|
DELETE FROM pending_interactions
|
|
12630
|
-
WHERE
|
|
12724
|
+
WHERE id IN (
|
|
12725
|
+
SELECT id FROM pending_interactions
|
|
12726
|
+
WHERE created_at < ${cutoff}
|
|
12727
|
+
LIMIT ${limit}
|
|
12728
|
+
)
|
|
12631
12729
|
RETURNING id
|
|
12632
12730
|
`;
|
|
12633
12731
|
return rows.map((r) => r.id);
|
|
12634
12732
|
}
|
|
12733
|
+
async function deletePendingQuestion(questionId, organizationId, connectionId, expectedUserId) {
|
|
12734
|
+
const sql = getDb();
|
|
12735
|
+
const rows = await sql`
|
|
12736
|
+
DELETE FROM pending_interactions
|
|
12737
|
+
WHERE id = ${questionId}
|
|
12738
|
+
AND organization_id = ${organizationId}
|
|
12739
|
+
AND connection_id = ${connectionId}
|
|
12740
|
+
AND expected_user_id = ${expectedUserId}
|
|
12741
|
+
RETURNING id
|
|
12742
|
+
`;
|
|
12743
|
+
return rows.length > 0;
|
|
12744
|
+
}
|
|
12745
|
+
var DEFAULT_SWEEP_LIMIT;
|
|
12635
12746
|
var init_pending_interaction_store = __esm({
|
|
12636
12747
|
"src/gateway/connections/pending-interaction-store.ts"() {
|
|
12637
12748
|
"use strict";
|
|
12638
12749
|
init_client();
|
|
12750
|
+
DEFAULT_SWEEP_LIMIT = 1e3;
|
|
12639
12751
|
}
|
|
12640
12752
|
});
|
|
12641
12753
|
|
|
@@ -12740,34 +12852,14 @@ function registerInteractionBridge(interactionService, manager, connection, chat
|
|
|
12740
12852
|
const PENDING_SENT_SWEEP_INTERVAL_MS = 60 * 60 * 1e3;
|
|
12741
12853
|
const pendingSentMessages = /* @__PURE__ */ new Map();
|
|
12742
12854
|
const pendingSentSweepTimer = setInterval(() => {
|
|
12743
|
-
sweepPendingSent().catch((error) => {
|
|
12744
|
-
logger45.warn(
|
|
12745
|
-
{ connectionId, error: String(error) },
|
|
12746
|
-
"pendingSentMessages sweep failed"
|
|
12747
|
-
);
|
|
12748
|
-
});
|
|
12749
|
-
}, PENDING_SENT_SWEEP_INTERVAL_MS);
|
|
12750
|
-
pendingSentSweepTimer.unref?.();
|
|
12751
|
-
async function sweepPendingSent() {
|
|
12752
12855
|
const ttlCutoff = Date.now() - PENDING_SENT_TTL_MS;
|
|
12753
12856
|
for (const [id, entry] of pendingSentMessages) {
|
|
12754
12857
|
if (entry.registeredAt <= ttlCutoff) {
|
|
12755
12858
|
pendingSentMessages.delete(id);
|
|
12756
12859
|
}
|
|
12757
12860
|
}
|
|
12758
|
-
|
|
12759
|
-
|
|
12760
|
-
deletedIds = await sweepStalePendingInteractions();
|
|
12761
|
-
} catch (error) {
|
|
12762
|
-
logger45.debug(
|
|
12763
|
-
{ connectionId, error: String(error) },
|
|
12764
|
-
"sweepStalePendingInteractions failed during local sweep"
|
|
12765
|
-
);
|
|
12766
|
-
}
|
|
12767
|
-
for (const id of deletedIds) {
|
|
12768
|
-
pendingSentMessages.delete(id);
|
|
12769
|
-
}
|
|
12770
|
-
}
|
|
12861
|
+
}, PENDING_SENT_SWEEP_INTERVAL_MS);
|
|
12862
|
+
pendingSentSweepTimer.unref?.();
|
|
12771
12863
|
function rememberSentMessage(questionId, sent) {
|
|
12772
12864
|
if (!sent) return;
|
|
12773
12865
|
pendingSentMessages.set(questionId, {
|
|
@@ -12856,7 +12948,7 @@ ${event.options.map((o, i) => `${i + 1}. ${o}`).join("\n")}`;
|
|
|
12856
12948
|
);
|
|
12857
12949
|
if (!sent) {
|
|
12858
12950
|
try {
|
|
12859
|
-
await
|
|
12951
|
+
await deletePendingQuestion(
|
|
12860
12952
|
event.id,
|
|
12861
12953
|
organizationId,
|
|
12862
12954
|
connectionId,
|
|
@@ -14202,6 +14294,7 @@ var init_message_handler_bridge = __esm({
|
|
|
14202
14294
|
conversationId,
|
|
14203
14295
|
teamId: teamId || platform,
|
|
14204
14296
|
agentId,
|
|
14297
|
+
organizationId: this.connection.organizationId,
|
|
14205
14298
|
messageId,
|
|
14206
14299
|
messageText: value,
|
|
14207
14300
|
channelId,
|
|
@@ -14400,15 +14493,47 @@ var init_chat_instance_manager = __esm({
|
|
|
14400
14493
|
continue;
|
|
14401
14494
|
}
|
|
14402
14495
|
try {
|
|
14403
|
-
if (connection.status === "active") {
|
|
14496
|
+
if (connection.status === "active" || connection.status === "error") {
|
|
14404
14497
|
await this.startInstance(connection);
|
|
14498
|
+
if (connection.status === "error") {
|
|
14499
|
+
const recoveryOrgId = connection.organizationId;
|
|
14500
|
+
const clearError = () => this.connectionStore.updateConnection(connection.id, {
|
|
14501
|
+
status: "active",
|
|
14502
|
+
errorMessage: void 0
|
|
14503
|
+
});
|
|
14504
|
+
if (recoveryOrgId) {
|
|
14505
|
+
await orgContext.run(
|
|
14506
|
+
{ organizationId: recoveryOrgId },
|
|
14507
|
+
clearError
|
|
14508
|
+
);
|
|
14509
|
+
} else {
|
|
14510
|
+
await clearError();
|
|
14511
|
+
}
|
|
14512
|
+
logger47.info(
|
|
14513
|
+
{ id: connection.id },
|
|
14514
|
+
"Recovered previously-errored connection"
|
|
14515
|
+
);
|
|
14516
|
+
}
|
|
14405
14517
|
}
|
|
14406
14518
|
} catch (error) {
|
|
14407
14519
|
logger47.error({ id: connection.id, error: String(error) }, "Failed to load connection");
|
|
14408
|
-
|
|
14520
|
+
const errOrgId = connection.organizationId;
|
|
14521
|
+
const markErrored = () => this.connectionStore.updateConnection(connection.id, {
|
|
14409
14522
|
status: "error",
|
|
14410
14523
|
errorMessage: `Startup failed: ${error instanceof Error ? error.message : String(error)}`
|
|
14411
14524
|
});
|
|
14525
|
+
try {
|
|
14526
|
+
if (errOrgId) {
|
|
14527
|
+
await orgContext.run({ organizationId: errOrgId }, markErrored);
|
|
14528
|
+
} else {
|
|
14529
|
+
await markErrored();
|
|
14530
|
+
}
|
|
14531
|
+
} catch (markErr) {
|
|
14532
|
+
logger47.error(
|
|
14533
|
+
{ id: connection.id, error: String(markErr) },
|
|
14534
|
+
"Failed to mark connection as errored"
|
|
14535
|
+
);
|
|
14536
|
+
}
|
|
14412
14537
|
}
|
|
14413
14538
|
}
|
|
14414
14539
|
}
|
|
@@ -14663,8 +14788,7 @@ var init_chat_instance_manager = __esm({
|
|
|
14663
14788
|
return instance;
|
|
14664
14789
|
}
|
|
14665
14790
|
async startInstance(connection) {
|
|
14666
|
-
|
|
14667
|
-
if (!callerOrgId && connection.organizationId) {
|
|
14791
|
+
if (connection.organizationId) {
|
|
14668
14792
|
return orgContext.run(
|
|
14669
14793
|
{ organizationId: connection.organizationId },
|
|
14670
14794
|
() => this.startInstanceUnscoped(connection)
|
|
@@ -15302,6 +15426,7 @@ var init_chat_instance_manager = __esm({
|
|
|
15302
15426
|
channelId: options.channelId,
|
|
15303
15427
|
teamId: options.teamId,
|
|
15304
15428
|
agentId: options.agentId,
|
|
15429
|
+
organizationId: connection.organizationId,
|
|
15305
15430
|
botId: `${name}-platform`,
|
|
15306
15431
|
platform: name,
|
|
15307
15432
|
messageText: message,
|
|
@@ -15748,6 +15873,7 @@ var init_chat_response_bridge = __esm({
|
|
|
15748
15873
|
"src/gateway/connections/chat-response-bridge.ts"() {
|
|
15749
15874
|
"use strict";
|
|
15750
15875
|
init_src();
|
|
15876
|
+
init_client();
|
|
15751
15877
|
init_link_buttons();
|
|
15752
15878
|
init_platform_strategies();
|
|
15753
15879
|
logger49 = createLogger("chat-response-bridge");
|
|
@@ -15885,6 +16011,31 @@ var init_chat_response_bridge = __esm({
|
|
|
15885
16011
|
"No session file to delete on reset"
|
|
15886
16012
|
);
|
|
15887
16013
|
}
|
|
16014
|
+
if (process.env.LOBU_SESSION_STORE !== "file") {
|
|
16015
|
+
try {
|
|
16016
|
+
const sql = getDb();
|
|
16017
|
+
const deleted = await sql`
|
|
16018
|
+
DELETE FROM public.agent_transcript_snapshot s
|
|
16019
|
+
USING public.agents a
|
|
16020
|
+
WHERE s.agent_id = ${agentId}
|
|
16021
|
+
AND s.conversation_id = ${payload.conversationId}
|
|
16022
|
+
AND a.id = s.agent_id
|
|
16023
|
+
AND a.organization_id = s.organization_id
|
|
16024
|
+
RETURNING s.id
|
|
16025
|
+
`;
|
|
16026
|
+
if (deleted.length > 0) {
|
|
16027
|
+
logger49.info(
|
|
16028
|
+
{ agentId, conversationId: payload.conversationId, count: deleted.length },
|
|
16029
|
+
"Purged agent_transcript_snapshot rows for session reset"
|
|
16030
|
+
);
|
|
16031
|
+
}
|
|
16032
|
+
} catch (error) {
|
|
16033
|
+
logger49.warn(
|
|
16034
|
+
{ agentId, conversationId: payload.conversationId, error: String(error) },
|
|
16035
|
+
"Failed to purge transcript snapshots on session reset (next boot may rehydrate stale history)"
|
|
16036
|
+
);
|
|
16037
|
+
}
|
|
16038
|
+
}
|
|
15888
16039
|
}
|
|
15889
16040
|
}
|
|
15890
16041
|
logger49.info(
|
|
@@ -20074,7 +20225,8 @@ var init_runs_queue = __esm({
|
|
|
20074
20225
|
const runAtSql = delayMs > 0 ? `now() + ${Number(delayMs) / 1e3}::float * interval '1 second'` : "now()";
|
|
20075
20226
|
const expiresAtSql = expireInSeconds && expireInSeconds > 0 ? `now() + ${Number(expireInSeconds)}::int * interval '1 second'` : "NULL";
|
|
20076
20227
|
const sql = getDb();
|
|
20077
|
-
const actionInput =
|
|
20228
|
+
const actionInput = sql.json(data ?? {});
|
|
20229
|
+
const organizationIdFromPayload = typeof data?.organizationId === "string" && data.organizationId.length > 0 ? data.organizationId : null;
|
|
20078
20230
|
const id = await sql.begin(async (tx) => {
|
|
20079
20231
|
const result = await tx.unsafe(
|
|
20080
20232
|
`INSERT INTO public.runs (
|
|
@@ -20089,9 +20241,10 @@ var init_runs_queue = __esm({
|
|
|
20089
20241
|
run_at,
|
|
20090
20242
|
priority,
|
|
20091
20243
|
expires_at,
|
|
20092
|
-
retry_delay_seconds
|
|
20244
|
+
retry_delay_seconds,
|
|
20245
|
+
organization_id
|
|
20093
20246
|
) VALUES (
|
|
20094
|
-
$1, $2, $3, $4
|
|
20247
|
+
$1, $2, $3, $4, $5, $6, 0, 'pending', ${runAtSql}, $7, ${expiresAtSql}, $8, $9
|
|
20095
20248
|
)
|
|
20096
20249
|
ON CONFLICT (idempotency_key)
|
|
20097
20250
|
WHERE idempotency_key IS NOT NULL
|
|
@@ -20106,7 +20259,8 @@ var init_runs_queue = __esm({
|
|
|
20106
20259
|
idempotencyKey,
|
|
20107
20260
|
maxAttempts,
|
|
20108
20261
|
priority,
|
|
20109
|
-
retryDelaySeconds
|
|
20262
|
+
retryDelaySeconds,
|
|
20263
|
+
organizationIdFromPayload
|
|
20110
20264
|
]
|
|
20111
20265
|
);
|
|
20112
20266
|
if (result.length === 0 && idempotencyKey) {
|
|
@@ -21020,6 +21174,34 @@ var init_user_agents_store = __esm({
|
|
|
21020
21174
|
const agents = await this.listAgents(platform, userId, organizationId);
|
|
21021
21175
|
return agents.includes(agentId);
|
|
21022
21176
|
}
|
|
21177
|
+
/**
|
|
21178
|
+
* Resolve the orgs in which `(platform, userId)` owns `agentId`.
|
|
21179
|
+
*
|
|
21180
|
+
* Reads `agent_users` directly, which IS the per-org owner mapping —
|
|
21181
|
+
* unlike `agents.{owner_platform, owner_user_id}` (used by the prior
|
|
21182
|
+
* `resolveAuthorizedOrgId` in agent-ownership.ts) those columns are
|
|
21183
|
+
* legacy and unique on `(owner_platform, owner_user_id, id)` only by
|
|
21184
|
+
* convention; they can return the wrong org when the same human owns
|
|
21185
|
+
* the same agentId across two orgs. The authoritative mapping for
|
|
21186
|
+
* "this user is allowed to read this agent's snapshot in org X" lives
|
|
21187
|
+
* here. Codex round 2 finding B on PR #865.
|
|
21188
|
+
*
|
|
21189
|
+
* Returns an empty array if the user owns no instance of `agentId`.
|
|
21190
|
+
* Typically returns 1 element; >1 means the same user has the same
|
|
21191
|
+
* agentId in multiple orgs (rare but legal).
|
|
21192
|
+
*/
|
|
21193
|
+
async findAgentOrganizations(platform, userId, agentId) {
|
|
21194
|
+
const sql = getDb();
|
|
21195
|
+
const rows = await sql`
|
|
21196
|
+
SELECT organization_id
|
|
21197
|
+
FROM agent_users
|
|
21198
|
+
WHERE platform = ${platform}
|
|
21199
|
+
AND user_id = ${userId}
|
|
21200
|
+
AND agent_id = ${agentId}
|
|
21201
|
+
ORDER BY organization_id
|
|
21202
|
+
`;
|
|
21203
|
+
return rows.map((r) => r.organization_id);
|
|
21204
|
+
}
|
|
21023
21205
|
};
|
|
21024
21206
|
}
|
|
21025
21207
|
});
|
|
@@ -21879,10 +22061,183 @@ var init_job_router = __esm({
|
|
|
21879
22061
|
}
|
|
21880
22062
|
});
|
|
21881
22063
|
|
|
21882
|
-
// src/gateway/gateway/
|
|
22064
|
+
// src/gateway/gateway/transcript-routes.ts
|
|
21883
22065
|
import { Hono as Hono18 } from "hono";
|
|
22066
|
+
function authenticate(c) {
|
|
22067
|
+
const authHeader = c.req.header("authorization");
|
|
22068
|
+
if (!authHeader?.startsWith("Bearer ")) return null;
|
|
22069
|
+
const token = authHeader.substring(7);
|
|
22070
|
+
return verifyWorkerToken(token);
|
|
22071
|
+
}
|
|
22072
|
+
async function isRunOwnedByJwtScope(runId, organizationId, agentId, conversationId) {
|
|
22073
|
+
const sql = getDb();
|
|
22074
|
+
const rows = await sql`
|
|
22075
|
+
SELECT 1 AS ok FROM public.runs
|
|
22076
|
+
WHERE id = ${runId}
|
|
22077
|
+
AND organization_id = ${organizationId}
|
|
22078
|
+
AND CASE jsonb_typeof(action_input)
|
|
22079
|
+
WHEN 'object' THEN action_input ->> 'agentId'
|
|
22080
|
+
WHEN 'string' THEN (action_input #>> '{}')::jsonb ->> 'agentId'
|
|
22081
|
+
ELSE NULL
|
|
22082
|
+
END = ${agentId}
|
|
22083
|
+
AND CASE jsonb_typeof(action_input)
|
|
22084
|
+
WHEN 'object' THEN action_input ->> 'conversationId'
|
|
22085
|
+
WHEN 'string' THEN (action_input #>> '{}')::jsonb ->> 'conversationId'
|
|
22086
|
+
ELSE NULL
|
|
22087
|
+
END = ${conversationId}
|
|
22088
|
+
LIMIT 1
|
|
22089
|
+
`;
|
|
22090
|
+
return rows.length > 0;
|
|
22091
|
+
}
|
|
22092
|
+
function createTranscriptRoutes() {
|
|
22093
|
+
const app3 = new Hono18();
|
|
22094
|
+
app3.get("/snapshot", async (c) => {
|
|
22095
|
+
const token = authenticate(c);
|
|
22096
|
+
if (!token) return c.json({ error: "Invalid token" }, 401);
|
|
22097
|
+
const { organizationId, agentId, conversationId } = token;
|
|
22098
|
+
if (!organizationId || !agentId || !conversationId) {
|
|
22099
|
+
return c.json({ error: "Token missing required scope" }, 400);
|
|
22100
|
+
}
|
|
22101
|
+
const sql = getDb();
|
|
22102
|
+
const rows = await sql`
|
|
22103
|
+
SELECT snapshot_jsonl
|
|
22104
|
+
FROM public.agent_transcript_snapshot
|
|
22105
|
+
WHERE organization_id = ${organizationId}
|
|
22106
|
+
AND agent_id = ${agentId}
|
|
22107
|
+
AND conversation_id = ${conversationId}
|
|
22108
|
+
AND terminal_status = 'completed'
|
|
22109
|
+
ORDER BY run_id DESC
|
|
22110
|
+
LIMIT 1
|
|
22111
|
+
`;
|
|
22112
|
+
const row = rows[0];
|
|
22113
|
+
if (!row) {
|
|
22114
|
+
return c.json({ error: "No snapshot found" }, 404);
|
|
22115
|
+
}
|
|
22116
|
+
return c.body(row.snapshot_jsonl, 200, {
|
|
22117
|
+
"content-type": "application/x-ndjson; charset=utf-8"
|
|
22118
|
+
});
|
|
22119
|
+
});
|
|
22120
|
+
app3.post("/snapshot", async (c) => {
|
|
22121
|
+
const token = authenticate(c);
|
|
22122
|
+
if (!token) return c.json({ error: "Invalid token" }, 401);
|
|
22123
|
+
const { organizationId, agentId, conversationId } = token;
|
|
22124
|
+
if (!organizationId || !agentId || !conversationId) {
|
|
22125
|
+
return c.json({ error: "Token missing required scope" }, 400);
|
|
22126
|
+
}
|
|
22127
|
+
let body2;
|
|
22128
|
+
try {
|
|
22129
|
+
body2 = await c.req.json();
|
|
22130
|
+
} catch {
|
|
22131
|
+
return c.json({ error: "Invalid JSON body" }, 400);
|
|
22132
|
+
}
|
|
22133
|
+
const terminalStatus = body2.terminalStatus;
|
|
22134
|
+
const snapshotJsonl = body2.snapshotJsonl;
|
|
22135
|
+
if (terminalStatus !== "completed" && terminalStatus !== "failed" && terminalStatus !== "timeout" && terminalStatus !== "cancelled") {
|
|
22136
|
+
return c.json({ error: "Invalid terminalStatus" }, 400);
|
|
22137
|
+
}
|
|
22138
|
+
if (typeof snapshotJsonl !== "string" || snapshotJsonl.length === 0) {
|
|
22139
|
+
return c.json({ error: "Missing snapshotJsonl" }, 400);
|
|
22140
|
+
}
|
|
22141
|
+
const byteSize = Buffer.byteLength(snapshotJsonl, "utf-8");
|
|
22142
|
+
if (byteSize > MAX_SNAPSHOT_BYTES) {
|
|
22143
|
+
logger72.warn(
|
|
22144
|
+
`Rejecting oversize snapshot (${byteSize} > ${MAX_SNAPSHOT_BYTES} bytes) for (${organizationId}, ${agentId}, ${conversationId})`
|
|
22145
|
+
);
|
|
22146
|
+
return c.json({ error: "Snapshot too large" }, 413);
|
|
22147
|
+
}
|
|
22148
|
+
const rawRunId = body2.runId;
|
|
22149
|
+
const runId = typeof rawRunId === "number" && Number.isFinite(rawRunId) && rawRunId > 0 ? rawRunId : null;
|
|
22150
|
+
if (runId === null) {
|
|
22151
|
+
return c.json({ error: "Missing or invalid runId" }, 400);
|
|
22152
|
+
}
|
|
22153
|
+
if (token.runId !== runId) {
|
|
22154
|
+
logger72.warn(
|
|
22155
|
+
`Token runId mismatch: token.runId=${token.runId ?? "<absent>"} body.runId=${runId}; rejecting snapshot`
|
|
22156
|
+
);
|
|
22157
|
+
return c.json({ error: "runId out of scope" }, 403);
|
|
22158
|
+
}
|
|
22159
|
+
if (!await isRunOwnedByJwtScope(
|
|
22160
|
+
runId,
|
|
22161
|
+
organizationId,
|
|
22162
|
+
agentId,
|
|
22163
|
+
conversationId
|
|
22164
|
+
)) {
|
|
22165
|
+
logger72.warn(
|
|
22166
|
+
`Run ${runId} does not belong to (${organizationId}, ${agentId}, ${conversationId}); rejecting snapshot`
|
|
22167
|
+
);
|
|
22168
|
+
return c.json({ error: "runId out of scope" }, 403);
|
|
22169
|
+
}
|
|
22170
|
+
const sql = getDb();
|
|
22171
|
+
try {
|
|
22172
|
+
const inserted = await sql`
|
|
22173
|
+
INSERT INTO public.agent_transcript_snapshot
|
|
22174
|
+
(organization_id, agent_id, conversation_id, run_id,
|
|
22175
|
+
snapshot_jsonl, byte_size, terminal_status)
|
|
22176
|
+
VALUES
|
|
22177
|
+
(${organizationId}, ${agentId}, ${conversationId}, ${runId},
|
|
22178
|
+
${snapshotJsonl}, ${byteSize}, ${terminalStatus})
|
|
22179
|
+
ON CONFLICT (organization_id, agent_id, conversation_id, run_id)
|
|
22180
|
+
DO NOTHING
|
|
22181
|
+
RETURNING id
|
|
22182
|
+
`;
|
|
22183
|
+
if (inserted.length === 0) {
|
|
22184
|
+
return c.json({ error: "Snapshot already exists for run" }, 409);
|
|
22185
|
+
}
|
|
22186
|
+
logger72.info(
|
|
22187
|
+
`Wrote snapshot id=${inserted[0].id} run_id=${runId} byte_size=${byteSize} status=${terminalStatus}`
|
|
22188
|
+
);
|
|
22189
|
+
return c.json({ id: inserted[0].id });
|
|
22190
|
+
} catch (err) {
|
|
22191
|
+
logger72.error(
|
|
22192
|
+
`Snapshot INSERT failed: ${err instanceof Error ? err.message : String(err)}`
|
|
22193
|
+
);
|
|
22194
|
+
return c.json({ error: "Internal error" }, 500);
|
|
22195
|
+
}
|
|
22196
|
+
});
|
|
22197
|
+
app3.delete("/snapshot", async (c) => {
|
|
22198
|
+
const token = authenticate(c);
|
|
22199
|
+
if (!token) return c.json({ error: "Invalid token" }, 401);
|
|
22200
|
+
const { organizationId, agentId, conversationId } = token;
|
|
22201
|
+
if (!organizationId || !agentId || !conversationId) {
|
|
22202
|
+
return c.json({ error: "Token missing required scope" }, 400);
|
|
22203
|
+
}
|
|
22204
|
+
const sql = getDb();
|
|
22205
|
+
try {
|
|
22206
|
+
const deleted = await sql`
|
|
22207
|
+
DELETE FROM public.agent_transcript_snapshot
|
|
22208
|
+
WHERE organization_id = ${organizationId}
|
|
22209
|
+
AND agent_id = ${agentId}
|
|
22210
|
+
AND conversation_id = ${conversationId}
|
|
22211
|
+
RETURNING id
|
|
22212
|
+
`;
|
|
22213
|
+
logger72.info(
|
|
22214
|
+
`Purged ${deleted.length} snapshot row(s) for (${organizationId}, ${agentId}, ${conversationId}) on session reset`
|
|
22215
|
+
);
|
|
22216
|
+
return c.json({ deleted: deleted.length });
|
|
22217
|
+
} catch (err) {
|
|
22218
|
+
logger72.error(
|
|
22219
|
+
`Snapshot DELETE failed: ${err instanceof Error ? err.message : String(err)}`
|
|
22220
|
+
);
|
|
22221
|
+
return c.json({ error: "Internal error" }, 500);
|
|
22222
|
+
}
|
|
22223
|
+
});
|
|
22224
|
+
return app3;
|
|
22225
|
+
}
|
|
22226
|
+
var logger72, MAX_SNAPSHOT_BYTES;
|
|
22227
|
+
var init_transcript_routes = __esm({
|
|
22228
|
+
"src/gateway/gateway/transcript-routes.ts"() {
|
|
22229
|
+
"use strict";
|
|
22230
|
+
init_src();
|
|
22231
|
+
init_client();
|
|
22232
|
+
logger72 = createLogger("worker-transcript");
|
|
22233
|
+
MAX_SNAPSHOT_BYTES = 4 * 1024 * 1024;
|
|
22234
|
+
}
|
|
22235
|
+
});
|
|
22236
|
+
|
|
22237
|
+
// src/gateway/gateway/index.ts
|
|
22238
|
+
import { Hono as Hono19 } from "hono";
|
|
21884
22239
|
import { stream } from "hono/streaming";
|
|
21885
|
-
var
|
|
22240
|
+
var logger73, WorkerGateway;
|
|
21886
22241
|
var init_gateway2 = __esm({
|
|
21887
22242
|
"src/gateway/gateway/index.ts"() {
|
|
21888
22243
|
"use strict";
|
|
@@ -21893,7 +22248,8 @@ var init_gateway2 = __esm({
|
|
|
21893
22248
|
init_model_selection();
|
|
21894
22249
|
init_connection_manager();
|
|
21895
22250
|
init_job_router();
|
|
21896
|
-
|
|
22251
|
+
init_transcript_routes();
|
|
22252
|
+
logger73 = createLogger("worker-gateway");
|
|
21897
22253
|
WorkerGateway = class {
|
|
21898
22254
|
app;
|
|
21899
22255
|
connectionManager;
|
|
@@ -21917,7 +22273,7 @@ var init_gateway2 = __esm({
|
|
|
21917
22273
|
this.providerCatalogService = providerCatalogService;
|
|
21918
22274
|
this.agentSettingsStore = agentSettingsStore;
|
|
21919
22275
|
this.secretStore = secretStore;
|
|
21920
|
-
this.app = new
|
|
22276
|
+
this.app = new Hono19();
|
|
21921
22277
|
this.setupRoutes();
|
|
21922
22278
|
}
|
|
21923
22279
|
/**
|
|
@@ -21942,7 +22298,8 @@ var init_gateway2 = __esm({
|
|
|
21942
22298
|
"/session-context",
|
|
21943
22299
|
(c) => this.handleSessionContextRequest(c)
|
|
21944
22300
|
);
|
|
21945
|
-
|
|
22301
|
+
this.app.route("/transcript", createTranscriptRoutes());
|
|
22302
|
+
logger73.debug("Worker gateway routes registered");
|
|
21946
22303
|
}
|
|
21947
22304
|
async enrichMcpStatus(mcpStatus, agentId, userId) {
|
|
21948
22305
|
const secretStore = this.secretStore;
|
|
@@ -21971,7 +22328,7 @@ var init_gateway2 = __esm({
|
|
|
21971
22328
|
mcp.id
|
|
21972
22329
|
);
|
|
21973
22330
|
} catch (error) {
|
|
21974
|
-
|
|
22331
|
+
logger73.warn("Failed to look up stored MCP credential", {
|
|
21975
22332
|
mcpId: mcp.id,
|
|
21976
22333
|
agentId,
|
|
21977
22334
|
userId,
|
|
@@ -22028,6 +22385,29 @@ var init_gateway2 = __esm({
|
|
|
22028
22385
|
});
|
|
22029
22386
|
}
|
|
22030
22387
|
};
|
|
22388
|
+
let connectionAdded = false;
|
|
22389
|
+
let cleanupRan = false;
|
|
22390
|
+
let aborted = false;
|
|
22391
|
+
const runCleanup = () => {
|
|
22392
|
+
if (cleanupRan) return;
|
|
22393
|
+
cleanupRan = true;
|
|
22394
|
+
aborted = true;
|
|
22395
|
+
if (!connectionAdded) {
|
|
22396
|
+
return;
|
|
22397
|
+
}
|
|
22398
|
+
const current = this.connectionManager.getConnection(deploymentName);
|
|
22399
|
+
if (current && current.writer !== sseWriter) {
|
|
22400
|
+
logger73.debug(
|
|
22401
|
+
`Ignoring stale disconnect for ${deploymentName} (replaced by newer SSE)`
|
|
22402
|
+
);
|
|
22403
|
+
return;
|
|
22404
|
+
}
|
|
22405
|
+
this.jobRouter.pauseWorker(deploymentName).catch((err) => {
|
|
22406
|
+
logger73.error(`Failed to pause worker ${deploymentName}:`, err);
|
|
22407
|
+
});
|
|
22408
|
+
this.connectionManager.removeConnection(deploymentName);
|
|
22409
|
+
};
|
|
22410
|
+
sseWriter.onClose(runCleanup);
|
|
22031
22411
|
const detachAbortBridge = bindRequestAbortToStream(
|
|
22032
22412
|
requestSignal,
|
|
22033
22413
|
streamWriter
|
|
@@ -22037,8 +22417,13 @@ var init_gateway2 = __esm({
|
|
|
22037
22417
|
c.header("Connection", "keep-alive");
|
|
22038
22418
|
c.header("X-Accel-Buffering", "no");
|
|
22039
22419
|
await this.jobRouter.pauseWorker(deploymentName);
|
|
22420
|
+
if (aborted || requestSignal?.aborted) {
|
|
22421
|
+
detachAbortBridge();
|
|
22422
|
+
runCleanup();
|
|
22423
|
+
return;
|
|
22424
|
+
}
|
|
22040
22425
|
if (this.connectionManager.isConnected(deploymentName)) {
|
|
22041
|
-
|
|
22426
|
+
logger73.info(
|
|
22042
22427
|
`Cleaning up stale connection for ${deploymentName} before new SSE`
|
|
22043
22428
|
);
|
|
22044
22429
|
this.connectionManager.removeConnection(deploymentName);
|
|
@@ -22051,21 +22436,14 @@ var init_gateway2 = __esm({
|
|
|
22051
22436
|
sseWriter,
|
|
22052
22437
|
httpPort
|
|
22053
22438
|
);
|
|
22439
|
+
connectionAdded = true;
|
|
22440
|
+
if (aborted || requestSignal?.aborted) {
|
|
22441
|
+
detachAbortBridge();
|
|
22442
|
+
runCleanup();
|
|
22443
|
+
return;
|
|
22444
|
+
}
|
|
22054
22445
|
await this.jobRouter.registerWorker(deploymentName);
|
|
22055
22446
|
await this.jobRouter.resumeWorker(deploymentName);
|
|
22056
|
-
sseWriter.onClose(() => {
|
|
22057
|
-
const current = this.connectionManager.getConnection(deploymentName);
|
|
22058
|
-
if (current && current.writer !== sseWriter) {
|
|
22059
|
-
logger72.debug(
|
|
22060
|
-
`Ignoring stale disconnect for ${deploymentName} (replaced by newer SSE)`
|
|
22061
|
-
);
|
|
22062
|
-
return;
|
|
22063
|
-
}
|
|
22064
|
-
this.jobRouter.pauseWorker(deploymentName).catch((err) => {
|
|
22065
|
-
logger72.error(`Failed to pause worker ${deploymentName}:`, err);
|
|
22066
|
-
});
|
|
22067
|
-
this.connectionManager.removeConnection(deploymentName);
|
|
22068
|
-
});
|
|
22069
22447
|
try {
|
|
22070
22448
|
while (!isClosed) {
|
|
22071
22449
|
await streamWriter.sleep(1e3);
|
|
@@ -22088,36 +22466,37 @@ var init_gateway2 = __esm({
|
|
|
22088
22466
|
try {
|
|
22089
22467
|
const body2 = await c.req.json();
|
|
22090
22468
|
const { jobId, ...responseData } = body2;
|
|
22091
|
-
const
|
|
22092
|
-
|
|
22469
|
+
const orgEnriched = auth.tokenData.organizationId && !responseData.organizationId ? { ...responseData, organizationId: auth.tokenData.organizationId } : responseData;
|
|
22470
|
+
const enrichedResponse = auth.tokenData.connectionId && (!orgEnriched.platformMetadata || typeof orgEnriched.platformMetadata === "object") ? {
|
|
22471
|
+
...orgEnriched,
|
|
22093
22472
|
platformMetadata: {
|
|
22094
|
-
...
|
|
22473
|
+
...orgEnriched.platformMetadata || {},
|
|
22095
22474
|
connectionId: auth.tokenData.connectionId
|
|
22096
22475
|
}
|
|
22097
|
-
} :
|
|
22476
|
+
} : orgEnriched;
|
|
22098
22477
|
if (jobId) {
|
|
22099
22478
|
this.jobRouter.acknowledgeJob(jobId);
|
|
22100
22479
|
}
|
|
22101
22480
|
if (enrichedResponse.received) {
|
|
22102
22481
|
if (enrichedResponse.heartbeat) {
|
|
22103
|
-
|
|
22482
|
+
logger73.debug(
|
|
22104
22483
|
`[WORKER-GATEWAY] Received heartbeat ACK from ${deploymentName}`
|
|
22105
22484
|
);
|
|
22106
22485
|
}
|
|
22107
22486
|
return c.json({ success: true });
|
|
22108
22487
|
}
|
|
22109
|
-
|
|
22488
|
+
logger73.info(
|
|
22110
22489
|
`[WORKER-GATEWAY] Received response with fields: ${Object.keys(enrichedResponse).join(", ")}`
|
|
22111
22490
|
);
|
|
22112
22491
|
if (enrichedResponse.delta) {
|
|
22113
|
-
|
|
22492
|
+
logger73.info(
|
|
22114
22493
|
`[WORKER-GATEWAY] Stream delta: deltaLength=${enrichedResponse.delta.length}`
|
|
22115
22494
|
);
|
|
22116
22495
|
}
|
|
22117
22496
|
await this.queue.send("thread_response", enrichedResponse);
|
|
22118
22497
|
return c.json({ success: true });
|
|
22119
22498
|
} catch (error) {
|
|
22120
|
-
|
|
22499
|
+
logger73.error(`Error handling worker response: ${error}`);
|
|
22121
22500
|
return c.json({ error: "Failed to process response" }, 500);
|
|
22122
22501
|
}
|
|
22123
22502
|
}
|
|
@@ -22206,7 +22585,7 @@ var init_gateway2 = __esm({
|
|
|
22206
22585
|
mcpInstructions[result.value.mcpId] = result.value.instructions;
|
|
22207
22586
|
}
|
|
22208
22587
|
} else {
|
|
22209
|
-
|
|
22588
|
+
logger73.error("MCP tool fetch rejected", {
|
|
22210
22589
|
reason: result.reason instanceof Error ? result.reason.message : String(result.reason)
|
|
22211
22590
|
});
|
|
22212
22591
|
}
|
|
@@ -22234,13 +22613,13 @@ var init_gateway2 = __esm({
|
|
|
22234
22613
|
}
|
|
22235
22614
|
}
|
|
22236
22615
|
} catch (error) {
|
|
22237
|
-
|
|
22616
|
+
logger73.error("Failed to fetch skills config for worker sync", {
|
|
22238
22617
|
error
|
|
22239
22618
|
});
|
|
22240
22619
|
}
|
|
22241
22620
|
}
|
|
22242
22621
|
const mergedSkillsInstructions = contextData.skillsInstructions || "";
|
|
22243
|
-
|
|
22622
|
+
logger73.info(
|
|
22244
22623
|
`Session context for ${userId}: ${Object.keys(mcpConfig.mcpServers || {}).length} MCPs, ${contextData.agentInstructions.length} chars agent instructions, ${contextData.platformInstructions.length} chars platform instructions, ${contextData.networkInstructions.length} chars network instructions, ${mergedSkillsInstructions.length} chars skills instructions, ${enrichedMcpStatus.length} MCP status entries, ${Object.keys(mcpTools).length} MCP tool lists, ${Object.keys(mcpInstructions).length} MCP instructions, ${skillsConfig.length} skills, provider: ${providerConfig.defaultProvider || "none"}`
|
|
22245
22624
|
);
|
|
22246
22625
|
return c.json({
|
|
@@ -22257,7 +22636,7 @@ var init_gateway2 = __esm({
|
|
|
22257
22636
|
skillsConfig
|
|
22258
22637
|
});
|
|
22259
22638
|
} catch (error) {
|
|
22260
|
-
|
|
22639
|
+
logger73.error("Failed to generate session context", { err: error });
|
|
22261
22640
|
return c.json({ error: "session_context_error" }, 500);
|
|
22262
22641
|
}
|
|
22263
22642
|
}
|
|
@@ -22269,11 +22648,11 @@ var init_gateway2 = __esm({
|
|
|
22269
22648
|
const token = authHeader.substring(7);
|
|
22270
22649
|
const tokenData = verifyWorkerToken(token);
|
|
22271
22650
|
if (!tokenData) {
|
|
22272
|
-
|
|
22651
|
+
logger73.warn("Invalid token");
|
|
22273
22652
|
return null;
|
|
22274
22653
|
}
|
|
22275
22654
|
if (tokenData.jti && await getRevokedTokenStore().isRevoked(tokenData.jti)) {
|
|
22276
|
-
|
|
22655
|
+
logger73.warn("Revoked worker token");
|
|
22277
22656
|
return null;
|
|
22278
22657
|
}
|
|
22279
22658
|
return { tokenData, token };
|
|
@@ -22389,12 +22768,12 @@ var init_gateway2 = __esm({
|
|
|
22389
22768
|
});
|
|
22390
22769
|
|
|
22391
22770
|
// src/gateway/infrastructure/queue/queue-producer.ts
|
|
22392
|
-
var
|
|
22771
|
+
var logger74, QueueProducer;
|
|
22393
22772
|
var init_queue_producer = __esm({
|
|
22394
22773
|
"src/gateway/infrastructure/queue/queue-producer.ts"() {
|
|
22395
22774
|
"use strict";
|
|
22396
22775
|
init_src();
|
|
22397
|
-
|
|
22776
|
+
logger74 = createLogger("queue-producer");
|
|
22398
22777
|
QueueProducer = class {
|
|
22399
22778
|
queue;
|
|
22400
22779
|
isInitialized = false;
|
|
@@ -22409,9 +22788,9 @@ var init_queue_producer = __esm({
|
|
|
22409
22788
|
try {
|
|
22410
22789
|
await this.queue.createQueue("messages");
|
|
22411
22790
|
this.isInitialized = true;
|
|
22412
|
-
|
|
22791
|
+
logger74.debug("Queue producer initialized");
|
|
22413
22792
|
} catch (error) {
|
|
22414
|
-
|
|
22793
|
+
logger74.error("Failed to initialize queue producer:", error);
|
|
22415
22794
|
throw error;
|
|
22416
22795
|
}
|
|
22417
22796
|
}
|
|
@@ -22420,7 +22799,7 @@ var init_queue_producer = __esm({
|
|
|
22420
22799
|
*/
|
|
22421
22800
|
async stop() {
|
|
22422
22801
|
this.isInitialized = false;
|
|
22423
|
-
|
|
22802
|
+
logger74.debug("Queue producer stopped");
|
|
22424
22803
|
}
|
|
22425
22804
|
/**
|
|
22426
22805
|
* Enqueue any message (direct or thread) to the single 'messages' queue
|
|
@@ -22441,12 +22820,12 @@ var init_queue_producer = __esm({
|
|
|
22441
22820
|
singletonKey: rawSingletonKey.replace(/:/g, "-")
|
|
22442
22821
|
// Prevent duplicates within canonical conversation identity
|
|
22443
22822
|
});
|
|
22444
|
-
|
|
22823
|
+
logger74.info(
|
|
22445
22824
|
`Enqueued message job ${jobId} for user ${payload.userId}, conversation ${payload.conversationId}`
|
|
22446
22825
|
);
|
|
22447
22826
|
return jobId || "job-sent";
|
|
22448
22827
|
} catch (error) {
|
|
22449
|
-
|
|
22828
|
+
logger74.error(
|
|
22450
22829
|
`Failed to enqueue message for user ${payload.userId}:`,
|
|
22451
22830
|
error
|
|
22452
22831
|
);
|
|
@@ -22495,12 +22874,12 @@ function assertConnectionId(connectionId, kind) {
|
|
|
22495
22874
|
);
|
|
22496
22875
|
}
|
|
22497
22876
|
}
|
|
22498
|
-
var
|
|
22877
|
+
var logger75, SAFE_LINK_BUTTON_SCHEMES, InteractionService;
|
|
22499
22878
|
var init_interactions2 = __esm({
|
|
22500
22879
|
"src/gateway/interactions.ts"() {
|
|
22501
22880
|
"use strict";
|
|
22502
22881
|
init_src();
|
|
22503
|
-
|
|
22882
|
+
logger75 = createLogger("interactions");
|
|
22504
22883
|
SAFE_LINK_BUTTON_SCHEMES = /* @__PURE__ */ new Set(["http:", "https:"]);
|
|
22505
22884
|
InteractionService = class extends EventEmitter {
|
|
22506
22885
|
beforeCreateHook;
|
|
@@ -22531,7 +22910,7 @@ var init_interactions2 = __esm({
|
|
|
22531
22910
|
question,
|
|
22532
22911
|
options
|
|
22533
22912
|
};
|
|
22534
|
-
|
|
22913
|
+
logger75.info(
|
|
22535
22914
|
`Posted question ${posted.id} for conversation ${conversationId}`
|
|
22536
22915
|
);
|
|
22537
22916
|
this.emit("question:created", posted);
|
|
@@ -22564,7 +22943,7 @@ var init_interactions2 = __esm({
|
|
|
22564
22943
|
args,
|
|
22565
22944
|
grantPattern
|
|
22566
22945
|
};
|
|
22567
|
-
|
|
22946
|
+
logger75.info(
|
|
22568
22947
|
`Posted tool approval ${posted.id} for ${mcpId}/${toolName} agent=${agentId}`
|
|
22569
22948
|
);
|
|
22570
22949
|
this.emit("tool:approval-needed", posted);
|
|
@@ -22593,7 +22972,7 @@ var init_interactions2 = __esm({
|
|
|
22593
22972
|
body: body2,
|
|
22594
22973
|
linkType
|
|
22595
22974
|
};
|
|
22596
|
-
|
|
22975
|
+
logger75.info(
|
|
22597
22976
|
`Posted link button ${posted.id} for conversation ${conversationId} (${linkType})`
|
|
22598
22977
|
);
|
|
22599
22978
|
this.emit("link-button:created", posted);
|
|
@@ -22634,7 +23013,7 @@ var init_interactions2 = __esm({
|
|
|
22634
23013
|
platform,
|
|
22635
23014
|
text
|
|
22636
23015
|
};
|
|
22637
|
-
|
|
23016
|
+
logger75.info(
|
|
22638
23017
|
`Posted status message ${posted.id} for conversation ${conversationId}`
|
|
22639
23018
|
);
|
|
22640
23019
|
this.emit("status-message:created", posted);
|
|
@@ -22654,7 +23033,7 @@ var init_interactions2 = __esm({
|
|
|
22654
23033
|
blocking: false,
|
|
22655
23034
|
prompts
|
|
22656
23035
|
};
|
|
22657
|
-
|
|
23036
|
+
logger75.info(
|
|
22658
23037
|
`Created suggestion ${suggestion.id} for conversation ${conversationId}`
|
|
22659
23038
|
);
|
|
22660
23039
|
this.emit("suggestion:created", suggestion);
|
|
@@ -22725,12 +23104,12 @@ function findMatchingRule(hostname, rules) {
|
|
|
22725
23104
|
function hashPolicy(organizationId, agentId, judgeName, policy) {
|
|
22726
23105
|
return crypto3.createHash("sha256").update(`${organizationId} ${agentId} ${judgeName} ${policy}`).digest("hex").slice(0, 16);
|
|
22727
23106
|
}
|
|
22728
|
-
var
|
|
23107
|
+
var logger76, PolicyStore;
|
|
22729
23108
|
var init_policy_store = __esm({
|
|
22730
23109
|
"src/gateway/permissions/policy-store.ts"() {
|
|
22731
23110
|
"use strict";
|
|
22732
23111
|
init_src();
|
|
22733
|
-
|
|
23112
|
+
logger76 = createLogger("policy-store");
|
|
22734
23113
|
PolicyStore = class _PolicyStore {
|
|
22735
23114
|
policies = /* @__PURE__ */ new Map();
|
|
22736
23115
|
static composeKey(organizationId, agentId) {
|
|
@@ -22739,7 +23118,7 @@ var init_policy_store = __esm({
|
|
|
22739
23118
|
set(organizationId, agentId, bundle) {
|
|
22740
23119
|
const prepared = prepareBundle(organizationId, agentId, bundle);
|
|
22741
23120
|
this.policies.set(_PolicyStore.composeKey(organizationId, agentId), prepared);
|
|
22742
|
-
|
|
23121
|
+
logger76.debug("Set egress policy bundle", {
|
|
22743
23122
|
organizationId,
|
|
22744
23123
|
agentId,
|
|
22745
23124
|
domains: prepared.judgedDomains.length,
|
|
@@ -22770,7 +23149,7 @@ var init_policy_store = __esm({
|
|
|
22770
23149
|
const judgeName = matched.judge ?? "default";
|
|
22771
23150
|
const judge = prepared.preparedJudges[judgeName];
|
|
22772
23151
|
if (!judge) {
|
|
22773
|
-
|
|
23152
|
+
logger76.warn(
|
|
22774
23153
|
"Judge rule matched but named policy not found \u2014 failing closed",
|
|
22775
23154
|
{ organizationId, agentId, hostname, judgeName }
|
|
22776
23155
|
);
|
|
@@ -22788,7 +23167,7 @@ var init_policy_store = __esm({
|
|
|
22788
23167
|
});
|
|
22789
23168
|
|
|
22790
23169
|
// src/gateway/proxy/secret-proxy.ts
|
|
22791
|
-
import { Hono as
|
|
23170
|
+
import { Hono as Hono20 } from "hono";
|
|
22792
23171
|
function defaultPlaceholderTtlSeconds() {
|
|
22793
23172
|
const raw = process.env.SECRET_PLACEHOLDER_TTL_MS;
|
|
22794
23173
|
if (raw) {
|
|
@@ -22803,8 +23182,8 @@ function lookupPlaceholderMapping(placeholder, expectedOrganizationId) {
|
|
|
22803
23182
|
const uuid = placeholder.slice(prefixIdx + PLACEHOLDER_PREFIX.length);
|
|
22804
23183
|
const mapping = placeholderCache.get(uuid);
|
|
22805
23184
|
if (!mapping) return null;
|
|
22806
|
-
if (expectedOrganizationId
|
|
22807
|
-
|
|
23185
|
+
if (expectedOrganizationId !== void 0 && mapping.organizationId !== expectedOrganizationId) {
|
|
23186
|
+
logger77.warn(
|
|
22808
23187
|
{
|
|
22809
23188
|
mappingAgentId: mapping.agentId,
|
|
22810
23189
|
mappingOrg: mapping.organizationId,
|
|
@@ -22814,6 +23193,15 @@ function lookupPlaceholderMapping(placeholder, expectedOrganizationId) {
|
|
|
22814
23193
|
);
|
|
22815
23194
|
return null;
|
|
22816
23195
|
}
|
|
23196
|
+
if (!mapping.organizationId) {
|
|
23197
|
+
logger77.warn(
|
|
23198
|
+
{
|
|
23199
|
+
mappingAgentId: mapping.agentId,
|
|
23200
|
+
expectedOrg: expectedOrganizationId
|
|
23201
|
+
},
|
|
23202
|
+
"Placeholder mapping accessed without organizationId \u2014 legacy row, schedule rotation"
|
|
23203
|
+
);
|
|
23204
|
+
}
|
|
22817
23205
|
return mapping;
|
|
22818
23206
|
}
|
|
22819
23207
|
function safeDecodePathSegment(value) {
|
|
@@ -22845,13 +23233,14 @@ function generatePlaceholder(agentId, envVarName, secretRef, deploymentName, opt
|
|
|
22845
23233
|
);
|
|
22846
23234
|
return `${PLACEHOLDER_PREFIX}${uuid}`;
|
|
22847
23235
|
}
|
|
22848
|
-
var
|
|
23236
|
+
var logger77, PLACEHOLDER_PREFIX, RESOLVE_FAILURE_THRESHOLD, RESOLVE_FAILURE_WINDOW_MS, ResolutionFailureLimiter, resolutionFailureLimiter, PlaceholderCache, placeholderCache, SecretProxy;
|
|
22849
23237
|
var init_secret_proxy = __esm({
|
|
22850
23238
|
"src/gateway/proxy/secret-proxy.ts"() {
|
|
22851
23239
|
"use strict";
|
|
22852
23240
|
init_src();
|
|
23241
|
+
init_org_context();
|
|
22853
23242
|
init_rate_limiter();
|
|
22854
|
-
|
|
23243
|
+
logger77 = createLogger("secret-proxy");
|
|
22855
23244
|
PLACEHOLDER_PREFIX = "lobu_secret_";
|
|
22856
23245
|
RESOLVE_FAILURE_THRESHOLD = 20;
|
|
22857
23246
|
RESOLVE_FAILURE_WINDOW_MS = 5 * 60 * 1e3;
|
|
@@ -22961,11 +23350,11 @@ var init_secret_proxy = __esm({
|
|
|
22961
23350
|
this.slugMap = /* @__PURE__ */ new Map();
|
|
22962
23351
|
for (const upstream of config.providerUpstreams ?? []) {
|
|
22963
23352
|
this.slugMap.set(upstream.slug, upstream.upstreamBaseUrl);
|
|
22964
|
-
|
|
23353
|
+
logger77.debug(
|
|
22965
23354
|
`Registered provider upstream: ${upstream.slug} -> ${upstream.upstreamBaseUrl}`
|
|
22966
23355
|
);
|
|
22967
23356
|
}
|
|
22968
|
-
this.app = new
|
|
23357
|
+
this.app = new Hono20();
|
|
22969
23358
|
this.setupRoutes();
|
|
22970
23359
|
}
|
|
22971
23360
|
setAuthProfilesManager(manager) {
|
|
@@ -22996,7 +23385,7 @@ var init_secret_proxy = __esm({
|
|
|
22996
23385
|
if (providerId) {
|
|
22997
23386
|
this.slugToProviderId.set(upstream.slug, providerId);
|
|
22998
23387
|
}
|
|
22999
|
-
|
|
23388
|
+
logger77.debug(
|
|
23000
23389
|
`Registered provider upstream: ${upstream.slug} -> ${upstream.upstreamBaseUrl}${providerId ? ` (providerId: ${providerId})` : ""}`
|
|
23001
23390
|
);
|
|
23002
23391
|
}
|
|
@@ -23018,7 +23407,7 @@ var init_secret_proxy = __esm({
|
|
|
23018
23407
|
try {
|
|
23019
23408
|
return await this.forward(c);
|
|
23020
23409
|
} catch (error) {
|
|
23021
|
-
|
|
23410
|
+
logger77.error("Secret proxy error:", error);
|
|
23022
23411
|
return c.json({ error: "Internal proxy error" }, 500);
|
|
23023
23412
|
}
|
|
23024
23413
|
}
|
|
@@ -23114,12 +23503,12 @@ var init_secret_proxy = __esm({
|
|
|
23114
23503
|
const { shouldLog, nowThrottled } = resolutionFailureLimiter.recordFailure(source);
|
|
23115
23504
|
if (shouldLog) {
|
|
23116
23505
|
if (nowThrottled) {
|
|
23117
|
-
|
|
23506
|
+
logger77.warn(
|
|
23118
23507
|
{ source },
|
|
23119
23508
|
"Throttling placeholder resolution for source after repeated failures"
|
|
23120
23509
|
);
|
|
23121
23510
|
} else {
|
|
23122
|
-
|
|
23511
|
+
logger77.warn({ source }, "Failed to resolve secret placeholder");
|
|
23123
23512
|
}
|
|
23124
23513
|
}
|
|
23125
23514
|
return "";
|
|
@@ -23170,9 +23559,13 @@ var init_secret_proxy = __esm({
|
|
|
23170
23559
|
const orgId = await this.agentOrgResolver(urlAgentId);
|
|
23171
23560
|
if (orgId) expectedOrganizationId = orgId;
|
|
23172
23561
|
} catch (err) {
|
|
23173
|
-
|
|
23562
|
+
logger77.error(
|
|
23174
23563
|
{ urlAgentId, err: String(err) },
|
|
23175
|
-
"agentOrgResolver failed \u2014
|
|
23564
|
+
"agentOrgResolver failed \u2014 rejecting request to preserve org isolation"
|
|
23565
|
+
);
|
|
23566
|
+
return c.json(
|
|
23567
|
+
{ error: "Service Unavailable: failed to resolve agent organization" },
|
|
23568
|
+
503
|
|
23176
23569
|
);
|
|
23177
23570
|
}
|
|
23178
23571
|
}
|
|
@@ -23183,26 +23576,26 @@ var init_secret_proxy = __esm({
|
|
|
23183
23576
|
expectedOrganizationId
|
|
23184
23577
|
);
|
|
23185
23578
|
if (!mapping) {
|
|
23186
|
-
|
|
23579
|
+
logger77.warn(
|
|
23187
23580
|
{ urlAgentId },
|
|
23188
23581
|
"Rejecting proxy request: placeholder did not resolve"
|
|
23189
23582
|
);
|
|
23190
23583
|
return c.json({ error: "Unauthorized" }, 401);
|
|
23191
23584
|
}
|
|
23192
23585
|
if (mapping.agentId !== urlAgentId) {
|
|
23193
|
-
|
|
23586
|
+
logger77.warn(
|
|
23194
23587
|
{ urlAgentId, mappingAgentId: mapping.agentId },
|
|
23195
23588
|
"Rejecting proxy request: placeholder agentId does not match URL"
|
|
23196
23589
|
);
|
|
23197
23590
|
return c.json({ error: "Forbidden" }, 403);
|
|
23198
23591
|
}
|
|
23199
23592
|
} else if (callerToken) {
|
|
23200
|
-
|
|
23593
|
+
logger77.debug(
|
|
23201
23594
|
{ urlAgentId },
|
|
23202
23595
|
"Proxy request authenticated by non-placeholder token; agentId binding skipped"
|
|
23203
23596
|
);
|
|
23204
23597
|
} else {
|
|
23205
|
-
|
|
23598
|
+
logger77.warn(
|
|
23206
23599
|
{ urlAgentId },
|
|
23207
23600
|
"Rejecting proxy request: names an agent but carries no auth header"
|
|
23208
23601
|
);
|
|
@@ -23225,17 +23618,23 @@ var init_secret_proxy = __esm({
|
|
|
23225
23618
|
if (urlAgentId && resolvedSlug && this.authProfilesManager) {
|
|
23226
23619
|
const providerId = this.slugToProviderId.get(resolvedSlug);
|
|
23227
23620
|
if (providerId) {
|
|
23228
|
-
const
|
|
23229
|
-
|
|
23230
|
-
|
|
23231
|
-
|
|
23232
|
-
|
|
23621
|
+
const runWithOrg = (fn) => expectedOrganizationId ? orgContext.run({ organizationId: expectedOrganizationId }, fn) : fn();
|
|
23622
|
+
const authProfilesManager = this.authProfilesManager;
|
|
23623
|
+
const profile = await runWithOrg(
|
|
23624
|
+
() => authProfilesManager.getBestProfile(
|
|
23625
|
+
urlAgentId,
|
|
23626
|
+
providerId,
|
|
23627
|
+
void 0,
|
|
23628
|
+
providerContext
|
|
23629
|
+
)
|
|
23233
23630
|
);
|
|
23234
23631
|
const userIdForRefresh = providerContext?.userId;
|
|
23235
|
-
const credential = profile && userIdForRefresh ? await
|
|
23236
|
-
|
|
23237
|
-
|
|
23238
|
-
|
|
23632
|
+
const credential = profile && userIdForRefresh ? await runWithOrg(
|
|
23633
|
+
() => authProfilesManager.ensureFreshCredential(profile, {
|
|
23634
|
+
userId: userIdForRefresh,
|
|
23635
|
+
agentId: urlAgentId
|
|
23636
|
+
})
|
|
23637
|
+
) : profile?.credential;
|
|
23239
23638
|
if (credential) {
|
|
23240
23639
|
headers.authorization = `Bearer ${credential}`;
|
|
23241
23640
|
} else if (this.systemKeyResolver) {
|
|
@@ -23243,7 +23642,7 @@ var init_secret_proxy = __esm({
|
|
|
23243
23642
|
if (systemKey) {
|
|
23244
23643
|
headers.authorization = `Bearer ${systemKey}`;
|
|
23245
23644
|
} else {
|
|
23246
|
-
|
|
23645
|
+
logger77.warn(
|
|
23247
23646
|
`No auth profile or system key for agent ${urlAgentId}, provider ${providerId}`
|
|
23248
23647
|
);
|
|
23249
23648
|
return c.json(
|
|
@@ -23258,7 +23657,7 @@ var init_secret_proxy = __esm({
|
|
|
23258
23657
|
);
|
|
23259
23658
|
}
|
|
23260
23659
|
} else {
|
|
23261
|
-
|
|
23660
|
+
logger77.warn(
|
|
23262
23661
|
`No auth profile for agent ${urlAgentId}, provider ${providerId}`
|
|
23263
23662
|
);
|
|
23264
23663
|
return c.json(
|
|
@@ -23273,7 +23672,7 @@ var init_secret_proxy = __esm({
|
|
|
23273
23672
|
);
|
|
23274
23673
|
}
|
|
23275
23674
|
} else {
|
|
23276
|
-
|
|
23675
|
+
logger77.warn(`No providerId mapping for slug "${resolvedSlug}"`);
|
|
23277
23676
|
}
|
|
23278
23677
|
} else {
|
|
23279
23678
|
const source = urlAgentId ?? getClientIp({
|
|
@@ -23301,10 +23700,10 @@ var init_secret_proxy = __esm({
|
|
|
23301
23700
|
}
|
|
23302
23701
|
}
|
|
23303
23702
|
}
|
|
23304
|
-
|
|
23703
|
+
logger77.info(`Forwarding to upstream: ${method} ${upstream}`);
|
|
23305
23704
|
const response = await fetch(upstream, { method, headers, body: body2 });
|
|
23306
23705
|
if (!response.ok) {
|
|
23307
|
-
|
|
23706
|
+
logger77.warn(
|
|
23308
23707
|
`Upstream returned ${response.status} for ${method} ${upstream}`
|
|
23309
23708
|
);
|
|
23310
23709
|
}
|
|
@@ -23341,14 +23740,14 @@ var init_secret_proxy = __esm({
|
|
|
23341
23740
|
});
|
|
23342
23741
|
|
|
23343
23742
|
// src/gateway/proxy/token-refresh-job.ts
|
|
23344
|
-
var
|
|
23743
|
+
var logger78, EXPIRY_BUFFER_MS, TokenRefreshJob;
|
|
23345
23744
|
var init_token_refresh_job = __esm({
|
|
23346
23745
|
"src/gateway/proxy/token-refresh-job.ts"() {
|
|
23347
23746
|
"use strict";
|
|
23348
23747
|
init_src();
|
|
23349
23748
|
init_client();
|
|
23350
23749
|
init_org_context();
|
|
23351
|
-
|
|
23750
|
+
logger78 = createLogger("token-refresh-job");
|
|
23352
23751
|
EXPIRY_BUFFER_MS = 5 * 60 * 1e3;
|
|
23353
23752
|
TokenRefreshJob = class {
|
|
23354
23753
|
constructor(authProfilesManager, refreshableProviders) {
|
|
@@ -23375,7 +23774,7 @@ var init_token_refresh_job = __esm({
|
|
|
23375
23774
|
() => this.maybeRefresh(userId, agentId)
|
|
23376
23775
|
);
|
|
23377
23776
|
} catch (err) {
|
|
23378
|
-
|
|
23777
|
+
logger78.warn(
|
|
23379
23778
|
{ userId, agentId, organizationId, err: String(err) },
|
|
23380
23779
|
"Token refresh failed for user/agent \u2014 continuing scan"
|
|
23381
23780
|
);
|
|
@@ -23398,7 +23797,7 @@ var init_token_refresh_job = __esm({
|
|
|
23398
23797
|
async refreshForUserAgent(userId, agentId) {
|
|
23399
23798
|
const organizationId = await this.lookupAgentOrg(agentId);
|
|
23400
23799
|
if (organizationId === null) {
|
|
23401
|
-
|
|
23800
|
+
logger78.warn(
|
|
23402
23801
|
{ userId, agentId },
|
|
23403
23802
|
"Skipping token refresh \u2014 agent has no org row (deleted?)"
|
|
23404
23803
|
);
|
|
@@ -23445,7 +23844,7 @@ var init_token_refresh_job = __esm({
|
|
|
23445
23844
|
const expiresAt = oauthProfile.metadata.expiresAt || 0;
|
|
23446
23845
|
const isExpiring = expiresAt <= Date.now() + EXPIRY_BUFFER_MS;
|
|
23447
23846
|
if (!isExpiring) continue;
|
|
23448
|
-
|
|
23847
|
+
logger78.info(
|
|
23449
23848
|
`Refreshing ${providerId} token for user ${userId} agent ${agentId} profile ${oauthProfile.id}`,
|
|
23450
23849
|
{ expiresAt: new Date(expiresAt).toISOString() }
|
|
23451
23850
|
);
|
|
@@ -23469,11 +23868,11 @@ var init_token_refresh_job = __esm({
|
|
|
23469
23868
|
},
|
|
23470
23869
|
makePrimary: false
|
|
23471
23870
|
});
|
|
23472
|
-
|
|
23871
|
+
logger78.info(
|
|
23473
23872
|
`Token refreshed for user ${userId} agent ${agentId} (${providerId})`
|
|
23474
23873
|
);
|
|
23475
23874
|
} catch (error) {
|
|
23476
|
-
|
|
23875
|
+
logger78.error(
|
|
23477
23876
|
`Failed to refresh ${providerId} token for user ${userId} agent ${agentId}`,
|
|
23478
23877
|
{
|
|
23479
23878
|
error,
|
|
@@ -24045,12 +24444,12 @@ async function loadBedrockModelsFromAws() {
|
|
|
24045
24444
|
function loadFallbackRegistryModels() {
|
|
24046
24445
|
return getModels("amazon-bedrock").map(normalizeRegistryModel).sort(sortModels);
|
|
24047
24446
|
}
|
|
24048
|
-
var
|
|
24447
|
+
var logger79, CACHE_TTL_MS3, BedrockModelCatalog;
|
|
24049
24448
|
var init_bedrock_model_catalog = __esm({
|
|
24050
24449
|
"src/gateway/services/bedrock-model-catalog.ts"() {
|
|
24051
24450
|
"use strict";
|
|
24052
24451
|
init_src();
|
|
24053
|
-
|
|
24452
|
+
logger79 = createLogger("bedrock-model-catalog");
|
|
24054
24453
|
CACHE_TTL_MS3 = 5 * 60 * 1e3;
|
|
24055
24454
|
BedrockModelCatalog = class {
|
|
24056
24455
|
cacheTtlMs;
|
|
@@ -24073,7 +24472,7 @@ var init_bedrock_model_catalog = __esm({
|
|
|
24073
24472
|
};
|
|
24074
24473
|
return models;
|
|
24075
24474
|
} catch (error) {
|
|
24076
|
-
|
|
24475
|
+
logger79.warn(
|
|
24077
24476
|
{
|
|
24078
24477
|
error: error instanceof Error ? error.message : String(error)
|
|
24079
24478
|
},
|
|
@@ -24107,7 +24506,7 @@ var init_bedrock_model_catalog = __esm({
|
|
|
24107
24506
|
// src/gateway/services/bedrock-openai-service.ts
|
|
24108
24507
|
import { getModel } from "@mariozechner/pi-ai";
|
|
24109
24508
|
import { streamBedrock } from "@mariozechner/pi-ai/dist/providers/amazon-bedrock.js";
|
|
24110
|
-
import { Hono as
|
|
24509
|
+
import { Hono as Hono21 } from "hono";
|
|
24111
24510
|
function parseDataUrl(url) {
|
|
24112
24511
|
if (!url) return null;
|
|
24113
24512
|
const match = url.match(/^data:([^;,]+);base64,(.+)$/);
|
|
@@ -24394,16 +24793,16 @@ function createSseStream(requestModel, stream2, includeUsage) {
|
|
|
24394
24793
|
}
|
|
24395
24794
|
});
|
|
24396
24795
|
}
|
|
24397
|
-
var
|
|
24796
|
+
var logger80, BedrockOpenAIService;
|
|
24398
24797
|
var init_bedrock_openai_service = __esm({
|
|
24399
24798
|
"src/gateway/services/bedrock-openai-service.ts"() {
|
|
24400
24799
|
"use strict";
|
|
24401
24800
|
init_src();
|
|
24402
24801
|
init_middleware();
|
|
24403
24802
|
init_bedrock_model_catalog();
|
|
24404
|
-
|
|
24803
|
+
logger80 = createLogger("bedrock-openai-service");
|
|
24405
24804
|
BedrockOpenAIService = class {
|
|
24406
|
-
app = new
|
|
24805
|
+
app = new Hono21();
|
|
24407
24806
|
modelCatalog;
|
|
24408
24807
|
modelResolver;
|
|
24409
24808
|
bedrockStreamer;
|
|
@@ -24492,7 +24891,7 @@ var init_bedrock_openai_service = __esm({
|
|
|
24492
24891
|
const context2 = buildBedrockContext(request2);
|
|
24493
24892
|
const stopSequences = Array.isArray(request2.stop) ? request2.stop : typeof request2.stop === "string" ? [request2.stop] : [];
|
|
24494
24893
|
const toolChoice = mapToolChoice(request2.tool_choice);
|
|
24495
|
-
|
|
24894
|
+
logger80.info(
|
|
24496
24895
|
{
|
|
24497
24896
|
modelId: request2.model,
|
|
24498
24897
|
region,
|
|
@@ -24577,12 +24976,12 @@ function buildRegistryMap(configAgents) {
|
|
|
24577
24976
|
}
|
|
24578
24977
|
return result;
|
|
24579
24978
|
}
|
|
24580
|
-
var
|
|
24979
|
+
var logger81, DeclaredAgentRegistry;
|
|
24581
24980
|
var init_declared_agent_registry = __esm({
|
|
24582
24981
|
"src/gateway/services/declared-agent-registry.ts"() {
|
|
24583
24982
|
"use strict";
|
|
24584
24983
|
init_src();
|
|
24585
|
-
|
|
24984
|
+
logger81 = createLogger("declared-agent-registry");
|
|
24586
24985
|
DeclaredAgentRegistry = class {
|
|
24587
24986
|
entries = /* @__PURE__ */ new Map();
|
|
24588
24987
|
has(agentId) {
|
|
@@ -24603,7 +25002,7 @@ var init_declared_agent_registry = __esm({
|
|
|
24603
25002
|
for (const [agentId, entry] of next) {
|
|
24604
25003
|
this.entries.set(agentId, entry);
|
|
24605
25004
|
}
|
|
24606
|
-
|
|
25005
|
+
logger81.debug(`Registry now holds ${this.entries.size} declared agent(s)`);
|
|
24607
25006
|
}
|
|
24608
25007
|
};
|
|
24609
25008
|
}
|
|
@@ -24645,12 +25044,12 @@ function hasImageGenerationAccess(profileProviderId, profile) {
|
|
|
24645
25044
|
if (!scopes) return true;
|
|
24646
25045
|
return scopes.has("api.model.image.request") || scopes.has("api.model.request") || scopes.has("model.image.request");
|
|
24647
25046
|
}
|
|
24648
|
-
var
|
|
25047
|
+
var logger82, IMAGE_CAPABLE_PROVIDERS, ImageGenerationService;
|
|
24649
25048
|
var init_image_generation_service = __esm({
|
|
24650
25049
|
"src/gateway/services/image-generation-service.ts"() {
|
|
24651
25050
|
"use strict";
|
|
24652
25051
|
init_src();
|
|
24653
|
-
|
|
25052
|
+
logger82 = createLogger("image-generation-service");
|
|
24654
25053
|
IMAGE_CAPABLE_PROVIDERS = [
|
|
24655
25054
|
{
|
|
24656
25055
|
profileProviderId: "chatgpt",
|
|
@@ -24685,7 +25084,7 @@ var init_image_generation_service = __esm({
|
|
|
24685
25084
|
);
|
|
24686
25085
|
if (!profile?.credential) continue;
|
|
24687
25086
|
if (!hasImageGenerationAccess(profileProviderId, profile)) {
|
|
24688
|
-
|
|
25087
|
+
logger82.info("Skipping provider without image-generation scope", {
|
|
24689
25088
|
agentId,
|
|
24690
25089
|
profileProviderId,
|
|
24691
25090
|
authType: profile.authType
|
|
@@ -24715,7 +25114,7 @@ var init_image_generation_service = __esm({
|
|
|
24715
25114
|
agentId
|
|
24716
25115
|
);
|
|
24717
25116
|
}
|
|
24718
|
-
|
|
25117
|
+
logger82.info("Generating image", {
|
|
24719
25118
|
agentId,
|
|
24720
25119
|
provider: config.provider,
|
|
24721
25120
|
profileProviderId: config.profileProviderId,
|
|
@@ -24734,7 +25133,7 @@ var init_image_generation_service = __esm({
|
|
|
24734
25133
|
};
|
|
24735
25134
|
} catch (error) {
|
|
24736
25135
|
const errorMessage3 = error instanceof Error ? error.message : String(error);
|
|
24737
|
-
|
|
25136
|
+
logger82.error("Image generation failed", {
|
|
24738
25137
|
agentId,
|
|
24739
25138
|
provider: config.provider,
|
|
24740
25139
|
profileProviderId: config.profileProviderId,
|
|
@@ -24748,7 +25147,7 @@ var init_image_generation_service = __esm({
|
|
|
24748
25147
|
}
|
|
24749
25148
|
noProviderError(message, agentId) {
|
|
24750
25149
|
const availableProviders = IMAGE_CAPABLE_PROVIDERS.map((p) => p.provider);
|
|
24751
|
-
|
|
25150
|
+
logger82.info(message, { agentId, availableProviders });
|
|
24752
25151
|
return { error: message, availableProviders };
|
|
24753
25152
|
}
|
|
24754
25153
|
async generateWithOpenAI(prompt, apiKey, options) {
|
|
@@ -24837,13 +25236,13 @@ var init_session = __esm({
|
|
|
24837
25236
|
});
|
|
24838
25237
|
|
|
24839
25238
|
// src/gateway/services/session-manager.ts
|
|
24840
|
-
var
|
|
25239
|
+
var logger83, StateAdapterSessionStore, SessionManager;
|
|
24841
25240
|
var init_session_manager = __esm({
|
|
24842
25241
|
"src/gateway/services/session-manager.ts"() {
|
|
24843
25242
|
"use strict";
|
|
24844
25243
|
init_src();
|
|
24845
25244
|
init_session();
|
|
24846
|
-
|
|
25245
|
+
logger83 = createLogger("session-manager");
|
|
24847
25246
|
StateAdapterSessionStore = class {
|
|
24848
25247
|
constructor(conversations) {
|
|
24849
25248
|
this.conversations = conversations;
|
|
@@ -24853,23 +25252,23 @@ var init_session_manager = __esm({
|
|
|
24853
25252
|
try {
|
|
24854
25253
|
return await this.conversations.getSession(sessionKey3);
|
|
24855
25254
|
} catch (error) {
|
|
24856
|
-
|
|
25255
|
+
logger83.error(`Failed to get session ${sessionKey3}:`, error);
|
|
24857
25256
|
return null;
|
|
24858
25257
|
}
|
|
24859
25258
|
}
|
|
24860
25259
|
async set(sessionKey3, session) {
|
|
24861
25260
|
await this.conversations.setSession(sessionKey3, session);
|
|
24862
|
-
|
|
25261
|
+
logger83.debug(`Stored session ${sessionKey3}`);
|
|
24863
25262
|
}
|
|
24864
25263
|
async delete(sessionKey3) {
|
|
24865
25264
|
await this.conversations.deleteSession(sessionKey3);
|
|
24866
|
-
|
|
25265
|
+
logger83.debug(`Deleted session ${sessionKey3}`);
|
|
24867
25266
|
}
|
|
24868
25267
|
async getByThread(channelId, threadTs) {
|
|
24869
25268
|
try {
|
|
24870
25269
|
return await this.conversations.getSessionByThread(channelId, threadTs);
|
|
24871
25270
|
} catch (error) {
|
|
24872
|
-
|
|
25271
|
+
logger83.error(
|
|
24873
25272
|
`Failed to get session by thread ${channelId}:${threadTs}:`,
|
|
24874
25273
|
error
|
|
24875
25274
|
);
|
|
@@ -24878,7 +25277,7 @@ var init_session_manager = __esm({
|
|
|
24878
25277
|
}
|
|
24879
25278
|
/** Optional cleanup - state adapter TTL handles this automatically */
|
|
24880
25279
|
async cleanup() {
|
|
24881
|
-
|
|
25280
|
+
logger83.debug("StateAdapter TTL handles automatic cleanup");
|
|
24882
25281
|
return 0;
|
|
24883
25282
|
}
|
|
24884
25283
|
};
|
|
@@ -25148,17 +25547,17 @@ data: ${JSON.stringify({ reason })}
|
|
|
25148
25547
|
});
|
|
25149
25548
|
|
|
25150
25549
|
// src/gateway/watchers/run-tracker.ts
|
|
25151
|
-
var
|
|
25550
|
+
var logger84, WatcherRunTracker;
|
|
25152
25551
|
var init_run_tracker = __esm({
|
|
25153
25552
|
"src/gateway/watchers/run-tracker.ts"() {
|
|
25154
25553
|
"use strict";
|
|
25155
25554
|
init_src();
|
|
25156
|
-
|
|
25555
|
+
logger84 = createLogger("watcher-run-tracker");
|
|
25157
25556
|
WatcherRunTracker = class {
|
|
25158
25557
|
pending = /* @__PURE__ */ new Map();
|
|
25159
25558
|
register(handle) {
|
|
25160
25559
|
if (this.pending.has(handle.messageId)) {
|
|
25161
|
-
|
|
25560
|
+
logger84.warn(
|
|
25162
25561
|
{ messageId: handle.messageId, runId: handle.runId },
|
|
25163
25562
|
"duplicate watcher run registration ignored"
|
|
25164
25563
|
);
|
|
@@ -25173,7 +25572,7 @@ var init_run_tracker = __esm({
|
|
|
25173
25572
|
try {
|
|
25174
25573
|
await handle.onResolve(result);
|
|
25175
25574
|
} catch (err) {
|
|
25176
|
-
|
|
25575
|
+
logger84.error(
|
|
25177
25576
|
{ err, messageId, runId: handle.runId },
|
|
25178
25577
|
"watcher run onResolve callback failed"
|
|
25179
25578
|
);
|
|
@@ -25235,12 +25634,12 @@ function resolveProviderRegistryFromRaw(raw) {
|
|
|
25235
25634
|
try {
|
|
25236
25635
|
rawParsed = JSON.parse(raw);
|
|
25237
25636
|
} catch {
|
|
25238
|
-
|
|
25637
|
+
logger85.error("Invalid providers JSON");
|
|
25239
25638
|
return null;
|
|
25240
25639
|
}
|
|
25241
25640
|
const substituted = raw.replace(/\$\{env:([^}]+)\}/g, (_match, varName) => {
|
|
25242
25641
|
if (isBlockedEnvSubstitution(varName)) {
|
|
25243
|
-
|
|
25642
|
+
logger85.warn(`Blocked env substitution for sensitive var: ${varName}`);
|
|
25244
25643
|
return "";
|
|
25245
25644
|
}
|
|
25246
25645
|
return process.env[varName] || "";
|
|
@@ -25249,21 +25648,21 @@ function resolveProviderRegistryFromRaw(raw) {
|
|
|
25249
25648
|
try {
|
|
25250
25649
|
parsed = JSON.parse(substituted);
|
|
25251
25650
|
} catch {
|
|
25252
|
-
|
|
25651
|
+
logger85.error("Invalid providers JSON after env substitution");
|
|
25253
25652
|
return null;
|
|
25254
25653
|
}
|
|
25255
25654
|
if (!Array.isArray(parsed.providers)) {
|
|
25256
|
-
|
|
25655
|
+
logger85.error("Invalid providers config: missing 'providers' array");
|
|
25257
25656
|
return null;
|
|
25258
25657
|
}
|
|
25259
25658
|
return { raw: rawParsed, resolved: parsed };
|
|
25260
25659
|
}
|
|
25261
|
-
var
|
|
25660
|
+
var logger85, ENV_SUBSTITUTION_BLOCKLIST, ProviderRegistryService;
|
|
25262
25661
|
var init_provider_registry_service = __esm({
|
|
25263
25662
|
"src/gateway/services/provider-registry-service.ts"() {
|
|
25264
25663
|
"use strict";
|
|
25265
25664
|
init_src();
|
|
25266
|
-
|
|
25665
|
+
logger85 = createLogger("provider-registry-service");
|
|
25267
25666
|
ENV_SUBSTITUTION_BLOCKLIST = /* @__PURE__ */ new Set([
|
|
25268
25667
|
"ENCRYPTION_KEY",
|
|
25269
25668
|
"DATABASE_URL",
|
|
@@ -25285,7 +25684,7 @@ var init_provider_registry_service = __esm({
|
|
|
25285
25684
|
const config = { providers: preloadedProviders };
|
|
25286
25685
|
this.loaded = config;
|
|
25287
25686
|
this.rawLoaded = config;
|
|
25288
|
-
|
|
25687
|
+
logger85.info(
|
|
25289
25688
|
`Loaded ${preloadedProviders.length} bundled provider(s) (injected)`
|
|
25290
25689
|
);
|
|
25291
25690
|
}
|
|
@@ -25322,7 +25721,7 @@ var init_provider_registry_service = __esm({
|
|
|
25322
25721
|
if (this.configUrl.startsWith("http://") || this.configUrl.startsWith("https://")) {
|
|
25323
25722
|
const response = await fetch(this.configUrl);
|
|
25324
25723
|
if (!response.ok) {
|
|
25325
|
-
|
|
25724
|
+
logger85.error(`Failed to fetch providers config: ${response.status}`);
|
|
25326
25725
|
return null;
|
|
25327
25726
|
}
|
|
25328
25727
|
raw = await response.text();
|
|
@@ -25333,10 +25732,10 @@ var init_provider_registry_service = __esm({
|
|
|
25333
25732
|
if (!resolved) return null;
|
|
25334
25733
|
this.rawLoaded = resolved.raw;
|
|
25335
25734
|
this.loaded = resolved.resolved;
|
|
25336
|
-
|
|
25735
|
+
logger85.info(`Loaded ${this.loaded.providers.length} bundled provider(s)`);
|
|
25337
25736
|
return this.loaded;
|
|
25338
25737
|
} catch (error) {
|
|
25339
|
-
|
|
25738
|
+
logger85.debug("Providers config not available", { error });
|
|
25340
25739
|
return null;
|
|
25341
25740
|
}
|
|
25342
25741
|
}
|
|
@@ -25348,12 +25747,12 @@ var init_provider_registry_service = __esm({
|
|
|
25348
25747
|
function displayName(provider2) {
|
|
25349
25748
|
return TTS_CAPABLE_PROVIDERS.find((p) => p.ttsProvider === provider2)?.displayName ?? provider2;
|
|
25350
25749
|
}
|
|
25351
|
-
var
|
|
25750
|
+
var logger86, TTS_CAPABLE_PROVIDERS, TranscriptionService;
|
|
25352
25751
|
var init_transcription_service = __esm({
|
|
25353
25752
|
"src/gateway/services/transcription-service.ts"() {
|
|
25354
25753
|
"use strict";
|
|
25355
25754
|
init_src();
|
|
25356
|
-
|
|
25755
|
+
logger86 = createLogger("transcription-service");
|
|
25357
25756
|
TTS_CAPABLE_PROVIDERS = [
|
|
25358
25757
|
{
|
|
25359
25758
|
profileProviderId: "chatgpt",
|
|
@@ -25394,7 +25793,7 @@ var init_transcription_service = __esm({
|
|
|
25394
25793
|
}
|
|
25395
25794
|
const attemptErrors = [];
|
|
25396
25795
|
for (const config of configs) {
|
|
25397
|
-
|
|
25796
|
+
logger86.info("Transcribing audio", {
|
|
25398
25797
|
agentId,
|
|
25399
25798
|
provider: config.provider,
|
|
25400
25799
|
profileProviderId: config.profileProviderId,
|
|
@@ -25407,7 +25806,7 @@ var init_transcription_service = __esm({
|
|
|
25407
25806
|
config,
|
|
25408
25807
|
mimeType
|
|
25409
25808
|
);
|
|
25410
|
-
|
|
25809
|
+
logger86.info("Transcription successful", {
|
|
25411
25810
|
agentId,
|
|
25412
25811
|
provider: config.provider,
|
|
25413
25812
|
profileProviderId: config.profileProviderId,
|
|
@@ -25416,7 +25815,7 @@ var init_transcription_service = __esm({
|
|
|
25416
25815
|
return { text, provider: config.provider };
|
|
25417
25816
|
} catch (error) {
|
|
25418
25817
|
const errorMessage3 = error instanceof Error ? error.message : String(error);
|
|
25419
|
-
|
|
25818
|
+
logger86.error("Transcription failed", {
|
|
25420
25819
|
agentId,
|
|
25421
25820
|
provider: config.provider,
|
|
25422
25821
|
profileProviderId: config.profileProviderId,
|
|
@@ -25484,7 +25883,7 @@ var init_transcription_service = __esm({
|
|
|
25484
25883
|
try {
|
|
25485
25884
|
providerConfigs = await this.providerConfigSource();
|
|
25486
25885
|
} catch (error) {
|
|
25487
|
-
|
|
25886
|
+
logger86.warn("Failed to load provider configs for STT", {
|
|
25488
25887
|
error: error instanceof Error ? error.message : String(error)
|
|
25489
25888
|
});
|
|
25490
25889
|
return [];
|
|
@@ -25496,7 +25895,7 @@ var init_transcription_service = __esm({
|
|
|
25496
25895
|
const sttEnabled = stt ? stt.enabled !== false : compat === "openai";
|
|
25497
25896
|
if (!sttEnabled) continue;
|
|
25498
25897
|
if (compat !== "openai") {
|
|
25499
|
-
|
|
25898
|
+
logger86.warn("Unsupported config-driven STT compatibility", {
|
|
25500
25899
|
providerId,
|
|
25501
25900
|
compat
|
|
25502
25901
|
});
|
|
@@ -25507,7 +25906,7 @@ var init_transcription_service = __esm({
|
|
|
25507
25906
|
stt?.baseUrl || entry.upstreamBaseUrl
|
|
25508
25907
|
);
|
|
25509
25908
|
if (!endpoint) {
|
|
25510
|
-
|
|
25909
|
+
logger86.warn("Invalid STT endpoint configuration", {
|
|
25511
25910
|
providerId,
|
|
25512
25911
|
transcriptionPath: stt?.transcriptionPath,
|
|
25513
25912
|
baseUrl: stt?.baseUrl || entry.upstreamBaseUrl
|
|
@@ -25546,7 +25945,7 @@ var init_transcription_service = __esm({
|
|
|
25546
25945
|
if (!config) {
|
|
25547
25946
|
return this.noProviderError("No audio provider configured", agentId);
|
|
25548
25947
|
}
|
|
25549
|
-
|
|
25948
|
+
logger86.info("Synthesizing audio", {
|
|
25550
25949
|
agentId,
|
|
25551
25950
|
provider: config.provider,
|
|
25552
25951
|
textLength: text.length,
|
|
@@ -25554,7 +25953,7 @@ var init_transcription_service = __esm({
|
|
|
25554
25953
|
});
|
|
25555
25954
|
try {
|
|
25556
25955
|
const result = await this.synthesizeWithProvider(text, config, options);
|
|
25557
|
-
|
|
25956
|
+
logger86.info("Synthesis successful", {
|
|
25558
25957
|
agentId,
|
|
25559
25958
|
provider: config.provider,
|
|
25560
25959
|
audioSize: result.audioBuffer.length
|
|
@@ -25562,7 +25961,7 @@ var init_transcription_service = __esm({
|
|
|
25562
25961
|
return { ...result, provider: config.provider };
|
|
25563
25962
|
} catch (error) {
|
|
25564
25963
|
const errorMessage3 = error instanceof Error ? error.message : String(error);
|
|
25565
|
-
|
|
25964
|
+
logger86.error("Synthesis failed", {
|
|
25566
25965
|
agentId,
|
|
25567
25966
|
provider: config.provider,
|
|
25568
25967
|
error: errorMessage3
|
|
@@ -25575,7 +25974,7 @@ var init_transcription_service = __esm({
|
|
|
25575
25974
|
}
|
|
25576
25975
|
noProviderError(message, agentId) {
|
|
25577
25976
|
const availableProviders = TTS_CAPABLE_PROVIDERS.map((p) => p.ttsProvider);
|
|
25578
|
-
|
|
25977
|
+
logger86.info(message, { agentId, availableProviders });
|
|
25579
25978
|
return { error: message, availableProviders };
|
|
25580
25979
|
}
|
|
25581
25980
|
// ==========================================================================
|
|
@@ -25828,7 +26227,7 @@ var init_transcription_service = __esm({
|
|
|
25828
26227
|
});
|
|
25829
26228
|
|
|
25830
26229
|
// src/gateway/services/core-services.ts
|
|
25831
|
-
var
|
|
26230
|
+
var logger87, CoreServices;
|
|
25832
26231
|
var init_core_services = __esm({
|
|
25833
26232
|
"src/gateway/services/core-services.ts"() {
|
|
25834
26233
|
"use strict";
|
|
@@ -25884,7 +26283,7 @@ var init_core_services = __esm({
|
|
|
25884
26283
|
init_provider_config_resolver();
|
|
25885
26284
|
init_provider_registry_service();
|
|
25886
26285
|
init_transcription_service();
|
|
25887
|
-
|
|
26286
|
+
logger87 = createLogger("core-services");
|
|
25888
26287
|
CoreServices = class {
|
|
25889
26288
|
constructor(config, options) {
|
|
25890
26289
|
this.config = config;
|
|
@@ -25990,20 +26389,20 @@ var init_core_services = __esm({
|
|
|
25990
26389
|
* Initialize all core services in dependency order
|
|
25991
26390
|
*/
|
|
25992
26391
|
async initialize() {
|
|
25993
|
-
|
|
26392
|
+
logger87.debug("Initializing core services...");
|
|
25994
26393
|
await this.initializeQueue();
|
|
25995
|
-
|
|
26394
|
+
logger87.debug("Queue initialized");
|
|
25996
26395
|
await this.initializeSessionServices();
|
|
25997
|
-
|
|
26396
|
+
logger87.debug("Session services initialized");
|
|
25998
26397
|
await this.initializeClaudeServices();
|
|
25999
|
-
|
|
26398
|
+
logger87.debug("Auth & provider services initialized");
|
|
26000
26399
|
await this.initializeMcpServices();
|
|
26001
|
-
|
|
26400
|
+
logger87.debug("MCP services initialized");
|
|
26002
26401
|
await this.initializeQueueProducer();
|
|
26003
|
-
|
|
26402
|
+
logger87.debug("Queue producer initialized");
|
|
26004
26403
|
this.initializeCommandRegistry();
|
|
26005
|
-
|
|
26006
|
-
|
|
26404
|
+
logger87.debug("Command registry initialized");
|
|
26405
|
+
logger87.info("Core services initialized successfully");
|
|
26007
26406
|
}
|
|
26008
26407
|
/** Public entry point for the scheduler-registered ephemeral-table sweep.
|
|
26009
26408
|
* Deletes expired rows from oauth_states, rate_limits, grants, and
|
|
@@ -26020,13 +26419,13 @@ var init_core_services = __esm({
|
|
|
26020
26419
|
]);
|
|
26021
26420
|
const pendingInteractions = pendingIds.length;
|
|
26022
26421
|
if (oauthStates + rate + grants + completedRuns + pendingInteractions > 0) {
|
|
26023
|
-
|
|
26422
|
+
logger87.debug(
|
|
26024
26423
|
{ oauthStates, rate, grants, completedRuns, pendingInteractions },
|
|
26025
26424
|
"Ephemeral table sweeper deleted expired rows"
|
|
26026
26425
|
);
|
|
26027
26426
|
}
|
|
26028
26427
|
} catch (error) {
|
|
26029
|
-
|
|
26428
|
+
logger87.warn(
|
|
26030
26429
|
{ error: error instanceof Error ? error.message : String(error) },
|
|
26031
26430
|
"Ephemeral table sweeper failed"
|
|
26032
26431
|
);
|
|
@@ -26038,7 +26437,7 @@ var init_core_services = __esm({
|
|
|
26038
26437
|
async initializeQueue() {
|
|
26039
26438
|
this.queue = new RunsQueue();
|
|
26040
26439
|
await this.queue.start();
|
|
26041
|
-
|
|
26440
|
+
logger87.debug("Queue connection established (runs-table substrate)");
|
|
26042
26441
|
}
|
|
26043
26442
|
async initializeQueueProducer() {
|
|
26044
26443
|
if (!this.queue) {
|
|
@@ -26046,7 +26445,7 @@ var init_core_services = __esm({
|
|
|
26046
26445
|
}
|
|
26047
26446
|
this.queueProducer = new QueueProducer(this.queue);
|
|
26048
26447
|
await this.queueProducer.start();
|
|
26049
|
-
|
|
26448
|
+
logger87.debug("Queue producer initialized");
|
|
26050
26449
|
}
|
|
26051
26450
|
// ============================================================================
|
|
26052
26451
|
// 2. Session Services Initialization
|
|
@@ -26061,17 +26460,17 @@ var init_core_services = __esm({
|
|
|
26061
26460
|
new ConversationStateStore(stateAdapter)
|
|
26062
26461
|
);
|
|
26063
26462
|
this.sessionManager = new SessionManager(sessionStore);
|
|
26064
|
-
|
|
26463
|
+
logger87.debug("Session manager initialized");
|
|
26065
26464
|
this.interactionService = new InteractionService();
|
|
26066
|
-
|
|
26465
|
+
logger87.debug("Interaction service initialized");
|
|
26067
26466
|
this.sseManager = new SseManager();
|
|
26068
|
-
|
|
26467
|
+
logger87.debug("SSE manager initialized");
|
|
26069
26468
|
this.watcherRunTracker = new WatcherRunTracker();
|
|
26070
|
-
|
|
26469
|
+
logger87.debug("Watcher run tracker initialized");
|
|
26071
26470
|
this.grantStore = new GrantStore();
|
|
26072
|
-
|
|
26471
|
+
logger87.debug("Grant store initialized");
|
|
26073
26472
|
this.policyStore = new PolicyStore();
|
|
26074
|
-
|
|
26473
|
+
logger87.debug("Policy store initialized");
|
|
26075
26474
|
const defaultSecretStore = new PostgresSecretStore();
|
|
26076
26475
|
this.secretStore = this.options?.secretStore ?? new SecretStoreRegistry(
|
|
26077
26476
|
defaultSecretStore,
|
|
@@ -26084,7 +26483,7 @@ var init_core_services = __esm({
|
|
|
26084
26483
|
}
|
|
26085
26484
|
}
|
|
26086
26485
|
);
|
|
26087
|
-
|
|
26486
|
+
logger87.debug("Secret store initialized");
|
|
26088
26487
|
this.channelBindingService = new ChannelBindingService();
|
|
26089
26488
|
this.userAgentsStore = new UserAgentsStore();
|
|
26090
26489
|
if (!this.configStore || !this.connectionStore || !this.accessStore) {
|
|
@@ -26097,7 +26496,7 @@ var init_core_services = __esm({
|
|
|
26097
26496
|
inMemoryStore,
|
|
26098
26497
|
this.config.agents
|
|
26099
26498
|
);
|
|
26100
|
-
|
|
26499
|
+
logger87.debug(
|
|
26101
26500
|
`Agent sub-stores initialized (in-memory, ${this.config.agents.length} agent(s) from config)`
|
|
26102
26501
|
);
|
|
26103
26502
|
} else {
|
|
@@ -26106,11 +26505,11 @@ var init_core_services = __esm({
|
|
|
26106
26505
|
);
|
|
26107
26506
|
}
|
|
26108
26507
|
} else {
|
|
26109
|
-
|
|
26508
|
+
logger87.debug("Using host-provided agent sub-stores (embedded mode)");
|
|
26110
26509
|
}
|
|
26111
26510
|
this.agentSettingsStore = new AgentSettingsStore(this.configStore);
|
|
26112
26511
|
this.agentMetadataStore = new AgentMetadataStore(this.configStore);
|
|
26113
|
-
|
|
26512
|
+
logger87.debug(
|
|
26114
26513
|
"Agent settings, channel binding, user agents & metadata stores initialized"
|
|
26115
26514
|
);
|
|
26116
26515
|
const externalAuthKv = /* @__PURE__ */ new Map();
|
|
@@ -26136,7 +26535,7 @@ var init_core_services = __esm({
|
|
|
26136
26535
|
}
|
|
26137
26536
|
}) ?? void 0;
|
|
26138
26537
|
if (this.externalAuthClient) {
|
|
26139
|
-
|
|
26538
|
+
logger87.debug("External OAuth client initialized");
|
|
26140
26539
|
}
|
|
26141
26540
|
}
|
|
26142
26541
|
// ============================================================================
|
|
@@ -26197,7 +26596,7 @@ var init_core_services = __esm({
|
|
|
26197
26596
|
}
|
|
26198
26597
|
}
|
|
26199
26598
|
}
|
|
26200
|
-
|
|
26599
|
+
logger87.debug(
|
|
26201
26600
|
"Auth profile, model preference, transcription, and image generation services initialized"
|
|
26202
26601
|
);
|
|
26203
26602
|
this.secretProxy = new SecretProxy(
|
|
@@ -26206,7 +26605,7 @@ var init_core_services = __esm({
|
|
|
26206
26605
|
},
|
|
26207
26606
|
this.secretStore
|
|
26208
26607
|
);
|
|
26209
|
-
|
|
26608
|
+
logger87.debug(
|
|
26210
26609
|
`Secret proxy initialized (upstream: ${this.config.anthropicProxy.anthropicBaseUrl || "https://api.anthropic.com"})`
|
|
26211
26610
|
);
|
|
26212
26611
|
if (!this.authProfilesManager) {
|
|
@@ -26217,24 +26616,24 @@ var init_core_services = __esm({
|
|
|
26217
26616
|
this.tokenRefreshJob = new TokenRefreshJob(this.authProfilesManager, [
|
|
26218
26617
|
{ providerId: "claude", oauthClient: new OAuthClient(CLAUDE_PROVIDER) }
|
|
26219
26618
|
]);
|
|
26220
|
-
|
|
26619
|
+
logger87.debug("Token refresh job constructed");
|
|
26221
26620
|
this.oauthStateStore = createOAuthStateStore("claude");
|
|
26222
26621
|
const claudeOAuthModule = new ClaudeOAuthModule(
|
|
26223
26622
|
this.authProfilesManager,
|
|
26224
26623
|
this.modelPreferenceStore
|
|
26225
26624
|
);
|
|
26226
26625
|
moduleRegistry.register(claudeOAuthModule);
|
|
26227
|
-
|
|
26626
|
+
logger87.debug(
|
|
26228
26627
|
`Claude OAuth module registered (system token: ${claudeOAuthModule.hasSystemKey() ? "available" : "not available"})`
|
|
26229
26628
|
);
|
|
26230
26629
|
const chatgptOAuthModule = new ChatGPTOAuthModule(this.authProfilesManager);
|
|
26231
26630
|
moduleRegistry.register(chatgptOAuthModule);
|
|
26232
|
-
|
|
26631
|
+
logger87.debug(
|
|
26233
26632
|
`ChatGPT OAuth module registered (system token: ${chatgptOAuthModule.hasSystemKey() ? "available" : "not available"})`
|
|
26234
26633
|
);
|
|
26235
26634
|
const geminiCliModule = new GeminiCliModule(this.authProfilesManager);
|
|
26236
26635
|
moduleRegistry.register(geminiCliModule);
|
|
26237
|
-
|
|
26636
|
+
logger87.debug("Gemini CLI module registered (acpx sub-agent shell-out)");
|
|
26238
26637
|
const bedrockModelCatalog = new BedrockModelCatalog();
|
|
26239
26638
|
const bedrockProviderModule = new BedrockProviderModule(
|
|
26240
26639
|
this.authProfilesManager,
|
|
@@ -26244,23 +26643,23 @@ var init_core_services = __esm({
|
|
|
26244
26643
|
this.bedrockOpenAIService = new BedrockOpenAIService({
|
|
26245
26644
|
modelCatalog: bedrockModelCatalog
|
|
26246
26645
|
});
|
|
26247
|
-
|
|
26646
|
+
logger87.debug("Bedrock provider module registered");
|
|
26248
26647
|
const injectedProviders = this.options?.providerRegistry;
|
|
26249
26648
|
if (injectedProviders) {
|
|
26250
26649
|
this.providerRegistryService = new ProviderRegistryService(
|
|
26251
26650
|
void 0,
|
|
26252
26651
|
injectedProviders
|
|
26253
26652
|
);
|
|
26254
|
-
|
|
26653
|
+
logger87.debug(
|
|
26255
26654
|
`Provider registry initialized from injected providers (${injectedProviders.length})`
|
|
26256
26655
|
);
|
|
26257
26656
|
} else {
|
|
26258
26657
|
const registryPath = resolveProviderRegistryPath();
|
|
26259
26658
|
this.providerRegistryService = new ProviderRegistryService(registryPath);
|
|
26260
26659
|
if (registryPath) {
|
|
26261
|
-
|
|
26660
|
+
logger87.info(`Provider registry path: ${registryPath}`);
|
|
26262
26661
|
} else {
|
|
26263
|
-
|
|
26662
|
+
logger87.warn(
|
|
26264
26663
|
"No providers registry found (set LOBU_PROVIDER_REGISTRY_PATH or run from a dir with config/providers.json) \u2014 config-driven providers will be unavailable"
|
|
26265
26664
|
);
|
|
26266
26665
|
}
|
|
@@ -26272,7 +26671,7 @@ var init_core_services = __esm({
|
|
|
26272
26671
|
() => this.providerConfigResolver ? this.providerConfigResolver.getProviderConfigs() : Promise.resolve({})
|
|
26273
26672
|
);
|
|
26274
26673
|
const configProviders = await this.providerConfigResolver.getProviderConfigs();
|
|
26275
|
-
|
|
26674
|
+
logger87.info(
|
|
26276
26675
|
`Provider registry loaded ${Object.keys(configProviders).length} config-driven provider(s)`
|
|
26277
26676
|
);
|
|
26278
26677
|
const registeredIds = new Set(
|
|
@@ -26280,7 +26679,7 @@ var init_core_services = __esm({
|
|
|
26280
26679
|
);
|
|
26281
26680
|
for (const [id, entry] of Object.entries(configProviders)) {
|
|
26282
26681
|
if (registeredIds.has(id)) {
|
|
26283
|
-
|
|
26682
|
+
logger87.info(
|
|
26284
26683
|
`Skipping config-driven provider "${id}" \u2014 already registered`
|
|
26285
26684
|
);
|
|
26286
26685
|
continue;
|
|
@@ -26302,7 +26701,7 @@ var init_core_services = __esm({
|
|
|
26302
26701
|
});
|
|
26303
26702
|
moduleRegistry.register(module);
|
|
26304
26703
|
registeredIds.add(id);
|
|
26305
|
-
|
|
26704
|
+
logger87.debug(
|
|
26306
26705
|
`Registered config-driven provider: ${id} (system key: ${module.hasSystemKey() ? "available" : "not available"})`
|
|
26307
26706
|
);
|
|
26308
26707
|
}
|
|
@@ -26311,7 +26710,7 @@ var init_core_services = __esm({
|
|
|
26311
26710
|
this.authProfilesManager,
|
|
26312
26711
|
this.declaredAgentRegistry
|
|
26313
26712
|
);
|
|
26314
|
-
|
|
26713
|
+
logger87.debug("Provider catalog service initialized");
|
|
26315
26714
|
if (this.secretProxy) {
|
|
26316
26715
|
this.secretProxy.setAuthProfilesManager(this.authProfilesManager);
|
|
26317
26716
|
this.secretProxy.setAgentOrgResolver(
|
|
@@ -26334,7 +26733,7 @@ var init_core_services = __esm({
|
|
|
26334
26733
|
}
|
|
26335
26734
|
return testEnv[mod.getCredentialEnvVarName()] || void 0;
|
|
26336
26735
|
});
|
|
26337
|
-
|
|
26736
|
+
logger87.debug("Provider upstreams registered with secret proxy");
|
|
26338
26737
|
}
|
|
26339
26738
|
}
|
|
26340
26739
|
// ============================================================================
|
|
@@ -26366,7 +26765,7 @@ var init_core_services = __esm({
|
|
|
26366
26765
|
this.mcpConfigService,
|
|
26367
26766
|
this.agentSettingsStore
|
|
26368
26767
|
);
|
|
26369
|
-
|
|
26768
|
+
logger87.debug("Instruction service initialized");
|
|
26370
26769
|
if (!this.secretStore) {
|
|
26371
26770
|
throw new Error("Secret store must be initialized before MCP proxy");
|
|
26372
26771
|
}
|
|
@@ -26417,7 +26816,7 @@ var init_core_services = __esm({
|
|
|
26417
26816
|
payload.message
|
|
26418
26817
|
);
|
|
26419
26818
|
};
|
|
26420
|
-
|
|
26819
|
+
logger87.debug("MCP proxy initialized");
|
|
26421
26820
|
this.workerGateway = new WorkerGateway(
|
|
26422
26821
|
this.queue,
|
|
26423
26822
|
this.config.mcp.publicGatewayUrl,
|
|
@@ -26428,10 +26827,10 @@ var init_core_services = __esm({
|
|
|
26428
26827
|
this.agentSettingsStore,
|
|
26429
26828
|
this.secretStore
|
|
26430
26829
|
);
|
|
26431
|
-
|
|
26830
|
+
logger87.debug("Worker gateway initialized");
|
|
26432
26831
|
await moduleRegistry.registerAvailableModules();
|
|
26433
26832
|
await moduleRegistry.initAll();
|
|
26434
|
-
|
|
26833
|
+
logger87.debug("Modules initialized");
|
|
26435
26834
|
}
|
|
26436
26835
|
// ============================================================================
|
|
26437
26836
|
// 7. Command Registry Initialization
|
|
@@ -26446,7 +26845,7 @@ var init_core_services = __esm({
|
|
|
26446
26845
|
registerBuiltInCommands(this.commandRegistry, {
|
|
26447
26846
|
agentSettingsStore: this.agentSettingsStore
|
|
26448
26847
|
});
|
|
26449
|
-
|
|
26848
|
+
logger87.debug("Command registry initialized with built-in commands");
|
|
26450
26849
|
}
|
|
26451
26850
|
// ============================================================================
|
|
26452
26851
|
// SDK-Embedded Helpers
|
|
@@ -26474,18 +26873,18 @@ var init_core_services = __esm({
|
|
|
26474
26873
|
// Shutdown
|
|
26475
26874
|
// ============================================================================
|
|
26476
26875
|
async shutdown() {
|
|
26477
|
-
|
|
26876
|
+
logger87.info("Shutting down core services...");
|
|
26478
26877
|
if (this.queueProducer) {
|
|
26479
26878
|
await this.queueProducer.stop();
|
|
26480
26879
|
}
|
|
26481
26880
|
if (this.workerGateway) {
|
|
26482
26881
|
this.workerGateway.shutdown();
|
|
26483
|
-
|
|
26882
|
+
logger87.info("Worker gateway shutdown complete");
|
|
26484
26883
|
}
|
|
26485
26884
|
if (this.queue) {
|
|
26486
26885
|
await this.queue.stop();
|
|
26487
26886
|
}
|
|
26488
|
-
|
|
26887
|
+
logger87.info("Core services shutdown complete");
|
|
26489
26888
|
}
|
|
26490
26889
|
// ============================================================================
|
|
26491
26890
|
// Service Accessors (implements ICoreServices interface)
|
|
@@ -26637,7 +27036,7 @@ var init_core_services = __esm({
|
|
|
26637
27036
|
});
|
|
26638
27037
|
|
|
26639
27038
|
// src/gateway/gateway-main.ts
|
|
26640
|
-
var
|
|
27039
|
+
var logger88, Gateway;
|
|
26641
27040
|
var init_gateway_main = __esm({
|
|
26642
27041
|
"src/gateway/gateway-main.ts"() {
|
|
26643
27042
|
"use strict";
|
|
@@ -26645,7 +27044,7 @@ var init_gateway_main = __esm({
|
|
|
26645
27044
|
init_platform2();
|
|
26646
27045
|
init_unified_thread_consumer();
|
|
26647
27046
|
init_core_services();
|
|
26648
|
-
|
|
27047
|
+
logger88 = createLogger("gateway");
|
|
26649
27048
|
Gateway = class {
|
|
26650
27049
|
constructor(config, options) {
|
|
26651
27050
|
this.config = config;
|
|
@@ -26682,7 +27081,7 @@ var init_gateway_main = __esm({
|
|
|
26682
27081
|
this.coreServices.getInstructionService()?.registerPlatformProvider(platform.name, provider2);
|
|
26683
27082
|
}
|
|
26684
27083
|
}
|
|
26685
|
-
|
|
27084
|
+
logger88.debug(`Platform registered: ${platform.name}`);
|
|
26686
27085
|
return this;
|
|
26687
27086
|
}
|
|
26688
27087
|
/**
|
|
@@ -26693,10 +27092,10 @@ var init_gateway_main = __esm({
|
|
|
26693
27092
|
* 4. Start all platforms
|
|
26694
27093
|
*/
|
|
26695
27094
|
async start() {
|
|
26696
|
-
|
|
27095
|
+
logger88.debug("Starting gateway...");
|
|
26697
27096
|
await this.coreServices.initialize();
|
|
26698
27097
|
for (const [name, platform] of this.platforms) {
|
|
26699
|
-
|
|
27098
|
+
logger88.debug(`Initializing platform: ${name}`);
|
|
26700
27099
|
await platform.initialize(this.coreServices);
|
|
26701
27100
|
}
|
|
26702
27101
|
const instructionService = this.coreServices.getInstructionService();
|
|
@@ -26711,7 +27110,7 @@ var init_gateway_main = __esm({
|
|
|
26711
27110
|
}
|
|
26712
27111
|
}
|
|
26713
27112
|
for (const [name, platform] of this.platforms) {
|
|
26714
|
-
|
|
27113
|
+
logger88.debug(`Starting platform: ${name}`);
|
|
26715
27114
|
await platform.start();
|
|
26716
27115
|
}
|
|
26717
27116
|
this.unifiedConsumer = new UnifiedThreadResponseConsumer(
|
|
@@ -26729,26 +27128,26 @@ var init_gateway_main = __esm({
|
|
|
26729
27128
|
* 3. Shutdown core services
|
|
26730
27129
|
*/
|
|
26731
27130
|
async stop() {
|
|
26732
|
-
|
|
27131
|
+
logger88.info("Stopping gateway...");
|
|
26733
27132
|
if (this.unifiedConsumer) {
|
|
26734
|
-
|
|
27133
|
+
logger88.info("Stopping unified thread response consumer");
|
|
26735
27134
|
try {
|
|
26736
27135
|
await this.unifiedConsumer.stop();
|
|
26737
27136
|
} catch (error) {
|
|
26738
|
-
|
|
27137
|
+
logger88.error("Failed to stop unified consumer:", error);
|
|
26739
27138
|
}
|
|
26740
27139
|
}
|
|
26741
27140
|
for (const [name, platform] of this.platforms) {
|
|
26742
|
-
|
|
27141
|
+
logger88.info(`Stopping platform: ${name}`);
|
|
26743
27142
|
try {
|
|
26744
27143
|
await platform.stop();
|
|
26745
27144
|
} catch (error) {
|
|
26746
|
-
|
|
27145
|
+
logger88.error(`Failed to stop platform ${name}:`, error);
|
|
26747
27146
|
}
|
|
26748
27147
|
}
|
|
26749
27148
|
await this.coreServices.shutdown();
|
|
26750
27149
|
this.isRunning = false;
|
|
26751
|
-
|
|
27150
|
+
logger88.info("\u2705 Gateway stopped");
|
|
26752
27151
|
}
|
|
26753
27152
|
/**
|
|
26754
27153
|
* Get gateway status
|
|
@@ -26813,7 +27212,7 @@ function isSecretEnvVar(name, providerModules) {
|
|
|
26813
27212
|
const upper = name.toUpperCase();
|
|
26814
27213
|
return upper.includes("_KEY") || upper.includes("_TOKEN") || upper.includes("_SECRET") || upper.includes("_PASSWORD");
|
|
26815
27214
|
}
|
|
26816
|
-
var
|
|
27215
|
+
var logger89, SECRET_PLACEHOLDER_TTL_SECONDS, GRANT_SYNC_CACHE_MAX, BaseDeploymentManager;
|
|
26817
27216
|
var init_base_deployment_manager = __esm({
|
|
26818
27217
|
"src/gateway/orchestration/base-deployment-manager.ts"() {
|
|
26819
27218
|
"use strict";
|
|
@@ -26821,7 +27220,7 @@ var init_base_deployment_manager = __esm({
|
|
|
26821
27220
|
init_policy_store();
|
|
26822
27221
|
init_secret_proxy();
|
|
26823
27222
|
init_secrets();
|
|
26824
|
-
|
|
27223
|
+
logger89 = createLogger("orchestrator");
|
|
26825
27224
|
SECRET_PLACEHOLDER_TTL_SECONDS = (() => {
|
|
26826
27225
|
const raw = process.env.SECRET_PLACEHOLDER_TTL_MS;
|
|
26827
27226
|
if (raw) {
|
|
@@ -26930,7 +27329,7 @@ var init_base_deployment_manager = __esm({
|
|
|
26930
27329
|
};
|
|
26931
27330
|
const deploymentName = generateDeploymentName(deploymentIdentity);
|
|
26932
27331
|
const canonicalConversationKey = buildCanonicalConversationKey(deploymentIdentity);
|
|
26933
|
-
|
|
27332
|
+
logger89.info(
|
|
26934
27333
|
`Worker deployment - conversationId: ${conversationId}, canonicalKey: ${canonicalConversationKey}, deploymentName: ${deploymentName}`
|
|
26935
27334
|
);
|
|
26936
27335
|
try {
|
|
@@ -26943,14 +27342,14 @@ var init_base_deployment_manager = __esm({
|
|
|
26943
27342
|
await this.scaleDeployment(deploymentName, 1);
|
|
26944
27343
|
return;
|
|
26945
27344
|
} catch (scaleErr) {
|
|
26946
|
-
|
|
27345
|
+
logger89.warn(
|
|
26947
27346
|
`scaleDeployment(${deploymentName}, 1) failed (${scaleErr instanceof Error ? scaleErr.message : String(scaleErr)}); re-spawning`
|
|
26948
27347
|
);
|
|
26949
27348
|
}
|
|
26950
27349
|
}
|
|
26951
27350
|
const maxDeployments = this.config.worker.maxDeployments;
|
|
26952
27351
|
if (maxDeployments > 0 && deployments.length >= maxDeployments) {
|
|
26953
|
-
|
|
27352
|
+
logger89.warn(
|
|
26954
27353
|
`\u26A0\uFE0F Maximum deployments limit reached (${deployments.length}/${maxDeployments}). Running cleanup before creating new deployment.`
|
|
26955
27354
|
);
|
|
26956
27355
|
await this.reconcileDeployments();
|
|
@@ -27014,7 +27413,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27014
27413
|
const organizationId = messageData.organizationId;
|
|
27015
27414
|
if (!this.policyStore || !agentId || !organizationId) {
|
|
27016
27415
|
if (!organizationId && agentId) {
|
|
27017
|
-
|
|
27416
|
+
logger89.warn(
|
|
27018
27417
|
{ agentId, deploymentName },
|
|
27019
27418
|
"Skipping egress policy sync \u2014 message has no organizationId"
|
|
27020
27419
|
);
|
|
@@ -27029,11 +27428,11 @@ var init_base_deployment_manager = __esm({
|
|
|
27029
27428
|
if (bundle) {
|
|
27030
27429
|
this.policyStore.set(organizationId, agentId, bundle);
|
|
27031
27430
|
if (deploymentName) {
|
|
27032
|
-
|
|
27431
|
+
logger89.info(
|
|
27033
27432
|
`Synced egress judge policy for ${deploymentName}: ${bundle.judgedDomains.length} rule(s), ${Object.keys(bundle.judges).length} judge(s)`
|
|
27034
27433
|
);
|
|
27035
27434
|
} else {
|
|
27036
|
-
|
|
27435
|
+
logger89.debug("Synced egress judge policy", {
|
|
27037
27436
|
organizationId,
|
|
27038
27437
|
agentId,
|
|
27039
27438
|
rules: bundle.judgedDomains.length,
|
|
@@ -27056,7 +27455,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27056
27455
|
for (const domain of messageData.networkConfig.allowedDomains) {
|
|
27057
27456
|
await this.grantStore.grant(agentId, domain, null, void 0, orgId);
|
|
27058
27457
|
}
|
|
27059
|
-
|
|
27458
|
+
logger89.info(
|
|
27060
27459
|
`Synced network config domains as grants for ${deploymentName}: ${messageData.networkConfig.allowedDomains.join(", ")}`
|
|
27061
27460
|
);
|
|
27062
27461
|
}
|
|
@@ -27064,7 +27463,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27064
27463
|
for (const pattern of messageData.preApprovedTools) {
|
|
27065
27464
|
await this.grantStore.grant(agentId, pattern, null, void 0, orgId);
|
|
27066
27465
|
}
|
|
27067
|
-
|
|
27466
|
+
logger89.info(
|
|
27068
27467
|
`Synced pre-approved tool patterns as grants for ${deploymentName}: ${messageData.preApprovedTools.join(", ")}`
|
|
27069
27468
|
);
|
|
27070
27469
|
}
|
|
@@ -27078,7 +27477,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27078
27477
|
for (const domain of NIX_DOMAINS) {
|
|
27079
27478
|
await this.grantStore.grant(agentId, domain, null, void 0, orgId);
|
|
27080
27479
|
}
|
|
27081
|
-
|
|
27480
|
+
logger89.info(
|
|
27082
27481
|
`Added Nix cache domains as grants for ${deploymentName}: ${NIX_DOMAINS.join(", ")}`
|
|
27083
27482
|
);
|
|
27084
27483
|
}
|
|
@@ -27217,7 +27616,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27217
27616
|
if (flakeUrl) envVars.NIX_FLAKE_URL = flakeUrl;
|
|
27218
27617
|
if (packages && packages.length > 0)
|
|
27219
27618
|
envVars.NIX_PACKAGES = packages.join(",");
|
|
27220
|
-
|
|
27619
|
+
logger89.debug(
|
|
27221
27620
|
`Nix config for ${deploymentName}: flakeUrl=${flakeUrl || "none"}, packages=${packages?.length || 0}`
|
|
27222
27621
|
);
|
|
27223
27622
|
}
|
|
@@ -27280,12 +27679,12 @@ var init_base_deployment_manager = __esm({
|
|
|
27280
27679
|
envVars[key] = placeholder;
|
|
27281
27680
|
hasSecrets = true;
|
|
27282
27681
|
} catch (error) {
|
|
27283
|
-
|
|
27682
|
+
logger89.warn(`Failed to generate placeholder for ${key}:`, error);
|
|
27284
27683
|
}
|
|
27285
27684
|
}
|
|
27286
27685
|
}
|
|
27287
27686
|
if (hasSecrets) {
|
|
27288
|
-
|
|
27687
|
+
logger89.info(
|
|
27289
27688
|
`\u{1F510} Generated secret placeholders for ${deploymentName}, routing through proxy`
|
|
27290
27689
|
);
|
|
27291
27690
|
}
|
|
@@ -27348,7 +27747,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27348
27747
|
providerContext
|
|
27349
27748
|
);
|
|
27350
27749
|
} catch (error) {
|
|
27351
|
-
|
|
27750
|
+
logger89.warn("Failed to build module environment variables:", error);
|
|
27352
27751
|
}
|
|
27353
27752
|
}
|
|
27354
27753
|
if (this.config.worker.env) {
|
|
@@ -27383,7 +27782,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27383
27782
|
}
|
|
27384
27783
|
}
|
|
27385
27784
|
if (primaryProvider) {
|
|
27386
|
-
|
|
27785
|
+
logger89.info(
|
|
27387
27786
|
{
|
|
27388
27787
|
agentId,
|
|
27389
27788
|
primaryProviderId: primaryProvider.providerId,
|
|
@@ -27429,7 +27828,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27429
27828
|
for (const domain of NPM_DOMAINS) {
|
|
27430
27829
|
await this.grantStore.grant(agentId, domain, null, void 0, orgId);
|
|
27431
27830
|
}
|
|
27432
|
-
|
|
27831
|
+
logger89.info(
|
|
27433
27832
|
`Added npm registry domains as grants for ${deploymentName}: ${NPM_DOMAINS.join(", ")}`
|
|
27434
27833
|
);
|
|
27435
27834
|
}
|
|
@@ -27448,12 +27847,12 @@ var init_base_deployment_manager = __esm({
|
|
|
27448
27847
|
`deployments/${deploymentName}/`
|
|
27449
27848
|
);
|
|
27450
27849
|
if (cleared > 0) {
|
|
27451
|
-
|
|
27850
|
+
logger89.debug(
|
|
27452
27851
|
`Cleared ${cleared} deployment secret(s) for ${deploymentName}`
|
|
27453
27852
|
);
|
|
27454
27853
|
}
|
|
27455
27854
|
} catch (error) {
|
|
27456
|
-
|
|
27855
|
+
logger89.warn(
|
|
27457
27856
|
`Failed to clear deployment secrets for ${deploymentName}:`,
|
|
27458
27857
|
error
|
|
27459
27858
|
);
|
|
@@ -27476,7 +27875,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27476
27875
|
async reconcileDeployments() {
|
|
27477
27876
|
try {
|
|
27478
27877
|
const maxDeployments = this.config.worker.maxDeployments;
|
|
27479
|
-
|
|
27878
|
+
logger89.debug("Running deployment cleanup...");
|
|
27480
27879
|
const activeDeployments = await this.listDeployments();
|
|
27481
27880
|
if (activeDeployments.length === 0) {
|
|
27482
27881
|
return;
|
|
@@ -27517,7 +27916,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27517
27916
|
if (results[j]?.status === "fulfilled") {
|
|
27518
27917
|
processedCount++;
|
|
27519
27918
|
} else {
|
|
27520
|
-
|
|
27919
|
+
logger89.error(
|
|
27521
27920
|
`\u274C Failed to delete deployment ${batch[j]}:`,
|
|
27522
27921
|
results[j].reason
|
|
27523
27922
|
);
|
|
@@ -27533,7 +27932,7 @@ var init_base_deployment_manager = __esm({
|
|
|
27533
27932
|
if (results[j]?.status === "fulfilled") {
|
|
27534
27933
|
processedCount++;
|
|
27535
27934
|
} else {
|
|
27536
|
-
|
|
27935
|
+
logger89.error(
|
|
27537
27936
|
`\u274C Failed to scale down deployment ${batch[j]}:`,
|
|
27538
27937
|
results[j].reason
|
|
27539
27938
|
);
|
|
@@ -27541,12 +27940,12 @@ var init_base_deployment_manager = __esm({
|
|
|
27541
27940
|
}
|
|
27542
27941
|
}
|
|
27543
27942
|
if (processedCount > 0) {
|
|
27544
|
-
|
|
27943
|
+
logger89.info(
|
|
27545
27944
|
`\u2705 Cleanup completed: processed ${processedCount} deployment(s)`
|
|
27546
27945
|
);
|
|
27547
27946
|
}
|
|
27548
27947
|
} catch (error) {
|
|
27549
|
-
|
|
27948
|
+
logger89.error(
|
|
27550
27949
|
"Error during deployment reconciliation:",
|
|
27551
27950
|
error instanceof Error ? error.message : String(error)
|
|
27552
27951
|
);
|
|
@@ -27668,6 +28067,111 @@ function makeUnitName(deploymentName) {
|
|
|
27668
28067
|
const tag = Math.random().toString(36).slice(2, 8);
|
|
27669
28068
|
return `lobu-worker-${safe}-${tag}`;
|
|
27670
28069
|
}
|
|
28070
|
+
function getDefaultMaxReservedLocks() {
|
|
28071
|
+
const poolMax = Number.parseInt(process.env.DB_POOL_MAX || "20", 10);
|
|
28072
|
+
if (!Number.isFinite(poolMax) || poolMax <= 0) {
|
|
28073
|
+
return Math.max(1, 20 - POOL_HEADROOM);
|
|
28074
|
+
}
|
|
28075
|
+
return Math.max(1, poolMax - POOL_HEADROOM);
|
|
28076
|
+
}
|
|
28077
|
+
function getMaxReservedLocks() {
|
|
28078
|
+
const raw = process.env.LOBU_MAX_RESERVED_LOCKS;
|
|
28079
|
+
if (!raw) return getDefaultMaxReservedLocks();
|
|
28080
|
+
const n = Number.parseInt(raw, 10);
|
|
28081
|
+
if (!Number.isFinite(n) || n < 0) return getDefaultMaxReservedLocks();
|
|
28082
|
+
return n;
|
|
28083
|
+
}
|
|
28084
|
+
function getReservedLockCount() {
|
|
28085
|
+
return reservedLockCount;
|
|
28086
|
+
}
|
|
28087
|
+
async function acquireConversationLock(organizationId, agentId, conversationId) {
|
|
28088
|
+
if (process.env.LOBU_DISABLE_PREPARE === "1") {
|
|
28089
|
+
return { release: async () => {
|
|
28090
|
+
} };
|
|
28091
|
+
}
|
|
28092
|
+
const max = getMaxReservedLocks();
|
|
28093
|
+
if (reservedLockCount >= max) {
|
|
28094
|
+
logger90.warn(
|
|
28095
|
+
`Reserved-lock cap reached (${reservedLockCount}/${max}); deferring spawn for ${organizationId}/${agentId}/${conversationId}`
|
|
28096
|
+
);
|
|
28097
|
+
return null;
|
|
28098
|
+
}
|
|
28099
|
+
reservedLockCount += 1;
|
|
28100
|
+
if (!warnedNearCap && reservedLockCount >= Math.ceil(max * 0.8)) {
|
|
28101
|
+
logger90.warn(
|
|
28102
|
+
`Reserved-lock count near cap: ${reservedLockCount}/${max}. Tune via LOBU_MAX_RESERVED_LOCKS or scale pods.`
|
|
28103
|
+
);
|
|
28104
|
+
warnedNearCap = true;
|
|
28105
|
+
}
|
|
28106
|
+
let decremented = false;
|
|
28107
|
+
const decrementOnce = () => {
|
|
28108
|
+
if (decremented) return;
|
|
28109
|
+
decremented = true;
|
|
28110
|
+
reservedLockCount = Math.max(0, reservedLockCount - 1);
|
|
28111
|
+
if (warnedNearCap && reservedLockCount < Math.ceil(max * 0.8)) {
|
|
28112
|
+
warnedNearCap = false;
|
|
28113
|
+
}
|
|
28114
|
+
};
|
|
28115
|
+
const sql = getDb();
|
|
28116
|
+
let reserved;
|
|
28117
|
+
try {
|
|
28118
|
+
reserved = await sql.reserve();
|
|
28119
|
+
} catch (err) {
|
|
28120
|
+
decrementOnce();
|
|
28121
|
+
throw err;
|
|
28122
|
+
}
|
|
28123
|
+
const key2 = hashConvKey2(organizationId, agentId, conversationId);
|
|
28124
|
+
try {
|
|
28125
|
+
const rows = await reserved`SELECT pg_try_advisory_lock(${CONV_LOCK_KEY1}, ${key2}) AS acquired`;
|
|
28126
|
+
if (!rows[0]?.acquired) {
|
|
28127
|
+
reserved.release();
|
|
28128
|
+
decrementOnce();
|
|
28129
|
+
return null;
|
|
28130
|
+
}
|
|
28131
|
+
} catch (err) {
|
|
28132
|
+
reserved.release();
|
|
28133
|
+
decrementOnce();
|
|
28134
|
+
throw err;
|
|
28135
|
+
}
|
|
28136
|
+
return {
|
|
28137
|
+
async release() {
|
|
28138
|
+
const MAX_ATTEMPTS = 3;
|
|
28139
|
+
const BACKOFF_MS = 100;
|
|
28140
|
+
let lastErr = null;
|
|
28141
|
+
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
|
|
28142
|
+
try {
|
|
28143
|
+
await reserved`SELECT pg_advisory_unlock(${CONV_LOCK_KEY1}, ${key2})`;
|
|
28144
|
+
lastErr = null;
|
|
28145
|
+
break;
|
|
28146
|
+
} catch (err) {
|
|
28147
|
+
lastErr = err;
|
|
28148
|
+
if (attempt < MAX_ATTEMPTS) {
|
|
28149
|
+
await new Promise((r) => setTimeout(r, BACKOFF_MS * attempt));
|
|
28150
|
+
}
|
|
28151
|
+
}
|
|
28152
|
+
}
|
|
28153
|
+
if (lastErr) {
|
|
28154
|
+
logger90.error(
|
|
28155
|
+
`Failed to release advisory lock after ${MAX_ATTEMPTS} attempts for ${organizationId}/${agentId}/${conversationId}: ${lastErr instanceof Error ? lastErr.message : String(lastErr)}`
|
|
28156
|
+
);
|
|
28157
|
+
}
|
|
28158
|
+
try {
|
|
28159
|
+
reserved.release();
|
|
28160
|
+
} catch {
|
|
28161
|
+
}
|
|
28162
|
+
decrementOnce();
|
|
28163
|
+
}
|
|
28164
|
+
};
|
|
28165
|
+
}
|
|
28166
|
+
function hashConvKey2(organizationId, agentId, conversationId) {
|
|
28167
|
+
const input = `${organizationId}:${agentId}:${conversationId}`;
|
|
28168
|
+
let hash = 2166136261;
|
|
28169
|
+
for (let i = 0; i < input.length; i++) {
|
|
28170
|
+
hash ^= input.charCodeAt(i);
|
|
28171
|
+
hash = hash + ((hash << 1) + (hash << 4) + (hash << 7) + (hash << 8) + (hash << 24)) | 0;
|
|
28172
|
+
}
|
|
28173
|
+
return hash;
|
|
28174
|
+
}
|
|
27671
28175
|
function buildEmbeddedWorkerPath(binPathEntries, existingPath) {
|
|
27672
28176
|
const segments = (existingPath || "").split(":").filter(Boolean);
|
|
27673
28177
|
for (const candidate of [...binPathEntries ?? []].reverse()) {
|
|
@@ -27724,15 +28228,20 @@ function nixPackageAttrRef(pkg) {
|
|
|
27724
28228
|
}
|
|
27725
28229
|
return `pkgs.${namespace}.${leaf}`;
|
|
27726
28230
|
}
|
|
27727
|
-
var
|
|
28231
|
+
var logger90, KILL_TIMEOUT_MS, cachedSystemdRun, CONV_LOCK_KEY1, POOL_HEADROOM, reservedLockCount, warnedNearCap, NIX_PACKAGE_NAMESPACES, NIX_LEAF_RE, NIX_ATTR_LEAF_RE, EmbeddedDeploymentManager;
|
|
27728
28232
|
var init_embedded_deployment = __esm({
|
|
27729
28233
|
"src/gateway/orchestration/impl/embedded-deployment.ts"() {
|
|
27730
28234
|
"use strict";
|
|
27731
28235
|
init_src();
|
|
28236
|
+
init_client();
|
|
27732
28237
|
init_base_deployment_manager();
|
|
27733
28238
|
init_deployment_utils();
|
|
27734
|
-
|
|
28239
|
+
logger90 = createLogger("orchestrator");
|
|
27735
28240
|
KILL_TIMEOUT_MS = 5e3;
|
|
28241
|
+
CONV_LOCK_KEY1 = 1819239029;
|
|
28242
|
+
POOL_HEADROOM = 5;
|
|
28243
|
+
reservedLockCount = 0;
|
|
28244
|
+
warnedNearCap = false;
|
|
27736
28245
|
NIX_PACKAGE_NAMESPACES = /* @__PURE__ */ new Set([
|
|
27737
28246
|
"python3Packages",
|
|
27738
28247
|
"python311Packages",
|
|
@@ -27784,7 +28293,7 @@ var init_embedded_deployment = __esm({
|
|
|
27784
28293
|
`Worker entry point not found: ${entryPoint}`
|
|
27785
28294
|
);
|
|
27786
28295
|
}
|
|
27787
|
-
|
|
28296
|
+
logger90.debug(`Worker entry point verified: ${entryPoint}`);
|
|
27788
28297
|
}
|
|
27789
28298
|
async spawnDeployment(deploymentName, username, userId, messageData) {
|
|
27790
28299
|
if (this.workers.has(deploymentName)) {
|
|
@@ -27805,111 +28314,164 @@ var init_embedded_deployment = __esm({
|
|
|
27805
28314
|
}
|
|
27806
28315
|
const workspaceDir = path5.resolve(`workspaces/${agentId}`);
|
|
27807
28316
|
fs2.mkdirSync(workspaceDir, { recursive: true, mode: 448 });
|
|
27808
|
-
const
|
|
27809
|
-
|
|
27810
|
-
|
|
27811
|
-
|
|
27812
|
-
|
|
27813
|
-
|
|
27814
|
-
|
|
27815
|
-
|
|
27816
|
-
|
|
27817
|
-
|
|
27818
|
-
|
|
27819
|
-
|
|
27820
|
-
|
|
27821
|
-
|
|
27822
|
-
|
|
27823
|
-
|
|
27824
|
-
|
|
27825
|
-
|
|
27826
|
-
|
|
27827
|
-
|
|
27828
|
-
|
|
27829
|
-
|
|
27830
|
-
|
|
27831
|
-
|
|
27832
|
-
|
|
27833
|
-
|
|
27834
|
-
|
|
27835
|
-
|
|
27836
|
-
|
|
27837
|
-
|
|
27838
|
-
|
|
27839
|
-
|
|
27840
|
-
|
|
27841
|
-
|
|
27842
|
-
|
|
28317
|
+
const snapshotModeEnabled = process.env.LOBU_SESSION_STORE !== "file";
|
|
28318
|
+
const conversationId = typeof messageData?.conversationId === "string" ? messageData.conversationId : null;
|
|
28319
|
+
const organizationId = typeof messageData?.organizationId === "string" ? messageData.organizationId : null;
|
|
28320
|
+
let convLock = null;
|
|
28321
|
+
if (snapshotModeEnabled && organizationId && conversationId) {
|
|
28322
|
+
try {
|
|
28323
|
+
convLock = await acquireConversationLock(
|
|
28324
|
+
organizationId,
|
|
28325
|
+
agentId,
|
|
28326
|
+
conversationId
|
|
28327
|
+
);
|
|
28328
|
+
} catch (err) {
|
|
28329
|
+
logger90.error(
|
|
28330
|
+
`Failed to acquire conversation lock: ${err instanceof Error ? err.message : String(err)}`
|
|
28331
|
+
);
|
|
28332
|
+
throw new OrchestratorError(
|
|
28333
|
+
"DEPLOYMENT_CREATE_FAILED" /* DEPLOYMENT_CREATE_FAILED */,
|
|
28334
|
+
"Could not acquire per-conversation lock"
|
|
28335
|
+
);
|
|
28336
|
+
}
|
|
28337
|
+
if (!convLock) {
|
|
28338
|
+
logger90.info(
|
|
28339
|
+
`Conversation lock busy for ${organizationId}/${agentId}/${conversationId}; deferring spawn`
|
|
28340
|
+
);
|
|
28341
|
+
throw new OrchestratorError(
|
|
28342
|
+
"DEPLOYMENT_CREATE_FAILED" /* DEPLOYMENT_CREATE_FAILED */,
|
|
28343
|
+
"Conversation lock busy on another pod"
|
|
28344
|
+
);
|
|
28345
|
+
}
|
|
28346
|
+
}
|
|
28347
|
+
let child;
|
|
28348
|
+
let commonEnvVars;
|
|
28349
|
+
try {
|
|
28350
|
+
commonEnvVars = await this.generateEnvironmentVariables(
|
|
28351
|
+
username,
|
|
28352
|
+
userId,
|
|
28353
|
+
deploymentName,
|
|
28354
|
+
messageData,
|
|
28355
|
+
true
|
|
27843
28356
|
);
|
|
27844
|
-
|
|
27845
|
-
|
|
27846
|
-
|
|
27847
|
-
|
|
27848
|
-
|
|
27849
|
-
|
|
27850
|
-
|
|
27851
|
-
const innerCommand = command;
|
|
27852
|
-
const innerArgs = spawnArgs;
|
|
27853
|
-
command = systemdRun;
|
|
27854
|
-
spawnArgs = [
|
|
27855
|
-
...buildSystemdRunArgs({ unitName, workspaceDir }),
|
|
27856
|
-
"--",
|
|
27857
|
-
innerCommand,
|
|
27858
|
-
...innerArgs
|
|
27859
|
-
];
|
|
27860
|
-
logger89.info(
|
|
27861
|
-
`Spawning embedded worker ${deploymentName} under systemd-run scope ${unitName}`
|
|
28357
|
+
commonEnvVars.WORKSPACE_DIR = workspaceDir;
|
|
28358
|
+
if (!snapshotModeEnabled) {
|
|
28359
|
+
commonEnvVars.LOBU_SESSION_STORE = "file";
|
|
28360
|
+
}
|
|
28361
|
+
const embeddedPath = buildEmbeddedWorkerPath(
|
|
28362
|
+
this.config.worker.binPathEntries,
|
|
28363
|
+
commonEnvVars.PATH || process.env.PATH
|
|
27862
28364
|
);
|
|
28365
|
+
if (embeddedPath) {
|
|
28366
|
+
commonEnvVars.PATH = embeddedPath;
|
|
28367
|
+
}
|
|
28368
|
+
const allowedDomains = messageData?.networkConfig?.allowedDomains ?? [];
|
|
28369
|
+
if (allowedDomains.length > 0) {
|
|
28370
|
+
commonEnvVars.JUST_BASH_ALLOWED_DOMAINS = JSON.stringify(allowedDomains);
|
|
28371
|
+
}
|
|
28372
|
+
const nixPackages = messageData?.nixConfig?.packages ?? [];
|
|
28373
|
+
const workerEntryPoint = this.getWorkerEntryPoint();
|
|
28374
|
+
const workerInvocation = buildWorkerInvocation(workerEntryPoint);
|
|
28375
|
+
let command;
|
|
28376
|
+
let spawnArgs;
|
|
28377
|
+
if (nixPackages.length > 0) {
|
|
28378
|
+
const packageRefs = nixPackages.map(nixPackageAttrRef);
|
|
28379
|
+
command = "nix-shell";
|
|
28380
|
+
spawnArgs = [
|
|
28381
|
+
"-E",
|
|
28382
|
+
`let pkgs = import <nixpkgs> {}; in pkgs.mkShell { buildInputs = [ ${packageRefs.join(" ")} ]; }`,
|
|
28383
|
+
"--run",
|
|
28384
|
+
buildShellCommand(workerInvocation.command, workerInvocation.args)
|
|
28385
|
+
];
|
|
28386
|
+
logger90.info(
|
|
28387
|
+
`Spawning embedded worker ${deploymentName} with nix packages: ${nixPackages.join(", ")}`
|
|
28388
|
+
);
|
|
28389
|
+
} else {
|
|
28390
|
+
command = workerInvocation.command;
|
|
28391
|
+
spawnArgs = workerInvocation.args;
|
|
28392
|
+
}
|
|
28393
|
+
const systemdRun = locateSystemdRun();
|
|
28394
|
+
if (systemdRun) {
|
|
28395
|
+
const unitName = makeUnitName(deploymentName);
|
|
28396
|
+
const innerCommand = command;
|
|
28397
|
+
const innerArgs = spawnArgs;
|
|
28398
|
+
command = systemdRun;
|
|
28399
|
+
spawnArgs = [
|
|
28400
|
+
...buildSystemdRunArgs({ unitName, workspaceDir }),
|
|
28401
|
+
"--",
|
|
28402
|
+
innerCommand,
|
|
28403
|
+
...innerArgs
|
|
28404
|
+
];
|
|
28405
|
+
logger90.info(
|
|
28406
|
+
`Spawning embedded worker ${deploymentName} under systemd-run scope ${unitName}`
|
|
28407
|
+
);
|
|
28408
|
+
}
|
|
28409
|
+
child = spawn(command, spawnArgs, {
|
|
28410
|
+
// Workers must not inherit gateway-only secrets or telemetry settings
|
|
28411
|
+
// (DATABASE_URL, SENTRY_DSN, OAuth secrets, etc.). Everything a worker
|
|
28412
|
+
// needs is assembled explicitly above, with optional operator-provided
|
|
28413
|
+
// values forwarded only via WORKER_ENV_*.
|
|
28414
|
+
env: commonEnvVars,
|
|
28415
|
+
cwd: workspaceDir,
|
|
28416
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
28417
|
+
});
|
|
28418
|
+
} catch (err) {
|
|
28419
|
+
if (convLock) {
|
|
28420
|
+
void convLock.release();
|
|
28421
|
+
}
|
|
28422
|
+
throw err;
|
|
27863
28423
|
}
|
|
27864
|
-
|
|
27865
|
-
|
|
27866
|
-
|
|
27867
|
-
|
|
27868
|
-
|
|
27869
|
-
|
|
27870
|
-
|
|
27871
|
-
|
|
27872
|
-
});
|
|
28424
|
+
let lockReleased = false;
|
|
28425
|
+
const releaseLockOnce = async () => {
|
|
28426
|
+
if (lockReleased) return;
|
|
28427
|
+
lockReleased = true;
|
|
28428
|
+
if (convLock) {
|
|
28429
|
+
await convLock.release();
|
|
28430
|
+
}
|
|
28431
|
+
};
|
|
27873
28432
|
child.once("error", (err) => {
|
|
27874
|
-
|
|
28433
|
+
logger90.error(
|
|
27875
28434
|
`Embedded worker ${deploymentName} spawn error: ${err.message}`
|
|
27876
28435
|
);
|
|
27877
28436
|
this.workers.delete(deploymentName);
|
|
28437
|
+
releaseLockOnce();
|
|
27878
28438
|
});
|
|
27879
28439
|
child.stdout?.on("data", (data) => {
|
|
27880
28440
|
for (const line of data.toString().trimEnd().split("\n")) {
|
|
27881
|
-
|
|
28441
|
+
logger90.info({ worker: deploymentName }, line);
|
|
27882
28442
|
}
|
|
27883
28443
|
});
|
|
27884
28444
|
child.stderr?.on("data", (data) => {
|
|
27885
28445
|
for (const line of data.toString().trimEnd().split("\n")) {
|
|
27886
|
-
|
|
28446
|
+
logger90.warn({ worker: deploymentName }, line);
|
|
27887
28447
|
}
|
|
27888
28448
|
});
|
|
27889
28449
|
child.once("exit", (code, signal) => {
|
|
27890
|
-
|
|
27891
|
-
|
|
27892
|
-
this.workers.delete(deploymentName);
|
|
27893
|
-
}
|
|
28450
|
+
this.workers.delete(deploymentName);
|
|
28451
|
+
releaseLockOnce();
|
|
27894
28452
|
if (signal) {
|
|
27895
|
-
|
|
28453
|
+
logger90.info(
|
|
27896
28454
|
`Embedded worker ${deploymentName} exited with signal ${signal}`
|
|
27897
28455
|
);
|
|
27898
28456
|
} else if (code !== 0) {
|
|
27899
|
-
|
|
28457
|
+
logger90.error(
|
|
27900
28458
|
`Embedded worker ${deploymentName} exited with code ${code}`
|
|
27901
28459
|
);
|
|
27902
28460
|
} else {
|
|
27903
|
-
|
|
28461
|
+
logger90.info(`Embedded worker ${deploymentName} exited cleanly`);
|
|
27904
28462
|
}
|
|
27905
28463
|
});
|
|
27906
28464
|
this.workers.set(deploymentName, {
|
|
27907
28465
|
process: child,
|
|
27908
28466
|
env: commonEnvVars,
|
|
27909
28467
|
lastActivity: /* @__PURE__ */ new Date(),
|
|
27910
|
-
workspaceDir
|
|
28468
|
+
workspaceDir,
|
|
28469
|
+
// Expose the idempotent release on the entry for introspection /
|
|
28470
|
+
// tests. The exit handler is the authoritative release site;
|
|
28471
|
+
// killWorker no longer touches this field.
|
|
28472
|
+
...convLock ? { releaseConvLock: releaseLockOnce } : {}
|
|
27911
28473
|
});
|
|
27912
|
-
|
|
28474
|
+
logger90.info(
|
|
27913
28475
|
`Started embedded worker subprocess for ${deploymentName} (pid=${child.pid})`
|
|
27914
28476
|
);
|
|
27915
28477
|
}
|
|
@@ -27917,7 +28479,7 @@ var init_embedded_deployment = __esm({
|
|
|
27917
28479
|
const entry = this.workers.get(deploymentName);
|
|
27918
28480
|
if (replicas === 0 && entry) {
|
|
27919
28481
|
await this.killWorker(entry, deploymentName);
|
|
27920
|
-
|
|
28482
|
+
logger90.info(`Stopped embedded worker ${deploymentName}`);
|
|
27921
28483
|
} else if (replicas === 1 && !entry) {
|
|
27922
28484
|
throw new Error(
|
|
27923
28485
|
`Embedded worker ${deploymentName} is not running \u2014 must re-create`
|
|
@@ -27928,7 +28490,7 @@ var init_embedded_deployment = __esm({
|
|
|
27928
28490
|
const entry = this.workers.get(deploymentName);
|
|
27929
28491
|
if (entry) {
|
|
27930
28492
|
await this.killWorker(entry, deploymentName);
|
|
27931
|
-
|
|
28493
|
+
logger90.info(`Stopped embedded worker: ${deploymentName}`);
|
|
27932
28494
|
}
|
|
27933
28495
|
}
|
|
27934
28496
|
async listDeployments() {
|
|
@@ -27956,7 +28518,14 @@ var init_embedded_deployment = __esm({
|
|
|
27956
28518
|
entry.lastActivity = /* @__PURE__ */ new Date();
|
|
27957
28519
|
}
|
|
27958
28520
|
}
|
|
27959
|
-
/** Send SIGTERM, then SIGKILL after timeout. Resolves on child exit.
|
|
28521
|
+
/** Send SIGTERM, then SIGKILL after timeout. Resolves on child exit.
|
|
28522
|
+
*
|
|
28523
|
+
* Does NOT release the conversation lock — the child's exit handler is
|
|
28524
|
+
* the authoritative release site, and the release call there is
|
|
28525
|
+
* idempotent. Releasing here before `await exited` (as a prior version
|
|
28526
|
+
* did) lets a sibling pod claim the conversation while this worker is
|
|
28527
|
+
* still flushing its cleanup() snapshot. Codex P1#3 on PR #865.
|
|
28528
|
+
*/
|
|
27960
28529
|
async killWorker(entry, deploymentName) {
|
|
27961
28530
|
const child = entry.process;
|
|
27962
28531
|
this.workers.delete(deploymentName);
|
|
@@ -27967,7 +28536,7 @@ var init_embedded_deployment = __esm({
|
|
|
27967
28536
|
child.kill("SIGTERM");
|
|
27968
28537
|
const killTimer = setTimeout(() => {
|
|
27969
28538
|
if (child.exitCode === null && child.signalCode === null) {
|
|
27970
|
-
|
|
28539
|
+
logger90.warn(
|
|
27971
28540
|
`Embedded worker ${deploymentName} did not exit after SIGTERM, sending SIGKILL`
|
|
27972
28541
|
);
|
|
27973
28542
|
child.kill("SIGKILL");
|
|
@@ -27985,14 +28554,14 @@ var init_embedded_deployment = __esm({
|
|
|
27985
28554
|
|
|
27986
28555
|
// src/gateway/orchestration/message-consumer.ts
|
|
27987
28556
|
import * as Sentry4 from "@sentry/node";
|
|
27988
|
-
var
|
|
28557
|
+
var logger91, MessageConsumer;
|
|
27989
28558
|
var init_message_consumer = __esm({
|
|
27990
28559
|
"src/gateway/orchestration/message-consumer.ts"() {
|
|
27991
28560
|
"use strict";
|
|
27992
28561
|
init_src();
|
|
27993
28562
|
init_queue();
|
|
27994
28563
|
init_base_deployment_manager();
|
|
27995
|
-
|
|
28564
|
+
logger91 = createLogger("orchestrator");
|
|
27996
28565
|
MessageConsumer = class {
|
|
27997
28566
|
queue;
|
|
27998
28567
|
deploymentManager;
|
|
@@ -28016,7 +28585,7 @@ var init_message_consumer = __esm({
|
|
|
28016
28585
|
await this.queue.start();
|
|
28017
28586
|
this.isRunning = true;
|
|
28018
28587
|
await this.queue.createQueue("messages");
|
|
28019
|
-
|
|
28588
|
+
logger91.debug("Created/verified messages queue");
|
|
28020
28589
|
await this.queue.work(
|
|
28021
28590
|
"messages",
|
|
28022
28591
|
async (job) => {
|
|
@@ -28034,7 +28603,7 @@ var init_message_consumer = __esm({
|
|
|
28034
28603
|
);
|
|
28035
28604
|
}
|
|
28036
28605
|
);
|
|
28037
|
-
|
|
28606
|
+
logger91.debug("Queue consumer started");
|
|
28038
28607
|
} catch (error) {
|
|
28039
28608
|
throw new OrchestratorError(
|
|
28040
28609
|
"QUEUE_JOB_PROCESSING_FAILED" /* QUEUE_JOB_PROCESSING_FAILED */,
|
|
@@ -28064,7 +28633,7 @@ var init_message_consumer = __esm({
|
|
|
28064
28633
|
"lobu.conversation_id": data?.conversationId || "unknown"
|
|
28065
28634
|
});
|
|
28066
28635
|
const childTraceparent = getTraceparent(queueSpan) || traceparent;
|
|
28067
|
-
|
|
28636
|
+
logger91.info(
|
|
28068
28637
|
{
|
|
28069
28638
|
traceparent,
|
|
28070
28639
|
traceId,
|
|
@@ -28075,6 +28644,10 @@ var init_message_consumer = __esm({
|
|
|
28075
28644
|
"Processing job with trace context"
|
|
28076
28645
|
);
|
|
28077
28646
|
try {
|
|
28647
|
+
const parsedRunId = Number(jobId);
|
|
28648
|
+
if (Number.isFinite(parsedRunId) && parsedRunId > 0) {
|
|
28649
|
+
data.runId = parsedRunId;
|
|
28650
|
+
}
|
|
28078
28651
|
const effectiveConversationId = data.conversationId;
|
|
28079
28652
|
if (!effectiveConversationId) {
|
|
28080
28653
|
throw new OrchestratorError(
|
|
@@ -28095,7 +28668,22 @@ var init_message_consumer = __esm({
|
|
|
28095
28668
|
channelId: data.channelId,
|
|
28096
28669
|
conversationId: effectiveConversationId
|
|
28097
28670
|
});
|
|
28098
|
-
|
|
28671
|
+
if (data.runId !== void 0) {
|
|
28672
|
+
data.runJobToken = generateWorkerToken(
|
|
28673
|
+
data.userId,
|
|
28674
|
+
effectiveConversationId,
|
|
28675
|
+
deploymentName,
|
|
28676
|
+
{
|
|
28677
|
+
channelId: data.channelId,
|
|
28678
|
+
teamId: data.teamId,
|
|
28679
|
+
agentId: data.agentId,
|
|
28680
|
+
organizationId: data.organizationId,
|
|
28681
|
+
platform: data.platform,
|
|
28682
|
+
runId: data.runId
|
|
28683
|
+
}
|
|
28684
|
+
);
|
|
28685
|
+
}
|
|
28686
|
+
logger91.info(
|
|
28099
28687
|
`Conversation routing - effectiveConversationId: ${effectiveConversationId}, canonicalKey: ${canonicalConversationKey}, deploymentName: ${deploymentName}`
|
|
28100
28688
|
);
|
|
28101
28689
|
await Sentry4.startSpan(
|
|
@@ -28112,7 +28700,7 @@ var init_message_consumer = __esm({
|
|
|
28112
28700
|
await this.sendToWorkerQueue(data, deploymentName);
|
|
28113
28701
|
}
|
|
28114
28702
|
);
|
|
28115
|
-
|
|
28703
|
+
logger91.info(
|
|
28116
28704
|
{ traceId, traceparent: childTraceparent, deploymentName },
|
|
28117
28705
|
"Enqueued message to thread queue"
|
|
28118
28706
|
);
|
|
@@ -28132,7 +28720,7 @@ var init_message_consumer = __esm({
|
|
|
28132
28720
|
},
|
|
28133
28721
|
level: "error"
|
|
28134
28722
|
});
|
|
28135
|
-
|
|
28723
|
+
logger91.error(
|
|
28136
28724
|
{
|
|
28137
28725
|
traceId,
|
|
28138
28726
|
error: bgError instanceof Error ? bgError.message : String(bgError),
|
|
@@ -28145,13 +28733,13 @@ var init_message_consumer = __esm({
|
|
|
28145
28733
|
);
|
|
28146
28734
|
this.trackFailedDeployment(deploymentName, data, bgError).catch(
|
|
28147
28735
|
(trackError) => {
|
|
28148
|
-
|
|
28736
|
+
logger91.error("Failed to track deployment failure:", trackError);
|
|
28149
28737
|
}
|
|
28150
28738
|
);
|
|
28151
28739
|
});
|
|
28152
28740
|
queueSpan?.setStatus({ code: SpanStatusCode.OK });
|
|
28153
28741
|
queueSpan?.end();
|
|
28154
|
-
|
|
28742
|
+
logger91.info({ traceId, jobId }, "Message job queued successfully");
|
|
28155
28743
|
} catch (error) {
|
|
28156
28744
|
queueSpan?.setStatus({
|
|
28157
28745
|
code: SpanStatusCode.ERROR,
|
|
@@ -28159,7 +28747,7 @@ var init_message_consumer = __esm({
|
|
|
28159
28747
|
});
|
|
28160
28748
|
queueSpan?.end();
|
|
28161
28749
|
Sentry4.captureException(error);
|
|
28162
|
-
|
|
28750
|
+
logger91.error({ traceId, jobId, error }, "Message job failed");
|
|
28163
28751
|
throw new OrchestratorError(
|
|
28164
28752
|
"QUEUE_JOB_PROCESSING_FAILED" /* QUEUE_JOB_PROCESSING_FAILED */,
|
|
28165
28753
|
`Failed to process message job: ${error instanceof Error ? error.message : String(error)}`,
|
|
@@ -28191,11 +28779,11 @@ var init_message_consumer = __esm({
|
|
|
28191
28779
|
true
|
|
28192
28780
|
);
|
|
28193
28781
|
}
|
|
28194
|
-
|
|
28782
|
+
logger91.info(
|
|
28195
28783
|
`\u2705 Sent message to thread queue ${threadQueueName} for conversation ${data.conversationId}, jobId: ${jobId}`
|
|
28196
28784
|
);
|
|
28197
28785
|
} catch (error) {
|
|
28198
|
-
|
|
28786
|
+
logger91.error(`\u274C [ERROR] sendToWorkerQueue failed:`, error);
|
|
28199
28787
|
throw new OrchestratorError(
|
|
28200
28788
|
"QUEUE_JOB_PROCESSING_FAILED" /* QUEUE_JOB_PROCESSING_FAILED */,
|
|
28201
28789
|
`Failed to send message to thread queue: ${error instanceof Error ? error.message : String(error)}`,
|
|
@@ -28241,7 +28829,7 @@ var init_message_consumer = __esm({
|
|
|
28241
28829
|
if (isNewThread) {
|
|
28242
28830
|
const acquired = this.acquireDeploymentLock(deploymentName);
|
|
28243
28831
|
if (!acquired) {
|
|
28244
|
-
|
|
28832
|
+
logger91.info(
|
|
28245
28833
|
{ traceId, deploymentName },
|
|
28246
28834
|
"Another handler is creating this deployment, waiting"
|
|
28247
28835
|
);
|
|
@@ -28249,7 +28837,7 @@ var init_message_consumer = __esm({
|
|
|
28249
28837
|
const rechecked = await this.deploymentManager.listDeployments();
|
|
28250
28838
|
if (rechecked.some((d) => d.deploymentName === deploymentName)) {
|
|
28251
28839
|
await this.deploymentManager.scaleDeployment(deploymentName, 1);
|
|
28252
|
-
|
|
28840
|
+
logger91.info(
|
|
28253
28841
|
{ traceId, deploymentName },
|
|
28254
28842
|
"Deployment created by other handler, scaled up"
|
|
28255
28843
|
);
|
|
@@ -28263,7 +28851,7 @@ var init_message_consumer = __esm({
|
|
|
28263
28851
|
try {
|
|
28264
28852
|
const recheckAfterLock = await this.deploymentManager.listDeployments();
|
|
28265
28853
|
if (recheckAfterLock.some((d) => d.deploymentName === deploymentName)) {
|
|
28266
|
-
|
|
28854
|
+
logger91.info(
|
|
28267
28855
|
{ traceId, deploymentName },
|
|
28268
28856
|
"Deployment already created by another handler after lock acquired"
|
|
28269
28857
|
);
|
|
@@ -28273,7 +28861,7 @@ var init_message_consumer = __esm({
|
|
|
28273
28861
|
);
|
|
28274
28862
|
return;
|
|
28275
28863
|
}
|
|
28276
|
-
|
|
28864
|
+
logger91.info(
|
|
28277
28865
|
{ traceId, traceparent, conversationId, deploymentName },
|
|
28278
28866
|
"New thread - creating deployment"
|
|
28279
28867
|
);
|
|
@@ -28283,24 +28871,24 @@ var init_message_consumer = __esm({
|
|
|
28283
28871
|
dataWithTrace,
|
|
28284
28872
|
recheckAfterLock
|
|
28285
28873
|
);
|
|
28286
|
-
|
|
28874
|
+
logger91.info({ traceId, deploymentName }, "Created deployment");
|
|
28287
28875
|
} finally {
|
|
28288
28876
|
this.releaseDeploymentLock(deploymentName);
|
|
28289
28877
|
}
|
|
28290
28878
|
} else {
|
|
28291
|
-
|
|
28879
|
+
logger91.info(
|
|
28292
28880
|
{ traceId, conversationId, deploymentName },
|
|
28293
28881
|
"Existing thread - ensuring worker exists"
|
|
28294
28882
|
);
|
|
28295
28883
|
await this.deploymentManager.syncNetworkConfigGrants(dataWithTrace);
|
|
28296
28884
|
try {
|
|
28297
28885
|
await this.deploymentManager.scaleDeployment(deploymentName, 1);
|
|
28298
|
-
|
|
28886
|
+
logger91.info(
|
|
28299
28887
|
{ traceId, deploymentName },
|
|
28300
28888
|
"Scaled existing worker to 1"
|
|
28301
28889
|
);
|
|
28302
28890
|
} catch {
|
|
28303
|
-
|
|
28891
|
+
logger91.info(
|
|
28304
28892
|
{ traceId, conversationId, deploymentName },
|
|
28305
28893
|
"Worker doesn't exist, creating it"
|
|
28306
28894
|
);
|
|
@@ -28309,11 +28897,11 @@ var init_message_consumer = __esm({
|
|
|
28309
28897
|
conversationId,
|
|
28310
28898
|
dataWithTrace
|
|
28311
28899
|
);
|
|
28312
|
-
|
|
28900
|
+
logger91.info({ traceId, deploymentName }, "Created worker");
|
|
28313
28901
|
}
|
|
28314
28902
|
}
|
|
28315
28903
|
await this.deploymentManager.updateDeploymentActivity(deploymentName);
|
|
28316
|
-
|
|
28904
|
+
logger91.info({ traceId, deploymentName }, "Worker is ready");
|
|
28317
28905
|
},
|
|
28318
28906
|
{
|
|
28319
28907
|
maxRetries: 3,
|
|
@@ -28321,7 +28909,7 @@ var init_message_consumer = __esm({
|
|
|
28321
28909
|
strategy: "linear",
|
|
28322
28910
|
jitter: true,
|
|
28323
28911
|
onRetry: (attempt, error) => {
|
|
28324
|
-
|
|
28912
|
+
logger91.warn(
|
|
28325
28913
|
{ traceId, deploymentName, attempt, maxAttempts: 3 },
|
|
28326
28914
|
`Retry attempt failed: ${error.message}`
|
|
28327
28915
|
);
|
|
@@ -28335,7 +28923,7 @@ var init_message_consumer = __esm({
|
|
|
28335
28923
|
*/
|
|
28336
28924
|
async trackFailedDeployment(deploymentName, data, error) {
|
|
28337
28925
|
try {
|
|
28338
|
-
|
|
28926
|
+
logger91.error(
|
|
28339
28927
|
{
|
|
28340
28928
|
deploymentName,
|
|
28341
28929
|
userId: data.userId,
|
|
@@ -28361,10 +28949,10 @@ var init_message_consumer = __esm({
|
|
|
28361
28949
|
processedMessageIds: [data.messageId]
|
|
28362
28950
|
});
|
|
28363
28951
|
} catch (notifyError) {
|
|
28364
|
-
|
|
28952
|
+
logger91.error("Failed to send error notification to user:", notifyError);
|
|
28365
28953
|
}
|
|
28366
28954
|
} catch (trackError) {
|
|
28367
|
-
|
|
28955
|
+
logger91.error("Failed to track deployment failure:", trackError);
|
|
28368
28956
|
}
|
|
28369
28957
|
}
|
|
28370
28958
|
/**
|
|
@@ -28378,7 +28966,7 @@ var init_message_consumer = __esm({
|
|
|
28378
28966
|
isRunning: this.isRunning
|
|
28379
28967
|
};
|
|
28380
28968
|
} catch (error) {
|
|
28381
|
-
|
|
28969
|
+
logger91.error("Failed to get queue stats:", error);
|
|
28382
28970
|
return {
|
|
28383
28971
|
isRunning: this.isRunning,
|
|
28384
28972
|
error: error instanceof Error ? error.message : String(error)
|
|
@@ -28390,7 +28978,7 @@ var init_message_consumer = __esm({
|
|
|
28390
28978
|
});
|
|
28391
28979
|
|
|
28392
28980
|
// src/gateway/orchestration/index.ts
|
|
28393
|
-
var
|
|
28981
|
+
var logger92, Orchestrator;
|
|
28394
28982
|
var init_orchestration = __esm({
|
|
28395
28983
|
"src/gateway/orchestration/index.ts"() {
|
|
28396
28984
|
"use strict";
|
|
@@ -28402,7 +28990,7 @@ var init_orchestration = __esm({
|
|
|
28402
28990
|
init_deployment_utils();
|
|
28403
28991
|
init_embedded_deployment();
|
|
28404
28992
|
init_message_consumer();
|
|
28405
|
-
|
|
28993
|
+
logger92 = createLogger("orchestrator");
|
|
28406
28994
|
Orchestrator = class {
|
|
28407
28995
|
config;
|
|
28408
28996
|
deploymentManager;
|
|
@@ -28442,7 +29030,7 @@ var init_orchestration = __esm({
|
|
|
28442
29030
|
}
|
|
28443
29031
|
const providerModules = getModelProviderModules();
|
|
28444
29032
|
this.deploymentManager.setProviderModules(providerModules);
|
|
28445
|
-
|
|
29033
|
+
logger92.debug(
|
|
28446
29034
|
`Provider modules injected into orchestrator (${providerModules.length})`
|
|
28447
29035
|
);
|
|
28448
29036
|
}
|
|
@@ -28455,9 +29043,9 @@ var init_orchestration = __esm({
|
|
|
28455
29043
|
await this.queueConsumer.start();
|
|
28456
29044
|
this.setupIdleCleanup();
|
|
28457
29045
|
this.isRunning = true;
|
|
28458
|
-
|
|
29046
|
+
logger92.debug("Orchestrator started");
|
|
28459
29047
|
} catch (error) {
|
|
28460
|
-
|
|
29048
|
+
logger92.error("\u274C Failed to start orchestrator:", error);
|
|
28461
29049
|
throw error;
|
|
28462
29050
|
}
|
|
28463
29051
|
}
|
|
@@ -28475,9 +29063,9 @@ var init_orchestration = __esm({
|
|
|
28475
29063
|
this.initialReconcileTimer = void 0;
|
|
28476
29064
|
}
|
|
28477
29065
|
if (this.activeReconciliation) {
|
|
28478
|
-
|
|
29066
|
+
logger92.info("Waiting for in-flight reconciliation to complete...");
|
|
28479
29067
|
const safeReconciliation = this.activeReconciliation.catch((error) => {
|
|
28480
|
-
|
|
29068
|
+
logger92.error(
|
|
28481
29069
|
"In-flight reconciliation failed during shutdown:",
|
|
28482
29070
|
error
|
|
28483
29071
|
);
|
|
@@ -28497,11 +29085,11 @@ var init_orchestration = __esm({
|
|
|
28497
29085
|
)
|
|
28498
29086
|
);
|
|
28499
29087
|
} catch (error) {
|
|
28500
|
-
|
|
29088
|
+
logger92.error("Error draining workers during shutdown:", error);
|
|
28501
29089
|
}
|
|
28502
|
-
|
|
29090
|
+
logger92.info("\u2705 Orchestrator stopped");
|
|
28503
29091
|
} catch (error) {
|
|
28504
|
-
|
|
29092
|
+
logger92.error("\u274C Error stopping orchestrator:", error);
|
|
28505
29093
|
}
|
|
28506
29094
|
}
|
|
28507
29095
|
setupIdleCleanup() {
|
|
@@ -28509,7 +29097,7 @@ var init_orchestration = __esm({
|
|
|
28509
29097
|
this.initialReconcileTimer = void 0;
|
|
28510
29098
|
if (this.shuttingDown) return;
|
|
28511
29099
|
const p = this.deploymentManager.reconcileDeployments().catch((error) => {
|
|
28512
|
-
|
|
29100
|
+
logger92.error("\u274C Initial deployment reconciliation failed:", error);
|
|
28513
29101
|
});
|
|
28514
29102
|
this.activeReconciliation = p;
|
|
28515
29103
|
p.finally(() => {
|
|
@@ -28519,7 +29107,7 @@ var init_orchestration = __esm({
|
|
|
28519
29107
|
this.cleanupInterval = setInterval(async () => {
|
|
28520
29108
|
if (this.shuttingDown) return;
|
|
28521
29109
|
if (this.isReconciling) {
|
|
28522
|
-
|
|
29110
|
+
logger92.debug(
|
|
28523
29111
|
"Skipping reconciliation interval: previous run still in progress"
|
|
28524
29112
|
);
|
|
28525
29113
|
return;
|
|
@@ -28530,7 +29118,7 @@ var init_orchestration = __esm({
|
|
|
28530
29118
|
this.activeReconciliation = p;
|
|
28531
29119
|
await p;
|
|
28532
29120
|
} catch (error) {
|
|
28533
|
-
|
|
29121
|
+
logger92.error(
|
|
28534
29122
|
"Error during deployment reconciliation:",
|
|
28535
29123
|
error instanceof Error ? error.message : String(error)
|
|
28536
29124
|
);
|
|
@@ -28569,20 +29157,20 @@ var init_orchestration = __esm({
|
|
|
28569
29157
|
function loadAllowedDomains() {
|
|
28570
29158
|
const allowedDomains = process.env.WORKER_ALLOWED_DOMAINS;
|
|
28571
29159
|
if (!allowedDomains) {
|
|
28572
|
-
|
|
29160
|
+
logger93.warn(
|
|
28573
29161
|
"\u26A0\uFE0F WORKER_ALLOWED_DOMAINS not set - workers will have NO internet access (complete isolation)"
|
|
28574
29162
|
);
|
|
28575
29163
|
return [];
|
|
28576
29164
|
}
|
|
28577
29165
|
const trimmed = allowedDomains.trim();
|
|
28578
29166
|
if (trimmed === "*") {
|
|
28579
|
-
|
|
29167
|
+
logger93.debug("WORKER_ALLOWED_DOMAINS=* - unrestricted internet access");
|
|
28580
29168
|
return ["*"];
|
|
28581
29169
|
}
|
|
28582
29170
|
const parsed = normalizeDomainPatterns(
|
|
28583
29171
|
trimmed.split(",").map((d) => d.trim()).filter((d) => d.length > 0)
|
|
28584
29172
|
) ?? [];
|
|
28585
|
-
|
|
29173
|
+
logger93.debug(
|
|
28586
29174
|
`Loaded ${parsed.length} allowed domains from WORKER_ALLOWED_DOMAINS`
|
|
28587
29175
|
);
|
|
28588
29176
|
return parsed;
|
|
@@ -28596,17 +29184,17 @@ function loadDisallowedDomains() {
|
|
|
28596
29184
|
const parsed = normalizeDomainPatterns(
|
|
28597
29185
|
disallowedDomains.split(",").map((d) => d.trim()).filter((d) => d.length > 0)
|
|
28598
29186
|
) ?? [];
|
|
28599
|
-
|
|
29187
|
+
logger93.debug(
|
|
28600
29188
|
`Loaded ${parsed.length} disallowed domains from WORKER_DISALLOWED_DOMAINS`
|
|
28601
29189
|
);
|
|
28602
29190
|
return parsed;
|
|
28603
29191
|
}
|
|
28604
|
-
var
|
|
29192
|
+
var logger93;
|
|
28605
29193
|
var init_network_allowlist = __esm({
|
|
28606
29194
|
"src/gateway/config/network-allowlist.ts"() {
|
|
28607
29195
|
"use strict";
|
|
28608
29196
|
init_src();
|
|
28609
|
-
|
|
29197
|
+
logger93 = createLogger("network-allowlist");
|
|
28610
29198
|
}
|
|
28611
29199
|
});
|
|
28612
29200
|
|
|
@@ -28640,7 +29228,7 @@ var init_policy_composer = __esm({
|
|
|
28640
29228
|
});
|
|
28641
29229
|
|
|
28642
29230
|
// src/gateway/proxy/egress-judge/judge.ts
|
|
28643
|
-
var
|
|
29231
|
+
var logger94;
|
|
28644
29232
|
var init_judge = __esm({
|
|
28645
29233
|
"src/gateway/proxy/egress-judge/judge.ts"() {
|
|
28646
29234
|
"use strict";
|
|
@@ -28649,7 +29237,7 @@ var init_judge = __esm({
|
|
|
28649
29237
|
init_cache();
|
|
28650
29238
|
init_circuit_breaker();
|
|
28651
29239
|
init_policy_composer();
|
|
28652
|
-
|
|
29240
|
+
logger94 = createLogger("egress-judge");
|
|
28653
29241
|
}
|
|
28654
29242
|
});
|
|
28655
29243
|
|
|
@@ -28686,7 +29274,7 @@ async function checkDomainAccess(hostname, agentId, organizationId, requestConte
|
|
|
28686
29274
|
organizationId
|
|
28687
29275
|
);
|
|
28688
29276
|
if (denied) {
|
|
28689
|
-
|
|
29277
|
+
logger95.debug(`Domain ${hostname} denied via grant (agent: ${agentId})`);
|
|
28690
29278
|
return { allowed: false, source: "grant" };
|
|
28691
29279
|
}
|
|
28692
29280
|
}
|
|
@@ -28699,7 +29287,7 @@ async function checkDomainAccess(hostname, agentId, organizationId, requestConte
|
|
|
28699
29287
|
organizationId
|
|
28700
29288
|
);
|
|
28701
29289
|
if (granted) {
|
|
28702
|
-
|
|
29290
|
+
logger95.debug(`Domain ${hostname} allowed via grant (agent: ${agentId})`);
|
|
28703
29291
|
return { allowed: true, source: "grant" };
|
|
28704
29292
|
}
|
|
28705
29293
|
}
|
|
@@ -28895,13 +29483,13 @@ function validateProxyAuth(req) {
|
|
|
28895
29483
|
}
|
|
28896
29484
|
const tokenData = verifyWorkerToken(creds.token);
|
|
28897
29485
|
if (!tokenData) {
|
|
28898
|
-
|
|
29486
|
+
logger95.warn(
|
|
28899
29487
|
`Proxy auth failed: invalid token (claimed deployment: ${creds.deploymentName})`
|
|
28900
29488
|
);
|
|
28901
29489
|
return null;
|
|
28902
29490
|
}
|
|
28903
29491
|
if (tokenData.jti && getRevokedTokenStore().isRevokedCached(tokenData.jti)) {
|
|
28904
|
-
|
|
29492
|
+
logger95.warn(
|
|
28905
29493
|
`Proxy auth failed: revoked jti (claimed deployment: ${creds.deploymentName})`
|
|
28906
29494
|
);
|
|
28907
29495
|
return null;
|
|
@@ -28911,7 +29499,7 @@ function validateProxyAuth(req) {
|
|
|
28911
29499
|
Buffer.from(creds.deploymentName)
|
|
28912
29500
|
);
|
|
28913
29501
|
if (!deploymentMatch) {
|
|
28914
|
-
|
|
29502
|
+
logger95.warn(
|
|
28915
29503
|
`Proxy auth failed: deployment mismatch (claimed: ${creds.deploymentName}, token: ${tokenData.deploymentName})`
|
|
28916
29504
|
);
|
|
28917
29505
|
return null;
|
|
@@ -28953,7 +29541,7 @@ function logAccessDecision(method, hostname, deploymentName, agentId, decision)
|
|
|
28953
29541
|
if (decision.allowed && decision.source === "global") {
|
|
28954
29542
|
return;
|
|
28955
29543
|
}
|
|
28956
|
-
|
|
29544
|
+
logger95.info("egress-decision", {
|
|
28957
29545
|
method,
|
|
28958
29546
|
hostname,
|
|
28959
29547
|
deploymentName,
|
|
@@ -28981,7 +29569,7 @@ async function handleConnect(req, clientSocket, head) {
|
|
|
28981
29569
|
const url = req.url || "";
|
|
28982
29570
|
const hostname = extractConnectHostname(url);
|
|
28983
29571
|
if (!hostname) {
|
|
28984
|
-
|
|
29572
|
+
logger95.warn(`Invalid CONNECT request: ${url}`);
|
|
28985
29573
|
clientSocket.write("HTTP/1.1 400 Bad Request\r\n\r\n");
|
|
28986
29574
|
clientSocket.end();
|
|
28987
29575
|
return;
|
|
@@ -28989,9 +29577,9 @@ async function handleConnect(req, clientSocket, head) {
|
|
|
28989
29577
|
let targetSocket = null;
|
|
28990
29578
|
clientSocket.on("error", (err) => {
|
|
28991
29579
|
if (err.code === "ECONNRESET") {
|
|
28992
|
-
|
|
29580
|
+
logger95.debug(`Client disconnected for ${hostname} (ECONNRESET)`);
|
|
28993
29581
|
} else {
|
|
28994
|
-
|
|
29582
|
+
logger95.debug(`Client connection error for ${hostname}: ${err.message}`);
|
|
28995
29583
|
}
|
|
28996
29584
|
try {
|
|
28997
29585
|
targetSocket?.end();
|
|
@@ -29006,7 +29594,7 @@ async function handleConnect(req, clientSocket, head) {
|
|
|
29006
29594
|
});
|
|
29007
29595
|
const auth = validateProxyAuth(req);
|
|
29008
29596
|
if (!auth) {
|
|
29009
|
-
|
|
29597
|
+
logger95.warn(`Proxy auth required for CONNECT to ${hostname}`);
|
|
29010
29598
|
try {
|
|
29011
29599
|
clientSocket.write(
|
|
29012
29600
|
'HTTP/1.1 407 Proxy Authentication Required\r\nProxy-Authenticate: Basic realm="lobu-proxy"\r\n\r\n'
|
|
@@ -29031,7 +29619,7 @@ async function handleConnect(req, clientSocket, head) {
|
|
|
29031
29619
|
);
|
|
29032
29620
|
if (!decision.allowed) {
|
|
29033
29621
|
const reason = decision.judge?.reason ?? `Domain not allowed: ${hostname}`;
|
|
29034
|
-
|
|
29622
|
+
logger95.warn(
|
|
29035
29623
|
`Blocked CONNECT to ${hostname} (deployment: ${deploymentName}) - ${reason}`
|
|
29036
29624
|
);
|
|
29037
29625
|
try {
|
|
@@ -29049,7 +29637,7 @@ Content-Type: text/plain\r
|
|
|
29049
29637
|
}
|
|
29050
29638
|
const targetResolution = await resolveAndValidateTarget(hostname);
|
|
29051
29639
|
if (!targetResolution.ok) {
|
|
29052
|
-
|
|
29640
|
+
logger95.warn(
|
|
29053
29641
|
`Blocked CONNECT to ${hostname} (deployment: ${deploymentName}) - ${targetResolution.reason}`
|
|
29054
29642
|
);
|
|
29055
29643
|
try {
|
|
@@ -29071,17 +29659,17 @@ ${targetResolution.clientMessage}\r
|
|
|
29071
29659
|
clientSocket.end();
|
|
29072
29660
|
return;
|
|
29073
29661
|
}
|
|
29074
|
-
|
|
29662
|
+
logger95.debug(`Allowing CONNECT to ${hostname} via ${resolvedIp}`);
|
|
29075
29663
|
const [host, portStr] = url.split(":");
|
|
29076
29664
|
const port = portStr ? Number.parseInt(portStr, 10) : 443;
|
|
29077
29665
|
if (!host) {
|
|
29078
|
-
|
|
29666
|
+
logger95.warn(`Invalid CONNECT host: ${url}`);
|
|
29079
29667
|
clientSocket.write("HTTP/1.1 400 Bad Request\r\n\r\n");
|
|
29080
29668
|
clientSocket.end();
|
|
29081
29669
|
return;
|
|
29082
29670
|
}
|
|
29083
29671
|
if (!Number.isInteger(port) || port < 1 || port > 65535) {
|
|
29084
|
-
|
|
29672
|
+
logger95.warn(`Invalid CONNECT port: ${url}`);
|
|
29085
29673
|
clientSocket.write("HTTP/1.1 400 Bad Request\r\n\r\n");
|
|
29086
29674
|
clientSocket.end();
|
|
29087
29675
|
return;
|
|
@@ -29094,7 +29682,7 @@ ${targetResolution.clientMessage}\r
|
|
|
29094
29682
|
});
|
|
29095
29683
|
targetSocket = tunnelSocket;
|
|
29096
29684
|
tunnelSocket.on("error", (err) => {
|
|
29097
|
-
|
|
29685
|
+
logger95.debug(`Target connection error for ${hostname}: ${err.message}`);
|
|
29098
29686
|
try {
|
|
29099
29687
|
clientSocket.end();
|
|
29100
29688
|
} catch {
|
|
@@ -29125,7 +29713,7 @@ async function handleProxyRequest(req, res) {
|
|
|
29125
29713
|
const hostname = parsedUrl.hostname;
|
|
29126
29714
|
const auth = validateProxyAuth(req);
|
|
29127
29715
|
if (!auth) {
|
|
29128
|
-
|
|
29716
|
+
logger95.warn(`Proxy auth required for ${req.method} ${hostname}`);
|
|
29129
29717
|
res.writeHead(407, {
|
|
29130
29718
|
"Content-Type": "text/plain",
|
|
29131
29719
|
"Proxy-Authenticate": 'Basic realm="lobu-proxy"'
|
|
@@ -29147,7 +29735,7 @@ async function handleProxyRequest(req, res) {
|
|
|
29147
29735
|
);
|
|
29148
29736
|
if (!decision.allowed) {
|
|
29149
29737
|
const reason = decision.judge?.reason ?? `Domain not allowed: ${hostname}`;
|
|
29150
|
-
|
|
29738
|
+
logger95.warn(
|
|
29151
29739
|
`Blocked request to ${hostname} (deployment: ${deploymentName}) - ${reason}`
|
|
29152
29740
|
);
|
|
29153
29741
|
res.writeHead(403, escapeHeaderValue(reason), {
|
|
@@ -29161,7 +29749,7 @@ async function handleProxyRequest(req, res) {
|
|
|
29161
29749
|
}
|
|
29162
29750
|
const targetResolution = await resolveAndValidateTarget(hostname);
|
|
29163
29751
|
if (!targetResolution.ok) {
|
|
29164
|
-
|
|
29752
|
+
logger95.warn(
|
|
29165
29753
|
`Blocked request to ${hostname} (deployment: ${deploymentName}) - ${targetResolution.reason}`
|
|
29166
29754
|
);
|
|
29167
29755
|
res.writeHead(targetResolution.statusCode ?? 502, {
|
|
@@ -29177,7 +29765,7 @@ async function handleProxyRequest(req, res) {
|
|
|
29177
29765
|
res.end("Internal proxy error\n");
|
|
29178
29766
|
return;
|
|
29179
29767
|
}
|
|
29180
|
-
|
|
29768
|
+
logger95.debug(
|
|
29181
29769
|
`Proxying ${req.method} ${hostname}${parsedUrl.pathname} via ${resolvedIp}`
|
|
29182
29770
|
);
|
|
29183
29771
|
const forwardHeaders = { ...req.headers };
|
|
@@ -29194,7 +29782,7 @@ async function handleProxyRequest(req, res) {
|
|
|
29194
29782
|
proxyRes.pipe(res);
|
|
29195
29783
|
});
|
|
29196
29784
|
proxyReq.on("error", (err) => {
|
|
29197
|
-
|
|
29785
|
+
logger95.error(`Proxy request error for ${hostname}:`, err.message);
|
|
29198
29786
|
if (!res.headersSent) {
|
|
29199
29787
|
res.writeHead(502, { "Content-Type": "text/plain" });
|
|
29200
29788
|
res.end("Bad Gateway: Could not reach target server\n");
|
|
@@ -29209,7 +29797,7 @@ function startHttpProxy(port = 8118, host = "::") {
|
|
|
29209
29797
|
const global = getGlobalConfig();
|
|
29210
29798
|
const server = http.createServer((req, res) => {
|
|
29211
29799
|
handleProxyRequest(req, res).catch((err) => {
|
|
29212
|
-
|
|
29800
|
+
logger95.error("Error handling proxy request:", err);
|
|
29213
29801
|
if (!res.headersSent) {
|
|
29214
29802
|
res.writeHead(500, { "Content-Type": "text/plain" });
|
|
29215
29803
|
res.end("Internal proxy error\n");
|
|
@@ -29218,7 +29806,7 @@ function startHttpProxy(port = 8118, host = "::") {
|
|
|
29218
29806
|
});
|
|
29219
29807
|
server.on("connect", (req, clientSocket, head) => {
|
|
29220
29808
|
handleConnect(req, clientSocket, head).catch((err) => {
|
|
29221
|
-
|
|
29809
|
+
logger95.error("Error handling CONNECT:", err);
|
|
29222
29810
|
try {
|
|
29223
29811
|
clientSocket.write("HTTP/1.1 500 Internal Server Error\r\n\r\n");
|
|
29224
29812
|
clientSocket.end();
|
|
@@ -29227,13 +29815,13 @@ function startHttpProxy(port = 8118, host = "::") {
|
|
|
29227
29815
|
});
|
|
29228
29816
|
});
|
|
29229
29817
|
server.on("error", (err) => {
|
|
29230
|
-
|
|
29818
|
+
logger95.error("HTTP proxy server error:", err);
|
|
29231
29819
|
reject(err);
|
|
29232
29820
|
});
|
|
29233
29821
|
server.listen(port, host, () => {
|
|
29234
29822
|
server.removeAllListeners("error");
|
|
29235
29823
|
server.on("error", (err) => {
|
|
29236
|
-
|
|
29824
|
+
logger95.error("HTTP proxy server error:", err);
|
|
29237
29825
|
});
|
|
29238
29826
|
let mode;
|
|
29239
29827
|
if (isUnrestrictedMode(global.allowedDomains)) {
|
|
@@ -29243,7 +29831,7 @@ function startHttpProxy(port = 8118, host = "::") {
|
|
|
29243
29831
|
} else {
|
|
29244
29832
|
mode = "complete-isolation";
|
|
29245
29833
|
}
|
|
29246
|
-
|
|
29834
|
+
logger95.debug(
|
|
29247
29835
|
`HTTP proxy started on ${host}:${port} (mode=${mode}, allowed=${global.allowedDomains.length}, denied=${global.deniedDomains.length})`
|
|
29248
29836
|
);
|
|
29249
29837
|
resolve6(server);
|
|
@@ -29254,16 +29842,16 @@ function stopHttpProxy(server) {
|
|
|
29254
29842
|
return new Promise((resolve6, reject) => {
|
|
29255
29843
|
server.close((err) => {
|
|
29256
29844
|
if (err) {
|
|
29257
|
-
|
|
29845
|
+
logger95.error("Error stopping HTTP proxy:", err);
|
|
29258
29846
|
reject(err);
|
|
29259
29847
|
} else {
|
|
29260
|
-
|
|
29848
|
+
logger95.info("HTTP proxy stopped");
|
|
29261
29849
|
resolve6();
|
|
29262
29850
|
}
|
|
29263
29851
|
});
|
|
29264
29852
|
});
|
|
29265
29853
|
}
|
|
29266
|
-
var
|
|
29854
|
+
var logger95, blockedIpv4Ranges, blockedIpv6Ranges, blockedIpv4List, blockedIpv6List, globalConfig, proxyGrantStore, proxyPolicyStore, proxyEgressJudge, dnsLookupOverride;
|
|
29267
29855
|
var init_http_proxy = __esm({
|
|
29268
29856
|
"src/gateway/proxy/http-proxy.ts"() {
|
|
29269
29857
|
"use strict";
|
|
@@ -29271,7 +29859,7 @@ var init_http_proxy = __esm({
|
|
|
29271
29859
|
init_network_allowlist();
|
|
29272
29860
|
init_revoked_token_store();
|
|
29273
29861
|
init_judge();
|
|
29274
|
-
|
|
29862
|
+
logger95 = createLogger("http-proxy");
|
|
29275
29863
|
blockedIpv4Ranges = [
|
|
29276
29864
|
["0.0.0.0", 8],
|
|
29277
29865
|
["10.0.0.0", 8],
|
|
@@ -29320,26 +29908,26 @@ async function startFilteringProxy() {
|
|
|
29320
29908
|
const host = "127.0.0.1";
|
|
29321
29909
|
try {
|
|
29322
29910
|
proxyServer = await startHttpProxy(port, host);
|
|
29323
|
-
|
|
29911
|
+
logger96.debug(`HTTP proxy started on ${host}:${port}`);
|
|
29324
29912
|
} catch (error) {
|
|
29325
|
-
|
|
29913
|
+
logger96.error("Failed to start HTTP proxy:", error);
|
|
29326
29914
|
throw error;
|
|
29327
29915
|
}
|
|
29328
29916
|
}
|
|
29329
29917
|
async function stopFilteringProxy() {
|
|
29330
29918
|
if (proxyServer) {
|
|
29331
|
-
|
|
29919
|
+
logger96.info("Stopping HTTP proxy...");
|
|
29332
29920
|
await stopHttpProxy(proxyServer);
|
|
29333
29921
|
proxyServer = null;
|
|
29334
29922
|
}
|
|
29335
29923
|
}
|
|
29336
|
-
var
|
|
29924
|
+
var logger96, proxyServer;
|
|
29337
29925
|
var init_proxy_manager = __esm({
|
|
29338
29926
|
"src/gateway/proxy/proxy-manager.ts"() {
|
|
29339
29927
|
"use strict";
|
|
29340
29928
|
init_src();
|
|
29341
29929
|
init_http_proxy();
|
|
29342
|
-
|
|
29930
|
+
logger96 = createLogger("proxy-manager");
|
|
29343
29931
|
proxyServer = null;
|
|
29344
29932
|
process.on("SIGTERM", async () => {
|
|
29345
29933
|
await stopFilteringProxy();
|
|
@@ -43037,7 +43625,7 @@ async function createAuthRun(params) {
|
|
|
43037
43625
|
async function createConnectorOperationRun(params) {
|
|
43038
43626
|
const sql = getDb();
|
|
43039
43627
|
const approvalStatus = params.approvalMode === "queued" ? "pending" : "auto";
|
|
43040
|
-
const status = params.approvalMode === "
|
|
43628
|
+
const status = params.approvalMode === "inline" ? "running" : "pending";
|
|
43041
43629
|
const defRows = await sql`
|
|
43042
43630
|
SELECT version FROM connector_definitions
|
|
43043
43631
|
WHERE key = ${params.connectorKey}
|
|
@@ -47806,7 +48394,7 @@ import path11 from "node:path";
|
|
|
47806
48394
|
import { fileURLToPath as fileURLToPath8 } from "node:url";
|
|
47807
48395
|
import v8 from "node:v8";
|
|
47808
48396
|
import { getRequestListener } from "@hono/node-server";
|
|
47809
|
-
import { Hono as
|
|
48397
|
+
import { Hono as Hono29 } from "hono";
|
|
47810
48398
|
|
|
47811
48399
|
// src/dev-vite.ts
|
|
47812
48400
|
import { existsSync as existsSync7 } from "node:fs";
|
|
@@ -47817,7 +48405,7 @@ import { fileURLToPath as fileURLToPath7 } from "node:url";
|
|
|
47817
48405
|
import fs5 from "node:fs/promises";
|
|
47818
48406
|
import path9 from "node:path";
|
|
47819
48407
|
import { fileURLToPath as fileURLToPath6 } from "node:url";
|
|
47820
|
-
import { Hono as
|
|
48408
|
+
import { Hono as Hono28 } from "hono";
|
|
47821
48409
|
import { compress } from "hono/compress";
|
|
47822
48410
|
import { cors as cors2 } from "hono/cors";
|
|
47823
48411
|
import { pinoLogger } from "hono-pino";
|
|
@@ -47909,7 +48497,7 @@ async function mcpAuth(c, next) {
|
|
|
47909
48497
|
|
|
47910
48498
|
// src/auth/oauth/routes.ts
|
|
47911
48499
|
init_client();
|
|
47912
|
-
import { Hono as
|
|
48500
|
+
import { Hono as Hono22 } from "hono";
|
|
47913
48501
|
|
|
47914
48502
|
// src/utils/rate-limiter.ts
|
|
47915
48503
|
var RateLimiter = class {
|
|
@@ -48052,7 +48640,7 @@ init_provider();
|
|
|
48052
48640
|
init_scopes();
|
|
48053
48641
|
init_utils();
|
|
48054
48642
|
init_public_origin();
|
|
48055
|
-
var oauthRoutes = new
|
|
48643
|
+
var oauthRoutes = new Hono22();
|
|
48056
48644
|
async function parseRequestBody(c) {
|
|
48057
48645
|
const contentType = c.req.header("content-type") || "";
|
|
48058
48646
|
if (contentType.includes("application/x-www-form-urlencoded")) {
|
|
@@ -48654,10 +49242,10 @@ init_errors2();
|
|
|
48654
49242
|
init_base_url();
|
|
48655
49243
|
init_auth2();
|
|
48656
49244
|
import { createHmac } from "node:crypto";
|
|
48657
|
-
import { Hono as
|
|
49245
|
+
import { Hono as Hono23 } from "hono";
|
|
48658
49246
|
init_clients();
|
|
48659
49247
|
init_tokens();
|
|
48660
|
-
var credentialRoutes = new
|
|
49248
|
+
var credentialRoutes = new Hono23();
|
|
48661
49249
|
function getAuthenticatedUser(c) {
|
|
48662
49250
|
const user = c.get("user");
|
|
48663
49251
|
if (!user) {
|
|
@@ -48973,7 +49561,7 @@ init_run_statuses();
|
|
|
48973
49561
|
init_url_builder();
|
|
48974
49562
|
init_oauth_providers();
|
|
48975
49563
|
import { createHash as createHash12, randomBytes as randomBytes12 } from "node:crypto";
|
|
48976
|
-
import { Hono as
|
|
49564
|
+
import { Hono as Hono24 } from "hono";
|
|
48977
49565
|
import { createMiddleware } from "hono/factory";
|
|
48978
49566
|
function buildPkceVerifier() {
|
|
48979
49567
|
return randomBytes12(32).toString("base64url");
|
|
@@ -48993,7 +49581,7 @@ var requireConnectToken = createMiddleware(async (c, next) => {
|
|
|
48993
49581
|
function getBaseUrl2(c) {
|
|
48994
49582
|
return getPublicWebUrl(c.req.url) ?? "";
|
|
48995
49583
|
}
|
|
48996
|
-
var connectRoutes = new
|
|
49584
|
+
var connectRoutes = new Hono24();
|
|
48997
49585
|
async function getLatestValidationRun(connectionId, createdAfter) {
|
|
48998
49586
|
const sql = getDb();
|
|
48999
49587
|
const rows = await sql`
|
|
@@ -49804,7 +50392,7 @@ async function restGetFeedForRun(c) {
|
|
|
49804
50392
|
init_src();
|
|
49805
50393
|
import { readFile as readFile9 } from "node:fs/promises";
|
|
49806
50394
|
import { resolve as resolve5 } from "node:path";
|
|
49807
|
-
import { Hono as
|
|
50395
|
+
import { Hono as Hono26 } from "hono";
|
|
49808
50396
|
init_client();
|
|
49809
50397
|
init_provider_secrets();
|
|
49810
50398
|
init_client2();
|
|
@@ -49814,11 +50402,12 @@ init_auth_profiles_manager();
|
|
|
49814
50402
|
init_public_origin();
|
|
49815
50403
|
|
|
49816
50404
|
// src/lobu/client-routes.ts
|
|
49817
|
-
import { Hono as
|
|
50405
|
+
import { Hono as Hono25 } from "hono";
|
|
49818
50406
|
init_clients();
|
|
49819
50407
|
init_client();
|
|
49820
50408
|
|
|
49821
50409
|
// src/mcp-handler.ts
|
|
50410
|
+
init_sse_abort_bridge();
|
|
49822
50411
|
init_clients();
|
|
49823
50412
|
init_tool_access();
|
|
49824
50413
|
init_client();
|
|
@@ -57563,6 +58152,86 @@ async function handleListAvailable(args, ctx) {
|
|
|
57563
58152
|
offset: result.offset
|
|
57564
58153
|
};
|
|
57565
58154
|
}
|
|
58155
|
+
async function waitForDeviceActionRun(runId, organizationId) {
|
|
58156
|
+
const sql = getDb();
|
|
58157
|
+
const QUEUE_BUDGET_MS = 6e4;
|
|
58158
|
+
const POST_CLAIM_BUDGET_MS = 95e3;
|
|
58159
|
+
const POLL_MS = 500;
|
|
58160
|
+
const queueDeadline = Date.now() + QUEUE_BUDGET_MS;
|
|
58161
|
+
let claimedAtMs = null;
|
|
58162
|
+
while (true) {
|
|
58163
|
+
const rows = await sql`
|
|
58164
|
+
SELECT status, action_output, error_message, claimed_at
|
|
58165
|
+
FROM runs
|
|
58166
|
+
WHERE id = ${runId} AND organization_id = ${organizationId}
|
|
58167
|
+
LIMIT 1
|
|
58168
|
+
`;
|
|
58169
|
+
const row = rows[0];
|
|
58170
|
+
if (!row) {
|
|
58171
|
+
return {
|
|
58172
|
+
status: "failed",
|
|
58173
|
+
error_message: `Run ${runId} disappeared from runs table while waiting.`
|
|
58174
|
+
};
|
|
58175
|
+
}
|
|
58176
|
+
if (row.status === "completed") {
|
|
58177
|
+
return {
|
|
58178
|
+
status: "completed",
|
|
58179
|
+
output: row.action_output ?? {}
|
|
58180
|
+
};
|
|
58181
|
+
}
|
|
58182
|
+
if (row.status === "failed" || row.status === "timeout") {
|
|
58183
|
+
return {
|
|
58184
|
+
status: row.status,
|
|
58185
|
+
error_message: row.error_message ?? `Run ${runId} ${row.status}`
|
|
58186
|
+
};
|
|
58187
|
+
}
|
|
58188
|
+
if (row.claimed_at && claimedAtMs == null) {
|
|
58189
|
+
claimedAtMs = row.claimed_at instanceof Date ? row.claimed_at.getTime() : new Date(row.claimed_at).getTime();
|
|
58190
|
+
}
|
|
58191
|
+
const now = Date.now();
|
|
58192
|
+
if (claimedAtMs != null) {
|
|
58193
|
+
if (now - claimedAtMs >= POST_CLAIM_BUDGET_MS) break;
|
|
58194
|
+
} else {
|
|
58195
|
+
if (now >= queueDeadline) break;
|
|
58196
|
+
}
|
|
58197
|
+
await new Promise((r) => setTimeout(r, POLL_MS));
|
|
58198
|
+
}
|
|
58199
|
+
const updated = await sql`
|
|
58200
|
+
UPDATE runs
|
|
58201
|
+
SET status = 'timeout',
|
|
58202
|
+
completed_at = current_timestamp,
|
|
58203
|
+
error_message = ${"waitForDeviceActionRun: device worker did not complete in time"}
|
|
58204
|
+
WHERE id = ${runId}
|
|
58205
|
+
AND organization_id = ${organizationId}
|
|
58206
|
+
AND status IN ('pending', 'running')
|
|
58207
|
+
RETURNING id
|
|
58208
|
+
`;
|
|
58209
|
+
if (updated.length === 0) {
|
|
58210
|
+
const finalRows = await sql`
|
|
58211
|
+
SELECT status, action_output, error_message
|
|
58212
|
+
FROM runs
|
|
58213
|
+
WHERE id = ${runId} AND organization_id = ${organizationId}
|
|
58214
|
+
LIMIT 1
|
|
58215
|
+
`;
|
|
58216
|
+
const final = finalRows[0];
|
|
58217
|
+
if (final?.status === "completed") {
|
|
58218
|
+
return {
|
|
58219
|
+
status: "completed",
|
|
58220
|
+
output: final.action_output ?? {}
|
|
58221
|
+
};
|
|
58222
|
+
}
|
|
58223
|
+
if (final?.status === "failed") {
|
|
58224
|
+
return {
|
|
58225
|
+
status: "failed",
|
|
58226
|
+
error_message: final.error_message ?? `Run ${runId} failed`
|
|
58227
|
+
};
|
|
58228
|
+
}
|
|
58229
|
+
}
|
|
58230
|
+
return {
|
|
58231
|
+
status: "timeout",
|
|
58232
|
+
error_message: claimedAtMs != null ? `Run ${runId} claimed but the device worker didn't finish within ${POST_CLAIM_BUDGET_MS}ms.` : `Run ${runId} was never claimed within ${QUEUE_BUDGET_MS}ms \u2014 the chrome-extension / device worker may be offline.`
|
|
58233
|
+
};
|
|
58234
|
+
}
|
|
57566
58235
|
async function handleExecute(args, env2, ctx) {
|
|
57567
58236
|
const sql = getDb();
|
|
57568
58237
|
const resolved = await getOperationForConnection(
|
|
@@ -57585,13 +58254,23 @@ async function handleExecute(args, env2, ctx) {
|
|
|
57585
58254
|
};
|
|
57586
58255
|
}
|
|
57587
58256
|
const shouldQueue = mode === "approval";
|
|
58257
|
+
const defRows = await sql`
|
|
58258
|
+
SELECT runtime FROM connector_definitions
|
|
58259
|
+
WHERE key = ${connection.connector_key}
|
|
58260
|
+
AND organization_id = ${ctx.organizationId}
|
|
58261
|
+
AND status = 'active'
|
|
58262
|
+
ORDER BY updated_at DESC, id DESC
|
|
58263
|
+
LIMIT 1
|
|
58264
|
+
`;
|
|
58265
|
+
const isDeviceBound = defRows[0]?.runtime != null;
|
|
58266
|
+
const approvalMode = shouldQueue ? "queued" : isDeviceBound ? "device" : "inline";
|
|
57588
58267
|
const runId = await createConnectorOperationRun({
|
|
57589
58268
|
organizationId: ctx.organizationId,
|
|
57590
58269
|
connectionId: connection.id,
|
|
57591
58270
|
connectorKey: connection.connector_key,
|
|
57592
58271
|
operationKey: operation.operation_key,
|
|
57593
58272
|
operationInput: input,
|
|
57594
|
-
approvalMode
|
|
58273
|
+
approvalMode,
|
|
57595
58274
|
requireCompiledCode: operation.backend === "local_action"
|
|
57596
58275
|
});
|
|
57597
58276
|
if (args.watcher_source) {
|
|
@@ -57666,6 +58345,31 @@ async function handleExecute(args, env2, ctx) {
|
|
|
57666
58345
|
message: `Operation '${operation.name}' requires approval. Share the approval_url with the user to confirm.`
|
|
57667
58346
|
};
|
|
57668
58347
|
}
|
|
58348
|
+
if (approvalMode === "device") {
|
|
58349
|
+
const result2 = await waitForDeviceActionRun(runId, ctx.organizationId);
|
|
58350
|
+
if (result2.status === "completed") {
|
|
58351
|
+
return {
|
|
58352
|
+
action: "execute",
|
|
58353
|
+
run_id: runId,
|
|
58354
|
+
status: "completed",
|
|
58355
|
+
output: result2.output ?? {}
|
|
58356
|
+
};
|
|
58357
|
+
}
|
|
58358
|
+
if (result2.status === "timeout") {
|
|
58359
|
+
return {
|
|
58360
|
+
action: "execute",
|
|
58361
|
+
run_id: runId,
|
|
58362
|
+
status: "timeout",
|
|
58363
|
+
error_message: result2.error_message ?? "Device action run timed out."
|
|
58364
|
+
};
|
|
58365
|
+
}
|
|
58366
|
+
return {
|
|
58367
|
+
action: "execute",
|
|
58368
|
+
run_id: runId,
|
|
58369
|
+
status: "failed",
|
|
58370
|
+
error_message: result2.error_message ?? "Device action run failed."
|
|
58371
|
+
};
|
|
58372
|
+
}
|
|
57669
58373
|
const result = await executeOperationInline(
|
|
57670
58374
|
runId,
|
|
57671
58375
|
ctx.organizationId,
|
|
@@ -67205,7 +67909,7 @@ async function sseToJson(response) {
|
|
|
67205
67909
|
return response;
|
|
67206
67910
|
}
|
|
67207
67911
|
var SSE_HEARTBEAT_INTERVAL_MS = 15e3;
|
|
67208
|
-
function withSSEHeartbeat(response) {
|
|
67912
|
+
function withSSEHeartbeat(response, signal) {
|
|
67209
67913
|
if (!response.headers.get("content-type")?.includes("text/event-stream") || !response.body) {
|
|
67210
67914
|
return response;
|
|
67211
67915
|
}
|
|
@@ -67226,22 +67930,37 @@ function withSSEHeartbeat(response) {
|
|
|
67226
67930
|
if (intervalId) clearInterval(intervalId);
|
|
67227
67931
|
writer.abort(reason).catch(() => void 0);
|
|
67228
67932
|
};
|
|
67933
|
+
const adapter = {
|
|
67934
|
+
get aborted() {
|
|
67935
|
+
return terminated;
|
|
67936
|
+
},
|
|
67937
|
+
get closed() {
|
|
67938
|
+
return terminated;
|
|
67939
|
+
},
|
|
67940
|
+
abort() {
|
|
67941
|
+
abortWriter(new Error("Request aborted"));
|
|
67942
|
+
}
|
|
67943
|
+
};
|
|
67229
67944
|
intervalId = setInterval(() => {
|
|
67230
67945
|
writer.write(heartbeat2).catch(() => abortWriter(new Error("SSE heartbeat write failed")));
|
|
67231
67946
|
}, SSE_HEARTBEAT_INTERVAL_MS);
|
|
67947
|
+
const detachAbortBridge = bindRequestAbortToStream(signal, adapter);
|
|
67232
67948
|
response.body.pipeTo(
|
|
67233
67949
|
new WritableStream({
|
|
67234
67950
|
write(chunk) {
|
|
67235
67951
|
return writer.write(chunk);
|
|
67236
67952
|
},
|
|
67237
67953
|
close() {
|
|
67954
|
+
detachAbortBridge();
|
|
67238
67955
|
closeWriter();
|
|
67239
67956
|
},
|
|
67240
67957
|
abort(reason) {
|
|
67958
|
+
detachAbortBridge();
|
|
67241
67959
|
abortWriter(reason);
|
|
67242
67960
|
}
|
|
67243
67961
|
})
|
|
67244
67962
|
).catch(() => {
|
|
67963
|
+
detachAbortBridge();
|
|
67245
67964
|
abortWriter(new Error("Source SSE stream error"));
|
|
67246
67965
|
});
|
|
67247
67966
|
return new Response(readable, { status: response.status, headers: response.headers });
|
|
@@ -67251,7 +67970,7 @@ async function handleAndMaybeConvert(transport, req, wantsSSE) {
|
|
|
67251
67970
|
if (!wantsSSE && response.headers.get("content-type")?.includes("text/event-stream")) {
|
|
67252
67971
|
return sseToJson(response);
|
|
67253
67972
|
}
|
|
67254
|
-
return withSSEHeartbeat(response);
|
|
67973
|
+
return withSSEHeartbeat(response, req.signal);
|
|
67255
67974
|
}
|
|
67256
67975
|
async function resolveAuthWithInstructions(c, req) {
|
|
67257
67976
|
const authCtx = extractAuthContext(c);
|
|
@@ -67484,8 +68203,8 @@ async function handleMcp(c) {
|
|
|
67484
68203
|
|
|
67485
68204
|
// src/lobu/client-routes.ts
|
|
67486
68205
|
init_org_context();
|
|
67487
|
-
var routes = new
|
|
67488
|
-
var platformSchemaRoutes = new
|
|
68206
|
+
var routes = new Hono25();
|
|
68207
|
+
var platformSchemaRoutes = new Hono25();
|
|
67489
68208
|
var LOBU_FIRST_PARTY_SOFTWARE_IDS = /* @__PURE__ */ new Set([
|
|
67490
68209
|
"lobu-cli",
|
|
67491
68210
|
"lobu",
|
|
@@ -67874,7 +68593,7 @@ platformSchemaRoutes.get("/:platform", (c) => {
|
|
|
67874
68593
|
init_gateway3();
|
|
67875
68594
|
init_postgres_stores();
|
|
67876
68595
|
init_org_context();
|
|
67877
|
-
var routes2 = new
|
|
68596
|
+
var routes2 = new Hono26();
|
|
67878
68597
|
function toStringArray(value) {
|
|
67879
68598
|
if (Array.isArray(value)) return value.map(String);
|
|
67880
68599
|
if (typeof value === "string") {
|
|
@@ -70620,6 +71339,7 @@ function generateOpenAPISpec(serverUrl) {
|
|
|
70620
71339
|
|
|
70621
71340
|
// src/index.ts
|
|
70622
71341
|
init_public_origin();
|
|
71342
|
+
init_embedded_deployment();
|
|
70623
71343
|
|
|
70624
71344
|
// src/scheduled/scheduler-health.ts
|
|
70625
71345
|
init_client();
|
|
@@ -70796,6 +71516,186 @@ async function joinPublicOrganization({
|
|
|
70796
71516
|
// src/index.ts
|
|
70797
71517
|
init_multi_tenant();
|
|
70798
71518
|
|
|
71519
|
+
// src/gateway/routes/internal/smoke.ts
|
|
71520
|
+
init_src();
|
|
71521
|
+
init_client();
|
|
71522
|
+
import { timingSafeEqual as timingSafeEqual3 } from "node:crypto";
|
|
71523
|
+
import { Hono as Hono27 } from "hono";
|
|
71524
|
+
var logger97 = createLogger("smoke-dispatch");
|
|
71525
|
+
function compareTokens(provided, expected) {
|
|
71526
|
+
if (provided.length !== expected.length) return false;
|
|
71527
|
+
try {
|
|
71528
|
+
return timingSafeEqual3(Buffer.from(provided), Buffer.from(expected));
|
|
71529
|
+
} catch {
|
|
71530
|
+
return false;
|
|
71531
|
+
}
|
|
71532
|
+
}
|
|
71533
|
+
var FORWARDED_HEADERS = [
|
|
71534
|
+
"x-forwarded-for",
|
|
71535
|
+
"x-forwarded-host",
|
|
71536
|
+
"x-forwarded-proto",
|
|
71537
|
+
"x-forwarded-port",
|
|
71538
|
+
"x-forwarded-server",
|
|
71539
|
+
"x-real-ip",
|
|
71540
|
+
"forwarded"
|
|
71541
|
+
];
|
|
71542
|
+
function createSmokeRoutes() {
|
|
71543
|
+
const app3 = new Hono27();
|
|
71544
|
+
app3.post("/dispatch", async (c) => {
|
|
71545
|
+
const expected = process.env.SMOKE_TEST_TOKEN ?? "";
|
|
71546
|
+
const smokeAgentId = (process.env.SMOKE_TEST_AGENT_ID ?? "").trim();
|
|
71547
|
+
const smokeOrgId = (process.env.SMOKE_TEST_ORG_ID ?? "").trim();
|
|
71548
|
+
if (expected.length === 0 || smokeAgentId === "" || smokeOrgId === "") {
|
|
71549
|
+
return c.json(
|
|
71550
|
+
{
|
|
71551
|
+
error: "Smoke dispatch disabled (SMOKE_TEST_TOKEN/SMOKE_TEST_AGENT_ID/SMOKE_TEST_ORG_ID unset)"
|
|
71552
|
+
},
|
|
71553
|
+
503
|
|
71554
|
+
);
|
|
71555
|
+
}
|
|
71556
|
+
for (const h of FORWARDED_HEADERS) {
|
|
71557
|
+
if (c.req.header(h)) {
|
|
71558
|
+
logger97.warn(
|
|
71559
|
+
`Smoke dispatch refused: ${h} header present (request came through ingress)`
|
|
71560
|
+
);
|
|
71561
|
+
return c.json({ error: "Forwarded request refused" }, 403);
|
|
71562
|
+
}
|
|
71563
|
+
}
|
|
71564
|
+
const allowedHost = (process.env.SMOKE_TEST_ALLOWED_HOST ?? "").trim();
|
|
71565
|
+
if (allowedHost !== "") {
|
|
71566
|
+
const rawHost = (c.req.header("host") ?? "").toLowerCase();
|
|
71567
|
+
const hostPart = rawHost.split(":")[0] ?? "";
|
|
71568
|
+
const expected2 = allowedHost.toLowerCase();
|
|
71569
|
+
if (hostPart !== expected2 && !hostPart.startsWith(`${expected2}.`)) {
|
|
71570
|
+
logger97.warn(
|
|
71571
|
+
`Smoke dispatch refused: Host '${rawHost}' does not match SMOKE_TEST_ALLOWED_HOST '${allowedHost}'`
|
|
71572
|
+
);
|
|
71573
|
+
return c.json({ error: "Host header refused" }, 403);
|
|
71574
|
+
}
|
|
71575
|
+
}
|
|
71576
|
+
const auth = c.req.header("authorization") ?? "";
|
|
71577
|
+
if (!auth.startsWith("Bearer ")) {
|
|
71578
|
+
return c.json({ error: "Missing bearer token" }, 401);
|
|
71579
|
+
}
|
|
71580
|
+
const provided = auth.substring(7);
|
|
71581
|
+
if (!compareTokens(provided, expected)) {
|
|
71582
|
+
return c.json({ error: "Invalid smoke token" }, 401);
|
|
71583
|
+
}
|
|
71584
|
+
let body2;
|
|
71585
|
+
try {
|
|
71586
|
+
body2 = await c.req.json();
|
|
71587
|
+
} catch {
|
|
71588
|
+
return c.json({ error: "Invalid JSON body" }, 400);
|
|
71589
|
+
}
|
|
71590
|
+
const agentId = smokeAgentId;
|
|
71591
|
+
const organizationId = smokeOrgId;
|
|
71592
|
+
const conversationId = body2.conversationId?.trim();
|
|
71593
|
+
const messageText = body2.messageText?.trim() || "smoke-test ping";
|
|
71594
|
+
if (!conversationId) {
|
|
71595
|
+
return c.json({ error: "conversationId is required" }, 400);
|
|
71596
|
+
}
|
|
71597
|
+
if (!conversationId.startsWith("smoke-")) {
|
|
71598
|
+
return c.json(
|
|
71599
|
+
{
|
|
71600
|
+
error: "conversationId must start with 'smoke-' for safety"
|
|
71601
|
+
},
|
|
71602
|
+
400
|
|
71603
|
+
);
|
|
71604
|
+
}
|
|
71605
|
+
const idempotencyKey = `smoke:${conversationId}`;
|
|
71606
|
+
const messageId = `smoke-msg-${Date.now()}`;
|
|
71607
|
+
const payload = {
|
|
71608
|
+
platform: "smoke",
|
|
71609
|
+
userId: "smoke-user",
|
|
71610
|
+
botId: "smoke",
|
|
71611
|
+
conversationId,
|
|
71612
|
+
teamId: "smoke",
|
|
71613
|
+
agentId,
|
|
71614
|
+
organizationId,
|
|
71615
|
+
messageId,
|
|
71616
|
+
messageText,
|
|
71617
|
+
channelId: conversationId,
|
|
71618
|
+
platformMetadata: {
|
|
71619
|
+
agentId,
|
|
71620
|
+
chatId: conversationId,
|
|
71621
|
+
senderId: "smoke-user",
|
|
71622
|
+
responseChannel: conversationId,
|
|
71623
|
+
responseId: messageId,
|
|
71624
|
+
responseThreadId: conversationId
|
|
71625
|
+
},
|
|
71626
|
+
agentOptions: {}
|
|
71627
|
+
};
|
|
71628
|
+
const sql = getDb();
|
|
71629
|
+
try {
|
|
71630
|
+
const result = await sql`
|
|
71631
|
+
INSERT INTO public.runs (
|
|
71632
|
+
run_type,
|
|
71633
|
+
queue_name,
|
|
71634
|
+
action_input,
|
|
71635
|
+
idempotency_key,
|
|
71636
|
+
max_attempts,
|
|
71637
|
+
attempts,
|
|
71638
|
+
status,
|
|
71639
|
+
run_at,
|
|
71640
|
+
priority,
|
|
71641
|
+
retry_delay_seconds
|
|
71642
|
+
) VALUES (
|
|
71643
|
+
'chat_message',
|
|
71644
|
+
'messages',
|
|
71645
|
+
${sql.json(payload)},
|
|
71646
|
+
${idempotencyKey},
|
|
71647
|
+
1,
|
|
71648
|
+
0,
|
|
71649
|
+
'pending',
|
|
71650
|
+
now(),
|
|
71651
|
+
0,
|
|
71652
|
+
NULL
|
|
71653
|
+
)
|
|
71654
|
+
ON CONFLICT (idempotency_key)
|
|
71655
|
+
WHERE idempotency_key IS NOT NULL
|
|
71656
|
+
AND status IN ('pending', 'claimed', 'running')
|
|
71657
|
+
DO NOTHING
|
|
71658
|
+
RETURNING id
|
|
71659
|
+
`;
|
|
71660
|
+
let runId = null;
|
|
71661
|
+
if (result.length > 0 && result[0]) {
|
|
71662
|
+
runId = Number(result[0].id);
|
|
71663
|
+
} else {
|
|
71664
|
+
const existing = await sql`
|
|
71665
|
+
SELECT id FROM public.runs
|
|
71666
|
+
WHERE idempotency_key = ${idempotencyKey}
|
|
71667
|
+
AND status IN ('pending', 'claimed', 'running')
|
|
71668
|
+
ORDER BY id DESC
|
|
71669
|
+
LIMIT 1
|
|
71670
|
+
`;
|
|
71671
|
+
if (existing.length > 0 && existing[0]) {
|
|
71672
|
+
runId = Number(existing[0].id);
|
|
71673
|
+
}
|
|
71674
|
+
}
|
|
71675
|
+
if (runId === null) {
|
|
71676
|
+
return c.json({ error: "Failed to enqueue smoke run" }, 500);
|
|
71677
|
+
}
|
|
71678
|
+
try {
|
|
71679
|
+
await sql`SELECT pg_notify('runs_lobu:messages', 'chat_message')`;
|
|
71680
|
+
} catch (err) {
|
|
71681
|
+
logger97.warn(
|
|
71682
|
+
`pg_notify after smoke dispatch failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`
|
|
71683
|
+
);
|
|
71684
|
+
}
|
|
71685
|
+
logger97.info(
|
|
71686
|
+
`Smoke dispatch: runId=${runId} agentId=${agentId} org=${organizationId} conv=${conversationId}`
|
|
71687
|
+
);
|
|
71688
|
+
return c.json({ runId, idempotencyKey });
|
|
71689
|
+
} catch (err) {
|
|
71690
|
+
logger97.error(
|
|
71691
|
+
`Smoke dispatch INSERT failed: ${err instanceof Error ? err.message : String(err)}`
|
|
71692
|
+
);
|
|
71693
|
+
return c.json({ error: "Internal error" }, 500);
|
|
71694
|
+
}
|
|
71695
|
+
});
|
|
71696
|
+
return app3;
|
|
71697
|
+
}
|
|
71698
|
+
|
|
70799
71699
|
// src/worker-api.ts
|
|
70800
71700
|
init_src();
|
|
70801
71701
|
init_auth2();
|
|
@@ -70866,7 +71766,7 @@ import { createHash as createHash14, randomUUID as randomUUID16 } from "node:cry
|
|
|
70866
71766
|
// src/gateway/services/agent-threads.ts
|
|
70867
71767
|
init_src();
|
|
70868
71768
|
import { randomUUID as randomUUID15 } from "node:crypto";
|
|
70869
|
-
var
|
|
71769
|
+
var logger98 = createLogger("agent-threads");
|
|
70870
71770
|
var TOKEN_EXPIRATION_MS2 = 24 * 60 * 60 * 1e3;
|
|
70871
71771
|
async function createThreadForAgent(deps, args) {
|
|
70872
71772
|
const { sessionManager } = deps;
|
|
@@ -70898,7 +71798,7 @@ async function createThreadForAgent(deps, args) {
|
|
|
70898
71798
|
isEphemeral: false
|
|
70899
71799
|
};
|
|
70900
71800
|
await sessionManager.setSession(session);
|
|
70901
|
-
|
|
71801
|
+
logger98.info(
|
|
70902
71802
|
`Created thread ${conversationId} for agent ${agentId}${reason ? ` (reason=${reason})` : ""}`
|
|
70903
71803
|
);
|
|
70904
71804
|
return { threadId: conversationId, token, expiresAt };
|
|
@@ -73362,6 +74262,8 @@ async function postAuthSignal(c) {
|
|
|
73362
74262
|
async function completeActionRun(c) {
|
|
73363
74263
|
try {
|
|
73364
74264
|
const req = await c.req.json();
|
|
74265
|
+
const denied = await authorizeRunForWorker(c, req.run_id, req.worker_id);
|
|
74266
|
+
if (denied) return denied;
|
|
73365
74267
|
const sql = getDb();
|
|
73366
74268
|
const updatedRuns = await sql`
|
|
73367
74269
|
UPDATE runs
|
|
@@ -73370,8 +74272,17 @@ async function completeActionRun(c) {
|
|
|
73370
74272
|
action_output = ${req.action_output ? sql.json(req.action_output) : null},
|
|
73371
74273
|
error_message = ${req.error_message ?? null}
|
|
73372
74274
|
WHERE id = ${req.run_id}
|
|
74275
|
+
AND status = 'running'
|
|
74276
|
+
AND claimed_by = ${req.worker_id}
|
|
73373
74277
|
RETURNING organization_id, action_key
|
|
73374
74278
|
`;
|
|
74279
|
+
if (updatedRuns.length === 0) {
|
|
74280
|
+
logger_default.info(
|
|
74281
|
+
{ run_id: req.run_id, worker_id: req.worker_id, claimed_status: req.status },
|
|
74282
|
+
"[completeActionRun] no-op: run already in terminal state (likely gateway timeout)"
|
|
74283
|
+
);
|
|
74284
|
+
return c.json({ success: false, reason: "already_finalized" });
|
|
74285
|
+
}
|
|
73375
74286
|
const organizationId = updatedRuns[0]?.organization_id;
|
|
73376
74287
|
const actionKey = updatedRuns[0]?.action_key ?? "Action";
|
|
73377
74288
|
if (organizationId) {
|
|
@@ -74218,7 +75129,7 @@ async function serveStaticFile(c, filePath) {
|
|
|
74218
75129
|
new Uint8Array(ab).set(body2);
|
|
74219
75130
|
return c.body(new Uint8Array(ab));
|
|
74220
75131
|
}
|
|
74221
|
-
var app = new
|
|
75132
|
+
var app = new Hono28();
|
|
74222
75133
|
app.use("/*", compress({ threshold: 1024 }));
|
|
74223
75134
|
app.use(
|
|
74224
75135
|
"/*",
|
|
@@ -74250,8 +75161,13 @@ app.use("/*", async (c, next) => {
|
|
|
74250
75161
|
if (contentType.startsWith("text/html")) {
|
|
74251
75162
|
const rawFrameAncestors = c.env.FRAME_ANCESTORS?.trim();
|
|
74252
75163
|
const frameAncestors = rawFrameAncestors ? rawFrameAncestors.split(/[\s,]+/).map((entry) => entry.trim()).filter((entry) => isValidFrameAncestor(entry)).join(" ") : "https://lobu.ai https://*.lobu.ai";
|
|
74253
|
-
const
|
|
74254
|
-
const
|
|
75164
|
+
const extraExtensionIds = (c.env.LOBU_OWLETTO_EXTENSION_IDS ?? "").split(",").map((s) => s.trim()).filter((s) => /^[a-p]{32}$/.test(s));
|
|
75165
|
+
const ownedExtensionIds = [
|
|
75166
|
+
// canonical, derived from apps/chrome/manifest.json's `key`
|
|
75167
|
+
"amnnhclgmbldmfcfamonoggjhfidemmm",
|
|
75168
|
+
...extraExtensionIds
|
|
75169
|
+
];
|
|
75170
|
+
const extensionAllowed = ownedExtensionIds.map((id) => ` chrome-extension://${id}`).join("");
|
|
74255
75171
|
c.header(
|
|
74256
75172
|
"Content-Security-Policy",
|
|
74257
75173
|
`frame-ancestors 'self' ${frameAncestors}${extensionAllowed}`
|
|
@@ -74323,6 +75239,17 @@ app.get("/health/ready", async (c) => {
|
|
|
74323
75239
|
);
|
|
74324
75240
|
}
|
|
74325
75241
|
});
|
|
75242
|
+
app.get("/health/orchestrator", (c) => {
|
|
75243
|
+
const count = getReservedLockCount();
|
|
75244
|
+
const cap = getMaxReservedLocks();
|
|
75245
|
+
const nearCap = cap > 0 && count >= Math.ceil(cap * 0.8);
|
|
75246
|
+
return c.json({
|
|
75247
|
+
status: "ok",
|
|
75248
|
+
reserved_conversation_locks: count,
|
|
75249
|
+
reserved_conversation_locks_cap: cap,
|
|
75250
|
+
near_cap: nearCap
|
|
75251
|
+
});
|
|
75252
|
+
});
|
|
74326
75253
|
app.get("/health/scheduler", async (c) => {
|
|
74327
75254
|
try {
|
|
74328
75255
|
const health = await getSchedulerHealth(c.env);
|
|
@@ -74401,6 +75328,7 @@ app.get("/legal", (c) => {
|
|
|
74401
75328
|
</html>`);
|
|
74402
75329
|
});
|
|
74403
75330
|
app.get("/api/health", restHealth);
|
|
75331
|
+
app.route("/api/internal/smoke", createSmokeRoutes());
|
|
74404
75332
|
app.use("/api/workers/*", async (c, next) => {
|
|
74405
75333
|
const expected = c.env.WORKER_API_TOKEN;
|
|
74406
75334
|
const provided = c.req.header("Authorization")?.replace("Bearer ", "");
|
|
@@ -74416,7 +75344,13 @@ app.use("/api/workers/*", async (c, next) => {
|
|
|
74416
75344
|
"/api/workers/poll",
|
|
74417
75345
|
"/api/workers/heartbeat",
|
|
74418
75346
|
"/api/workers/stream",
|
|
74419
|
-
"/api/workers/complete"
|
|
75347
|
+
"/api/workers/complete",
|
|
75348
|
+
// Action runs (run_type='action') finalize via /complete-action,
|
|
75349
|
+
// which persists action_output. The handler still goes through
|
|
75350
|
+
// authorizeRunForWorker so a user worker can only finalize runs
|
|
75351
|
+
// it claimed. Required for chrome-extension action tools to
|
|
75352
|
+
// return their observation back to the gateway.
|
|
75353
|
+
"/api/workers/complete-action"
|
|
74420
75354
|
]);
|
|
74421
75355
|
const requestPath = new URL(c.req.url).pathname;
|
|
74422
75356
|
const isAuthProfileSubpath = requestPath.startsWith("/api/workers/me/auth-profiles");
|
|
@@ -74970,7 +75904,6 @@ init_gateway3();
|
|
|
74970
75904
|
init_client();
|
|
74971
75905
|
init_connect_tokens();
|
|
74972
75906
|
init_logger();
|
|
74973
|
-
init_pg_errors();
|
|
74974
75907
|
var REAPER_ADVISORY_LOCK_KEY = 1919841138;
|
|
74975
75908
|
var DEFAULT_STALE_AFTER_SECONDS = 120;
|
|
74976
75909
|
function staleAfterSeconds() {
|
|
@@ -74992,56 +75925,75 @@ async function reapStaleRuns() {
|
|
|
74992
75925
|
try {
|
|
74993
75926
|
const errorMessage3 = "worker_heartbeat_lost";
|
|
74994
75927
|
const reaped = await reserved`
|
|
74995
|
-
|
|
74996
|
-
|
|
74997
|
-
|
|
74998
|
-
|
|
74999
|
-
|
|
75000
|
-
|
|
75001
|
-
|
|
75002
|
-
(
|
|
75003
|
-
|
|
75004
|
-
|
|
75005
|
-
|
|
75006
|
-
|
|
75007
|
-
|
|
75008
|
-
|
|
75928
|
+
WITH timed_out AS (
|
|
75929
|
+
UPDATE public.runs
|
|
75930
|
+
SET status = 'timeout',
|
|
75931
|
+
completed_at = current_timestamp,
|
|
75932
|
+
error_message = ${errorMessage3}
|
|
75933
|
+
WHERE run_type IN ('sync', 'action', 'embed_backfill', 'auth')
|
|
75934
|
+
AND status IN ('claimed', 'running')
|
|
75935
|
+
AND (
|
|
75936
|
+
(last_heartbeat_at IS NULL
|
|
75937
|
+
AND COALESCE(claimed_at, created_at)
|
|
75938
|
+
< current_timestamp - (${thresholdSeconds}::int * interval '1 second'))
|
|
75939
|
+
OR
|
|
75940
|
+
(last_heartbeat_at IS NOT NULL
|
|
75941
|
+
AND last_heartbeat_at
|
|
75942
|
+
< current_timestamp - (${thresholdSeconds}::int * interval '1 second'))
|
|
75943
|
+
)
|
|
75944
|
+
RETURNING id, run_type, feed_id, connection_id, connector_key, connector_version, organization_id
|
|
75945
|
+
),
|
|
75946
|
+
retries AS (
|
|
75947
|
+
INSERT INTO public.runs (
|
|
75948
|
+
organization_id, run_type, feed_id, connection_id,
|
|
75949
|
+
connector_key, connector_version, status, approval_status, created_at
|
|
75009
75950
|
)
|
|
75010
|
-
|
|
75951
|
+
SELECT
|
|
75952
|
+
t.organization_id, 'sync', t.feed_id, t.connection_id,
|
|
75953
|
+
t.connector_key, t.connector_version, 'pending', 'auto', current_timestamp
|
|
75954
|
+
FROM timed_out t
|
|
75955
|
+
WHERE t.run_type = 'sync'
|
|
75956
|
+
AND t.feed_id IS NOT NULL
|
|
75957
|
+
AND NOT EXISTS (
|
|
75958
|
+
-- Look for an unrelated active sync run on the same feed.
|
|
75959
|
+
-- Exclude timed_out.id because in PostgreSQL the sibling
|
|
75960
|
+
-- CTE UPDATE is not visible here (all CTEs see the same
|
|
75961
|
+
-- snapshot), so the row we just reaped still appears as
|
|
75962
|
+
-- running. Without this exclusion, every reap would
|
|
75963
|
+
-- dedupe against itself and no retries would ever land.
|
|
75964
|
+
SELECT 1 FROM public.runs r
|
|
75965
|
+
WHERE r.feed_id = t.feed_id
|
|
75966
|
+
AND r.run_type = 'sync'
|
|
75967
|
+
AND r.status IN ('pending', 'claimed', 'running')
|
|
75968
|
+
AND r.id NOT IN (SELECT id FROM timed_out)
|
|
75969
|
+
)
|
|
75970
|
+
RETURNING id, feed_id
|
|
75971
|
+
)
|
|
75972
|
+
SELECT
|
|
75973
|
+
(SELECT count(*)::int FROM timed_out) AS reaped,
|
|
75974
|
+
(SELECT count(*)::int FROM retries) AS retries_created,
|
|
75975
|
+
(SELECT count(*)::int FROM timed_out
|
|
75976
|
+
WHERE run_type = 'sync' AND feed_id IS NOT NULL) AS sync_eligible
|
|
75011
75977
|
`;
|
|
75012
|
-
|
|
75978
|
+
const reapedRow = reaped[0];
|
|
75979
|
+
const reapedCount = reapedRow?.reaped ?? 0;
|
|
75980
|
+
const retriesCreated = reapedRow?.retries_created ?? 0;
|
|
75981
|
+
const syncEligible = reapedRow?.sync_eligible ?? 0;
|
|
75982
|
+
if (reapedCount === 0) {
|
|
75013
75983
|
return { acquired: true, reaped: 0, retriesCreated: 0 };
|
|
75014
75984
|
}
|
|
75015
75985
|
logger_default.warn(
|
|
75016
|
-
{ reaped:
|
|
75986
|
+
{ reaped: reapedCount, retriesCreated, thresholdSeconds },
|
|
75017
75987
|
"[reaper] Marked stale connector runs as timeout (worker_heartbeat_lost)"
|
|
75018
75988
|
);
|
|
75019
|
-
|
|
75020
|
-
|
|
75021
|
-
|
|
75022
|
-
|
|
75023
|
-
|
|
75024
|
-
|
|
75025
|
-
organization_id, run_type, feed_id, connection_id,
|
|
75026
|
-
connector_key, connector_version, status, approval_status, created_at
|
|
75027
|
-
) VALUES (
|
|
75028
|
-
${row.organization_id}, 'sync', ${row.feed_id}, ${row.connection_id},
|
|
75029
|
-
${row.connector_key}, ${row.connector_version}, 'pending', 'auto', current_timestamp
|
|
75030
|
-
)
|
|
75031
|
-
`;
|
|
75032
|
-
retriesCreated += 1;
|
|
75033
|
-
} catch (err) {
|
|
75034
|
-
if (isUniqueViolation(err, "idx_runs_active_sync_per_feed")) {
|
|
75035
|
-
logger_default.info(
|
|
75036
|
-
{ feedId: row.feed_id },
|
|
75037
|
-
"[reaper] Skipped sync retry \u2014 another active sync run exists"
|
|
75038
|
-
);
|
|
75039
|
-
} else {
|
|
75040
|
-
logger_default.error({ err, runId: row.id }, "[reaper] Failed to insert sync retry");
|
|
75041
|
-
}
|
|
75042
|
-
}
|
|
75989
|
+
const skippedRetries = syncEligible - retriesCreated;
|
|
75990
|
+
if (skippedRetries > 0) {
|
|
75991
|
+
logger_default.info(
|
|
75992
|
+
{ count: skippedRetries },
|
|
75993
|
+
"[reaper] Skipped sync retries \u2014 another active sync run exists (ON CONFLICT DO NOTHING)"
|
|
75994
|
+
);
|
|
75043
75995
|
}
|
|
75044
|
-
return { acquired: true, reaped:
|
|
75996
|
+
return { acquired: true, reaped: reapedCount, retriesCreated };
|
|
75045
75997
|
} finally {
|
|
75046
75998
|
await reserved`SELECT pg_advisory_unlock(${REAPER_ADVISORY_LOCK_KEY})`;
|
|
75047
75999
|
}
|
|
@@ -75610,7 +76562,7 @@ async function runClassificationReconciliation(_env) {
|
|
|
75610
76562
|
init_src();
|
|
75611
76563
|
init_cron();
|
|
75612
76564
|
import * as Sentry8 from "@sentry/node";
|
|
75613
|
-
var
|
|
76565
|
+
var logger99 = createLogger("task-scheduler");
|
|
75614
76566
|
var TASK_QUEUE_NAME = "task";
|
|
75615
76567
|
function cronSeedKey(name, tick) {
|
|
75616
76568
|
return `cron:${name}:${tick.toISOString()}`;
|
|
@@ -75663,7 +76615,7 @@ var TaskScheduler = class {
|
|
|
75663
76615
|
try {
|
|
75664
76616
|
await this.seedNextCronTick(reg);
|
|
75665
76617
|
} catch (err) {
|
|
75666
|
-
|
|
76618
|
+
logger99.error(
|
|
75667
76619
|
{ err, taskName: reg.name, cron: reg.cron },
|
|
75668
76620
|
"[task-scheduler] Failed to seed cron row at boot; will retry in background"
|
|
75669
76621
|
);
|
|
@@ -75672,7 +76624,7 @@ var TaskScheduler = class {
|
|
|
75672
76624
|
}
|
|
75673
76625
|
await this.queue.work(TASK_QUEUE_NAME, (job) => this.dispatch(job));
|
|
75674
76626
|
const periodic = [...this.handlers.values()].filter((r) => r.cron).length;
|
|
75675
|
-
|
|
76627
|
+
logger99.info(
|
|
75676
76628
|
{ total: this.handlers.size, periodic },
|
|
75677
76629
|
"[task-scheduler] Started"
|
|
75678
76630
|
);
|
|
@@ -75684,12 +76636,12 @@ var TaskScheduler = class {
|
|
|
75684
76636
|
setTimeout(() => {
|
|
75685
76637
|
if (!this.started) return;
|
|
75686
76638
|
this.seedNextCronTick(reg).then(
|
|
75687
|
-
() =>
|
|
76639
|
+
() => logger99.info(
|
|
75688
76640
|
{ taskName: reg.name, cron: reg.cron, attempt: i + 1 },
|
|
75689
76641
|
"[task-scheduler] Recovered cron seed in background"
|
|
75690
76642
|
)
|
|
75691
76643
|
).catch((err) => {
|
|
75692
|
-
|
|
76644
|
+
logger99.error(
|
|
75693
76645
|
{ err, taskName: reg.name, cron: reg.cron, attempt: i + 1 },
|
|
75694
76646
|
"[task-scheduler] Background cron seed retry failed"
|
|
75695
76647
|
);
|
|
@@ -75712,7 +76664,7 @@ var TaskScheduler = class {
|
|
|
75712
76664
|
const data = job.data ?? { name: "", payload: {} };
|
|
75713
76665
|
const reg = this.handlers.get(data.name);
|
|
75714
76666
|
if (!reg) {
|
|
75715
|
-
|
|
76667
|
+
logger99.error(
|
|
75716
76668
|
{ taskName: data.name, runId: job.id },
|
|
75717
76669
|
"[task-scheduler] No handler registered for task; failing run"
|
|
75718
76670
|
);
|
|
@@ -76123,7 +77075,7 @@ async function assertSchemaUpToDate(sql, options) {
|
|
|
76123
77075
|
// src/server.ts
|
|
76124
77076
|
init_workspace();
|
|
76125
77077
|
dotenv2.config();
|
|
76126
|
-
var app2 = new
|
|
77078
|
+
var app2 = new Hono29();
|
|
76127
77079
|
var PACKAGE_REPO_ROOT2 = path11.resolve(
|
|
76128
77080
|
fileURLToPath8(new URL(".", import.meta.url)),
|
|
76129
77081
|
"../../.."
|