@aion0/forge 0.9.16 → 0.9.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/RELEASE_NOTES.md +12 -8
- package/app/api/connectors/[id]/settings/route.ts +68 -10
- package/app/api/connectors/[id]/test/route.ts +28 -5
- package/app/api/memory/blocks/route.ts +56 -0
- package/app/api/monitor/route.ts +2 -0
- package/app/chat/page.tsx +189 -2
- package/bin/forge-server.mjs +3 -2
- package/components/ConnectorsPanel.tsx +141 -1
- package/components/MonitorPanel.tsx +2 -0
- package/lib/chat/agent-loop.ts +39 -8
- package/lib/chat/build-memory-context.ts +91 -0
- package/lib/chat/llm/openai.ts +4 -1
- package/lib/chat/local-memory.ts +22 -5
- package/lib/chat/protocols/http.ts +198 -24
- package/lib/chat/session-store.ts +49 -0
- package/lib/chat/tool-dispatcher.ts +84 -7
- package/lib/chat-standalone.ts +6 -0
- package/lib/connectors/registry.ts +76 -18
- package/lib/connectors/types.ts +87 -1
- package/lib/help-docs/21-build-connector.md +139 -0
- package/lib/init.ts +16 -0
- package/lib/memory/compress-messages.ts +65 -0
- package/lib/memory/keys.ts +82 -0
- package/lib/memory/temper-summary.ts +485 -0
- package/lib/memory/token-estimate.ts +28 -0
- package/lib/memory-standalone.ts +108 -0
- package/package.json +1 -1
- package/scripts/test-memory-local.ts +139 -0
- package/scripts/test-memory-upsert.ts +106 -0
|
@@ -9,6 +9,7 @@ interface MonitorData {
|
|
|
9
9
|
telegram: { running: boolean; pid: string; startedAt?: string };
|
|
10
10
|
workspace: { running: boolean; pid: string; startedAt?: string };
|
|
11
11
|
browserBridge?: { running: boolean; pid: string; startedAt?: string };
|
|
12
|
+
memory?: { running: boolean; pid: string; startedAt?: string };
|
|
12
13
|
chat?: {
|
|
13
14
|
running: boolean;
|
|
14
15
|
pid: string;
|
|
@@ -60,6 +61,7 @@ export default function MonitorPanel({ onClose }: { onClose: () => void }) {
|
|
|
60
61
|
{ label: 'Telegram Bot', ...data.processes.telegram },
|
|
61
62
|
{ label: 'Workspace Daemon', ...data.processes.workspace },
|
|
62
63
|
...(data.processes.browserBridge ? [{ label: 'Browser Bridge', ...data.processes.browserBridge }] : []),
|
|
64
|
+
...(data.processes.memory ? [{ label: 'Memory Worker', ...data.processes.memory }] : []),
|
|
63
65
|
{ label: 'Tunnel', ...data.processes.tunnel },
|
|
64
66
|
].map(p => (
|
|
65
67
|
<div key={p.label} className="flex items-center gap-2 text-xs">
|
package/lib/chat/agent-loop.ts
CHANGED
|
@@ -16,16 +16,17 @@ import { loadSettings } from '../settings';
|
|
|
16
16
|
import {
|
|
17
17
|
appendMessage,
|
|
18
18
|
getSession,
|
|
19
|
-
|
|
19
|
+
listMessagesCapped,
|
|
20
20
|
} from './session-store';
|
|
21
21
|
import {
|
|
22
22
|
dispatchTool,
|
|
23
23
|
BUILTIN_TOOL_DEFS,
|
|
24
24
|
type BuiltinHandler,
|
|
25
25
|
} from './tool-dispatcher';
|
|
26
|
-
import { renderMemoryContext } from './temper';
|
|
27
26
|
import { getMemoryStore } from './memory-store';
|
|
27
|
+
import { buildMemoryContext } from './build-memory-context';
|
|
28
28
|
import { buildMemoryTools } from './memory-tools';
|
|
29
|
+
import { estimateTokens } from '../memory/token-estimate';
|
|
29
30
|
import {
|
|
30
31
|
listInstalledConnectors,
|
|
31
32
|
getConnector,
|
|
@@ -41,6 +42,28 @@ import type {
|
|
|
41
42
|
|
|
42
43
|
const MAX_ITERATIONS = 6;
|
|
43
44
|
const MAX_TOKENS = 16000;
|
|
45
|
+
// Working-window budgets for the LLM history. Capped by message count
|
|
46
|
+
// AND by token estimate (whichever hits first), see design §8. Older
|
|
47
|
+
// raw is summarized by the memory-standalone Temper Summary sub-task
|
|
48
|
+
// and recalled via buildMemoryContext as compact blocks instead.
|
|
49
|
+
const HISTORY_MSG_BUDGET = 60;
|
|
50
|
+
const HISTORY_TOKEN_BUDGET = 8000;
|
|
51
|
+
|
|
52
|
+
// After clipping to last N, the first kept message may be a tool_result
|
|
53
|
+
// whose tool_use was cut. Anthropic/OpenAI both reject that, so drop
|
|
54
|
+
// leading tool_result-bearing user messages until the slice starts clean.
|
|
55
|
+
function trimOrphanToolResults(history: Message[]): Message[] {
|
|
56
|
+
let i = 0;
|
|
57
|
+
while (i < history.length) {
|
|
58
|
+
const m = history[i];
|
|
59
|
+
const hasToolResult = m.role === 'user'
|
|
60
|
+
&& Array.isArray(m.blocks)
|
|
61
|
+
&& m.blocks.some((b) => (b as any).type === 'tool_result');
|
|
62
|
+
if (!hasToolResult) break;
|
|
63
|
+
i += 1;
|
|
64
|
+
}
|
|
65
|
+
return i === 0 ? history : history.slice(i);
|
|
66
|
+
}
|
|
44
67
|
|
|
45
68
|
export interface AgentEvent {
|
|
46
69
|
type:
|
|
@@ -59,7 +82,7 @@ export type AgentCallbacks = {
|
|
|
59
82
|
onEvent: (event: AgentEvent) => void;
|
|
60
83
|
};
|
|
61
84
|
|
|
62
|
-
interface ProviderResolution {
|
|
85
|
+
export interface ProviderResolution {
|
|
63
86
|
name: string;
|
|
64
87
|
type: 'anthropic' | 'openai';
|
|
65
88
|
apiKey: string;
|
|
@@ -126,7 +149,7 @@ export function pickApiKey(profile: { apiKey?: string; env?: Record<string, stri
|
|
|
126
149
|
return env.OPENAI_API_KEY || '';
|
|
127
150
|
}
|
|
128
151
|
|
|
129
|
-
function resolveProvider(sessionProvider: string | null, sessionModel: string | null): ProviderResolution | { error: string } {
|
|
152
|
+
export function resolveProvider(sessionProvider: string | null, sessionModel: string | null): ProviderResolution | { error: string } {
|
|
130
153
|
const settings = loadSettings();
|
|
131
154
|
const agents = settings.agents || {};
|
|
132
155
|
|
|
@@ -372,18 +395,24 @@ export async function runTurn(args: RunTurnArgs): Promise<{ ok: boolean; error?:
|
|
|
372
395
|
for (const t of memTools) memHandlers[t.def.name] = t.handle;
|
|
373
396
|
|
|
374
397
|
if (memStore.enabled) {
|
|
375
|
-
|
|
398
|
+
// Inspector strip (memory_status event) wants the full inventory —
|
|
399
|
+
// keep its own listBlocks call. The prompt-injection text comes
|
|
400
|
+
// from buildMemoryContext which excludes internal bookkeeping
|
|
401
|
+
// (cursor / health) and combines pinned + query-driven retrieval
|
|
402
|
+
// hits in one pass.
|
|
403
|
+
const [bp, ba, sp, ctx] = await Promise.allSettled([
|
|
376
404
|
memStore.listBlocks({ pinned: true, scope: 'both' }),
|
|
377
405
|
memStore.listBlocks({ scope: 'both' }),
|
|
378
406
|
memStore.search(args.userText, 8),
|
|
407
|
+
buildMemoryContext({ store: memStore, currentUserMessage: args.userText }),
|
|
379
408
|
]);
|
|
380
409
|
const pinnedBlocks = bp.status === 'fulfilled' ? bp.value : [];
|
|
381
410
|
const allBlocks = ba.status === 'fulfilled' ? ba.value : [];
|
|
382
411
|
const searchHits = sp.status === 'fulfilled' ? sp.value : [];
|
|
383
|
-
const firstErr = [bp, ba, sp].find((r) => r.status === 'rejected') as PromiseRejectedResult | undefined;
|
|
412
|
+
const firstErr = [bp, ba, sp, ctx].find((r) => r.status === 'rejected') as PromiseRejectedResult | undefined;
|
|
384
413
|
const memError = firstErr ? (firstErr.reason instanceof Error ? firstErr.reason.message : String(firstErr.reason)) : undefined;
|
|
385
414
|
|
|
386
|
-
memContext =
|
|
415
|
+
memContext = ctx.status === 'fulfilled' ? ctx.value.text : '';
|
|
387
416
|
|
|
388
417
|
cb({
|
|
389
418
|
type: 'memory_status',
|
|
@@ -470,7 +499,9 @@ export async function runTurn(args: RunTurnArgs): Promise<{ ok: boolean; error?:
|
|
|
470
499
|
while (iter < MAX_ITERATIONS) {
|
|
471
500
|
iter += 1;
|
|
472
501
|
|
|
473
|
-
const history =
|
|
502
|
+
const history = trimOrphanToolResults(
|
|
503
|
+
listMessagesCapped(args.sessionId, HISTORY_MSG_BUDGET, HISTORY_TOKEN_BUDGET, estimateTokens),
|
|
504
|
+
);
|
|
474
505
|
|
|
475
506
|
assistantBlocksAccum = [];
|
|
476
507
|
let currentTextBuf = '';
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* buildMemoryContext — assemble the memory chunk for the agent-loop
|
|
3
|
+
* system prompt.
|
|
4
|
+
*
|
|
5
|
+
* Wraps store.listBlocks (for pinned + recall) and store.search (for
|
|
6
|
+
* query-driven retrieval) and post-filters out internal bookkeeping
|
|
7
|
+
* blocks like the summarizer cursor/health by key prefix. The actual
|
|
8
|
+
* string rendering reuses renderMemoryContext(blocks, hits) — this
|
|
9
|
+
* helper is just the assembly + filtering layer so callers don't have
|
|
10
|
+
* to think about it.
|
|
11
|
+
*
|
|
12
|
+
* Why post-filter instead of extending MemoryStore.search/listBlocks
|
|
13
|
+
* with scope filters: the existing API is flat key/value across both
|
|
14
|
+
* backends (LocalMemoryStore + Temper) and we want zero changes there.
|
|
15
|
+
* Forge owns the key naming convention (see lib/memory/keys.ts), so we
|
|
16
|
+
* own the prefix-exclusion decision client-side.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import type { MemoryBlock, MemoryStore, SearchHit } from './memory-store';
|
|
20
|
+
import { renderMemoryContext } from './temper';
|
|
21
|
+
import { INTERNAL_KEY_PREFIXES } from '../memory/keys';
|
|
22
|
+
|
|
23
|
+
export interface BuildMemoryContextOpts {
|
|
24
|
+
store: MemoryStore;
|
|
25
|
+
/** Used as `store.search(query)` — typically the latest user message. */
|
|
26
|
+
currentUserMessage?: string;
|
|
27
|
+
/** Cap on hits returned from store.search. Default 6. */
|
|
28
|
+
topK?: number;
|
|
29
|
+
/** Cap on inlined pinned blocks. Default 50 (renderMemoryContext default). */
|
|
30
|
+
maxBlocks?: number;
|
|
31
|
+
/** Prefixes that mark internal-only blocks (cursor / health / etc).
|
|
32
|
+
* Defaults to lib/memory/keys.INTERNAL_KEY_PREFIXES. */
|
|
33
|
+
excludeKeyPrefixes?: readonly string[];
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface BuildMemoryContextResult {
|
|
37
|
+
text: string;
|
|
38
|
+
blocks: MemoryBlock[];
|
|
39
|
+
hits: SearchHit[];
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export async function buildMemoryContext(opts: BuildMemoryContextOpts): Promise<BuildMemoryContextResult> {
|
|
43
|
+
const {
|
|
44
|
+
store,
|
|
45
|
+
currentUserMessage,
|
|
46
|
+
topK = 6,
|
|
47
|
+
maxBlocks = 50,
|
|
48
|
+
excludeKeyPrefixes = INTERNAL_KEY_PREFIXES,
|
|
49
|
+
} = opts;
|
|
50
|
+
|
|
51
|
+
const blocks = filterInternal(
|
|
52
|
+
await safe(() => store.listBlocks({ pinned: true }), [] as MemoryBlock[]),
|
|
53
|
+
excludeKeyPrefixes,
|
|
54
|
+
).slice(0, maxBlocks);
|
|
55
|
+
|
|
56
|
+
const q = (currentUserMessage || '').trim();
|
|
57
|
+
let hits: SearchHit[] = [];
|
|
58
|
+
if (q) {
|
|
59
|
+
const rawHits = await safe(() => store.search(q, topK), [] as SearchHit[]);
|
|
60
|
+
hits = filterInternalHits(rawHits, excludeKeyPrefixes);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return { text: renderMemoryContext(blocks, hits), blocks, hits };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function filterInternal(blocks: MemoryBlock[], prefixes: readonly string[]): MemoryBlock[] {
|
|
67
|
+
if (prefixes.length === 0) return blocks;
|
|
68
|
+
return blocks.filter((b) => !prefixes.some((p) => b.key.startsWith(p)));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function filterInternalHits(hits: SearchHit[], prefixes: readonly string[]): SearchHit[] {
|
|
72
|
+
if (prefixes.length === 0) return hits;
|
|
73
|
+
// SearchHit.id encodes its source: LocalMemoryStore returns 'block:<key>'
|
|
74
|
+
// for block-derived hits. Temper returns Graphiti UUIDs — those won't
|
|
75
|
+
// match prefixes, so they pass through (correct: Temper hits aren't
|
|
76
|
+
// direct block references).
|
|
77
|
+
return hits.filter((h) => {
|
|
78
|
+
if (!h.id?.startsWith('block:')) return true;
|
|
79
|
+
const key = h.id.slice('block:'.length);
|
|
80
|
+
return !prefixes.some((p) => key.startsWith(p));
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
async function safe<T>(fn: () => Promise<T>, fallback: T): Promise<T> {
|
|
85
|
+
try {
|
|
86
|
+
return await fn();
|
|
87
|
+
} catch (err) {
|
|
88
|
+
console.warn('[buildMemoryContext]', err instanceof Error ? err.message : err);
|
|
89
|
+
return fallback;
|
|
90
|
+
}
|
|
91
|
+
}
|
package/lib/chat/llm/openai.ts
CHANGED
|
@@ -77,11 +77,14 @@ export const openaiAdapter: LlmAdapter = {
|
|
|
77
77
|
};
|
|
78
78
|
}
|
|
79
79
|
|
|
80
|
+
// Some providers (litellm/vLLM) reject `tools: []` — they want the
|
|
81
|
+
// field omitted entirely when there are no tools.
|
|
82
|
+
const hasTools = Object.keys(tools).length > 0;
|
|
80
83
|
const result = streamText({
|
|
81
84
|
model: client(req.model),
|
|
82
85
|
system: req.system,
|
|
83
86
|
messages: historyToModelMessages(req.history),
|
|
84
|
-
tools,
|
|
87
|
+
...(hasTools ? { tools } : {}),
|
|
85
88
|
maxOutputTokens: req.maxTokens,
|
|
86
89
|
});
|
|
87
90
|
|
package/lib/chat/local-memory.ts
CHANGED
|
@@ -133,26 +133,43 @@ export class LocalMemoryStore implements MemoryStore {
|
|
|
133
133
|
const q = (query || '').trim();
|
|
134
134
|
if (!q) return [];
|
|
135
135
|
const cap = Math.min(50, Math.max(1, limit));
|
|
136
|
-
|
|
136
|
+
// Tokenize on whitespace and OR-match. Natural-language queries
|
|
137
|
+
// like "tell me about the X" can't be AND-matched (stop words
|
|
138
|
+
// wouldn't appear in stored content), so OR keeps recall useful.
|
|
139
|
+
// Drop tokens shorter than 3 chars to avoid runaway noise. If
|
|
140
|
+
// every token is too short, fall back to a single-substring match
|
|
141
|
+
// on the raw query.
|
|
142
|
+
const allTokens = q.split(/\s+/).filter((t) => t.length > 0);
|
|
143
|
+
const tokens = allTokens.filter((t) => t.length >= 3);
|
|
144
|
+
const useTokens = tokens.length > 0 ? tokens : [q];
|
|
145
|
+
const likes = useTokens.map((t) => `%${t.replace(/[%_]/g, (m) => '\\' + m)}%`);
|
|
137
146
|
const conn = db();
|
|
138
147
|
|
|
148
|
+
const blockWhere = useTokens
|
|
149
|
+
.map(() => `(value LIKE ? ESCAPE '\\' OR key LIKE ? ESCAPE '\\' OR description LIKE ? ESCAPE '\\')`)
|
|
150
|
+
.join(' OR ');
|
|
151
|
+
const blockParams: unknown[] = [this.ns];
|
|
152
|
+
for (const like of likes) { blockParams.push(like, like, like); }
|
|
153
|
+
blockParams.push(cap);
|
|
139
154
|
const blockHits = conn.prepare(
|
|
140
155
|
`SELECT key, value, description, updated_at
|
|
141
156
|
FROM memory_blocks
|
|
142
157
|
WHERE ns = ?
|
|
143
|
-
AND (
|
|
158
|
+
AND (${blockWhere})
|
|
144
159
|
ORDER BY pinned DESC, updated_at DESC
|
|
145
160
|
LIMIT ?`,
|
|
146
|
-
).all(
|
|
161
|
+
).all(...blockParams) as Array<Pick<BlockRow, 'key' | 'value' | 'description' | 'updated_at'>>;
|
|
147
162
|
|
|
163
|
+
const episodeWhere = useTokens.map(() => `content LIKE ? ESCAPE '\\'`).join(' OR ');
|
|
164
|
+
const episodeParams: unknown[] = [this.ns, ...likes, cap];
|
|
148
165
|
const episodeHits = conn.prepare(
|
|
149
166
|
`SELECT id, content, reference_time, created_at
|
|
150
167
|
FROM memory_episodes
|
|
151
168
|
WHERE ns = ?
|
|
152
|
-
AND
|
|
169
|
+
AND (${episodeWhere})
|
|
153
170
|
ORDER BY created_at DESC
|
|
154
171
|
LIMIT ?`,
|
|
155
|
-
).all(
|
|
172
|
+
).all(...episodeParams) as Array<Pick<EpisodeRow, 'id' | 'content' | 'reference_time' | 'created_at'>>;
|
|
156
173
|
|
|
157
174
|
const hits: SearchHit[] = [];
|
|
158
175
|
for (const b of blockHits) {
|
|
@@ -15,13 +15,20 @@
|
|
|
15
15
|
* is_error so the LLM can react.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
import type { HttpRequestSpec, ConnectorTool } from '../../connectors/types';
|
|
18
|
+
import type { HttpRequestSpec, ConnectorTool, ConnectorAuth, ConnectorFieldSchema } from '../../connectors/types';
|
|
19
19
|
import { expandAllTokens } from '../../plugins/templates';
|
|
20
20
|
|
|
21
21
|
export interface HttpProtocolArgs {
|
|
22
22
|
tool: ConnectorTool;
|
|
23
23
|
settings: Record<string, any>;
|
|
24
24
|
args: Record<string, any>;
|
|
25
|
+
/**
|
|
26
|
+
* Connector-level auth. Tool-level `tool.auth` takes precedence.
|
|
27
|
+
* Forge resolves the scheme into the right header/query at dispatch
|
|
28
|
+
* time so manifests don't have to hand-craft Authorization headers
|
|
29
|
+
* or base64-encode credentials.
|
|
30
|
+
*/
|
|
31
|
+
connectorAuth?: ConnectorAuth;
|
|
25
32
|
/**
|
|
26
33
|
* When true, return the full response body without the 8KB cap. Used by
|
|
27
34
|
* the Jobs scheduler — it parses JSON, not feeds the response into an
|
|
@@ -53,22 +60,42 @@ function expandObjectLeaves(obj: any, settings: Record<string, any>, args: Recor
|
|
|
53
60
|
}
|
|
54
61
|
|
|
55
62
|
/**
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
63
|
+
* Encode a string value per its parameter's `url_encoding` declaration.
|
|
64
|
+
* Default `uri_component` matches encodeURIComponent (slashes encoded).
|
|
65
|
+
* `none` is raw, for pre-formatted paths (e.g. Jenkins folder paths
|
|
66
|
+
* `job/team/job/build`). `path_segments` encodes each `/`-separated
|
|
67
|
+
* piece but preserves the slashes — good for human-readable paths
|
|
68
|
+
* that contain spaces or unicode.
|
|
69
|
+
*/
|
|
70
|
+
function encodePathValue(raw: string, mode: ConnectorFieldSchema['url_encoding'] | undefined): string {
|
|
71
|
+
switch (mode) {
|
|
72
|
+
case 'none':
|
|
73
|
+
return raw;
|
|
74
|
+
case 'path_segments':
|
|
75
|
+
return raw.split('/').map(encodeURIComponent).join('/');
|
|
76
|
+
case 'uri_component':
|
|
77
|
+
case undefined:
|
|
78
|
+
default:
|
|
79
|
+
return encodeURIComponent(raw);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Expand `{args.X}` placeholders in a URL path. Each arg's encoding is
|
|
85
|
+
* decided by its parameter schema's `url_encoding` field (default
|
|
86
|
+
* `uri_component` — see `encodePathValue` for the modes). `{settings.X}`
|
|
87
|
+
* is NOT encoded — `{settings.base_url}` is the scheme + host (with its
|
|
88
|
+
* own `://` and `/`), which must stay literal.
|
|
67
89
|
*/
|
|
68
|
-
function expandUrlPath(
|
|
90
|
+
function expandUrlPath(
|
|
91
|
+
template: string,
|
|
92
|
+
settings: Record<string, any>,
|
|
93
|
+
args: Record<string, any>,
|
|
94
|
+
paramSchemas?: Record<string, ConnectorFieldSchema>,
|
|
95
|
+
): string {
|
|
69
96
|
// First handle settings.* with raw substitution (keeps base_url intact).
|
|
70
97
|
let out = expandAllTokens(template, settings, {});
|
|
71
|
-
// Then handle args.* with URL
|
|
98
|
+
// Then handle args.* with per-parameter URL encoding.
|
|
72
99
|
out = out.replace(/\{args\.([^{}]+)\}/g, (full, rawKey) => {
|
|
73
100
|
const path = String(rawKey).trim().split('.');
|
|
74
101
|
let v: any = args;
|
|
@@ -78,13 +105,23 @@ function expandUrlPath(template: string, settings: Record<string, any>, args: Re
|
|
|
78
105
|
}
|
|
79
106
|
if (v == null) return full;
|
|
80
107
|
const s = typeof v === 'string' ? v : (typeof v === 'number' || typeof v === 'boolean' ? String(v) : JSON.stringify(v));
|
|
81
|
-
|
|
108
|
+
// Encoding mode comes from the top-level parameter's schema. Nested
|
|
109
|
+
// arg paths inherit their root parameter's encoding — common case is
|
|
110
|
+
// a flat scalar parameter so this matters rarely.
|
|
111
|
+
const rootParam = path[0];
|
|
112
|
+
const mode = paramSchemas?.[rootParam]?.url_encoding;
|
|
113
|
+
return encodePathValue(s, mode);
|
|
82
114
|
});
|
|
83
115
|
return out;
|
|
84
116
|
}
|
|
85
117
|
|
|
86
|
-
function buildUrl(
|
|
87
|
-
|
|
118
|
+
function buildUrl(
|
|
119
|
+
spec: HttpRequestSpec,
|
|
120
|
+
settings: Record<string, any>,
|
|
121
|
+
args: Record<string, any>,
|
|
122
|
+
paramSchemas?: Record<string, ConnectorFieldSchema>,
|
|
123
|
+
): string {
|
|
124
|
+
const base = expandUrlPath(spec.url, settings, args, paramSchemas);
|
|
88
125
|
if (!spec.query) return base;
|
|
89
126
|
const url = new URL(base);
|
|
90
127
|
for (const [k, raw] of Object.entries(spec.query)) {
|
|
@@ -98,6 +135,48 @@ function buildUrl(spec: HttpRequestSpec, settings: Record<string, any>, args: Re
|
|
|
98
135
|
return url.toString();
|
|
99
136
|
}
|
|
100
137
|
|
|
138
|
+
/**
|
|
139
|
+
* Apply a connector/tool auth scheme onto an outbound request. Resolves
|
|
140
|
+
* templated `{settings.*}` inside auth values, base64-encodes basic
|
|
141
|
+
* credentials, and chooses between header / query placement. The URL is
|
|
142
|
+
* passed by reference (returned as a new string if the auth scheme
|
|
143
|
+
* appends a query param). Centralised so the chat dispatcher and the
|
|
144
|
+
* connector-test probe stay consistent.
|
|
145
|
+
*/
|
|
146
|
+
export function applyAuth(
|
|
147
|
+
url: string,
|
|
148
|
+
headers: Headers,
|
|
149
|
+
auth: ConnectorAuth | undefined,
|
|
150
|
+
settings: Record<string, any>,
|
|
151
|
+
args: Record<string, any> = {},
|
|
152
|
+
): string {
|
|
153
|
+
if (!auth || auth.type === 'none') return url;
|
|
154
|
+
const exp = (s: string) => expandAllTokens(String(s ?? ''), settings, args);
|
|
155
|
+
switch (auth.type) {
|
|
156
|
+
case 'basic': {
|
|
157
|
+
const u = exp(auth.username);
|
|
158
|
+
const p = exp(auth.password);
|
|
159
|
+
const token = Buffer.from(`${u}:${p}`, 'utf-8').toString('base64');
|
|
160
|
+
headers.set('Authorization', `Basic ${token}`);
|
|
161
|
+
return url;
|
|
162
|
+
}
|
|
163
|
+
case 'bearer': {
|
|
164
|
+
headers.set('Authorization', `Bearer ${exp(auth.token)}`);
|
|
165
|
+
return url;
|
|
166
|
+
}
|
|
167
|
+
case 'header': {
|
|
168
|
+
headers.set(auth.name, exp(auth.value));
|
|
169
|
+
return url;
|
|
170
|
+
}
|
|
171
|
+
case 'query': {
|
|
172
|
+
const u = new URL(url);
|
|
173
|
+
u.searchParams.set(auth.name, exp(auth.value));
|
|
174
|
+
return u.toString();
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return url;
|
|
178
|
+
}
|
|
179
|
+
|
|
101
180
|
function buildHeaders(spec: HttpRequestSpec, settings: Record<string, any>, args: Record<string, any>): Headers {
|
|
102
181
|
const h = new Headers();
|
|
103
182
|
if (spec.headers) {
|
|
@@ -109,12 +188,102 @@ function buildHeaders(spec: HttpRequestSpec, settings: Record<string, any>, args
|
|
|
109
188
|
}
|
|
110
189
|
|
|
111
190
|
function buildBody(spec: HttpRequestSpec, settings: Record<string, any>, args: Record<string, any>): { body?: string; contentType?: string } {
|
|
112
|
-
if (spec.body
|
|
113
|
-
|
|
114
|
-
|
|
191
|
+
if (spec.body != null) {
|
|
192
|
+
if (typeof spec.body === 'string') {
|
|
193
|
+
return { body: expandAllTokens(spec.body, settings, args) };
|
|
194
|
+
}
|
|
195
|
+
const obj = expandObjectLeaves(spec.body, settings, args);
|
|
196
|
+
return { body: JSON.stringify(obj), contentType: 'application/json' };
|
|
197
|
+
}
|
|
198
|
+
if (spec.body_form != null || spec.body_form_inject != null || spec.body_form_inject_from != null) {
|
|
199
|
+
return buildFormBody(spec.body_form, spec.body_form_inject, spec.body_form_inject_from, settings, args);
|
|
115
200
|
}
|
|
116
|
-
|
|
117
|
-
|
|
201
|
+
return {};
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Serialise an object into application/x-www-form-urlencoded body.
|
|
206
|
+
* The spec value can be:
|
|
207
|
+
* - a literal placeholder `{args.NAME}` — resolved to the named arg
|
|
208
|
+
* (must be a plain object); used by Jenkins trigger_build to take a
|
|
209
|
+
* dynamic `params` map of build parameters.
|
|
210
|
+
* - an inline object whose leaves get template-expanded — used when
|
|
211
|
+
* the form keys are static.
|
|
212
|
+
* - any other string — treated as a JSON-string template, parsed, then
|
|
213
|
+
* serialised (less common, but lets manifests build the body inline).
|
|
214
|
+
*
|
|
215
|
+
* null/undefined values are dropped (no empty `KEY=`). Non-string values
|
|
216
|
+
* are stringified.
|
|
217
|
+
*/
|
|
218
|
+
function buildFormBody(spec: string | Record<string, unknown> | undefined, inject: Record<string, string> | undefined, injectFrom: string | undefined, settings: Record<string, any>, args: Record<string, any>): { body?: string; contentType?: string } {
|
|
219
|
+
let obj: any = null;
|
|
220
|
+
if (spec != null) {
|
|
221
|
+
if (typeof spec === 'string') {
|
|
222
|
+
const m = spec.match(/^\{args\.([^{}]+)\}$/);
|
|
223
|
+
if (m) {
|
|
224
|
+
obj = args[m[1]];
|
|
225
|
+
// LLMs frequently JSON-stringify an object arg even when the
|
|
226
|
+
// tool schema declares it as `type: json`. Parse it back so the
|
|
227
|
+
// form serialisation works either way.
|
|
228
|
+
if (typeof obj === 'string') {
|
|
229
|
+
try { obj = JSON.parse(obj); } catch { /* leave as null below */ }
|
|
230
|
+
}
|
|
231
|
+
} else {
|
|
232
|
+
const expanded = expandAllTokens(spec, settings, args);
|
|
233
|
+
try { obj = JSON.parse(expanded); } catch { obj = null; }
|
|
234
|
+
}
|
|
235
|
+
} else {
|
|
236
|
+
obj = expandObjectLeaves(spec, settings, args);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
if (obj == null) obj = {};
|
|
240
|
+
if (typeof obj !== 'object' || Array.isArray(obj)) obj = {};
|
|
241
|
+
|
|
242
|
+
// Server-side inject — typically secrets pulled from settings.
|
|
243
|
+
// BOTH key and value are templated (against settings only, NOT args,
|
|
244
|
+
// so the LLM can't shadow injected keys). Templated keys let one
|
|
245
|
+
// manifest target different Jenkins jobs whose param names vary —
|
|
246
|
+
// each instance config sets the key name (e.g. TOKEN_PASSWORD)
|
|
247
|
+
// alongside the value source (e.g. {settings.gitlab_pat}). Entries
|
|
248
|
+
// where the key OR value comes back empty / unresolved get dropped.
|
|
249
|
+
if (inject) {
|
|
250
|
+
for (const [rawKey, rawVal] of Object.entries(inject)) {
|
|
251
|
+
const k = expandAllTokens(String(rawKey), settings, {});
|
|
252
|
+
if (!k || /\{(settings|args)\./.test(k)) continue;
|
|
253
|
+
const v = expandAllTokens(String(rawVal), settings, {});
|
|
254
|
+
if (!v || /\{(settings|args)\./.test(v)) continue;
|
|
255
|
+
obj[k] = v;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
// body_form_inject_from — settings[X] is expected to be an
|
|
260
|
+
// `instances`-shaped array (each row { name, value, ... }). Inject
|
|
261
|
+
// every row as one form key/value pair. Lets the connector defer
|
|
262
|
+
// the actual key+value choices to per-user instance config without
|
|
263
|
+
// hardcoding them in the manifest. Rows with empty name or value
|
|
264
|
+
// are dropped.
|
|
265
|
+
if (injectFrom) {
|
|
266
|
+
let rows: any = (settings as any)[injectFrom];
|
|
267
|
+
if (typeof rows === 'string') {
|
|
268
|
+
try { rows = JSON.parse(rows); } catch { rows = null; }
|
|
269
|
+
}
|
|
270
|
+
if (Array.isArray(rows)) {
|
|
271
|
+
for (const row of rows) {
|
|
272
|
+
if (!row || typeof row !== 'object') continue;
|
|
273
|
+
const k = typeof row.name === 'string' ? row.name.trim() : '';
|
|
274
|
+
const v = typeof row.value === 'string' ? row.value : (row.value == null ? '' : String(row.value));
|
|
275
|
+
if (!k || !v) continue;
|
|
276
|
+
obj[k] = v;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
const usp = new URLSearchParams();
|
|
282
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
283
|
+
if (v == null) continue;
|
|
284
|
+
usp.append(k, typeof v === 'string' ? v : String(v));
|
|
285
|
+
}
|
|
286
|
+
return { body: usp.toString(), contentType: 'application/x-www-form-urlencoded' };
|
|
118
287
|
}
|
|
119
288
|
|
|
120
289
|
function truncate(s: string): { text: string; truncated: boolean; totalBytes: number } {
|
|
@@ -124,7 +293,7 @@ function truncate(s: string): { text: string; truncated: boolean; totalBytes: nu
|
|
|
124
293
|
return { text: slice, truncated: true, totalBytes: buf.byteLength };
|
|
125
294
|
}
|
|
126
295
|
|
|
127
|
-
export async function runHttp({ tool, settings, args, noTruncation }: HttpProtocolArgs): Promise<HttpProtocolResult> {
|
|
296
|
+
export async function runHttp({ tool, settings, args, connectorAuth, noTruncation }: HttpProtocolArgs): Promise<HttpProtocolResult> {
|
|
128
297
|
const spec = tool.request;
|
|
129
298
|
if (!spec || !spec.url) {
|
|
130
299
|
return { content: 'http tool missing `request.url`', is_error: true };
|
|
@@ -145,11 +314,16 @@ export async function runHttp({ tool, settings, args, noTruncation }: HttpProtoc
|
|
|
145
314
|
}
|
|
146
315
|
}
|
|
147
316
|
|
|
148
|
-
|
|
317
|
+
let url = buildUrl(spec, settings, argsWithDefaults, tool.parameters);
|
|
149
318
|
const headers = buildHeaders(spec, settings, argsWithDefaults);
|
|
150
319
|
const { body, contentType } = buildBody(spec, settings, argsWithDefaults);
|
|
151
320
|
if (body != null && contentType && !headers.has('content-type')) headers.set('content-type', contentType);
|
|
152
321
|
|
|
322
|
+
// Tool-level auth overrides connector-level. `{ type: 'none' }` is a
|
|
323
|
+
// valid override that disables auth entirely (public endpoint).
|
|
324
|
+
const effectiveAuth = tool.auth ?? connectorAuth;
|
|
325
|
+
url = applyAuth(url, headers, effectiveAuth, settings, argsWithDefaults);
|
|
326
|
+
|
|
153
327
|
const controller = new AbortController();
|
|
154
328
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
155
329
|
|
|
@@ -265,6 +265,55 @@ export function listMessages(session_id: string, opts?: { limit?: number; after_
|
|
|
265
265
|
return rows.map(rowToMessage);
|
|
266
266
|
}
|
|
267
267
|
|
|
268
|
+
/** Last N messages in chronological order — used by agent-loop to cap LLM context. */
|
|
269
|
+
export function listRecentMessages(session_id: string, limit: number): Message[] {
|
|
270
|
+
ensureSchema();
|
|
271
|
+
const rows = db().prepare(`
|
|
272
|
+
SELECT * FROM chat_messages WHERE session_id = ?
|
|
273
|
+
ORDER BY ts DESC LIMIT ?
|
|
274
|
+
`).all(session_id, limit) as MessageRow[];
|
|
275
|
+
return rows.map(rowToMessage).reverse();
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Take the most recent messages, stopping when either the message-count
|
|
280
|
+
* budget OR the token-estimate budget would be exceeded. Walks
|
|
281
|
+
* newest-first so the most recent dialogue is always kept; returns
|
|
282
|
+
* chronological order for the LLM history slot.
|
|
283
|
+
*
|
|
284
|
+
* msgBudget is enforced via SQL LIMIT (cheap). tokenBudget is enforced
|
|
285
|
+
* via the caller-supplied estimator (decoupled to avoid pulling the
|
|
286
|
+
* token-estimator into the storage layer).
|
|
287
|
+
*/
|
|
288
|
+
export function listMessagesCapped(
|
|
289
|
+
session_id: string,
|
|
290
|
+
msgBudget: number,
|
|
291
|
+
tokenBudget: number,
|
|
292
|
+
estimateTokens: (m: Message) => number,
|
|
293
|
+
): Message[] {
|
|
294
|
+
ensureSchema();
|
|
295
|
+
const cap = Math.max(1, Math.floor(msgBudget));
|
|
296
|
+
// Pull newest-first via SQL — bounded by msgBudget so we never load
|
|
297
|
+
// more rows than we could possibly keep.
|
|
298
|
+
const rows = db().prepare(`
|
|
299
|
+
SELECT * FROM chat_messages WHERE session_id = ?
|
|
300
|
+
ORDER BY ts DESC LIMIT ?
|
|
301
|
+
`).all(session_id, cap) as MessageRow[];
|
|
302
|
+
const newestFirst = rows.map(rowToMessage);
|
|
303
|
+
|
|
304
|
+
// Now apply tokenBudget walking newest → oldest. Always keep at
|
|
305
|
+
// least one (so an oversized last message doesn't strand the loop).
|
|
306
|
+
const kept: Message[] = [];
|
|
307
|
+
let used = 0;
|
|
308
|
+
for (const m of newestFirst) {
|
|
309
|
+
const cost = estimateTokens(m);
|
|
310
|
+
if (kept.length > 0 && used + cost > tokenBudget) break;
|
|
311
|
+
kept.push(m);
|
|
312
|
+
used += cost;
|
|
313
|
+
}
|
|
314
|
+
return kept.reverse();
|
|
315
|
+
}
|
|
316
|
+
|
|
268
317
|
export function deleteMessage(id: string): boolean {
|
|
269
318
|
ensureSchema();
|
|
270
319
|
const r = db().prepare(`DELETE FROM chat_messages WHERE id = ?`).run(id);
|