@andreprado/agentkit 0.1.0-alpha.17 → 0.1.0-alpha.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/docs/guides/add-channel.md +14 -8
- package/docs/guides/add-knowledge.md +10 -0
- package/docs/guides/add-managed-composio.md +43 -17
- package/docs/guides/channel-security.md +60 -39
- package/docs/guides/connect-discord.md +178 -0
- package/docs/guides/create-agent.md +13 -0
- package/docs/guides/debug-channel.md +147 -0
- package/docs/guides/improve-from-production.md +151 -0
- package/docs/guides/prepare-deploy.md +30 -14
- package/docs/guides/replay-production-traces.md +72 -0
- package/docs/guides/run-evals.md +18 -0
- package/docs/guides/security-rules.md +5 -5
- package/docs/guides/use-provider.md +11 -1
- package/docs/llms-full.txt +106 -15
- package/docs/llms.txt +22 -4
- package/package.json +1 -3
- package/src/cli/args.ts +23 -2
- package/src/cli/cloud-client.ts +75 -0
- package/src/cli/commands/channels.ts +139 -14
- package/src/cli/deploy-chat-ui.ts +146 -3
- package/src/cli/deploy-readiness.ts +57 -1
- package/src/cli/help.ts +32 -8
- package/src/cli/index.ts +447 -17
- package/src/create-project.ts +13 -2
- package/src/index.ts +46 -3
- package/src/providers/pi.ts +49 -15
- package/src/runtime/channel-test-harness.ts +4 -1
- package/src/runtime/channels/discord.ts +887 -0
- package/src/runtime/channels.ts +15 -0
- package/src/runtime/config.ts +39 -3
- package/src/runtime/core/manifest.ts +2 -0
- package/src/runtime/dev-server.ts +149 -8
- package/src/runtime/evals.ts +27 -6
- package/src/runtime/improve.ts +868 -0
- package/src/runtime/inspect.ts +1 -0
- package/src/runtime/integrations/composio.ts +168 -2
- package/src/runtime/knowledge/retrieve.ts +25 -5
- package/src/runtime/knowledge/schema.ts +45 -1
- package/src/runtime/runtime-contract.ts +54 -0
- package/src/runtime/targets/cloudflare/build.ts +479 -194
- package/src/runtime/targets/vps/deploy.ts +1 -1
- package/src/storage/sqlite.ts +7 -2
- package/src/templates/skills/agentkit-capsule/SKILL.md +8 -1
- package/src/templates/skills/agentkit-capsule/references/docs-router.md +1 -2
- package/src/templates/skills/agentkit-channels/SKILL.md +6 -1
- package/src/templates/skills/agentkit-channels/references/channel-debugging.md +2 -1
- package/src/templates/skills/agentkit-channels/references/discord.md +93 -0
- package/src/templates/skills/agentkit-deploy/SKILL.md +6 -0
- package/src/templates/skills/agentkit-evals/SKILL.md +12 -3
- package/src/templates/skills/agentkit-improve/SKILL.md +86 -0
- package/src/templates/skills/agentkit-improve/references/replay-side-effects.md +18 -0
- package/src/templates/skills/agentkit-improve/references/trace-packets.md +22 -0
- package/src/templates/skills/agentkit-improve/templates/regression.eval.md +18 -0
- package/src/templates/skills/agentkit-integrations/SKILL.md +12 -2
- package/src/templates/skills/agentkit-knowledge/SKILL.md +4 -1
- package/src/templates/skills/agentkit-provider/SKILL.md +4 -1
- package/src/templates/skills/agentkit-security/SKILL.md +3 -2
- package/src/templates/skills/agentkit-troubleshooting/SKILL.md +9 -0
- package/src/templates/support.ts +4 -2
- package/docs/guides/agentkit-skills-architecture.md +0 -472
- package/docs/guides/channels-implementation-map.md +0 -243
- package/docs/guides/channels-production-handoff.md +0 -118
- package/docs/portable-deploy-release-checklist.md +0 -41
package/src/runtime/channels.ts
CHANGED
|
@@ -49,6 +49,19 @@ export type ChannelExternalIdentity = {
|
|
|
49
49
|
redactedDisplay?: string;
|
|
50
50
|
};
|
|
51
51
|
|
|
52
|
+
export type ChannelReplyTarget =
|
|
53
|
+
| {
|
|
54
|
+
type: "discord_interaction";
|
|
55
|
+
applicationId: string;
|
|
56
|
+
interactionId: string;
|
|
57
|
+
interactionToken: string;
|
|
58
|
+
}
|
|
59
|
+
| {
|
|
60
|
+
type: "discord_message";
|
|
61
|
+
channelId: string;
|
|
62
|
+
messageId: string;
|
|
63
|
+
};
|
|
64
|
+
|
|
52
65
|
export type ChannelAudioAttachment = {
|
|
53
66
|
providerMediaId: string;
|
|
54
67
|
kind: "voice" | "audio";
|
|
@@ -77,6 +90,7 @@ export type NormalizedChannelEvent = ChannelAdapterId & {
|
|
|
77
90
|
receivedAt: string;
|
|
78
91
|
unsupportedReason?: string;
|
|
79
92
|
providerMetadata?: Record<string, unknown>;
|
|
93
|
+
replyTarget?: ChannelReplyTarget;
|
|
80
94
|
};
|
|
81
95
|
|
|
82
96
|
export type WebhookVerificationInput = RawWebhookEvent & {
|
|
@@ -98,6 +112,7 @@ export type ChannelSendInput = ChannelAdapterId & {
|
|
|
98
112
|
channelId: string;
|
|
99
113
|
conversationId: string;
|
|
100
114
|
externalIdentity: ChannelExternalIdentity;
|
|
115
|
+
replyTarget?: ChannelReplyTarget;
|
|
101
116
|
message: NormalizedChannelMessage;
|
|
102
117
|
secrets: Record<string, string>;
|
|
103
118
|
};
|
package/src/runtime/config.ts
CHANGED
|
@@ -13,6 +13,7 @@ import type {
|
|
|
13
13
|
AgentChannel,
|
|
14
14
|
ChannelProvider,
|
|
15
15
|
ChannelType,
|
|
16
|
+
DiscordChannelMode,
|
|
16
17
|
TelegramAllowedUpdate,
|
|
17
18
|
AgentKnowledgeEmbeddingProvider,
|
|
18
19
|
ChannelBufferMode,
|
|
@@ -29,10 +30,11 @@ const AGENT_RUNTIMES = new Set<AgentRuntime>(["local", "edge"]);
|
|
|
29
30
|
const PROVIDERS = new Set<AgentProviderName>(["test", "openai", "anthropic", "openrouter", "custom"]);
|
|
30
31
|
const ACCESS_MODES = new Set<AccessMode>(["private", "token", "public"]);
|
|
31
32
|
const TOOL_VISIBILITIES = new Set<ToolVisibility>(["user", "internal"]);
|
|
32
|
-
const CHANNEL_TYPES = new Set<ChannelType>(["website", "telegram", "whatsapp"]);
|
|
33
|
-
const CHANNEL_PROVIDERS = new Set<ChannelProvider>(["agentkit", "telegram", "zapster", "meta"]);
|
|
33
|
+
const CHANNEL_TYPES = new Set<ChannelType>(["website", "telegram", "whatsapp", "discord"]);
|
|
34
|
+
const CHANNEL_PROVIDERS = new Set<ChannelProvider>(["agentkit", "telegram", "zapster", "meta", "discord"]);
|
|
34
35
|
const CHANNEL_BUFFER_MODES = new Set<ChannelBufferMode>(["off", "debounce"]);
|
|
35
36
|
const CHANNEL_AUDIO_MODES = new Set<ChannelAudioMode>(["off", "reject", "transcribe"]);
|
|
37
|
+
const DISCORD_CHANNEL_MODES = new Set<DiscordChannelMode>(["interactions", "bot"]);
|
|
36
38
|
const TELEGRAM_ALLOWED_UPDATES = new Set<TelegramAllowedUpdate>(["message"]);
|
|
37
39
|
const KNOWLEDGE_EMBEDDING_PROVIDERS = new Set<AgentKnowledgeEmbeddingProvider>(["none", "test", "openai"]);
|
|
38
40
|
const TRANSCRIPTION_PROVIDERS = new Set<TranscriptionProviderName>(["test", "openai", "groq"]);
|
|
@@ -260,6 +262,10 @@ function validateAgentConfig(value: unknown): asserts value is AgentConfig {
|
|
|
260
262
|
throw invalidConfig("storage.path must be a string when provided.");
|
|
261
263
|
}
|
|
262
264
|
|
|
265
|
+
if (value.storage.driver === "sqlite" && typeof value.storage.path !== "string") {
|
|
266
|
+
throw invalidConfig('storage.path is required when storage.driver is "sqlite".');
|
|
267
|
+
}
|
|
268
|
+
|
|
263
269
|
validateDatabaseConfig(value.storage.database);
|
|
264
270
|
validateFilesConfig(value.storage.files);
|
|
265
271
|
}
|
|
@@ -371,6 +377,10 @@ function validateComposioManagedIntegration(value: AgentIntegration, index: numb
|
|
|
371
377
|
if (value.tags !== undefined) {
|
|
372
378
|
validateIntegrationTags(value.tags, `integrations[${index}].tags`);
|
|
373
379
|
}
|
|
380
|
+
|
|
381
|
+
if (value.confirmExternalWrites !== undefined && typeof value.confirmExternalWrites !== "boolean") {
|
|
382
|
+
throw invalidConfig(`integrations[${index}].confirmExternalWrites must be a boolean when provided.`);
|
|
383
|
+
}
|
|
374
384
|
}
|
|
375
385
|
|
|
376
386
|
function validateIntegrationTags(value: unknown, label: string): void {
|
|
@@ -606,6 +616,12 @@ function validateChannel(value: unknown, index: number, hasTranscriptionConfig:
|
|
|
606
616
|
validateSecretNames(value.secrets, `channels[${index}].secrets`);
|
|
607
617
|
validateChannelLimits(value.limits, index);
|
|
608
618
|
validateChannelBuffer(value.buffer, index);
|
|
619
|
+
|
|
620
|
+
if (value.type === "discord") {
|
|
621
|
+
validateDiscordChannel(value, index);
|
|
622
|
+
return;
|
|
623
|
+
}
|
|
624
|
+
|
|
609
625
|
validateChannelAudio(value.audio, index, hasTranscriptionConfig);
|
|
610
626
|
|
|
611
627
|
if (value.type === "website") {
|
|
@@ -678,7 +694,7 @@ function validateTelegramChannel(value: Record<string, unknown>, index: number):
|
|
|
678
694
|
|
|
679
695
|
function validateWhatsappChannel(value: Record<string, unknown>, index: number): void {
|
|
680
696
|
if (value.type !== "whatsapp") {
|
|
681
|
-
throw invalidConfig(`channels[${index}].type must be "website", "telegram", or "
|
|
697
|
+
throw invalidConfig(`channels[${index}].type must be "website", "telegram", "whatsapp", or "discord".`);
|
|
682
698
|
}
|
|
683
699
|
|
|
684
700
|
if (value.provider === "zapster") {
|
|
@@ -702,6 +718,26 @@ function validateWhatsappChannel(value: Record<string, unknown>, index: number):
|
|
|
702
718
|
throw invalidConfig(`channels[${index}].provider must be "zapster" or "meta" when type is "whatsapp".`);
|
|
703
719
|
}
|
|
704
720
|
|
|
721
|
+
function validateDiscordChannel(value: Record<string, unknown>, index: number): void {
|
|
722
|
+
if (value.provider !== "discord") {
|
|
723
|
+
throw invalidConfig(`channels[${index}].provider must be "discord" when type is "discord".`);
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
if (value.mode !== undefined && !DISCORD_CHANNEL_MODES.has(value.mode as DiscordChannelMode)) {
|
|
727
|
+
throw invalidConfig(`channels[${index}].mode must be "interactions" or "bot" when type is "discord".`);
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
if (value.audio !== undefined) {
|
|
731
|
+
throw invalidConfig(`channels[${index}].audio is not supported when type is "discord".`);
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
expectRequiredSecrets(
|
|
735
|
+
value.secrets,
|
|
736
|
+
value.mode === "bot" ? ["DISCORD_BOT_TOKEN"] : ["DISCORD_PUBLIC_KEY"],
|
|
737
|
+
`channels[${index}]`,
|
|
738
|
+
);
|
|
739
|
+
}
|
|
740
|
+
|
|
705
741
|
function validateChannelLimits(value: unknown, index: number): void {
|
|
706
742
|
if (value === undefined) {
|
|
707
743
|
return;
|
|
@@ -40,6 +40,7 @@ export type SharedAgentManifest = {
|
|
|
40
40
|
toolkits: string[];
|
|
41
41
|
tools: string[];
|
|
42
42
|
userId: string;
|
|
43
|
+
confirmExternalWrites: boolean;
|
|
43
44
|
}>;
|
|
44
45
|
channels: NonNullable<AgentConfig["channels"]>;
|
|
45
46
|
transcription: {
|
|
@@ -174,6 +175,7 @@ export function buildSharedAgentManifest(
|
|
|
174
175
|
toolkits: integration.toolkits,
|
|
175
176
|
tools: integration.allowedTools,
|
|
176
177
|
userId: integration.userId,
|
|
178
|
+
confirmExternalWrites: integration.confirmExternalWrites !== false,
|
|
177
179
|
})),
|
|
178
180
|
channels: capsule.config.channels ?? [],
|
|
179
181
|
transcription: {
|
|
@@ -8,6 +8,7 @@ import { Readable } from "node:stream";
|
|
|
8
8
|
import { openCapsuleStore, type ConversationRecord } from "../storage/sqlite";
|
|
9
9
|
import type { AgentChannel, ChannelProvider, ChannelType } from "../index";
|
|
10
10
|
import type { ChannelAdapter, NormalizedChannelMessage, RawWebhookEvent } from "./channels";
|
|
11
|
+
import { discordChannelAdapter, isDiscordPingInteraction } from "./channels/discord";
|
|
11
12
|
import { telegramChannelAdapter } from "./channels/telegram";
|
|
12
13
|
import { websiteChannelAdapter } from "./channels/website";
|
|
13
14
|
import { metaWhatsappChannelAdapter } from "./channels/whatsapp-meta";
|
|
@@ -50,6 +51,8 @@ export type AgentDevServer = {
|
|
|
50
51
|
|
|
51
52
|
const DEFAULT_PORT = 4123;
|
|
52
53
|
const DEFAULT_HOSTNAME = "localhost";
|
|
54
|
+
const CHANNEL_DEDUPE_TTL_MS = 6 * 60 * 60 * 1000;
|
|
55
|
+
const CHANNEL_DEDUPE_MAX_KEYS = 10_000;
|
|
53
56
|
|
|
54
57
|
export type AgentDevServerAccessPolicy = "local" | "enforce";
|
|
55
58
|
|
|
@@ -58,7 +61,7 @@ type DevHttpServerOptions = {
|
|
|
58
61
|
};
|
|
59
62
|
|
|
60
63
|
type DevChannelRuntimeState = {
|
|
61
|
-
dedupeKeys:
|
|
64
|
+
dedupeKeys: BoundedDedupeCache;
|
|
62
65
|
buffers: Map<string, DevChannelBuffer>;
|
|
63
66
|
};
|
|
64
67
|
|
|
@@ -155,7 +158,7 @@ async function listenForDevServer(
|
|
|
155
158
|
|
|
156
159
|
function createDevHttpServer(capsule: LoadedAgentCapsule, options: DevHttpServerOptions): Server {
|
|
157
160
|
const channelRuntime: DevChannelRuntimeState = {
|
|
158
|
-
dedupeKeys: new
|
|
161
|
+
dedupeKeys: new BoundedDedupeCache(CHANNEL_DEDUPE_MAX_KEYS, CHANNEL_DEDUPE_TTL_MS),
|
|
159
162
|
buffers: new Map<string, DevChannelBuffer>(),
|
|
160
163
|
};
|
|
161
164
|
const capsuleRoot = capsule.root;
|
|
@@ -316,7 +319,7 @@ async function handleDevServerRequest(
|
|
|
316
319
|
return await handleLocalFile(capsule, request, decodeURIComponent(fileMatch[1]));
|
|
317
320
|
}
|
|
318
321
|
|
|
319
|
-
const channelMatch = route.match(/^\/channels\/([^/]+)\/(website|telegram|whatsapp)\/([^/]+)\/webhook$/);
|
|
322
|
+
const channelMatch = route.match(/^\/channels\/([^/]+)\/(website|telegram|whatsapp|discord)\/([^/]+)\/webhook$/);
|
|
320
323
|
|
|
321
324
|
if (channelMatch) {
|
|
322
325
|
return await handlePortableChannel(capsule, request, channelRuntime, {
|
|
@@ -505,6 +508,10 @@ async function handlePortableChannel(
|
|
|
505
508
|
return new Response(new URL(request.url).searchParams.get("hub.challenge") ?? "");
|
|
506
509
|
}
|
|
507
510
|
|
|
511
|
+
if (channel.type === "discord" && channel.provider === "discord" && isDiscordPingInteraction(rawBody)) {
|
|
512
|
+
return jsonResponse({ type: 1 });
|
|
513
|
+
}
|
|
514
|
+
|
|
508
515
|
const events = await adapter.normalizeInbound(rawEvent);
|
|
509
516
|
const deliveries = [];
|
|
510
517
|
|
|
@@ -560,6 +567,10 @@ async function handlePortableChannel(
|
|
|
560
567
|
});
|
|
561
568
|
}
|
|
562
569
|
|
|
570
|
+
if (channel.type === "discord" && channel.provider === "discord") {
|
|
571
|
+
return jsonResponse(discordLocalInteractionResponse(deliveries));
|
|
572
|
+
}
|
|
573
|
+
|
|
563
574
|
return jsonResponse({ deliveries });
|
|
564
575
|
}
|
|
565
576
|
|
|
@@ -745,12 +756,8 @@ async function handleLocalFile(capsule: LoadedAgentCapsule, request: Request, ra
|
|
|
745
756
|
}
|
|
746
757
|
|
|
747
758
|
if (request.method === "PUT") {
|
|
748
|
-
const bytes = new Uint8Array(await request.arrayBuffer());
|
|
749
759
|
const maxUploadBytes = capsule.config.storage.files.maxUploadBytes ?? 10 * 1024 * 1024;
|
|
750
|
-
|
|
751
|
-
if (bytes.byteLength > maxUploadBytes) {
|
|
752
|
-
throw new AgentKitError("file_too_large", `File exceeds max upload size of ${maxUploadBytes} bytes.`);
|
|
753
|
-
}
|
|
760
|
+
const bytes = await readRequestBytesWithLimit(request, maxUploadBytes);
|
|
754
761
|
|
|
755
762
|
await mkdir(dirname(path), { recursive: true });
|
|
756
763
|
await writeFile(path, bytes);
|
|
@@ -773,6 +780,102 @@ async function handleLocalFile(capsule: LoadedAgentCapsule, request: Request, ra
|
|
|
773
780
|
throw new AgentKitError("method_not_allowed", "Unsupported file method.");
|
|
774
781
|
}
|
|
775
782
|
|
|
783
|
+
export class BoundedDedupeCache {
|
|
784
|
+
private readonly entries = new Map<string, number>();
|
|
785
|
+
|
|
786
|
+
constructor(
|
|
787
|
+
private readonly maxKeys: number,
|
|
788
|
+
private readonly ttlMs: number,
|
|
789
|
+
) {}
|
|
790
|
+
|
|
791
|
+
has(key: string): boolean {
|
|
792
|
+
this.pruneExpired();
|
|
793
|
+
return this.entries.has(key);
|
|
794
|
+
}
|
|
795
|
+
|
|
796
|
+
add(key: string): void {
|
|
797
|
+
this.pruneExpired();
|
|
798
|
+
this.entries.set(key, Date.now());
|
|
799
|
+
this.pruneOverflow();
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
get size(): number {
|
|
803
|
+
this.pruneExpired();
|
|
804
|
+
return this.entries.size;
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
private pruneExpired(): void {
|
|
808
|
+
const cutoff = Date.now() - this.ttlMs;
|
|
809
|
+
|
|
810
|
+
for (const [key, timestamp] of this.entries) {
|
|
811
|
+
if (timestamp >= cutoff) {
|
|
812
|
+
continue;
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
this.entries.delete(key);
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
private pruneOverflow(): void {
|
|
820
|
+
while (this.entries.size > this.maxKeys) {
|
|
821
|
+
const oldest = this.entries.keys().next().value;
|
|
822
|
+
|
|
823
|
+
if (typeof oldest !== "string") {
|
|
824
|
+
return;
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
this.entries.delete(oldest);
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
async function readRequestBytesWithLimit(request: Request, maxBytes: number): Promise<Uint8Array> {
|
|
833
|
+
const contentLength = request.headers.get("content-length");
|
|
834
|
+
|
|
835
|
+
if (contentLength) {
|
|
836
|
+
const declaredBytes = Number(contentLength);
|
|
837
|
+
|
|
838
|
+
if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {
|
|
839
|
+
throw new AgentKitError("file_too_large", `File exceeds max upload size of ${maxBytes} bytes.`);
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
if (!request.body) {
|
|
844
|
+
return new Uint8Array();
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
const reader = request.body.getReader();
|
|
848
|
+
const chunks: Uint8Array[] = [];
|
|
849
|
+
let total = 0;
|
|
850
|
+
|
|
851
|
+
while (true) {
|
|
852
|
+
const { done, value } = await reader.read();
|
|
853
|
+
|
|
854
|
+
if (done) {
|
|
855
|
+
break;
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
total += value.byteLength;
|
|
859
|
+
|
|
860
|
+
if (total > maxBytes) {
|
|
861
|
+
await reader.cancel().catch(() => undefined);
|
|
862
|
+
throw new AgentKitError("file_too_large", `File exceeds max upload size of ${maxBytes} bytes.`);
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
chunks.push(value);
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
const bytes = new Uint8Array(total);
|
|
869
|
+
let offset = 0;
|
|
870
|
+
|
|
871
|
+
for (const chunk of chunks) {
|
|
872
|
+
bytes.set(chunk, offset);
|
|
873
|
+
offset += chunk.byteLength;
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
return bytes;
|
|
877
|
+
}
|
|
878
|
+
|
|
776
879
|
async function listConversations(capsule: LoadedAgentCapsule) {
|
|
777
880
|
const store = await openCapsuleStore(capsule);
|
|
778
881
|
|
|
@@ -1033,9 +1136,47 @@ function adapterForChannel(type: ChannelType, provider: ChannelProvider): Channe
|
|
|
1033
1136
|
return metaWhatsappChannelAdapter;
|
|
1034
1137
|
}
|
|
1035
1138
|
|
|
1139
|
+
if (type === "discord" && provider === "discord") {
|
|
1140
|
+
return discordChannelAdapter;
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1036
1143
|
throw new AgentKitError("channel_not_found", `No channel adapter for ${type}/${provider}.`);
|
|
1037
1144
|
}
|
|
1038
1145
|
|
|
1146
|
+
function discordLocalInteractionResponse(
|
|
1147
|
+
deliveries: Array<{
|
|
1148
|
+
state: string;
|
|
1149
|
+
message?: {
|
|
1150
|
+
content?: string;
|
|
1151
|
+
};
|
|
1152
|
+
}>,
|
|
1153
|
+
): Record<string, unknown> {
|
|
1154
|
+
const sent = deliveries.find((delivery) => delivery.state === "sent" && delivery.message?.content);
|
|
1155
|
+
|
|
1156
|
+
if (sent?.message?.content) {
|
|
1157
|
+
return {
|
|
1158
|
+
type: 4,
|
|
1159
|
+
data: {
|
|
1160
|
+
content: sent.message.content,
|
|
1161
|
+
allowed_mentions: {
|
|
1162
|
+
parse: [],
|
|
1163
|
+
},
|
|
1164
|
+
},
|
|
1165
|
+
};
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
return {
|
|
1169
|
+
type: 4,
|
|
1170
|
+
data: {
|
|
1171
|
+
content: "This Discord interaction was received but could not be processed.",
|
|
1172
|
+
flags: 64,
|
|
1173
|
+
allowed_mentions: {
|
|
1174
|
+
parse: [],
|
|
1175
|
+
},
|
|
1176
|
+
},
|
|
1177
|
+
};
|
|
1178
|
+
}
|
|
1179
|
+
|
|
1039
1180
|
async function resolveChannelSecrets(root: string, channel: AgentChannel): Promise<Record<string, string>> {
|
|
1040
1181
|
const env = await loadCapsuleEnv(root);
|
|
1041
1182
|
return Object.fromEntries(channel.secrets.map((name) => [name, env[name] ?? ""]));
|
package/src/runtime/evals.ts
CHANGED
|
@@ -184,6 +184,7 @@ export async function writeEvalFromConversation(
|
|
|
184
184
|
const root = await findAgentCapsuleRoot(cwd);
|
|
185
185
|
const trace = await getConversationTraceFromCwd(root, input.conversationId);
|
|
186
186
|
const turns = replayTurnsFromMessages(trace.messages);
|
|
187
|
+
const safeTitle = redactEvalText(trace.title ?? trace.id);
|
|
187
188
|
|
|
188
189
|
if (turns.length === 0) {
|
|
189
190
|
throw new AgentKitError(
|
|
@@ -192,11 +193,11 @@ export async function writeEvalFromConversation(
|
|
|
192
193
|
);
|
|
193
194
|
}
|
|
194
195
|
|
|
195
|
-
const file = resolveEvalOutputPath(root, input.out,
|
|
196
|
+
const file = resolveEvalOutputPath(root, input.out, safeTitle);
|
|
196
197
|
const source = `import { defineEval } from "@andreprado/agentkit";
|
|
197
198
|
|
|
198
199
|
export default defineEval({
|
|
199
|
-
name: ${JSON.stringify(input.name
|
|
200
|
+
name: ${JSON.stringify(input.name ? redactEvalText(input.name) : `replay ${safeTitle}`)},
|
|
200
201
|
turns: ${formatEvalTurns(turns)},
|
|
201
202
|
});
|
|
202
203
|
`;
|
|
@@ -729,14 +730,22 @@ function replayTurnsFromMessages(
|
|
|
729
730
|
}
|
|
730
731
|
|
|
731
732
|
const nextAssistant = messages.slice(index + 1).find((candidate) => candidate.role === "assistant");
|
|
733
|
+
const input = redactEvalText(message.content);
|
|
734
|
+
const contains = nextAssistant ? redactEvalText(nextAssistant.content) : undefined;
|
|
735
|
+
const redactionPresent = input.includes("[redacted_") || Boolean(contains?.includes("[redacted_"));
|
|
736
|
+
|
|
732
737
|
turns.push({
|
|
733
|
-
input
|
|
738
|
+
input,
|
|
734
739
|
...(nextAssistant
|
|
735
740
|
? {
|
|
736
741
|
expect: {
|
|
737
|
-
response:
|
|
738
|
-
|
|
739
|
-
|
|
742
|
+
response: redactionPresent
|
|
743
|
+
? {
|
|
744
|
+
notRegex: ["API_KEY|Bearer\\s+|(?:sk|agk|ak|dpat)[_-]|[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}"],
|
|
745
|
+
}
|
|
746
|
+
: {
|
|
747
|
+
contains: contains ?? "",
|
|
748
|
+
},
|
|
740
749
|
},
|
|
741
750
|
}
|
|
742
751
|
: {}),
|
|
@@ -746,6 +755,18 @@ function replayTurnsFromMessages(
|
|
|
746
755
|
return turns;
|
|
747
756
|
}
|
|
748
757
|
|
|
758
|
+
function redactEvalText(value: string): string {
|
|
759
|
+
return value
|
|
760
|
+
.replace(/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi, "[redacted_email]")
|
|
761
|
+
.replace(
|
|
762
|
+
/\b[A-Z0-9_]*(?:API[_-]?KEY|TOKEN|SECRET|PASSWORD|PRIVATE[_-]?KEY|CLIENT[_-]?SECRET)\s*=\s*[^\s,;]+/gi,
|
|
763
|
+
(match) => `${match.slice(0, match.indexOf("=")).trim()}=[redacted_secret]`,
|
|
764
|
+
)
|
|
765
|
+
.replace(/\b(?:sk|agk|ak|dpat)[_-][A-Za-z0-9_-]{8,}\b/g, "[redacted_token]")
|
|
766
|
+
.replace(/\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi, "Bearer [redacted_token]")
|
|
767
|
+
.replace(/\+?\d[\d\s().-]{7,}\d/g, "[redacted_phone]");
|
|
768
|
+
}
|
|
769
|
+
|
|
749
770
|
function formatEvalTurns(turns: Array<{ input: string; expect?: EvalExpect }>): string {
|
|
750
771
|
return JSON.stringify(turns, null, 4)
|
|
751
772
|
.split("\n")
|