@segmentstream/agent 0.1.4 → 0.2.0

package/dist/extension/lib/telegram/chats.mjs

@@ -0,0 +1,123 @@
+import { fileURLToPath as __eveFileURLToPath } from "node:url";
+import { dirname as __eveDirname } from "node:path";
+import { createRequire as __eveCreateRequire } from "node:module";
+const __filename = __eveFileURLToPath(import.meta.url);
+__eveDirname(__filename);
+__eveCreateRequire(import.meta.url);
+const MAXIMUM_WHEN_TO_USE_LENGTH = 1e3;
+const REPORT_INTERVAL_MS = 6e5;
+const MAXIMUM_CHATS = 500;
+function agentChats(input) {
+	const chats = new Map();
+	const now = input.now ?? Date.now;
+	const log = input.log ?? ((message, error) => console.warn(message, error));
+	function remember(chatId, value) {
+		chats.delete(chatId);
+		chats.set(chatId, value);
+		if (chats.size > MAXIMUM_CHATS) {
+			const oldest = chats.keys().next();
+			if (oldest.done !== true) chats.delete(oldest.value);
+		}
+	}
+	function restore(chatId, written, previous) {
+		if (chats.get(chatId) !== written) return;
+		if (previous === void 0) chats.delete(chatId);
+		else chats.set(chatId, previous);
+	}
+	async function report(chatId, fallback, body) {
+		try {
+			const chat = await input.identities().seenChat({
+				provider: "telegram",
+				externalId: chatId,
+				...body
+			});
+			remember(chatId, {
+				...chats.get(chatId) ?? fallback,
+				chat
+			});
+			return true;
+		} catch (error) {
+			log("The agent could not report the chat to the ontology's registry.", error);
+			return false;
+		}
+	}
+	return {
+		async seen({ chatId, title, memberId }) {
+			const known = chats.get(chatId);
+			const at = now();
+			if (!(known === void 0 || at - known.reportedAt >= 6e5 || memberId !== void 0 && !known.members.has(memberId))) return;
+			const members = new Set(known?.members);
+			if (memberId !== void 0) members.add(memberId);
+			const written = {
+				reportedAt: at,
+				members,
+				chat: known?.chat
+			};
+			remember(chatId, written);
+			if (!await report(chatId, {
+				reportedAt: at,
+				members,
+				chat: void 0
+			}, {
+				title,
+				...memberId === void 0 ? {} : { memberId }
+			})) restore(chatId, written, known);
+		},
+		async left({ chatId, title, memberId }) {
+			const known = chats.get(chatId);
+			const remembered = {
+				reportedAt: known?.reportedAt ?? now(),
+				members: new Set([...known?.members ?? []].filter((member) => member !== memberId)),
+				chat: known?.chat
+			};
+			remember(chatId, remembered);
+			if (!await report(chatId, remembered, {
+				title,
+				leftMemberId: memberId
+			})) restore(chatId, remembered, known);
+		},
+		async agentLeft({ chatId, title }) {
+			const known = chats.get(chatId);
+			const remembered = {
+				reportedAt: known?.reportedAt ?? now(),
+				members: new Set(),
+				chat: known?.chat
+			};
+			remember(chatId, remembered);
+			if (!await report(chatId, remembered, {
+				title,
+				agentLeft: true
+			})) restore(chatId, remembered, known);
+		},
+		async migrated({ chatId, title, migratedFrom }) {
+			const old = chats.get(migratedFrom);
+			const known = chats.get(chatId);
+			const remembered = {
+				reportedAt: now(),
+				members: new Set([...old?.members ?? [], ...known?.members ?? []]),
+				chat: known?.chat
+			};
+			chats.delete(migratedFrom);
+			remember(chatId, remembered);
+			if (!await report(chatId, remembered, {
+				title,
+				migratedFrom
+			})) {
+				restore(chatId, remembered, known);
+				if (old !== void 0 && !chats.has(migratedFrom)) remember(migratedFrom, old);
+			}
+		},
+		chat(chatId) {
+			return chats.get(chatId)?.chat;
+		}
+	};
+}
+function chatContext(chat) {
+	if (chat === void 0) return [];
+	const lines = ["<chat>", `title: ${chat.title}`];
+	if (chat.purpose !== null && chat.purpose !== "") lines.push(`purpose: ${chat.purpose}`);
+	if (chat.whenToUse !== null && chat.whenToUse !== "") lines.push(`when_to_use: ${chat.whenToUse.slice(0, MAXIMUM_WHEN_TO_USE_LENGTH)}`);
+	lines.push("</chat>");
+	return [lines.join("\n")];
+}
+export { REPORT_INTERVAL_MS, agentChats, chatContext };

package/dist/extension/lib/telegram/files.d.ts

@@ -0,0 +1,25 @@
+import type { TelegramMessage } from "eve/channels/telegram";
+import { type AgentAttachments } from "../attachments.js";
+import type { StagedFile } from "./inbound-request.js";
+/**
+ * Photos and documents a member sends the agent (`docs/agent.md`, "Attachments"). Before the turn
+ * starts, each file is downloaded through the connection and stored once as a temporary attachment
+ * of the member, in this chat, for 24 hours. The turn reads the stored copy, and the model learns
+ * each attachment's id from an `<attachments>` block, so that it can read, discard or promote it
+ * with the attachment tools. A file that cannot be stored is never passed on silently: the turn
+ * gets a note saying why.
+ */
+/** The chat an attachment belongs to: `telegram:<chat id>`. */
+export declare function telegramConversation(chatId: string): string;
+/** Stores the message's files one after another, so that a large album holds one file at a time. */
+export declare function stageFiles(input: {
+    readonly message: TelegramMessage;
+    readonly memberId: string;
+    readonly attachments: AgentAttachments;
+    readonly download: (fileId: string) => Promise<Uint8Array>;
+}): Promise<ReadonlyMap<string, StagedFile>>;
+/** The `<attachments>` block that tells the model which attachments the message carries. */
+export declare function attachmentsContext(input: {
+    readonly files: ReadonlyMap<string, StagedFile>;
+    readonly group: boolean;
+}): ReadonlyArray<string>;

package/dist/extension/lib/telegram/files.mjs

@@ -0,0 +1,77 @@
+import { fileURLToPath as __eveFileURLToPath } from "node:url";
+import { dirname as __eveDirname } from "node:path";
+import { createRequire as __eveCreateRequire } from "node:module";
+const __filename = __eveFileURLToPath(import.meta.url);
+__eveDirname(__filename);
+__eveCreateRequire(import.meta.url);
+import { MAXIMUM_ATTACHMENT_BYTES, attachmentUrl } from "../attachments.mjs";
+function telegramConversation(chatId) {
+	return `telegram:${chatId}`;
+}
+function described(file, index) {
+	return {
+		filename: file.fileName ?? (file.kind === "photo" ? `photo-${index}.jpg` : `file-${index}`),
+		contentType: file.mediaType ?? (file.kind === "photo" ? "image/jpeg" : "application/octet-stream")
+	};
+}
+function megabytes(bytes) {
+	return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
+}
+async function stageFiles(input) {
+	const { message } = input;
+	const conversation = telegramConversation(message.chat.id);
+	const staged = new Map();
+	for (const [index, file] of message.attachments.entries()) {
+		const { filename, contentType } = described(file, index);
+		if (file.size !== void 0 && file.size > MAXIMUM_ATTACHMENT_BYTES) {
+			staged.set(file.fileId, {
+				status: "refused",
+				filename,
+				reason: `it is ${megabytes(file.size)}, over the ${megabytes(MAXIMUM_ATTACHMENT_BYTES)} a bot may download`
+			});
+			continue;
+		}
+		try {
+			const bytes = await input.download(file.fileId);
+			const attachment = await input.attachments.receive({
+				memberId: input.memberId,
+				conversation
+			}, {
+				bytes,
+				filename,
+				contentType,
+				sourceRef: `${message.chat.id}:${message.messageId}:${index}`
+			});
+			staged.set(file.fileId, {
+				status: "stored",
+				attachment,
+				url: attachmentUrl({
+					id: attachment.id,
+					memberId: input.memberId,
+					conversation
+				})
+			});
+		} catch (error) {
+			console.error("[segmentstream] attachment could not be stored", {
+				kind: file.kind,
+				size: file.size,
+				error
+			});
+			staged.set(file.fileId, {
+				status: "refused",
+				filename,
+				reason: error instanceof Error ? error.message : "it could not be stored"
+			});
+		}
+	}
+	return staged;
+}
+function attachmentsContext(input) {
+	if (input.files.size === 0) return [];
+	const lines = ["<attachments>", `The member's files from this message, kept for 24 hours for this member in this chat only. Read, discard or promote them with the segmentstream__attachments_* tools and these ids; promote only what should be kept.`];
+	for (const staged of input.files.values()) lines.push(staged.status === "stored" ? `- id: ${staged.attachment.id}, filename: ${JSON.stringify(staged.attachment.filename)}, type: ${staged.attachment.contentType}, size: ${staged.attachment.size} bytes, expires_at: ${staged.attachment.expiresAt}` : `- ${JSON.stringify(staged.filename)}: not received, ${staged.reason}`);
+	if (input.group) lines.push("This is a group chat: everyone in it shares this conversation's history, so do not repeat a private file's contents here. Suggest sending private files in a direct message.");
+	lines.push("</attachments>");
+	return [lines.join("\n")];
+}
+export { attachmentsContext, stageFiles, telegramConversation };

package/dist/extension/lib/telegram/inbound-request.d.ts

@@ -1,5 +1,16 @@
 import type { ChannelSource } from "eve/channels";
+import type { Attachment } from "../attachments.js";
 import type { SessionOperations } from "./sessions.js";
+/** What became of one Telegram file of the message: stored as an attachment, or why not. */
+export type StagedFile = {
+    readonly status: "stored";
+    readonly attachment: Attachment;
+    readonly url: URL;
+} | {
+    readonly status: "refused";
+    readonly filename: string;
+    readonly reason: string;
+};
 /**
  * What one webhook request shares between eve's `onMessage` hook and the route wrapper around it.
  * `onMessage` may only return `{ auth, context, title }`; it cannot change the message eve sends.
@@ -12,8 +23,15 @@ export declare class InboundRequest {
     readonly agentDescription: () => Promise<string>;
     /** The text the turn should carry instead of the parsed message, once a transcript exists. */
     text: string | undefined;
+    /**
+     * The message's Telegram files by file id, once stored as attachments: the turn reads each from
+     * the attachment store instead of Telegram, and one that could not be stored becomes a note.
+     */
+    files: ReadonlyMap<string, StagedFile> | undefined;
     /** The route's session operations, on addresses already mapped to the configured scope. */
     sessions: SessionOperations | undefined;
+    /** Keeps work the request started, such as a chat report, alive after it answers. */
+    waitUntil: ((task: Promise<unknown>) => void) | undefined;
     constructor(
     /** The agent's description from the compiled manifest, for the routing decision. */
     agentDescription: () => Promise<string>);
@@ -21,5 +39,8 @@ export declare class InboundRequest {
 export declare function runInboundRequest<Result>(request: InboundRequest, run: () => Result): Result;
 /** The current webhook request, or `undefined` outside the wrapped route. */
 export declare function currentInboundRequest(): InboundRequest | undefined;
-/** A channel source whose `send` delivers the request's replacement text, when one was recorded. */
+/**
+ * A channel source whose `send` delivers the request's replacement text, when one was recorded,
+ * and its stored attachments in place of the Telegram files.
+ */
 export declare function withReplacedText<State>(source: ChannelSource<State>, request: InboundRequest): ChannelSource<State>;

package/dist/extension/lib/telegram/inbound-request.mjs

@@ -8,7 +8,9 @@ import { AsyncLocalStorage } from "node:async_hooks";
 var InboundRequest = class {
 	agentDescription;
 	text;
+	files;
 	sessions;
+	waitUntil;
 	constructor(agentDescription) {
 		this.agentDescription = agentDescription;
 	}
@@ -27,11 +29,38 @@ function replaceText(message, text) {
 		text
 	}, ...message.filter((part) => part.type !== "text")];
 }
+function telegramFileId(data) {
+	const href = data instanceof URL ? data.href : typeof data === "string" ? data : void 0;
+	if (href === void 0 || !href.startsWith("telegram-file:")) return;
+	try {
+		return decodeURIComponent(new URL(href).pathname);
+	} catch {
+		return;
+	}
+}
+function replaceFiles(message, files) {
+	if (typeof message === "string") return message;
+	return message.map((part) => {
+		if (part.type !== "file") return part;
+		const fileId = telegramFileId(part.data);
+		const staged = fileId === void 0 ? void 0 : files.get(fileId);
+		if (staged === void 0) return part;
+		if (staged.status === "refused") return {
+			type: "text",
+			text: `[file ${staged.filename} was not received: ${staged.reason}]`
+		};
+		return {
+			...part,
+			data: staged.url
+		};
+	});
+}
 function withReplacedText(source, request) {
 	return {
 		...source,
 		send(message, options) {
-			return source.send(request.text === void 0 ? message : replaceText(message, request.text), options);
+			const text = request.text === void 0 ? message : replaceText(message, request.text);
+			return source.send(request.files === void 0 ? text : replaceFiles(text, request.files), options);
 		}
 	};
 }

package/dist/extension/lib/telegram/inbound.d.ts

@@ -1,7 +1,9 @@
 import type { TelegramContext, TelegramInboundResult, TelegramMessage } from "eve/channels/telegram";
 import type { SessionSettings, TelegramPolicy } from "../../config.js";
+import type { AgentAttachments } from "../attachments.js";
 import type { AgentIdentities } from "../identities.js";
 import type { BotIdentityLookup } from "./bot.js";
+import { type AgentChats } from "./chats.js";
 import type { InboundRequest } from "./inbound-request.js";
 import { type ObservedMessages } from "./observed.js";
 import { type RouteDecider } from "./router.js";
@@ -11,13 +13,22 @@ import { type Transcriber } from "./voice.js";
  * eve's Telegram `onMessage` hook, in the order `agent-design.md` gives:
  *
  * 1. The hard gate, with no model: bots, channel posts and senders without a live pairing never
- *    start a turn. An unknown sender who writes to the bot privately gets a pairing button.
+ *    start a turn. An unknown sender who writes to the bot privately gets a pairing button. A group
+ *    message reports its chat to the ontology's registry in the background, whatever follows. A
+ *    group's notice that a member left is never a turn: a paired member's departure is reported,
+ *    and so is the agent's own, which unlinks every member. Neither is the notice, in the old chat
+ *    or the new one, that a group became a supergroup: the migration is reported.
  * 2. Voice becomes text, so the router and the agent read what was said.
  * 3. The router answers or observes. An observed message goes to the chat's ring and stops here.
  * 4. The session policy runs on the session the message continues: an idle one is cleared, the
  *    first message after the day's compaction time compacts it.
- * 5. An answered message dispatches as the paired member, with the ring and a `<speaker>` block
- *    naming the member, their roles and the message they replied to ahead of it as context.
+ * 5. An answered message's photos and documents are stored as the member's temporary attachments,
+ *    which the turn reads in place of the Telegram files.
+ * 6. An answered message dispatches as the paired member, with the ring, the registry's `<chat>`
+ *    block, a `<speaker>` block naming the member, their roles and the message they replied to and
+ *    an `<attachments>` block naming its stored files ahead of it as context. An answered group
+ *    message waits for its chat's report first, so that a chat new to the process already carries
+ *    its `<chat>` block.
  */
 export interface InboundDependencies {
     readonly policy: TelegramPolicy;
@@ -29,7 +40,10 @@ export interface InboundDependencies {
     readonly decide: RouteDecider;
     readonly transcribe: Transcriber;
     readonly download: (fileId: string) => Promise<Uint8Array>;
+    /** The attachment store; without it, files reach the turn as eve's Telegram file parts. */
+    readonly attachments?: AgentAttachments;
     readonly observed: ObservedMessages;
+    readonly chats: AgentChats;
     /** The app's `/agent/connect` link for a pairing code. */
     readonly connectUrl: (code: string) => string;
 }

package/dist/extension/lib/telegram/inbound.mjs

@@ -5,6 +5,8 @@ const __filename = __eveFileURLToPath(import.meta.url);
 __eveDirname(__filename);
 __eveCreateRequire(import.meta.url);
 import { telegramPrincipal } from "./principal.mjs";
+import { chatContext } from "./chats.mjs";
+import { attachmentsContext, stageFiles } from "./files.mjs";
 import { observedChatKey, observedContext } from "./observed.mjs";
 import { isAddressed, routeMessage } from "./router.mjs";
 import { messageAddress, scopedAddress } from "./sessions.mjs";
@@ -90,9 +92,67 @@ async function answerUnpaired(input) {
 		reply_markup: buttonTo("Connect account", dependencies.connectUrl(pairing.code))
 	});
 }
+function isGroupChat(message) {
+	return message.chat.type === "group" || message.chat.type === "supergroup";
+}
+async function reportDeparture(input) {
+	const { context, message, left, dependencies } = input;
+	const id = left.id;
+	if (typeof id !== "number" && typeof id !== "string") return;
+	if (left.is_bot === true) {
+		const bot = await dependencies.bot(context.telegram);
+		if (bot !== void 0 && String(id) === bot.id) await dependencies.chats.agentLeft({
+			chatId: message.chat.id,
+			title: message.chat.title ?? ""
+		});
+		return;
+	}
+	const lookup = await dependencies.identities.lookup({
+		provider: "telegram",
+		externalId: String(id)
+	});
+	if (lookup.status === "paired") await dependencies.chats.left({
+		chatId: message.chat.id,
+		title: message.chat.title ?? "",
+		memberId: lookup.member.id
+	});
+}
+function rawChatId(value) {
+	return typeof value === "number" || typeof value === "string" && value !== "" ? String(value) : void 0;
+}
+function migrationOf(message) {
+	const to = rawChatId(message.raw.migrate_to_chat_id);
+	const from = rawChatId(message.raw.migrate_from_chat_id);
+	if (to !== void 0) return {
+		chatId: to,
+		migratedFrom: message.chat.id
+	};
+	return from === void 0 ? void 0 : {
+		chatId: message.chat.id,
+		migratedFrom: from
+	};
+}
 async function handleInboundMessage(input) {
 	const { context, dependencies } = input;
 	const message = input.message;
+	const left = message.raw.left_chat_member;
+	const migration = isGroupChat(message) ? migrationOf(message) : void 0;
+	if (isGroupChat(message) && isRecord(left)) {
+		await reportDeparture({
+			context,
+			message,
+			left,
+			dependencies
+		});
+		return null;
+	}
+	if (migration !== void 0) {
+		await dependencies.chats.migrated({
+			...migration,
+			title: message.chat.title ?? ""
+		});
+		return null;
+	}
 	if (message.from === void 0 || message.from.isBot || message.chat.type === "channel") return null;
 	const sent = {
 		...message,
@@ -104,6 +164,13 @@ async function handleInboundMessage(input) {
 		provider: "telegram",
 		externalId: sent.from.id
 	});
+	const group = isGroupChat(sent);
+	const report = group ? dependencies.chats.seen({
+		chatId: sent.chat.id,
+		title: sent.chat.title ?? "",
+		...lookup.status === "paired" ? { memberId: lookup.member.id } : {}
+	}) : void 0;
+	if (report !== void 0) input.request?.waitUntil?.(report);
 	if (lookup.status !== "paired") {
 		if (addressed) await answerUnpaired({
 			context,
@@ -160,6 +227,14 @@ async function handleInboundMessage(input) {
 		dependencies.sessionPolicy.delivered(address);
 	}
 	await context.telegram.startTyping();
+	const files = sent.attachments.length === 0 || dependencies.attachments === void 0 || input.request === void 0 ? new Map() : await stageFiles({
+		message: sent,
+		memberId: lookup.member.id,
+		attachments: dependencies.attachments,
+		download: dependencies.download
+	});
+	if (input.request !== void 0 && files.size > 0) input.request.files = files;
+	await report;
 	return {
 		auth: telegramPrincipal({
 			message: sent,
@@ -167,11 +242,16 @@ async function handleInboundMessage(input) {
 		}),
 		context: [
 			...observedContext(dependencies.observed.take(chat)),
+			...group ? chatContext(dependencies.chats.chat(sent.chat.id)) : [],
 			speakerBlock({
 				message: sent,
 				member: lookup.member,
 				bot
 			}),
+			...attachmentsContext({
+				files,
+				group
+			}),
 			...notes
 		]
 	};

package/dist/extension/lib/telegram/proxy-fetch.mjs

@@ -53,13 +53,14 @@ function hasNullBody(status) {
 function telegramProxyFetch(proxy) {
 	return async (input, init) => {
 		const route = telegramProxyRoute(requestUrl(input));
-		const reply = await proxy.request({
+		const call = {
 			api: route.api,
 			method: proxyMethod(init?.method),
 			path: route.path,
 			headers: headerPairs(init?.headers),
 			body: await bodyBytes(init?.body)
-		});
+		};
+		const reply = route.api === "file" ? await proxy.download(call) : await proxy.request(call);
 		return new Response(hasNullBody(reply.status) ? null : Buffer.from(reply.body), {
 			status: reply.status,
 			headers: reply.headers.map(([name, value]) => [name, value])

package/dist/extension/lib/telegram/requests.d.ts

@@ -0,0 +1,71 @@
+import { type RouteDefinition } from "eve/channels";
+import type { TelegramChannel, TelegramChannelState } from "eve/channels/telegram";
+import type { SessionAuthContext } from "eve/context";
+import type { SessionSettings } from "../../config.js";
+import type { RuntimeTokenVerifier } from "../runtime-identity.js";
+import { type SessionPolicy } from "./sessions.js";
+/**
+ * Requests from the ontology's `agent_chat.send_message` action: an external agent or a member asks
+ * the workspace agent to contact someone in a group or a private chat. The ontology service resolves
+ * the chat and posts here with its own Vercel OIDC token; the request is delivered into the chat's
+ * session through eve's proactive `receive`, without a seed message, so it continues the session
+ * the chat's own messages reach (`chatId::`, the chat-scope address) and the replies land there too.
+ * The session policy runs on that session first, as it does for an inbound message.
+ */
+export declare const MESSAGES_ROUTE = "/eve/v1/telegram/messages";
+export interface AgentRequest {
+    readonly target: {
+        readonly kind: "group" | "private";
+        readonly provider: "telegram";
+        /** The group's chat id, or the Telegram user id of a private chat. */
+        readonly chatId: string;
+    };
+    readonly message: string;
+    /** Whom to contact, when the requester named someone. */
+    readonly person: {
+        readonly id: string;
+        readonly name: string;
+    } | null;
+    readonly requester: {
+        readonly id: string;
+        readonly name: string;
+        readonly kind: "member" | "service";
+        /** The roles the requester holds; empty from an ontology that does not send them. */
+        readonly roles: ReadonlyArray<string>;
+        /**
+         * Whether the turn can act as the requester: a member with a live paired identity. An ontology
+         * that does not send it means a member can.
+         */
+        readonly actAs: boolean;
+    };
+}
+/** The request body, or `undefined` when it does not match the contract. */
+export declare function agentRequest(value: unknown): AgentRequest | undefined;
+/**
+ * Whom the request's turn runs as. A member requester the ontology says can be acted as is the
+ * turn's member, so the ontology connection acts as them and the ontology service checks, as for any
+ * turn, that they are active and paired. Their roles, as the action's principal held them, choose the
+ * model of a session the request starts; the service resolves them afresh on every call. A service
+ * user, or a member who has not paired, cannot be acted as, so the request runs without a member and
+ * the turn has no ontology; their roles still choose the model, and grant nothing, since the
+ * ontology connection acts only for a `memberId`.
+ */
+export declare function requestPrincipal(request: AgentRequest): SessionAuthContext;
+/** What the turn is about, for the model, ahead of the request itself. */
+export declare function requestContext(request: AgentRequest): string;
+/**
+ * `POST /eve/v1/telegram/messages`. Gated like the webhook: outside production it answers 503 and a
+ * workspace without a Telegram channel 404. The caller must be a function of this deployment's
+ * project, verified from its Vercel OIDC token.
+ */
+export declare function messagesRoute(input: {
+    readonly channel: TelegramChannel;
+    readonly enabled: () => boolean;
+    readonly configured: () => boolean;
+    readonly verify: () => RuntimeTokenVerifier;
+    /** The adapter's session policy, which the webhook route applies too. */
+    readonly sessionPolicy: SessionPolicy;
+    /** The mounted `sessions.scope`. */
+    readonly scope: () => SessionSettings["scope"];
+    readonly log?: (message: string, detail: unknown) => void;
+}): RouteDefinition<TelegramChannelState>;