niahere 0.3.0 → 0.3.2

package/src/channels/sms.ts

@@ -0,0 +1,179 @@
+/**
+ * SMS channel via Twilio.
+ *
+ * Same Twilio number as voice (channels.phone.from_number by default,
+ * overridable via channels.sms.from_number). Inbound webhook →
+ * chat engine → REST reply. Reuses the shared TwilioWebhookServer for
+ * routing, signature validation, dedup, and rate-limiting.
+ *
+ * Use case: cellular-only-no-data reachability — Aman can text Nia from
+ * patchy zones (Ladakh highways, basements, etc.) where Telegram /
+ * WhatsApp / voice over data won't work but SMS over SS7 still does.
+ *
+ * Note: outbound from US Twilio long codes to Indian mobile numbers has
+ * variable deliverability under TRAI scrubbing rules. Test empirically;
+ * if outbound fails, the inbound leg (Aman → Nia) is more reliable.
+ */
+import { getMcpServers } from "../mcp";
+import { runMigrations } from "../db/migrate";
+import type { Channel, ChatState, Outbound, TwilioConfig } from "../types";
+import { getConfig } from "../utils/config";
+import { log } from "../utils/log";
+import { sendMessage as twilioSendMessage } from "./twilio/rest";
+import { getTwilioServer } from "./twilio/server";
+import { chainLock, openChatEngine } from "./common/chat-session";
+
+const EMPTY_TWIML = '<?xml version="1.0" encoding="UTF-8"?><Response></Response>';
+
+class SmsChannel implements Channel {
+  name = "sms" as const;
+  private readonly twilio: TwilioConfig;
+  /** Cached resolved "from" number: sms.from_number || phone.from_number */
+  private readonly fromNumber: string;
+  private readonly chats = new Map<string, ChatState>();
+
+  constructor(twilio: TwilioConfig, fromNumber: string) {
+    this.twilio = twilio;
+    this.fromNumber = fromNumber;
+  }
+
+  async start(): Promise<void> {
+    await runMigrations();
+
+    const server = getTwilioServer();
+    server.configure({
+      port: this.twilio.port,
+      publicBaseUrl: this.twilio.public_base_url,
+      signingToken: this.twilio.auth_token || this.twilio.secret,
+    });
+
+    server.registerHttp("/twilio/sms/incoming", (_req, ctx) => this.handleInbound(ctx.params), {
+      dedupOn: "MessageSid",
+      rateLimitOn: "From",
+    });
+    server.registerHttp("/twilio/sms/status", (_req, ctx) => this.handleStatus(ctx.params), {
+      dedupOn: "MessageSid",
+    });
+
+    if (this.twilio.owner_number) server.exemptFromRateLimit(this.twilio.owner_number);
+
+    await server.start();
+
+    log.info(
+      {
+        from: this.fromNumber,
+        owner: this.twilio.owner_number,
+        publicBaseUrl: this.twilio.public_base_url,
+      },
+      "sms channel started",
+    );
+  }
+
+  async stop(): Promise<void> {
+    for (const state of this.chats.values()) state.engine.close();
+    this.chats.clear();
+  }
+
+  /** Outbound — used by send_message MCP tool. SMS is text-only; media is dropped with a warning. */
+  async deliver(out: Outbound): Promise<void> {
+    if (!this.twilio.owner_number) throw new Error("sms: owner_number not set");
+    // SMS has no threading; recipient kind is ignored.
+    if (out.media) {
+      log.warn({ filename: out.media.filename }, "sms: media payload dropped (channel is text-only)");
+    }
+    if (out.text) {
+      await this.sendTo(this.twilio.owner_number, out.text);
+    }
+  }
+
+  // --- Inbound webhook ---
+
+  private async handleInbound(params: Record<string, string>): Promise<Response> {
+    const from = params.From || "";
+    const body = params.Body || "";
+
+    if (!this.isAllowed(from)) {
+      log.warn({ from }, "sms: rejecting non-allowlisted sender");
+      return new Response(EMPTY_TWIML, { status: 200, headers: { "Content-Type": "text/xml" } });
+    }
+
+    const state = await this.getState(from);
+    // Ack the webhook immediately; reply via REST asynchronously to avoid
+    // Twilio's ~15s webhook timeout when the engine takes longer.
+    chainLock(state, async () => {
+      try {
+        const { result } = await state.engine.send(body);
+        const reply = result.trim() || "(no response)";
+        await this.sendTo(from, reply);
+      } catch (err) {
+        log.error({ err, from }, "sms: engine error");
+        await this.sendTo(from, `[error] ${err instanceof Error ? err.message : String(err)}`).catch(() => {});
+      }
+    });
+
+    return new Response(EMPTY_TWIML, { status: 200, headers: { "Content-Type": "text/xml" } });
+  }
+
+  private handleStatus(params: Record<string, string>): Response {
+    log.info(
+      {
+        messageSid: params.MessageSid,
+        status: params.MessageStatus,
+        errorCode: params.ErrorCode,
+        to: params.To,
+      },
+      "sms: delivery status",
+    );
+    return new Response("", { status: 204 });
+  }
+
+  // --- Outbound ---
+
+  private async sendTo(remoteE164: string, body: string): Promise<void> {
+    if (!this.twilio.sid || !this.twilio.secret) {
+      log.warn("sms: twilio sid/secret missing, cannot send");
+      return;
+    }
+    try {
+      const res = await twilioSendMessage({
+        accountSid: this.twilio.sid,
+        authSid: this.twilio.sid,
+        authSecret: this.twilio.secret,
+        to: remoteE164,
+        from: this.fromNumber,
+        body,
+        statusCallbackUrl: this.twilio.public_base_url ? `${this.twilio.public_base_url}/twilio/sms/status` : undefined,
+      });
+      log.info({ to: remoteE164, sid: res.messageSid, status: res.status }, "sms: sent");
+    } catch (err) {
+      log.error({ err, to: remoteE164 }, "sms: send failed");
+    }
+  }
+
+  // --- Helpers ---
+
+  private isAllowed(remoteE164: string): boolean {
+    if (this.twilio.owner_number && remoteE164 === this.twilio.owner_number) return true;
+    return this.twilio.allowlist.includes(remoteE164);
+  }
+
+  private async getState(remoteE164: string): Promise<ChatState> {
+    let state = this.chats.get(remoteE164);
+    if (state) return state;
+    state = await openChatEngine(`sms-${remoteE164}`, () => ({ channel: "sms", mcpServers: getMcpServers() }));
+    this.chats.set(remoteE164, state);
+    return state;
+  }
+}
+
+export function createSmsChannel(): SmsChannel | null {
+  const { twilio, sms, phone } = getConfig().channels;
+  if (!sms.enabled) return null;
+  if (!twilio.sid || !twilio.secret) return null;
+  // sms.from_number falls back to phone.from_number (same number for voice + SMS).
+  const fromNumber = sms.from_number ?? phone.from_number;
+  if (!fromNumber) return null;
+  return new SmsChannel(twilio, fromNumber);
+}
+
+export type { SmsChannel };

package/src/channels/telegram.ts

@@ -2,15 +2,15 @@ import { Bot, InputFile } from "grammy";
 import { createHash } from "crypto";
 import { mkdirSync, writeFileSync } from "fs";
 import { join } from "path";
-import { createChatEngine } from "../chat/engine";
-import type { Channel, ChatState, Attachment } from "../types";
+import type { Channel, ChatState, Attachment, Outbound } from "../types";
 import { getConfig, updateRawConfig } from "../utils/config";
 import { runMigrations } from "../db/migrate";
-import { Session, Message } from "../db/models";
+import { Message } from "../db/models";
 import { log } from "../utils/log";
 import { getMcpServers } from "../mcp";
 import { classifyMime, validateAttachment, prepareImage } from "../utils/attachment";
 import { getNiaHome } from "../utils/paths";
+import { chainLock, openChatEngine, rotateRoom } from "./common/chat-session";
 
 function safeExtension(filename?: string): string {
   const ext = filename?.split(".").pop();
@@ -23,27 +23,27 @@ function cacheExtension(filename: string | undefined, mimeType: string): string
 }
 
 class TelegramChannel implements Channel {
-  name = "telegram";
+  name = "telegram" as const;
   private bot: Bot | null = null;
   private outboundChatId: number | null = null;
 
-  async sendMessage(text: string): Promise<void> {
+  async deliver(out: Outbound): Promise<void> {
     if (!this.bot) throw new Error("Telegram not started");
     const chatId = this.outboundChatId;
     if (!chatId) throw new Error("No outbound chat ID registered");
-    await this.bot.api.sendMessage(chatId, text);
-  }
-
-  async sendMedia(data: Buffer, mimeType: string, filename?: string): Promise<void> {
-    if (!this.bot) throw new Error("Telegram not started");
-    const chatId = this.outboundChatId;
-    if (!chatId) throw new Error("No outbound chat ID registered");
-
-    const file = new InputFile(data, filename);
-    if (mimeType.startsWith("image/")) {
-      await this.bot.api.sendPhoto(chatId, file);
-    } else {
-      await this.bot.api.sendDocument(chatId, file);
+    // Telegram has no native threading; thread recipients fall back to the
+    // configured DM chat (the same place we'd send to for `owner`).
+
+    if (out.media) {
+      const file = new InputFile(Buffer.from(out.media.data), out.media.filename);
+      if (out.media.mimeType.startsWith("image/")) {
+        await this.bot.api.sendPhoto(chatId, file);
+      } else {
+        await this.bot.api.sendDocument(chatId, file);
+      }
+    }
+    if (out.text) {
+      await this.bot.api.sendMessage(chatId, out.text);
     }
   }
 
@@ -57,7 +57,13 @@ class TelegramChannel implements Channel {
     return Buffer.from(await resp.arrayBuffer());
   }
 
-  private cacheAttachment(chatId: number, roomIndex: number, data: Buffer, mimeType: string, filename?: string): string {
+  private cacheAttachment(
+    chatId: number,
+    roomIndex: number,
+    data: Buffer,
+    mimeType: string,
+    filename?: string,
+  ): string {
     const scope = `telegram-${chatId}-${roomIndex}`;
     const dir = join(getNiaHome(), "tmp", "attachments", scope);
     mkdirSync(dir, { recursive: true });
@@ -78,44 +84,23 @@ class TelegramChannel implements Channel {
 
     const chats = new Map<number, ChatState>();
 
-    function roomPrefix(chatId: number): string {
+    function keyOf(chatId: number): string {
       return `tg-${chatId}`;
     }
 
-    function roomName(chatId: number, index: number): string {
-      return `tg-${chatId}-${index}`;
-    }
-
     async function getState(chatId: number): Promise<ChatState> {
       let state = chats.get(chatId);
-      if (!state) {
-        const prefix = roomPrefix(chatId);
-        const idx = await Session.getLatestRoomIndex(prefix);
-        const room = roomName(chatId, idx);
-        log.info({ chatId, room }, "telegram: creating chat engine");
-        const engine = await createChatEngine({ room, channel: "telegram", resume: true, mcpServers: getMcpServers() });
-        state = { engine, roomIndex: idx, lock: Promise.resolve() };
-        chats.set(chatId, state);
-        log.info({ chatId, room, activeSessions: chats.size }, "telegram: engine ready");
-      }
+      if (state) return state;
+      state = await openChatEngine(keyOf(chatId), () => ({ channel: "telegram", mcpServers: getMcpServers() }));
+      chats.set(chatId, state);
       return state;
     }
 
     async function restartChat(chatId: number): Promise<ChatState> {
-      const old = chats.get(chatId);
-      if (old) old.engine.close();
-
-      const prefix = roomPrefix(chatId);
-      const prevIdx = await Session.getLatestRoomIndex(prefix);
-      const newIdx = prevIdx + 1;
-      const room = roomName(chatId, newIdx);
-
-      // Persist a placeholder session immediately so the room index survives
-      // daemon restarts (otherwise getState falls back to the old room).
-      await Session.create(`placeholder-${room}`, room);
-
-      const engine = await createChatEngine({ room, channel: "telegram", resume: false, mcpServers: getMcpServers() });
-      const state: ChatState = { engine, roomIndex: newIdx, lock: Promise.resolve() };
+      const state = await rotateRoom(keyOf(chatId), chats.get(chatId), () => ({
+        channel: "telegram",
+        mcpServers: getMcpServers(),
+      }));
       chats.set(chatId, state);
       return state;
     }
@@ -126,9 +111,7 @@ class TelegramChannel implements Channel {
         fn().catch((err) => log.error({ err, chatId }, "unhandled error in locked handler"));
         return;
       }
-      const queued = state.lock !== Promise.resolve();
-      if (queued) log.debug({ chatId }, "telegram: message queued behind active lock");
-      state.lock = state.lock.then(fn, fn);
+      chainLock(state, fn);
     }
 
     const isOpen = config.channels.telegram.open;
@@ -151,7 +134,10 @@ class TelegramChannel implements Channel {
 
     async function processMessage(ctx: any, state: ChatState, text: string, attachments?: Attachment[]): Promise<void> {
       const chatId = ctx.chatId;
-      log.info({ chatId, text: text.slice(0, 100), attachments: attachments?.length || 0 }, "telegram message received");
+      log.info(
+        { chatId, text: text.slice(0, 100), attachments: attachments?.length || 0 },
+        "telegram message received",
+      );
 
       // Show typing indicator throughout
       const typingInterval = setInterval(() => {
@@ -251,7 +237,7 @@ class TelegramChannel implements Channel {
           const mime = doc.mime_type || "application/octet-stream";
           const attType = classifyMime(mime) || "file";
           let data = await self.downloadFile(doc.file_id);
-          const error = validateAttachment(data, mime);
+          const error = validateAttachment(data);
           if (error) {
             await ctx.reply(error);
             return;
@@ -263,7 +249,13 @@ class TelegramChannel implements Channel {
             finalMime = prepared.mimeType;
           }
           const sourcePath = self.cacheAttachment(ctx.chatId, state.roomIndex, data, finalMime, doc.file_name);
-          const attachment: Attachment = { type: attType, data, mimeType: finalMime, filename: doc.file_name, sourcePath };
+          const attachment: Attachment = {
+            type: attType,
+            data,
+            mimeType: finalMime,
+            filename: doc.file_name,
+            sourcePath,
+          };
           const caption = ctx.message.caption || (attType === "image" ? "What's in this image?" : "Here's a file.");
           await processMessage(ctx, state, caption, [attachment]);
         } catch (err) {

package/src/channels/twilio/dedup.ts

@@ -0,0 +1,36 @@
+/**
+ * Time-bounded set for deduplicating webhook deliveries.
+ *
+ * Twilio retries webhooks on 5xx/timeouts, so the same MessageSid /
+ * CallSid can arrive multiple times. We track recently-seen IDs and
+ * drop duplicates, expiring entries after `ttlMs`.
+ */
+export class Dedup {
+  private readonly seen = new Map<string, number>();
+
+  constructor(
+    private readonly ttlMs: number = 10 * 60 * 1000,
+    private readonly maxEntries: number = 5000,
+  ) {}
+
+  /** Returns true if this id was already seen recently; false (and records it) otherwise. */
+  check(id: string): boolean {
+    const now = Date.now();
+    const cutoff = now - this.ttlMs;
+    const seenAt = this.seen.get(id);
+    if (seenAt !== undefined && seenAt > cutoff) return true;
+    this.seen.set(id, now);
+    if (this.seen.size > this.maxEntries) this.prune(cutoff);
+    return false;
+  }
+
+  private prune(cutoff: number): void {
+    for (const [k, v] of this.seen) {
+      if (v <= cutoff) this.seen.delete(k);
+    }
+  }
+
+  size(): number {
+    return this.seen.size;
+  }
+}

package/src/channels/twilio/media-cache.ts

@@ -0,0 +1,107 @@
+/**
+ * Disk-backed cache for outbound Twilio media.
+ *
+ * Twilio fetches outbound MMS/WhatsApp media by URL, so we write the
+ * payload to ~/.niahere/tmp/outbound/<sha>.<ext>, expose it under
+ * GET /twilio/media/<sha>.<ext>, and Twilio retrieves it. Disk (not
+ * memory) so the URL survives daemon restarts and Twilio's webhook
+ * retries within the eviction window.
+ *
+ * Eviction is opportunistic: capped at 100 files, 10MB total, 24h max
+ * age. Oldest-first; expired-first. Runs after every write; cheap
+ * enough at this scale (single user, low traffic).
+ */
+import { mkdir, readdir, readFile, stat, unlink, writeFile } from "fs/promises";
+import { createHash } from "crypto";
+import { join } from "path";
+import { getNiaHome } from "../../utils/paths";
+import { log } from "../../utils/log";
+
+const MAX_FILES = 100;
+const MAX_BYTES = 10 * 1024 * 1024;
+const MAX_AGE_MS = 24 * 60 * 60 * 1000;
+
+const MIME_TO_EXT: Record<string, string> = {
+  "image/jpeg": "jpg",
+  "image/png": "png",
+  "image/webp": "webp",
+  "image/gif": "gif",
+  "audio/mpeg": "mp3",
+  "audio/mp4": "m4a",
+  "audio/ogg": "ogg",
+  "audio/wav": "wav",
+  "video/mp4": "mp4",
+  "application/pdf": "pdf",
+};
+
+const FILENAME_RE = /^[a-f0-9]{16,64}\.[a-z0-9]{1,8}$/i;
+
+export function getMediaDir(): string {
+  return join(getNiaHome(), "tmp", "outbound");
+}
+
+export interface CachedMedia {
+  filename: string;
+  path: string;
+}
+
+export async function cacheMedia(buffer: Uint8Array, mime: string, ext?: string): Promise<CachedMedia> {
+  const dir = getMediaDir();
+  await mkdir(dir, { recursive: true });
+  const resolvedExt = (ext ?? MIME_TO_EXT[mime] ?? "bin").replace(/[^a-z0-9]/gi, "").slice(0, 8) || "bin";
+  const hash = createHash("sha256").update(buffer).digest("hex").slice(0, 32);
+  const filename = `${hash}.${resolvedExt}`;
+  const path = join(dir, filename);
+  await writeFile(path, buffer);
+  await evict().catch((err) => log.warn({ err }, "media-cache: eviction failed"));
+  return { filename, path };
+}
+
+export async function readCachedMedia(filename: string): Promise<{ buffer: Buffer; mime: string } | null> {
+  if (!FILENAME_RE.test(filename)) return null;
+  const path = join(getMediaDir(), filename);
+  try {
+    const buffer = await readFile(path);
+    const ext = filename.split(".").pop()!.toLowerCase();
+    const mime = Object.entries(MIME_TO_EXT).find(([, e]) => e === ext)?.[0] ?? "application/octet-stream";
+    return { buffer, mime };
+  } catch {
+    return null;
+  }
+}
+
+async function evict(): Promise<void> {
+  const dir = getMediaDir();
+  const entries = await readdir(dir);
+  const stats = await Promise.all(
+    entries.map(async (name) => {
+      const path = join(dir, name);
+      const s = await stat(path);
+      return { name, path, mtime: s.mtimeMs, size: s.size };
+    }),
+  );
+
+  const now = Date.now();
+  let alive: typeof stats = [];
+  for (const s of stats) {
+    if (now - s.mtime > MAX_AGE_MS) {
+      await unlink(s.path).catch(() => {});
+    } else {
+      alive.push(s);
+    }
+  }
+
+  alive.sort((a, b) => a.mtime - b.mtime);
+
+  while (alive.length > MAX_FILES) {
+    const victim = alive.shift()!;
+    await unlink(victim.path).catch(() => {});
+  }
+
+  let totalBytes = alive.reduce((sum, f) => sum + f.size, 0);
+  while (totalBytes > MAX_BYTES && alive.length > 0) {
+    const victim = alive.shift()!;
+    totalBytes -= victim.size;
+    await unlink(victim.path).catch(() => {});
+  }
+}

package/src/channels/twilio/media.ts

@@ -0,0 +1,79 @@
+/**
+ * Download inbound media attached to a Twilio webhook.
+ *
+ * Twilio's MediaUrlN values are HTTPS URLs that 302-redirect to S3.
+ * They sit behind Basic auth (same Twilio creds), so a vanilla fetch
+ * without an Authorization header gets 401. Bun's fetch follows the
+ * redirect transparently.
+ *
+ * Concurrency is bounded (4 in flight, 10s per item) so a sender that
+ * attaches many large files cannot stall the webhook handler.
+ */
+import { log } from "../../utils/log";
+import type { TwilioCreds } from "./rest";
+
+const MAX_CONCURRENT = 4;
+const TIMEOUT_MS = 10_000;
+
+export interface InboundMedia {
+  index: number;
+  url: string;
+  mime: string;
+  data: Buffer;
+}
+
+export interface MediaDescriptor {
+  index: number;
+  url: string;
+  mime: string;
+}
+
+/**
+ * Pull out the NumMedia / MediaUrlN / MediaContentTypeN fields from a
+ * Twilio webhook form body.
+ */
+export function extractMedia(params: Record<string, string>): MediaDescriptor[] {
+  const num = parseInt(params.NumMedia || "0", 10);
+  if (!Number.isFinite(num) || num <= 0) return [];
+  const items: MediaDescriptor[] = [];
+  for (let i = 0; i < num; i++) {
+    const url = params[`MediaUrl${i}`];
+    const mime = params[`MediaContentType${i}`];
+    if (url && mime) items.push({ index: i, url, mime });
+  }
+  return items;
+}
+
+export async function downloadInboundMedia(
+  descriptors: MediaDescriptor[],
+  creds: TwilioCreds,
+): Promise<InboundMedia[]> {
+  const out: InboundMedia[] = [];
+  for (let i = 0; i < descriptors.length; i += MAX_CONCURRENT) {
+    const slice = descriptors.slice(i, i + MAX_CONCURRENT);
+    const results = await Promise.allSettled(slice.map((d) => downloadOne(d, creds)));
+    for (let j = 0; j < results.length; j++) {
+      const r = results[j];
+      if (r.status === "fulfilled") {
+        out.push(r.value);
+      } else {
+        log.warn({ err: r.reason, descriptor: slice[j] }, "twilio: media download failed");
+      }
+    }
+  }
+  return out;
+}
+
+async function downloadOne(d: MediaDescriptor, creds: TwilioCreds): Promise<InboundMedia> {
+  const auth = `Basic ${Buffer.from(`${creds.authSid}:${creds.authSecret}`).toString("base64")}`;
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), TIMEOUT_MS);
+  try {
+    const resp = await fetch(d.url, { headers: { Authorization: auth }, signal: controller.signal });
+    if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
+    const buffer = Buffer.from(await resp.arrayBuffer());
+    return { index: d.index, url: d.url, mime: d.mime, data: buffer };
+  } finally {
+    clearTimeout(timer);
+  }
+}

package/src/channels/twilio/rate-limit.ts

@@ -0,0 +1,33 @@
+/**
+ * Sliding-window rate limiter keyed by an arbitrary string (e.g. caller
+ * E.164). Protects against runaway costs when the WhatsApp Sandbox's
+ * shared number gets random opt-ins and someone spams.
+ */
+export class RateLimiter {
+  private readonly hits = new Map<string, number[]>();
+
+  constructor(
+    private readonly maxPerWindow: number = 30,
+    private readonly windowMs: number = 60_000,
+  ) {}
+
+  /** Returns true if this hit was allowed (and recorded); false if over limit. */
+  allow(key: string): boolean {
+    const now = Date.now();
+    const cutoff = now - this.windowMs;
+    const arr = this.hits.get(key) ?? [];
+    const recent = arr.filter((t) => t > cutoff);
+    if (recent.length >= this.maxPerWindow) {
+      this.hits.set(key, recent);
+      return false;
+    }
+    recent.push(now);
+    this.hits.set(key, recent);
+    return true;
+  }
+
+  /** Drop tracking for a key (e.g. owner number — never rate-limit yourself). */
+  exempt(key: string): void {
+    this.hits.delete(key);
+  }
+}