memex-mvp 0.5.0

package/bot/nexara.js

@@ -0,0 +1,61 @@
+/**
+ * Nexara transcription — POST /audio/transcriptions.
+ *
+ * Endpoint: https://api.nexara.ru/api/v1/audio/transcriptions
+ * Auth:     Authorization: Bearer <key>
+ *
+ * Telegram voice messages arrive as audio/ogg (Opus). Nexara's docs note
+ * that ogg/opus may need ffmpeg conversion to mp3 first; this implementation
+ * sends the raw OGG. If transcription fails on a particular file, install
+ * ffmpeg and we can add a conversion step later.
+ */
+
+const ENDPOINT = 'https://api.nexara.ru/api/v1/audio/transcriptions';
+
+/**
+ * @param {object} opts
+ * @param {Buffer|Uint8Array} opts.audioBuffer
+ * @param {string} opts.apiKey
+ * @param {string} [opts.filename='voice.oga']
+ * @param {string} [opts.mimeType='audio/ogg']
+ * @param {string} [opts.language='ru']
+ * @returns {Promise<{ text: string, duration?: number, language?: string, raw: object }>}
+ */
+export async function transcribe({
+  audioBuffer,
+  apiKey,
+  filename = 'voice.oga',
+  mimeType = 'audio/ogg',
+  language = 'ru',
+}) {
+  if (!apiKey) throw new Error('nexara: apiKey required');
+  if (!audioBuffer) throw new Error('nexara: audioBuffer required');
+
+  const blob = new Blob([audioBuffer], { type: mimeType });
+  const form = new FormData();
+  form.append('file', blob, filename);
+  form.append('response_format', 'verbose_json');
+  form.append('language', language);
+
+  const resp = await fetch(ENDPOINT, {
+    method: 'POST',
+    headers: { Authorization: `Bearer ${apiKey}` },
+    body: form,
+    signal: AbortSignal.timeout(120000),
+  });
+
+  if (!resp.ok) {
+    const body = await resp.text().catch(() => '');
+    throw new Error(`nexara HTTP ${resp.status}: ${body.slice(0, 200)}`);
+  }
+  const json = await resp.json();
+  if (!json || typeof json.text !== 'string') {
+    throw new Error('nexara: response missing .text field');
+  }
+  return {
+    text: json.text.trim(),
+    duration: typeof json.duration === 'number' ? json.duration : undefined,
+    language: typeof json.language === 'string' ? json.language : undefined,
+    raw: json,
+  };
+}

package/bot/poll.js

@@ -0,0 +1,304 @@
+/**
+ * Long-poll runner — owns the getUpdates loop, allowlist gating, update
+ * dispatch (text / forward / voice / command), and error backoff.
+ *
+ * Each accepted message is converted into a Telegram-Desktop-export-format
+ * JSON snippet and dropped into the memex inbox. The MCP server's existing
+ * inbox watcher does the rest (parse → DB → archive). Bot does NOT touch
+ * SQLite for ingest, only for /search and /recent reads.
+ *
+ * State (last update offset) lives in ~/.memex/data/bot-state.json so a
+ * restart resumes where it stopped instead of replaying Telegram's 24h
+ * server-side buffer.
+ */
+
+import { existsSync, readFileSync, writeFileSync, mkdirSync, renameSync, unlinkSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { TelegramClient } from './telegram.js';
+import { writeInboxMessage, tgUpdateToExportMessage } from './inbox.js';
+import { transcribe } from './nexara.js';
+import { searchMemex, recentMemex, renderSearchResults, renderRecent } from './search.js';
+
+const HELP_TEXT =
+  '*Memex bot* — your mobile capture surface.\n\n' +
+  'Send any text, forward any message, or send a voice note — it goes ' +
+  'straight into your memex memory.\n\n' +
+  'Commands:\n' +
+  '  /search <query> — search across all memex sources\n' +
+  '  /recent — most recent captures\n' +
+  '  /help — this message\n\n' +
+  '_Bot is local-only: when your laptop is asleep, messages buffer on ' +
+  "Telegram's side for ~24h and catch up on next poll. For longer gaps, " +
+  'export the chat from Telegram Desktop and drop result.json into ' +
+  '~/.memex/inbox/.';
+
+export class BotRunner {
+  constructor({ config, log }) {
+    this.config = config;
+    this.log = log || ((...a) => console.error('[bot]', ...a));
+    this.tg = new TelegramClient(config.telegram_bot_token);
+    this.allowlist = new Set(config.allowlist_user_ids);
+    this.lastUpdateId = this._loadOffset();
+    this.shuttingDown = false;
+  }
+
+  _loadOffset() {
+    try {
+      if (existsSync(this.config.state_path)) {
+        const s = JSON.parse(readFileSync(this.config.state_path, 'utf-8'));
+        if (typeof s.lastUpdateId === 'number') return s.lastUpdateId;
+      }
+    } catch (_) {}
+    return 0;
+  }
+
+  _saveOffset() {
+    try {
+      mkdirSync(dirname(this.config.state_path), { recursive: true });
+      const tmp = this.config.state_path + '.tmp';
+      writeFileSync(tmp, JSON.stringify({ lastUpdateId: this.lastUpdateId }));
+      renameSync(tmp, this.config.state_path);
+    } catch (e) {
+      this.log(`! could not save offset: ${e.message}`);
+    }
+  }
+
+  async start() {
+    let me;
+    try { me = await this.tg.getMe(); }
+    catch (e) {
+      throw new Error(`getMe failed — token invalid? ${e.message}`);
+    }
+    this.log(`✓ connected as @${me.username} (id=${me.id})`);
+    this.log(`  allowlist: ${[...this.allowlist].join(', ')}`);
+    this.log(`  inbox:     ${this.config.inbox_path}`);
+    this.log(`  voice:     ${this.config.voice_enabled ? 'enabled (Nexara)' : 'disabled'}`);
+    this.log(`  resuming from update_id ${this.lastUpdateId}`);
+
+    let backoffMs = 1000;
+    while (!this.shuttingDown) {
+      try {
+        const updates = await this.tg.getUpdates({
+          offset: this.lastUpdateId + 1,
+          timeout: 30,
+        });
+        backoffMs = 1000;
+        for (const update of updates) {
+          if (this.shuttingDown) break;
+          this.lastUpdateId = Math.max(this.lastUpdateId, update.update_id);
+          try {
+            await this._handleUpdate(update);
+          } catch (e) {
+            this.log(`! handler error on update ${update.update_id}: ${e.stack || e.message}`);
+          }
+        }
+        if (updates.length > 0) this._saveOffset();
+      } catch (e) {
+        if (this.shuttingDown) break;
+        this.log(`! poll error: ${e.message}, retrying in ${backoffMs}ms`);
+        await sleep(backoffMs);
+        backoffMs = Math.min(backoffMs * 2, 60000);
+      }
+    }
+    this.log('poll loop exited');
+  }
+
+  stop() {
+    this.shuttingDown = true;
+  }
+
+  async _handleUpdate(update) {
+    const msg = update.message;
+    if (!msg) return;
+    const fromId = msg.from?.id;
+    const chatId = msg.chat?.id;
+    if (!fromId || !chatId) return;
+
+    if (!this.allowlist.has(Number(fromId))) {
+      this.log(`! rejecting message from non-allowlisted user ${fromId} (${msg.from?.username || '?'})`);
+      try {
+        await this.tg.sendMessage(
+          chatId,
+          'This bot is private and not accepting messages. (Memex personal capture surface.)'
+        );
+      } catch (_) {}
+      return;
+    }
+
+    // Commands first.
+    const text = typeof msg.text === 'string' ? msg.text.trim() : '';
+    if (text.startsWith('/')) {
+      return this._handleCommand({ chatId, msg, text });
+    }
+
+    // Voice message
+    if (msg.voice) {
+      return this._handleVoice({ chatId, msg });
+    }
+
+    // Text or forwarded text/caption
+    if (text || msg.caption || msg.forward_from || msg.forward_from_chat || msg.forward_sender_name) {
+      return this._handleText({ chatId, msg });
+    }
+
+    // Anything else (photo / sticker / document without caption etc.) — not
+    // in v0.1 scope; acknowledge so user knows it didn't silently succeed.
+    try {
+      await this.tg.sendMessage(
+        chatId,
+        '⚠️ Only text, forwarded messages, and voice notes are captured in this version.'
+      );
+    } catch (_) {}
+  }
+
+  async _handleText({ chatId, msg }) {
+    const exportMsg = tgUpdateToExportMessage({
+      tgMessage: msg,
+      userId: this.allowlist.values().next().value, // single-user bot — first allowlisted id
+    });
+    if (!exportMsg.text) {
+      // Empty after assembly — nothing to capture.
+      return;
+    }
+    try {
+      const path = writeInboxMessage({
+        inboxPath: this.config.inbox_path,
+        userId: this.allowlist.values().next().value,
+        message: exportMsg,
+      });
+      this.log(`+ captured msg ${msg.message_id} (${exportMsg.text.length} chars) → ${path.split('/').pop()}`);
+      const ack = msg.forward_from || msg.forward_from_chat || msg.forward_sender_name
+        ? '✓ saved (forward)'
+        : '✓ saved';
+      await this.tg.sendMessage(chatId, ack, { reply_to_message_id: msg.message_id, disable_notification: true });
+    } catch (e) {
+      this.log(`! capture failed: ${e.message}`);
+      try { await this.tg.sendMessage(chatId, `⚠️ Couldn't save: ${e.message.slice(0, 200)}`); } catch (_) {}
+    }
+  }
+
+  async _handleVoice({ chatId, msg }) {
+    if (!this.config.voice_enabled) {
+      try {
+        await this.tg.sendMessage(
+          chatId,
+          '⚠️ Voice capture is disabled (no Nexara API key configured).',
+          { reply_to_message_id: msg.message_id }
+        );
+      } catch (_) {}
+      return;
+    }
+    try {
+      await this.tg.sendChatAction(chatId, 'typing');
+    } catch (_) {}
+
+    let audioBuf, mediaPath = null;
+    try {
+      const file = await this.tg.getFile(msg.voice.file_id);
+      audioBuf = await this.tg.downloadFile(file.file_path);
+    } catch (e) {
+      this.log(`! voice download failed: ${e.message}`);
+      try { await this.tg.sendMessage(chatId, `⚠️ Couldn't download voice: ${e.message.slice(0, 200)}`, { reply_to_message_id: msg.message_id }); } catch (_) {}
+      return;
+    }
+
+    // Persist OGG before transcription so the original is recoverable
+    // even if Nexara is down.
+    try {
+      mkdirSync(this.config.media_path, { recursive: true });
+      mediaPath = join(this.config.media_path, `${msg.message_id}.oga`);
+      writeFileSync(mediaPath, audioBuf);
+    } catch (e) {
+      this.log(`! could not save voice file: ${e.message}`);
+      mediaPath = null;
+    }
+
+    let transcript;
+    try {
+      const result = await transcribe({
+        audioBuffer: audioBuf,
+        apiKey: this.config.nexara_api_key,
+        filename: `${msg.message_id}.oga`,
+        mimeType: 'audio/ogg',
+        language: 'ru',
+      });
+      transcript = result.text;
+    } catch (e) {
+      this.log(`! transcription failed: ${e.message}`);
+      try { await this.tg.sendMessage(chatId, `⚠️ Transcription failed: ${e.message.slice(0, 200)}`, { reply_to_message_id: msg.message_id }); } catch (_) {}
+      return;
+    }
+    if (!transcript) {
+      try { await this.tg.sendMessage(chatId, '⚠️ Transcription was empty.', { reply_to_message_id: msg.message_id }); } catch (_) {}
+      return;
+    }
+
+    const userId = this.allowlist.values().next().value;
+    const exportMsg = tgUpdateToExportMessage({
+      tgMessage: msg,
+      userId,
+      textOverride: `🎙 ${transcript}`,
+      mediaPath,
+    });
+
+    try {
+      const path = writeInboxMessage({
+        inboxPath: this.config.inbox_path,
+        userId,
+        message: exportMsg,
+      });
+      this.log(`+ captured voice ${msg.message_id} (${transcript.length} chars) → ${path.split('/').pop()}`);
+      const preview = transcript.length > 140 ? transcript.slice(0, 140) + '…' : transcript;
+      await this.tg.sendMessage(
+        chatId,
+        `✓ transcribed: ${preview}`,
+        { reply_to_message_id: msg.message_id, disable_notification: true }
+      );
+    } catch (e) {
+      this.log(`! capture failed: ${e.message}`);
+      try { await this.tg.sendMessage(chatId, `⚠️ Couldn't save transcript: ${e.message.slice(0, 200)}`); } catch (_) {}
+    }
+  }
+
+  async _handleCommand({ chatId, msg, text }) {
+    // Strip @botname suffix Telegram appends in groups (e.g. /search@MyBot foo)
+    const m = text.match(/^\/(\w+)(?:@\w+)?(?:\s+([\s\S]*))?$/);
+    if (!m) return;
+    const cmd = m[1].toLowerCase();
+    const arg = (m[2] || '').trim();
+
+    if (cmd === 'help' || cmd === 'start') {
+      await this.tg.sendMessage(chatId, HELP_TEXT, { parse_mode: 'Markdown' });
+      return;
+    }
+    if (cmd === 'search') {
+      if (!arg) {
+        await this.tg.sendMessage(chatId, 'Usage: `/search <query>`', { parse_mode: 'Markdown' });
+        return;
+      }
+      const { rows, error } = searchMemex({ dbPath: this.config.db_path, query: arg, limit: 3 });
+      if (error) {
+        await this.tg.sendMessage(chatId, `⚠️ ${error}`);
+        return;
+      }
+      const body = renderSearchResults(arg, rows);
+      await this.tg.sendMessage(chatId, body, { parse_mode: 'Markdown' });
+      return;
+    }
+    if (cmd === 'recent') {
+      const { rows, error } = recentMemex({ dbPath: this.config.db_path, limit: 5 });
+      if (error) {
+        await this.tg.sendMessage(chatId, `⚠️ ${error}`);
+        return;
+      }
+      await this.tg.sendMessage(chatId, renderRecent(rows), { parse_mode: 'Markdown' });
+      return;
+    }
+    // Unknown command — capture it as text so it isn't lost.
+    return this._handleText({ chatId, msg });
+  }
+}
+
+function sleep(ms) {
+  return new Promise((res) => setTimeout(res, ms));
+}

package/bot/search.js

@@ -0,0 +1,155 @@
+/**
+ * Read-only search against memex.db for the bot's /search and /recent
+ * slash-commands. Uses better-sqlite3 directly — server.js uses WAL mode
+ * so concurrent readers are safe.
+ *
+ * Lazy DB open: don't error at startup if memex.db doesn't exist yet
+ * (first-time install where the bot fires before the MCP server has run).
+ * Open on first call instead and surface a friendly message.
+ */
+
+import Database from 'better-sqlite3';
+import { existsSync } from 'node:fs';
+
+let db = null;
+let openError = null;
+
+function ensureDb(dbPath) {
+  if (db) return db;
+  if (openError) return null;
+  if (!existsSync(dbPath)) {
+    openError = `memex.db not found at ${dbPath}. Run the memex MCP server at least once first.`;
+    return null;
+  }
+  try {
+    db = new Database(dbPath, { readonly: true, fileMustExist: true });
+    db.pragma('busy_timeout = 5000');
+  } catch (e) {
+    openError = `failed to open memex.db: ${e.message}`;
+    return null;
+  }
+  return db;
+}
+
+function shortDate(unixTs) {
+  if (!unixTs) return '?';
+  const d = new Date(unixTs * 1000);
+  // YYYY-MM-DD HH:MM
+  return d.toISOString().slice(0, 16).replace('T', ' ');
+}
+
+function sourceLabel(source) {
+  switch (source) {
+    case 'claude-code': return 'Claude Code';
+    case 'claude-cowork': return 'Cowork';
+    case 'cursor': return 'Cursor';
+    case 'obsidian': return 'Obsidian';
+    case 'telegram': return 'Telegram';
+    default: return source || '?';
+  }
+}
+
+function escapeMarkdown(s) {
+  // Telegram MarkdownV1 is forgiving — only escape backticks and underscores
+  // we actually emit as raw text; we wrap snippets in code blocks so this is
+  // mostly defensive.
+  return String(s || '').replace(/`/g, "'");
+}
+
+/**
+ * FTS5 search. Returns up to `limit` rows, one row per match (not grouped).
+ * Each row: { snippet, title, source, conversation_id, ts }
+ */
+export function searchMemex({ dbPath, query, limit = 3 }) {
+  const conn = ensureDb(dbPath);
+  if (!conn) return { error: openError };
+  const q = String(query || '').trim();
+  if (!q) return { error: 'empty query' };
+
+  let rows;
+  try {
+    rows = conn
+      .prepare(
+        `SELECT m.text AS text,
+                m.ts AS ts,
+                m.source AS source,
+                m.conversation_id AS conversation_id,
+                c.title AS title,
+                snippet(messages_fts, 0, '«', '»', '…', 24) AS snippet
+           FROM messages_fts
+           JOIN messages m ON messages_fts.rowid = m.id
+      LEFT JOIN conversations c ON c.conversation_id = m.conversation_id
+          WHERE messages_fts MATCH ?
+            AND (c.archived_at IS NULL OR c.archived_at = 0)
+       ORDER BY rank
+          LIMIT ?`
+      )
+      .all(q, limit);
+  } catch (e) {
+    return { error: `search failed: ${e.message}` };
+  }
+  return { rows };
+}
+
+/** Most recent N user-facing messages, time-sorted. */
+export function recentMemex({ dbPath, limit = 5 }) {
+  const conn = ensureDb(dbPath);
+  if (!conn) return { error: openError };
+  let rows;
+  try {
+    rows = conn
+      .prepare(
+        `SELECT m.text AS text,
+                m.ts AS ts,
+                m.source AS source,
+                m.conversation_id AS conversation_id,
+                c.title AS title
+           FROM messages m
+      LEFT JOIN conversations c ON c.conversation_id = m.conversation_id
+          WHERE (c.archived_at IS NULL OR c.archived_at = 0)
+            AND m.text IS NOT NULL
+            AND length(m.text) > 0
+       ORDER BY m.ts DESC
+          LIMIT ?`
+      )
+      .all(limit);
+  } catch (e) {
+    return { error: `recent failed: ${e.message}` };
+  }
+  return { rows };
+}
+
+/** Render search results as a single Telegram message body. */
+export function renderSearchResults(query, rows) {
+  if (!rows || rows.length === 0) {
+    return `🔍 No matches for *${escapeMarkdown(query)}*`;
+  }
+  const parts = [`🔍 Top ${rows.length} for *${escapeMarkdown(query)}*\n`];
+  for (let i = 0; i < rows.length; i++) {
+    const r = rows[i];
+    const title = r.title ? r.title.slice(0, 80) : r.conversation_id;
+    const snippet = (r.snippet || r.text || '').replace(/\s+/g, ' ').slice(0, 240);
+    parts.push(
+      `*${i + 1}.* [${sourceLabel(r.source)}] ${escapeMarkdown(title)}` +
+      `\n_${shortDate(r.ts)}_` +
+      `\n\`\`\`\n${escapeMarkdown(snippet)}\n\`\`\``
+    );
+  }
+  return parts.join('\n\n');
+}
+
+export function renderRecent(rows) {
+  if (!rows || rows.length === 0) return '📭 Nothing in memex yet.';
+  const parts = ['🕒 Most recent:\n'];
+  for (let i = 0; i < rows.length; i++) {
+    const r = rows[i];
+    const title = r.title ? r.title.slice(0, 80) : r.conversation_id;
+    const snippet = (r.text || '').replace(/\s+/g, ' ').slice(0, 200);
+    parts.push(
+      `*${i + 1}.* [${sourceLabel(r.source)}] ${escapeMarkdown(title)}` +
+      `\n_${shortDate(r.ts)}_` +
+      `\n${escapeMarkdown(snippet)}`
+    );
+  }
+  return parts.join('\n\n');
+}

package/bot/telegram.js

@@ -0,0 +1,96 @@
+/**
+ * Thin wrapper around Telegram Bot API HTTP endpoints used by the bot.
+ * Native fetch only — no node-telegram / telegraf / grammy dependency.
+ *
+ * - getUpdates: long-polling with offset persistence
+ * - sendMessage / sendChatAction
+ * - getFile + downloadFile (for voice messages)
+ *
+ * Errors that aren't fatal (network blips, Telegram 5xx, 429) are retried
+ * with exponential backoff in the run loop, not here.
+ */
+
+const API_BASE = 'https://api.telegram.org';
+
+export class TelegramClient {
+  constructor(token) {
+    if (!token) throw new Error('TelegramClient: token required');
+    this.token = token;
+    this.api = `${API_BASE}/bot${token}`;
+    this.fileApi = `${API_BASE}/file/bot${token}`;
+  }
+
+  async _post(method, body) {
+    const resp = await fetch(`${this.api}/${method}`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(60000),
+    });
+    const json = await resp.json().catch(() => ({}));
+    if (!resp.ok || json.ok === false) {
+      const err = new Error(`telegram ${method}: HTTP ${resp.status} ${json.description || ''}`);
+      err.status = resp.status;
+      err.tgError = json;
+      throw err;
+    }
+    return json.result;
+  }
+
+  /** Long-poll for updates. Returns array of update objects. */
+  async getUpdates({ offset, timeout = 30, allowedUpdates = ['message'] }) {
+    const url = new URL(`${this.api}/getUpdates`);
+    if (offset !== undefined && offset !== null) url.searchParams.set('offset', String(offset));
+    url.searchParams.set('timeout', String(timeout));
+    url.searchParams.set('allowed_updates', JSON.stringify(allowedUpdates));
+
+    // HTTP timeout = polling timeout + buffer for connection setup.
+    const resp = await fetch(url, { signal: AbortSignal.timeout((timeout + 15) * 1000) });
+    const json = await resp.json().catch(() => ({}));
+    if (!resp.ok || json.ok === false) {
+      const err = new Error(`telegram getUpdates: HTTP ${resp.status} ${json.description || ''}`);
+      err.status = resp.status;
+      throw err;
+    }
+    return json.result || [];
+  }
+
+  async sendMessage(chatId, text, opts = {}) {
+    const body = { chat_id: chatId, text };
+    if (opts.parse_mode) body.parse_mode = opts.parse_mode;
+    if (opts.reply_to_message_id) body.reply_to_message_id = opts.reply_to_message_id;
+    if (opts.disable_notification) body.disable_notification = true;
+    return this._post('sendMessage', body);
+  }
+
+  async sendChatAction(chatId, action) {
+    return this._post('sendChatAction', { chat_id: chatId, action });
+  }
+
+  /** getFile returns { file_id, file_unique_id, file_size, file_path }. */
+  async getFile(fileId) {
+    const url = new URL(`${this.api}/getFile`);
+    url.searchParams.set('file_id', fileId);
+    const resp = await fetch(url, { signal: AbortSignal.timeout(30000) });
+    const json = await resp.json().catch(() => ({}));
+    if (!resp.ok || json.ok === false) {
+      throw new Error(`telegram getFile: HTTP ${resp.status} ${json.description || ''}`);
+    }
+    return json.result;
+  }
+
+  /** Download a file by file_path (returned by getFile). Returns Buffer. */
+  async downloadFile(filePath) {
+    const url = `${this.fileApi}/${filePath}`;
+    const resp = await fetch(url, { signal: AbortSignal.timeout(120000) });
+    if (!resp.ok) {
+      throw new Error(`telegram downloadFile: HTTP ${resp.status}`);
+    }
+    const buf = await resp.arrayBuffer();
+    return Buffer.from(buf);
+  }
+
+  async getMe() {
+    return this._post('getMe', {});
+  }
+}