arisa 2.0.0

package/src/shared/db.ts

@@ -0,0 +1,304 @@
+/**
+ * @module shared/db
+ * @role Unified persistence layer using deepbase
+ * @responsibilities
+ *   - Persist scheduled tasks, authorized users, onboarded users, queue messages
+ *   - Provide type-safe operations with JSON storage
+ *   - Auto-connect on first operation
+ * @dependencies deepbase, shared/types, shared/config
+ * @effects Disk I/O (runtime db directory)
+ */
+
+import DeepBase from "deepbase";
+import { config } from "./config";
+import type { ScheduledTask, AttachmentRecord, MessageRecord } from "./types";
+import { DeepbaseSecure } from "./deepbase-secure";
+import { existsSync, readFileSync, writeFileSync, mkdirSync, copyFileSync } from "fs";
+import { randomBytes } from "crypto";
+import { dirname } from "path";
+
+const DB_PATH = `${config.arisaDir}/db`;
+const ARISA_DB_FILE = `${DB_PATH}/arisa.json`;
+const LEGACY_DB_FILE = `${DB_PATH}/tinyclaw.json`;
+
+function readDbJson(path: string): Record<string, any> {
+  try {
+    if (!existsSync(path)) return {};
+    const raw = readFileSync(path, "utf8").trim();
+    if (!raw) return {};
+    const parsed = JSON.parse(raw);
+    return parsed && typeof parsed === "object" ? parsed : {};
+  } catch {
+    return {};
+  }
+}
+
+function mergeLegacyIntoArisa(): void {
+  if (!existsSync(LEGACY_DB_FILE)) return;
+
+  // If arisa DB doesn't exist yet, seed it from legacy and keep legacy file as backup/history.
+  if (!existsSync(ARISA_DB_FILE)) {
+    try {
+      mkdirSync(DB_PATH, { recursive: true });
+      copyFileSync(LEGACY_DB_FILE, ARISA_DB_FILE);
+    } catch {
+      // Best-effort migration; if copy fails, app can still run on a fresh arisa DB.
+    }
+    return;
+  }
+
+  const arisa = readDbJson(ARISA_DB_FILE);
+  const legacy = readDbJson(LEGACY_DB_FILE);
+  let changed = false;
+
+  for (const [collection, legacyCollection] of Object.entries(legacy)) {
+    if (!legacyCollection || typeof legacyCollection !== "object") continue;
+
+    const arisaCollection = arisa[collection];
+    if (!arisaCollection || typeof arisaCollection !== "object") {
+      arisa[collection] = legacyCollection;
+      changed = true;
+      continue;
+    }
+
+    for (const [key, value] of Object.entries(legacyCollection as Record<string, any>)) {
+      if (!(key in arisaCollection)) {
+        arisaCollection[key] = value;
+        changed = true;
+      }
+    }
+  }
+
+  if (changed) {
+    try {
+      writeFileSync(ARISA_DB_FILE, JSON.stringify(arisa, null, 4));
+    } catch {
+      // Ignore write failures; runtime will continue with current DB state.
+    }
+  }
+}
+
+// Ensure legacy data is available in arisa DB before DeepBase picks a file.
+mergeLegacyIntoArisa();
+
+// Initialize deepbase with the storage directory
+const db = new DeepBase({
+  path: DB_PATH,
+  name: "arisa",
+});
+
+// Initialize encrypted secrets database
+function getOrCreateEncryptionKey(): string {
+  const keyPath = `${config.arisaDir}/.encryption_key`;
+  const keyDir = dirname(keyPath);
+
+  if (!existsSync(keyDir)) {
+    mkdirSync(keyDir, { recursive: true });
+  }
+
+  if (existsSync(keyPath)) {
+    return readFileSync(keyPath, 'utf-8').trim();
+  }
+
+  const key = randomBytes(32).toString('hex');
+  writeFileSync(keyPath, key, { mode: 0o600 });
+  return key;
+}
+
+const secretsDb = new DeepbaseSecure({
+  path: DB_PATH,
+  name: "secrets",
+  encryptionKey: getOrCreateEncryptionKey(),
+});
+
+/**
+ * Helper functions for common operations
+ */
+
+// Tasks
+export async function getTasks(): Promise<ScheduledTask[]> {
+  const tasks = await db.values("tasks");
+  return tasks || [];
+}
+
+export async function getTask(id: string): Promise<ScheduledTask | null> {
+  return await db.get("tasks", id);
+}
+
+export async function addTask(task: ScheduledTask): Promise<void> {
+  await db.set("tasks", task.id, task);
+}
+
+export async function updateTask(id: string, updates: Partial<ScheduledTask>): Promise<void> {
+  const existing = await db.get("tasks", id);
+  if (existing) {
+    await db.set("tasks", id, { ...existing, ...updates });
+  }
+}
+
+export async function deleteTask(id: string): Promise<void> {
+  await db.del("tasks", id);
+}
+
+export async function deleteTasks(filter: Partial<ScheduledTask>): Promise<number> {
+  const tasks = await getTasks();
+  let deleted = 0;
+
+  for (const task of tasks) {
+    const matches = Object.entries(filter).every(([key, value]) => {
+      return task[key as keyof ScheduledTask] === value;
+    });
+
+    if (matches) {
+      await db.del("tasks", task.id);
+      deleted++;
+    }
+  }
+
+  return deleted;
+}
+
+// Authorized users
+export async function isAuthorized(userId: string): Promise<boolean> {
+  const user = await db.get("authorized", userId);
+  return user !== null && user !== undefined;
+}
+
+export async function addAuthorized(userId: string): Promise<void> {
+  await db.set("authorized", userId, { userId });
+}
+
+export async function getAuthorizedUsers(): Promise<string[]> {
+  const users = await db.keys("authorized");
+  return users || [];
+}
+
+// Onboarded users
+export async function isOnboarded(userId: string): Promise<boolean> {
+  const user = await db.get("onboarded", userId);
+  return user !== null && user !== undefined;
+}
+
+export async function addOnboarded(userId: string): Promise<void> {
+  await db.set("onboarded", userId, { userId });
+}
+
+export async function getOnboardedUsers(): Promise<string[]> {
+  const users = await db.keys("onboarded");
+  return users || [];
+}
+
+// Queue operations
+export async function enqueueMessage(message: {
+  id: string;
+  chatId: string | number;
+  text: string;
+  type: "heartbeat" | "message";
+}): Promise<void> {
+  await db.set("queue", message.id, {
+    ...message,
+    timestamp: Date.now(),
+  });
+}
+
+export async function dequeueMessages(limit?: number): Promise<any[]> {
+  const messages = await db.values("queue");
+  if (!messages || messages.length === 0) return [];
+
+  const sorted = messages.sort((a: any, b: any) => a.timestamp - b.timestamp);
+  const batch = limit ? sorted.slice(0, limit) : sorted;
+
+  // Delete the dequeued messages
+  for (const msg of batch) {
+    await db.del("queue", msg.id);
+  }
+
+  return batch;
+}
+
+export async function getQueueSize(): Promise<number> {
+  const messages = await db.keys("queue");
+  return messages ? messages.length : 0;
+}
+
+export async function clearQueue(): Promise<void> {
+  const keys = await db.keys("queue");
+  if (keys) {
+    for (const key of keys) {
+      await db.del("queue", key);
+    }
+  }
+}
+
+// Settings (auth token, etc.)
+export async function getSetting(key: string): Promise<string | null> {
+  const val = await db.get("settings", key);
+  return val?.value ?? null;
+}
+
+export async function setSetting(key: string, value: string): Promise<void> {
+  await db.set("settings", key, { value });
+}
+
+// Attachments
+export async function addAttachment(record: AttachmentRecord): Promise<void> {
+  await db.set("attachments", record.id, record);
+}
+
+export async function getAttachments(chatId?: string): Promise<AttachmentRecord[]> {
+  const all = (await db.values("attachments")) || [];
+  if (!chatId) return all;
+  return all.filter((a: AttachmentRecord) => a.chatId === chatId);
+}
+
+export async function getAttachment(id: string): Promise<AttachmentRecord | null> {
+  return await db.get("attachments", id);
+}
+
+export async function deleteAttachment(id: string): Promise<void> {
+  await db.del("attachments", id);
+}
+
+export async function getExpiredAttachments(maxAgeDays: number): Promise<AttachmentRecord[]> {
+  const all = (await db.values("attachments")) || [];
+  const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000;
+  return all.filter((a: AttachmentRecord) => a.createdAt < cutoff);
+}
+
+// Messages (ledger)
+export async function saveMessageRecord(record: MessageRecord): Promise<void> {
+  await db.set("messages", record.id, record);
+}
+
+export async function getMessageRecord(chatId: string, messageId: number): Promise<MessageRecord | null> {
+  return await db.get("messages", `${chatId}_${messageId}`);
+}
+
+export async function cleanupOldMessages(maxAgeDays: number): Promise<number> {
+  const all = (await db.values("messages")) || [];
+  const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000;
+  let deleted = 0;
+  for (const record of all) {
+    if ((record as MessageRecord).timestamp < cutoff) {
+      await db.del("messages", (record as MessageRecord).id);
+      deleted++;
+    }
+  }
+  return deleted;
+}
+
+// Secrets (API keys stored encrypted)
+export async function getSecret(key: string): Promise<string | null> {
+  const val = await secretsDb.get("secrets", key);
+  return val?.value ?? null;
+}
+
+export async function setSecret(key: string, value: string): Promise<void> {
+  await secretsDb.set("secrets", key, { value });
+}
+
+export async function deleteSecret(key: string): Promise<void> {
+  await secretsDb.del("secrets", key);
+}
+
+export { db, secretsDb };

package/src/shared/deepbase-secure.ts

@@ -0,0 +1,39 @@
+import CryptoJS from 'crypto-js';
+import DeepBase from 'deepbase';
+import { JsonDriver } from 'deepbase-json';
+
+interface DeepbaseSecureOptions {
+  encryptionKey: string;
+  path: string;
+  name: string;
+}
+
+export class DeepbaseSecure extends DeepBase {
+  constructor(opts: DeepbaseSecureOptions) {
+    const encryptionKey = opts.encryptionKey;
+    const { path, name } = opts;
+
+    // Create JSON driver with encryption
+    const driver = new JsonDriver({
+      path,
+      name,
+      stringify: (obj: any) => {
+        const iv = CryptoJS.lib.WordArray.random(128 / 8);
+        const encrypted = CryptoJS.AES.encrypt(
+          JSON.stringify(obj),
+          encryptionKey,
+          { iv }
+        );
+        return iv.toString(CryptoJS.enc.Hex) + ':' + encrypted.toString();
+      },
+      parse: (encryptedData: string) => {
+        const [ivHex, encrypted] = encryptedData.split(':');
+        const iv = CryptoJS.enc.Hex.parse(ivHex);
+        const bytes = CryptoJS.AES.decrypt(encrypted, encryptionKey, { iv });
+        return JSON.parse(bytes.toString(CryptoJS.enc.Utf8));
+      }
+    });
+
+    super(driver);
+  }
+}

package/src/shared/logger.ts

@@ -0,0 +1,42 @@
+/**
+ * @module shared/logger
+ * @role Structured logging to runtime logs dir and stdout.
+ * @responsibilities
+ *   - Create named loggers per component (core, daemon, telegram, scheduler)
+ *   - Write timestamped log lines to file + console
+ *   - Ensure log directory exists
+ * @dependencies shared/config
+ * @effects Writes to disk (logs dir), writes to stdout
+ */
+
+import { appendFileSync, mkdirSync, existsSync } from "fs";
+import { join } from "path";
+import { config } from "./config";
+
+type Level = "DEBUG" | "INFO" | "WARN" | "ERROR";
+
+export function createLogger(component: string) {
+  const logFile = join(config.logsDir, `${component}.log`);
+
+  if (!existsSync(config.logsDir)) {
+    mkdirSync(config.logsDir, { recursive: true });
+  }
+
+  function write(level: Level, message: string) {
+    const timestamp = new Date().toISOString();
+    const line = `[${timestamp}] [${level}] ${message}\n`;
+    console.log(line.trim());
+    try {
+      appendFileSync(logFile, line);
+    } catch {
+      // If we can't write to log file, at least console output happened
+    }
+  }
+
+  return {
+    debug: (msg: string) => write("DEBUG", msg),
+    info: (msg: string) => write("INFO", msg),
+    warn: (msg: string) => write("WARN", msg),
+    error: (msg: string) => write("ERROR", msg),
+  };
+}

package/src/shared/paths.ts

@@ -0,0 +1,90 @@
+/**
+ * @module shared/paths
+ * @role Resolve project and runtime data directories with migration-safe defaults.
+ * @responsibilities
+ *   - Resolve project directory (supports ARISA_PROJECT_DIR override)
+ *   - Resolve runtime data directory (prefers ~/.arisa by default)
+ *   - Migrate legacy project-local dirs (.tinyclaw/.arisa) into ~/.arisa
+ *   - Support ARISA_DATA_DIR override for advanced deployments
+ */
+
+import { cpSync, existsSync, mkdirSync, readdirSync, renameSync, rmSync } from "fs";
+import { homedir } from "os";
+import { isAbsolute, join, resolve } from "path";
+
+const DEFAULT_PROJECT_DIR = join(import.meta.dir, "..", "..");
+const PROJECT_DIR = process.env.ARISA_PROJECT_DIR
+  ? resolve(process.env.ARISA_PROJECT_DIR)
+  : DEFAULT_PROJECT_DIR;
+
+const HOME_DATA_DIR = join(homedir(), ".arisa");
+const PROJECT_ARISA_DIR = join(PROJECT_DIR, ".arisa");
+const LEGACY_DATA_DIR = join(PROJECT_DIR, ".tinyclaw");
+
+function samePath(a: string, b: string): boolean {
+  return resolve(a) === resolve(b);
+}
+
+function hasFiles(dir: string): boolean {
+  try {
+    return existsSync(dir) && readdirSync(dir).length > 0;
+  } catch {
+    return false;
+  }
+}
+
+function moveOrMerge(sourceDir: string, targetDir: string): void {
+  if (!existsSync(sourceDir) || samePath(sourceDir, targetDir)) return;
+
+  try {
+    if (!existsSync(targetDir)) {
+      mkdirSync(resolve(targetDir, ".."), { recursive: true });
+      renameSync(sourceDir, targetDir);
+      return;
+    }
+
+    // Merge source into target and drop source afterward.
+    cpSync(sourceDir, targetDir, { recursive: true, force: false, errorOnExist: false });
+    rmSync(sourceDir, { recursive: true, force: true });
+  } catch {
+    // Do not crash startup if migration has a filesystem issue.
+  }
+}
+
+function resolveOverrideDataDir(): string | null {
+  const override = process.env.ARISA_DATA_DIR?.trim();
+  if (!override) return null;
+  return isAbsolute(override) ? override : resolve(PROJECT_DIR, override);
+}
+
+function resolveDataDir(): string {
+  const overrideDir = resolveOverrideDataDir();
+  if (overrideDir) {
+    return overrideDir;
+  }
+
+  // One-time migration from local runtime dirs to ~/.arisa.
+  if (!hasFiles(HOME_DATA_DIR)) {
+    if (hasFiles(PROJECT_ARISA_DIR)) {
+      moveOrMerge(PROJECT_ARISA_DIR, HOME_DATA_DIR);
+    } else if (hasFiles(LEGACY_DATA_DIR)) {
+      moveOrMerge(LEGACY_DATA_DIR, HOME_DATA_DIR);
+    }
+  } else {
+    // If ~/.arisa already exists, still merge any local leftovers into it.
+    if (hasFiles(PROJECT_ARISA_DIR)) moveOrMerge(PROJECT_ARISA_DIR, HOME_DATA_DIR);
+    if (hasFiles(LEGACY_DATA_DIR)) moveOrMerge(LEGACY_DATA_DIR, HOME_DATA_DIR);
+  }
+
+  if (!existsSync(HOME_DATA_DIR)) {
+    mkdirSync(HOME_DATA_DIR, { recursive: true });
+  }
+
+  return HOME_DATA_DIR;
+}
+
+export const projectDir = PROJECT_DIR;
+export const preferredDataDir = HOME_DATA_DIR;
+export const projectLocalDataDir = PROJECT_ARISA_DIR;
+export const legacyDataDir = LEGACY_DATA_DIR;
+export const dataDir = resolveDataDir();

package/src/shared/ports.ts

@@ -0,0 +1,98 @@
+/**
+ * @module shared/ports
+ * @role Process cleanup via PID files + /proc scan, retry-aware Bun.serve.
+ * @effects Reads/writes runtime pid files, kills processes via SIGKILL
+ */
+
+import { existsSync, readFileSync, readdirSync, writeFileSync, unlinkSync, mkdirSync } from "fs";
+import { join, dirname } from "path";
+import { dataDir } from "./paths";
+
+const ARISA_DIR = dataDir;
+
+function pidPath(name: string): string {
+  return join(ARISA_DIR, `${name}.pid`);
+}
+
+// Patterns to match in /proc cmdline for each process type
+const CMDLINE_PATTERNS: Record<string, string> = {
+  daemon: "daemon/index.ts",
+  core: "core/index.ts",
+};
+
+/**
+ * Kill previous instances of a named process, then write our PID.
+ * Uses PID file + /proc scan for robustness in containers.
+ */
+export function claimProcess(name: string): void {
+  const myPid = process.pid;
+
+  // 1. Kill from PID file
+  const path = pidPath(name);
+  if (existsSync(path)) {
+    try {
+      const oldPid = parseInt(readFileSync(path, "utf8").trim(), 10);
+      if (oldPid && oldPid !== myPid) {
+        try { process.kill(oldPid, "SIGKILL"); } catch {}
+      }
+    } catch {}
+  }
+
+  // 2. Scan /proc for any matching processes (Linux containers)
+  const pattern = CMDLINE_PATTERNS[name];
+  if (pattern) {
+    killByPattern(pattern, myPid);
+  }
+
+  // 3. Write our PID
+  if (!existsSync(dirname(path))) mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, String(myPid));
+
+  // 4. Brief pause to let OS release resources
+  Bun.sleepSync(200);
+}
+
+// Scan /proc cmdline and kill processes matching a pattern (Linux only).
+function killByPattern(pattern: string, excludePid: number): void {
+  try {
+    if (!existsSync("/proc")) return;
+    const dirs = readdirSync("/proc").filter((d) => /^\d+$/.test(d));
+    for (const pid of dirs) {
+      const numPid = Number(pid);
+      if (numPid === excludePid) continue;
+      try {
+        const cmdline = readFileSync(`/proc/${pid}/cmdline`, "utf8");
+        if (cmdline.includes(pattern)) {
+          process.kill(numPid, "SIGKILL");
+        }
+      } catch {}
+    }
+  } catch {}
+}
+
+/**
+ * Remove our PID file on clean shutdown.
+ */
+export function releaseProcess(name: string): void {
+  try { unlinkSync(pidPath(name)); } catch {}
+}
+
+/**
+ * Bun.serve() with retry — if port is busy, waits for previous process to die.
+ */
+export async function serveWithRetry(
+  options: Parameters<typeof Bun.serve>[0],
+  retries = 5,
+): Promise<ReturnType<typeof Bun.serve>> {
+  for (let i = 0; i < retries; i++) {
+    try {
+      return Bun.serve(options);
+    } catch (e: any) {
+      if (e?.code !== "EADDRINUSE" || i === retries - 1) throw e;
+      const port = (options as any).port ?? "?";
+      console.log(`[ports] Port ${port} busy, retrying (${i + 1}/${retries})...`);
+      await new Promise((r) => setTimeout(r, 1000));
+    }
+  }
+  throw new Error("unreachable");
+}

package/src/shared/secrets.ts

@@ -0,0 +1,136 @@
+/**
+ * @module shared/secrets
+ * @role Encrypted secrets storage using DeepbaseSecure
+ * @responsibilities
+ *   - Generate/load encryption key
+ *   - Store API keys encrypted at rest
+ *   - Provide type-safe getters for secrets
+ * @dependencies DeepbaseSecure, crypto-js
+ */
+
+import { join } from "path";
+import { existsSync, readFileSync, writeFileSync, mkdirSync, readdirSync, renameSync } from "fs";
+import CryptoJS from "crypto-js";
+import { DeepbaseSecure } from "./deepbase-secure";
+import { dataDir } from "./paths";
+
+const ARISA_DIR = dataDir;
+const ENCRYPTION_KEY_PATH = join(ARISA_DIR, ".encryption_key");
+const SECRETS_DB_PATH = join(ARISA_DIR, "db");
+
+// Ensure runtime data and db dirs exist
+mkdirSync(join(ARISA_DIR, "db"), { recursive: true });
+
+/**
+ * Load or generate encryption key
+ */
+function getEncryptionKey(): string {
+  if (existsSync(ENCRYPTION_KEY_PATH)) {
+    return readFileSync(ENCRYPTION_KEY_PATH, "utf8").trim();
+  }
+
+  // Generate random 256-bit key
+  const key = CryptoJS.lib.WordArray.random(256 / 8).toString(CryptoJS.enc.Hex);
+  writeFileSync(ENCRYPTION_KEY_PATH, key, { mode: 0o600 });
+  return key;
+}
+
+const encryptionKey = getEncryptionKey();
+let secretsDb = createSecretsDb();
+
+function createSecretsDb(): DeepbaseSecure {
+  return new DeepbaseSecure({
+    path: SECRETS_DB_PATH,
+    name: "secrets",
+    encryptionKey,
+  });
+}
+
+// Initialize connection
+let connectionPromise: Promise<void> | null = null;
+let recoveredOnce = false;
+
+function looksCorrupted(err: unknown): boolean {
+  const msg = err instanceof Error ? `${err.message}\n${err.stack || ""}` : String(err);
+  return /Malformed UTF-8 data|Unexpected token|JSON|decrypt|invalid/i.test(msg);
+}
+
+function backupCorruptSecretsDb(): void {
+  try {
+    const timestamp = Date.now();
+    for (const file of readdirSync(SECRETS_DB_PATH)) {
+      if (!file.startsWith("secrets")) continue;
+      const src = join(SECRETS_DB_PATH, file);
+      const dst = join(SECRETS_DB_PATH, `${file}.corrupt.${timestamp}`);
+      try {
+        renameSync(src, dst);
+      } catch {
+        // Best-effort backup; ignore per-file failures.
+      }
+    }
+  } catch {
+    // Ignore backup errors; recovery will still retry with a fresh DB handle.
+  }
+}
+
+async function ensureConnected(): Promise<void> {
+  if (!connectionPromise) {
+    connectionPromise = (async () => {
+      try {
+        await secretsDb.connect();
+      } catch (err) {
+        if (!recoveredOnce && looksCorrupted(err)) {
+          recoveredOnce = true;
+          console.warn("[secrets] Encrypted secrets DB looks corrupted; backing up and recreating it.");
+          backupCorruptSecretsDb();
+          secretsDb = createSecretsDb();
+          await secretsDb.connect();
+          return;
+        }
+        throw err;
+      }
+    })().catch((err) => {
+      connectionPromise = null;
+      throw err;
+    });
+  }
+  await connectionPromise;
+}
+
+/**
+ * Get a secret by key
+ */
+export async function getSecret(key: string): Promise<string | undefined> {
+  try {
+    await ensureConnected();
+    return await secretsDb.get("secrets", key);
+  } catch (err) {
+    console.warn(`[secrets] Could not read ${key} from encrypted DB: ${err}`);
+    return undefined;
+  }
+}
+
+/**
+ * Set a secret by key
+ */
+export async function setSecret(key: string, value: string): Promise<void> {
+  await ensureConnected();
+  await secretsDb.set("secrets", key, value);
+}
+
+/**
+ * Delete a secret by key
+ */
+export async function deleteSecret(key: string): Promise<void> {
+  await ensureConnected();
+  await secretsDb.del("secrets", key);
+}
+
+/**
+ * Type-safe getters for known secrets
+ */
+export const secrets = {
+  telegram: () => getSecret("TELEGRAM_BOT_TOKEN"),
+  openai: () => getSecret("OPENAI_API_KEY"),
+  elevenlabs: () => getSecret("ELEVENLABS_API_KEY"),
+};