@tenex-chat/backend 0.9.1

package/src/services/pubkey-gate/PubkeyGateService.ts

@@ -0,0 +1,93 @@
+import { getTrustPubkeyService, type TrustResult } from "@/services/trust-pubkeys";
+import { logger } from "@/utils/logger";
+import { trace } from "@opentelemetry/api";
+import type { NDKEvent } from "@nostr-dev-kit/ndk";
+
+/**
+ * PubkeyGateService gates incoming events based on pubkey authorization.
+ *
+ * An event is allowed through if its author's pubkey is trusted by TrustPubkeyService
+ * (whitelisted, backend, or known agent). All other events are silently dropped.
+ *
+ * Design principles:
+ * - Fail-closed: if the trust check errors, the event is denied
+ * - Sanitized logging: only pubkey prefix + event kind for observability
+ * - OpenTelemetry telemetry for audit trail
+ * - Sync trust checks for performance (requires backend pubkey cache initialization)
+ */
+export class PubkeyGateService {
+    private static instance: PubkeyGateService;
+
+    private constructor() {}
+
+    static getInstance(): PubkeyGateService {
+        if (!PubkeyGateService.instance) {
+            PubkeyGateService.instance = new PubkeyGateService();
+        }
+        return PubkeyGateService.instance;
+    }
+
+    /**
+     * Check if an incoming event should be allowed through the gate.
+     *
+     * Uses synchronous trust checks for performance. The backend pubkey cache
+     * must be initialized via TrustPubkeyService.initializeBackendPubkeyCache()
+     * before calling this method for accurate backend pubkey checks.
+     *
+     * @param event The incoming NDKEvent to check
+     * @returns true if the event should be routed, false if it should be dropped
+     */
+    shouldAllowEvent(event: NDKEvent): boolean {
+        const pubkey = event.pubkey;
+
+        if (!pubkey) {
+            logger.debug("[PUBKEY_GATE] Event denied: missing pubkey", {
+                kind: event.kind,
+            });
+            this.recordDenied(event, "no_pubkey");
+            return false;
+        }
+
+        let trustResult: TrustResult;
+        try {
+            trustResult = getTrustPubkeyService().isTrustedEventSync(event);
+        } catch (error) {
+            // Fail-closed: if the trust check errors, deny
+            logger.warn("[PUBKEY_GATE] Trust check failed, denying event (fail-closed)", {
+                pubkey: pubkey.substring(0, 8),
+                kind: event.kind,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            this.recordDenied(event, "error");
+            return false;
+        }
+
+        if (!trustResult.trusted) {
+            logger.debug("[PUBKEY_GATE] Event denied: untrusted pubkey", {
+                pubkey: pubkey.substring(0, 8),
+                kind: event.kind,
+            });
+            this.recordDenied(event, "untrusted");
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Record a denied event in telemetry for audit trail.
+     */
+    private recordDenied(event: NDKEvent, reason: string): void {
+        trace.getActiveSpan()?.addEvent("pubkey_gate.denied", {
+            "gate.pubkey": event.pubkey?.substring(0, 8) ?? "unknown",
+            "gate.kind": event.kind ?? 0,
+            "gate.reason": reason,
+        });
+    }
+}
+
+/**
+ * Get the PubkeyGateService singleton instance
+ */
+export const getPubkeyGateService = (): PubkeyGateService =>
+    PubkeyGateService.getInstance();

package/src/services/pubkey-gate/index.ts

@@ -0,0 +1 @@
+export { PubkeyGateService, getPubkeyGateService } from "./PubkeyGateService";

package/src/services/rag/EmbeddingProviderFactory.ts

@@ -0,0 +1,292 @@
+import * as path from "node:path";
+import { fileExists, readJsonFile } from "@/lib/fs";
+import { config } from "@/services/ConfigService";
+import { resolveApiKey } from "@/services/config/types";
+import { logger } from "@/utils/logger";
+import {
+    type EmbeddingProvider,
+    LocalTransformerEmbeddingProvider,
+    OpenAIEmbeddingProvider,
+} from "@/services/embedding";
+
+/**
+ * Configuration for embedding providers
+ */
+export interface EmbeddingConfig {
+    provider: string; // "local" or any configured provider ID (openai, openrouter, etc.)
+    model: string;
+    apiKey?: string;
+    baseUrl?: string;
+}
+
+export interface EmbeddingConfigOptions {
+    /**
+     * Path to the project metadata directory (e.g., ~/.tenex/projects/<dTag>)
+     */
+    metadataPath?: string;
+    /**
+     * Path to the project repository root (used to derive project .tenex path)
+     */
+    projectPath?: string;
+    /**
+     * Scope for resolution. "auto" (default) tries project then global.
+     * "project" prioritizes project config but will still fall back to global defaults.
+     * "global" only reads the global config.
+     */
+    scope?: "auto" | "project" | "global";
+}
+
+/**
+ * Raw configuration as it may appear in JSON files
+ * Supports both old format (string or partial object) and new format (full object)
+ */
+type RawEmbeddingConfig =
+    | string // Old format: just model name
+    | { model: string; provider?: never } // Old format: object with just model
+    | { provider: string; model: string; apiKey?: string; baseUrl?: string }; // New format
+
+function isRawEmbeddingConfig(value: unknown): value is RawEmbeddingConfig {
+    if (typeof value === "string") {
+        return true;
+    }
+
+    if (typeof value !== "object" || value === null) {
+        return false;
+    }
+
+    const obj = value as Record<string, unknown>;
+
+    // Must have a model
+    if (!("model" in obj) || typeof obj.model !== "string") {
+        return false;
+    }
+
+    // If provider is present, must be a string
+    if ("provider" in obj && typeof obj.provider !== "string") {
+        return false;
+    }
+
+    // If apiKey is present, must be string
+    if ("apiKey" in obj && typeof obj.apiKey !== "string") {
+        return false;
+    }
+
+    // If baseUrl is present, must be string
+    if ("baseUrl" in obj && typeof obj.baseUrl !== "string") {
+        return false;
+    }
+
+    return true;
+}
+
+/**
+ * Factory for creating embedding providers based on configuration
+ */
+export class EmbeddingProviderFactory {
+    private static readonly EMBED_CONFIG_FILE = "embed.json";
+    private static readonly DEFAULT_CONFIG: EmbeddingConfig = {
+        provider: "local",
+        model: "Xenova/all-MiniLM-L6-v2",
+    };
+
+    /**
+     * Base URLs for known OpenAI-compatible providers
+     */
+    private static readonly PROVIDER_BASE_URLS: Record<string, string> = {
+        openai: "https://api.openai.com/v1",
+        openrouter: "https://openrouter.ai/api/v1",
+    };
+
+    /**
+     * Create an embedding provider based on configuration
+     */
+    static async create(
+        customConfig?: EmbeddingConfig,
+        options?: EmbeddingConfigOptions
+    ): Promise<EmbeddingProvider> {
+        const embeddingConfig = customConfig || (await EmbeddingProviderFactory.loadConfiguration(options));
+
+        logger.debug(`Creating embedding provider: ${embeddingConfig.provider}/${embeddingConfig.model}`);
+
+        // Local provider
+        if (embeddingConfig.provider === "local") {
+            return new LocalTransformerEmbeddingProvider(embeddingConfig.model);
+        }
+
+        // All other providers are treated as OpenAI-compatible
+        if (!embeddingConfig.apiKey) {
+            throw new Error(
+                `API key required for ${embeddingConfig.provider}. Configure with 'tenex setup embed'.`
+            );
+        }
+
+        const baseUrl = embeddingConfig.baseUrl ||
+            EmbeddingProviderFactory.PROVIDER_BASE_URLS[embeddingConfig.provider] ||
+            "https://api.openai.com/v1";
+
+        return new OpenAIEmbeddingProvider(embeddingConfig.apiKey, embeddingConfig.model, baseUrl);
+    }
+
+    /**
+     * Load embedding configuration using configured paths.
+     * If a project path is provided, project config is tried first, then global.
+     * If no paths are provided, falls back to CWD project path (legacy) and then global.
+     */
+    static async loadConfiguration(options?: EmbeddingConfigOptions): Promise<EmbeddingConfig> {
+        try {
+            const basePaths = EmbeddingProviderFactory.resolveConfigBases(options);
+
+            for (const basePath of basePaths) {
+                const configPath = path.join(basePath, EmbeddingProviderFactory.EMBED_CONFIG_FILE);
+                if (!(await fileExists(configPath))) continue;
+
+                const rawConfig = await readJsonFile<unknown>(configPath);
+                if (!isRawEmbeddingConfig(rawConfig)) {
+                    logger.warn(`Invalid embedding config at ${configPath}, using defaults`);
+                    return EmbeddingProviderFactory.DEFAULT_CONFIG;
+                }
+
+                logger.debug(`Loaded embedding config from ${configPath}`);
+                return await EmbeddingProviderFactory.parseConfig(rawConfig);
+            }
+
+            logger.debug("No embedding configuration found, using defaults");
+            return EmbeddingProviderFactory.DEFAULT_CONFIG;
+        } catch (error) {
+            logger.warn("Failed to load embedding configuration, using defaults", { error });
+            return EmbeddingProviderFactory.DEFAULT_CONFIG;
+        }
+    }
+
+    /**
+     * Load provider credentials from providers.json
+     */
+    private static async loadProviderCredentials(providerId: string): Promise<{ apiKey?: string; baseUrl?: string }> {
+        try {
+            const globalPath = config.getGlobalPath();
+            const providersConfig = await config.loadTenexProviders(globalPath);
+            const providerCreds = providersConfig.providers[providerId];
+            return {
+                apiKey: resolveApiKey(providerCreds?.apiKey),
+                baseUrl: providerCreds?.baseUrl,
+            };
+        } catch (error) {
+            logger.debug(`Failed to load provider credentials for ${providerId}`, { error });
+            return {};
+        }
+    }
+
+    /**
+     * Parse and validate embedding configuration
+     */
+    private static async parseConfig(raw: RawEmbeddingConfig): Promise<EmbeddingConfig> {
+        // Support both old format (just model string) and new format
+        if (typeof raw === "string" || (raw?.model && !raw.provider)) {
+            // Old format or just model specified
+            const modelId = typeof raw === "string" ? raw : raw.model;
+
+            // Infer provider from model name
+            if (modelId.includes("text-embedding")) {
+                const creds = await EmbeddingProviderFactory.loadProviderCredentials("openai");
+                return {
+                    provider: "openai",
+                    model: modelId,
+                    apiKey: creds.apiKey,
+                    baseUrl: creds.baseUrl,
+                };
+            }
+
+            return {
+                provider: "local",
+                model: modelId,
+            };
+        }
+
+        // New format with explicit provider
+        const rawConfig = raw as { provider?: string; model?: string; apiKey?: string; baseUrl?: string };
+        const provider = rawConfig.provider || "local";
+
+        // Load credentials from providers.json if not provided inline
+        let apiKey = rawConfig.apiKey;
+        let baseUrl = rawConfig.baseUrl;
+
+        if (provider !== "local" && !apiKey) {
+            const creds = await EmbeddingProviderFactory.loadProviderCredentials(provider);
+            apiKey = creds.apiKey;
+            baseUrl = baseUrl || creds.baseUrl;
+        }
+
+        return {
+            provider,
+            model: rawConfig.model || EmbeddingProviderFactory.DEFAULT_CONFIG.model,
+            apiKey,
+            baseUrl,
+        };
+    }
+
+    /**
+     * Save embedding configuration
+     */
+    static async saveConfiguration(
+        embeddingConfig: EmbeddingConfig,
+        scope: "global" | "project" = "global",
+        options?: EmbeddingConfigOptions
+    ): Promise<void> {
+        const basePath =
+            scope === "global"
+                ? config.getGlobalPath()
+                : EmbeddingProviderFactory.resolveProjectBase(options) ||
+                  config.getProjectPath(process.cwd());
+
+        const configPath = path.join(basePath, EmbeddingProviderFactory.EMBED_CONFIG_FILE);
+
+        // Don't save API key to file - it's stored in providers.json
+        const configToSave: { provider: string; model: string; baseUrl?: string } = {
+            provider: embeddingConfig.provider,
+            model: embeddingConfig.model,
+        };
+
+        // Only save baseUrl if it's a custom one (not the default for the provider)
+        const defaultBaseUrl = EmbeddingProviderFactory.PROVIDER_BASE_URLS[embeddingConfig.provider];
+        if (embeddingConfig.baseUrl && embeddingConfig.baseUrl !== defaultBaseUrl) {
+            configToSave.baseUrl = embeddingConfig.baseUrl;
+        }
+
+        const { writeJsonFile, ensureDirectory } = await import("@/lib/fs");
+        await ensureDirectory(basePath);
+        await writeJsonFile(configPath, configToSave);
+
+        logger.info(
+            `Embedding configuration saved to ${scope} config: ${embeddingConfig.provider}/${embeddingConfig.model}`
+        );
+    }
+
+    private static resolveConfigBases(options?: EmbeddingConfigOptions): string[] {
+        const scope = options?.scope ?? "auto";
+        const bases: string[] = [];
+
+        if (scope !== "global") {
+            const projectBase = EmbeddingProviderFactory.resolveProjectBase(options);
+            if (projectBase) {
+                bases.push(projectBase);
+            } else if (!options) {
+                // Legacy fallback to current working directory
+                bases.push(config.getProjectPath(process.cwd()));
+            }
+        }
+
+        // Always append global as fallback/default
+        bases.push(config.getGlobalPath());
+        return bases;
+    }
+
+    private static resolveProjectBase(options?: EmbeddingConfigOptions): string | undefined {
+        if (options?.metadataPath) {
+            return options.metadataPath;
+        }
+        if (options?.projectPath) {
+            return config.getProjectPath(options.projectPath);
+        }
+        return undefined;
+    }
+}

package/src/services/rag/LanceDBMaintenanceService.ts

@@ -0,0 +1,211 @@
+/**
+ * LanceDBMaintenanceService - Periodic compaction and cleanup
+ *
+ * Runs optimize() on all LanceDB tables to:
+ * - Compact small fragment files into larger ones
+ * - Prune old version manifests
+ *
+ * This prevents fragmentation accumulation from frequent writes.
+ * Uses a self-scheduling setTimeout pattern (like ConversationIndexingJob)
+ * to prevent overlapping runs.
+ */
+
+import { logger } from "@/utils/logger";
+import { RAGDatabaseService } from "./RAGDatabaseService";
+
+/** Default interval: 2 hours (in milliseconds) */
+const DEFAULT_INTERVAL_MS = 2 * 60 * 60 * 1000;
+
+/**
+ * Cleanup versions older than this duration.
+ * We keep 1 day of history for safety; the current version is never removed.
+ */
+const DEFAULT_CLEANUP_OLDER_THAN_DAYS = 1;
+
+/**
+ * Periodic LanceDB maintenance service
+ */
+export class LanceDBMaintenanceService {
+    private static instance: LanceDBMaintenanceService | null = null;
+    private timer: NodeJS.Timeout | null = null;
+    private isRunning = false;
+    private isMaintenanceRunning = false;
+    private intervalMs: number;
+
+    private constructor(intervalMs: number = DEFAULT_INTERVAL_MS) {
+        this.intervalMs = intervalMs;
+    }
+
+    /**
+     * Get singleton instance
+     */
+    public static getInstance(intervalMs?: number): LanceDBMaintenanceService {
+        if (!LanceDBMaintenanceService.instance) {
+            LanceDBMaintenanceService.instance = new LanceDBMaintenanceService(intervalMs);
+        }
+        return LanceDBMaintenanceService.instance;
+    }
+
+    /**
+     * Start the periodic maintenance job
+     */
+    public start(): void {
+        if (this.isRunning) {
+            logger.warn("LanceDBMaintenanceService is already running");
+            return;
+        }
+
+        this.isRunning = true;
+        logger.info("Starting LanceDBMaintenanceService", {
+            intervalMs: this.intervalMs,
+            intervalHours: this.intervalMs / (60 * 60 * 1000),
+        });
+
+        // Run first maintenance after a short delay (30 seconds)
+        // to let the system finish booting
+        this.scheduleNextRun(30_000);
+    }
+
+    /**
+     * Schedule the next maintenance run (self-scheduling to prevent overlaps)
+     */
+    private scheduleNextRun(delayMs: number): void {
+        if (!this.isRunning) return;
+
+        this.timer = setTimeout(async () => {
+            try {
+                await this.runMaintenance();
+            } catch (error) {
+                logger.error("LanceDB maintenance failed", { error });
+            } finally {
+                // Schedule next run only after this one completes
+                this.scheduleNextRun(this.intervalMs);
+            }
+        }, delayMs);
+    }
+
+    /**
+     * Stop the periodic maintenance job
+     */
+    public stop(): void {
+        if (!this.isRunning) {
+            return;
+        }
+
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+
+        this.isRunning = false;
+        logger.info("LanceDBMaintenanceService stopped");
+    }
+
+    /**
+     * Run maintenance on all LanceDB tables
+     */
+    private async runMaintenance(): Promise<void> {
+        if (this.isMaintenanceRunning) {
+            logger.warn("Skipping LanceDB maintenance - previous run still in progress");
+            return;
+        }
+
+        this.isMaintenanceRunning = true;
+
+        // Use a temporary DB service to avoid coupling to RAGService singleton
+        // (which requires embedding provider initialization)
+        let dbService: RAGDatabaseService | null = null;
+
+        try {
+            dbService = new RAGDatabaseService();
+            const tableNames = await dbService.listTables();
+
+            if (tableNames.length === 0) {
+                logger.debug("No LanceDB tables to optimize");
+                return;
+            }
+
+            logger.info("Starting LanceDB maintenance", { tableCount: tableNames.length });
+
+            const cleanupOlderThan = new Date();
+            cleanupOlderThan.setDate(cleanupOlderThan.getDate() - DEFAULT_CLEANUP_OLDER_THAN_DAYS);
+
+            let optimizedCount = 0;
+
+            for (const tableName of tableNames) {
+                try {
+                    const table = await dbService.getTable(tableName);
+                    const stats = await table.optimize({
+                        cleanupOlderThan,
+                    });
+
+                    optimizedCount++;
+                    logger.info(`Optimized table '${tableName}'`, {
+                        compaction: stats.compaction,
+                        prune: stats.prune,
+                    });
+                } catch (error) {
+                    const message = error instanceof Error ? error.message : String(error);
+                    logger.error(`Failed to optimize table '${tableName}'`, {
+                        error: message,
+                    });
+                    // Continue with other tables
+                }
+            }
+
+            logger.info("LanceDB maintenance complete", {
+                tablesOptimized: optimizedCount,
+                totalTables: tableNames.length,
+            });
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            logger.error("LanceDB maintenance failed", { error: message });
+        } finally {
+            if (dbService) {
+                try {
+                    await dbService.close();
+                } catch (closeError) {
+                    logger.debug("Failed to close maintenance DB service", { error: closeError });
+                }
+            }
+            this.isMaintenanceRunning = false;
+        }
+    }
+
+    /**
+     * Force an immediate maintenance run (for manual triggering)
+     */
+    public async forceRun(): Promise<void> {
+        await this.runMaintenance();
+    }
+
+    /**
+     * Get current service status
+     */
+    public getStatus(): {
+        isRunning: boolean;
+        isMaintenanceRunning: boolean;
+        intervalMs: number;
+    } {
+        return {
+            isRunning: this.isRunning,
+            isMaintenanceRunning: this.isMaintenanceRunning,
+            intervalMs: this.intervalMs,
+        };
+    }
+
+    /**
+     * Reset singleton instance (for testing)
+     */
+    public static resetInstance(): void {
+        if (LanceDBMaintenanceService.instance) {
+            LanceDBMaintenanceService.instance.stop();
+            LanceDBMaintenanceService.instance = null;
+        }
+    }
+}
+
+// Export lazy getter
+export function getLanceDBMaintenanceService(): LanceDBMaintenanceService {
+    return LanceDBMaintenanceService.getInstance();
+}