@hexabot-ai/api 3.0.0-alpha.3 → 3.1.1-alpha.0

package/src/cms/services/rag-retriever.service.ts

@@ -0,0 +1,342 @@
+/*
+ * Hexabot — Fair Core License (FCL-1.0-ALv2)
+ * Copyright (c) 2026 Hexastack.
+ * Full terms: see LICENSE.md.
+ */
+
+import { Settings } from '@llamaindex/core/global';
+import { MetadataMode } from '@llamaindex/core/schema';
+import {
+  FilterCondition,
+  FilterOperator,
+  MetadataFilters,
+} from '@llamaindex/core/vector-store';
+import { Injectable } from '@nestjs/common';
+import { KeywordTableRetrieverMode } from 'llamaindex';
+
+import { SettingService } from '@/setting/services/setting.service';
+
+import { RagHit, RagMode, RagQueryOptions } from '../types/rag';
+
+import { RagBackendService } from './rag-backend.service';
+
+type ResolvedRagQueryOptions = Omit<RagQueryOptions, 'limit'> & {
+  limit: number;
+};
+
+type RagCandidateHit = RagHit & { _status: number };
+
+type RetrieverEntry = {
+  node: {
+    id_: string;
+    metadata: Record<string, unknown>;
+    getContent(mode: MetadataMode): string;
+  };
+  score?: number;
+};
+
+@Injectable()
+export class RagRetrieverService {
+  constructor(
+    private readonly settingService: SettingService,
+    private readonly ragBackendService: RagBackendService,
+  ) {}
+
+  /**
+   * Retrieves RAG hits using the requested retrieval mode.
+   * @param query Query text.
+   * @param options Retrieval options.
+   * @returns Matching RAG hits.
+   */
+  async retrieve(
+    query: string,
+    options: RagQueryOptions = {},
+  ): Promise<RagHit[]> {
+    const settings = await this.settingService.getSettings();
+    const ragSettings = settings.rag_settings;
+    if (!ragSettings.enabled || !query?.trim()) {
+      return [];
+    }
+
+    const resolvedOptions = this.resolveQueryOptions(options, ragSettings);
+    if (resolvedOptions.mode !== 'embedding') {
+      return await this.retrieveLexical(query, resolvedOptions);
+    }
+
+    return await this.retrieveEmbedding(query, resolvedOptions);
+  }
+
+  /**
+   * Retrieves lexical matches from the keyword index.
+   * @param query Query text.
+   * @param options Resolved query options.
+   * @returns Lexical retrieval hits.
+   */
+  private async retrieveLexical(
+    query: string,
+    options: ResolvedRagQueryOptions,
+  ): Promise<RagHit[]> {
+    const index = await this.ragBackendService.getLexicalIndex();
+    const entries = await Settings.withLLM({} as any, async () => {
+      const retriever = index.asRetriever({
+        mode: KeywordTableRetrieverMode.SIMPLE,
+        numChunksPerQuery: options.limit,
+      });
+
+      return (await retriever.retrieve(query)) as RetrieverEntry[];
+    });
+    const filteredHits = entries
+      .map((entry) => this.toCandidateHit(entry, 'lexical'))
+      .filter((hit) => this.shouldIncludeHit(hit, options));
+
+    return this.dedupeLexicalHits(filteredHits).slice(0, options.limit);
+  }
+
+  /**
+   * Retrieves semantic matches from the embedding index.
+   * @param query Query text.
+   * @param options Resolved query options.
+   * @returns Embedding retrieval hits.
+   */
+  private async retrieveEmbedding(
+    query: string,
+    options: ResolvedRagQueryOptions,
+  ): Promise<RagHit[]> {
+    const embeddingIndex = await this.ragBackendService.getEmbeddingIndex();
+    const filters = this.buildEmbeddingFilters(options);
+    const retriever = embeddingIndex.asRetriever({
+      similarityTopK: options.limit,
+      ...(filters ? { filters } : {}),
+    });
+    const entries = (await retriever.retrieve(query)) as RetrieverEntry[];
+    const filteredHits = entries
+      .map((entry) => this.toCandidateHit(entry, 'embedding'))
+      .filter((hit) => this.shouldIncludeHit(hit, options));
+
+    return this.dedupeEmbeddingHits(filteredHits).slice(0, options.limit);
+  }
+
+  /**
+   * Builds metadata filters for embedding retrieval.
+   * @param options Query options.
+   * @returns Metadata filters or undefined when none are needed.
+   */
+  private buildEmbeddingFilters(
+    options: RagQueryOptions,
+  ): MetadataFilters | undefined {
+    const filters: MetadataFilters['filters'] = [];
+    const includeInactive = options.includeInactive ?? false;
+
+    if (!includeInactive) {
+      filters.push({
+        key: 'status',
+        value: 1,
+        operator: FilterOperator.EQ,
+      });
+    }
+
+    if (options.contentTypeId) {
+      filters.push({
+        key: 'contentTypeId',
+        value: options.contentTypeId,
+        operator: FilterOperator.EQ,
+      });
+    }
+
+    if (filters.length === 0) {
+      return undefined;
+    }
+
+    return {
+      filters,
+      condition: FilterCondition.AND,
+    };
+  }
+
+  /**
+   * Resolves query options with defaults from RAG settings.
+   * @param options User-provided options.
+   * @param ragSettings RAG defaults.
+   * @returns Fully resolved query options.
+   */
+  private resolveQueryOptions(
+    options: RagQueryOptions,
+    ragSettings: {
+      default_mode: RagMode;
+      top_k: number;
+    },
+  ): ResolvedRagQueryOptions {
+    return {
+      ...options,
+      mode: options.mode ?? ragSettings.default_mode,
+      limit: options.limit ?? ragSettings.top_k,
+    };
+  }
+
+  /**
+   * Converts a retriever entry to an internal candidate hit.
+   * @param entry Retriever entry.
+   * @param source Retrieval mode source.
+   * @returns Parsed candidate hit.
+   */
+  private toCandidateHit(
+    entry: RetrieverEntry,
+    source: RagMode,
+  ): RagCandidateHit {
+    const metadata = entry.node.metadata;
+    const contentId = this.parseContentId(metadata.contentId, entry.node.id_);
+    const title = this.parseTitle(metadata.title, contentId);
+
+    return {
+      contentId,
+      title,
+      text: entry.node.getContent(MetadataMode.NONE),
+      score: source === 'embedding' ? this.parseScore(entry.score) : undefined,
+      contentTypeId: this.parseContentTypeId(metadata.contentTypeId),
+      source,
+      _status: this.parseStatus(metadata.status),
+    };
+  }
+
+  /**
+   * Parses a content identifier with fallback support.
+   * @param contentId Candidate content identifier.
+   * @param fallback Fallback identifier.
+   * @returns Parsed content identifier.
+   */
+  private parseContentId(contentId: unknown, fallback: string): string {
+    return this.isNonEmptyString(contentId) ? contentId : fallback;
+  }
+
+  /**
+   * Parses a title with fallback support.
+   * @param title Candidate title value.
+   * @param fallback Fallback title.
+   * @returns Parsed title.
+   */
+  private parseTitle(title: unknown, fallback: string): string {
+    return this.isNonEmptyString(title) ? title : fallback;
+  }
+
+  /**
+   * Parses a content type identifier.
+   * @param contentTypeId Candidate content type value.
+   * @returns Parsed content type identifier.
+   */
+  private parseContentTypeId(contentTypeId: unknown): string | undefined {
+    return this.isNonEmptyString(contentTypeId) ? contentTypeId : undefined;
+  }
+
+  /**
+   * Parses a content status value.
+   * @param status Candidate status value.
+   * @returns Parsed status value.
+   */
+  private parseStatus(status: unknown): number {
+    const statusValue = typeof status === 'number' ? status : Number(status);
+
+    return Number.isFinite(statusValue) ? statusValue : 1;
+  }
+
+  /**
+   * Parses a numeric relevance score.
+   * @param score Candidate score value.
+   * @returns Parsed score.
+   */
+  private parseScore(score: unknown): number | undefined {
+    return typeof score === 'number' ? score : undefined;
+  }
+
+  /**
+   * Checks whether a candidate hit satisfies query filters.
+   * @param hit Candidate hit.
+   * @param options Query options.
+   * @returns True when the hit should be included.
+   */
+  private shouldIncludeHit(
+    hit: RagCandidateHit,
+    options: RagQueryOptions,
+  ): boolean {
+    const includeInactive = options.includeInactive ?? false;
+    if (!includeInactive && hit._status !== 1) {
+      return false;
+    }
+
+    if (options.contentTypeId && hit.contentTypeId !== options.contentTypeId) {
+      return false;
+    }
+
+    return true;
+  }
+
+  /**
+   * Deduplicates lexical hits by content identifier.
+   * @param hits Candidate hits.
+   * @returns Deduplicated lexical hits.
+   */
+  private dedupeLexicalHits(hits: RagCandidateHit[]): RagHit[] {
+    const byContentId = new Map<string, RagHit>();
+    for (const hit of hits) {
+      if (byContentId.has(hit.contentId)) {
+        continue;
+      }
+
+      byContentId.set(hit.contentId, this.toPublicHit(hit));
+    }
+
+    return Array.from(byContentId.values());
+  }
+
+  /**
+   * Deduplicates embedding hits and keeps the highest score per content item.
+   * @param hits Candidate hits.
+   * @returns Deduplicated and score-sorted embedding hits.
+   */
+  private dedupeEmbeddingHits(hits: RagCandidateHit[]): RagHit[] {
+    const byContentId = new Map<string, RagHit>();
+    for (const hit of hits) {
+      const currentHit = byContentId.get(hit.contentId);
+      if (!currentHit) {
+        byContentId.set(hit.contentId, this.toPublicHit(hit));
+        continue;
+      }
+
+      if (this.getScore(hit) > this.getScore(currentHit)) {
+        byContentId.set(hit.contentId, this.toPublicHit(hit));
+      }
+    }
+
+    return Array.from(byContentId.values()).sort(
+      (left, right) => this.getScore(right) - this.getScore(left),
+    );
+  }
+
+  /**
+   * Removes internal fields from a candidate hit.
+   * @param hit Candidate hit.
+   * @returns Public hit representation.
+   */
+  private toPublicHit(hit: RagCandidateHit): RagHit {
+    const { _status: _, ...publicHit } = hit;
+
+    return publicHit;
+  }
+
+  /**
+   * Returns a comparable score value for sorting.
+   * @param hit Hit with optional score.
+   * @returns Comparable score value.
+   */
+  private getScore(hit: Pick<RagHit, 'score'>): number {
+    return hit.score ?? Number.NEGATIVE_INFINITY;
+  }
+
+  /**
+   * Checks whether a value is a non-empty string.
+   * @param value Candidate value.
+   * @returns True when the value is a non-empty string.
+   */
+  private isNonEmptyString(value: unknown): value is string {
+    return typeof value === 'string' && value.length > 0;
+  }
+}

package/src/cms/services/rag.service.ts

@@ -0,0 +1,226 @@
+/*
+ * Hexabot — Fair Core License (FCL-1.0-ALv2)
+ * Copyright (c) 2026 Hexastack.
+ * Full terms: see LICENSE.md.
+ */
+
+import { Injectable } from '@nestjs/common';
+import { OnEvent } from '@nestjs/event-emitter';
+
+import { LoggerService } from '@/logger/logger.service';
+import { SettingService } from '@/setting/services/setting.service';
+import { UserOrmEntity } from '@/user';
+import {
+  DeleteEntityEvent,
+  InsertEntityEvent,
+  UpdateEntityEvent,
+} from '@/utils/types/entity-event.types';
+
+import { ContentOrmEntity } from '../entities/content.entity';
+import { RagHit, RagQueryOptions } from '../types/rag';
+
+import { RagBackendService } from './rag-backend.service';
+import { RagIndexerService } from './rag-indexer.service';
+import { RagRetrieverService } from './rag-retriever.service';
+
+@Injectable()
+export class RagService {
+  private ragSettingsReindexPromise?: Promise<void>;
+
+  private manualReindexPromise?: Promise<void>;
+
+  constructor(
+    private readonly retrieverService: RagRetrieverService,
+    private readonly indexerService: RagIndexerService,
+    private readonly embeddingBackendService: RagBackendService,
+    private readonly settingService: SettingService,
+    private readonly logger: LoggerService,
+  ) {}
+
+  /**
+   * Retrieves RAG hits for the provided query.
+   * @param query User query text.
+   * @param options Retrieval options.
+   * @returns Matching RAG hits.
+   */
+  async retrieve(
+    query: string,
+    options: RagQueryOptions = {},
+  ): Promise<RagHit[]> {
+    return await this.retrieverService.retrieve(query, options);
+  }
+
+  /**
+   * Upserts one content item in RAG indexes.
+   * @param contentId Content identifier.
+   * @returns Resolves when indexing is complete.
+   */
+  async upsertContentIndex(contentId: string): Promise<void> {
+    await this.indexerService.upsertContentIndex(contentId);
+  }
+
+  /**
+   * Removes one content item from RAG indexes.
+   * @param contentId Content identifier.
+   * @returns Resolves when removal is complete.
+   */
+  async removeContentIndex(contentId: string): Promise<void> {
+    await this.indexerService.removeContentIndex(contentId);
+  }
+
+  /**
+   * Rebuilds RAG indexes for all content.
+   * @returns Resolves when reindexing is complete.
+   */
+  async reindexAll(): Promise<void> {
+    await this.indexerService.reindexAll();
+  }
+
+  /**
+   * Schedules a full manual reindex if one is not already running.
+   * @returns No return value.
+   */
+  scheduleReindexAll(): void {
+    if (!this.manualReindexPromise) {
+      this.manualReindexPromise = this.reindexAll()
+        .catch((error) => {
+          this.logger.error(
+            'Unable to complete manual RAG reindex request.',
+            error,
+          );
+        })
+        .finally(() => {
+          this.manualReindexPromise = undefined;
+        });
+    }
+  }
+
+  /**
+   * Handles changes to the RAG enabled setting.
+   * @returns Resolves when follow-up reindex handling completes.
+   */
+  @OnEvent('hook:rag_settings:enabled')
+  async handleRagEnabledSettingChanged(): Promise<void> {
+    await this.reindexAfterRagSettingChange('enabled');
+  }
+
+  /**
+   * Handles changes to RAG backend settings that require reindexing.
+   * @returns Resolves when follow-up reindex handling completes.
+   */
+  @OnEvent('hook:rag_settings:embedding_model')
+  @OnEvent('hook:rag_settings:embedding_provider')
+  @OnEvent('hook:rag_settings:embedding_api_key')
+  @OnEvent('hook:rag_settings:embedding_base_url')
+  @OnEvent('hook:rag_settings:embedding_dimensions')
+  @OnEvent('hook:rag_settings:index_only_active_content')
+  async handleRagBackendSettingChanged(): Promise<void> {
+    await this.reindexAfterRagSettingChange('backend');
+  }
+
+  /**
+   * Handles content creation events and indexes the new content.
+   * @param event Insert event payload.
+   * @returns Resolves when event handling completes.
+   */
+  @OnEvent('hook:content:postCreate')
+  async handleContentCreated(
+    event: InsertEntityEvent<UserOrmEntity>,
+  ): Promise<void> {
+    const contentId = event.entity?.id;
+    if (!contentId) {
+      return;
+    }
+
+    try {
+      await this.upsertContentIndex(contentId);
+    } catch (error) {
+      this.logger.error('Unable to index created content', error, {
+        contentId,
+      });
+    }
+  }
+
+  /**
+   * Handles content update events and reindexes the updated content.
+   * @param event Update event payload.
+   * @returns Resolves when event handling completes.
+   */
+  @OnEvent('hook:content:postUpdate')
+  async handleContentUpdated(
+    event: UpdateEntityEvent<ContentOrmEntity>,
+  ): Promise<void> {
+    const contentId = event.entity?.id;
+    if (!contentId || !event.entity.status) {
+      return;
+    }
+
+    try {
+      await this.upsertContentIndex(contentId);
+    } catch (error) {
+      this.logger.error('Unable to index updated content', error, {
+        contentId,
+      });
+    }
+  }
+
+  /**
+   * Handles content deletion events and removes deleted content from indexes.
+   * @param event Remove event payload.
+   * @returns Resolves when event handling completes.
+   */
+  @OnEvent('hook:content:postDelete')
+  async handleContentDeleted(
+    event: DeleteEntityEvent<ContentOrmEntity>,
+  ): Promise<void> {
+    const contentId = event.entity?.id;
+    if (!contentId) {
+      return;
+    }
+
+    try {
+      await this.removeContentIndex(contentId);
+    } catch (error) {
+      this.logger.error('Unable to remove deleted content from index', error, {
+        contentId,
+      });
+    }
+  }
+
+  /**
+   * Queues a reindex after a relevant RAG setting change.
+   * @param trigger Setting key that triggered reindexing.
+   * @returns Resolves when the queued reindex attempt finishes.
+   */
+  private async reindexAfterRagSettingChange(trigger: string): Promise<void> {
+    if (!this.ragSettingsReindexPromise) {
+      this.ragSettingsReindexPromise = this.performRagSettingsReindex()
+        .catch((error) => {
+          this.logger.error(
+            `Unable to reindex content after RAG setting change (${trigger}).`,
+            error,
+          );
+        })
+        .finally(() => {
+          this.ragSettingsReindexPromise = undefined;
+        });
+    }
+
+    await this.ragSettingsReindexPromise;
+  }
+
+  /**
+   * Resets RAG backend state and performs a full reindex when RAG is enabled.
+   * @returns Resolves when reset and reindexing complete.
+   */
+  private async performRagSettingsReindex(): Promise<void> {
+    const settings = await this.settingService.getSettings();
+
+    if (!settings.rag_settings.enabled) {
+      return;
+    }
+
+    this.embeddingBackendService.reset();
+    await this.reindexAll();
+  }
+}

package/src/cms/types/content.ts

@@ -7,10 +7,6 @@
 import { ContentTypeOrmEntity } from '../entities/content-type.entity';
 import { ContentOrmEntity } from '../entities/content.entity';
 
-export type ContentPopulate = 'entity';
-
 export type ContentEntityWithType = Omit<ContentOrmEntity, 'entity'> & {
   entity: ContentTypeOrmEntity | null;
 };
-
-export const CONTENT_POPULATE: ContentPopulate[] = ['entity'];

package/src/cms/types/rag.ts

@@ -0,0 +1,23 @@
+/*
+ * Hexabot — Fair Core License (FCL-1.0-ALv2)
+ * Copyright (c) 2026 Hexastack.
+ * Full terms: see LICENSE.md.
+ */
+
+export type RagMode = 'embedding' | 'lexical';
+
+export interface RagQueryOptions {
+  mode?: RagMode;
+  limit?: number;
+  contentTypeId?: string;
+  includeInactive?: boolean;
+}
+
+export interface RagHit {
+  contentId: string;
+  title: string;
+  text: string;
+  score?: number;
+  contentTypeId?: string;
+  source: RagMode;
+}

package/src/config/i18n/en/chatbot_settings.json

@@ -0,0 +1,17 @@
+{
+  "Chatbot": "Chatbot",
+  "Default NLU helper": "Default NLU helper",
+  "Helper used by default to run NLU tasks.": "Helper used by default to run NLU tasks.",
+  "Default NLU penalty factor": "Default NLU penalty factor",
+  "Penalty factor applied to NLU confidence scoring.": "Penalty factor applied to NLU confidence scoring.",
+  "Default LLM helper": "Default LLM helper",
+  "Helper used by default for LLM generation tasks.": "Helper used by default for LLM generation tasks.",
+  "Default storage helper": "Default storage helper",
+  "Helper used to persist chatbot data by default.": "Helper used to persist chatbot data by default.",
+  "License key": "License key",
+  "Provide the license key associated with your subscription. Learn more about available plans at https://hexabot.ai/pricing#pricing.": "Provide the license key associated with your subscription. Learn more about available plans at https://hexabot.ai/pricing#pricing.",
+  "Enable global fallback": "Enable global fallback",
+  "Enable fallback handling when no intent or flow matches.": "Enable fallback handling when no intent or flow matches.",
+  "Fallback messages": "Fallback messages",
+  "Messages shown when fallback handling is triggered.": "Messages shown when fallback handling is triggered."
+}

package/src/config/i18n/en/contact.json

@@ -0,0 +1,23 @@
+{
+  "Contact": "Contact",
+  "Contact recipient email": "Contact recipient email",
+  "Email address that receives contact form submissions.": "Email address that receives contact form submissions.",
+  "Company name": "Company name",
+  "Company name displayed to end users.": "Company name displayed to end users.",
+  "Company phone": "Company phone",
+  "Primary phone number displayed in contact information.": "Primary phone number displayed in contact information.",
+  "Company email": "Company email",
+  "Public contact email address shown to users.": "Public contact email address shown to users.",
+  "Address line 1": "Address line 1",
+  "First line of the company postal address.": "First line of the company postal address.",
+  "Address line 2": "Address line 2",
+  "Second line of the company postal address.": "Second line of the company postal address.",
+  "City": "City",
+  "City for the company postal address.": "City for the company postal address.",
+  "Postal code": "Postal code",
+  "Postal or ZIP code for the company address.": "Postal or ZIP code for the company address.",
+  "State or region": "State or region",
+  "State, region, or province for the company address.": "State, region, or province for the company address.",
+  "Country": "Country",
+  "Country code for the company address.": "Country code for the company address."
+}

package/src/config/i18n/en/messages.json

@@ -1,16 +1,8 @@
 {
-  "invitation_subject": "[Hexabot] Sign-Up Invitation",
   "account_confirmation_subject": "[Hexabot] Account Confirmation",
   "password_reset_subject": "[Hexabot] Password Reset",
   "welcome": "Welcome",
   "hi": "Hi",
-  "invitation_to_join": "Invitation to join",
-  "invitation_for_account_creation": "You have been invited to create a",
-  "account": "account",
-  "create_account": "Click on the button below to create your account:",
-  "registration_failed": "Registration failed! Either email address or invitation token is invalid.",
-  "create_account_button": "Click on the button below to create your account",
-  "join": "Join",
   "account_successfully_created_confirm_password": "You have successfully created an account but you will need to confirm your email address.",
   "confirm_account": "Click on the button below in order to confirm your account:",
   "password_reset_request": "A request to reset the password for your account has been made.",

package/src/config/i18n/en/rag_settings.json

@@ -0,0 +1,21 @@
+{
+  "RAG": "RAG",
+  "Enable RAG": "Enable RAG",
+  "Enable retrieval-augmented generation.": "Enable retrieval-augmented generation.",
+  "Default RAG mode": "Default RAG mode",
+  "Default retrieval mode used for RAG queries.": "Default retrieval mode used for RAG queries.",
+  "Embedding provider": "Embedding provider",
+  "Provider used to generate embedding vectors.": "Provider used to generate embedding vectors.",
+  "Embedding model": "Embedding model",
+  "Embedding model identifier used by the selected provider.": "Embedding model identifier used by the selected provider.",
+  "Embedding API key": "Embedding API key",
+  "API key used by the configured embedding provider.": "API key used by the configured embedding provider.",
+  "Embedding base URL": "Embedding base URL",
+  "Custom base URL used for embedding API requests.": "Custom base URL used for embedding API requests.",
+  "Embedding dimensions": "Embedding dimensions",
+  "Number of dimensions expected for embedding vectors.": "Number of dimensions expected for embedding vectors.",
+  "Top K results": "Top K results",
+  "Maximum number of retrieved chunks returned per query.": "Maximum number of retrieved chunks returned per query.",
+  "Index only active content": "Index only active content",
+  "Include only active content entries in the retrieval index.": "Include only active content entries in the retrieval index."
+}