@ixo/common 1.1.0

package/src/ai/utils/__tests__/stringify-docs.test.ts

@@ -0,0 +1,61 @@
+import { Document } from '@langchain/core/documents';
+import { stringifyDocs } from '../stringify-docs.js';
+
+describe('stringifyDocs', () => {
+  it('should stringify a single document', () => {
+    const doc = new Document({
+      pageContent: 'Test content',
+      metadata: { id: '123' },
+    });
+    const result = stringifyDocs([doc]);
+    expect(result).toBe('#ID:123\n Document: Test content \n \n ');
+  });
+
+  it('should stringify multiple documents', () => {
+    const docs = [
+      new Document({
+        pageContent: 'First content',
+        metadata: { id: '1' },
+      }),
+      new Document({
+        pageContent: 'Second content',
+        metadata: { id: '2' },
+      }),
+    ];
+    const expected =
+      '#ID:1\n Document: First content \n \n \n\n#ID:2\n Document: Second content \n \n ';
+    expect(stringifyDocs(docs)).toBe(expected);
+  });
+
+  it('should handle empty array', () => {
+    expect(stringifyDocs([])).toBe('');
+  });
+
+  it('should handle documents with special characters', () => {
+    const doc = new Document({
+      pageContent: 'Content with \n newline and \t tab',
+      metadata: { id: 'special' },
+    });
+    const expected =
+      '#ID:special\n Document: Content with \n newline and \t tab \n \n ';
+    expect(stringifyDocs([doc])).toBe(expected);
+  });
+
+  it('should handle documents with empty content', () => {
+    const doc = new Document({
+      pageContent: '',
+      metadata: { id: 'empty' },
+    });
+    const expected = '#ID:empty\n Document:  \n \n ';
+    expect(stringifyDocs([doc])).toBe(expected);
+  });
+
+  it('should handle documents with missing id in metadata', () => {
+    const doc = new Document({
+      pageContent: 'Test content',
+      metadata: {},
+    });
+    const expected = '#ID:undefined\n Document: Test content \n \n ';
+    expect(stringifyDocs([doc])).toBe(expected);
+  });
+});

package/src/ai/utils/chunk-arr.ts

@@ -0,0 +1,13 @@
+function chunkArr<T>(array: T[], chunkSize: number): T[][] {
+  if (chunkSize <= 0) {
+    throw new Error('Chunk size must be greater than 0');
+  }
+
+  const chunks: T[][] = [];
+  for (let i = 0; i < array.length; i += chunkSize) {
+    const chunk = array.slice(i, i + chunkSize);
+    chunks.push(chunk);
+  }
+  return chunks;
+}
+export { chunkArr };

package/src/ai/utils/doc-relevance-checker.ts

@@ -0,0 +1,58 @@
+import { type Document } from '@langchain/core/documents';
+import { type BaseLanguageModel } from '@langchain/core/language_models/base';
+import { ChatPromptTemplate } from '@langchain/core/prompts';
+import z from 'zod';
+import { getChatOpenAiModel } from '../models/openai.js';
+
+type TCheckDocRelevanceArgs = {
+  doc: Document | string;
+  query: string;
+  model?: BaseLanguageModel;
+};
+
+async function checkDocRelevance({
+  doc,
+  model = getChatOpenAiModel(),
+  query,
+}: TCheckDocRelevanceArgs): Promise<boolean> {
+  const prompt = ChatPromptTemplate.fromTemplate(
+    `You are an AI agent responsible for determining the relevance of a document to a given query. 
+    
+		DOCUMENT: ${typeof doc === 'string' ? doc : doc.pageContent} 
+		QUERY: ${query} 
+    
+		### Acceptance Criteria
+		(1) Your goal is to identify DOCUMENTs that are completely unrelated to the QUERY.
+		(2) If the DOCUMENT contain any keywords or semantic meaning related to the QUERY, consider them relevant.
+		(3) It is acceptable if the DOCUMENT have some information that is unrelated to the QUERY, as long as (2) is met.
+    
+		### Instructions
+		(1) Review the DOCUMENT and determine if it is relevant to the QUERY.
+		(2) Provide a boolean answer indicating the relevance of the DOCUMENT to the QUERY.
+		(3) If the DOCUMENT is relevant, set the answer to true. Otherwise, set the answer to false.
+		`,
+  );
+
+  const zodSchema = z.object({
+    answer: z.boolean(),
+  });
+
+  if (!model) {
+    throw new Error('Model is not set');
+  }
+
+  if (!model.withStructuredOutput) {
+    throw new Error('Model does not support structured output');
+  }
+
+  const structuredLlm = model.withStructuredOutput(zodSchema);
+
+  const chain = prompt.pipe(structuredLlm);
+  const response = await chain.invoke({
+    query,
+  });
+
+  return response.answer;
+}
+
+export default checkDocRelevance;

package/src/ai/utils/doc-splitter.ts

@@ -0,0 +1,28 @@
+import { type Document } from '@langchain/core/documents';
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
+
+import  z  from 'zod';
+
+const splitter = new RecursiveCharacterTextSplitter();
+
+export const docSplitter = (text: string | string[]): Promise<Document[]> => {
+  if (!text) throw new Error('No text provided');
+
+  const _text = typeof text === 'string' ? [text] : text;
+  arrSchema.parse(_text);
+
+  return splitter.createDocuments(_text);
+};
+
+const arrSchema = z
+  .array(
+    z.string({
+      message: 'Invalid text provided',
+    }),
+    {
+      message: 'Invalid text provided in array',
+    },
+  )
+  .nonempty({
+    message: 'Text array cannot be empty',
+  });

package/src/ai/utils/filter-similarity-search-results.ts

@@ -0,0 +1,15 @@
+import { type Document } from '@langchain/core/documents';
+
+function filterSimilaritySearchResults<
+  Doc extends Document,
+  Result extends [Doc, number][],
+>(results: Result, threshold: number): Doc[] {
+  const filteredResults = [];
+  for (const [doc, score] of results) {
+    if (score >= threshold) {
+      filteredResults.push(doc);
+    }
+  }
+  return filteredResults;
+}
+export { filterSimilaritySearchResults };

package/src/ai/utils/generate-questions-from-chunks.ts

@@ -0,0 +1,114 @@
+import { Document } from 'langchain';
+import { getOpenAiClient } from '../models/openai.js';
+
+export const generateQuestionsFromChunks = async (
+  content: string,
+  chunks: Document[],
+): Promise<QuestionGeneratorOutput['json_output']> => {
+  const openai = getOpenAiClient();
+  const response = await openai.chat.completions.parse({
+    model: 'gpt-4o-mini',
+    messages: [
+      {
+        role: 'system',
+        content: [
+          {
+            type: 'text',
+            text: '# SYSTEM — “Question Generator with Visible Chain-of-Thought”\n\nYou are an expert dataset-builder.  \nYour mission: turn raw knowledge into high-quality Q-A pairs for semantic search **while showing your reasoning**.\n\n---\n## INPUTS\n- **chunks** – array → `{ "id": string, "text": string }`  \n- **fullText** – the complete source document.\n\n---\n## TASK\nFor **each** chunk:\n\n1. **THINK (visible)**  \n   • Analyse the chunk’s content.  \n   • Note key facts, entities, dates, numbers, causal links, etc.  \n   • Decide which facts yield **Direct** vs **Derived** questions.  \n   • *Keep thoughts ≤ 120 words.*\n\n2. **GENERATE**  \n   • **Direct Q-A** – 1-3 questions whose full answer text appears **verbatim (or nearly so)** in the *same chunk*.  \n   • **Derived Q-A** – 1-2 questions that require light inference or synthesis; answers may use the chunk **or** *fullText*.  \n   • Ensure **at least two distinct sections** of the chunk are represented among the questions.\n\n---\n## OUTPUT FORMAT  \nReturn **one valid JSON object** that matches the schema below.  \nOutput **nothing** else.\n\n```json\n{\n  "thoughts": "<your chain-of-thought here>",\n  "json_output": [\n    {\n      "chunkId": "...",\n      "qas": [\n        { "type": "direct",  "question": "...?", "answer": "..." },\n        { "type": "derived", "question": "...?", "answer": "..." }\n        // …\n      ]\n    }\n    // …more chunks\n  ]\n}\n',
+          },
+        ],
+      },
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: `## CHUNKS
+          ${chunks
+            .map((chunk) => {
+              return `### CHUNK ${chunk.id}
+            ${chunk.pageContent}`;
+            })
+            .join('\n')}
+          ## FULL TEXT
+          ${content}`,
+          },
+        ],
+      },
+    ],
+    response_format: {
+      type: 'json_schema',
+      json_schema: {
+        name: 'question_generator_output',
+        strict: true,
+        schema: {
+          type: 'object',
+          properties: {
+            thoughts: {
+              type: 'string',
+              description: 'Visible chain-of-thought reasoning.',
+            },
+            json_output: {
+              type: 'array',
+              description: 'Array of chunks with their generated Q&A pairs.',
+              items: {
+                type: 'object',
+                properties: {
+                  chunkId: {
+                    type: 'string',
+                    description: 'Identifier of the chunk.',
+                  },
+                  qas: {
+                    type: 'array',
+                    items: {
+                      type: 'object',
+                      properties: {
+                        type: {
+                          type: 'string',
+                          enum: ['direct', 'derived'],
+                          description:
+                            'Question type: direct (verbatim) or derived (inference).',
+                        },
+                        question: {
+                          type: 'string',
+                          description: 'The question text.',
+                        },
+                        answer: {
+                          type: 'string',
+                          description: 'The answer text.',
+                        },
+                      },
+                      required: ['type', 'question', 'answer'],
+                      additionalProperties: false,
+                    },
+                  },
+                },
+                required: ['chunkId', 'qas'],
+                additionalProperties: false,
+              },
+            },
+          },
+          required: ['thoughts', 'json_output'],
+          additionalProperties: false,
+        },
+      },
+    },
+    temperature: 0,
+    max_completion_tokens: 9957,
+    top_p: 0.28,
+    frequency_penalty: 0,
+    presence_penalty: 0,
+    store: false,
+  });
+  const json = response.choices[0]?.message
+    .parsed as unknown as QuestionGeneratorOutput;
+  return json.json_output;
+};
+
+type QuestionGeneratorOutput = {
+  thoughts: string;
+  json_output: {
+    chunkId: string;
+    qas: { type: string; question: string; answer: string }[];
+  }[];
+};

package/src/ai/utils/index.ts

@@ -0,0 +1,10 @@
+export * from './chunk-arr.js';
+export * from './doc-relevance-checker.js';
+export * from './doc-splitter.js';
+export * from './filter-similarity-search-results.js';
+export * from './generate-questions-from-chunks.js';
+export * from './json-to-yaml.js';
+export * from './load-file.js';
+export * from './stringify-docs.js';
+export * from './transformGraphStateMessageToListMessageResponse.js';
+export * from './verify-matrix-openId-token.js';

package/src/ai/utils/json-to-yaml.ts

@@ -0,0 +1,32 @@
+export const jsonToYaml = <T extends object>(
+  json: T,
+  indentLevel = 0,
+): string => {
+  const indent = '  '.repeat(indentLevel); // Two spaces per level of indentation
+
+  const isArray = Array.isArray(json);
+
+  if (isArray) {
+    return (json as unknown as any[])
+      .map((item) => {
+        if (item !== null && typeof item === 'object') {
+          // Recursively process nested objects/arrays with increased indentation
+          const nestedYaml = jsonToYaml(item, indentLevel + 1);
+          return `${indent}- \n${nestedYaml}`;
+        }
+        return `${indent}- ${item}`;
+      })
+      .join('\n');
+  }
+
+  return Object.entries(json)
+    .map(([key, value]) => {
+      if (value !== null && typeof value === 'object') {
+        // Recursively process nested objects with increased indentation
+        const nestedYaml = jsonToYaml(value, indentLevel + 1);
+        return `${indent}${key}:\n${nestedYaml}`;
+      }
+      return `${indent}${key}: ${value}`;
+    })
+    .join('\n');
+};

package/src/ai/utils/load-file.ts

@@ -0,0 +1,170 @@
+import { Logger } from '@ixo/logger';
+import { DocxLoader } from '@langchain/community/document_loaders/fs/docx';
+import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf';
+import { HtmlToTextTransformer } from '@langchain/community/document_transformers/html_to_text';
+import { Document } from '@langchain/core/documents';
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
+import fs from 'node:fs/promises';
+
+type SupportedFileType = 'pdf' | 'markdown' | 'html' | 'text' | 'doc';
+const splitter = new RecursiveCharacterTextSplitter({
+  chunkSize: 1000,
+  chunkOverlap: 200,
+});
+
+const determineFileType = (
+  path: string,
+  contentType?: string,
+): SupportedFileType | null => {
+  const extension = path.split('.').pop()?.toLowerCase();
+
+  // Determine based on file extension
+  if (extension) {
+    if (['pdf'].includes(extension)) return 'pdf';
+    if (['md', 'markdown'].includes(extension)) return 'markdown';
+    if (['html', 'htm'].includes(extension)) return 'html';
+    if (['txt'].includes(extension)) return 'text';
+    if (['doc', 'docx'].includes(extension)) return 'doc';
+  }
+
+  // Determine based on MIME type
+  if (contentType) {
+    if (contentType.includes('application/pdf')) return 'pdf';
+    if (contentType.includes('text/markdown')) return 'markdown';
+    if (contentType.includes('text/html')) return 'html';
+    if (contentType.includes('text/plain')) return 'text';
+    if (
+      contentType.includes('application/msword') ||
+      contentType.includes(
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+      )
+    ) {
+      return 'doc';
+    }
+  }
+
+  return null;
+};
+
+/**
+ * Loads a file (PDF, Markdown, HTML, Text, DOC) from a given path or URL and returns processed content.
+ *
+ * @param path - The file path or URL of the document to load.
+ * @param fetchOptions - Optional fetch options (e.g., headers for authorization).
+ * @returns A promise that resolves to an array of Document objects.
+ * @throws If the file type is unsupported or the file cannot be processed.
+ */
+export const loadFile = async (
+  path: string,
+  fetchOptions?: RequestInit,
+): Promise<Document[]> => {
+  const isUrl = path.startsWith('http');
+
+  if (isUrl) {
+    try {
+      const response = await fetch(path, fetchOptions);
+
+      if (!response.ok) {
+        const error = await getErrorFromResponse(response);
+        // eslint-disable-next-line no-console -- debug
+        console.error(`Error fetching file from URL: ${path}`, error);
+        throw new Error(`Failed to fetch file from URL: ${path}`);
+      }
+
+      const contentType = response.headers.get('content-type') ?? '';
+      const fileType = determineFileType(path, contentType);
+
+      if (!fileType) {
+        throw new Error(`Unsupported file type. Content-Type: ${contentType}`);
+      }
+
+      const data = await response.arrayBuffer();
+      const blob = new Blob([data], {
+        type: contentType || 'application/octet-stream',
+      });
+
+      return processFile(blob, fileType);
+    } catch (error) {
+      Logger.error(`Failed to load file from URL: ${path}`, error);
+      throw error;
+    }
+  }
+
+  try {
+    const buffer = await fs.readFile(path);
+    const blob = new Blob([buffer]);
+
+    const fileType = determineFileType(path);
+
+    if (!fileType) {
+      throw new Error(`Unsupported file type for file: ${path}`);
+    }
+
+    return processFile(blob, fileType);
+  } catch (error) {
+    Logger.error(`Failed to load local file from path: ${path}`, error);
+    throw error;
+  }
+};
+
+const processFile = async (
+  blob: Blob,
+  fileType: SupportedFileType,
+): Promise<Document[]> => {
+  switch (fileType) {
+    case 'pdf': {
+      const pdfLoader = new PDFLoader(blob);
+      return pdfLoader.load();
+    }
+
+    case 'markdown': {
+      return loadMarkdown(blob);
+    }
+
+    case 'html': {
+      const html = await blobToString(blob);
+      const htmlSplitter = RecursiveCharacterTextSplitter.fromLanguage('html');
+      const transformer = new HtmlToTextTransformer();
+      const sequence = htmlSplitter.pipe(transformer);
+      const docs = await sequence.invoke([
+        new Document({
+          pageContent: html,
+        }),
+      ]);
+      return docs;
+    }
+
+    case 'text':
+      return splitter.createDocuments([await blobToString(blob)]);
+
+    case 'doc': {
+      const docxLoader = new DocxLoader(blob);
+      return docxLoader.load();
+    }
+
+    default:
+      throw new Error(`Unsupported file type: ${fileType as string}`);
+  }
+};
+
+const loadMarkdown = async (blob: Blob): Promise<Document[]> => {
+  const markdown = await blobToString(blob);
+  return splitter.createDocuments([markdown]);
+};
+
+const blobToString = async (blob: Blob): Promise<string> => {
+  const buffer = await blob.arrayBuffer(); // Convert Blob to ArrayBuffer
+  return Buffer.from(buffer).toString('utf-8'); // Convert ArrayBuffer to string
+};
+
+const getErrorFromResponse = async (response: Response): Promise<unknown> => {
+  try {
+    const json = await response.json();
+
+    return json;
+  } catch (error) {
+    if (response.bodyUsed) return 'Unknown error';
+    const errorMessage = await response.text();
+    return errorMessage;
+  }
+};

package/src/ai/utils/stringify-docs.ts

@@ -0,0 +1,14 @@
+import { type Document } from '@langchain/core/documents';
+
+/**
+ *
+ * @param docs - Array of documents to stringify
+ * @returns  - Stringified documents with ID in Header
+ */
+export const stringifyDocs = (docs: Document[]): string => {
+  return docs
+    .map(
+      (doc) => `#ID:${doc.metadata.id}\n Document: ${doc.pageContent} \n \n `,
+    )
+    .join('\n\n');
+};

package/src/ai/utils/transformGraphStateMessageToListMessageResponse.ts

@@ -0,0 +1,108 @@
+import {
+  ToolMessage,
+  type AIMessage,
+  type BaseMessage,
+} from '@langchain/core/messages';
+import { isUUID } from 'class-validator';
+import crypto from 'node:crypto';
+
+interface ToolCall {
+  name: string;
+  id: string;
+  args: unknown;
+  status?: 'isRunning' | 'done';
+  output?: string;
+}
+
+interface MessageDto {
+  id: string;
+  type: 'ai' | 'human';
+  content: string;
+  toolCalls?: ToolCall[];
+  reasoning?: string;
+  isComplete?: boolean;
+  isReasoning?: boolean;
+}
+
+export interface ListOracleMessagesResponse {
+  messages: MessageDto[];
+}
+export interface CleanAdditionalKwargs {
+  msgFromMatrixRoom: boolean;
+  timestamp: string;
+  oracleName: string;
+  reasoning?: string;
+  reasoningDetails?: Array<{
+    type: string;
+    text: string;
+  }>;
+  [key: string]: unknown; // Allow additional properties for LangChain compatibility
+}
+
+export function transformGraphStateMessageToListMessageResponse(
+  messages: BaseMessage[],
+): ListOracleMessagesResponse {
+  return {
+    messages: messages.reduce<MessageDto[]>((acc, message) => {
+      const toolMsg = message.type === 'tool' ? (message as ToolMessage) : null;
+      if (message.type !== 'system' && message.type !== 'tool' && !message.additional_kwargs?.isError) {
+        // Extract reasoning from additional_kwargs
+        const additionalKwargs =
+          message.additional_kwargs as CleanAdditionalKwargs;
+        const reasoning = additionalKwargs?.reasoning;
+
+        acc.push({
+          type: message.type === 'ai' ? 'ai' : 'human',
+          content: message.content.toString(),
+          id: uuidFromString(message.id ?? message.content.toString()),
+          toolCalls: (message as AIMessage).tool_calls?.map((toolCall) => ({
+            name: toolCall.name,
+            args: toolCall.args,
+            id: toolCall.id ?? uuidFromString(JSON.stringify(toolCall.args)),
+            output: undefined,
+          })),
+          reasoning,
+          isComplete: true, // Messages from DB are always complete
+          isReasoning: false, // since this is not a reasoning message and the request is done
+        });
+      }
+      if (toolMsg) {
+        const toolCallId =
+          toolMsg.lc_kwargs.tool_call_id ??
+          uuidFromString(JSON.stringify(toolMsg.lc_kwargs.args));
+        const messageWithToolCallIdIdx = acc.findIndex((m) =>
+          m.toolCalls?.find((t) => t.id === toolCallId),
+        );
+
+        // if the message with the tool call id exits then update the tool Call to add the output
+        const el =
+          messageWithToolCallIdIdx !== -1
+            ? acc[messageWithToolCallIdIdx]
+            : null;
+        if (el) {
+          el.toolCalls = el.toolCalls?.map((t) =>
+            t.id === toolCallId
+              ? {
+                  ...t,
+                  output: JSON.stringify(toolMsg.content),
+                  status: 'done',
+                }
+              : t,
+          );
+          acc[messageWithToolCallIdIdx] = el;
+        }
+      }
+
+      return acc;
+    }, []),
+  };
+}
+
+export const uuidFromString = (str: string): string => {
+  const isStrUUID = isUUID(str);
+  if (isStrUUID) return str;
+  // generate a uuid from a string
+  const hash = crypto.createHash('sha256');
+  hash.update(str);
+  return hash.digest('hex');
+};

package/src/ai/utils/verify-matrix-openId-token.ts

@@ -0,0 +1,46 @@
+import { Logger } from "@ixo/logger";
+
+const ERROR_CODES = ['M_UNKNOWN_TOKEN', 'M_FORBIDDEN'];
+export async function verifyMatrixOpenIdToken(
+  openIdToken: string,
+  baseUrl: string,
+): Promise<{ isValid: boolean; userId?: string; error?: string }> {
+  try {
+    // Make request to Matrix federation endpoint
+    const response = await fetch(
+      `${baseUrl.endsWith('/') ? baseUrl.slice(0, -1) : baseUrl}/_matrix/federation/v1/openid/userinfo?access_token=${openIdToken}`,
+      {
+        method: 'GET',
+      },
+    );
+
+    if (response.ok) {
+      const userInfo = (await response.json()) as { sub: string };
+      console.log('User info from server:', userInfo);
+
+      return {
+        isValid: true,
+        userId: userInfo.sub, // The verified user ID
+      };
+    }
+    const error = await response.json() as { errcode: string };
+    const isAuthError = ERROR_CODES.includes(error.errcode);
+    if (!isAuthError) { 
+      Logger.error('Error verifying Matrix OpenID token:', error);
+    }
+    return {
+      isValid: false,
+      error: `Server returned ${response.status}: ${response.statusText}`,
+    };
+  } catch (error) {
+    const errorMessage =
+      error instanceof Error ? error.message : String(error);
+    const errorStack = error instanceof Error ? error.stack : undefined;
+
+    console.error('Error verifying Matrix OpenID token:', errorMessage, errorStack);
+    return {
+      isValid: false,
+      error: error instanceof Error ? error.message : 'Unknown error',
+    };
+  }
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from './ai/index.js';
+export * from './services/index.js';
+export * from './utils/index.js';