@ixo/common 1.1.2 → 1.1.4

package/dist/ai/nodes/find-docs/find-docs.prompt.d.ts

@@ -1,3 +0,0 @@
-declare const SELF_QUERY_RAG_PROMPT = "You are an advanced AI agent specializing in self-query Retrieval-Augmented Generation (RAG). Your primary function is to analyze client questions, generate optimized search queries for a knowledge base, and iteratively refine these queries based on initial results. Follow these comprehensive guidelines:\n\n1. Initial Analysis:\n   - Thoroughly examine the client's question or task\n   - Identify key concepts, entities, relationships, and any implicit information needs\n   - Determine the scope, context, and potential complexities of the inquiry\n\n2. Query Generation - First Pass:\n   - Create an initial set of 3-5 focused, non-redundant search queries\n   - Ensure each query addresses a unique aspect of the client's input\n   - Incorporate key terms from the original question for keyword optimization\n   - Vary query formulations to cover different potential phrasings and synonyms\n\n3. Iterative Refinement:\n   - After receiving initial results, analyze their relevance and comprehensiveness\n   - Identify gaps in the retrieved information\n   - Generate additional queries to fill these gaps or explore new angles\n   - Refine existing queries based on the quality of results they produced\n\n4. Query Diversification:\n   - Ensure a mix of query types:\n     a. Broad conceptual queries for general information\n     b. Specific detail-oriented queries for precise facts\n     c. Comparative queries to explore relationships between concepts\n     d. Contextual queries to gather background information\n\n5. Handling Complex Questions:\n   - For multi-faceted questions, break them down into component parts\n   - Generate separate query sets for each component\n   - Create integrative queries that explore connections between components\n\n6. Adapting to Query Results:\n   - If initial queries yield insufficient information, broaden the scope\n   - If results are too general, create more specific, targeted queries\n   - Adjust the vocabulary and technical level based on the knowledge base content\n\n7. Temporal and Contextual Considerations:\n   - Include queries that account for potential time-sensitive information\n   - Generate queries that explore historical context if relevant\n   - Consider geographical or domain-specific variations in terminology\n\n8. Output Format and Explanation:\n   - Present a numbered list of refined search queries\n   - For each query, provide:\n     a. The query itself\n     b. A brief explanation of its purpose and how it relates to the client's question\n     c. Any notable findings or gaps identified from previous iterations\n\n9. Continuous Learning:\n   - Track the effectiveness of different query structures and patterns\n   - Adapt your query generation strategy based on successful retrievals\n   - Note any recurring challenges or limitations in the knowledge base\n\n10. Final Review and Optimization:\n    - Ensure the final set of queries comprehensively covers the client's needs\n    - Eliminate any remaining redundancies or overly similar queries\n    - Prioritize queries based on their potential to provide the most relevant and valuable information\n\nRemember: Your goal is to generate a dynamic, adaptive set of queries that evolves based on initial results and comprehensively addresses the client's information needs. Quality, relevance, and adaptability are key to successful self-query RAG implementation.";
-export { SELF_QUERY_RAG_PROMPT };
-//# sourceMappingURL=find-docs.prompt.d.ts.map

package/dist/ai/nodes/find-docs/find-docs.prompt.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"find-docs.prompt.d.ts","sourceRoot":"","sources":["../../../../src/ai/nodes/find-docs/find-docs.prompt.ts"],"names":[],"mappings":"AAAA,QAAA,MAAM,qBAAqB,+0GA0DyO,CAAC;AAErQ,OAAO,EAAE,qBAAqB,EAAE,CAAC"}

package/dist/ai/nodes/find-docs/find-docs.prompt.js

@@ -1,61 +0,0 @@
-const SELF_QUERY_RAG_PROMPT = `You are an advanced AI agent specializing in self-query Retrieval-Augmented Generation (RAG). Your primary function is to analyze client questions, generate optimized search queries for a knowledge base, and iteratively refine these queries based on initial results. Follow these comprehensive guidelines:
-
-1. Initial Analysis:
-   - Thoroughly examine the client's question or task
-   - Identify key concepts, entities, relationships, and any implicit information needs
-   - Determine the scope, context, and potential complexities of the inquiry
-
-2. Query Generation - First Pass:
-   - Create an initial set of 3-5 focused, non-redundant search queries
-   - Ensure each query addresses a unique aspect of the client's input
-   - Incorporate key terms from the original question for keyword optimization
-   - Vary query formulations to cover different potential phrasings and synonyms
-
-3. Iterative Refinement:
-   - After receiving initial results, analyze their relevance and comprehensiveness
-   - Identify gaps in the retrieved information
-   - Generate additional queries to fill these gaps or explore new angles
-   - Refine existing queries based on the quality of results they produced
-
-4. Query Diversification:
-   - Ensure a mix of query types:
-     a. Broad conceptual queries for general information
-     b. Specific detail-oriented queries for precise facts
-     c. Comparative queries to explore relationships between concepts
-     d. Contextual queries to gather background information
-
-5. Handling Complex Questions:
-   - For multi-faceted questions, break them down into component parts
-   - Generate separate query sets for each component
-   - Create integrative queries that explore connections between components
-
-6. Adapting to Query Results:
-   - If initial queries yield insufficient information, broaden the scope
-   - If results are too general, create more specific, targeted queries
-   - Adjust the vocabulary and technical level based on the knowledge base content
-
-7. Temporal and Contextual Considerations:
-   - Include queries that account for potential time-sensitive information
-   - Generate queries that explore historical context if relevant
-   - Consider geographical or domain-specific variations in terminology
-
-8. Output Format and Explanation:
-   - Present a numbered list of refined search queries
-   - For each query, provide:
-     a. The query itself
-     b. A brief explanation of its purpose and how it relates to the client's question
-     c. Any notable findings or gaps identified from previous iterations
-
-9. Continuous Learning:
-   - Track the effectiveness of different query structures and patterns
-   - Adapt your query generation strategy based on successful retrievals
-   - Note any recurring challenges or limitations in the knowledge base
-
-10. Final Review and Optimization:
-    - Ensure the final set of queries comprehensively covers the client's needs
-    - Eliminate any remaining redundancies or overly similar queries
-    - Prioritize queries based on their potential to provide the most relevant and valuable information
-
-Remember: Your goal is to generate a dynamic, adaptive set of queries that evolves based on initial results and comprehensively addresses the client's information needs. Quality, relevance, and adaptability are key to successful self-query RAG implementation.`;
-export { SELF_QUERY_RAG_PROMPT };
-//# sourceMappingURL=find-docs.prompt.js.map

package/dist/ai/nodes/find-docs/find-docs.prompt.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"find-docs.prompt.js","sourceRoot":"","sources":["../../../../src/ai/nodes/find-docs/find-docs.prompt.ts"],"names":[],"mappings":"AAAA,MAAM,qBAAqB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oQA0DsO,CAAC;AAErQ,OAAO,EAAE,qBAAqB,EAAE,CAAC"}

package/dist/ai/nodes/find-docs/index.d.ts

@@ -1,3 +0,0 @@
-export * from './find-docs.prompt.js';
-export * from './node.js';
-//# sourceMappingURL=index.d.ts.map

package/dist/ai/nodes/find-docs/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/ai/nodes/find-docs/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC"}

package/dist/ai/nodes/find-docs/index.js

@@ -1,3 +0,0 @@
-export * from './find-docs.prompt.js';
-export * from './node.js';
-//# sourceMappingURL=index.js.map

package/dist/ai/nodes/find-docs/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/ai/nodes/find-docs/index.ts"],"names":[],"mappings":"AAAA,cAAc,uBAAuB,CAAC;AACtC,cAAc,WAAW,CAAC"}

package/dist/ai/nodes/find-docs/node.d.ts

@@ -1,17 +0,0 @@
-import { type VectorDBDataStore } from '@ixo/data-store';
-import { type Document } from '@langchain/core/documents';
-import { type BaseMessage } from '@langchain/core/messages';
-import { type RunnableConfig } from '@langchain/core/runnables';
-import 'dotenv/config';
-interface IFindDocsNodeRequiredState {
-    question: string;
-    docs: Document[];
-    messages?: BaseMessage[];
-    config?: {
-        isInternal: boolean;
-    };
-    status?: 'completed' | 'inProgress';
-}
-export declare const findDocsNode: (store: VectorDBDataStore) => (state: IFindDocsNodeRequiredState, config?: RunnableConfig) => Promise<Partial<IFindDocsNodeRequiredState>>;
-export {};
-//# sourceMappingURL=node.d.ts.map

package/dist/ai/nodes/find-docs/node.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"node.d.ts","sourceRoot":"","sources":["../../../../src/ai/nodes/find-docs/node.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEzD,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAE5D,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,eAAe,CAAC;AAQvB,UAAU,0BAA0B;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,QAAQ,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,WAAW,EAAE,CAAC;IACzB,MAAM,CAAC,EAAE;QACP,UAAU,EAAE,OAAO,CAAC;KACrB,CAAC;IACF,MAAM,CAAC,EAAE,WAAW,GAAG,YAAY,CAAC;CACrC;AAOD,eAAO,MAAM,YAAY,GACtB,OAAO,iBAAiB,MAEvB,OAAO,0BAA0B,EACjC,SAAS,cAAc,KACtB,OAAO,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAgD7C,CAAC"}

package/dist/ai/nodes/find-docs/node.js

@@ -1,46 +0,0 @@
-import { Logger } from '@ixo/logger';
-import { ChatPromptTemplate } from '@langchain/core/prompts';
-import 'dotenv/config';
-import z from 'zod';
-import { getChatOpenAiModel, retrieverToolFactory, SELF_QUERY_RAG_PROMPT, } from '../../index.js';
-export const findDocsNode = (store) => async (state, config) => {
-    try {
-        const model = getChatOpenAiModel();
-        const modelWithTools = model.withStructuredOutput(z.object({
-            questions: z.array(z.string('Generated Queries')),
-        }));
-        const chain = ChatPromptTemplate.fromMessages([
-            ['system', SELF_QUERY_RAG_PROMPT],
-            [
-                'user',
-                state.messages
-                    ? `this is the last three messages from the user's conversation ${state.messages
-                        .map((message) => message.content)
-                        .slice(-3)
-                        .join(',')}`
-                    : state.question,
-            ],
-        ]).pipe(modelWithTools);
-        const response = await chain.invoke({}, config);
-        const retrieval = retrieverToolFactory({
-            filters: state.config?.isInternal
-                ? undefined
-                : { approved: true, visibility: 'public' },
-            store,
-        });
-        const docs = await Promise.all(response.questions.map((question) => retrieval.invoke({ query: question })));
-        return {
-            docs: docs.flat().map((value) => ({
-                metadata: value.metadata,
-                pageContent: value.pageContent.toString(),
-                id: value.id,
-            })),
-            status: undefined,
-        };
-    }
-    catch (error) {
-        Logger.error('Error finding docs', error);
-        throw error;
-    }
-};
-//# sourceMappingURL=node.js.map

package/dist/ai/nodes/find-docs/node.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"node.js","sourceRoot":"","sources":["../../../../src/ai/nodes/find-docs/node.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAE7D,OAAO,eAAe,CAAC;AACvB,OAAO,CAAC,MAAM,KAAK,CAAC;AACpB,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,qBAAqB,GACtB,MAAM,gBAAgB,CAAC;AAiBxB,MAAM,CAAC,MAAM,YAAY,GACvB,CAAC,KAAwB,EAAE,EAAE,CAC7B,KAAK,EACH,KAAiC,EACjC,MAAuB,EACuB,EAAE;IAChD,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,kBAAkB,EAAE,CAAC;QACnC,MAAM,cAAc,GAAG,KAAK,CAAC,oBAAoB,CAC/C,CAAC,CAAC,MAAM,CAAC;YACP,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;SAClD,CAAC,CACH,CAAC;QAEF,MAAM,KAAK,GAAG,kBAAkB,CAAC,YAAY,CAAC;YAC5C,CAAC,QAAQ,EAAE,qBAAqB,CAAC;YACjC;gBACE,MAAM;gBACN,KAAK,CAAC,QAAQ;oBACZ,CAAC,CAAC,gEAAgE,KAAK,CAAC,QAAQ;yBAC3E,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;yBACjC,KAAK,CAAC,CAAC,CAAC,CAAC;yBACT,IAAI,CAAC,GAAG,CAAC,EAAE;oBAChB,CAAC,CAAC,KAAK,CAAC,QAAQ;aACnB;SACF,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAExB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAChD,MAAM,SAAS,GAAG,oBAAoB,CAAC;YACrC,OAAO,EAAE,KAAK,CAAC,MAAM,EAAE,UAAU;gBAC/B,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE;YAC5C,KAAK;SACN,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5B,QAAQ,CAAC,SAAS,CAAC,GAAG,CACpB,CAAC,QAAQ,EAAE,EAAE,CACX,SAAS,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAwB,CAC/D,CACF,CAAC;QAEF,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAChC,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,WAAW,EAAE,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE;gBACzC,EAAE,EAAE,KAAK,CAAC,EAAE;aACb,CAAC,CAAC;YACH,MAAM,EAAE,SAAS;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,KAAK,CAAC,CAAC;QAC1C,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC,CAAC"}

package/dist/ai/tools/retriever-tool/index.d.ts

@@ -1,2 +0,0 @@
-export * from './retriever-tool.js';
-//# sourceMappingURL=index.d.ts.map

package/dist/ai/tools/retriever-tool/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/ai/tools/retriever-tool/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC"}

package/dist/ai/tools/retriever-tool/index.js

@@ -1,2 +0,0 @@
-export * from './retriever-tool.js';
-//# sourceMappingURL=index.js.map

package/dist/ai/tools/retriever-tool/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/ai/tools/retriever-tool/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC"}

package/dist/ai/tools/retriever-tool/retriever-tool.d.ts

@@ -1,18 +0,0 @@
-import { type VectorDBDataStore } from '@ixo/data-store';
-import { type BaseChatModel } from '@langchain/core/language_models/chat_models';
-import { type DynamicStructuredTool } from '@langchain/core/tools';
-import z from 'zod';
-type RetrieverToolFactoryArgs<Filters extends Record<string, unknown> = Record<string, unknown>, MapType extends Map<string, unknown> = Map<string, unknown>> = {
-    model?: BaseChatModel;
-    filters?: Filters;
-    similarThreshold?: number;
-    store: VectorDBDataStore;
-    map?: MapType;
-    requestId?: string;
-};
-declare const schema: z.ZodObject<{
-    query: z.ZodString;
-}, z.z.core.$strip>;
-export declare const retrieverToolFactory: ({ model, filters, similarThreshold, map, requestId, store, }: RetrieverToolFactoryArgs) => DynamicStructuredTool<typeof schema>;
-export {};
-//# sourceMappingURL=retriever-tool.d.ts.map

package/dist/ai/tools/retriever-tool/retriever-tool.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"retriever-tool.d.ts","sourceRoot":"","sources":["../../../../src/ai/tools/retriever-tool/retriever-tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAGzD,OAAO,EAAE,KAAK,aAAa,EAAE,MAAM,6CAA6C,CAAC;AACjF,OAAO,EAAQ,KAAK,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACzE,OAAO,CAAC,MAAM,KAAK,CAAC;AAIpB,KAAK,wBAAwB,CAC3B,OAAO,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACjE,OAAO,SAAS,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,IACzD;IACF,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,KAAK,EAAE,iBAAiB,CAAC;IACzB,GAAG,CAAC,EAAE,OAAO,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,QAAA,MAAM,MAAM;;mBAEV,CAAC;AAEH,eAAO,MAAM,oBAAoB,GAAI,8DAOlC,wBAAwB,KAAG,qBAAqB,CAAC,OAAO,MAAM,CA0E9D,CAAC"}

package/dist/ai/tools/retriever-tool/retriever-tool.js

@@ -1,62 +0,0 @@
-import { Logger } from '@ixo/logger';
-import { Document } from '@langchain/core/documents';
-import { tool } from '@langchain/core/tools';
-import z from 'zod';
-import { getChatOpenAiModel } from '../../models/openai.js';
-import checkDocRelevance from '../../utils/doc-relevance-checker.js';
-const schema = z.object({
-    query: z.string(),
-});
-export const retrieverToolFactory = ({ model = getChatOpenAiModel(), filters, similarThreshold = 0.3, map, requestId, store, }) => tool(async ({ query }) => {
-    const docs = await store.queryWithSimilarity(query, {
-        similarityThreshold: similarThreshold,
-        filters,
-    });
-    if (map) {
-        map.set(requestId ?? query, docs.map((doc) => doc.metadata));
-    }
-    if (docs.length === 0) {
-        return undefined;
-    }
-    if (similarThreshold >= 0.3) {
-        const relevantDocs = await Promise.all(docs.map(async (doc) => {
-            try {
-                return (await checkDocRelevance({
-                    doc: new Document({
-                        pageContent: doc.content,
-                        id: doc.id,
-                        metadata: doc.metadata,
-                    }),
-                    query,
-                    model,
-                })).valueOf()
-                    ? doc
-                    : null;
-            }
-            catch (error) {
-                Logger.error(`Error checking relevance of document:`, error);
-                return null;
-            }
-        }));
-        return relevantDocs.reduce((acc, doc) => {
-            if (doc) {
-                acc.push(new Document({
-                    pageContent: doc.content,
-                    id: doc.id,
-                    metadata: doc.metadata,
-                }));
-            }
-            return acc;
-        }, []);
-    }
-    return docs.map((doc) => new Document({
-        pageContent: doc.content,
-        id: doc.id,
-        metadata: doc.metadata,
-    }));
-}, {
-    name: 'retrieverTool',
-    description: 'Retrieves documents from the knowledge base based on the query',
-    schema,
-});
-//# sourceMappingURL=retriever-tool.js.map

package/dist/ai/tools/retriever-tool/retriever-tool.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"retriever-tool.js","sourceRoot":"","sources":["../../../../src/ai/tools/retriever-tool/retriever-tool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAErD,OAAO,EAAE,IAAI,EAA8B,MAAM,uBAAuB,CAAC;AACzE,OAAO,CAAC,MAAM,KAAK,CAAC;AACpB,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,iBAAiB,MAAM,sCAAsC,CAAC;AAcrE,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IACtB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;CAClB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,EACnC,KAAK,GAAG,kBAAkB,EAAE,EAC5B,OAAO,EACP,gBAAgB,GAAG,GAAG,EACtB,GAAG,EACH,SAAS,EACT,KAAK,GACoB,EAAwC,EAAE,CACnE,IAAI,CACF,KAAK,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE;IAClB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,mBAAmB,CAAC,KAAK,EAAE;QAClD,mBAAmB,EAAE,gBAAgB;QACrC,OAAO;KACR,CAAC,CAAC;IAGH,IAAI,GAAG,EAAE,CAAC;QACR,GAAG,CAAC,GAAG,CACL,SAAS,IAAI,KAAK,EAClB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAChC,CAAC;IACJ,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,IAAI,gBAAgB,IAAI,GAAG,EAAE,CAAC;QAE5B,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACrB,IAAI,CAAC;gBACH,OAAO,CACL,MAAM,iBAAiB,CAAC;oBACtB,GAAG,EAAE,IAAI,QAAQ,CAAC;wBAChB,WAAW,EAAE,GAAG,CAAC,OAAO;wBACxB,EAAE,EAAE,GAAG,CAAC,EAAE;wBACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;qBACvB,CAAC;oBACF,KAAK;oBACL,KAAK;iBACN,CAAC,CACH,CAAC,OAAO,EAAE;oBACT,CAAC,CAAC,GAAG;oBACL,CAAC,CAAC,IAAI,CAAC;YACX,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,KAAK,CAAC,CAAC;gBAC7D,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QAEF,OAAO,YAAY,CAAC,MAAM,CAAa,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YAClD,IAAI,GAAG,EAAE,CAAC;gBACR,GAAG,CAAC,IAAI,CACN,IAAI,QAAQ,CAAC;oBACX,WAAW,EAAE,GAAG,CAAC,OAAO;oBACxB,EAAE,EAAE,GAAG,CAAC,EAAE;oBACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;iBACvB,CAAC,CACH,CAAC;YACJ,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC,EAAE,EAAE,CAAC,CAAC;IACT,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CACb,CAAC,GAAG,EAAE,EAAE,CACN,IAAI,QAAQ,CAAC;QACX,WAAW,EAAE,GAAG,CAAC,OAAO;QACxB,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,QAAQ,EAAE,GAAG,CAAC,QAAQ;KACvB,CAAC,CACL,CAAC;AACJ,CAAC,EACD;IACE,IAAI,EAAE,eAAe;IACrB,WAAW,EACT,gEAAgE;IAClE,MAAM;CACP,CACF,CAAC"}

package/src/ai/nodes/find-docs/find-docs.prompt.ts

@@ -1,61 +0,0 @@
-const SELF_QUERY_RAG_PROMPT = `You are an advanced AI agent specializing in self-query Retrieval-Augmented Generation (RAG). Your primary function is to analyze client questions, generate optimized search queries for a knowledge base, and iteratively refine these queries based on initial results. Follow these comprehensive guidelines:
-
-1. Initial Analysis:
-   - Thoroughly examine the client's question or task
-   - Identify key concepts, entities, relationships, and any implicit information needs
-   - Determine the scope, context, and potential complexities of the inquiry
-
-2. Query Generation - First Pass:
-   - Create an initial set of 3-5 focused, non-redundant search queries
-   - Ensure each query addresses a unique aspect of the client's input
-   - Incorporate key terms from the original question for keyword optimization
-   - Vary query formulations to cover different potential phrasings and synonyms
-
-3. Iterative Refinement:
-   - After receiving initial results, analyze their relevance and comprehensiveness
-   - Identify gaps in the retrieved information
-   - Generate additional queries to fill these gaps or explore new angles
-   - Refine existing queries based on the quality of results they produced
-
-4. Query Diversification:
-   - Ensure a mix of query types:
-     a. Broad conceptual queries for general information
-     b. Specific detail-oriented queries for precise facts
-     c. Comparative queries to explore relationships between concepts
-     d. Contextual queries to gather background information
-
-5. Handling Complex Questions:
-   - For multi-faceted questions, break them down into component parts
-   - Generate separate query sets for each component
-   - Create integrative queries that explore connections between components
-
-6. Adapting to Query Results:
-   - If initial queries yield insufficient information, broaden the scope
-   - If results are too general, create more specific, targeted queries
-   - Adjust the vocabulary and technical level based on the knowledge base content
-
-7. Temporal and Contextual Considerations:
-   - Include queries that account for potential time-sensitive information
-   - Generate queries that explore historical context if relevant
-   - Consider geographical or domain-specific variations in terminology
-
-8. Output Format and Explanation:
-   - Present a numbered list of refined search queries
-   - For each query, provide:
-     a. The query itself
-     b. A brief explanation of its purpose and how it relates to the client's question
-     c. Any notable findings or gaps identified from previous iterations
-
-9. Continuous Learning:
-   - Track the effectiveness of different query structures and patterns
-   - Adapt your query generation strategy based on successful retrievals
-   - Note any recurring challenges or limitations in the knowledge base
-
-10. Final Review and Optimization:
-    - Ensure the final set of queries comprehensively covers the client's needs
-    - Eliminate any remaining redundancies or overly similar queries
-    - Prioritize queries based on their potential to provide the most relevant and valuable information
-
-Remember: Your goal is to generate a dynamic, adaptive set of queries that evolves based on initial results and comprehensively addresses the client's information needs. Quality, relevance, and adaptability are key to successful self-query RAG implementation.`;
-
-export { SELF_QUERY_RAG_PROMPT };

package/src/ai/nodes/find-docs/index.ts

@@ -1,2 +0,0 @@
-export * from './find-docs.prompt.js';
-export * from './node.js';

package/src/ai/nodes/find-docs/node.ts

@@ -1,83 +0,0 @@
-import { type VectorDBDataStore } from '@ixo/data-store';
-import { Logger } from '@ixo/logger';
-import { type Document } from '@langchain/core/documents';
-import { type BaseMessage } from '@langchain/core/messages';
-import { ChatPromptTemplate } from '@langchain/core/prompts';
-import { type RunnableConfig } from '@langchain/core/runnables';
-import 'dotenv/config';
-import z from 'zod';
-import {
-  getChatOpenAiModel,
-  retrieverToolFactory,
-  SELF_QUERY_RAG_PROMPT,
-} from '../../index.js';
-
-interface IFindDocsNodeRequiredState {
-  question: string;
-  docs: Document[];
-  messages?: BaseMessage[];
-  config?: {
-    isInternal: boolean;
-  };
-  status?: 'completed' | 'inProgress';
-}
-
-/**
- * factory function that returns a Node that takes a state and config and returns a promise that resolves to a partial state.
- * @param store - The vector database data store.
- * @returns The Node that takes a state and config and returns a promise that resolves to a partial state.
- */
-export const findDocsNode =
-  (store: VectorDBDataStore) =>
-  async (
-    state: IFindDocsNodeRequiredState,
-    config?: RunnableConfig,
-  ): Promise<Partial<IFindDocsNodeRequiredState>> => {
-    try {
-      const model = getChatOpenAiModel();
-      const modelWithTools = model.withStructuredOutput(
-        z.object({
-          questions: z.array(z.string('Generated Queries')),
-        }),
-      );
-
-      const chain = ChatPromptTemplate.fromMessages([
-        ['system', SELF_QUERY_RAG_PROMPT],
-        [
-          'user',
-          state.messages
-            ? `this is the last three messages from the user's conversation ${state.messages
-                .map((message) => message.content)
-                .slice(-3)
-                .join(',')}`
-            : state.question,
-        ],
-      ]).pipe(modelWithTools);
-
-      const response = await chain.invoke({}, config);
-      const retrieval = retrieverToolFactory({
-        filters: state.config?.isInternal
-          ? undefined
-          : { approved: true, visibility: 'public' },
-        store,
-      });
-      const docs = await Promise.all(
-        response.questions.map(
-          (question) =>
-            retrieval.invoke({ query: question }) as Promise<Document[]>,
-        ),
-      );
-
-      return {
-        docs: docs.flat().map((value) => ({
-          metadata: value.metadata,
-          pageContent: value.pageContent.toString(),
-          id: value.id,
-        })),
-        status: undefined,
-      };
-    } catch (error) {
-      Logger.error('Error finding docs', error);
-      throw error;
-    }
-  };

package/src/ai/tools/retriever-tool/index.ts

@@ -1 +0,0 @@
-export * from './retriever-tool.js';

package/src/ai/tools/retriever-tool/retriever-tool.test.ts

@@ -1,163 +0,0 @@
-import { type VectorDBDataStore } from '@ixo/data-store';
-import { Document } from '@langchain/core/documents';
-import { FakeChatModel } from '@langchain/core/utils/testing';
-import { retrieverToolFactory } from './retriever-tool.js';
-
-const fakeModel = new FakeChatModel({});
-
-// Mock the logger to prevent console output during tests
-vi.mock('@ixo/logger', () => ({
-  Logger: {
-    error: vi.fn(),
-  },
-}));
-
-// Mock the OpenAI model
-vi.mock('../../models/openai', () => ({
-  getChatOpenAiModel: () => fakeModel,
-}));
-
-// Mock the doc relevance checker
-vi.mock('../../utils/doc-relevance-checker', () => ({
-  default: vi.fn().mockResolvedValue(true),
-}));
-const fakeDocRelevanceChecker = vi.fn().mockResolvedValue(true);
-
-vi.spyOn(
-  await import('../../utils/doc-relevance-checker'),
-  'default',
-).mockImplementation(fakeDocRelevanceChecker);
-
-describe('RetrieverTool', () => {
-  let mockStore: { queryWithSimilarity: ReturnType<typeof vi.fn> };
-  const mockModel: FakeChatModel = new FakeChatModel({});
-
-  beforeEach(() => {
-    mockStore = {
-      queryWithSimilarity: vi.fn(),
-    };
-  });
-
-  it('should retrieve documents with default settings', async () => {
-    const mockDocs = [
-      { id: '1', content: 'test content 1', metadata: { source: 'test1' } },
-      { id: '2', content: 'test content 2', metadata: { source: 'test2' } },
-    ];
-
-    mockStore.queryWithSimilarity.mockResolvedValue(mockDocs);
-
-    const tool = retrieverToolFactory({
-      store: mockStore as unknown as VectorDBDataStore,
-    });
-    const result = (await tool.invoke({ query: 'test query' })) as Document[];
-
-    expect(mockStore.queryWithSimilarity).toHaveBeenCalledWith('test query', {
-      similarityThreshold: 0.3,
-      filters: undefined,
-    });
-
-    expect(result).toHaveLength(2);
-    expect(result[0]).toBeInstanceOf(Document);
-    expect(result[0]?.pageContent).toBe('test content 1');
-    expect(result[0]?.metadata).toEqual({ source: 'test1' });
-
-    expect(fakeDocRelevanceChecker).toHaveBeenCalledWith({
-      doc: result[0],
-      query: 'test query',
-      model: fakeModel,
-    });
-  });
-
-  it('should store results in provided map', async () => {
-    const mockDocs = [
-      { id: '1', content: 'test content', metadata: { source: 'test' } },
-    ];
-    mockStore.queryWithSimilarity.mockResolvedValue(mockDocs);
-
-    const resultsMap = new Map();
-    const requestId = 'test-request-id';
-
-    const tool = retrieverToolFactory({
-      store: mockStore as unknown as VectorDBDataStore,
-      map: resultsMap,
-      requestId,
-    });
-
-    await tool.invoke({ query: 'test query' });
-
-    expect(resultsMap.get(requestId)).toEqual([{ source: 'test' }]);
-  });
-
-  it('should return undefined when no documents found', async () => {
-    mockStore.queryWithSimilarity.mockResolvedValue([]);
-
-    const tool = retrieverToolFactory({
-      store: mockStore as unknown as VectorDBDataStore,
-    });
-    const result = await tool.invoke({ query: 'test query' });
-
-    expect(result).toBeUndefined();
-  });
-
-  it('should check document relevance when similarity threshold is higher', async () => {
-    const mockDocs = [
-      { id: '1', content: 'test content', metadata: { source: 'test' } },
-    ];
-    mockStore.queryWithSimilarity.mockResolvedValue(mockDocs);
-
-    const tool = retrieverToolFactory({
-      store: mockStore as unknown as VectorDBDataStore,
-      similarThreshold: 0.5,
-      model: mockModel,
-    });
-
-    const result = await tool.invoke({ query: 'test query' });
-
-    expect(result).toHaveLength(1);
-    expect(result[0]).toBeInstanceOf(Document);
-  });
-
-  it('should handle errors during relevance checking', async () => {
-    const mockDocs = [
-      { id: '1', content: 'test content', metadata: { source: 'test' } },
-    ];
-    mockStore.queryWithSimilarity.mockResolvedValue(mockDocs);
-
-    // Mock the relevance checker to throw an error
-    const docRelevanceModule = await import(
-      '../../utils/doc-relevance-checker'
-    );
-    (
-      docRelevanceModule.default as ReturnType<typeof vi.fn>
-    ).mockRejectedValueOnce(new Error('Test error'));
-
-    const tool = retrieverToolFactory({
-      store: mockStore as unknown as VectorDBDataStore,
-      similarThreshold: 0.5,
-      model: mockModel,
-    });
-
-    const result = await tool.invoke({ query: 'test query' });
-
-    expect(result).toHaveLength(0);
-  });
-
-  it('should apply filters when provided', async () => {
-    const mockDocs = [
-      { id: '1', content: 'test content', metadata: { source: 'test' } },
-    ];
-    mockStore.queryWithSimilarity.mockResolvedValue(mockDocs);
-    const filters = { category: 'test' };
-    const tool = retrieverToolFactory({
-      store: mockStore as unknown as VectorDBDataStore,
-      filters,
-    });
-
-    await tool.invoke({ query: 'test query' });
-
-    expect(mockStore.queryWithSimilarity).toHaveBeenCalledWith('test query', {
-      similarityThreshold: 0.3,
-      filters,
-    });
-  });
-});