@delofarag/ai-utils 1.0.0

package/dist/src/rag.d.ts

@@ -0,0 +1,58 @@
+import { SupabaseVectorStore, FaissStore, Document, Embeddings, ChatPromptTemplate, BaseChatModel, VectorStore, BaseRetriever } from "./imports";
+import { SupabaseClient } from "@delofarag/supabase-utils";
+interface SupabaseStoreConfig {
+    data: string[];
+    table_name?: string;
+    RPC_function?: string;
+}
+export declare function turn_to_docs<T>(docs: T[]): Document<Record<string, any>>[];
+export declare function createSupabaseVectoreStore({ supabase, data, table_name, RPC_function }: SupabaseStoreConfig & {
+    supabase: SupabaseClient;
+}): Promise<SupabaseVectorStore>;
+export declare function getSupabaseVectorStore({ supabase, table_name, RPC_function }: {
+    supabase: SupabaseClient;
+    table_name?: string;
+    RPC_function?: string;
+}): SupabaseVectorStore;
+export declare function createFaissStore({ data, save_path, embeddings }: {
+    data: string[];
+    save_path?: string;
+    embeddings?: Embeddings;
+}): Promise<FaissStore>;
+export declare function loadFaissStore({ path, embeddings }: {
+    path: string;
+    embeddings?: Embeddings;
+}): Promise<FaissStore>;
+export declare function createRAGChain({ vectorStore, llm, prompt, num_of_results_from_vdb }: {
+    vectorStore: VectorStore;
+    llm: BaseChatModel;
+    prompt?: ChatPromptTemplate;
+    num_of_results_from_vdb?: number;
+}): Promise<import("@langchain/core/dist/runnables/base").Runnable<{
+    input: string;
+    chat_history?: import("@langchain/core/dist/messages/base").BaseMessage[] | string;
+} & {
+    [key: string]: unknown;
+}, {
+    context: Document[];
+    answer: string;
+} & {
+    [key: string]: unknown;
+}, import("@langchain/core/dist/runnables/types").RunnableConfig<Record<string, any>>>>;
+export declare function createRAGChainFromRetriever({ retriever, llm, prompt }: {
+    retriever: BaseRetriever;
+    llm: BaseChatModel;
+    prompt?: ChatPromptTemplate;
+}): Promise<import("@langchain/core/dist/runnables/base").Runnable<{
+    input: string;
+    chat_history?: import("@langchain/core/dist/messages/base").BaseMessage[] | string;
+} & {
+    [key: string]: unknown;
+}, {
+    context: Document[];
+    answer: string;
+} & {
+    [key: string]: unknown;
+}, import("@langchain/core/dist/runnables/types").RunnableConfig<Record<string, any>>>>;
+export {};
+//# sourceMappingURL=rag.d.ts.map

package/dist/src/rag.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rag.d.ts","sourceRoot":"","sources":["../../src/rag.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,mBAAmB,EAGnB,UAAU,EACV,QAAQ,EACR,UAAU,EAGV,kBAAkB,EAClB,aAAa,EACb,WAAW,EACX,aAAa,EAChB,MAAM,WAAW,CAAA;AAClB,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAA;AAG1D,UAAU,mBAAmB;IACzB,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;CACxB;AAWD,wBAAgB,YAAY,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAC,GAAG,CAAC,CAAC,EAAE,CAKzE;AAED,wBAAsB,0BAA0B,CAAC,EAAC,QAAQ,EAAE,IAAI,EAAE,UAAwB,EAAE,YAAgC,EAAC,EAAE,mBAAmB,GAAG;IAAC,QAAQ,EAAE,cAAc,CAAA;CAAC,gCAa9K;AAGD,wBAAgB,sBAAsB,CAAC,EAAC,QAAQ,EAAE,UAAwB,EAAE,YAAgC,EAAC,EAAE;IAAC,QAAQ,EAAE,cAAc,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAC,uBAMpL;AAED,wBAAsB,gBAAgB,CAAC,EAAC,IAAI,EAAE,SAAuB,EAAE,UAA2B,EAAC,EAAE;IAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,UAAU,CAAA;CAAC,uBAWjK;AAED,wBAAsB,cAAc,CAAC,EAAC,IAAI,EAAE,UAA2B,EAAC,EAAE;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,UAAU,CAAA;CAAC,uBAEhH;AAED,wBAAsB,cAAc,CAAC,EACjC,WAAW,EACX,GAAG,EACH,MAAM,EACN,uBAA2B,EAC9B,EAAE;IACC,WAAW,EAAE,WAAW,CAAC;IACzB,GAAG,EAAE,aAAa,CAAC;IACnB,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,uBAAuB,CAAC,EAAE,MAAM,CAAA;CACnC;;;;;;;;;;wFAiBA;AAGD,wBAAsB,2BAA2B,CAAC,EAC9C,SAAS,EACT,GAAG,EACH,MAAM,EACT,EAAE;IACC,SAAS,EAAE,aAAa,CAAC;IACzB,GAAG,EAAE,aAAa,CAAC;IACnB,MAAM,CAAC,EAAE,kBAAkB,CAAA;CAC9B;;;;;;;;;;wFAcA"}

package/dist/src/rag.js

@@ -0,0 +1,72 @@
+import { SupabaseVectorStore, OllamaEmbeddings, RecursiveCharacterTextSplitter, FaissStore, Document, createStuffDocumentsChain, createRetrievalChain } from "./imports";
+const baseEmbeddings = new OllamaEmbeddings({
+    model: "nomic-embed-text"
+});
+const baseSplitter = new RecursiveCharacterTextSplitter({
+    chunkSize: 500,
+    chunkOverlap: 50
+});
+export function turn_to_docs(docs) {
+    return docs.map(doc => new Document({
+        pageContent: typeof doc === "string" ? doc : JSON.stringify(doc, null, 2),
+        metadata: {}
+    }));
+}
+export async function createSupabaseVectoreStore({ supabase, data, table_name = "documents", RPC_function = "match_documents" }) {
+    const docs = turn_to_docs(data);
+    const splitted_docs = await baseSplitter.splitDocuments(docs);
+    return await SupabaseVectorStore.fromDocuments(splitted_docs, baseEmbeddings, {
+        client: supabase,
+        tableName: table_name,
+        queryName: RPC_function
+    });
+}
+// Bestehenden Supabase Store holen (ohne neue Docs)
+export function getSupabaseVectorStore({ supabase, table_name = "documents", RPC_function = "match_documents" }) {
+    return new SupabaseVectorStore(baseEmbeddings, {
+        client: supabase,
+        tableName: table_name,
+        queryName: RPC_function
+    });
+}
+export async function createFaissStore({ data, save_path = "faiss_rag", embeddings = baseEmbeddings }) {
+    const docs = turn_to_docs(data);
+    const splitted_docs = await baseSplitter.splitDocuments(docs);
+    const vectore_store = await FaissStore.fromDocuments(splitted_docs, embeddings);
+    if (save_path) {
+        await vectore_store.save(save_path);
+    }
+    return vectore_store;
+}
+export async function loadFaissStore({ path, embeddings = baseEmbeddings }) {
+    return await FaissStore.load(path, embeddings);
+}
+export async function createRAGChain({ vectorStore, llm, prompt, num_of_results_from_vdb = 4 }) {
+    const model = llm;
+    const retriever = vectorStore.asRetriever({ k: num_of_results_from_vdb });
+    // Document Chain: Kombiniert Docs in den Prompt
+    const documentChain = await createStuffDocumentsChain({
+        llm: model,
+        prompt: prompt
+    });
+    // Retrieval Chain: Verbindet Retriever mit Document Chain
+    const retrievalChain = await createRetrievalChain({
+        retriever: retriever,
+        combineDocsChain: documentChain
+    });
+    return retrievalChain;
+}
+// Erstellt eine Retrieval Chain direkt aus einem Retriever
+export async function createRAGChainFromRetriever({ retriever, llm, prompt }) {
+    const model = llm;
+    const documentChain = await createStuffDocumentsChain({
+        llm: model,
+        prompt: prompt
+    });
+    const retrievalChain = await createRetrievalChain({
+        retriever: retriever,
+        combineDocsChain: documentChain
+    });
+    return retrievalChain;
+}
+//# sourceMappingURL=rag.js.map

package/dist/src/rag.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rag.js","sourceRoot":"","sources":["../../src/rag.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,mBAAmB,EACnB,gBAAgB,EAChB,8BAA8B,EAC9B,UAAU,EACV,QAAQ,EAER,yBAAyB,EACzB,oBAAoB,EAKvB,MAAM,WAAW,CAAA;AAUlB,MAAM,cAAc,GAAG,IAAI,gBAAgB,CAAC;IACxC,KAAK,EAAE,kBAAkB;CAC5B,CAAC,CAAA;AAEF,MAAM,YAAY,GAAG,IAAI,8BAA8B,CAAC;IACpD,SAAS,EAAE,GAAG;IACd,YAAY,EAAE,EAAE;CACnB,CAAC,CAAA;AAEF,MAAM,UAAU,YAAY,CAAI,IAAS;IACrC,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,QAAQ,CAAC;QAChC,WAAW,EAAE,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAC,IAAI,EAAC,CAAC,CAAC;QACvE,QAAQ,EAAE,EAAE;KACf,CAAC,CAAC,CAAA;AACP,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAAC,EAAC,QAAQ,EAAE,IAAI,EAAE,UAAU,GAAG,WAAW,EAAE,YAAY,GAAG,iBAAiB,EAAmD;IAC3K,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;IAC/B,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;IAC7D,OAAO,MAAM,mBAAmB,CAAC,aAAa,CAC1C,aAAa,EACb,cAAc,EACd;QACI,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,UAAU;QACrB,SAAS,EAAE,YAAY;KAC1B,CACJ,CAAA;AAEL,CAAC;AAED,oDAAoD;AACpD,MAAM,UAAU,sBAAsB,CAAC,EAAC,QAAQ,EAAE,UAAU,GAAG,WAAW,EAAE,YAAY,GAAG,iBAAiB,EAAyE;IACjL,OAAO,IAAI,mBAAmB,CAAC,cAAc,EAAE;QAC3C,MAAM,EAAE,QAAQ;QAChB,SAAS,EAAE,UAAU;QACrB,SAAS,EAAE,YAAY;KAC1B,CAAC,CAAA;AACN,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAC,IAAI,EAAE,SAAS,GAAG,WAAW,EAAE,UAAU,GAAG,cAAc,EAAgE;IAC9J,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;IAC/B,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;IAC7D,MAAM,aAAa,GAAG,MAAM,UAAU,CAAC,aAAa,CAChD,aAAa,EACb,UAAU,CACb,CAAA;IACD,IAAI,SAAS,EAAE,CAAC;QACZ,MAAM,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IACvC,CAAC;IACD,OAAO,aAAa,CAAA;AACxB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,EAAC,IAAI,EAAE,UAAU,GAAG,cAAc,EAA0C;IAC7G,OAAO,MAAM,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;AAClD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,EACjC,WAAW,EACX,GAAG,EACH,MAAM,EACN,uBAAuB,GAAG,CAAC,EAM9B;IACG,MAAM,KAAK,GAAG,GAAG,CAAA;IACjB,MAAM,SAAS,GAAG,WAAW,CAAC,WAAW,CAAC,EAAE,CAAC,EAAE,uBAAuB,EAAE,CAAC,CAAA;IAEzE,gDAAgD;IAChD,MAAM,aAAa,GAAG,MAAM,yBAAyB,CAAC;QAClD,GAAG,EAAE,KAAY;QACjB,MAAM,EAAE,MAAa;KACxB,CAAC,CAAA;IAEF,0DAA0D;IAC1D,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC;QAC9C,SAAS,EAAE,SAAgB;QAC3B,gBAAgB,EAAE,aAAa;KAClC,CAAC,CAAA;IAEF,OAAO,cAAc,CAAA;AACzB,CAAC;AAED,2DAA2D;AAC3D,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,EAC9C,SAAS,EACT,GAAG,EACH,MAAM,EAKT;IACG,MAAM,KAAK,GAAG,GAAG,CAAA;IAEjB,MAAM,aAAa,GAAG,MAAM,yBAAyB,CAAC;QAClD,GAAG,EAAE,KAAY;QACjB,MAAM,EAAE,MAAa;KACxB,CAAC,CAAA;IAEF,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC;QAC9C,SAAS,EAAE,SAAgB;QAC3B,gBAAgB,EAAE,aAAa;KAClC,CAAC,CAAA;IAEF,OAAO,cAAc,CAAA;AACzB,CAAC"}

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@delofarag/ai-utils",
+  "version": "1.0.0",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx",
+    "start": "tsx"
+  },
+  "dependencies": {
+    "@langchain/anthropic": "^1.3.3",
+    "@langchain/classic": "^1.0.7",
+    "@langchain/community": "^1.1.1",
+    "@langchain/core": "^1.1.8",
+    "@langchain/groq": "^1.0.2",
+    "@langchain/langgraph": "^1.0.7",
+    "@langchain/langgraph-checkpoint-postgres": "^1.0.0",
+    "@langchain/langgraph-checkpoint-sqlite": "^1.0.0",
+    "@langchain/ollama": "^1.1.0",
+    "@langchain/openai": "^1.2.0",
+    "@langchain/textsplitters": "^1.0.1",
+    "@delofarag/base-utils": "^1.0.0",
+    "readline": "^1.3.0",
+    "zod": "^3.23.0",
+    "zod-validation-error": "^2.1.0",
+    "zodios": "^5.1.0"
+  },
+  "peerDependencies": {
+    "@delofarag/supabase-utils": "^1.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "tsx": "^4.0.0",
+    "typescript": "^5.0.0"
+  }
+}