@intlayer/backend 7.2.1-canary.1 → 7.2.3

package/dist/cjs/utils/AI/askDocQuestion/askDocQuestion.cjs

@@ -1,201 +0,0 @@
-const require_rolldown_runtime = require('../../../_virtual/rolldown_runtime.cjs');
-const require__utils_asset = require('../../../_virtual/_utils_asset.cjs');
-let __intlayer_core = require("@intlayer/core");
-let __intlayer_docs = require("@intlayer/docs");
-let ai = require("ai");
-let openai = require("openai");
-
-//#region src/utils/AI/askDocQuestion/askDocQuestion.ts
-const readEmbeddingsForFile = (fileKey) => {
-	try {
-		return JSON.parse(require__utils_asset.readAsset(`./embeddings/${fileKey.replace(".md", ".json")}`, "utf-8"));
-	} catch {
-		return {};
-	}
-};
-/**
-* Simple in-memory vector store to hold document embeddings and their content.
-* Each entry contains:
-* - fileKey: A unique key identifying the file
-* - chunkNumber: The number of the chunk within the document
-* - content: The chunk content
-* - embedding: The numerical embedding vector for the chunk
-*/
-const vectorStore = [];
-const MAX_RELEVANT_CHUNKS_NB = 20;
-const MIN_RELEVANT_CHUNKS_SIMILARITY = .42;
-const EMBEDDING_MODEL = "text-embedding-3-large";
-const OVERLAP_TOKENS = 200;
-const MAX_CHUNK_TOKENS = 800;
-const CHAR_BY_TOKEN = 4.15;
-const MAX_CHARS = MAX_CHUNK_TOKENS * CHAR_BY_TOKEN;
-const OVERLAP_CHARS = OVERLAP_TOKENS * CHAR_BY_TOKEN;
-const skipDocEmbeddingsIndex = process.env.SKIP_DOC_EMBEDDINGS_INDEX === "true";
-/**
-* Splits a given text into chunks ensuring each chunk does not exceed MAX_CHARS.
-* @param text - The input text to split.
-* @returns - Array of text chunks.
-*/
-const chunkText = (text) => {
-	const chunks = [];
-	let start = 0;
-	while (start < text.length) {
-		let end = Math.min(start + MAX_CHARS, text.length);
-		if (end < text.length) {
-			const lastSpace = text.lastIndexOf(" ", end);
-			if (lastSpace > start) end = lastSpace;
-		}
-		chunks.push(text.substring(start, end));
-		const nextStart = end - OVERLAP_CHARS;
-		if (nextStart <= start) start = end;
-		else start = nextStart;
-	}
-	return chunks;
-};
-/**
-* Generates an embedding for a given text using OpenAI's embedding API.
-* Trims the text if it exceeds the maximum allowed characters.
-*
-* @param text - The input text to generate an embedding for
-* @returns The embedding vector as a number array
-*/
-const generateEmbedding = async (text) => {
-	try {
-		return (await new openai.OpenAI({ apiKey: process.env.OPENAI_API_KEY }).embeddings.create({
-			model: EMBEDDING_MODEL,
-			input: text
-		})).data[0].embedding;
-	} catch (error) {
-		console.error("Error generating embedding:", error);
-		return [];
-	}
-};
-/**
-* Calculates the cosine similarity between two vectors.
-* Cosine similarity measures the cosine of the angle between two vectors in an inner product space.
-* Used to determine the similarity between chunks of text.
-*
-* @param vecA - The first vector
-* @param vecB - The second vector
-* @returns The cosine similarity score
-*/
-const cosineSimilarity = (vecA, vecB) => {
-	return vecA.reduce((sum, a, idx) => sum + a * vecB[idx], 0) / (Math.sqrt(vecA.reduce((sum, a) => sum + a * a, 0)) * Math.sqrt(vecB.reduce((sum, b) => sum + b * b, 0)));
-};
-/**
-* Indexes all Markdown documents by generating embeddings for each chunk and storing them in memory.
-* Persists per-document embeddings under `embeddings/<fileKey>.json`.
-* Handles cases where files have been updated and chunk counts have changed.
-*/
-const loadMarkdownFiles = async () => {
-	const frequentQuestions = await (0, __intlayer_docs.getFrequentQuestions)();
-	const docs = await (0, __intlayer_docs.getDocs)();
-	const blogs = await (0, __intlayer_docs.getBlogs)();
-	const files = {
-		...docs,
-		...blogs,
-		...frequentQuestions
-	};
-	for await (const fileKey of Object.keys(files)) {
-		const fileMetadata = (0, __intlayer_core.getMarkdownMetadata)(files[fileKey]);
-		const fileChunks = chunkText(files[fileKey]);
-		const existingEmbeddings = readEmbeddingsForFile(fileKey);
-		const existingChunksForFile = Object.keys(existingEmbeddings);
-		const currentChunkCount = fileChunks.length;
-		const previousChunkCount = existingChunksForFile.length;
-		let shouldRegenerateFileEmbeddings = false;
-		if (currentChunkCount !== previousChunkCount) {
-			console.info(`File "${fileKey}" chunk count changed: ${previousChunkCount} -> ${currentChunkCount}. Regenerating embeddings.`);
-			shouldRegenerateFileEmbeddings = !skipDocEmbeddingsIndex;
-		}
-		let resultForFile = {};
-		for await (const chunkIndex of Object.keys(fileChunks)) {
-			const chunkNumber = Number(chunkIndex) + 1;
-			const chunksNumber = fileChunks.length;
-			const fileChunk = fileChunks[chunkIndex];
-			const chunkKeyName = `chunk_${chunkNumber}`;
-			const embedding = !shouldRegenerateFileEmbeddings ? existingEmbeddings[chunkKeyName] : void 0;
-			resultForFile = {
-				...resultForFile,
-				[chunkKeyName]: embedding
-			};
-			vectorStore.push({
-				fileKey,
-				chunkNumber,
-				embedding,
-				content: fileChunk,
-				docUrl: fileMetadata.url,
-				docName: fileMetadata.title
-			});
-			console.info(`- Loaded: ${fileKey}/${chunkKeyName}/${chunksNumber}`);
-		}
-	}
-};
-loadMarkdownFiles();
-/**
-* Searches the indexed documents for the most relevant chunks based on a query.
-* Utilizes cosine similarity to find the closest matching embeddings.
-*
-* @param query - The search query provided by the user
-* @returns An array of the top matching document chunks' content
-*/
-const searchChunkReference = async (query, maxResults = MAX_RELEVANT_CHUNKS_NB, minSimilarity = MIN_RELEVANT_CHUNKS_SIMILARITY) => {
-	const queryEmbedding = await generateEmbedding(query);
-	const selection = vectorStore.filter((chunk) => chunk.embedding).map((chunk) => ({
-		...chunk,
-		similarity: cosineSimilarity(queryEmbedding, chunk.embedding)
-	})).filter((chunk) => chunk.similarity > minSimilarity).sort((a, b) => b.similarity - a.similarity).slice(0, maxResults);
-	const orderedDocKeys = new Set(selection.map((chunk) => chunk.fileKey));
-	return vectorStore.sort((a, _b) => orderedDocKeys.has(a.fileKey) ? -1 : 1).filter((chunk) => selection.some((v) => v.fileKey === chunk.fileKey && v.chunkNumber === chunk.chunkNumber));
-};
-const CHAT_GPT_PROMPT = require__utils_asset.readAsset("./PROMPT.md");
-const initPrompt = {
-	role: "system",
-	content: CHAT_GPT_PROMPT
-};
-/**
-* Handles the "Ask a question" endpoint in an Express.js route.
-* Processes user messages, retrieves relevant documents, and interacts with AI models to generate responses.
-*
-* @param messages - An array of chat messages from the user and assistant
-* @returns The assistant's response as a string
-*/
-const askDocQuestion = async (messages, aiConfig, options) => {
-	const relevantFilesReferences = await searchChunkReference(messages.filter((message) => message.role === "user").map((message) => `- ${message.content}`).join("\n"));
-	const aiMessages = [{
-		role: "system",
-		content: initPrompt.content.replace("{{relevantFilesReferences}}", relevantFilesReferences.length === 0 ? "Not relevant file found related to the question." : relevantFilesReferences.map((doc, idx) => [
-			"-----",
-			"---",
-			`chunkId: ${idx}`,
-			`docChunk: "${doc.chunkNumber}/${doc.fileKey.length}"`,
-			`docName: "${doc.docName}"`,
-			`docUrl: "${doc.docUrl}"`,
-			`---`,
-			doc.content,
-			`-----`
-		].join("\n")).join("\n\n"))
-	}, ...messages.slice(-8)];
-	if (!aiConfig) throw new Error("Failed to initialize AI configuration");
-	let fullResponse = "";
-	const stream = (0, ai.streamText)({
-		...aiConfig,
-		messages: aiMessages
-	});
-	for await (const chunk of stream.textStream) {
-		fullResponse += chunk;
-		options?.onMessage?.(chunk);
-	}
-	const relatedFiles = [...new Set(relevantFilesReferences.map((doc) => doc.fileKey))];
-	return {
-		response: fullResponse ?? "Error: No result found",
-		relatedFiles
-	};
-};
-
-//#endregion
-exports.askDocQuestion = askDocQuestion;
-exports.initPrompt = initPrompt;
-exports.loadMarkdownFiles = loadMarkdownFiles;
-exports.searchChunkReference = searchChunkReference;
-//# sourceMappingURL=askDocQuestion.cjs.map

package/dist/cjs/utils/AI/askDocQuestion/askDocQuestion.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"askDocQuestion.cjs","names":["readAsset","vectorStore: VectorStoreEl[]","MAX_RELEVANT_CHUNKS_NB: number","MIN_RELEVANT_CHUNKS_SIMILARITY: number","EMBEDDING_MODEL: OpenAI.EmbeddingModel","OVERLAP_TOKENS: number","MAX_CHUNK_TOKENS: number","CHAR_BY_TOKEN: number","MAX_CHARS: number","OVERLAP_CHARS: number","chunks: string[]","OpenAI","resultForFile: Record<string, number[] | undefined>","initPrompt: ChatCompletionRequestMessage"],"sources":["../../../../../src/utils/AI/askDocQuestion/askDocQuestion.ts"],"sourcesContent":["import { readAsset } from 'utils:asset';\nimport { getMarkdownMetadata } from '@intlayer/core';\nimport { getBlogs, getDocs, getFrequentQuestions } from '@intlayer/docs';\nimport { streamText } from 'ai';\nimport { OpenAI } from 'openai';\nimport type { AIConfig, ChatCompletionRequestMessage } from '../aiSdk';\n\nconst readEmbeddingsForFile = (fileKey: string): Record<string, number[]> => {\n  try {\n    return JSON.parse(\n      readAsset(`./embeddings/${fileKey.replace('.md', '.json')}`, 'utf-8')\n    ) as Record<string, number[]>;\n  } catch {\n    return {};\n  }\n};\n\ntype VectorStoreEl = {\n  fileKey: string;\n  chunkNumber: number;\n  content: string;\n  embedding?: number[];\n  docUrl: string;\n  docName: string;\n};\n\n/**\n * Simple in-memory vector store to hold document embeddings and their content.\n * Each entry contains:\n * - fileKey: A unique key identifying the file\n * - chunkNumber: The number of the chunk within the document\n * - content: The chunk content\n * - embedding: The numerical embedding vector for the chunk\n */\nconst vectorStore: VectorStoreEl[] = [];\n\n/*\n * Ask question AI configuration\n */\nconst MAX_RELEVANT_CHUNKS_NB: number = 20; // Maximum number of relevant chunks to attach to chatGPT context\nconst MIN_RELEVANT_CHUNKS_SIMILARITY: number = 0.42; // Minimum similarity required for a chunk to be considered relevant\n\n/*\n * Embedding model configuration\n */\nconst EMBEDDING_MODEL: OpenAI.EmbeddingModel = 'text-embedding-3-large'; // Model to use for embedding generation\nconst OVERLAP_TOKENS: number = 200; // Number of tokens to overlap between chunks\nconst MAX_CHUNK_TOKENS: number = 800; // Maximum number of tokens per chunk\nconst CHAR_BY_TOKEN: number = 4.15; // Approximate pessimistically the number of characters per token // Can use `tiktoken` or other tokenizers to calculate it more precisely\nconst MAX_CHARS: number = MAX_CHUNK_TOKENS * CHAR_BY_TOKEN;\nconst OVERLAP_CHARS: number = OVERLAP_TOKENS * CHAR_BY_TOKEN;\n\nconst skipDocEmbeddingsIndex = process.env.SKIP_DOC_EMBEDDINGS_INDEX === 'true';\n\n/**\n * Splits a given text into chunks ensuring each chunk does not exceed MAX_CHARS.\n * @param text - The input text to split.\n * @returns - Array of text chunks.\n */\nconst chunkText = (text: string): string[] => {\n  const chunks: string[] = [];\n  let start = 0;\n\n  while (start < text.length) {\n    let end = Math.min(start + MAX_CHARS, text.length);\n\n    // Ensure we don't cut words in the middle (find nearest space)\n    if (end < text.length) {\n      const lastSpace = text.lastIndexOf(' ', end);\n      if (lastSpace > start) {\n        end = lastSpace;\n      }\n    }\n\n    chunks.push(text.substring(start, end));\n\n    // Move start forward correctly\n    const nextStart = end - OVERLAP_CHARS;\n    if (nextStart <= start) {\n      // Prevent infinite loop if overlap is too large\n      start = end;\n    } else {\n      start = nextStart;\n    }\n  }\n\n  return chunks;\n};\n\n/**\n * Generates an embedding for a given text using OpenAI's embedding API.\n * Trims the text if it exceeds the maximum allowed characters.\n *\n * @param text - The input text to generate an embedding for\n * @returns The embedding vector as a number array\n */\nconst generateEmbedding = async (text: string): Promise<number[]> => {\n  try {\n    const openaiClient = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });\n\n    const response = await openaiClient.embeddings.create({\n      model: EMBEDDING_MODEL,\n      input: text,\n    });\n\n    return response.data[0].embedding;\n  } catch (error) {\n    console.error('Error generating embedding:', error);\n    return [];\n  }\n};\n\n/**\n * Calculates the cosine similarity between two vectors.\n * Cosine similarity measures the cosine of the angle between two vectors in an inner product space.\n * Used to determine the similarity between chunks of text.\n *\n * @param vecA - The first vector\n * @param vecB - The second vector\n * @returns The cosine similarity score\n */\nconst cosineSimilarity = (vecA: number[], vecB: number[]): number => {\n  // Calculate the dot product of the two vectors\n  const dotProduct = vecA.reduce((sum, a, idx) => sum + a * vecB[idx], 0);\n\n  // Calculate the magnitude (Euclidean norm) of each vector\n  const magnitudeA = Math.sqrt(vecA.reduce((sum, a) => sum + a * a, 0));\n  const magnitudeB = Math.sqrt(vecB.reduce((sum, b) => sum + b * b, 0));\n\n  // Compute and return the cosine similarity\n  return dotProduct / (magnitudeA * magnitudeB);\n};\n\n/**\n * Indexes all Markdown documents by generating embeddings for each chunk and storing them in memory.\n * Persists per-document embeddings under `embeddings/<fileKey>.json`.\n * Handles cases where files have been updated and chunk counts have changed.\n */\nexport const loadMarkdownFiles = async (): Promise<void> => {\n  // Retrieve documentation and blog posts in English locale\n  const frequentQuestions = await getFrequentQuestions();\n  const docs = await getDocs();\n  const blogs = await getBlogs();\n\n  const files = { ...docs, ...blogs, ...frequentQuestions }; // Combine docs and blogs into a single object\n\n  // Iterate over each file key (identifier) in the combined files\n  for await (const fileKey of Object.keys(files)) {\n    // Get the metadata of the file\n    const fileMetadata = getMarkdownMetadata(\n      files[fileKey as keyof typeof files] as string\n    );\n\n    // Split the document into chunks based on headings\n    const fileChunks = chunkText(\n      files[fileKey as keyof typeof files] as string\n    );\n\n    // Read existing embeddings for this file\n    const existingEmbeddings = readEmbeddingsForFile(fileKey);\n\n    // Check if the number of chunks has changed for this file\n    const existingChunksForFile = Object.keys(existingEmbeddings);\n    const currentChunkCount = fileChunks.length;\n    const previousChunkCount = existingChunksForFile.length;\n\n    let shouldRegenerateFileEmbeddings = false;\n\n    // If chunk count differs, we need to regenerate embeddings for this file\n    if (currentChunkCount !== previousChunkCount) {\n      console.info(\n        `File \"${fileKey}\" chunk count changed: ${previousChunkCount} -> ${currentChunkCount}. Regenerating embeddings.`\n      );\n\n      shouldRegenerateFileEmbeddings = !skipDocEmbeddingsIndex;\n    }\n\n    // Iterate over each chunk within the current file\n    let resultForFile: Record<string, number[] | undefined> = {};\n    for await (const chunkIndex of Object.keys(fileChunks)) {\n      const chunkNumber = Number(chunkIndex) + 1; // Chunk number starts at 1\n      const chunksNumber = fileChunks.length;\n\n      const fileChunk = fileChunks[\n        chunkIndex as keyof typeof fileChunks\n      ] as string;\n\n      const chunkKeyName = `chunk_${chunkNumber}`; // Unique key for the chunk within the file\n\n      // Retrieve precomputed embedding if available and file hasn't changed\n      const docEmbedding = !shouldRegenerateFileEmbeddings\n        ? (existingEmbeddings[\n            chunkKeyName as keyof typeof existingEmbeddings\n          ] as number[] | undefined)\n        : undefined;\n\n      const embedding = docEmbedding; // Use existing embedding if available and valid\n\n      // Update the file-scoped result object with the embedding\n      resultForFile = { ...resultForFile, [chunkKeyName]: embedding };\n\n      // Store the embedding and content in the in-memory vector store\n      vectorStore.push({\n        fileKey,\n        chunkNumber,\n        embedding,\n        content: fileChunk,\n        docUrl: fileMetadata.url,\n        docName: fileMetadata.title,\n      });\n\n      console.info(`- Loaded: ${fileKey}/${chunkKeyName}/${chunksNumber}`);\n    }\n  }\n};\n\n// Automatically index Markdown files\nloadMarkdownFiles();\n\n/**\n * Searches the indexed documents for the most relevant chunks based on a query.\n * Utilizes cosine similarity to find the closest matching embeddings.\n *\n * @param query - The search query provided by the user\n * @returns An array of the top matching document chunks' content\n */\nexport const searchChunkReference = async (\n  query: string,\n  maxResults: number = MAX_RELEVANT_CHUNKS_NB,\n  minSimilarity: number = MIN_RELEVANT_CHUNKS_SIMILARITY\n): Promise<VectorStoreEl[]> => {\n  // Generate an embedding for the user's query\n  const queryEmbedding = await generateEmbedding(query);\n\n  // Calculate similarity scores between the query embedding and each document's embedding\n  const selection = vectorStore\n    .filter((chunk) => chunk.embedding)\n    .map((chunk) => ({\n      ...chunk,\n      similarity: cosineSimilarity(queryEmbedding, chunk.embedding!), // Add similarity score to each doc\n    }))\n    .filter((chunk) => chunk.similarity > minSimilarity) // Filter out documents with low similarity scores\n    .sort((a, b) => b.similarity - a.similarity) // Sort documents by highest similarity first\n    .slice(0, maxResults); // Select the top 6 most similar documents\n\n  const orderedDocKeys = new Set(selection.map((chunk) => chunk.fileKey));\n\n  const orderedVectorStore = vectorStore.sort((a, _b) =>\n    orderedDocKeys.has(a.fileKey) ? -1 : 1\n  );\n\n  const results = orderedVectorStore.filter((chunk) =>\n    selection.some(\n      (v) => v.fileKey === chunk.fileKey && v.chunkNumber === chunk.chunkNumber\n    )\n  );\n\n  // Return the content of the top matching documents\n  return results;\n};\n\nconst CHAT_GPT_PROMPT = readAsset('./PROMPT.md');\n\n// Initial prompt configuration for the chatbot\nexport const initPrompt: ChatCompletionRequestMessage = {\n  role: 'system',\n  content: CHAT_GPT_PROMPT,\n};\n\nexport type AskDocQuestionResult = {\n  response: string;\n  relatedFiles: string[];\n};\n\nexport type AskDocQuestionOptions = {\n  onMessage?: (chunk: string) => void;\n};\n\n/**\n * Handles the \"Ask a question\" endpoint in an Express.js route.\n * Processes user messages, retrieves relevant documents, and interacts with AI models to generate responses.\n *\n * @param messages - An array of chat messages from the user and assistant\n * @returns The assistant's response as a string\n */\nexport const askDocQuestion = async (\n  messages: ChatCompletionRequestMessage[],\n  aiConfig: AIConfig,\n  options?: AskDocQuestionOptions\n): Promise<AskDocQuestionResult> => {\n  // Format the user's question to keep only the relevant keywords\n  const query = messages\n    .filter((message) => message.role === 'user')\n    .map((message) => `- ${message.content}`)\n    .join('\\n');\n\n  // 1) Find relevant documents based on the user's question\n  const relevantFilesReferences = await searchChunkReference(query);\n\n  // 2) Integrate the relevant documents into the initial system prompt\n  const systemPrompt = initPrompt.content.replace(\n    '{{relevantFilesReferences}}',\n    relevantFilesReferences.length === 0\n      ? 'Not relevant file found related to the question.'\n      : relevantFilesReferences\n          .map((doc, idx) =>\n            [\n              '-----',\n              '---',\n              `chunkId: ${idx}`,\n              `docChunk: \"${doc.chunkNumber}/${doc.fileKey.length}\"`,\n              `docName: \"${doc.docName}\"`,\n              `docUrl: \"${doc.docUrl}\"`,\n              `---`,\n              doc.content,\n              `-----`,\n            ].join('\\n')\n          )\n          .join('\\n\\n') // Insert relevant docs into the prompt\n  );\n\n  // Format messages for AI SDK\n  const aiMessages = [\n    {\n      role: 'system' as const,\n      content: systemPrompt,\n    },\n    ...messages.slice(-8),\n  ];\n\n  if (!aiConfig) {\n    throw new Error('Failed to initialize AI configuration');\n  }\n\n  // 3) Use the AI SDK to stream the response\n  let fullResponse = '';\n  const stream = streamText({\n    ...aiConfig,\n    messages: aiMessages,\n  });\n\n  // Process the stream\n  for await (const chunk of stream.textStream) {\n    fullResponse += chunk;\n    options?.onMessage?.(chunk);\n  }\n\n  // 4) Extract unique related files\n  const relatedFiles = [\n    ...new Set(relevantFilesReferences.map((doc) => doc.fileKey)),\n  ];\n\n  // 5) Return the assistant's response to the user\n  return {\n    response: fullResponse ?? 'Error: No result found',\n    relatedFiles,\n  };\n};\n"],"mappings":";;;;;;;;AAOA,MAAM,yBAAyB,YAA8C;AAC3E,KAAI;AACF,SAAO,KAAK,MACVA,+BAAU,gBAAgB,QAAQ,QAAQ,OAAO,QAAQ,IAAI,QAAQ,CACtE;SACK;AACN,SAAO,EAAE;;;;;;;;;;;AAqBb,MAAMC,cAA+B,EAAE;AAKvC,MAAMC,yBAAiC;AACvC,MAAMC,iCAAyC;AAK/C,MAAMC,kBAAyC;AAC/C,MAAMC,iBAAyB;AAC/B,MAAMC,mBAA2B;AACjC,MAAMC,gBAAwB;AAC9B,MAAMC,YAAoB,mBAAmB;AAC7C,MAAMC,gBAAwB,iBAAiB;AAE/C,MAAM,yBAAyB,QAAQ,IAAI,8BAA8B;;;;;;AAOzE,MAAM,aAAa,SAA2B;CAC5C,MAAMC,SAAmB,EAAE;CAC3B,IAAI,QAAQ;AAEZ,QAAO,QAAQ,KAAK,QAAQ;EAC1B,IAAI,MAAM,KAAK,IAAI,QAAQ,WAAW,KAAK,OAAO;AAGlD,MAAI,MAAM,KAAK,QAAQ;GACrB,MAAM,YAAY,KAAK,YAAY,KAAK,IAAI;AAC5C,OAAI,YAAY,MACd,OAAM;;AAIV,SAAO,KAAK,KAAK,UAAU,OAAO,IAAI,CAAC;EAGvC,MAAM,YAAY,MAAM;AACxB,MAAI,aAAa,MAEf,SAAQ;MAER,SAAQ;;AAIZ,QAAO;;;;;;;;;AAUT,MAAM,oBAAoB,OAAO,SAAoC;AACnE,KAAI;AAQF,UALiB,MAFI,IAAIC,cAAO,EAAE,QAAQ,QAAQ,IAAI,gBAAgB,CAAC,CAEnC,WAAW,OAAO;GACpD,OAAO;GACP,OAAO;GACR,CAAC,EAEc,KAAK,GAAG;UACjB,OAAO;AACd,UAAQ,MAAM,+BAA+B,MAAM;AACnD,SAAO,EAAE;;;;;;;;;;;;AAab,MAAM,oBAAoB,MAAgB,SAA2B;AASnE,QAPmB,KAAK,QAAQ,KAAK,GAAG,QAAQ,MAAM,IAAI,KAAK,MAAM,EAAE,IAGpD,KAAK,KAAK,KAAK,QAAQ,KAAK,MAAM,MAAM,IAAI,GAAG,EAAE,CAAC,GAClD,KAAK,KAAK,KAAK,QAAQ,KAAK,MAAM,MAAM,IAAI,GAAG,EAAE,CAAC;;;;;;;AAWvE,MAAa,oBAAoB,YAA2B;CAE1D,MAAM,oBAAoB,iDAA4B;CACtD,MAAM,OAAO,oCAAe;CAC5B,MAAM,QAAQ,qCAAgB;CAE9B,MAAM,QAAQ;EAAE,GAAG;EAAM,GAAG;EAAO,GAAG;EAAmB;AAGzD,YAAW,MAAM,WAAW,OAAO,KAAK,MAAM,EAAE;EAE9C,MAAM,wDACJ,MAAM,SACP;EAGD,MAAM,aAAa,UACjB,MAAM,SACP;EAGD,MAAM,qBAAqB,sBAAsB,QAAQ;EAGzD,MAAM,wBAAwB,OAAO,KAAK,mBAAmB;EAC7D,MAAM,oBAAoB,WAAW;EACrC,MAAM,qBAAqB,sBAAsB;EAEjD,IAAI,iCAAiC;AAGrC,MAAI,sBAAsB,oBAAoB;AAC5C,WAAQ,KACN,SAAS,QAAQ,yBAAyB,mBAAmB,MAAM,kBAAkB,4BACtF;AAED,oCAAiC,CAAC;;EAIpC,IAAIC,gBAAsD,EAAE;AAC5D,aAAW,MAAM,cAAc,OAAO,KAAK,WAAW,EAAE;GACtD,MAAM,cAAc,OAAO,WAAW,GAAG;GACzC,MAAM,eAAe,WAAW;GAEhC,MAAM,YAAY,WAChB;GAGF,MAAM,eAAe,SAAS;GAS9B,MAAM,YANe,CAAC,iCACjB,mBACC,gBAEF;AAKJ,mBAAgB;IAAE,GAAG;KAAgB,eAAe;IAAW;AAG/D,eAAY,KAAK;IACf;IACA;IACA;IACA,SAAS;IACT,QAAQ,aAAa;IACrB,SAAS,aAAa;IACvB,CAAC;AAEF,WAAQ,KAAK,aAAa,QAAQ,GAAG,aAAa,GAAG,eAAe;;;;AAM1E,mBAAmB;;;;;;;;AASnB,MAAa,uBAAuB,OAClC,OACA,aAAqB,wBACrB,gBAAwB,mCACK;CAE7B,MAAM,iBAAiB,MAAM,kBAAkB,MAAM;CAGrD,MAAM,YAAY,YACf,QAAQ,UAAU,MAAM,UAAU,CAClC,KAAK,WAAW;EACf,GAAG;EACH,YAAY,iBAAiB,gBAAgB,MAAM,UAAW;EAC/D,EAAE,CACF,QAAQ,UAAU,MAAM,aAAa,cAAc,CACnD,MAAM,GAAG,MAAM,EAAE,aAAa,EAAE,WAAW,CAC3C,MAAM,GAAG,WAAW;CAEvB,MAAM,iBAAiB,IAAI,IAAI,UAAU,KAAK,UAAU,MAAM,QAAQ,CAAC;AAavE,QAX2B,YAAY,MAAM,GAAG,OAC9C,eAAe,IAAI,EAAE,QAAQ,GAAG,KAAK,EACtC,CAEkC,QAAQ,UACzC,UAAU,MACP,MAAM,EAAE,YAAY,MAAM,WAAW,EAAE,gBAAgB,MAAM,YAC/D,CACF;;AAMH,MAAM,kBAAkBZ,+BAAU,cAAc;AAGhD,MAAaa,aAA2C;CACtD,MAAM;CACN,SAAS;CACV;;;;;;;;AAkBD,MAAa,iBAAiB,OAC5B,UACA,UACA,YACkC;CAQlC,MAAM,0BAA0B,MAAM,qBANxB,SACX,QAAQ,YAAY,QAAQ,SAAS,OAAO,CAC5C,KAAK,YAAY,KAAK,QAAQ,UAAU,CACxC,KAAK,KAAK,CAGoD;CAyBjE,MAAM,aAAa,CACjB;EACE,MAAM;EACN,SAzBiB,WAAW,QAAQ,QACtC,+BACA,wBAAwB,WAAW,IAC/B,qDACA,wBACG,KAAK,KAAK,QACT;GACE;GACA;GACA,YAAY;GACZ,cAAc,IAAI,YAAY,GAAG,IAAI,QAAQ,OAAO;GACpD,aAAa,IAAI,QAAQ;GACzB,YAAY,IAAI,OAAO;GACvB;GACA,IAAI;GACJ;GACD,CAAC,KAAK,KAAK,CACb,CACA,KAAK,OAAO,CACpB;EAOE,EACD,GAAG,SAAS,MAAM,GAAG,CACtB;AAED,KAAI,CAAC,SACH,OAAM,IAAI,MAAM,wCAAwC;CAI1D,IAAI,eAAe;CACnB,MAAM,4BAAoB;EACxB,GAAG;EACH,UAAU;EACX,CAAC;AAGF,YAAW,MAAM,SAAS,OAAO,YAAY;AAC3C,kBAAgB;AAChB,WAAS,YAAY,MAAM;;CAI7B,MAAM,eAAe,CACnB,GAAG,IAAI,IAAI,wBAAwB,KAAK,QAAQ,IAAI,QAAQ,CAAC,CAC9D;AAGD,QAAO;EACL,UAAU,gBAAgB;EAC1B;EACD"}

package/dist/cjs/utils/AI/askDocQuestion/indexMarkdownFiles.cjs

@@ -1,150 +0,0 @@
-const require_rolldown_runtime = require('../../../_virtual/rolldown_runtime.cjs');
-let node_fs = require("node:fs");
-let node_path = require("node:path");
-let node_url = require("node:url");
-let __intlayer_core = require("@intlayer/core");
-let __intlayer_docs = require("@intlayer/docs");
-let openai = require("openai");
-let dotenv = require("dotenv");
-dotenv = require_rolldown_runtime.__toESM(dotenv);
-
-//#region src/utils/AI/askDocQuestion/indexMarkdownFiles.ts
-const OUTPUT_EMBEDDINGS_DIR = "src/utils/AI/askDocQuestion/embeddings";
-const __dirname$1 = (0, node_path.dirname)((0, node_url.fileURLToPath)(require("url").pathToFileURL(__filename).href));
-const readEmbeddingsForFile = (fileKey) => {
-	try {
-		return JSON.parse((0, node_fs.readFileSync)(`${__dirname$1}/embeddings/${fileKey.replace(".md", ".json")}`, "utf-8"));
-	} catch {
-		return {};
-	}
-};
-const writeEmbeddingsForFile = (fileKey, data) => {
-	const filePath = (0, node_path.join)(OUTPUT_EMBEDDINGS_DIR, `${fileKey.replace(".md", ".json")}`);
-	const dir = (0, node_path.dirname)(filePath);
-	if (!(0, node_fs.existsSync)(dir)) (0, node_fs.mkdirSync)(dir, { recursive: true });
-	(0, node_fs.writeFileSync)(filePath, JSON.stringify(data));
-};
-/**
-* Simple in-memory vector store to hold document embeddings and their content.
-* Each entry contains:
-* - fileKey: A unique key identifying the file
-* - chunkNumber: The number of the chunk within the document
-* - content: The chunk content
-* - embedding: The numerical embedding vector for the chunk
-*/
-const vectorStore = [];
-const EMBEDDING_MODEL = "text-embedding-3-large";
-const OVERLAP_TOKENS = 200;
-const MAX_CHUNK_TOKENS = 800;
-const CHAR_BY_TOKEN = 4.15;
-const MAX_CHARS = MAX_CHUNK_TOKENS * CHAR_BY_TOKEN;
-const OVERLAP_CHARS = OVERLAP_TOKENS * CHAR_BY_TOKEN;
-const skipDocEmbeddingsIndex = process.env.SKIP_DOC_EMBEDDINGS_INDEX === "true";
-/**
-* Splits a given text into chunks ensuring each chunk does not exceed MAX_CHARS.
-* @param text - The input text to split.
-* @returns - Array of text chunks.
-*/
-const chunkText = (text) => {
-	const chunks = [];
-	let start = 0;
-	while (start < text.length) {
-		let end = Math.min(start + MAX_CHARS, text.length);
-		if (end < text.length) {
-			const lastSpace = text.lastIndexOf(" ", end);
-			if (lastSpace > start) end = lastSpace;
-		}
-		chunks.push(text.substring(start, end));
-		const nextStart = end - OVERLAP_CHARS;
-		if (nextStart <= start) start = end;
-		else start = nextStart;
-	}
-	return chunks;
-};
-/**
-* Generates an embedding for a given text using OpenAI's embedding API.
-* Trims the text if it exceeds the maximum allowed characters.
-*
-* @param text - The input text to generate an embedding for
-* @returns The embedding vector as a number array
-*/
-const generateEmbedding = async (text) => {
-	try {
-		return (await new openai.OpenAI({ apiKey: process.env.OPENAI_API_KEY }).embeddings.create({
-			model: EMBEDDING_MODEL,
-			input: text
-		})).data[0].embedding;
-	} catch (error) {
-		console.error("Error generating embedding:", error);
-		return [];
-	}
-};
-/**
-* Indexes all Markdown documents by generating embeddings for each chunk and storing them in memory.
-* Persists per-document embeddings under `embeddings/<fileKey>.json`.
-* Handles cases where files have been updated and chunk counts have changed.
-*/
-const indexMarkdownFiles = async () => {
-	const env = process.env.NODE_ENV;
-	dotenv.default.config({ path: [
-		`.env.${env}.local`,
-		`.env.${env}`,
-		".env.local",
-		".env"
-	] });
-	const frequentQuestions = await (0, __intlayer_docs.getFrequentQuestions)();
-	const docs = await (0, __intlayer_docs.getDocs)();
-	const blogs = await (0, __intlayer_docs.getBlogs)();
-	const files = {
-		...docs,
-		...blogs,
-		...frequentQuestions
-	};
-	for await (const fileKey of Object.keys(files)) {
-		const fileMetadata = (0, __intlayer_core.getMarkdownMetadata)(files[fileKey]);
-		const fileChunks = chunkText(files[fileKey]);
-		const existingEmbeddings = readEmbeddingsForFile(fileKey);
-		const existingChunksForFile = Object.keys(existingEmbeddings);
-		const currentChunkCount = fileChunks.length;
-		const previousChunkCount = existingChunksForFile.length;
-		let shouldRegenerateFileEmbeddings = false;
-		if (currentChunkCount !== previousChunkCount) {
-			console.info(`File "${fileKey}" chunk count changed: ${previousChunkCount} -> ${currentChunkCount}. Regenerating embeddings.`);
-			shouldRegenerateFileEmbeddings = !skipDocEmbeddingsIndex;
-		}
-		let resultForFile = {};
-		for await (const chunkIndex of Object.keys(fileChunks)) {
-			const chunkNumber = Number(chunkIndex) + 1;
-			const chunksNumber = fileChunks.length;
-			const fileChunk = fileChunks[chunkIndex];
-			const chunkKeyName = `chunk_${chunkNumber}`;
-			let embedding = !shouldRegenerateFileEmbeddings ? existingEmbeddings[chunkKeyName] : void 0;
-			if (!embedding) {
-				embedding = await generateEmbedding(fileChunk);
-				console.info(`- Generated new embedding: ${fileKey}/${chunkKeyName}`);
-			}
-			resultForFile = {
-				...resultForFile,
-				[chunkKeyName]: embedding
-			};
-			vectorStore.push({
-				fileKey,
-				chunkNumber,
-				embedding,
-				content: fileChunk,
-				docUrl: fileMetadata.url,
-				docName: fileMetadata.title
-			});
-			console.info(`- Indexed: ${fileKey}/${chunkKeyName}/${chunksNumber}`);
-		}
-		try {
-			if (JSON.stringify(resultForFile) !== JSON.stringify(existingEmbeddings)) writeEmbeddingsForFile(fileKey, resultForFile);
-		} catch (error) {
-			console.error(error);
-		}
-	}
-};
-
-//#endregion
-exports.indexMarkdownFiles = indexMarkdownFiles;
-//# sourceMappingURL=indexMarkdownFiles.cjs.map

package/dist/cjs/utils/AI/askDocQuestion/indexMarkdownFiles.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"indexMarkdownFiles.cjs","names":["__dirname","vectorStore: VectorStoreEl[]","EMBEDDING_MODEL: OpenAI.EmbeddingModel","OVERLAP_TOKENS: number","MAX_CHUNK_TOKENS: number","CHAR_BY_TOKEN: number","MAX_CHARS: number","OVERLAP_CHARS: number","chunks: string[]","OpenAI","resultForFile: Record<string, number[]>"],"sources":["../../../../../src/utils/AI/askDocQuestion/indexMarkdownFiles.ts"],"sourcesContent":["import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath } from 'node:url';\nimport { getMarkdownMetadata } from '@intlayer/core';\nimport { getBlogs, getDocs, getFrequentQuestions } from '@intlayer/docs';\nimport dotenv from 'dotenv';\nimport { OpenAI } from 'openai';\nimport { type AIOptions, AIProvider } from '../aiSdk';\n\nconst OUTPUT_EMBEDDINGS_DIR = 'src/utils/AI/askDocQuestion/embeddings';\nconst __dirname = dirname(fileURLToPath(import.meta.url));\n\nconst readEmbeddingsForFile = (fileKey: string): Record<string, number[]> => {\n  try {\n    return JSON.parse(\n      readFileSync(\n        `${__dirname}/embeddings/${fileKey.replace('.md', '.json')}`,\n        'utf-8'\n      )\n    ) as Record<string, number[]>;\n  } catch {\n    return {};\n  }\n};\n\nconst writeEmbeddingsForFile = (\n  fileKey: string,\n  data: Record<string, number[]>\n): void => {\n  const filePath = join(\n    OUTPUT_EMBEDDINGS_DIR,\n    `${fileKey.replace('.md', '.json')}`\n  );\n  const dir = dirname(filePath);\n  if (!existsSync(dir)) {\n    mkdirSync(dir, { recursive: true });\n  }\n  writeFileSync(filePath, JSON.stringify(data));\n};\n\ntype VectorStoreEl = {\n  fileKey: string;\n  chunkNumber: number;\n  content: string;\n  embedding: number[];\n  docUrl: string;\n  docName: string;\n};\n\n/**\n * Simple in-memory vector store to hold document embeddings and their content.\n * Each entry contains:\n * - fileKey: A unique key identifying the file\n * - chunkNumber: The number of the chunk within the document\n * - content: The chunk content\n * - embedding: The numerical embedding vector for the chunk\n */\nconst vectorStore: VectorStoreEl[] = [];\n\n/*\n * Embedding model configuration\n */\nconst EMBEDDING_MODEL: OpenAI.EmbeddingModel = 'text-embedding-3-large'; // Model to use for embedding generation\nconst OVERLAP_TOKENS: number = 200; // Number of tokens to overlap between chunks\nconst MAX_CHUNK_TOKENS: number = 800; // Maximum number of tokens per chunk\nconst CHAR_BY_TOKEN: number = 4.15; // Approximate pessimistically the number of characters per token // Can use `tiktoken` or other tokenizers to calculate it more precisely\nconst MAX_CHARS: number = MAX_CHUNK_TOKENS * CHAR_BY_TOKEN;\nconst OVERLAP_CHARS: number = OVERLAP_TOKENS * CHAR_BY_TOKEN;\n\nconst skipDocEmbeddingsIndex = process.env.SKIP_DOC_EMBEDDINGS_INDEX === 'true';\n\n/**\n * Splits a given text into chunks ensuring each chunk does not exceed MAX_CHARS.\n * @param text - The input text to split.\n * @returns - Array of text chunks.\n */\nconst chunkText = (text: string): string[] => {\n  const chunks: string[] = [];\n  let start = 0;\n\n  while (start < text.length) {\n    let end = Math.min(start + MAX_CHARS, text.length);\n\n    // Ensure we don't cut words in the middle (find nearest space)\n    if (end < text.length) {\n      const lastSpace = text.lastIndexOf(' ', end);\n      if (lastSpace > start) {\n        end = lastSpace;\n      }\n    }\n\n    chunks.push(text.substring(start, end));\n\n    // Move start forward correctly\n    const nextStart = end - OVERLAP_CHARS;\n    if (nextStart <= start) {\n      // Prevent infinite loop if overlap is too large\n      start = end;\n    } else {\n      start = nextStart;\n    }\n  }\n\n  return chunks;\n};\n\n/**\n * Generates an embedding for a given text using OpenAI's embedding API.\n * Trims the text if it exceeds the maximum allowed characters.\n *\n * @param text - The input text to generate an embedding for\n * @returns The embedding vector as a number array\n */\nconst generateEmbedding = async (text: string): Promise<number[]> => {\n  try {\n    const openaiClient = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });\n\n    const response = await openaiClient.embeddings.create({\n      model: EMBEDDING_MODEL,\n      input: text,\n    });\n\n    return response.data[0].embedding;\n  } catch (error) {\n    console.error('Error generating embedding:', error);\n    return [];\n  }\n};\n\n/**\n * Indexes all Markdown documents by generating embeddings for each chunk and storing them in memory.\n * Persists per-document embeddings under `embeddings/<fileKey>.json`.\n * Handles cases where files have been updated and chunk counts have changed.\n */\nexport const indexMarkdownFiles = async (): Promise<void> => {\n  const env = process.env.NODE_ENV;\n  dotenv.config({\n    path: [`.env.${env}.local`, `.env.${env}`, '.env.local', '.env'],\n  });\n\n  // Retrieve documentation and blog posts in English locale\n  const frequentQuestions = await getFrequentQuestions();\n  const docs = await getDocs();\n  const blogs = await getBlogs();\n\n  const files = { ...docs, ...blogs, ...frequentQuestions }; // Combine docs and blogs into a single object\n\n  // Iterate over each file key (identifier) in the combined files\n  for await (const fileKey of Object.keys(files)) {\n    // Get the metadata of the file\n    const fileMetadata = getMarkdownMetadata(\n      files[fileKey as keyof typeof files] as string\n    );\n\n    // Split the document into chunks based on headings\n    const fileChunks = chunkText(\n      files[fileKey as keyof typeof files] as string\n    );\n\n    // Read existing embeddings for this file\n    const existingEmbeddings = readEmbeddingsForFile(fileKey);\n\n    // Check if the number of chunks has changed for this file\n    const existingChunksForFile = Object.keys(existingEmbeddings);\n    const currentChunkCount = fileChunks.length;\n    const previousChunkCount = existingChunksForFile.length;\n\n    let shouldRegenerateFileEmbeddings = false;\n\n    // If chunk count differs, we need to regenerate embeddings for this file\n    if (currentChunkCount !== previousChunkCount) {\n      console.info(\n        `File \"${fileKey}\" chunk count changed: ${previousChunkCount} -> ${currentChunkCount}. Regenerating embeddings.`\n      );\n\n      shouldRegenerateFileEmbeddings = !skipDocEmbeddingsIndex;\n    }\n\n    // Iterate over each chunk within the current file\n    let resultForFile: Record<string, number[]> = {};\n    for await (const chunkIndex of Object.keys(fileChunks)) {\n      const chunkNumber = Number(chunkIndex) + 1; // Chunk number starts at 1\n      const chunksNumber = fileChunks.length;\n\n      const fileChunk = fileChunks[\n        chunkIndex as keyof typeof fileChunks\n      ] as string;\n\n      const chunkKeyName = `chunk_${chunkNumber}`; // Unique key for the chunk within the file\n\n      // Retrieve precomputed embedding if available and file hasn't changed\n      const docEmbedding = !shouldRegenerateFileEmbeddings\n        ? (existingEmbeddings[\n            chunkKeyName as keyof typeof existingEmbeddings\n          ] as number[] | undefined)\n        : undefined;\n\n      let embedding = docEmbedding; // Use existing embedding if available and valid\n\n      if (!embedding) {\n        embedding = await generateEmbedding(fileChunk); // Generate embedding if not present or file changed\n        console.info(`- Generated new embedding: ${fileKey}/${chunkKeyName}`);\n      }\n\n      // Update the file-scoped result object with the embedding\n      resultForFile = { ...resultForFile, [chunkKeyName]: embedding };\n\n      // Store the embedding and content in the in-memory vector store\n      vectorStore.push({\n        fileKey,\n        chunkNumber,\n        embedding,\n        content: fileChunk,\n        docUrl: fileMetadata.url,\n        docName: fileMetadata.title,\n      });\n\n      console.info(`- Indexed: ${fileKey}/${chunkKeyName}/${chunksNumber}`);\n    }\n\n    // Persist per-file embeddings if changed\n    try {\n      if (\n        JSON.stringify(resultForFile) !== JSON.stringify(existingEmbeddings)\n      ) {\n        writeEmbeddingsForFile(fileKey, resultForFile);\n      }\n    } catch (error) {\n      console.error(error);\n    }\n  }\n};\n"],"mappings":";;;;;;;;;;;AASA,MAAM,wBAAwB;AAC9B,MAAMA,+GAAkD,CAAC;AAEzD,MAAM,yBAAyB,YAA8C;AAC3E,KAAI;AACF,SAAO,KAAK,gCAER,GAAGA,YAAU,cAAc,QAAQ,QAAQ,OAAO,QAAQ,IAC1D,QACD,CACF;SACK;AACN,SAAO,EAAE;;;AAIb,MAAM,0BACJ,SACA,SACS;CACT,MAAM,+BACJ,uBACA,GAAG,QAAQ,QAAQ,OAAO,QAAQ,GACnC;CACD,MAAM,6BAAc,SAAS;AAC7B,KAAI,yBAAY,IAAI,CAClB,wBAAU,KAAK,EAAE,WAAW,MAAM,CAAC;AAErC,4BAAc,UAAU,KAAK,UAAU,KAAK,CAAC;;;;;;;;;;AAoB/C,MAAMC,cAA+B,EAAE;AAKvC,MAAMC,kBAAyC;AAC/C,MAAMC,iBAAyB;AAC/B,MAAMC,mBAA2B;AACjC,MAAMC,gBAAwB;AAC9B,MAAMC,YAAoB,mBAAmB;AAC7C,MAAMC,gBAAwB,iBAAiB;AAE/C,MAAM,yBAAyB,QAAQ,IAAI,8BAA8B;;;;;;AAOzE,MAAM,aAAa,SAA2B;CAC5C,MAAMC,SAAmB,EAAE;CAC3B,IAAI,QAAQ;AAEZ,QAAO,QAAQ,KAAK,QAAQ;EAC1B,IAAI,MAAM,KAAK,IAAI,QAAQ,WAAW,KAAK,OAAO;AAGlD,MAAI,MAAM,KAAK,QAAQ;GACrB,MAAM,YAAY,KAAK,YAAY,KAAK,IAAI;AAC5C,OAAI,YAAY,MACd,OAAM;;AAIV,SAAO,KAAK,KAAK,UAAU,OAAO,IAAI,CAAC;EAGvC,MAAM,YAAY,MAAM;AACxB,MAAI,aAAa,MAEf,SAAQ;MAER,SAAQ;;AAIZ,QAAO;;;;;;;;;AAUT,MAAM,oBAAoB,OAAO,SAAoC;AACnE,KAAI;AAQF,UALiB,MAFI,IAAIC,cAAO,EAAE,QAAQ,QAAQ,IAAI,gBAAgB,CAAC,CAEnC,WAAW,OAAO;GACpD,OAAO;GACP,OAAO;GACR,CAAC,EAEc,KAAK,GAAG;UACjB,OAAO;AACd,UAAQ,MAAM,+BAA+B,MAAM;AACnD,SAAO,EAAE;;;;;;;;AASb,MAAa,qBAAqB,YAA2B;CAC3D,MAAM,MAAM,QAAQ,IAAI;AACxB,gBAAO,OAAO,EACZ,MAAM;EAAC,QAAQ,IAAI;EAAS,QAAQ;EAAO;EAAc;EAAO,EACjE,CAAC;CAGF,MAAM,oBAAoB,iDAA4B;CACtD,MAAM,OAAO,oCAAe;CAC5B,MAAM,QAAQ,qCAAgB;CAE9B,MAAM,QAAQ;EAAE,GAAG;EAAM,GAAG;EAAO,GAAG;EAAmB;AAGzD,YAAW,MAAM,WAAW,OAAO,KAAK,MAAM,EAAE;EAE9C,MAAM,wDACJ,MAAM,SACP;EAGD,MAAM,aAAa,UACjB,MAAM,SACP;EAGD,MAAM,qBAAqB,sBAAsB,QAAQ;EAGzD,MAAM,wBAAwB,OAAO,KAAK,mBAAmB;EAC7D,MAAM,oBAAoB,WAAW;EACrC,MAAM,qBAAqB,sBAAsB;EAEjD,IAAI,iCAAiC;AAGrC,MAAI,sBAAsB,oBAAoB;AAC5C,WAAQ,KACN,SAAS,QAAQ,yBAAyB,mBAAmB,MAAM,kBAAkB,4BACtF;AAED,oCAAiC,CAAC;;EAIpC,IAAIC,gBAA0C,EAAE;AAChD,aAAW,MAAM,cAAc,OAAO,KAAK,WAAW,EAAE;GACtD,MAAM,cAAc,OAAO,WAAW,GAAG;GACzC,MAAM,eAAe,WAAW;GAEhC,MAAM,YAAY,WAChB;GAGF,MAAM,eAAe,SAAS;GAS9B,IAAI,YANiB,CAAC,iCACjB,mBACC,gBAEF;AAIJ,OAAI,CAAC,WAAW;AACd,gBAAY,MAAM,kBAAkB,UAAU;AAC9C,YAAQ,KAAK,8BAA8B,QAAQ,GAAG,eAAe;;AAIvE,mBAAgB;IAAE,GAAG;KAAgB,eAAe;IAAW;AAG/D,eAAY,KAAK;IACf;IACA;IACA;IACA,SAAS;IACT,QAAQ,aAAa;IACrB,SAAS,aAAa;IACvB,CAAC;AAEF,WAAQ,KAAK,cAAc,QAAQ,GAAG,aAAa,GAAG,eAAe;;AAIvE,MAAI;AACF,OACE,KAAK,UAAU,cAAc,KAAK,KAAK,UAAU,mBAAmB,CAEpE,wBAAuB,SAAS,cAAc;WAEzC,OAAO;AACd,WAAQ,MAAM,MAAM"}

package/dist/cjs/utils/AI/auditDictionary/index.cjs

@@ -1,60 +0,0 @@
-const require_rolldown_runtime = require('../../../_virtual/rolldown_runtime.cjs');
-const require_logger_index = require('../../../logger/index.cjs');
-const require__utils_asset = require('../../../_virtual/_utils_asset.cjs');
-const require_utils_extractJSON = require('../../extractJSON.cjs');
-let __intlayer_types = require("@intlayer/types");
-let __intlayer_core = require("@intlayer/core");
-let ai = require("ai");
-
-//#region src/utils/AI/auditDictionary/index.ts
-const CHAT_GPT_PROMPT = require__utils_asset.readAsset("./PROMPT.md");
-const aiDefaultOptions = {};
-/**
-* Format a locale with its name.
-*
-* @param locale - The locale to format.
-* @returns A string in the format "locale: name", e.g. "en: English".
-*/
-const formatLocaleWithName = (locale) => {
-	return `${locale}: ${(0, __intlayer_core.getLocaleName)(locale, __intlayer_types.Locales.ENGLISH)}`;
-};
-/**
-* Formats tag instructions for the AI prompt.
-* Creates a string with all available tags and their descriptions.
-*
-* @param tags - The list of tags to format.
-* @returns A formatted string with tag instructions.
-*/
-const formatTagInstructions = (tags) => {
-	if (!tags || tags.length === 0) return "";
-	return [`Based on the dictionary content, identify specific tags from the list below that would be relevant:`, tags.map(({ key, description }) => `- ${key}: ${description}`).join("\n\n")].join("\n\n");
-};
-/**
-* Audits a content declaration file by constructing a prompt for AI models.
-* The prompt includes details about the project's locales, file paths of content declarations,
-* and requests for identifying issues or inconsistencies.
-*/
-const auditDictionary = async ({ fileContent, filePath, locales, defaultLocale, tags, aiConfig, applicationContext }) => {
-	const otherLocales = locales.filter((locale) => locale !== defaultLocale);
-	const prompt = CHAT_GPT_PROMPT.replace("{{defaultLocale}}", formatLocaleWithName(defaultLocale)).replace("{{otherLocales}}", `{${otherLocales.map(formatLocaleWithName).join(", ")}}`).replace("{{filePath}}", filePath ?? "").replace("{{applicationContext}}", applicationContext ?? "").replace("{{tagsInstructions}}", formatTagInstructions(tags));
-	const { text: newContent, usage } = await (0, ai.generateText)({
-		...aiConfig,
-		messages: [{
-			role: "system",
-			content: prompt
-		}, {
-			role: "user",
-			content: ["**File to Audit:**", fileContent].join("\n")
-		}]
-	});
-	require_logger_index.logger.info(`${usage?.totalTokens ?? 0} tokens used in the request`);
-	return {
-		fileContent: require_utils_extractJSON.extractJson(newContent),
-		tokenUsed: usage?.totalTokens ?? 0
-	};
-};
-
-//#endregion
-exports.aiDefaultOptions = aiDefaultOptions;
-exports.auditDictionary = auditDictionary;
-//# sourceMappingURL=index.cjs.map

package/dist/cjs/utils/AI/auditDictionary/index.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.cjs","names":["readAsset","aiDefaultOptions: AIOptions","Locales","extractJson"],"sources":["../../../../../src/utils/AI/auditDictionary/index.ts"],"sourcesContent":["import { readAsset } from 'utils:asset';\nimport { getLocaleName } from '@intlayer/core';\nimport { type Locale, Locales } from '@intlayer/types';\nimport { logger } from '@logger';\nimport { extractJson } from '@utils/extractJSON';\nimport { generateText } from 'ai';\nimport type { Tag } from '@/types/tag.types';\nimport type { AIConfig, AIOptions } from '../aiSdk';\n\nexport type AuditOptions = {\n  fileContent: string;\n  filePath?: string;\n  locales: Locale[];\n  defaultLocale: Locale;\n  tags: Tag[];\n  aiConfig: AIConfig;\n  applicationContext?: string;\n};\n\nexport type AuditFileResultData = {\n  fileContent: {\n    title: string;\n    description: string;\n    tags: string[];\n  };\n  tokenUsed: number;\n};\n\n// The prompt template to send to the AI model\nconst CHAT_GPT_PROMPT = readAsset('./PROMPT.md');\n\nexport const aiDefaultOptions: AIOptions = {\n  // Keep default options\n};\n\n/**\n * Format a locale with its name.\n *\n * @param locale - The locale to format.\n * @returns A string in the format \"locale: name\", e.g. \"en: English\".\n */\nconst formatLocaleWithName = (locale: Locale): string => {\n  return `${locale}: ${getLocaleName(locale, Locales.ENGLISH)}`;\n};\n\n/**\n * Formats tag instructions for the AI prompt.\n * Creates a string with all available tags and their descriptions.\n *\n * @param tags - The list of tags to format.\n * @returns A formatted string with tag instructions.\n */\nconst formatTagInstructions = (tags: Tag[]): string => {\n  if (!tags || tags.length === 0) return '';\n\n  // Prepare the tag instructions.\n  return [\n    `Based on the dictionary content, identify specific tags from the list below that would be relevant:`,\n    tags.map(({ key, description }) => `- ${key}: ${description}`).join('\\n\\n'),\n  ].join('\\n\\n');\n};\n\n/**\n * Audits a content declaration file by constructing a prompt for AI models.\n * The prompt includes details about the project's locales, file paths of content declarations,\n * and requests for identifying issues or inconsistencies.\n */\nexport const auditDictionary = async ({\n  fileContent,\n  filePath,\n  locales,\n  defaultLocale,\n  tags,\n  aiConfig,\n  applicationContext,\n}: AuditOptions): Promise<AuditFileResultData | undefined> => {\n  const otherLocales = locales.filter((locale) => locale !== defaultLocale);\n\n  // Prepare the prompt for AI by replacing placeholders with actual values.\n  const prompt = CHAT_GPT_PROMPT.replace(\n    '{{defaultLocale}}',\n    formatLocaleWithName(defaultLocale)\n  )\n    .replace(\n      '{{otherLocales}}',\n      `{${otherLocales.map(formatLocaleWithName).join(', ')}}`\n    )\n    .replace('{{filePath}}', filePath ?? '')\n    .replace('{{applicationContext}}', applicationContext ?? '')\n    .replace('{{tagsInstructions}}', formatTagInstructions(tags));\n\n  // Use the AI SDK to generate the completion\n  const { text: newContent, usage } = await generateText({\n    ...aiConfig,\n    messages: [\n      { role: 'system', content: prompt },\n      {\n        role: 'user',\n        content: ['**File to Audit:**', fileContent].join('\\n'),\n      },\n    ],\n  });\n\n  logger.info(`${usage?.totalTokens ?? 0} tokens used in the request`);\n\n  return {\n    fileContent: extractJson(newContent),\n    tokenUsed: usage?.totalTokens ?? 0,\n  };\n};\n"],"mappings":";;;;;;;;;AA6BA,MAAM,kBAAkBA,+BAAU,cAAc;AAEhD,MAAaC,mBAA8B,EAE1C;;;;;;;AAQD,MAAM,wBAAwB,WAA2B;AACvD,QAAO,GAAG,OAAO,uCAAkB,QAAQC,yBAAQ,QAAQ;;;;;;;;;AAU7D,MAAM,yBAAyB,SAAwB;AACrD,KAAI,CAAC,QAAQ,KAAK,WAAW,EAAG,QAAO;AAGvC,QAAO,CACL,uGACA,KAAK,KAAK,EAAE,KAAK,kBAAkB,KAAK,IAAI,IAAI,cAAc,CAAC,KAAK,OAAO,CAC5E,CAAC,KAAK,OAAO;;;;;;;AAQhB,MAAa,kBAAkB,OAAO,EACpC,aACA,UACA,SACA,eACA,MACA,UACA,yBAC4D;CAC5D,MAAM,eAAe,QAAQ,QAAQ,WAAW,WAAW,cAAc;CAGzE,MAAM,SAAS,gBAAgB,QAC7B,qBACA,qBAAqB,cAAc,CACpC,CACE,QACC,oBACA,IAAI,aAAa,IAAI,qBAAqB,CAAC,KAAK,KAAK,CAAC,GACvD,CACA,QAAQ,gBAAgB,YAAY,GAAG,CACvC,QAAQ,0BAA0B,sBAAsB,GAAG,CAC3D,QAAQ,wBAAwB,sBAAsB,KAAK,CAAC;CAG/D,MAAM,EAAE,MAAM,YAAY,UAAU,2BAAmB;EACrD,GAAG;EACH,UAAU,CACR;GAAE,MAAM;GAAU,SAAS;GAAQ,EACnC;GACE,MAAM;GACN,SAAS,CAAC,sBAAsB,YAAY,CAAC,KAAK,KAAK;GACxD,CACF;EACF,CAAC;AAEF,6BAAO,KAAK,GAAG,OAAO,eAAe,EAAE,6BAA6B;AAEpE,QAAO;EACL,aAAaC,sCAAY,WAAW;EACpC,WAAW,OAAO,eAAe;EAClC"}

package/dist/cjs/utils/AI/auditDictionaryField/index.cjs

@@ -1,63 +0,0 @@
-const require_rolldown_runtime = require('../../../_virtual/rolldown_runtime.cjs');
-const require_logger_index = require('../../../logger/index.cjs');
-const require__utils_asset = require('../../../_virtual/_utils_asset.cjs');
-let __intlayer_types = require("@intlayer/types");
-let __intlayer_core = require("@intlayer/core");
-let ai = require("ai");
-
-//#region src/utils/AI/auditDictionaryField/index.ts
-const CHAT_GPT_PROMPT = require__utils_asset.readAsset("./PROMPT.md");
-const aiDefaultOptions = {};
-/**
-* Format a locale with its name.
-*
-* @param locale - The locale to format.
-* @returns A string in the format "locale: name", e.g. "en: English".
-*/
-const formatLocaleWithName = (locale) => {
-	return `${locale}: ${(0, __intlayer_core.getLocaleName)(locale, __intlayer_types.Locales.ENGLISH)}`;
-};
-/**
-* Formats tag instructions for the AI prompt.
-*
-* @param tags - Array of tags to format
-* @returns A formatted string with tag instructions
-*/
-const formatTagInstructions = (tags) => {
-	if (!tags || tags.length === 0) return "";
-	return `Based on the dictionary content, identify specific tags from the list below that would be relevant:
-  
-${tags.map(({ key, description }) => `- ${key}: ${description}`).join("\n\n")}`;
-};
-/**
-* Audits a content declaration file by constructing a prompt for AI models.
-* The prompt includes details about the project's locales, file paths of content declarations,
-* and requests for identifying issues or inconsistencies.
-*/
-const auditDictionaryField = async ({ fileContent, applicationContext, locales, keyPath, tags, aiConfig }) => {
-	const prompt = CHAT_GPT_PROMPT.replace("{{otherLocales}}", `{${locales.map(formatLocaleWithName).join(", ")}}`).replace("{{keyPath}}", JSON.stringify(keyPath)).replace("{{applicationContext}}", applicationContext ?? "").replace("{{tagsInstructions}}", formatTagInstructions(tags));
-	if (!aiConfig) {
-		require_logger_index.logger.error("Failed to configure AI model");
-		return;
-	}
-	const { text: newContent, usage } = await (0, ai.generateText)({
-		...aiConfig,
-		messages: [{
-			role: "system",
-			content: prompt
-		}, {
-			role: "user",
-			content: ["**File to Audit:**", fileContent].join("\n")
-		}]
-	});
-	require_logger_index.logger.info(`${usage?.totalTokens ?? 0} tokens used in the request`);
-	return {
-		fileContent: newContent,
-		tokenUsed: usage?.totalTokens ?? 0
-	};
-};
-
-//#endregion
-exports.aiDefaultOptions = aiDefaultOptions;
-exports.auditDictionaryField = auditDictionaryField;
-//# sourceMappingURL=index.cjs.map

package/dist/cjs/utils/AI/auditDictionaryField/index.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.cjs","names":["readAsset","aiDefaultOptions: AIOptions","Locales"],"sources":["../../../../../src/utils/AI/auditDictionaryField/index.ts"],"sourcesContent":["import { readAsset } from 'utils:asset';\nimport { getLocaleName } from '@intlayer/core';\nimport type { KeyPath } from '@intlayer/types';\nimport { type Locale, Locales } from '@intlayer/types';\nimport { logger } from '@logger';\nimport { generateText } from 'ai';\nimport type { Tag } from '@/types/tag.types';\nimport type { AIConfig, AIOptions } from '../aiSdk';\n\nexport type AuditDictionaryFieldOptions = {\n  fileContent: string;\n  locales: Locale[];\n  keyPath: KeyPath[];\n  tags: Tag[];\n  aiConfig: AIConfig;\n  applicationContext?: string;\n};\n\nexport type AuditDictionaryFieldResultData = {\n  fileContent: string;\n  tokenUsed: number;\n};\n\n// The prompt template to send to the AI model\nconst CHAT_GPT_PROMPT = readAsset('./PROMPT.md');\n\nexport const aiDefaultOptions: AIOptions = {\n  // Keep default options\n};\n\n/**\n * Format a locale with its name.\n *\n * @param locale - The locale to format.\n * @returns A string in the format \"locale: name\", e.g. \"en: English\".\n */\nconst formatLocaleWithName = (locale: Locale): string => {\n  return `${locale}: ${getLocaleName(locale, Locales.ENGLISH)}`;\n};\n\n/**\n * Formats tag instructions for the AI prompt.\n *\n * @param tags - Array of tags to format\n * @returns A formatted string with tag instructions\n */\nconst formatTagInstructions = (tags: Tag[]): string => {\n  if (!tags || tags.length === 0) {\n    return '';\n  }\n\n  return `Based on the dictionary content, identify specific tags from the list below that would be relevant:\n  \n${tags.map(({ key, description }) => `- ${key}: ${description}`).join('\\n\\n')}`;\n};\n\n/**\n * Audits a content declaration file by constructing a prompt for AI models.\n * The prompt includes details about the project's locales, file paths of content declarations,\n * and requests for identifying issues or inconsistencies.\n */\nexport const auditDictionaryField = async ({\n  fileContent,\n  applicationContext,\n  locales,\n  keyPath,\n  tags,\n  aiConfig,\n}: AuditDictionaryFieldOptions): Promise<\n  AuditDictionaryFieldResultData | undefined\n> => {\n  // Prepare the prompt for AI by replacing placeholders with actual values.\n  const prompt = CHAT_GPT_PROMPT.replace(\n    '{{otherLocales}}',\n    `{${locales.map(formatLocaleWithName).join(', ')}}`\n  )\n    .replace('{{keyPath}}', JSON.stringify(keyPath))\n    .replace('{{applicationContext}}', applicationContext ?? '')\n    .replace('{{tagsInstructions}}', formatTagInstructions(tags));\n\n  if (!aiConfig) {\n    logger.error('Failed to configure AI model');\n    return undefined;\n  }\n\n  // Use the AI SDK to generate the completion\n  const { text: newContent, usage } = await generateText({\n    ...aiConfig,\n    messages: [\n      { role: 'system', content: prompt },\n      {\n        role: 'user',\n        content: ['**File to Audit:**', fileContent].join('\\n'),\n      },\n    ],\n  });\n\n  logger.info(`${usage?.totalTokens ?? 0} tokens used in the request`);\n\n  return {\n    fileContent: newContent,\n    tokenUsed: usage?.totalTokens ?? 0,\n  };\n};\n"],"mappings":";;;;;;;;AAwBA,MAAM,kBAAkBA,+BAAU,cAAc;AAEhD,MAAaC,mBAA8B,EAE1C;;;;;;;AAQD,MAAM,wBAAwB,WAA2B;AACvD,QAAO,GAAG,OAAO,uCAAkB,QAAQC,yBAAQ,QAAQ;;;;;;;;AAS7D,MAAM,yBAAyB,SAAwB;AACrD,KAAI,CAAC,QAAQ,KAAK,WAAW,EAC3B,QAAO;AAGT,QAAO;;EAEP,KAAK,KAAK,EAAE,KAAK,kBAAkB,KAAK,IAAI,IAAI,cAAc,CAAC,KAAK,OAAO;;;;;;;AAQ7E,MAAa,uBAAuB,OAAO,EACzC,aACA,oBACA,SACA,SACA,MACA,eAGG;CAEH,MAAM,SAAS,gBAAgB,QAC7B,oBACA,IAAI,QAAQ,IAAI,qBAAqB,CAAC,KAAK,KAAK,CAAC,GAClD,CACE,QAAQ,eAAe,KAAK,UAAU,QAAQ,CAAC,CAC/C,QAAQ,0BAA0B,sBAAsB,GAAG,CAC3D,QAAQ,wBAAwB,sBAAsB,KAAK,CAAC;AAE/D,KAAI,CAAC,UAAU;AACb,8BAAO,MAAM,+BAA+B;AAC5C;;CAIF,MAAM,EAAE,MAAM,YAAY,UAAU,2BAAmB;EACrD,GAAG;EACH,UAAU,CACR;GAAE,MAAM;GAAU,SAAS;GAAQ,EACnC;GACE,MAAM;GACN,SAAS,CAAC,sBAAsB,YAAY,CAAC,KAAK,KAAK;GACxD,CACF;EACF,CAAC;AAEF,6BAAO,KAAK,GAAG,OAAO,eAAe,EAAE,6BAA6B;AAEpE,QAAO;EACL,aAAa;EACb,WAAW,OAAO,eAAe;EAClC"}

package/dist/cjs/utils/AI/auditDictionaryMetadata/index.cjs

@@ -1,45 +0,0 @@
-const require_rolldown_runtime = require('../../../_virtual/rolldown_runtime.cjs');
-const require_logger_index = require('../../../logger/index.cjs');
-const require__utils_asset = require('../../../_virtual/_utils_asset.cjs');
-const require_utils_extractJSON = require('../../extractJSON.cjs');
-let ai = require("ai");
-
-//#region src/utils/AI/auditDictionaryMetadata/index.ts
-const CHAT_GPT_PROMPT = require__utils_asset.readAsset("./PROMPT.md");
-const aiDefaultOptions = {};
-/**
-* Audits a content declaration file by constructing a prompt for AI models.
-* The prompt includes details about the project's locales, file paths of content declarations,
-* and requests for identifying issues or inconsistencies.
-*/
-const auditDictionaryMetadata = async ({ tags, fileContent, applicationContext, aiConfig }) => {
-	const prompt = CHAT_GPT_PROMPT.replace("{{tags}}", `${JSON.stringify(tags.map(({ key, description }) => `- ${key}: ${description}`).join("\n\n"), null, 2)}`).replace("{{applicationContext}}", applicationContext ?? "");
-	if (!aiConfig) {
-		require_logger_index.logger.error("Failed to configure AI model");
-		return;
-	}
-	const { text: newContent, usage } = await (0, ai.generateText)({
-		...aiConfig,
-		messages: [{
-			role: "system",
-			content: prompt
-		}, {
-			role: "user",
-			content: [
-				"**Content declaration to describe:**",
-				"This is the content declaration that you should consider to describe:",
-				fileContent
-			].join("\n")
-		}]
-	});
-	require_logger_index.logger.info(`${usage?.totalTokens ?? 0} tokens used in the request`);
-	return {
-		fileContent: require_utils_extractJSON.extractJson(newContent),
-		tokenUsed: usage?.totalTokens ?? 0
-	};
-};
-
-//#endregion
-exports.aiDefaultOptions = aiDefaultOptions;
-exports.auditDictionaryMetadata = auditDictionaryMetadata;
-//# sourceMappingURL=index.cjs.map

package/dist/cjs/utils/AI/auditDictionaryMetadata/index.cjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.cjs","names":["readAsset","aiDefaultOptions: AIOptions","extractJson"],"sources":["../../../../../src/utils/AI/auditDictionaryMetadata/index.ts"],"sourcesContent":["import { readAsset } from 'utils:asset';\nimport { logger } from '@logger';\nimport { extractJson } from '@utils/extractJSON';\nimport { generateText } from 'ai';\nimport type { Tag } from '@/types/tag.types';\nimport type { AIConfig, AIOptions } from '../aiSdk';\n\nexport type AuditOptions = {\n  fileContent: string;\n  tags: Tag[];\n  aiConfig: AIConfig;\n  applicationContext?: string;\n};\n\nexport type AuditFileResultData = {\n  fileContent: {\n    title: string;\n    description: string;\n    tags: string[];\n  };\n  tokenUsed: number;\n};\n\n// The prompt template to send to AI models\nconst CHAT_GPT_PROMPT = readAsset('./PROMPT.md');\n\nexport const aiDefaultOptions: AIOptions = {\n  // Keep default options\n};\n\n/**\n * Audits a content declaration file by constructing a prompt for AI models.\n * The prompt includes details about the project's locales, file paths of content declarations,\n * and requests for identifying issues or inconsistencies.\n */\nexport const auditDictionaryMetadata = async ({\n  tags,\n  fileContent,\n  applicationContext,\n  aiConfig,\n}: AuditOptions): Promise<AuditFileResultData | undefined> => {\n  // Prepare the prompt for AI by replacing placeholders with actual values.\n  const prompt = CHAT_GPT_PROMPT.replace(\n    '{{tags}}',\n    `${JSON.stringify(\n      tags\n        .map(({ key, description }) => `- ${key}: ${description}`)\n        .join('\\n\\n'),\n      null,\n      2\n    )}`\n  ).replace('{{applicationContext}}', applicationContext ?? '');\n\n  if (!aiConfig) {\n    logger.error('Failed to configure AI model');\n    return undefined;\n  }\n\n  // Use the AI SDK to generate the completion\n  const { text: newContent, usage } = await generateText({\n    ...aiConfig,\n    messages: [\n      { role: 'system', content: prompt },\n      {\n        role: 'user',\n        content: [\n          '**Content declaration to describe:**',\n          'This is the content declaration that you should consider to describe:',\n          fileContent,\n        ].join('\\n'),\n      },\n    ],\n  });\n\n  logger.info(`${usage?.totalTokens ?? 0} tokens used in the request`);\n\n  return {\n    fileContent: extractJson(newContent),\n    tokenUsed: usage?.totalTokens ?? 0,\n  };\n};\n"],"mappings":";;;;;;;AAwBA,MAAM,kBAAkBA,+BAAU,cAAc;AAEhD,MAAaC,mBAA8B,EAE1C;;;;;;AAOD,MAAa,0BAA0B,OAAO,EAC5C,MACA,aACA,oBACA,eAC4D;CAE5D,MAAM,SAAS,gBAAgB,QAC7B,YACA,GAAG,KAAK,UACN,KACG,KAAK,EAAE,KAAK,kBAAkB,KAAK,IAAI,IAAI,cAAc,CACzD,KAAK,OAAO,EACf,MACA,EACD,GACF,CAAC,QAAQ,0BAA0B,sBAAsB,GAAG;AAE7D,KAAI,CAAC,UAAU;AACb,8BAAO,MAAM,+BAA+B;AAC5C;;CAIF,MAAM,EAAE,MAAM,YAAY,UAAU,2BAAmB;EACrD,GAAG;EACH,UAAU,CACR;GAAE,MAAM;GAAU,SAAS;GAAQ,EACnC;GACE,MAAM;GACN,SAAS;IACP;IACA;IACA;IACD,CAAC,KAAK,KAAK;GACb,CACF;EACF,CAAC;AAEF,6BAAO,KAAK,GAAG,OAAO,eAAe,EAAE,6BAA6B;AAEpE,QAAO;EACL,aAAaC,sCAAY,WAAW;EACpC,WAAW,OAAO,eAAe;EAClC"}