@dyrected/core 2.13.0 → 2.13.3

package/dist/chunk-ZYYDTHF3.js

@@ -1,635 +0,0 @@
-import {
-  sanitizeDocForAI
-} from "./chunk-254THFDS.js";
-import {
-  AI_CHUNKS_COLLECTION,
-  isAICollection
-} from "./chunk-4RAQJVJM.js";
-import {
-  isAccessAllowed
-} from "./chunk-HRRJ3RYT.js";
-
-// src/services/rag/normalizer.ts
-function stripHtml(html) {
-  return html.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").replace(/\s+([.,;:!?])/g, "$1").trim();
-}
-function extractTextFromNode(node) {
-  if (!node) return "";
-  if (typeof node === "string") return node;
-  if (typeof node === "number" || typeof node === "boolean") return String(node);
-  if (typeof node.text === "string") {
-    let text = node.text;
-    if (node.bold) text = `**${text}**`;
-    if (node.italic) text = `*${text}*`;
-    if (node.code) text = `\`${text}\``;
-    return text;
-  }
-  const childrenText = Array.isArray(node.children) ? node.children.map(extractTextFromNode).join("") : Array.isArray(node.content) ? node.content.map(extractTextFromNode).join("") : "";
-  const type = (node.type || node.tag || "").toLowerCase();
-  switch (type) {
-    case "h1":
-    case "heading-one":
-    case "heading_1":
-      return `
-# ${childrenText}
-`;
-    case "h2":
-    case "heading-two":
-    case "heading_2":
-      return `
-## ${childrenText}
-`;
-    case "h3":
-    case "heading-three":
-    case "heading_3":
-      return `
-### ${childrenText}
-`;
-    case "h4":
-    case "h5":
-    case "h6":
-      return `
-#### ${childrenText}
-`;
-    case "paragraph":
-    case "p":
-      return `
-${childrenText}
-`;
-    case "list-item":
-    case "li":
-      return `- ${childrenText}
-`;
-    case "bulleted-list":
-    case "ul":
-      return `
-${childrenText}
-`;
-    case "numbered-list":
-    case "ol":
-      return `
-${childrenText}
-`;
-    case "blockquote":
-    case "quote":
-      return `
-> ${childrenText}
-`;
-    case "code":
-    case "code-block":
-      return `
-\`\`\`
-${childrenText}
-\`\`\`
-`;
-    case "link":
-    case "a":
-      return node.url || node.href ? `[${childrenText}](${node.url || node.href})` : childrenText;
-    default:
-      return childrenText;
-  }
-}
-function normalizeFieldValue(value, _fieldType) {
-  if (value === null || value === void 0) return "";
-  if (typeof value === "string") {
-    const trimmed = value.trim();
-    if (!trimmed) return "";
-    if (trimmed.startsWith("<") && trimmed.endsWith(">")) {
-      return stripHtml(trimmed);
-    }
-    return trimmed;
-  }
-  if (typeof value === "number" || typeof value === "boolean") {
-    return String(value);
-  }
-  if (Array.isArray(value)) {
-    const parts = value.map((item) => normalizeFieldValue(item)).filter(Boolean);
-    return parts.join("\n\n");
-  }
-  if (typeof value === "object") {
-    const root = value.root || value;
-    const extracted = extractTextFromNode(root);
-    if (extracted && extracted.trim().length > 0) {
-      return extracted.replace(/\n{3,}/g, "\n\n").trim();
-    }
-    try {
-      const entries = Object.entries(value).filter(([k, v]) => !k.startsWith("_") && v !== null && v !== void 0 && typeof v !== "function").map(([k, v]) => `${k}: ${typeof v === "object" ? JSON.stringify(v) : v}`);
-      return entries.join("\n");
-    } catch {
-      return "";
-    }
-  }
-  return String(value);
-}
-function extractTextFromDoc(doc, targetFields) {
-  if (!doc) return [];
-  const results = [];
-  if (targetFields && targetFields.length > 0) {
-    for (const fieldName of targetFields) {
-      const val = doc[fieldName];
-      const text = normalizeFieldValue(val);
-      if (text && text.trim().length > 0) {
-        results.push({ field: fieldName, text: text.trim() });
-      }
-    }
-    return results;
-  }
-  for (const [key, val] of Object.entries(doc)) {
-    if (key.startsWith("_") || key === "id" || key === "password" || key === "salt" || key === "hash" || key === "resetPasswordToken" || key === "createdAt" || key === "updatedAt" || key === "createdBy" || key === "updatedBy") {
-      continue;
-    }
-    const text = normalizeFieldValue(val);
-    if (text && text.trim().length > 0) {
-      results.push({ field: key, text: text.trim() });
-    }
-  }
-  return results;
-}
-
-// src/services/rag/chunker.ts
-function hashContent(content, seed = 0) {
-  let h1 = 3735928559 ^ seed;
-  let h2 = 1103547991 ^ seed;
-  for (let i = 0, ch; i < content.length; i++) {
-    ch = content.charCodeAt(i);
-    h1 = Math.imul(h1 ^ ch, 2654435761);
-    h2 = Math.imul(h2 ^ ch, 1597334677);
-  }
-  h1 = Math.imul(h1 ^ h1 >>> 16, 2246822507) ^ Math.imul(h2 ^ h2 >>> 13, 3266489909);
-  h2 = Math.imul(h2 ^ h2 >>> 16, 2246822507) ^ Math.imul(h1 ^ h1 >>> 13, 3266489909);
-  return (4294967296 * (2097151 & h2) + (h1 >>> 0)).toString(16);
-}
-function estimateTokens(text) {
-  return Math.max(1, Math.ceil(text.length / 4));
-}
-function splitTextRecursively(text, maxSize, overlap, separators) {
-  if (text.length <= maxSize) {
-    return [text];
-  }
-  let chosenSeparator = separators[separators.length - 1] ?? "";
-  let nextSeparators = separators;
-  for (let i = 0; i < separators.length; i++) {
-    const sep = separators[i];
-    if (sep === "" || text.includes(sep)) {
-      chosenSeparator = sep;
-      nextSeparators = separators.slice(i + 1);
-      break;
-    }
-  }
-  const rawSplits = chosenSeparator === "" ? text.split("") : text.split(chosenSeparator);
-  const resultChunks = [];
-  let currentChunk = "";
-  for (const piece of rawSplits) {
-    const candidate = currentChunk ? currentChunk + (chosenSeparator ? chosenSeparator : "") + piece : piece;
-    if (candidate.length <= maxSize) {
-      currentChunk = candidate;
-    } else {
-      if (currentChunk.trim().length > 0) {
-        resultChunks.push(currentChunk.trim());
-      }
-      if (piece.length > maxSize) {
-        const subChunks = splitTextRecursively(piece, maxSize, overlap, nextSeparators);
-        resultChunks.push(...subChunks);
-        currentChunk = "";
-      } else {
-        if (overlap > 0 && currentChunk.length > overlap) {
-          const overlapPrefix = currentChunk.slice(currentChunk.length - overlap);
-          currentChunk = overlapPrefix + (chosenSeparator ? chosenSeparator : "") + piece;
-        } else {
-          currentChunk = piece;
-        }
-      }
-    }
-  }
-  if (currentChunk.trim().length > 0) {
-    resultChunks.push(currentChunk.trim());
-  }
-  return resultChunks;
-}
-function chunkText(text, options = {}) {
-  const {
-    maxChunkSize = 1500,
-    chunkOverlap = 150,
-    separators = ["\n## ", "\n### ", "\n#### ", "\n\n", "\n", ". ", "; ", ", ", " ", ""]
-  } = options;
-  if (!text || text.trim().length === 0) {
-    return [];
-  }
-  const trimmed = text.trim();
-  if (trimmed.length <= maxChunkSize) {
-    return [
-      {
-        text: trimmed,
-        tokenCount: estimateTokens(trimmed),
-        index: 0
-      }
-    ];
-  }
-  const rawChunks = splitTextRecursively(trimmed, maxChunkSize, chunkOverlap, separators);
-  return rawChunks.map((chunk) => chunk.trim()).filter((chunk) => chunk.length > 0).map((chunk, index) => ({
-    text: chunk,
-    tokenCount: estimateTokens(chunk),
-    index
-  }));
-}
-
-// src/services/rag/embedding.service.ts
-import { createGoogleGenerativeAI } from "@ai-sdk/google";
-import { createOpenAI } from "@ai-sdk/openai";
-import { embed, embedMany } from "ai";
-function getEmbeddingModel(config) {
-  const ai = config?.ai;
-  const provider = ai?.provider;
-  if (provider === "openrouter" || provider === "agentrouter" && process.env.OPENROUTER_API_KEY || !provider && process.env.OPENROUTER_API_KEY) {
-    const apiKey = ai?.apiKey || process.env.OPENROUTER_API_KEY;
-    if (apiKey) {
-      const baseURL = ai?.baseURL || "https://openrouter.ai/api/v1";
-      const modelName = ai?.rag?.embeddingModel || "openai/text-embedding-3-small";
-      const openRouterProvider = createOpenAI({ apiKey, baseURL });
-      return openRouterProvider.textEmbeddingModel(modelName);
-    }
-  }
-  if (provider === "openai" || provider === "agentrouter" && process.env.OPENAI_API_KEY || !provider && process.env.OPENAI_API_KEY && !process.env.GEMINI_API_KEY) {
-    const apiKey = ai?.apiKey || process.env.OPENAI_API_KEY;
-    if (apiKey) {
-      const baseURL = ai?.baseURL;
-      const modelName = ai?.rag?.embeddingModel || "text-embedding-3-small";
-      const openaiProvider = createOpenAI({ apiKey, baseURL });
-      return openaiProvider.textEmbeddingModel(modelName);
-    }
-  }
-  const geminiApiKey = ai?.apiKey || process.env.GEMINI_API_KEY || process.env.GOOGLE_GENERATIVE_AI_API_KEY;
-  if (geminiApiKey) {
-    const googleProvider = createGoogleGenerativeAI({ apiKey: geminiApiKey });
-    const modelName = ai?.rag?.embeddingModel || "text-embedding-004";
-    return googleProvider.textEmbeddingModel(modelName);
-  }
-  if (process.env.OPENROUTER_API_KEY) {
-    const openRouterProvider = createOpenAI({
-      apiKey: process.env.OPENROUTER_API_KEY,
-      baseURL: "https://openrouter.ai/api/v1"
-    });
-    return openRouterProvider.textEmbeddingModel(ai?.rag?.embeddingModel || "openai/text-embedding-3-small");
-  }
-  throw new Error(
-    "No AI API key found for embeddings. Please set OPENROUTER_API_KEY, GEMINI_API_KEY, or OPENAI_API_KEY in your .env.local file."
-  );
-}
-var EmbeddingService = class {
-  config;
-  model;
-  constructor(config) {
-    this.config = config;
-    this.model = getEmbeddingModel(config);
-  }
-  async embedText(text) {
-    const { embedding } = await embed({
-      model: this.model,
-      value: text
-    });
-    return embedding;
-  }
-  async embedBatch(texts) {
-    if (texts.length === 0) return [];
-    const batchSize = 64;
-    const results = [];
-    for (let i = 0; i < texts.length; i += batchSize) {
-      const batch = texts.slice(i, i + batchSize);
-      const { embeddings } = await embedMany({
-        model: this.model,
-        values: batch
-      });
-      results.push(...embeddings);
-    }
-    return results;
-  }
-};
-function cosineSimilarity(a, b) {
-  if (!a || !b || a.length === 0 || b.length === 0 || a.length !== b.length) {
-    return 0;
-  }
-  let dotProduct = 0;
-  let normA = 0;
-  let normB = 0;
-  for (let i = 0; i < a.length; i++) {
-    const ai = a[i];
-    const bi = b[i];
-    dotProduct += ai * bi;
-    normA += ai * ai;
-    normB += bi * bi;
-  }
-  if (normA === 0 || normB === 0) return 0;
-  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
-}
-
-// src/services/rag/rag.service.ts
-var RAGService = class {
-  static embeddingServiceCache = /* @__PURE__ */ new Map();
-  static getEmbeddingService(config) {
-    const key = config.ai?.apiKey || config.ai?.provider || "default";
-    if (!this.embeddingServiceCache.has(key)) {
-      this.embeddingServiceCache.set(key, new EmbeddingService(config));
-    }
-    return this.embeddingServiceCache.get(key);
-  }
-  /**
-   * Determines whether RAG indexing is enabled for a given collection.
-   */
-  static isCollectionRAGEnabled(config, col) {
-    if (isAICollection(col.slug) || col.slug.startsWith("_")) {
-      return false;
-    }
-    if (config.ai?.rag?.enabled === false) {
-      return false;
-    }
-    if (col.ai?.rag?.enabled === false) {
-      return false;
-    }
-    return true;
-  }
-  /**
-   * Indexes a single document's text fields into `_dyrected_ai_chunks`.
-   */
-  static async indexDocument(options) {
-    const { db, config, collection: collectionSlug, doc, projectId, force = false } = options;
-    const col = config.collections?.find((c) => c.slug === collectionSlug);
-    if (!col || !this.isCollectionRAGEnabled(config, col)) {
-      return { indexed: 0, skipped: 0 };
-    }
-    const docId = String(doc.id || doc._id || "");
-    if (!docId) return { indexed: 0, skipped: 0 };
-    const ragConfig = col.ai?.rag;
-    const maxChunkSize = ragConfig?.maxChunkSize || config.ai?.rag?.maxChunkSize || 1500;
-    const chunkOverlap = ragConfig?.chunkOverlap ?? config.ai?.rag?.chunkOverlap ?? 150;
-    const targetFields = ragConfig?.fields;
-    const sanitizedDoc = sanitizeDocForAI({
-      doc,
-      collectionConfig: col,
-      globalAIConfig: config.ai
-    });
-    const titleField = ragConfig?.titleField || col.admin?.useAsTitle || "title";
-    const documentTitle = String(sanitizedDoc[titleField] || sanitizedDoc.name || sanitizedDoc.headline || sanitizedDoc.question || `${col.labels?.singular || col.slug} #${docId}`);
-    const extractedFields = extractTextFromDoc(sanitizedDoc, targetFields);
-    if (extractedFields.length === 0) {
-      await this.deleteDocumentChunks({ db, collection: collectionSlug, documentId: docId, projectId });
-      return { indexed: 0, skipped: 0 };
-    }
-    let indexedCount = 0;
-    let skippedCount = 0;
-    for (const field of extractedFields) {
-      const fullFieldText = field.text;
-      const contentHash = hashContent(fullFieldText);
-      if (!force) {
-        try {
-          const existing = await db.find({
-            collection: AI_CHUNKS_COLLECTION,
-            where: {
-              projectId,
-              collection: collectionSlug,
-              documentId: docId,
-              field: field.field,
-              contentHash
-            },
-            limit: 1
-          });
-          if (existing?.docs && existing.docs.length > 0) {
-            skippedCount++;
-            continue;
-          }
-        } catch {
-        }
-      }
-      const chunks = chunkText(fullFieldText, { maxChunkSize, chunkOverlap });
-      if (chunks.length === 0) continue;
-      try {
-        const previousChunks = await db.find({
-          collection: AI_CHUNKS_COLLECTION,
-          where: {
-            projectId,
-            collection: collectionSlug,
-            documentId: docId,
-            field: field.field
-          },
-          limit: 200
-        });
-        if (previousChunks?.docs) {
-          for (const prev of previousChunks.docs) {
-            await db.delete({ collection: AI_CHUNKS_COLLECTION, id: prev.id });
-          }
-        }
-      } catch {
-      }
-      const embeddingService = this.getEmbeddingService(config);
-      const chunkTexts = chunks.map((c) => c.text);
-      const embeddings = await embeddingService.embedBatch(chunkTexts);
-      for (let i = 0; i < chunks.length; i++) {
-        const chunk = chunks[i];
-        const embedding = embeddings[i] || [];
-        const chunkRecord = {
-          id: `chunk_${docId}_${field.field}_${chunk.index}_${Math.random().toString(36).substring(2, 7)}`,
-          projectId,
-          collection: collectionSlug,
-          documentId: docId,
-          field: field.field,
-          chunkIndex: chunk.index,
-          text: chunk.text,
-          embedding,
-          tokenCount: chunk.tokenCount,
-          contentHash,
-          metadata: {
-            title: documentTitle,
-            slug: typeof doc.slug === "string" ? doc.slug : void 0,
-            status: typeof doc.status === "string" ? doc.status : void 0,
-            updatedAt: /* @__PURE__ */ new Date()
-          },
-          createdAt: /* @__PURE__ */ new Date(),
-          updatedAt: /* @__PURE__ */ new Date()
-        };
-        try {
-          await db.create({ collection: AI_CHUNKS_COLLECTION, data: chunkRecord });
-          indexedCount++;
-        } catch (err) {
-          console.error(`[dyrected/rag] Failed to save chunk ${chunkRecord.id}:`, err?.message || err);
-        }
-      }
-    }
-    return { indexed: indexedCount, skipped: skippedCount };
-  }
-  /**
-   * Deletes all vector chunks associated with a specific document.
-   */
-  static async deleteDocumentChunks(options) {
-    const { db, collection: collectionSlug, documentId, projectId } = options;
-    try {
-      const existing = await db.find({
-        collection: AI_CHUNKS_COLLECTION,
-        where: {
-          projectId,
-          collection: collectionSlug,
-          documentId
-        },
-        limit: 500
-      });
-      let count = 0;
-      if (existing?.docs) {
-        for (const doc of existing.docs) {
-          await db.delete({ collection: AI_CHUNKS_COLLECTION, id: doc.id });
-          count++;
-        }
-      }
-      return count;
-    } catch {
-      return 0;
-    }
-  }
-  /**
-   * Reindexes all documents in a specific collection.
-   */
-  static async reindexCollection(options) {
-    const { db, config, collection: collectionSlug, projectId, force = false } = options;
-    const col = config.collections?.find((c) => c.slug === collectionSlug);
-    if (!col || !this.isCollectionRAGEnabled(config, col)) {
-      return { collection: collectionSlug, totalDocs: 0, indexedChunks: 0 };
-    }
-    const result = await db.find({
-      collection: collectionSlug,
-      where: {},
-      limit: 1e3
-    });
-    const docs = result.docs || [];
-    let indexedChunks = 0;
-    for (const doc of docs) {
-      const res = await this.indexDocument({
-        db,
-        config,
-        collection: collectionSlug,
-        doc,
-        projectId,
-        force
-      });
-      indexedChunks += res.indexed;
-    }
-    return {
-      collection: collectionSlug,
-      totalDocs: docs.length,
-      indexedChunks
-    };
-  }
-  /**
-   * Reindexes all RAG-enabled collections across the project.
-   */
-  static async reindexAll(options) {
-    const { db, config, projectId, force = false } = options;
-    const results = [];
-    let totalChunks = 0;
-    const collections = (config.collections || []).filter((c) => this.isCollectionRAGEnabled(config, c));
-    for (const col of collections) {
-      try {
-        const stats = await this.reindexCollection({
-          db,
-          config,
-          collection: col.slug,
-          projectId,
-          force
-        });
-        results.push(stats);
-        totalChunks += stats.indexedChunks;
-      } catch (err) {
-        console.error(`[dyrected/rag] Reindex failed for collection "${col.slug}":`, err?.message || err);
-      }
-    }
-    return { collections: results, totalChunks };
-  }
-  /**
-   * Performs semantic vector search over project chunks with access filtering.
-   */
-  static async search(options) {
-    const {
-      db,
-      config,
-      query,
-      projectId,
-      collections: requestedCollections,
-      limit = config.ai?.rag?.topK || 4,
-      minScore = config.ai?.rag?.minScore ?? 0.45,
-      user
-    } = options;
-    if (!query || !query.trim()) {
-      return { query: "", resultsCount: 0, sources: [] };
-    }
-    const allCollections = config.collections || [];
-    const candidateCollections = [];
-    for (const col of allCollections) {
-      if (!this.isCollectionRAGEnabled(config, col)) continue;
-      if (requestedCollections && requestedCollections.length > 0 && !requestedCollections.includes(col.slug)) {
-        continue;
-      }
-      const canRead = await isAccessAllowed(config, col.access?.read, {
-        req: { user, siteId: projectId },
-        user
-      });
-      if (canRead) {
-        candidateCollections.push(col.slug);
-      }
-    }
-    if (candidateCollections.length === 0) {
-      return { query, resultsCount: 0, sources: [] };
-    }
-    const embeddingService = this.getEmbeddingService(config);
-    const queryVector = await embeddingService.embedText(query.trim());
-    let chunks;
-    try {
-      const result = await db.find({
-        collection: AI_CHUNKS_COLLECTION,
-        where: { projectId },
-        limit: 1e3
-      });
-      chunks = result.docs || [];
-    } catch {
-      chunks = [];
-    }
-    const filteredChunks = chunks.filter((c) => candidateCollections.includes(c.collection));
-    const scored = filteredChunks.map((chunk) => {
-      const score = cosineSimilarity(queryVector, chunk.embedding);
-      return {
-        chunk,
-        score
-      };
-    }).filter((item) => item.score >= minScore).sort((a, b) => b.score - a.score);
-    const topResults = scored.slice(0, limit);
-    const sources = topResults.map(({ chunk, score }) => {
-      const title = chunk.metadata?.title || `${chunk.collection} #${chunk.documentId}`;
-      const url = `/admin/collections/${chunk.collection}/${chunk.documentId}`;
-      return {
-        id: chunk.id,
-        collection: chunk.collection,
-        documentId: chunk.documentId,
-        title,
-        field: chunk.field,
-        score: Number(score.toFixed(3)),
-        text: `<untrusted_content>
-${chunk.text}
-</untrusted_content>`,
-        url,
-        metadata: chunk.metadata
-      };
-    });
-    return {
-      query,
-      resultsCount: sources.length,
-      sources
-    };
-  }
-};
-
-export {
-  normalizeFieldValue,
-  extractTextFromDoc,
-  hashContent,
-  estimateTokens,
-  chunkText,
-  getEmbeddingModel,
-  EmbeddingService,
-  cosineSimilarity,
-  RAGService
-};

package/dist/rag.service-YDGEYTXS.js

@@ -1,11 +0,0 @@
-import {
-  RAGService
-} from "./chunk-ZYYDTHF3.js";
-import "./chunk-254THFDS.js";
-import "./chunk-4RAQJVJM.js";
-import "./chunk-HRRJ3RYT.js";
-import "./chunk-27YRIGK5.js";
-import "./chunk-JSBRDJBE.js";
-export {
-  RAGService
-};