@devilsdev/rag-pipeline-utils 2.1.12 → 2.2.1

package/src/dx/index.js

@@ -0,0 +1,40 @@
+/**
+ * Developer Experience (DX) Enhancements
+ *
+ * This module provides advanced developer tools and interfaces for the RAG pipeline:
+ * - Visual Pipeline Builder: Web-based drag-and-drop interface
+ * - Real-time Debugging: Live pipeline inspection and debugging
+ * - Performance Profiler: Detailed bottleneck analysis
+ * - Integration Templates: Pre-built connectors for popular services
+ */
+
+const VisualPipelineBuilder = require("./visual-pipeline-builder");
+// eslint-disable-line global-require
+const RealtimeDebugger = require("./realtime-debugger");
+// eslint-disable-line global-require
+const PerformanceProfiler = require("./performance-profiler");
+// eslint-disable-line global-require
+const IntegrationTemplates = require("./integration-templates");
+// eslint-disable-line global-require
+
+module.exports = {
+  VisualPipelineBuilder,
+  RealtimeDebugger,
+  PerformanceProfiler,
+  IntegrationTemplates,
+
+  // Convenience factory methods
+  createVisualBuilder: (_options = {}) => new VisualPipelineBuilder(_options),
+  createDebugger: (_options = {}) => new RealtimeDebugger(_options),
+  createProfiler: (_options = {}) => new PerformanceProfiler(_options),
+  getTemplates: () => IntegrationTemplates.getAllTemplates(),
+
+  // DX utilities
+  getDXInfo: () => ({
+    version: "1.0.0",
+    components: ["visual-builder", "debugger", "profiler", "templates"],
+    status: "active",
+  }),
+};
+
+// Ensure module.exports is properly defined

package/src/dx/integration-templates.js

@@ -0,0 +1,665 @@
+/**
+ * Integration Templates
+ *
+ * Pre-built connectors and templates for popular services and platforms.
+ * Provides ready-to-use configurations for common RAG pipeline integrations.
+ */
+
+const fs = require("fs").promises; // eslint-disable-line global-require
+const path = require("path"); // eslint-disable-line global-require
+
+/**
+ * Normalize template shape and keep back-compat.
+ * Google-style: small helpers, explicit transformation.
+ */
+function normalize(t) {
+  return {
+    id: t.id,
+    name: t.name,
+    description: t.description,
+    type: t.type ?? t._type,
+    config: t.config ?? t._config,
+    generateIntegrationCode: t.generateIntegrationCode,
+    category: t.category,
+    dependencies: t.dependencies,
+    usage: t.usage,
+  };
+}
+
+function addBackCompat(template) {
+  if (!Object.getOwnPropertyDescriptor(template, "_type")) {
+    Object.defineProperty(template, "_type", {
+      get() {
+        return template.type;
+      },
+    });
+  }
+  if (!Object.getOwnPropertyDescriptor(template, "_config")) {
+    Object.defineProperty(template, "_config", {
+      get() {
+        return template.config;
+      },
+    });
+  }
+  return template;
+}
+
+class IntegrationTemplates {
+  constructor() {
+    this.templates = new Map();
+    this.categories = new Map();
+
+    this.initializeTemplates();
+  }
+
+  /**
+   * Initialize built-in integration templates
+   */
+  initializeTemplates() {
+    // Data Source Templates
+    this.addTemplate("confluence", {
+      name: "Atlassian Confluence",
+      category: "data-sources",
+      description: "Connect to Confluence spaces and pages",
+      type: "loader",
+      config: {
+        baseUrl: {
+          _type: "string",
+          required: true,
+          description: "Confluence base URL",
+        },
+        username: {
+          _type: "string",
+          required: true,
+          description: "Username or email",
+        },
+        apiToken: {
+          _type: "string",
+          required: true,
+          secret: true,
+          description: "API token",
+        },
+        spaceKey: {
+          _type: "string",
+          required: false,
+          description: "Specific space key (optional)",
+        },
+        pageLimit: {
+          _type: "number",
+          default: 100,
+          description: "Maximum pages to fetch",
+        },
+      },
+      dependencies: ["axios"],
+      usage: "Load documents from Confluence spaces and pages",
+    });
+
+    this.addTemplate("notion", {
+      name: "Notion Database",
+      category: "data-sources",
+      description: "Connect to Notion databases and pages",
+      type: "loader",
+      config: {
+        apiToken: {
+          _type: "string",
+          required: true,
+          secret: true,
+          description: "Notion API token",
+        },
+        databaseId: {
+          _type: "string",
+          required: true,
+          description: "Database ID to query",
+        },
+        filter: {
+          _type: "object",
+          required: false,
+          description: "Query filter (optional)",
+        },
+        pageSize: {
+          _type: "number",
+          default: 100,
+          description: "Results per page",
+        },
+      },
+      dependencies: ["@notionhq/client"],
+      usage: "Load documents from Notion databases",
+    });
+
+    this.addTemplate("sharepoint", {
+      name: "Microsoft SharePoint",
+      category: "data-sources",
+      description: "Connect to SharePoint document libraries",
+      type: "loader",
+      config: {
+        siteUrl: {
+          _type: "string",
+          required: true,
+          description: "SharePoint site URL",
+        },
+        clientId: {
+          _type: "string",
+          required: true,
+          description: "Azure app client ID",
+        },
+        clientSecret: {
+          _type: "string",
+          required: true,
+          secret: true,
+          description: "Azure app client secret",
+        },
+        tenantId: {
+          _type: "string",
+          required: true,
+          description: "Azure tenant ID",
+        },
+        libraryName: {
+          _type: "string",
+          default: "Documents",
+          description: "Document library name",
+        },
+      },
+      dependencies: ["@microsoft/microsoft-graph-client", "@azure/identity"],
+      usage: "Load documents from SharePoint libraries",
+    });
+
+    // Vector Database Templates
+    this.addTemplate("pinecone", {
+      name: "Pinecone Vector Database",
+      category: "vector-stores",
+      description: "Connect to Pinecone for vector storage and retrieval",
+      type: "retriever",
+      config: {
+        apiKey: {
+          _type: "string",
+          required: true,
+          secret: true,
+          description: "Pinecone API key",
+        },
+        environment: {
+          _type: "string",
+          required: true,
+          description: "Pinecone environment",
+        },
+        indexName: {
+          _type: "string",
+          required: true,
+          description: "Index name",
+        },
+        topK: {
+          _type: "number",
+          default: 5,
+          description: "Number of results to return",
+        },
+        includeMetadata: {
+          _type: "boolean",
+          default: true,
+          description: "Include metadata in results",
+        },
+      },
+      dependencies: ["@pinecone-database/pinecone"],
+      usage: "Store and retrieve vectors from Pinecone",
+    });
+
+    this.addTemplate("weaviate", {
+      name: "Weaviate Vector Database",
+      category: "vector-stores",
+      description: "Connect to Weaviate for semantic search",
+      type: "retriever",
+      config: {
+        host: {
+          _type: "string",
+          required: true,
+          description: "Weaviate host URL",
+        },
+        apiKey: {
+          _type: "string",
+          required: false,
+          secret: true,
+          description: "API key (if required)",
+        },
+        className: {
+          _type: "string",
+          required: true,
+          description: "Weaviate class name",
+        },
+        limit: {
+          _type: "number",
+          default: 5,
+          description: "Number of results to return",
+        },
+      },
+      dependencies: ["weaviate-ts-client"],
+      usage: "Semantic search with Weaviate",
+    });
+
+    // LLM Provider Templates
+    this.addTemplate("openai-gpt4", {
+      name: "OpenAI GPT-4",
+      category: "llm-providers",
+      description: "OpenAI GPT-4 language model integration",
+      type: "llm",
+      config: {
+        apiKey: {
+          _type: "string",
+          required: true,
+          secret: true,
+          description: "OpenAI API key",
+        },
+        model: { _type: "string", default: "gpt-4", description: "Model name" },
+        temperature: {
+          _type: "number",
+          default: 0.7,
+          description: "Sampling temperature",
+        },
+        maxTokens: {
+          _type: "number",
+          default: 1000,
+          description: "Maximum tokens to generate",
+        },
+        systemPrompt: {
+          _type: "string",
+          required: false,
+          description: "System prompt",
+        },
+      },
+      dependencies: ["openai"],
+      usage: "Generate responses using OpenAI GPT-4",
+    });
+
+    this.addTemplate("anthropic-claude", {
+      name: "Anthropic Claude",
+      category: "llm-providers",
+      description: "Anthropic Claude language model integration",
+      type: "llm",
+      config: {
+        apiKey: {
+          _type: "string",
+          required: true,
+          secret: true,
+          description: "Anthropic API key",
+        },
+        model: {
+          _type: "string",
+          default: "claude-3-sonnet-20240229",
+          description: "Model name",
+        },
+        maxTokens: {
+          _type: "number",
+          default: 1000,
+          description: "Maximum tokens to generate",
+        },
+        systemPrompt: {
+          _type: "string",
+          required: false,
+          description: "System prompt",
+        },
+      },
+      dependencies: ["@anthropic-ai/sdk"],
+      usage: "Generate responses using Anthropic Claude",
+    });
+
+    // Embedding Provider Templates
+    this.addTemplate("openai-embeddings", {
+      name: "OpenAI Embeddings",
+      category: "embedding-providers",
+      description: "OpenAI text embedding models",
+      type: "embedder",
+      config: {
+        apiKey: {
+          _type: "string",
+          required: true,
+          secret: true,
+          description: "OpenAI API key",
+        },
+        model: {
+          _type: "string",
+          default: "text-embedding-3-small",
+          description: "Embedding model",
+        },
+        batchSize: {
+          _type: "number",
+          default: 100,
+          description: "Batch size for processing",
+        },
+      },
+      dependencies: ["openai"],
+      usage: "Generate text embeddings using OpenAI",
+    });
+
+    // Monitoring & Analytics Templates
+    this.addTemplate("datadog", {
+      name: "Datadog Monitoring",
+      category: "monitoring",
+      description: "Send metrics and logs to Datadog",
+      type: "monitor",
+      config: {
+        apiKey: {
+          _type: "string",
+          required: true,
+          secret: true,
+          description: "Datadog API key",
+        },
+        site: {
+          _type: "string",
+          default: "datadoghq.com",
+          description: "Datadog site",
+        },
+        service: {
+          _type: "string",
+          default: "rag-pipeline",
+          description: "Service name",
+        },
+        environment: {
+          _type: "string",
+          default: "production",
+          description: "Environment",
+        },
+      },
+      dependencies: ["node-statsd", "axios"],
+      usage: "Monitor pipeline performance with Datadog",
+    });
+
+    // Initialize categories
+    this.categories.set("data-sources", {
+      name: "Data Sources",
+      description:
+        "Connectors for various data sources and content management systems",
+    });
+
+    this.categories.set("vector-stores", {
+      name: "Vector Databases",
+      description: "Vector database integrations for semantic search",
+    });
+
+    this.categories.set("llm-providers", {
+      name: "LLM Providers",
+      description: "Language model provider integrations",
+    });
+
+    this.categories.set("embedding-providers", {
+      name: "Embedding Providers",
+      description: "Text embedding service integrations",
+    });
+
+    this.categories.set("monitoring", {
+      name: "Monitoring & Analytics",
+      description: "Monitoring, logging, and analytics integrations",
+    });
+  }
+
+  /**
+   * Add a new template
+   */
+  addTemplate(id, template) {
+    this.templates.set(id, {
+      id,
+      ...template,
+      created: new Date().toISOString(),
+    });
+  }
+
+  /**
+   * Get template by ID
+   */
+  getTemplate(id) {
+    const t = this.templates.get(id);
+    return t ? addBackCompat(normalize(t)) : undefined;
+  }
+
+  /**
+   * Get all templates
+   */
+  getAllTemplates() {
+    return Array.from(this.templates.values());
+  }
+
+  /**
+   * Get templates by category
+   */
+  getTemplatesByCategory(category) {
+    return Array.from(this.templates.values()).filter(
+      (template) => template.category === category,
+    );
+  }
+
+  /**
+   * Get templates by type
+   */
+  getTemplatesByType(type) {
+    return Array.from(this.templates.values())
+      .filter((t) => (t.type ?? t._type) === type)
+      .map((t) => addBackCompat(normalize(t)));
+  }
+
+  /**
+   * Search templates
+   */
+  searchTemplates(query) {
+    const searchTerm = query.toLowerCase();
+    return Array.from(this.templates.values()).filter(
+      (template) =>
+        template.name.toLowerCase().includes(searchTerm) ||
+        template.description.toLowerCase().includes(searchTerm) ||
+        template.category.toLowerCase().includes(searchTerm),
+    );
+  }
+
+  /**
+   * Generate integration code from template
+   */
+  generateIntegration(templateId, userConfig = {}) {
+    const template = this.templates.get(templateId);
+    if (!template) {
+      throw new Error(`Template ${templateId} not found`);
+    }
+
+    // Validate required config
+    const missingRequired = [];
+    Object.entries(template.config).forEach(([key, configDef]) => {
+      if (configDef.required && !userConfig[key]) {
+        missingRequired.push(key);
+      }
+    });
+
+    if (missingRequired.length > 0) {
+      throw new Error(
+        `Missing required configuration: ${missingRequired.join(", ")}`,
+      );
+    }
+
+    // Merge with defaults
+    const finalConfig = {};
+    Object.entries(template.config).forEach(([key, configDef]) => {
+      finalConfig[key] =
+        userConfig[key] !== undefined ? userConfig[key] : configDef.default;
+    });
+
+    return {
+      name: template.name,
+      type: template.type,
+      config: finalConfig,
+      dependencies: template.dependencies || [],
+      usage: template.usage,
+      setupInstructions: this.generateSetupInstructions(template, finalConfig),
+    };
+  }
+
+  /**
+   * Generate setup instructions
+   */
+  generateSetupInstructions(template, _config) {
+    const instructions = [];
+
+    instructions.push(`# ${template.name} Integration Setup`);
+    instructions.push("");
+    instructions.push("## 1. Install Dependencies");
+    instructions.push("```bash");
+    instructions.push(`npm install ${template.dependencies.join(" ")}`);
+    instructions.push("```");
+    instructions.push("");
+    instructions.push("## 2. Configuration");
+    instructions.push("```javascript");
+    instructions.push(`const _config = ${JSON.stringify(_config, null, 2)};`);
+    instructions.push("```");
+    instructions.push("");
+    instructions.push("## 3. Usage");
+    instructions.push(`${template.usage}`);
+
+    return instructions.join("\n");
+  }
+
+  /**
+   * Get all categories
+   */
+  getAllCategories() {
+    return Array.from(this.categories.values());
+  }
+
+  /**
+   * Export template as file
+   */
+  async exportTemplate(templateId, outputDir = "./templates") {
+    const template = this.templates.get(templateId);
+    if (!template) {
+      throw new Error(`Template ${templateId} not found`);
+    }
+
+    await fs.mkdir(outputDir, { recursive: true });
+
+    const _filePath = path.join(outputDir, `${templateId}.json`);
+    await fs.writeFile(_filePath, JSON.stringify(template, null, 2));
+
+    return _filePath;
+  }
+
+  /**
+   * Import template from file
+   */
+  async importTemplate(_filePath) {
+    const data = await fs.readFile(_filePath, "utf8");
+    const template = JSON.parse(data);
+
+    if (!template.id) {
+      throw new Error("Template must have an id field");
+    }
+
+    this.addTemplate(template.id, template);
+    return template.id;
+  }
+
+  /**
+   * Get template statistics
+   */
+  getStatistics() {
+    const templates = Array.from(this.templates.values());
+    const categories = {};
+    const types = {};
+
+    templates.forEach((template) => {
+      categories[template.category] = (categories[template.category] || 0) + 1;
+      types[template.type] = (types[template.type] || 0) + 1;
+    });
+
+    return {
+      totalTemplates: templates.length,
+      categoryCounts: categories,
+      typeCounts: types,
+      totalCategories: this.categories.size,
+    };
+  }
+
+  /**
+   * Get template statistics (alias for tests)
+   */
+  getTemplateStats() {
+    return this.getStatistics();
+  }
+}
+
+// Create a singleton instance for module-level exports
+const defaultInstance = new IntegrationTemplates();
+
+// Module-level _config object with templates
+const _config = {
+  llm: {
+    "openai-gpt4": {
+      name: "OpenAI GPT-4",
+      category: "llm-providers",
+      description: "OpenAI GPT-4 language model integration",
+      type: "llm",
+      config: {
+        apiKey: {
+          _type: "string",
+          required: true,
+          secret: true,
+          description: "OpenAI API key",
+        },
+        model: { _type: "string", default: "gpt-4", description: "Model name" },
+        temperature: {
+          _type: "number",
+          default: 0.7,
+          description: "Sampling temperature",
+        },
+        maxTokens: {
+          _type: "number",
+          default: 1000,
+          description: "Maximum tokens to generate",
+        },
+      },
+      dependencies: ["openai"],
+      usage: "Generate responses using OpenAI GPT-4",
+    },
+  },
+  loader: {
+    confluence: {
+      name: "Atlassian Confluence",
+      category: "data-sources",
+      description: "Connect to Confluence spaces and pages",
+      type: "loader",
+      config: {
+        baseUrl: {
+          _type: "string",
+          required: true,
+          description: "Confluence base URL",
+        },
+        username: {
+          _type: "string",
+          required: true,
+          description: "Username or email",
+        },
+        apiToken: {
+          _type: "string",
+          required: true,
+          secret: true,
+          description: "API token",
+        },
+      },
+      dependencies: ["axios"],
+      usage: "Load documents from Confluence spaces and pages",
+    },
+  },
+};
+
+// Module-level functions
+function generateIntegration(templateKey, userConfig = {}) {
+  return defaultInstance.generateIntegration(templateKey, userConfig);
+}
+
+function getTemplateStats() {
+  return defaultInstance.getTemplateStats();
+}
+
+function addTemplate(key, def) {
+  return defaultInstance.addTemplate(key, def);
+}
+
+// Export both class and module-level API
+module.exports = IntegrationTemplates;
+module.exports._config = _config;
+module.exports.generateIntegration = generateIntegration;
+module.exports.getTemplateStats = getTemplateStats;
+module.exports.addTemplate = addTemplate;
+
+// CJS+ESM interop pattern
+module.exports.default = module.exports;