@purplesquirrel/watsonx-mcp-server 1.0.0

package/document-analyzer.js

@@ -0,0 +1,353 @@
+#!/usr/bin/env node
+/**
+ * watsonx Document Analyzer
+ * Analyzes documents from external drive using IBM watsonx.ai
+ */
+
+import { WatsonXAI } from "@ibm-cloud/watsonx-ai";
+import { IamAuthenticator } from "ibm-cloud-sdk-core";
+import fs from "fs/promises";
+import path from "path";
+
+// Configuration
+const WATSONX_API_KEY = process.env.WATSONX_API_KEY;
+const WATSONX_URL = process.env.WATSONX_URL || "https://us-south.ml.cloud.ibm.com";
+const WATSONX_SPACE_ID = process.env.WATSONX_SPACE_ID;
+
+// External drive paths
+const EXTERNAL_DRIVE = "/Volumes/Virtual Server/_NEW";
+const DOCUMENTS_PATH = `${EXTERNAL_DRIVE}/Documents`;
+const TRAINING_PATH = `${EXTERNAL_DRIVE}/Code/AI/Training`;
+
+// Initialize watsonx client
+let client = null;
+
+function getClient() {
+  if (!client && WATSONX_API_KEY) {
+    client = WatsonXAI.newInstance({
+      version: "2024-05-31",
+      serviceUrl: WATSONX_URL,
+      authenticator: new IamAuthenticator({
+        apikey: WATSONX_API_KEY,
+      }),
+    });
+  }
+  return client;
+}
+
+/**
+ * Summarize a document using watsonx Granite model
+ */
+async function summarizeDocument(text, maxLength = 200) {
+  const watsonx = getClient();
+  if (!watsonx) throw new Error("watsonx client not initialized");
+
+  // Truncate very long documents
+  const truncatedText = text.length > 4000 ? text.substring(0, 4000) + "..." : text;
+
+  const response = await watsonx.generateText({
+    modelId: "ibm/granite-3-3-8b-instruct",
+    spaceId: WATSONX_SPACE_ID,
+    input: `Summarize the following document in ${maxLength} words or less. Focus on the key points and main ideas.
+
+Document:
+${truncatedText}
+
+Summary:`,
+    parameters: {
+      max_new_tokens: 300,
+      temperature: 0.3,
+      stop_sequences: ["\n\n"],
+    },
+  });
+
+  return response.result.results?.[0]?.generated_text?.trim() || "";
+}
+
+/**
+ * Generate embeddings for a list of texts
+ */
+async function generateEmbeddings(texts) {
+  const watsonx = getClient();
+  if (!watsonx) throw new Error("watsonx client not initialized");
+
+  const response = await watsonx.embedText({
+    modelId: "ibm/slate-125m-english-rtrvr-v2",
+    spaceId: WATSONX_SPACE_ID,
+    inputs: texts,
+  });
+
+  return response.result.results?.map((r) => r.embedding) || [];
+}
+
+/**
+ * Analyze document type and extract key information
+ */
+async function analyzeDocument(text) {
+  const watsonx = getClient();
+  if (!watsonx) throw new Error("watsonx client not initialized");
+
+  const truncatedText = text.length > 3000 ? text.substring(0, 3000) + "..." : text;
+
+  const response = await watsonx.generateText({
+    modelId: "ibm/granite-3-3-8b-instruct",
+    spaceId: WATSONX_SPACE_ID,
+    input: `Analyze the following document and provide:
+1. Document Type (e.g., technical documentation, article, notes, code, etc.)
+2. Main Topics (comma-separated list of 3-5 topics)
+3. Key Entities (people, organizations, technologies mentioned)
+4. Sentiment (positive, negative, neutral)
+
+Document:
+${truncatedText}
+
+Analysis:`,
+    parameters: {
+      max_new_tokens: 300,
+      temperature: 0.2,
+    },
+  });
+
+  return response.result.results?.[0]?.generated_text?.trim() || "";
+}
+
+/**
+ * Answer questions about a document
+ */
+async function questionDocument(text, question) {
+  const watsonx = getClient();
+  if (!watsonx) throw new Error("watsonx client not initialized");
+
+  const truncatedText = text.length > 3500 ? text.substring(0, 3500) + "..." : text;
+
+  const response = await watsonx.generateText({
+    modelId: "ibm/granite-3-3-8b-instruct",
+    spaceId: WATSONX_SPACE_ID,
+    input: `Based on the following document, answer the question.
+
+Document:
+${truncatedText}
+
+Question: ${question}
+
+Answer:`,
+    parameters: {
+      max_new_tokens: 300,
+      temperature: 0.3,
+    },
+  });
+
+  return response.result.results?.[0]?.generated_text?.trim() || "";
+}
+
+/**
+ * Calculate cosine similarity between two vectors
+ */
+function cosineSimilarity(a, b) {
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+/**
+ * Search documents by semantic similarity
+ */
+async function semanticSearch(query, documentEmbeddings, documents, topK = 5) {
+  const [queryEmbedding] = await generateEmbeddings([query]);
+
+  const similarities = documentEmbeddings.map((embedding, index) => ({
+    index,
+    similarity: cosineSimilarity(queryEmbedding, embedding),
+    document: documents[index],
+  }));
+
+  similarities.sort((a, b) => b.similarity - a.similarity);
+  return similarities.slice(0, topK);
+}
+
+/**
+ * Load documents from the training catalog
+ */
+async function loadDocumentCatalog() {
+  const catalogPath = `${TRAINING_PATH}/documents_catalog.json`;
+  const data = await fs.readFile(catalogPath, "utf-8");
+  return JSON.parse(data);
+}
+
+/**
+ * Read a sample of documents
+ */
+async function readSampleDocuments(count = 10) {
+  const files = await fs.readdir(DOCUMENTS_PATH);
+  const txtFiles = files.filter((f) => f.endsWith(".txt")).slice(0, count);
+
+  const documents = [];
+  for (const file of txtFiles) {
+    try {
+      const content = await fs.readFile(`${DOCUMENTS_PATH}/${file}`, "utf-8");
+      documents.push({
+        filename: file,
+        content: content.substring(0, 5000), // Limit size
+      });
+    } catch (err) {
+      // Skip files that can't be read
+    }
+  }
+  return documents;
+}
+
+// Main execution
+async function main() {
+  const command = process.argv[2];
+  const arg = process.argv[3];
+
+  console.log("╔══════════════════════════════════════════════════════════════╗");
+  console.log("║           watsonx Document Analyzer                          ║");
+  console.log("║           Powered by IBM Granite 3.3                         ║");
+  console.log("╚══════════════════════════════════════════════════════════════╝");
+  console.log("");
+
+  if (!WATSONX_API_KEY || !WATSONX_SPACE_ID) {
+    console.error("Error: WATSONX_API_KEY and WATSONX_SPACE_ID must be set");
+    process.exit(1);
+  }
+
+  switch (command) {
+    case "catalog": {
+      console.log("📚 Loading document catalog...");
+      const catalog = await loadDocumentCatalog();
+      console.log(`   Total documents: ${catalog.total}`);
+      console.log(`   Sample documents:`);
+      catalog.documents.slice(0, 10).forEach((doc) => {
+        console.log(`   - ${doc.filename} (${doc.type}, ${doc.size_bytes} bytes)`);
+      });
+      break;
+    }
+
+    case "summarize": {
+      const filename = arg || "1002519.txt";
+      console.log(`📝 Summarizing: ${filename}`);
+      try {
+        const content = await fs.readFile(`${DOCUMENTS_PATH}/${filename}`, "utf-8");
+        console.log(`   Document length: ${content.length} characters`);
+        console.log("\n   Generating summary with watsonx...\n");
+        const summary = await summarizeDocument(content);
+        console.log("   Summary:");
+        console.log("   " + "-".repeat(60));
+        console.log("   " + summary.split("\n").join("\n   "));
+        console.log("   " + "-".repeat(60));
+      } catch (err) {
+        console.error(`   Error: ${err.message}`);
+      }
+      break;
+    }
+
+    case "analyze": {
+      const filename = arg || "1002519.txt";
+      console.log(`🔍 Analyzing: ${filename}`);
+      try {
+        const content = await fs.readFile(`${DOCUMENTS_PATH}/${filename}`, "utf-8");
+        console.log(`   Document length: ${content.length} characters`);
+        console.log("\n   Analyzing with watsonx...\n");
+        const analysis = await analyzeDocument(content);
+        console.log("   Analysis:");
+        console.log("   " + "-".repeat(60));
+        console.log("   " + analysis.split("\n").join("\n   "));
+        console.log("   " + "-".repeat(60));
+      } catch (err) {
+        console.error(`   Error: ${err.message}`);
+      }
+      break;
+    }
+
+    case "question": {
+      const filename = arg;
+      const question = process.argv[4];
+      if (!filename || !question) {
+        console.log("Usage: document-analyzer.js question <filename> '<question>'");
+        process.exit(1);
+      }
+      console.log(`❓ Asking question about: ${filename}`);
+      console.log(`   Question: ${question}`);
+      try {
+        const content = await fs.readFile(`${DOCUMENTS_PATH}/${filename}`, "utf-8");
+        console.log("\n   Answering with watsonx...\n");
+        const answer = await questionDocument(content, question);
+        console.log("   Answer:");
+        console.log("   " + "-".repeat(60));
+        console.log("   " + answer.split("\n").join("\n   "));
+        console.log("   " + "-".repeat(60));
+      } catch (err) {
+        console.error(`   Error: ${err.message}`);
+      }
+      break;
+    }
+
+    case "embed": {
+      console.log("🔢 Generating embeddings for sample documents...");
+      const docs = await readSampleDocuments(5);
+      console.log(`   Loaded ${docs.length} documents`);
+
+      const texts = docs.map((d) => d.content.substring(0, 500));
+      console.log("   Generating embeddings with watsonx...");
+      const embeddings = await generateEmbeddings(texts);
+
+      console.log("\n   Embeddings generated:");
+      docs.forEach((doc, i) => {
+        console.log(`   - ${doc.filename}: ${embeddings[i]?.length || 0} dimensions`);
+      });
+      break;
+    }
+
+    case "search": {
+      const query = arg;
+      if (!query) {
+        console.log("Usage: document-analyzer.js search '<query>'");
+        process.exit(1);
+      }
+      console.log(`🔍 Semantic search: "${query}"`);
+      console.log("   Loading sample documents...");
+      const docs = await readSampleDocuments(20);
+      console.log(`   Loaded ${docs.length} documents`);
+
+      console.log("   Generating embeddings...");
+      const texts = docs.map((d) => d.content.substring(0, 500));
+      const embeddings = await generateEmbeddings(texts);
+
+      console.log("   Searching...\n");
+      const results = await semanticSearch(query, embeddings, docs, 5);
+
+      console.log("   Top results:");
+      results.forEach((r, i) => {
+        console.log(`   ${i + 1}. ${r.document.filename} (similarity: ${r.similarity.toFixed(4)})`);
+        console.log(`      ${r.document.content.substring(0, 100).replace(/\n/g, " ")}...`);
+      });
+      break;
+    }
+
+    default:
+      console.log("Usage: document-analyzer.js <command> [args]");
+      console.log("");
+      console.log("Commands:");
+      console.log("  catalog              - List documents from training catalog");
+      console.log("  summarize [file]     - Summarize a document");
+      console.log("  analyze [file]       - Analyze document type and topics");
+      console.log("  question <file> <q>  - Ask a question about a document");
+      console.log("  embed                - Generate embeddings for sample docs");
+      console.log("  search <query>       - Semantic search across documents");
+      console.log("");
+      console.log("Examples:");
+      console.log("  document-analyzer.js summarize 1002519.txt");
+      console.log("  document-analyzer.js analyze 1002519.txt");
+      console.log("  document-analyzer.js question 1002519.txt 'What is this about?'");
+      console.log("  document-analyzer.js search 'IBM Cloud Satellite'");
+  }
+}
+
+main().catch(console.error);

package/embedding-index.js

@@ -0,0 +1,318 @@
+#!/usr/bin/env node
+/**
+ * watsonx Embedding Index
+ * Builds and queries a persistent embedding index for RAG
+ */
+
+import { WatsonXAI } from "@ibm-cloud/watsonx-ai";
+import { IamAuthenticator } from "ibm-cloud-sdk-core";
+import fs from "fs/promises";
+import path from "path";
+
+// Configuration
+const WATSONX_API_KEY = process.env.WATSONX_API_KEY;
+const WATSONX_URL = process.env.WATSONX_URL || "https://us-south.ml.cloud.ibm.com";
+const WATSONX_SPACE_ID = process.env.WATSONX_SPACE_ID;
+
+// Paths
+const EXTERNAL_DRIVE = "/Volumes/Virtual Server/_NEW";
+const DOCUMENTS_PATH = `${EXTERNAL_DRIVE}/Documents`;
+const INDEX_PATH = "/Users/matthewkarsten/watsonx-mcp-server/embeddings-index.json";
+
+let client = null;
+
+function getClient() {
+  if (!client && WATSONX_API_KEY) {
+    client = WatsonXAI.newInstance({
+      version: "2024-05-31",
+      serviceUrl: WATSONX_URL,
+      authenticator: new IamAuthenticator({
+        apikey: WATSONX_API_KEY,
+      }),
+    });
+  }
+  return client;
+}
+
+/**
+ * Generate embeddings for texts (batch)
+ */
+async function generateEmbeddings(texts) {
+  const watsonx = getClient();
+  const response = await watsonx.embedText({
+    modelId: "ibm/slate-125m-english-rtrvr-v2",
+    spaceId: WATSONX_SPACE_ID,
+    inputs: texts,
+  });
+  return response.result.results?.map((r) => r.embedding) || [];
+}
+
+/**
+ * Calculate cosine similarity
+ */
+function cosineSimilarity(a, b) {
+  let dot = 0, normA = 0, normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+/**
+ * Load or create index
+ */
+async function loadIndex() {
+  try {
+    const data = await fs.readFile(INDEX_PATH, "utf-8");
+    return JSON.parse(data);
+  } catch {
+    return { documents: [], embeddings: [], metadata: { created: new Date().toISOString(), count: 0 } };
+  }
+}
+
+/**
+ * Save index
+ */
+async function saveIndex(index) {
+  index.metadata.updated = new Date().toISOString();
+  index.metadata.count = index.documents.length;
+  await fs.writeFile(INDEX_PATH, JSON.stringify(index, null, 2));
+}
+
+/**
+ * Build index from documents
+ */
+async function buildIndex(maxDocs = 100) {
+  console.log("📚 Building embedding index...");
+  console.log(`   Source: ${DOCUMENTS_PATH}`);
+  console.log(`   Max documents: ${maxDocs}`);
+
+  const files = await fs.readdir(DOCUMENTS_PATH);
+  const txtFiles = files.filter(f => f.endsWith(".txt")).slice(0, maxDocs);
+
+  console.log(`   Found ${txtFiles.length} text files`);
+
+  const index = { documents: [], embeddings: [], metadata: { created: new Date().toISOString() } };
+
+  // Process in batches of 10
+  const batchSize = 10;
+  for (let i = 0; i < txtFiles.length; i += batchSize) {
+    const batch = txtFiles.slice(i, i + batchSize);
+    const texts = [];
+    const docs = [];
+
+    for (const file of batch) {
+      try {
+        const content = await fs.readFile(`${DOCUMENTS_PATH}/${file}`, "utf-8");
+        const truncated = content.substring(0, 500); // First 500 chars for embedding
+        texts.push(truncated);
+        docs.push({
+          filename: file,
+          preview: truncated.substring(0, 200).replace(/\n/g, " "),
+          length: content.length,
+        });
+      } catch {
+        // Skip unreadable files
+      }
+    }
+
+    if (texts.length > 0) {
+      console.log(`   Processing batch ${Math.floor(i/batchSize) + 1}/${Math.ceil(txtFiles.length/batchSize)}...`);
+      const embeddings = await generateEmbeddings(texts);
+
+      for (let j = 0; j < docs.length; j++) {
+        index.documents.push(docs[j]);
+        index.embeddings.push(embeddings[j]);
+      }
+    }
+  }
+
+  await saveIndex(index);
+  console.log(`\n✅ Index built with ${index.documents.length} documents`);
+  console.log(`   Saved to: ${INDEX_PATH}`);
+
+  return index;
+}
+
+/**
+ * Query the index
+ */
+async function queryIndex(query, topK = 5) {
+  console.log(`🔍 Searching: "${query}"`);
+
+  const index = await loadIndex();
+  if (index.documents.length === 0) {
+    console.log("   Index is empty. Run 'build' first.");
+    return [];
+  }
+
+  console.log(`   Searching ${index.documents.length} documents...`);
+
+  const [queryEmbedding] = await generateEmbeddings([query]);
+
+  const results = index.embeddings.map((emb, i) => ({
+    ...index.documents[i],
+    similarity: cosineSimilarity(queryEmbedding, emb),
+  }));
+
+  results.sort((a, b) => b.similarity - a.similarity);
+  return results.slice(0, topK);
+}
+
+/**
+ * RAG: Retrieve and Generate
+ */
+async function ragQuery(question, topK = 3) {
+  console.log(`💡 RAG Query: "${question}"`);
+
+  // Retrieve relevant documents
+  const results = await queryIndex(question, topK);
+
+  if (results.length === 0) {
+    console.log("   No documents found. Build index first.");
+    return;
+  }
+
+  // Load full content of top documents
+  const contexts = [];
+  for (const result of results) {
+    try {
+      const content = await fs.readFile(`${DOCUMENTS_PATH}/${result.filename}`, "utf-8");
+      contexts.push({
+        filename: result.filename,
+        content: content.substring(0, 1500),
+        similarity: result.similarity,
+      });
+    } catch {
+      // Skip
+    }
+  }
+
+  console.log(`\n   Retrieved ${contexts.length} relevant documents:`);
+  contexts.forEach((c, i) => {
+    console.log(`   ${i + 1}. ${c.filename} (similarity: ${c.similarity.toFixed(4)})`);
+  });
+
+  // Generate answer using watsonx
+  const watsonx = getClient();
+  const contextText = contexts.map(c => `[${c.filename}]\n${c.content}`).join("\n\n---\n\n");
+
+  console.log("\n   Generating answer with Granite 3.3...\n");
+
+  const response = await watsonx.generateText({
+    modelId: "ibm/granite-3-3-8b-instruct",
+    spaceId: WATSONX_SPACE_ID,
+    input: `You are a helpful assistant. Answer the question based on the provided context documents. If the answer is not in the context, say so.
+
+Context Documents:
+${contextText}
+
+Question: ${question}
+
+Answer:`,
+    parameters: {
+      max_new_tokens: 400,
+      temperature: 0.3,
+    },
+  });
+
+  const answer = response.result.results?.[0]?.generated_text?.trim() || "No answer generated";
+
+  console.log("   " + "─".repeat(60));
+  console.log("   Answer:");
+  console.log("   " + answer.split("\n").join("\n   "));
+  console.log("   " + "─".repeat(60));
+  console.log("\n   Sources:");
+  contexts.forEach(c => console.log(`   - ${c.filename}`));
+}
+
+/**
+ * Show index stats
+ */
+async function showStats() {
+  const index = await loadIndex();
+  console.log("📊 Index Statistics");
+  console.log("   " + "─".repeat(40));
+  console.log(`   Documents indexed: ${index.documents.length}`);
+  console.log(`   Created: ${index.metadata.created || "N/A"}`);
+  console.log(`   Updated: ${index.metadata.updated || "N/A"}`);
+  console.log(`   Index file: ${INDEX_PATH}`);
+
+  if (index.documents.length > 0) {
+    console.log("\n   Sample documents:");
+    index.documents.slice(0, 5).forEach(d => {
+      console.log(`   - ${d.filename} (${d.length} chars)`);
+    });
+  }
+}
+
+// Main
+async function main() {
+  const command = process.argv[2];
+  const arg = process.argv[3];
+
+  console.log("╔══════════════════════════════════════════════════════════════╗");
+  console.log("║           watsonx Embedding Index & RAG                      ║");
+  console.log("║           Powered by IBM Granite 3.3 + Slate                 ║");
+  console.log("╚══════════════════════════════════════════════════════════════╝");
+  console.log("");
+
+  if (!WATSONX_API_KEY || !WATSONX_SPACE_ID) {
+    console.error("Error: WATSONX_API_KEY and WATSONX_SPACE_ID must be set");
+    process.exit(1);
+  }
+
+  switch (command) {
+    case "build": {
+      const maxDocs = parseInt(arg) || 100;
+      await buildIndex(maxDocs);
+      break;
+    }
+
+    case "search": {
+      if (!arg) {
+        console.log("Usage: embedding-index.js search '<query>'");
+        process.exit(1);
+      }
+      const results = await queryIndex(arg, 10);
+      console.log("\n   Top results:");
+      results.forEach((r, i) => {
+        console.log(`   ${i + 1}. ${r.filename} (${r.similarity.toFixed(4)})`);
+        console.log(`      ${r.preview.substring(0, 80)}...`);
+      });
+      break;
+    }
+
+    case "rag": {
+      if (!arg) {
+        console.log("Usage: embedding-index.js rag '<question>'");
+        process.exit(1);
+      }
+      await ragQuery(arg);
+      break;
+    }
+
+    case "stats": {
+      await showStats();
+      break;
+    }
+
+    default:
+      console.log("Usage: embedding-index.js <command> [args]");
+      console.log("");
+      console.log("Commands:");
+      console.log("  build [count]     - Build embedding index (default: 100 docs)");
+      console.log("  search <query>    - Search the index");
+      console.log("  rag <question>    - RAG: Retrieve docs and generate answer");
+      console.log("  stats             - Show index statistics");
+      console.log("");
+      console.log("Examples:");
+      console.log("  embedding-index.js build 200");
+      console.log("  embedding-index.js search 'IBM Cloud'");
+      console.log("  embedding-index.js rag 'How do I set up AWS for Satellite?'");
+  }
+}
+
+main().catch(console.error);