@duckmind/deepquark-darwin-arm64 0.9.83 → 0.9.90

package/.deepquark/skills/bundled/knowledge-graph/server/compare-embeddings.ts

@@ -0,0 +1,175 @@
+#!/usr/bin/env bun
+/**
+ * Compare Ollama mxbai-embed-large vs OpenAI text-embedding-3-small
+ */
+
+const OLLAMA_HOST = 'http://10.0.0.150:11434';
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+
+// Test sentences for semantic similarity
+const TEST_PAIRS = [
+  {
+    a: 'The cat sat on the mat',
+    b: 'A feline rested on the rug',
+    label: 'Cat/Feline synonyms',
+    expected: 'high',
+  },
+  {
+    a: 'Alice is a software engineer',
+    b: 'Alice works as a developer',
+    label: 'Job title synonyms',
+    expected: 'high',
+  },
+  {
+    a: 'The weather is sunny today',
+    b: 'I love programming in TypeScript',
+    label: 'Weather vs Programming',
+    expected: 'low',
+  },
+  {
+    a: 'Machine learning is a subset of AI',
+    b: 'Deep learning uses neural networks',
+    label: 'ML/DL related',
+    expected: 'high',
+  },
+  {
+    a: 'Paris is the capital of France',
+    b: 'Tokyo is in Japan',
+    label: 'Different countries',
+    expected: 'medium',
+  },
+  {
+    a: 'The stock market crashed yesterday',
+    b: 'My favorite color is blue',
+    label: 'Finance vs Personal',
+    expected: 'low',
+  },
+];
+
+function cosineSimilarity(a: number[], b: number[]): number {
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+async function getOllamaEmbedding(text: string): Promise<{ embedding: number[]; ms: number }> {
+  const start = Date.now();
+  const res = await fetch(`${OLLAMA_HOST}/api/embeddings`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ model: 'mxbai-embed-large', prompt: text }),
+  });
+  const data = (await res.json()) as { embedding: number[] };
+  return { embedding: data.embedding, ms: Date.now() - start };
+}
+
+async function getOpenAIEmbedding(text: string): Promise<{ embedding: number[]; ms: number }> {
+  const start = Date.now();
+  const res = await fetch('https://api.openai.com/v1/embeddings', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${OPENAI_API_KEY}`,
+    },
+    body: JSON.stringify({ model: 'text-embedding-3-small', input: text }),
+  });
+  const data = (await res.json()) as { data: { embedding: number[] }[] };
+  return { embedding: data.data[0].embedding, ms: Date.now() - start };
+}
+
+async function main() {
+  if (!OPENAI_API_KEY) {
+    console.error('OPENAI_API_KEY not set');
+    process.exit(1);
+  }
+
+  console.log('==============================================');
+  console.log('Embedding Model Comparison');
+  console.log('Ollama mxbai-embed-large vs OpenAI text-embedding-3-small');
+  console.log('==============================================\n');
+
+  // Get dimensions
+  const ollamaTest = await getOllamaEmbedding('test');
+  const openaiTest = await getOpenAIEmbedding('test');
+
+  console.log(`Ollama mxbai-embed-large: ${ollamaTest.embedding.length} dimensions`);
+  console.log(`OpenAI text-embedding-3-small: ${openaiTest.embedding.length} dimensions\n`);
+
+  console.log('| Test Case | Expected | Ollama | OpenAI | Winner |');
+  console.log('|-----------|----------|--------|--------|--------|');
+
+  let ollamaTotalTime = 0;
+  let openaiTotalTime = 0;
+  let ollamaScore = 0;
+  let openaiScore = 0;
+
+  for (const pair of TEST_PAIRS) {
+    // Get embeddings
+    const [ollamaA, ollamaB] = await Promise.all([
+      getOllamaEmbedding(pair.a),
+      getOllamaEmbedding(pair.b),
+    ]);
+    const [openaiA, openaiB] = await Promise.all([
+      getOpenAIEmbedding(pair.a),
+      getOpenAIEmbedding(pair.b),
+    ]);
+
+    ollamaTotalTime += ollamaA.ms + ollamaB.ms;
+    openaiTotalTime += openaiA.ms + openaiB.ms;
+
+    const ollamaSim = cosineSimilarity(ollamaA.embedding, ollamaB.embedding);
+    const openaiSim = cosineSimilarity(openaiA.embedding, openaiB.embedding);
+
+    // Score based on expected result
+    let winner = 'Tie';
+    if (pair.expected === 'high') {
+      // Higher similarity is better
+      if (ollamaSim > openaiSim + 0.05) {
+        winner = 'Ollama';
+        ollamaScore++;
+      } else if (openaiSim > ollamaSim + 0.05) {
+        winner = 'OpenAI';
+        openaiScore++;
+      }
+    } else if (pair.expected === 'low') {
+      // Lower similarity is better (more discriminating)
+      if (ollamaSim < openaiSim - 0.05) {
+        winner = 'Ollama';
+        ollamaScore++;
+      } else if (openaiSim < ollamaSim - 0.05) {
+        winner = 'OpenAI';
+        openaiScore++;
+      }
+    }
+
+    console.log(
+      `| ${pair.label.padEnd(25)} | ${pair.expected.padEnd(8)} | ${(ollamaSim * 100).toFixed(1)}% | ${(openaiSim * 100).toFixed(1)}% | ${winner} |`
+    );
+  }
+
+  console.log('\n==============================================');
+  console.log('SUMMARY');
+  console.log('==============================================\n');
+
+  console.log('| Metric | Ollama mxbai | OpenAI small |');
+  console.log('|--------|--------------|--------------|');
+  console.log(`| Dimensions | ${ollamaTest.embedding.length} | ${openaiTest.embedding.length} |`);
+  console.log(
+    `| Avg response | ${Math.round(ollamaTotalTime / (TEST_PAIRS.length * 2))}ms | ${Math.round(openaiTotalTime / (TEST_PAIRS.length * 2))}ms |`
+  );
+  console.log(`| Test wins | ${ollamaScore} | ${openaiScore} |`);
+  console.log('| Cost | Free | ~$0.02/1M tokens |');
+
+  console.log(
+    `\n🏆 ${ollamaScore > openaiScore ? 'Ollama mxbai-embed-large' : ollamaScore < openaiScore ? 'OpenAI text-embedding-3-small' : 'Tie'} wins on quality`
+  );
+  console.log(`💰 Ollama is FREE vs OpenAI's paid API`);
+}
+
+main();

package/.deepquark/skills/bundled/knowledge-graph/server/config-falkordb.yaml

@@ -0,0 +1,108 @@
+# Graphiti MCP Server Configuration for FalkorDB Backend
+# This configuration is used with docker-compose-falkordb.yml
+#
+# FalkorDB advantages:
+#   - Redis-based (lighter resource footprint)
+#   - Simple deployment (single container)
+#   - Web UI at http://localhost:3000
+
+server:
+  transport: "http"  # HTTP transport (SSE is deprecated)
+  host: "0.0.0.0"
+  port: 8000
+
+llm:
+  provider: "openai"  # Options: openai, azure_openai, anthropic, gemini, groq
+  model: ${MODEL_NAME:google/gemini-2.0-flash-001}  # Default: Gemini Flash, override with MODEL_NAME env var
+  max_tokens: 4096
+
+  providers:
+    openai:
+      api_key: ${OPENAI_API_KEY}
+      api_url: ${OPENAI_BASE_URL:https://api.openai.com/v1}  # Ollama: http://host.docker.internal:11434/v1
+      organization_id: ${OPENAI_ORGANIZATION_ID:}
+
+    azure_openai:
+      api_key: ${AZURE_OPENAI_API_KEY}
+      api_url: ${AZURE_OPENAI_ENDPOINT}
+      api_version: ${AZURE_OPENAI_API_VERSION:2024-10-21}
+      deployment_name: ${AZURE_OPENAI_DEPLOYMENT}
+      use_azure_ad: ${USE_AZURE_AD:false}
+
+    anthropic:
+      api_key: ${ANTHROPIC_API_KEY}
+      api_url: ${ANTHROPIC_API_URL:https://api.anthropic.com}
+      max_retries: 3
+
+    gemini:
+      api_key: ${GOOGLE_API_KEY}
+      project_id: ${GOOGLE_PROJECT_ID:}
+      location: ${GOOGLE_LOCATION:us-central1}
+
+    groq:
+      api_key: ${GROQ_API_KEY}
+      api_url: ${GROQ_API_URL:https://api.groq.com/openai/v1}
+
+embedder:
+  provider: ${EMBEDDER_PROVIDER:openai}  # Options: openai, azure_openai, gemini, voyage, ollama
+  model: ${EMBEDDER_MODEL:text-embedding-3-small}  # Ollama: mxbai-embed-large (best), OpenAI: text-embedding-3-small
+  dimensions: ${EMBEDDER_DIMENSIONS:1536}  # Ollama mxbai-embed-large: 1024, OpenAI: 1536
+
+  providers:
+    openai:
+      api_key: ${OPENAI_API_KEY}
+      api_url: ${EMBEDDER_PROVIDER_URL:-https://api.openai.com/v1}  # Custom embedder URL or OpenAI default
+      organization_id: ${OPENAI_ORGANIZATION_ID:}
+
+    ollama:
+      api_url: ${EMBEDDER_PROVIDER_URL:http://host.containers.internal:11434}
+
+    azure_openai:
+      api_key: ${AZURE_OPENAI_API_KEY}
+      api_url: ${AZURE_OPENAI_EMBEDDINGS_ENDPOINT}
+      api_version: ${AZURE_OPENAI_API_VERSION:2024-10-21}
+      deployment_name: ${AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT}
+      use_azure_ad: ${USE_AZURE_AD:false}
+
+    gemini:
+      api_key: ${GOOGLE_API_KEY}
+      project_id: ${GOOGLE_PROJECT_ID:}
+      location: ${GOOGLE_LOCATION:us-central1}
+
+    voyage:
+      api_key: ${VOYAGE_API_KEY}
+      api_url: ${VOYAGE_API_URL:https://api.voyageai.com/v1}
+      model: "voyage-3"
+
+database:
+  provider: "falkordb"  # Using FalkorDB for this configuration
+
+  providers:
+    falkordb:
+      # Use environment variable if set, otherwise use Docker service hostname
+      uri: ${FALKORDB_URI:redis://falkordb:6379}
+      password: ${FALKORDB_PASSWORD:}
+
+graphiti:
+  group_id: ${GRAPHITI_GROUP_ID:main}
+  episode_id_prefix: ${EPISODE_ID_PREFIX:}
+  user_id: ${USER_ID:mcp_user}
+  entity_types:
+    - name: "Preference"
+      description: "User preferences, choices, opinions, or selections (PRIORITIZE over most other types except User/Assistant)"
+    - name: "Requirement"
+      description: "Specific needs, features, or functionality that must be fulfilled"
+    - name: "Procedure"
+      description: "Standard operating procedures and sequential instructions"
+    - name: "Location"
+      description: "Physical or virtual places where activities occur"
+    - name: "Event"
+      description: "Time-bound activities, occurrences, or experiences"
+    - name: "Organization"
+      description: "Companies, institutions, groups, or formal entities"
+    - name: "Document"
+      description: "Information content in various forms (books, articles, reports, etc.)"
+    - name: "Topic"
+      description: "Subject of conversation, interest, or knowledge domain (use as last resort)"
+    - name: "Object"
+      description: "Physical items, tools, devices, or possessions (use as last resort)"

package/.deepquark/skills/bundled/knowledge-graph/server/config-neo4j.yaml

@@ -0,0 +1,111 @@
+# Graphiti MCP Server Configuration for Neo4j Backend
+# This configuration is used with docker-compose-neo4j.yml
+#
+# Neo4j advantages over FalkorDB:
+#   - Mature ecosystem with Neo4j Browser visualization
+#   - Better handling of special characters in identifiers
+#   - APOC procedures for advanced graph operations
+
+server:
+  transport: "http"  # HTTP transport (SSE is deprecated)
+  host: "0.0.0.0"
+  port: 8000
+
+llm:
+  provider: "openai"  # Options: openai, azure_openai, anthropic, gemini, groq
+  model: ${MODEL_NAME:google/gemini-2.0-flash-001}  # Default: Gemini Flash, override with MODEL_NAME env var
+  max_tokens: 4096
+
+  providers:
+    openai:
+      api_key: ${OPENAI_API_KEY}
+      api_url: ${OPENAI_BASE_URL:https://api.openai.com/v1}  # Ollama: http://host.docker.internal:11434/v1
+      organization_id: ${OPENAI_ORGANIZATION_ID:}
+
+    azure_openai:
+      api_key: ${AZURE_OPENAI_API_KEY}
+      api_url: ${AZURE_OPENAI_ENDPOINT}
+      api_version: ${AZURE_OPENAI_API_VERSION:2024-10-21}
+      deployment_name: ${AZURE_OPENAI_DEPLOYMENT}
+      use_azure_ad: ${USE_AZURE_AD:false}
+
+    anthropic:
+      api_key: ${ANTHROPIC_API_KEY}
+      api_url: ${ANTHROPIC_API_URL:https://api.anthropic.com}
+      max_retries: 3
+
+    gemini:
+      api_key: ${GOOGLE_API_KEY}
+      project_id: ${GOOGLE_PROJECT_ID:}
+      location: ${GOOGLE_LOCATION:us-central1}
+
+    groq:
+      api_key: ${GROQ_API_KEY}
+      api_url: ${GROQ_API_URL:https://api.groq.com/openai/v1}
+
+embedder:
+  provider: ${EMBEDDER_PROVIDER:openai}  # Options: openai, azure_openai, gemini, voyage, ollama
+  model: ${EMBEDDER_MODEL:text-embedding-3-small}  # Ollama: mxbai-embed-large (best), OpenAI: text-embedding-3-small
+  dimensions: ${EMBEDDER_DIMENSIONS:1536}  # Ollama mxbai-embed-large: 1024, OpenAI: 1536
+
+  providers:
+    openai:
+      api_key: ${OPENAI_API_KEY}
+      api_url: ${EMBEDDER_PROVIDER_URL:-https://api.openai.com/v1}  # Custom embedder URL or OpenAI default
+      organization_id: ${OPENAI_ORGANIZATION_ID:}
+
+    ollama:
+      api_url: ${EMBEDDER_PROVIDER_URL:http://host.containers.internal:11434}
+
+    azure_openai:
+      api_key: ${AZURE_OPENAI_API_KEY}
+      api_url: ${AZURE_OPENAI_EMBEDDINGS_ENDPOINT}
+      api_version: ${AZURE_OPENAI_API_VERSION:2024-10-21}
+      deployment_name: ${AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT}
+      use_azure_ad: ${USE_AZURE_AD:false}
+
+    gemini:
+      api_key: ${GOOGLE_API_KEY}
+      project_id: ${GOOGLE_PROJECT_ID:}
+      location: ${GOOGLE_LOCATION:us-central1}
+
+    voyage:
+      api_key: ${VOYAGE_API_KEY}
+      api_url: ${VOYAGE_API_URL:https://api.voyageai.com/v1}
+      model: "voyage-3"
+
+database:
+  provider: "neo4j"  # Using Neo4j for this configuration
+
+  providers:
+    neo4j:
+      # Use environment variable if set, otherwise use Docker service hostname
+      uri: ${NEO4J_URI:bolt://neo4j:7687}
+      username: ${NEO4J_USER:neo4j}
+      password: madeinozknowledge  # TODO: Switch back to ${NEO4J_PASSWORD} after testing
+      database: ${NEO4J_DATABASE:neo4j}
+      use_parallel_runtime: ${USE_PARALLEL_RUNTIME:false}
+
+graphiti:
+  group_id: ${GRAPHITI_GROUP_ID:main}
+  episode_id_prefix: ${EPISODE_ID_PREFIX:}
+  user_id: ${USER_ID:mcp_user}
+  entity_types:
+    - name: "Preference"
+      description: "User preferences, choices, opinions, or selections (PRIORITIZE over most other types except User/Assistant)"
+    - name: "Requirement"
+      description: "Specific needs, features, or functionality that must be fulfilled"
+    - name: "Procedure"
+      description: "Standard operating procedures and sequential instructions"
+    - name: "Location"
+      description: "Physical or virtual places where activities occur"
+    - name: "Event"
+      description: "Time-bound activities, occurrences, or experiences"
+    - name: "Organization"
+      description: "Companies, institutions, groups, or formal entities"
+    - name: "Document"
+      description: "Information content in various forms (books, articles, reports, etc.)"
+    - name: "Topic"
+      description: "Subject of conversation, interest, or knowledge domain (use as last resort)"
+    - name: "Object"
+      description: "Physical items, tools, devices, or possessions (use as last resort)"