grimoire-framework 1.0.0

package/.grimoire/cli/commands/workers/search-semantic.js

@@ -0,0 +1,294 @@
+/**
+ * Semantic Search Module
+ *
+ * Implements semantic search using OpenAI embeddings for intelligent
+ * similarity matching based on meaning, not just keywords.
+ *
+ * @module cli/commands/workers/search-semantic
+ * @version 1.0.0
+ * @story 2.7 - Discovery CLI Search
+ */
+
+const path = require('path');
+const fs = require('fs');
+
+// Registry loader
+const { getRegistry } = require('../../../core/registry/registry-loader');
+
+// Cache for embeddings
+let embeddingsCache = null;
+let embeddingsCacheTimestamp = 0;
+const EMBEDDINGS_CACHE_TTL = 5 * 60 * 1000; // 5 minutes
+
+// Configurable timeouts and rate limits
+const OPENAI_REQUEST_TIMEOUT_MS = parseInt(process.env.OPENAI_REQUEST_TIMEOUT_MS) || 10000; // 10s default
+const RATE_LIMIT_DELAY_MS = parseInt(process.env.RATE_LIMIT_DELAY_MS) || 600; // 600ms for 100 RPM (Free tier)
+
+/**
+ * Check if semantic search is available
+ * @returns {boolean} True if OPENAI_API_KEY is set
+ */
+function isSemanticAvailable() {
+  return !!process.env.OPENAI_API_KEY;
+}
+
+/**
+ * Get OpenAI embeddings for text
+ * @param {string} text - Text to embed
+ * @returns {Promise<number[]>} Embedding vector
+ */
+async function getEmbedding(text) {
+  const apiKey = process.env.OPENAI_API_KEY;
+
+  if (!apiKey) {
+    throw new Error('OPENAI_API_KEY not set');
+  }
+
+  // Create AbortController for timeout
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), OPENAI_REQUEST_TIMEOUT_MS);
+
+  try {
+    const response = await fetch('https://api.openai.com/v1/embeddings', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${apiKey}`,
+      },
+      body: JSON.stringify({
+        model: 'text-embedding-3-small',
+        input: text,
+      }),
+      signal: controller.signal,
+    });
+
+    clearTimeout(timeoutId);
+
+    if (!response.ok) {
+      // Handle rate limiting with Retry-After header
+      if (response.status === 429) {
+        const retryAfter = response.headers.get('Retry-After');
+        throw new Error(`OpenAI rate limit exceeded. Retry after ${retryAfter || 'a few'} seconds.`);
+      }
+      const error = await response.json().catch(() => ({ error: { message: 'Unknown error' } }));
+      throw new Error(`OpenAI API error: ${error.error?.message || response.statusText}`);
+    }
+
+    const data = await response.json();
+    return data.data[0].embedding;
+  } catch (error) {
+    clearTimeout(timeoutId);
+    if (error.name === 'AbortError') {
+      throw new Error(`OpenAI request timed out after ${OPENAI_REQUEST_TIMEOUT_MS}ms`);
+    }
+    throw error;
+  }
+}
+
+/**
+ * Calculate cosine similarity between two vectors
+ * @param {number[]} a - First vector
+ * @param {number[]} b - Second vector
+ * @returns {number} Similarity score (0-1)
+ */
+function cosineSimilarity(a, b) {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have same length');
+  }
+
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  if (normA === 0 || normB === 0) return 0;
+
+  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+/**
+ * Load or generate worker embeddings
+ * @returns {Promise<Map<string, {worker: object, embedding: number[]}>>}
+ */
+async function loadWorkerEmbeddings() {
+  const now = Date.now();
+
+  // Return cached if valid
+  if (embeddingsCache && (now - embeddingsCacheTimestamp) < EMBEDDINGS_CACHE_TTL) {
+    return embeddingsCache;
+  }
+
+  // Check for pre-computed embeddings file
+  const embeddingsPath = path.join(
+    process.cwd(),
+    '.grimoire/core/registry/worker-embeddings.json',
+  );
+
+  if (fs.existsSync(embeddingsPath)) {
+    try {
+      const data = JSON.parse(fs.readFileSync(embeddingsPath, 'utf8'));
+      embeddingsCache = new Map(Object.entries(data.embeddings));
+      embeddingsCacheTimestamp = now;
+      return embeddingsCache;
+    } catch (error) {
+      console.warn('Warning: Could not load pre-computed embeddings, will compute on-the-fly');
+    }
+  }
+
+  // Load registry and compute embeddings on-the-fly
+  const registry = getRegistry();
+  const workers = await registry.getAll();
+
+  embeddingsCache = new Map();
+
+  // For performance, we'll batch the first N workers
+  // In production, embeddings should be pre-computed
+  const maxWorkersToEmbed = 50; // Limit for on-the-fly computation
+  const workersToEmbed = workers.slice(0, maxWorkersToEmbed);
+
+  for (const worker of workersToEmbed) {
+    const text = buildSearchText(worker);
+    try {
+      const embedding = await getEmbedding(text);
+      embeddingsCache.set(worker.id, { worker, embedding });
+    } catch (error) {
+      // Skip workers that fail embedding
+      console.warn(`Warning: Could not embed worker ${worker.id}: ${error.message}`);
+    }
+  }
+
+  // Add remaining workers without embeddings (will use keyword fallback)
+  for (let i = maxWorkersToEmbed; i < workers.length; i++) {
+    embeddingsCache.set(workers[i].id, { worker: workers[i], embedding: null });
+  }
+
+  embeddingsCacheTimestamp = now;
+  return embeddingsCache;
+}
+
+/**
+ * Build searchable text from worker
+ * @param {object} worker - Worker object
+ * @returns {string} Concatenated search text
+ */
+function buildSearchText(worker) {
+  return [
+    worker.name,
+    worker.description,
+    worker.category,
+    worker.subcategory || '',
+    ...(worker.tags || []),
+    ...(worker.inputs || []),
+    ...(worker.outputs || []),
+  ].filter(Boolean).join(' ');
+}
+
+/**
+ * Perform semantic search
+ * @param {string} query - Search query
+ * @returns {Promise<Array<{worker: object, score: number}>>} Search results with scores
+ */
+async function searchSemantic(query) {
+  // Get query embedding
+  const queryEmbedding = await getEmbedding(query);
+
+  // Load worker embeddings
+  const workerEmbeddings = await loadWorkerEmbeddings();
+
+  const results = [];
+
+  for (const [id, data] of workerEmbeddings) {
+    if (data.embedding) {
+      // Calculate semantic similarity
+      const similarity = cosineSimilarity(queryEmbedding, data.embedding);
+      const score = Math.round(similarity * 100);
+
+      // Only include results with reasonable similarity
+      if (score >= 20) {
+        results.push({
+          ...data.worker,
+          score: score,
+          matchType: 'semantic',
+        });
+      }
+    } else {
+      // Fallback to simple text matching for workers without embeddings
+      const searchText = buildSearchText(data.worker).toLowerCase();
+      if (searchText.includes(query.toLowerCase())) {
+        results.push({
+          ...data.worker,
+          score: 50, // Default score for keyword matches
+          matchType: 'keyword-fallback',
+        });
+      }
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Pre-compute and save embeddings for all workers
+ * @returns {Promise<void>}
+ */
+async function precomputeEmbeddings() {
+  console.log('Pre-computing embeddings for all workers...');
+
+  const registry = getRegistry();
+  const workers = await registry.getAll();
+
+  const embeddings = {};
+  let computed = 0;
+  let failed = 0;
+
+  for (const worker of workers) {
+    const text = buildSearchText(worker);
+    try {
+      const embedding = await getEmbedding(text);
+      embeddings[worker.id] = { worker, embedding };
+      computed++;
+      process.stdout.write(`\rComputed: ${computed}/${workers.length}`);
+    } catch (error) {
+      failed++;
+      console.warn(`\nWarning: Failed to embed ${worker.id}: ${error.message}`);
+    }
+
+    // Rate limiting - OpenAI Free tier allows 100 RPM, so 600ms delay
+    // Configurable via RATE_LIMIT_DELAY_MS environment variable
+    await new Promise(resolve => setTimeout(resolve, RATE_LIMIT_DELAY_MS));
+  }
+
+  console.log(`\nEmbeddings computed: ${computed}, failed: ${failed}`);
+
+  // Save to file
+  const embeddingsPath = path.join(
+    process.cwd(),
+    '.grimoire/core/registry/worker-embeddings.json',
+  );
+
+  const data = {
+    version: '1.0.0',
+    generated: new Date().toISOString(),
+    model: 'text-embedding-3-small',
+    count: computed,
+    embeddings: embeddings,
+  };
+
+  fs.writeFileSync(embeddingsPath, JSON.stringify(data, null, 2));
+  console.log(`Saved embeddings to: ${embeddingsPath}`);
+}
+
+module.exports = {
+  isSemanticAvailable,
+  searchSemantic,
+  getEmbedding,
+  cosineSimilarity,
+  precomputeEmbeddings,
+  buildSearchText,
+};
+

package/.grimoire/cli/commands/workers/search.js

@@ -0,0 +1,155 @@
+/**
+ * Search Command - Main entry point for worker search
+ *
+ * Implements `grimoire workers search <query>` CLI command.
+ * Supports semantic search (OpenAI embeddings) and keyword fallback.
+ *
+ * @module cli/commands/workers/search
+ * @version 1.0.0
+ * @story 2.7 - Discovery CLI Search
+ */
+
+const { Command } = require('commander');
+const path = require('path');
+
+// Local modules
+const { searchSemantic, isSemanticAvailable } = require('./search-semantic');
+const { searchKeyword } = require('./search-keyword');
+const { applyFilters } = require('./search-filters');
+const { formatOutput } = require('../../utils/output-formatter-cli');
+const { calculateScores, sortByScore } = require('../../utils/score-calculator');
+
+/**
+ * Create the search command
+ * @returns {Command} Commander command instance
+ */
+function createSearchCommand() {
+  const search = new Command('search');
+
+  search
+    .description('Search for workers in the service registry')
+    .argument('<query>', 'Search query (words, phrases, or patterns)')
+    .option('-c, --category <category>', 'Filter by category')
+    .option('-t, --tags <tags>', 'Filter by tags (comma-separated)')
+    .option('-f, --format <format>', 'Output format: table, json, yaml', 'table')
+    .option('-l, --limit <n>', 'Max results', '10')
+    .option('--semantic', 'Force semantic search (requires OPENAI_API_KEY)')
+    .option('--keyword', 'Force keyword search')
+    .option('-v, --verbose', 'Show verbose output')
+    .addHelpText('after', `
+Examples:
+  $ grimoire workers search "json csv"
+  $ grimoire workers search "data transformation" --category=etl
+  $ grimoire workers search "validation" --tags=schema,json
+  $ grimoire workers search "api" --format=json --limit=5
+  $ grimoire workers search "transform" --semantic
+  $ grimoire workers search "convert" --keyword
+`)
+    .action(executeSearch);
+
+  return search;
+}
+
+/**
+ * Execute the search command
+ * @param {string} query - Search query
+ * @param {object} options - Command options
+ */
+async function executeSearch(query, options) {
+  const startTime = Date.now();
+
+  try {
+    // Validate query
+    if (!query || query.trim().length === 0) {
+      console.error('Error: Search query cannot be empty');
+      process.exit(1);
+    }
+
+    const trimmedQuery = query.trim();
+    const limit = parseInt(options.limit, 10) || 10;
+    const tags = options.tags ? options.tags.split(',').map(t => t.trim()) : [];
+
+    if (options.verbose) {
+      console.log(`Searching for: "${trimmedQuery}"`);
+      if (options.category) console.log(`  Category filter: ${options.category}`);
+      if (tags.length > 0) console.log(`  Tag filters: ${tags.join(', ')}`);
+      console.log(`  Output format: ${options.format}`);
+      console.log(`  Limit: ${limit}`);
+      console.log('');
+    }
+
+    // Determine search strategy
+    let results;
+    let searchMethod;
+
+    if (options.keyword) {
+      // Force keyword search
+      searchMethod = 'keyword';
+      results = await searchKeyword(trimmedQuery);
+    } else if (options.semantic) {
+      // Force semantic search
+      if (!isSemanticAvailable()) {
+        console.error('Error: Semantic search requires OPENAI_API_KEY environment variable');
+        console.log('Hint: Set OPENAI_API_KEY or use --keyword for fallback search');
+        process.exit(1);
+      }
+      searchMethod = 'semantic';
+      results = await searchSemantic(trimmedQuery);
+    } else {
+      // Auto-detect: use semantic if available, otherwise keyword
+      if (isSemanticAvailable()) {
+        searchMethod = 'semantic';
+        results = await searchSemantic(trimmedQuery);
+      } else {
+        searchMethod = 'keyword';
+        results = await searchKeyword(trimmedQuery);
+      }
+    }
+
+    // Apply filters
+    results = applyFilters(results, {
+      category: options.category,
+      tags: tags,
+    });
+
+    // Calculate and sort by scores
+    results = calculateScores(results, trimmedQuery);
+    results = sortByScore(results);
+
+    // Apply limit
+    results = results.slice(0, limit);
+
+    // Calculate duration
+    const duration = ((Date.now() - startTime) / 1000).toFixed(1);
+
+    // Format and output results
+    const output = formatOutput(results, {
+      format: options.format,
+      query: trimmedQuery,
+      duration: duration,
+      searchMethod: searchMethod,
+      verbose: options.verbose,
+    });
+
+    console.log(output);
+
+    // Performance check
+    const durationMs = Date.now() - startTime;
+    if (durationMs > 30000) {
+      console.warn(`\nWarning: Search took ${(durationMs / 1000).toFixed(1)}s (target: < 30s)`);
+    }
+
+  } catch (error) {
+    console.error(`Search error: ${error.message}`);
+    if (options.verbose) {
+      console.error(error.stack);
+    }
+    process.exit(1);
+  }
+}
+
+module.exports = {
+  createSearchCommand,
+  executeSearch,
+};
+

package/.grimoire/cli/commands/workers/utils/pagination.js

@@ -0,0 +1,102 @@
+/**
+ * Pagination Utility
+ *
+ * Handles pagination logic for CLI list commands.
+ *
+ * @module cli/commands/workers/utils/pagination
+ * @version 1.0.0
+ * @story 2.8-2.9 - Discovery CLI Info & List
+ */
+
+/**
+ * Default pagination options
+ */
+const DEFAULT_PAGE_SIZE = 20;
+const MAX_PAGE_SIZE = 100;
+
+/**
+ * Paginate an array of items
+ * @param {Array} items - Items to paginate
+ * @param {object} options - Pagination options
+ * @param {number} options.page - Page number (1-based)
+ * @param {number} options.limit - Items per page
+ * @returns {object} Paginated result
+ */
+function paginate(items, options = {}) {
+  const page = Math.max(1, parseInt(options.page, 10) || 1);
+  const limit = Math.min(
+    MAX_PAGE_SIZE,
+    Math.max(1, parseInt(options.limit, 10) || DEFAULT_PAGE_SIZE),
+  );
+
+  const totalItems = items.length;
+  const totalPages = Math.ceil(totalItems / limit);
+  const startIndex = (page - 1) * limit;
+  const endIndex = Math.min(startIndex + limit, totalItems);
+
+  const paginatedItems = items.slice(startIndex, endIndex);
+
+  return {
+    items: paginatedItems,
+    pagination: {
+      page,
+      limit,
+      totalItems,
+      totalPages,
+      startIndex: startIndex + 1, // 1-based for display
+      endIndex,
+      hasNextPage: page < totalPages,
+      hasPrevPage: page > 1,
+    },
+  };
+}
+
+/**
+ * Format pagination info for display
+ * @param {object} pagination - Pagination object from paginate()
+ * @returns {string} Formatted pagination text
+ */
+function formatPaginationInfo(pagination) {
+  const { startIndex, endIndex, totalItems, page, totalPages } = pagination;
+
+  if (totalItems === 0) {
+    return 'No items found.';
+  }
+
+  if (totalPages === 1) {
+    return `Showing all ${totalItems} items.`;
+  }
+
+  return `Showing ${startIndex}-${endIndex} of ${totalItems} items (page ${page}/${totalPages})`;
+}
+
+/**
+ * Format pagination navigation hint
+ * @param {object} pagination - Pagination object
+ * @returns {string} Navigation hint text
+ */
+function formatPaginationHint(pagination) {
+  const hints = [];
+
+  if (pagination.hasPrevPage) {
+    hints.push(`--page=${pagination.page - 1} for previous`);
+  }
+
+  if (pagination.hasNextPage) {
+    hints.push(`--page=${pagination.page + 1} for next`);
+  }
+
+  if (hints.length === 0) {
+    return '';
+  }
+
+  return `Use ${hints.join(', ')}.`;
+}
+
+module.exports = {
+  paginate,
+  formatPaginationInfo,
+  formatPaginationHint,
+  DEFAULT_PAGE_SIZE,
+  MAX_PAGE_SIZE,
+};

package/.grimoire/cli/index.js

@@ -0,0 +1,150 @@
+/**
+ * grimoire CLI Entry Point
+ *
+ * Main entry point for the grimoire CLI with Commander.js integration.
+ * Registers all subcommands including workers, agents, etc.
+ *
+ * @module cli
+ * @version 1.0.0
+ * @story 2.7 - Discovery CLI Search
+ */
+
+const { Command } = require('commander');
+const path = require('path');
+const fs = require('fs');
+
+// Import command modules
+const { createWorkersCommand } = require('./commands/workers');
+const { createManifestCommand } = require('./commands/manifest');
+const { createQaCommand } = require('./commands/qa');
+const { createMcpCommand } = require('./commands/mcp');
+const { createMigrateCommand } = require('./commands/migrate');
+const { createGenerateCommand } = require('./commands/generate');
+const { createMetricsCommand } = require('./commands/metrics');
+const { createConfigCommand } = require('./commands/config');
+const { createProCommand } = require('./commands/pro');
+
+// Read package.json for version
+const packageJsonPath = path.join(__dirname, '..', '..', 'package.json');
+let packageVersion = '0.0.0';
+try {
+  const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8'));
+  packageVersion = packageJson.version;
+} catch (error) {
+  // Fallback version if package.json not found
+}
+
+/**
+ * Create the main CLI program
+ * @returns {Command} Commander program instance
+ */
+function createProgram() {
+  const program = new Command();
+
+  program
+    .name('grimoire')
+    .version(packageVersion)
+    .description('grimoire-FullStack: AI-Orchestrated System for Full Stack Development')
+    .addHelpText('after', `
+Commands:
+  workers           Manage and discover workers
+  manifest          Manage manifest files (validate, regenerate)
+  qa                Quality Gate Manager (run, status)
+  metrics           Quality Gate Metrics (record, show, seed, cleanup)
+  config            Manage layered configuration (show, diff, migrate, validate)
+  pro               grimoire Pro license management (activate, status, deactivate, features)
+  mcp               Manage global MCP configuration
+  migrate           Migrate from v2.0 to v4.0.4 structure
+  generate          Generate documents from templates (prd, adr, pmdr, etc.)
+  install           Install grimoire in current project
+  init <name>       Create new grimoire project
+  info              Show system information
+  doctor            Run system diagnostics
+
+For command help:
+  $ grimoire <command> --help
+
+Examples:
+  $ grimoire workers search "json transformation"
+  $ grimoire workers list --category=data
+  $ grimoire manifest validate
+  $ grimoire manifest regenerate
+  $ grimoire qa run
+  $ grimoire qa status
+  $ grimoire mcp setup --with-defaults
+  $ grimoire mcp link
+  $ grimoire mcp status
+  $ grimoire metrics show
+  $ grimoire metrics record --layer 1 --passed
+  $ grimoire metrics seed --days 30
+  $ grimoire migrate --dry-run
+  $ grimoire migrate --from=2.0 --to=2.1
+  $ grimoire generate pmdr --title "Feature X Decision"
+  $ grimoire generate adr --save
+  $ grimoire generate list
+  $ grimoire config show
+  $ grimoire config show --debug
+  $ grimoire config diff --levels L1,L2
+  $ grimoire config migrate --dry-run
+  $ grimoire config validate
+  $ grimoire config init-local
+  $ grimoire pro activate --key PRO-XXXX-XXXX-XXXX-XXXX
+  $ grimoire pro status
+  $ grimoire pro deactivate
+  $ grimoire pro features
+  $ grimoire pro validate
+  $ grimoire install
+  $ grimoire doctor
+`);
+
+  // Add workers command
+  program.addCommand(createWorkersCommand());
+
+  // Add manifest command (Story 2.13)
+  program.addCommand(createManifestCommand());
+
+  // Add qa command (Story 2.10)
+  program.addCommand(createQaCommand());
+
+  // Add mcp command (Story 2.11)
+  program.addCommand(createMcpCommand());
+
+  // Add migrate command (Story 2.14)
+  program.addCommand(createMigrateCommand());
+
+  // Add generate command (Story 3.9)
+  program.addCommand(createGenerateCommand());
+
+  // Add metrics command (Story 3.11a)
+  program.addCommand(createMetricsCommand());
+
+  // Add config command (Story PRO-4)
+  program.addCommand(createConfigCommand());
+
+  // Add pro command (Story PRO-6)
+  program.addCommand(createProCommand());
+
+  return program;
+}
+
+/**
+ * Run the CLI
+ * @param {string[]} args - Command line arguments
+ * @returns {Promise<void>}
+ */
+async function run(args = process.argv) {
+  const program = createProgram();
+
+  try {
+    await program.parseAsync(args);
+  } catch (error) {
+    console.error(`Error: ${error.message}`);
+    process.exit(1);
+  }
+}
+
+module.exports = {
+  createProgram,
+  run,
+};
+