@jokerized/getresearchdone 0.4.1

package/lib/evolve/_product-ideation.ts

@@ -0,0 +1,405 @@
+'use strict';
+
+/**
+ * GRD Evolve -- Product ideation discoverer
+ *
+ * Claude-powered product-level feature discovery. Unlike the code-quality
+ * discoverers in _dimensions.ts, this module thinks like a product manager:
+ * it reads PROJECT.md, LONG-TERM-ROADMAP.md, existing commands/agents,
+ * and generates creative feature ideas with business value.
+ *
+ * Private helper module -- imported only by ./discovery.ts.
+ *
+ * @dependencies ./types, ./state, ../utils, ../autopilot
+ */
+
+import type { WorkItem, WorkItemEffort, ProductIdeationContext, DiscoveryOptions } from './types';
+
+const path = require('path') as typeof import('path');
+const fs = require('fs') as typeof import('fs');
+const {
+  safeReadFile,
+}: {
+  safeReadFile: (filePath: string) => string | null;
+} = require('../utils');
+const {
+  spawnClaudeAsync,
+}: {
+  spawnClaudeAsync: (
+    cwd: string,
+    prompt: string,
+    opts?: {
+      captureOutput?: boolean;
+      model?: string;
+      maxTurns?: number;
+      timeout?: number;
+      outputFormat?: string;
+      captureStderr?: boolean;
+    }
+  ) => Promise<{
+    exitCode: number;
+    stdout?: string;
+    stderr?: string;
+    timedOut: boolean;
+  }>;
+} = require('../autopilot');
+const {
+  createWorkItem,
+  SONNET_MODEL,
+}: {
+  createWorkItem: (
+    dimension: string,
+    slug: string,
+    title: string,
+    description: string,
+    opts?: { effort?: string }
+  ) => WorkItem;
+  SONNET_MODEL: string;
+} = require('./state');
+
+// ── Context Gathering ────────────────────────────────────────────────────────
+
+/**
+ * Gather product context from planning files and codebase structure.
+ * Each file read is wrapped in try/catch; missing files produce null/empty fallbacks.
+ */
+function gatherProductContext(cwd: string): ProductIdeationContext {
+  const planningDir: string = path.join(cwd, '.planning');
+
+  // PROJECT.md -- extract first 3000 chars (vision section)
+  let projectVision: string | null = null;
+  try {
+    const raw: string | null = safeReadFile(path.join(planningDir, 'PROJECT.md'));
+    if (raw) {
+      projectVision = raw.substring(0, 3000);
+    }
+  } catch (err) {
+    process.stderr.write(
+      `[evolve] Product ideation: failed to read PROJECT.md: ${(err as Error).message}\n`
+    );
+  }
+
+  // LONG-TERM-ROADMAP.md -- extract first 2000 chars
+  let longTermGoals: string | null = null;
+  try {
+    const raw: string | null = safeReadFile(path.join(planningDir, 'LONG-TERM-ROADMAP.md'));
+    if (raw) {
+      longTermGoals = raw.substring(0, 2000);
+    }
+  } catch (err) {
+    process.stderr.write(
+      `[evolve] Product ideation: failed to read LONG-TERM-ROADMAP.md: ${(err as Error).message}\n`
+    );
+  }
+
+  // PRODUCT-QUALITY.md -- extract first 1500 chars if exists
+  let productQuality: string | null = null;
+  try {
+    const raw: string | null = safeReadFile(path.join(planningDir, 'PRODUCT-QUALITY.md'));
+    if (raw) {
+      productQuality = raw.substring(0, 1500);
+    }
+  } catch (err) {
+    process.stderr.write(
+      `[evolve] Product ideation: failed to read PRODUCT-QUALITY.md: ${(err as Error).message}\n`
+    );
+  }
+
+  // commands/ directory -- list .md files, extract command names
+  let existingCommands: string[] = [];
+  try {
+    const cmdDir: string = path.join(cwd, 'commands');
+    const entries = fs.readdirSync(cmdDir, { withFileTypes: true });
+    existingCommands = entries
+      .filter((e: { isFile: () => boolean; name: string }) => e.isFile() && e.name.endsWith('.md'))
+      .map((e: { name: string }) => e.name.replace(/\.md$/, ''));
+  } catch {
+    // commands/ directory missing -- not critical
+  }
+
+  // agents/ directory -- list .md files, extract agent names
+  let existingAgents: string[] = [];
+  try {
+    const agentDir: string = path.join(cwd, 'agents');
+    const entries = fs.readdirSync(agentDir, { withFileTypes: true });
+    existingAgents = entries
+      .filter((e: { isFile: () => boolean; name: string }) => e.isFile() && e.name.endsWith('.md'))
+      .map((e: { name: string }) => e.name.replace(/\.md$/, ''));
+  } catch {
+    // agents/ directory missing -- not critical
+  }
+
+  // ROADMAP.md -- extract last ~2000 chars (recent phases)
+  let recentPhases: string | null = null;
+  try {
+    const raw: string | null = safeReadFile(path.join(planningDir, 'ROADMAP.md'));
+    if (raw) {
+      recentPhases = raw.length > 2000 ? raw.substring(raw.length - 2000) : raw;
+    }
+  } catch (err) {
+    process.stderr.write(
+      `[evolve] Product ideation: failed to read ROADMAP.md: ${(err as Error).message}\n`
+    );
+  }
+
+  return {
+    projectVision,
+    longTermGoals,
+    existingCommands,
+    existingAgents,
+    recentPhases,
+    productQuality,
+  };
+}
+
+// ── Prompt Construction ──────────────────────────────────────────────────────
+
+/**
+ * Build a product-manager-focused prompt for Claude.
+ * This is fundamentally different from buildDiscoveryPrompt in discovery.ts --
+ * it asks for creative feature ideas, not code-quality improvements.
+ */
+function buildProductIdeationPrompt(context: ProductIdeationContext): string {
+  const sections: string[] = [];
+
+  sections.push(
+    'You are a product manager for a developer tools company. Your job is to generate creative, high-value feature ideas for this product.'
+  );
+
+  sections.push('');
+  sections.push('## Product Vision');
+  if (context.projectVision) {
+    sections.push(context.projectVision);
+  } else {
+    sections.push('(No PROJECT.md found -- infer from existing commands and agents.)');
+  }
+
+  if (context.longTermGoals) {
+    sections.push('');
+    sections.push('## Long-Term Roadmap Goals');
+    sections.push(context.longTermGoals);
+  }
+
+  if (context.productQuality) {
+    sections.push('');
+    sections.push('## Known Product Quality Gaps');
+    sections.push(context.productQuality);
+  }
+
+  sections.push('');
+  sections.push('## What Already Exists');
+
+  if (context.existingCommands.length > 0) {
+    sections.push('');
+    sections.push('### Commands');
+    sections.push(context.existingCommands.map((c) => `- /${c}`).join('\n'));
+  }
+
+  if (context.existingAgents.length > 0) {
+    sections.push('');
+    sections.push('### Agents');
+    sections.push(context.existingAgents.map((a) => `- ${a}`).join('\n'));
+  }
+
+  if (context.recentPhases) {
+    sections.push('');
+    sections.push('## Recently Built (for context on momentum)');
+    sections.push(context.recentPhases);
+  }
+
+  sections.push('');
+  sections.push('## Your Task');
+  sections.push('');
+  sections.push('Generate creative feature ideas across these categories:');
+  sections.push('- New user-facing commands or workflows');
+  sections.push('- New integrations with external tools/services');
+  sections.push('- UX improvements to existing workflows');
+  sections.push('- Developer experience enhancements');
+  sections.push('- New analysis or insight capabilities');
+  sections.push('- Automation or productivity features');
+
+  sections.push('');
+  sections.push('## Output Format');
+  sections.push('');
+  sections.push('Output ONLY a JSON array. Each item:');
+  sections.push(
+    '{"dimension":"product-ideation","slug":"<kebab-id>","title":"<short title>","description":"<what it does + why users need it + what pain point it solves>","effort":"small|medium|large"}'
+  );
+
+  sections.push('');
+  sections.push('## Rules');
+  sections.push('- Think about USER VALUE, not code hygiene');
+  sections.push('- Each idea should solve a real user problem or unlock a new capability');
+  sections.push('- Be creative but realistic -- ideas must be directly implementable as code');
+  sections.push('- Include the "why" -- what user pain point does this address?');
+  sections.push(
+    '- Each description MUST specify: which files to create/modify, what the implementation looks like, entry point'
+  );
+  sections.push('- Bad: "Add a dashboard for metrics" (too vague)');
+  sections.push(
+    '- Good: "Add `gd metrics` CLI command in bin/grd-tools.ts that reads .planning/EVOLVE-STATE.json and outputs iteration success rate, avg items/iteration, and top themes — helps users understand evolve effectiveness"'
+  );
+  sections.push('- 3-5 items only — each must be specific enough to implement in one session');
+  sections.push('- ONLY the JSON array, no other text');
+  sections.push('- Every item MUST have dimension "product-ideation"');
+  sections.push(
+    '- Slugs should be descriptive kebab-case (e.g. "auto-changelog-generation", "dependency-impact-analysis")'
+  );
+
+  return sections.join('\n');
+}
+
+// ── Output Parsing ───────────────────────────────────────────────────────────
+
+/**
+ * Parse Claude's product ideation output into validated WorkItem objects.
+ * Only accepts items where dimension === 'product-ideation'.
+ * Returns empty array if parsing fails (no crash).
+ */
+function parseProductIdeationOutput(raw: string): WorkItem[] {
+  let jsonStr: string = raw.trim();
+
+  // Strip markdown fences if present
+  const fenceMatch: RegExpMatchArray | null = jsonStr.match(
+    /```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/
+  );
+  if (fenceMatch) {
+    jsonStr = fenceMatch[1].trim();
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(jsonStr);
+  } catch {
+    return [];
+  }
+
+  if (!Array.isArray(parsed)) {
+    return [];
+  }
+
+  const validEfforts: string[] = ['small', 'medium', 'large'];
+  const items: WorkItem[] = [];
+
+  for (const entry of parsed as Array<Record<string, unknown>>) {
+    if (
+      !entry ||
+      typeof entry.dimension !== 'string' ||
+      typeof entry.slug !== 'string' ||
+      typeof entry.title !== 'string' ||
+      typeof entry.description !== 'string'
+    ) {
+      continue;
+    }
+
+    // Only accept product-ideation dimension
+    if (entry.dimension !== 'product-ideation') {
+      continue;
+    }
+
+    const effort: WorkItemEffort = validEfforts.includes(entry.effort as string)
+      ? (entry.effort as WorkItemEffort)
+      : 'medium';
+
+    try {
+      items.push(
+        createWorkItem(entry.dimension, entry.slug, entry.title, entry.description, { effort })
+      );
+    } catch {
+      // Skip invalid items (e.g., if createWorkItem rejects the dimension)
+    }
+  }
+
+  return items;
+}
+
+// ── Main Discovery Function ──────────────────────────────────────────────────
+
+/**
+ * Discover product-level feature ideas by spawning Claude with a product-manager prompt.
+ * Returns empty array on failure (graceful fallback, no crash).
+ */
+async function discoverProductIdeationItems(
+  cwd: string,
+  opts?: DiscoveryOptions
+): Promise<WorkItem[]> {
+  const context: ProductIdeationContext = gatherProductContext(cwd);
+
+  // If no PROJECT.md found, skip product ideation entirely
+  if (context.projectVision === null) {
+    process.stderr.write('[evolve] Product ideation skipped: no PROJECT.md found\n');
+    return [];
+  }
+
+  const DEFAULT_IDEATION_TIMEOUT: number = 10_800_000; // 3 hours
+  const effectiveTimeout: number = opts?.timeoutMs || DEFAULT_IDEATION_TIMEOUT;
+  const scheduler = opts?.scheduler;
+  const prompt: string = buildProductIdeationPrompt(context);
+
+  try {
+    const result = scheduler
+      ? await scheduler.spawn(prompt, {
+          model: SONNET_MODEL,
+          maxTurns: 15,
+          timeout: effectiveTimeout,
+          captureOutput: true,
+          cwd,
+          workItemId: `evolve-discovery-product-ideation-${Date.now()}`,
+          agentType: 'grd-planner',
+        })
+      : await spawnClaudeAsync(cwd, prompt, {
+          captureOutput: true,
+          model: SONNET_MODEL,
+          maxTurns: 15,
+          timeout: effectiveTimeout,
+          outputFormat: 'text',
+        });
+
+    if (result.timedOut) {
+      process.stderr.write(
+        `[evolve] Product ideation timed out after ${Math.round(effectiveTimeout / 1000)}s, returning empty\n`
+      );
+      return [];
+    }
+
+    if (result.exitCode !== 0 || !result.stdout) {
+      process.stderr.write(
+        `[evolve] Product ideation failed (exit=${result.exitCode}), returning empty\n`
+      );
+      return [];
+    }
+
+    // Check for error message in stdout (e.g., max-turns exceeded)
+    if (result.stdout.startsWith('Error:')) {
+      process.stderr.write(
+        `[evolve] Product ideation error: ${result.stdout.trim()}, returning empty\n`
+      );
+      return [];
+    }
+
+    const items: WorkItem[] = parseProductIdeationOutput(result.stdout);
+
+    if (items.length === 0) {
+      process.stderr.write(
+        `[evolve] Product ideation returned unparseable output (${result.stdout.length} chars), returning empty\n`
+      );
+      return [];
+    }
+
+    return items;
+  } catch (err) {
+    process.stderr.write(
+      `[evolve] Product ideation threw: ${(err as Error).message}, returning empty\n`
+    );
+    return [];
+  }
+}
+
+// ── Exports ──────────────────────────────────────────────────────────────────
+
+module.exports = {
+  gatherProductContext,
+  buildProductIdeationPrompt,
+  parseProductIdeationOutput,
+  discoverProductIdeationItems,
+};

package/lib/evolve/_prompts.ts

@@ -0,0 +1,178 @@
+'use strict';
+
+/**
+ * GRD Evolve -- Prompt templates
+ *
+ * Pure string template functions for building prompts used in the
+ * evolve iteration loop (plan, execute, review, batch variants).
+ *
+ * Private helper module -- imported only by ./orchestrator.ts.
+ *
+ * @dependencies ./types
+ */
+
+import type { WorkItem, WorkGroup } from './types';
+
+// ─── Prompt Builders ────────────────────────────────────────────────────────
+
+/**
+ * Build a prompt for planning a work item improvement.
+ */
+function buildPlanPrompt(item: WorkItem): string {
+  return [
+    'Read CLAUDE.md for project conventions.',
+    `Analyze the codebase for the following improvement opportunity:`,
+    `Title: ${item.title}`,
+    `Description: ${item.description}`,
+    `Dimension: ${item.dimension}`,
+    `Effort: ${item.effort}`,
+    'Create a brief implementation plan: what files to change and what changes to make.',
+    'Do NOT implement anything — only plan.',
+  ].join('\n');
+}
+
+/**
+ * Build a prompt for executing a work item improvement.
+ */
+function buildExecutePrompt(item: WorkItem): string {
+  return [
+    'Read CLAUDE.md for project conventions.',
+    `Implement the following improvement:`,
+    `Title: ${item.title}`,
+    `Description: ${item.description}`,
+    `Dimension: ${item.dimension}`,
+    `Effort: ${item.effort}`,
+    'Run `npm test` to verify changes do not break tests.',
+    'Fix any test failures before completing.',
+    'Keep changes focused and minimal.',
+  ].join('\n');
+}
+
+/**
+ * Build a prompt for reviewing an executed improvement.
+ */
+function buildReviewPrompt(item: WorkItem): string {
+  return [
+    `Review the improvement that was just made:`,
+    `Title: ${item.title}`,
+    `Description: ${item.description}`,
+    'Run `npm test` and `npm run lint` to check for regressions.',
+    'Verify the improvement was actually made.',
+    'Fix any issues found.',
+  ].join('\n');
+}
+
+/**
+ * Build execution prompt for an entire group of work items.
+ */
+function buildGroupExecutePrompt(group: WorkGroup): string {
+  const itemList: string = group.items
+    .map((item, i) => `${i + 1}. ${item.title}: ${item.description}`)
+    .join('\n');
+
+  return [
+    'Read CLAUDE.md for project conventions.',
+    `Implement the following improvements (theme: ${group.theme}, dimension: ${group.dimension}):`,
+    '',
+    itemList,
+    '',
+    'Run `npm test` to verify changes do not break tests.',
+    'Fix any test failures before completing.',
+    'Keep changes focused and minimal.',
+  ].join('\n');
+}
+
+/**
+ * Build review prompt for an entire group after execution.
+ */
+function buildGroupReviewPrompt(group: WorkGroup): string {
+  return [
+    `Review the improvements that were just made for group: ${group.title}`,
+    `Theme: ${group.theme}, Dimension: ${group.dimension}`,
+    `${group.items.length} items were addressed.`,
+    'Run `npm test` and `npm run lint` to check for regressions.',
+    'Verify the improvements were actually made.',
+    'Fix any issues found.',
+  ].join('\n');
+}
+
+/**
+ * Build a single execution prompt for ALL groups in an iteration.
+ */
+function buildBatchExecutePrompt(groups: WorkGroup[], iterationNum?: number): string {
+  const sections: string[] = groups.map((group) => {
+    const itemList: string = group.items
+      .map((item, i) => `${i + 1}. ${item.title}: ${item.description}`)
+      .join('\n');
+    return `### ${group.title} (${group.dimension}/${group.theme})\n${itemList}`;
+  });
+
+  const feedbackFile: string = `.planning/evolve-iteration-${iterationNum || 0}.json`;
+
+  return [
+    'Read CLAUDE.md for project conventions.',
+    `Implement the following improvements across ${groups.length} groups:`,
+    '',
+    ...sections,
+    '',
+    '## CRITICAL RULES — READ CAREFULLY',
+    '',
+    '1. You MUST modify actual source code files (.ts, .js, .tsx, .jsx, .py, etc.)',
+    '2. Do NOT create markdown files, todo files, or documentation stubs',
+    '3. Do NOT create files in .planning/, docs/, or any markdown directory',
+    '4. Do NOT just add JSDoc comments or rename variables — make real functional changes',
+    '5. If an item requires creating a new feature, write the actual implementation code',
+    '6. If an item is too vague to implement as code, SKIP it entirely',
+    '7. Every change must be a real code improvement: new functions, bug fixes, refactors, tests, or feature code',
+    '',
+    'Do NOT run tests between groups — implement everything first.',
+    'After ALL changes are done, run `npm test` once to verify nothing is broken.',
+    'Fix any test failures before completing.',
+    '',
+    `After completing all changes, write a JSON feedback file to \`${feedbackFile}\` with this exact structure:`,
+    '```json',
+    '{',
+    '  "decisions": ["Brief description of each design decision you made and why"],',
+    '  "patterns": ["Any code patterns you noticed — good or bad — worth documenting"],',
+    '  "takeaways": ["Observations about the codebase that could inform future improvements"],',
+    '  "files_changed": ["list/of/files/you/modified.ts"],',
+    '  "items_skipped": ["Items you skipped because they were too vague or not implementable as code"]',
+    '}',
+    '```',
+    'Include at least 1 entry per array. Be specific and concise.',
+  ].join('\n');
+}
+
+/**
+ * Build a single review prompt for ALL groups after batch execution.
+ */
+function buildBatchReviewPrompt(groups: WorkGroup[]): string {
+  const groupList: string = groups.map((g) => `- ${g.title} (${g.items.length} items)`).join('\n');
+
+  return [
+    `Review ALL improvements that were just made across ${groups.length} groups:`,
+    '',
+    groupList,
+    '',
+    'Run `npm test` and `npm run lint` to check for regressions.',
+    'Verify the improvements were actually made in source code files (.ts, .js, etc.).',
+    '',
+    'IMPORTANT: If you find any changes that are ONLY markdown files, todo files, or documentation stubs',
+    'without corresponding source code changes, revert those files with `git checkout -- <file>`.',
+    'Only real code changes should remain.',
+    '',
+    'Fix any issues found.',
+  ].join('\n');
+}
+
+// ─── Exports ────────────────────────────────────────────────────────────────
+
+module.exports = {
+  buildPlanPrompt,
+  buildExecutePrompt,
+  buildReviewPrompt,
+  buildGroupExecutePrompt,
+  buildGroupReviewPrompt,
+  buildBatchExecutePrompt,
+  buildBatchReviewPrompt,
+};