@far-world-labs/verblets 0.1.1 → 0.1.2

package/src/chains/dismantle/dismantle.examples.js

@@ -0,0 +1,27 @@
+import { describe, expect, it } from 'vitest';
+import { longTestTimeout } from '../../constants/common.js';
+import { dismantle } from './index.js';
+
+const examples = [
+  {
+    name: 'Basic usage',
+    inputs: { text: 'test' },
+    want: { result: {} },
+  },
+];
+
+describe('Dismantle chain', () => {
+  examples.forEach((example) => {
+    it(
+      example.name,
+      async () => {
+        const result = await dismantle(example.inputs.text);
+
+        if (example.want.typeOfResult) {
+          expect(JSON.stringify(result.tree)).toStrictEqual(JSON.stringify(example.want.result));
+        }
+      },
+      longTestTimeout
+    );
+  });
+});

package/src/chains/dismantle/index.js

@@ -1,12 +1,7 @@
-/* eslint-disable no-await-in-loop */
-
 import { v4 as uuid } from 'uuid';
 
 import chatGPT from '../../lib/chatgpt/index.js';
-import {
-  constants as promptConstants,
-  outputSuccinctNames,
-} from '../../prompts/index.js';
+import { outputSuccinctNames, constants as promptConstants } from '../../prompts/index.js';
 import modelService from '../../services/llm-model/index.js';
 import toObject from '../../verblets/to-object/index.js';
 
@@ -82,15 +77,11 @@ const defaultDecompose = async ({
   focus,
   rootName,
   fixes,
-  model = modelService.getBestAvailableModel(),
+  model = modelService.getBestPublicModel(),
 } = {}) => {
   const focusFormatted = focus ? `: ${focus}` : '';
 
-  const promptCreated = subComponentsPrompt(
-    `${name}${focusFormatted}`,
-    rootName,
-    fixes
-  );
+  const promptCreated = subComponentsPrompt(`${name}${focusFormatted}`, rootName, fixes);
   const budget = model.budgetTokens(promptCreated);
   return toObject(
     await chatGPT(promptCreated, {
@@ -107,7 +98,7 @@ const defaultEnhance = async ({
   name,
   rootName,
   fixes,
-  model = modelService.getBestAvailableModel(),
+  model = modelService.getBestPublicModel(),
 } = {}) => {
   const promptCreated = componentOptionsPrompt(name, rootName, fixes);
   const budget = model.budgetTokens(promptCreated);
@@ -207,6 +198,7 @@ const makeSubtree = async ({
 
   const children = [];
   for (const child of tree.children) {
+    // eslint-disable-next-line no-await-in-loop
     const subtree = await makeSubtree({
       tree: child,
       rootName,
@@ -245,10 +237,7 @@ export const simplifyTree = (node) => {
 };
 
 class ChainTree {
-  constructor(
-    name,
-    { decompose, enhance, makeId, enhanceFixes, decomposeFixes } = {}
-  ) {
+  constructor(name, { decompose, enhance, makeId, enhanceFixes, decomposeFixes } = {}) {
     this.rootName = name;
     this.tree = {};
     this.decompose = decompose;

package/src/chains/dismantle/index.spec.js

@@ -25,8 +25,7 @@ describe('Dismantle chain', () => {
       const result = await dismantle(example.inputs.text);
 
       if (example.want.typeOfResult) {
-        expect(JSON.stringify(result.tree))
-          .toStrictEqual(JSON.stringify(example.want.result));
+        expect(JSON.stringify(result.tree)).toStrictEqual(JSON.stringify(example.want.result));
       }
     });
   });

package/src/chains/expect/README.md

@@ -0,0 +1,171 @@
+# LLM Expect Chain
+
+Advanced intelligent assertions with debugging features, environment variable modes, and structured results. This chain provides enhanced functionality beyond the basic [expect verblet](../../verblets/expect/).
+
+## Why Use the Chain?
+
+While the verblet provides simple pass/fail assertions, the chain offers:
+
+- **Advanced Debugging**: Automatic code context analysis and intelligent advice
+- **Environment Modes**: Different behaviors for development, testing, and CI
+- **Structured Results**: Detailed information about assertions and failures
+- **Stack Trace Integration**: Automatic detection of calling file and line
+
+## Environment Variable Modes
+
+Control behavior with the `LLM_EXPECT_MODE` environment variable:
+
+### `none` (default)
+Silent mode - returns structured results without throwing or logging
+```bash
+export LLM_EXPECT_MODE=none
+```
+
+### `info` 
+Development mode - logs failures with context and advice to console
+```bash
+export LLM_EXPECT_MODE=info
+```
+
+### `error`
+CI/Testing mode - throws detailed errors on assertion failures
+```bash
+export LLM_EXPECT_MODE=error
+```
+
+## API Reference
+
+### Enhanced API: `expect(actual, expected?, constraint?)`
+
+Returns a tuple `[passed, details]` with structured results:
+
+```javascript
+import { expect } from './index.js';
+
+const [passed, details] = await expect(
+  actualValue,
+  "Does this meet our quality standards?"
+);
+
+console.log(details);
+/* Returns:
+{
+  passed: false,
+  advice: "ISSUE: Content lacks specific examples...",
+  file: "/path/to/test.js",
+  line: 42
+}
+*/
+```
+
+### Simple API: `expect(actual, expected?, constraint?)`
+
+Backward compatible with the verblet - returns boolean:
+
+```javascript
+import expect from './index.js';
+
+const result = await expect("hello", "hello");
+// Returns: true
+```
+
+## Real-World Use Cases
+
+### Content Quality Assurance with Debugging
+
+```javascript
+import { expect } from './index.js';
+
+// Set development mode for detailed feedback
+process.env.LLM_EXPECT_MODE = 'info';
+
+const [passed, details] = await expect(
+  generatedCopy,
+  "Is this marketing copy professional, engaging, and free of grammatical errors?"
+);
+
+if (!passed) {
+  console.log(`Failed at ${details.file}:${details.line}`);
+  console.log(details.advice);
+  // ISSUE: Copy contains informal language and lacks call-to-action
+  // FIX: Replace casual phrases with professional alternatives and add clear CTA
+  // CONTEXT: Marketing copy should maintain professional tone while being engaging
+}
+```
+
+## Advanced Features
+
+### Automatic Code Context Analysis
+
+The chain automatically:
+- Detects the calling file and line number
+- Reads 400 lines before and 100 lines after the assertion
+- Provides this context to the LLM for better debugging advice
+
+### Intelligent Advice Generation
+
+Failed assertions generate structured advice:
+- **ISSUE**: Brief description of why the assertion failed
+- **FIX**: Specific actionable steps to resolve the issue  
+- **CONTEXT**: Additional context about the problem and root causes
+
+### Environment-Aware Behavior
+
+Different modes for different environments:
+- **Development**: Rich console output with advice
+- **Testing**: Detailed error throwing for CI/CD
+- **Production**: Silent operation with structured results
+
+## Best Practices
+
+### **Write Specific Constraints**
+```javascript
+// ❌ Vague
+await expect(text, "Is this good?");
+
+// ✅ Specific  
+await expect(text, "Is this text grammatically correct, under 100 words, and written in a professional tone?");
+```
+
+### **Use Structured Results**
+```javascript
+const [passed, details] = await expect(content, constraint);
+
+// Access rich debugging information
+console.log('File:', details.file);
+console.log('Line:', details.line);
+console.log('Advice:', details.advice);
+```
+
+### **Combine with Traditional Tests**
+```javascript
+// Traditional assertion for structure
+expect(response).toHaveProperty('status');
+expect(response.status).toBe(200);
+
+// LLM assertion for content quality
+const [passed] = await expect(
+  response.message,
+  "Is this error message helpful and user-friendly?"
+);
+```
+
+### Environment Modes
+
+```bash
+# Silent operation (default)
+export LLM_EXPECT_MODE=none
+
+# Log debugging advice on failures
+export LLM_EXPECT_MODE=info
+
+# Throw with detailed debugging advice
+export LLM_EXPECT_MODE=error
+```
+
+## Best Practices
+
+- **Be specific**: Use clear, detailed constraints
+- **Test qualitatively**: Verify qualitative details with a clear yes/no answer
+- **Use robust constraints**: Write criteria to pass under a wide range of input hallucinations. Assert cases that classical software can't. Tune the level of rigorousness to the model performing the eval.
+- **Performance**: Remember this makes an LLM call - use judiciously

package/src/chains/expect/index.examples.js

@@ -0,0 +1,146 @@
+import { describe, expect, it, beforeAll, afterAll } from 'vitest';
+
+import { expect as aiExpect } from './index.js';
+import { longTestTimeout } from '../../constants/common.js';
+
+const examples = [
+  {
+    inputs: {
+      actual: 'Hello world!',
+      constraint: 'Is this a greeting?',
+    },
+    want: { result: true },
+  },
+  {
+    inputs: {
+      actual: 'Goodbye cruel world',
+      constraint: 'Is this a greeting?',
+    },
+    want: { result: false },
+  },
+  {
+    inputs: {
+      actual: 'hello',
+      expected: 'hello',
+    },
+    want: { result: true },
+  },
+  {
+    inputs: {
+      actual: 'hello',
+      expected: 'goodbye',
+    },
+    want: { result: false },
+  },
+  {
+    inputs: {
+      actual: 'This is a well-written, professional email with proper grammar and clear intent.',
+      constraint: 'Is this text professional and grammatically correct?',
+    },
+    want: { result: true },
+  },
+  {
+    inputs: {
+      actual: { name: 'John Doe', age: 30, city: 'New York' },
+      constraint: 'Does this person data look realistic?',
+    },
+    want: { result: true },
+  },
+];
+
+describe('LLM Expect Chain', () => {
+  // Set environment mode to 'none' for all tests to avoid throwing
+  const originalMode = process.env.LLM_EXPECT_MODE;
+
+  beforeAll(() => {
+    process.env.LLM_EXPECT_MODE = 'none';
+  });
+
+  afterAll(() => {
+    if (originalMode !== undefined) {
+      process.env.LLM_EXPECT_MODE = originalMode;
+    } else {
+      delete process.env.LLM_EXPECT_MODE;
+    }
+  });
+
+  examples.forEach((example) => {
+    const description = example.inputs.constraint
+      ? `${JSON.stringify(example.inputs.actual).slice(0, 30)}... - ${example.inputs.constraint}`
+      : `${JSON.stringify(example.inputs.actual)} === ${JSON.stringify(example.inputs.expected)}`;
+
+    it(
+      description,
+      async () => {
+        const [result, details] = await aiExpect(
+          example.inputs.actual,
+          example.inputs.expected,
+          example.inputs.constraint
+        );
+
+        expect(result).toBe(example.want.result);
+        expect(details).toHaveProperty('passed', example.want.result);
+        expect(details).toHaveProperty('file');
+        expect(details).toHaveProperty('line');
+        expect(typeof details.line).toBe('number');
+      },
+      longTestTimeout
+    );
+  });
+
+  it(
+    'should provide detailed debugging information on failure',
+    async () => {
+      const [result, details] = await aiExpect(
+        'This is clearly wrong content',
+        undefined,
+        'Is this a professional business email?'
+      );
+
+      expect(result).toBe(false);
+      expect(details.passed).toBe(false);
+      expect(details.file).toBeDefined();
+      expect(details.line).toBeGreaterThan(0);
+
+      // In none mode, advice should be null for failures
+      expect(details.advice).toBeNull();
+    },
+    longTestTimeout
+  );
+
+  it(
+    'should handle complex business logic validation',
+    async () => {
+      const businessRecommendation =
+        'Increase marketing budget by 20% for Q4 to boost holiday sales and target demographics aged 25-45 through social media campaigns';
+
+      const [result, details] = await aiExpect(
+        businessRecommendation,
+        undefined,
+        'Is this recommendation specific, actionable, and includes measurable targets?'
+      );
+
+      expect(result).toBe(true);
+      expect(details.passed).toBe(true);
+      expect(details.file).toBeDefined();
+    },
+    longTestTimeout
+  );
+
+  it(
+    'should validate creative content quality',
+    async () => {
+      const storyOpening =
+        'Once upon a time, in a land far away, there lived a brave knight who embarked on a quest to save the kingdom from an ancient curse that had plagued the realm for centuries.';
+
+      const [result] = await aiExpect(
+        storyOpening,
+        undefined,
+        'Is this story opening engaging, sets up clear conflict, and follows good narrative structure?'
+      );
+
+      expect(result).toBe(true);
+    },
+    longTestTimeout
+  );
+});

package/src/chains/expect/index.js

@@ -0,0 +1,173 @@
+import chatgpt from '../../lib/chatgpt/index.js';
+import fs from 'fs';
+import path from 'path';
+
+/**
+ * Get the calling file and line number from stack trace
+ */
+function getCallerInfo() {
+  const { stack } = new Error();
+  const lines = stack.split('\n');
+
+  // Find the first line that's not this file
+  for (let i = 2; i < lines.length; i++) {
+    const line = lines[i];
+    if (line.includes('at ') && !line.includes('expect')) {
+      const match = line.match(/at .* \((.+):(\d+):\d+\)/);
+      if (match) {
+        return { file: match[1], line: parseInt(match[2]) };
+      }
+      // Handle cases without parentheses
+      const simpleMatch = line.match(/at (.+):(\d+):\d+/);
+      if (simpleMatch) {
+        return { file: simpleMatch[1], line: parseInt(simpleMatch[2]) };
+      }
+    }
+  }
+  return { file: 'unknown', line: 0 };
+}
+
+/**
+ * Read code context around the assertion
+ */
+function getCodeContext(filePath, lineNumber) {
+  try {
+    if (!fs.existsSync(filePath)) return null;
+
+    const content = fs.readFileSync(filePath, 'utf8');
+    const lines = content.split('\n');
+
+    // Get 400 lines before and 100 lines after
+    const start = Math.max(0, lineNumber - 400);
+    const end = Math.min(lines.length, lineNumber + 100);
+
+    return {
+      lines: lines.slice(start, end),
+      startLine: start + 1,
+      assertionLine: lineNumber,
+    };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Generate intelligent advice for failed assertions
+ */
+async function generateAdvice(actual, expected, constraint, codeContext) {
+  const contextInfo = codeContext
+    ? `
+Code context around assertion (line ${codeContext.assertionLine}):
+\`\`\`
+${codeContext.lines.join('\n')}
+\`\`\`
+`
+    : '';
+
+  const prompt = `You are a debugging assistant helping with a failed LLM assertion.
+
+ASSERTION DETAILS:
+- Actual value: ${JSON.stringify(actual, null, 2)}
+- Expected value: ${expected ? JSON.stringify(expected, null, 2) : 'N/A'}
+- Constraint: ${constraint || 'N/A'}
+
+${contextInfo}
+
+Provide structured debugging advice in this format:
+
+ISSUE: [Brief description of why the assertion failed]
+FIX: [Specific actionable steps to resolve the issue]
+CONTEXT: [Additional context about the problem and potential root causes]
+
+Keep your response concise but actionable. Focus on practical solutions.`;
+
+  try {
+    return await chatgpt(prompt);
+  } catch {
+    return 'Unable to generate debugging advice due to LLM error.';
+  }
+}
+
+/**
+ * Enhanced LLM expectation with debugging features
+ */
+export async function expect(actual, expected, constraint) {
+  const mode = process.env.LLM_EXPECT_MODE || 'none';
+  const callerInfo = getCallerInfo();
+
+  // Build the assertion prompt
+  let prompt;
+  if (constraint && expected === undefined) {
+    // Constraint-only mode
+    prompt = `Given this constraint: "${constraint}"
+    
+Actual value: ${JSON.stringify(actual, null, 2)}
+
+Does the actual value satisfy the constraint? Answer only "True" or "False".`;
+  } else if (constraint && expected !== undefined) {
+    // Both expected and constraint provided - use constraint
+    prompt = `Given this constraint: "${constraint}"
+    
+Actual value: ${JSON.stringify(actual, null, 2)}
+Expected value: ${JSON.stringify(expected, null, 2)}
+
+Does the actual value satisfy the constraint? Answer only "True" or "False".`;
+  } else if (expected !== undefined) {
+    // Expected value only
+    prompt = `Does the actual value strictly equal the expected value?
+
+Actual: ${JSON.stringify(actual, null, 2)}
+Expected: ${JSON.stringify(expected, null, 2)}
+
+Answer only "True" or "False".`;
+  } else {
+    throw new Error('Either expected value or constraint must be provided');
+  }
+
+  try {
+    const response = await chatgpt(prompt);
+    const passed = response.trim().toLowerCase() === 'true';
+
+    // Prepare result structure
+    const result = {
+      passed,
+      advice: null,
+      file: callerInfo.file,
+      line: callerInfo.line,
+    };
+
+    // Handle failure cases based on mode
+    if (!passed) {
+      if (mode === 'info' || mode === 'error') {
+        const codeContext = getCodeContext(callerInfo.file, callerInfo.line);
+        result.advice = await generateAdvice(actual, expected, constraint, codeContext);
+
+        const message = `LLM Assertion Failed at ${path.basename(callerInfo.file)}:${
+          callerInfo.line
+        }
+${result.advice}`;
+
+        if (mode === 'error') {
+          throw new Error(message);
+        } else if (mode === 'info') {
+          console.info(message);
+        }
+      }
+    }
+
+    return [passed, result];
+  } catch (error) {
+    if (error.message.includes('LLM Assertion Failed')) {
+      throw error; // Re-throw our custom errors
+    }
+    throw new Error(`LLM expectation failed due to error: ${error.message}`);
+  }
+}
+
+/**
+ * Simple LLM expectation (backward compatibility)
+ */
+export default async function expectSimple(actual, expected, constraint) {
+  const [passed] = await expect(actual, expected, constraint);
+  return passed;
+}