@far-world-labs/verblets 0.1.1 → 0.1.3

package/src/chains/expect/index.js

@@ -0,0 +1,173 @@
+import chatgpt from '../../lib/chatgpt/index.js';
+import fs from 'fs';
+import path from 'path';
+
+/**
+ * Get the calling file and line number from stack trace
+ */
+function getCallerInfo() {
+  const { stack } = new Error();
+  const lines = stack.split('\n');
+
+  // Find the first line that's not this file
+  for (let i = 2; i < lines.length; i++) {
+    const line = lines[i];
+    if (line.includes('at ') && !line.includes('expect')) {
+      const match = line.match(/at .* \((.+):(\d+):\d+\)/);
+      if (match) {
+        return { file: match[1], line: parseInt(match[2]) };
+      }
+      // Handle cases without parentheses
+      const simpleMatch = line.match(/at (.+):(\d+):\d+/);
+      if (simpleMatch) {
+        return { file: simpleMatch[1], line: parseInt(simpleMatch[2]) };
+      }
+    }
+  }
+  return { file: 'unknown', line: 0 };
+}
+
+/**
+ * Read code context around the assertion
+ */
+function getCodeContext(filePath, lineNumber) {
+  try {
+    if (!fs.existsSync(filePath)) return null;
+
+    const content = fs.readFileSync(filePath, 'utf8');
+    const lines = content.split('\n');
+
+    // Get 400 lines before and 100 lines after
+    const start = Math.max(0, lineNumber - 400);
+    const end = Math.min(lines.length, lineNumber + 100);
+
+    return {
+      lines: lines.slice(start, end),
+      startLine: start + 1,
+      assertionLine: lineNumber,
+    };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Generate intelligent advice for failed assertions
+ */
+async function generateAdvice(actual, expected, constraint, codeContext) {
+  const contextInfo = codeContext
+    ? `
+Code context around assertion (line ${codeContext.assertionLine}):
+\`\`\`
+${codeContext.lines.join('\n')}
+\`\`\`
+`
+    : '';
+
+  const prompt = `You are a debugging assistant helping with a failed LLM assertion.
+
+ASSERTION DETAILS:
+- Actual value: ${JSON.stringify(actual, null, 2)}
+- Expected value: ${expected ? JSON.stringify(expected, null, 2) : 'N/A'}
+- Constraint: ${constraint || 'N/A'}
+
+${contextInfo}
+
+Provide structured debugging advice in this format:
+
+ISSUE: [Brief description of why the assertion failed]
+FIX: [Specific actionable steps to resolve the issue]
+CONTEXT: [Additional context about the problem and potential root causes]
+
+Keep your response concise but actionable. Focus on practical solutions.`;
+
+  try {
+    return await chatgpt(prompt);
+  } catch {
+    return 'Unable to generate debugging advice due to LLM error.';
+  }
+}
+
+/**
+ * Enhanced LLM expectation with debugging features
+ */
+export async function expect(actual, expected, constraint) {
+  const mode = process.env.LLM_EXPECT_MODE || 'none';
+  const callerInfo = getCallerInfo();
+
+  // Build the assertion prompt
+  let prompt;
+  if (constraint && expected === undefined) {
+    // Constraint-only mode
+    prompt = `Given this constraint: "${constraint}"
+    
+Actual value: ${JSON.stringify(actual, null, 2)}
+
+Does the actual value satisfy the constraint? Answer only "True" or "False".`;
+  } else if (constraint && expected !== undefined) {
+    // Both expected and constraint provided - use constraint
+    prompt = `Given this constraint: "${constraint}"
+    
+Actual value: ${JSON.stringify(actual, null, 2)}
+Expected value: ${JSON.stringify(expected, null, 2)}
+
+Does the actual value satisfy the constraint? Answer only "True" or "False".`;
+  } else if (expected !== undefined) {
+    // Expected value only
+    prompt = `Does the actual value strictly equal the expected value?
+
+Actual: ${JSON.stringify(actual, null, 2)}
+Expected: ${JSON.stringify(expected, null, 2)}
+
+Answer only "True" or "False".`;
+  } else {
+    throw new Error('Either expected value or constraint must be provided');
+  }
+
+  try {
+    const response = await chatgpt(prompt);
+    const passed = response.trim().toLowerCase() === 'true';
+
+    // Prepare result structure
+    const result = {
+      passed,
+      advice: null,
+      file: callerInfo.file,
+      line: callerInfo.line,
+    };
+
+    // Handle failure cases based on mode
+    if (!passed) {
+      if (mode === 'info' || mode === 'error') {
+        const codeContext = getCodeContext(callerInfo.file, callerInfo.line);
+        result.advice = await generateAdvice(actual, expected, constraint, codeContext);
+
+        const message = `LLM Assertion Failed at ${path.basename(callerInfo.file)}:${
+          callerInfo.line
+        }
+${result.advice}`;
+
+        if (mode === 'error') {
+          throw new Error(message);
+        } else if (mode === 'info') {
+          console.info(message);
+        }
+      }
+    }
+
+    return [passed, result];
+  } catch (error) {
+    if (error.message.includes('LLM Assertion Failed')) {
+      throw error; // Re-throw our custom errors
+    }
+    throw new Error(`LLM expectation failed due to error: ${error.message}`);
+  }
+}
+
+/**
+ * Simple LLM expectation (backward compatibility)
+ */
+export default async function expectSimple(actual, expected, constraint) {
+  const [passed] = await expect(actual, expected, constraint);
+  return passed;
+}

package/src/chains/expect/index.spec.js

@@ -0,0 +1,324 @@
+import { describe, expect as vitestExpect, it, vi, beforeEach, afterEach } from 'vitest';
+import aiExpectSimple, { expect as aiExpect } from './index.js';
+import { longTestTimeout } from '../../constants/common.js';
+
+// Mock the chatgpt function to avoid actual API calls
+vi.mock('../../lib/chatgpt/index.js', () => ({
+  default: vi.fn().mockImplementation((prompt) => {
+    // Handle exact equality checks
+    if (prompt.includes('Does the actual value strictly equal the expected value?')) {
+      if (prompt.includes('Actual: "hello"') && prompt.includes('Expected: "hello"')) {
+        return 'True';
+      }
+      if (prompt.includes('Actual: "goodbye"') && prompt.includes('Expected: "hello"')) {
+        return 'False';
+      }
+      if (prompt.includes('Actual: "hello"') && prompt.includes('Expected: "goodbye"')) {
+        return 'False';
+      }
+    }
+
+    // Handle constraint-based validations (format: "Given this constraint:")
+    if (prompt.includes('Given this constraint:')) {
+      if (prompt.includes('Is this a greeting?') && prompt.includes('Hello world!')) {
+        return 'True';
+      }
+
+      if (prompt.includes('Is this text professional and grammatically correct?')) {
+        if (prompt.includes('well-written, professional email')) {
+          return 'True';
+        }
+      }
+
+      if (prompt.includes('Does this person data look realistic?')) {
+        if (prompt.includes('John Doe') && prompt.includes('"age": 30')) {
+          return 'True';
+        }
+      }
+
+      if (prompt.includes('Is this recommendation specific and actionable?')) {
+        if (prompt.includes('Increase marketing budget by 20%')) {
+          return 'True';
+        }
+      }
+
+      if (prompt.includes('Does this profile represent an experienced software developer')) {
+        if (prompt.includes('Alice Johnson') && prompt.includes('JavaScript')) {
+          return 'True';
+        }
+      }
+
+      if (prompt.includes('Is this story opening engaging')) {
+        if (prompt.includes('Once upon a time')) {
+          return 'True';
+        }
+      }
+    }
+
+    // Default to False for unmatched cases
+    return 'False';
+  }),
+}));
+
+describe('expect chain', () => {
+  let originalEnv;
+
+  beforeEach(() => {
+    originalEnv = process.env.LLM_EXPECT_MODE;
+  });
+
+  afterEach(() => {
+    if (originalEnv !== undefined) {
+      process.env.LLM_EXPECT_MODE = originalEnv;
+    } else {
+      delete process.env.LLM_EXPECT_MODE;
+    }
+  });
+
+  describe('Enhanced API', () => {
+    it(
+      'should return structured results in none mode',
+      async () => {
+        process.env.LLM_EXPECT_MODE = 'none';
+
+        const [passed, details] = await aiExpect('hello', 'hello');
+
+        vitestExpect(passed).toBe(true);
+        vitestExpect(details).toHaveProperty('passed', true);
+        vitestExpect(details).toHaveProperty('advice');
+        vitestExpect(details).toHaveProperty('file');
+        vitestExpect(details).toHaveProperty('line');
+      },
+      longTestTimeout
+    );
+
+    it(
+      'should handle failed assertions in none mode',
+      async () => {
+        process.env.LLM_EXPECT_MODE = 'none';
+
+        const [passed, details] = await aiExpect('hello', 'goodbye');
+
+        vitestExpect(passed).toBe(false);
+        vitestExpect(details.passed).toBe(false);
+        vitestExpect(details).toHaveProperty('advice');
+        vitestExpect(details).toHaveProperty('file');
+        vitestExpect(details).toHaveProperty('line');
+      },
+      longTestTimeout
+    );
+
+    it(
+      'should throw errors in error mode',
+      async () => {
+        process.env.LLM_EXPECT_MODE = 'error';
+
+        await vitestExpect(async () => {
+          await aiExpect('hello', 'goodbye');
+        }).rejects.toThrow('LLM Assertion Failed');
+      },
+      longTestTimeout
+    );
+
+    it(
+      'should log in info mode',
+      async () => {
+        process.env.LLM_EXPECT_MODE = 'info';
+        const consoleSpy = vi.spyOn(console, 'info').mockImplementation(() => {});
+
+        const [passed] = await aiExpect('hello', 'goodbye');
+
+        vitestExpect(passed).toBe(false);
+        vitestExpect(consoleSpy).toHaveBeenCalledWith(
+          vitestExpect.stringContaining('LLM Assertion Failed')
+        );
+
+        consoleSpy.mockRestore();
+      },
+      longTestTimeout
+    );
+
+    it(
+      'should handle constraint-based validation',
+      async () => {
+        process.env.LLM_EXPECT_MODE = 'none';
+
+        const [passed, details] = await aiExpect('Hello world!', undefined, 'Is this a greeting?');
+
+        vitestExpect(passed).toBe(true);
+        vitestExpect(details.passed).toBe(true);
+      },
+      longTestTimeout
+    );
+
+    it(
+      'should validate content quality',
+      async () => {
+        const [passed, details] = await aiExpect(
+          'This is a well-written, professional email with proper grammar and clear intent.',
+          undefined,
+          'Is this text professional and grammatically correct?'
+        );
+
+        vitestExpect(passed).toBe(true);
+        vitestExpect(details).toHaveProperty('file');
+        vitestExpect(details).toHaveProperty('line');
+      },
+      longTestTimeout
+    );
+
+    it(
+      'should validate data structures',
+      async () => {
+        const [passed] = await aiExpect(
+          { name: 'John Doe', age: 30, city: 'New York' },
+          undefined,
+          'Does this person data look realistic?'
+        );
+
+        vitestExpect(passed).toBe(true);
+      },
+      longTestTimeout
+    );
+
+    it(
+      'should handle business logic validation',
+      async () => {
+        const [passed] = await aiExpect(
+          'Increase marketing budget by 20% for Q4 to boost holiday sales',
+          undefined,
+          'Is this recommendation specific and actionable?'
+        );
+
+        vitestExpect(passed).toBe(true);
+      },
+      longTestTimeout
+    );
+
+    it('should throw error when neither expected nor constraint provided', async () => {
+      await vitestExpect(async () => {
+        await aiExpect('test value');
+      }).rejects.toThrow('Either expected value or constraint must be provided');
+    });
+  });
+
+  describe('Simple API (backward compatibility)', () => {
+    it(
+      'should pass for exact equality',
+      async () => {
+        const result = await aiExpectSimple('hello', 'hello');
+        vitestExpect(result).toBe(true);
+      },
+      longTestTimeout
+    );
+
+    it(
+      'should pass for constraint-based validation',
+      async () => {
+        const result = await aiExpectSimple('Hello world!', undefined, 'Is this a greeting?');
+        vitestExpect(result).toBe(true);
+      },
+      longTestTimeout
+    );
+
+    it(
+      'should fail for non-matching values',
+      async () => {
+        const result = await aiExpectSimple('goodbye', 'hello');
+        vitestExpect(result).toBe(false);
+      },
+      longTestTimeout
+    );
+
+    it(
+      'should validate content quality',
+      async () => {
+        const result = await aiExpectSimple(
+          'This is a well-written, professional email with proper grammar.',
+          undefined,
+          'Is this text professional and grammatically correct?'
+        );
+        vitestExpect(result).toBe(true);
+      },
+      longTestTimeout
+    );
+  });
+
+  describe('Environment variable handling', () => {
+    it(
+      'should default to none mode when env var is not set',
+      async () => {
+        delete process.env.LLM_EXPECT_MODE;
+
+        const [passed] = await aiExpect('hello', 'goodbye');
+        vitestExpect(passed).toBe(false);
+        // Should not throw in none mode
+      },
+      longTestTimeout
+    );
+
+    it(
+      'should handle invalid env var values',
+      async () => {
+        process.env.LLM_EXPECT_MODE = 'invalid';
+
+        const [passed] = await aiExpect('hello', 'goodbye');
+        vitestExpect(passed).toBe(false);
+        // Should default to none mode and not throw
+      },
+      longTestTimeout
+    );
+  });
+
+  describe('Advanced features', () => {
+    it(
+      'should provide file and line information',
+      async () => {
+        const [, details] = await aiExpect('hello', 'hello');
+
+        vitestExpect(details.file).toBeDefined();
+        vitestExpect(details.line).toBeTypeOf('number');
+        vitestExpect(details.line).toBeGreaterThan(0);
+      },
+      longTestTimeout
+    );
+
+    it(
+      'should handle complex object comparisons',
+      async () => {
+        const userProfile = {
+          name: 'Alice Johnson',
+          skills: ['JavaScript', 'Python', 'React'],
+          experience: '5 years',
+          level: 'Senior Developer',
+        };
+
+        const [passed] = await aiExpect(
+          userProfile,
+          undefined,
+          'Does this profile represent an experienced software developer with modern skills?'
+        );
+
+        vitestExpect(passed).toBe(true);
+      },
+      longTestTimeout
+    );
+
+    it(
+      'should validate creative content',
+      async () => {
+        const storyOpening =
+          'Once upon a time, in a land far away, there lived a brave knight who embarked on a quest to save the kingdom from an ancient curse.';
+
+        const [passed] = await aiExpect(
+          storyOpening,
+          undefined,
+          'Is this story opening engaging and sets up a clear adventure narrative?'
+        );
+
+        vitestExpect(passed).toBe(true);
+      },
+      longTestTimeout
+    );
+  });
+});

package/src/chains/filter-ambiguous/README.md

@@ -0,0 +1,11 @@
+# filter-ambiguous
+
+Identify ambiguous words or phrases in text. Sentences are first scored for overall uncertainty and the highest scoring ones are examined for unclear terms. Each term is scored in context so you can zero in on the most confusing language.
+
+```javascript
+import filterAmbiguous from './index.js';
+
+const text = `I saw her duck\nThe magician made a little girl disappear\nHe fed her dog food`;
+const result = await filterAmbiguous(text, { topN: 3 });
+// => [{ term: 'duck', sentence: 'I saw her duck', score: 9 }, ...]
+```

package/src/chains/filter-ambiguous/index.examples.js

@@ -0,0 +1,20 @@
+import { describe, it, expect } from 'vitest';
+import { longTestTimeout } from '../../constants/common.js';
+import filterAmbiguous from './index.js';
+
+describe('filterAmbiguous examples', () => {
+  it(
+    'highlights unclear language',
+    async () => {
+      const text = `I saw her duck\nThe magician made a little girl disappear\nHe fed her dog food`;
+      const result = await filterAmbiguous(text, { topN: 2 });
+      expect(result.length).toBeGreaterThan(0);
+      result.forEach((r) => {
+        expect(r).toHaveProperty('term');
+        expect(r).toHaveProperty('sentence');
+        expect(typeof r.score).toBe('number');
+      });
+    },
+    longTestTimeout
+  );
+});

package/src/chains/filter-ambiguous/index.js

@@ -0,0 +1,49 @@
+import list from '../list/index.js';
+import bulkScore from '../bulk-score/index.js';
+
+export default async function filterAmbiguous(text, config = {}) {
+  const { topN = 10, chunkSize = 5, llm, ...options } = config;
+  if (!text) return [];
+  const sentences = text
+    .split('\n')
+    .map((s) => s.trim())
+    .filter(Boolean);
+  if (sentences.length === 0) return [];
+
+  const { scores: sentenceScores } = await bulkScore(
+    sentences,
+    'How ambiguous or easily misinterpreted is this sentence?',
+    { chunkSize, llm, ...options }
+  );
+
+  const rankedSentences = sentences
+    .map((s, i) => ({ sentence: s, score: sentenceScores[i] ?? 0 }))
+    .sort((a, b) => b.score - a.score)
+    .slice(0, topN);
+
+  const termPairs = [];
+  for (const { sentence } of rankedSentences) {
+    // eslint-disable-next-line no-await-in-loop
+    const terms = await list('Ambiguous words or short phrases', {
+      attachments: { text: sentence },
+      targetNewItemsCount: 5,
+      llm,
+      ...options,
+    });
+    terms.forEach((term) => {
+      termPairs.push({ term, sentence });
+    });
+  }
+
+  if (termPairs.length === 0) return [];
+
+  const { scores } = await bulkScore(
+    termPairs.map((p) => `${p.term} | ${p.sentence}`),
+    'Score how ambiguous the term is within the sentence.',
+    { chunkSize, llm, ...options }
+  );
+
+  const scored = termPairs.map((p, i) => ({ ...p, score: scores[i] }));
+  scored.sort((a, b) => (b.score || 0) - (a.score || 0));
+  return scored.slice(0, topN);
+}

package/src/chains/filter-ambiguous/index.spec.js

@@ -0,0 +1,31 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import filterAmbiguous from './index.js';
+import bulkScore from '../bulk-score/index.js';
+import list from '../list/index.js';
+
+vi.mock('../bulk-score/index.js', () => ({
+  default: vi.fn(),
+}));
+
+vi.mock('../list/index.js', () => ({
+  default: vi.fn(),
+}));
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe('filterAmbiguous chain', () => {
+  it('returns scored ambiguous terms', async () => {
+    bulkScore
+      .mockResolvedValueOnce({ scores: [1, 9], reference: [] })
+      .mockResolvedValueOnce({ scores: [8, 3], reference: [] });
+    list.mockResolvedValueOnce(['alpha', 'beta']).mockResolvedValueOnce([]);
+
+    const result = await filterAmbiguous('s1\ns2', { topN: 1 });
+
+    expect(result).toStrictEqual([{ term: 'alpha', sentence: 's2', score: 8 }]);
+    expect(bulkScore).toHaveBeenCalled();
+    expect(list).toHaveBeenCalled();
+  });
+});

package/src/chains/glossary/README.md

@@ -0,0 +1,19 @@
+# glossary
+
+Identify difficult or technical terms in any text so you can explain them later.
+The chain breaks long passages into paragraphs and uses the `bulk-map` retry
+utility to collect candidate terms from each chunk. Any failed paragraphs are
+automatically retried. Finally the terms are ranked by importance using `sort`.
+
+```javascript
+import glossary from './index.js';
+
+const blog = `The chef explained how umami develops through the Maillard reaction alongside sous-vide techniques.`;
+
+const terms = await glossary(blog, { maxTerms: 3 });
+console.log(terms);
+// => ['Maillard reaction', 'sous-vide', 'umami']
+```
+
+This is handy when you want to add a quick glossary sidebar to a dense article
+so everyday readers aren't left guessing what key terms mean.