@far-world-labs/verblets 0.1.1 → 0.1.3

package/src/chains/disambiguate/index.js

@@ -0,0 +1,92 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import chatGPT from '../../lib/chatgpt/index.js';
+import listFilter from '../../verblets/list-filter/index.js';
+import { constants as promptConstants } from '../../prompts/index.js';
+import modelService from '../../services/llm-model/index.js';
+
+const { onlyJSONStringArray } = promptConstants;
+
+// Get the directory of this module
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+/**
+ * Load the JSON schema for disambiguate meanings results
+ * @returns {Promise<Object>} JSON schema for validation
+ */
+async function getDisambiguateMeaningsSchema() {
+  const schemaPath = path.join(__dirname, 'disambiguate-meanings-result.json');
+  return JSON.parse(await fs.readFile(schemaPath, 'utf8'));
+}
+
+/**
+ * Create model options for structured outputs
+ * @param {string|Object} llm - LLM model name or configuration object
+ * @returns {Promise<Object>} Model options for chatGPT
+ */
+async function createModelOptions(llm = 'fastGoodCheap') {
+  const schema = await getDisambiguateMeaningsSchema();
+
+  const responseFormat = {
+    type: 'json_schema',
+    json_schema: {
+      name: 'disambiguate_meanings_result',
+      schema,
+    },
+  };
+
+  if (typeof llm === 'string') {
+    return {
+      modelName: llm,
+      response_format: responseFormat,
+    };
+  } else {
+    return {
+      ...llm,
+      response_format: responseFormat,
+    };
+  }
+}
+
+const meaningsPrompt = (term) => {
+  return `${onlyJSONStringArray}
+List all distinct dictionary meanings or common uses of "${term}".
+Return a JSON object with a "meanings" array containing the distinct meanings.
+${onlyJSONStringArray}`;
+};
+
+export const getMeanings = async (term, config = {}) => {
+  const { model = modelService.getBestPublicModel(), llm, ...options } = config;
+  const prompt = meaningsPrompt(term);
+  const budget = model.budgetTokens(prompt);
+  const modelOptions = await createModelOptions(llm);
+  const response = await chatGPT(prompt, {
+    maxTokens: budget.completion,
+    modelOptions,
+    ...options,
+  });
+
+  // With structured outputs, response should already be parsed and validated
+  const parsed = typeof response === 'string' ? JSON.parse(response) : response;
+  // Extract meanings from the object structure
+  const resultArray = parsed?.meanings || parsed;
+  return Array.isArray(resultArray) ? resultArray.filter(Boolean) : [];
+};
+
+export default async function disambiguate({
+  term,
+  context,
+  model = modelService.getBestPublicModel(),
+  ...config
+} = {}) {
+  const { llm, ...options } = config;
+  const meanings = await getMeanings(term, { model, llm, ...options });
+  const best = await listFilter(
+    meanings,
+    `the meaning of "${term}" in context: ${context}. Keep only the single best matching meaning.`,
+    { llm, ...options }
+  );
+  return { meaning: best[0], meanings };
+}

package/src/chains/disambiguate/index.spec.js

@@ -0,0 +1,25 @@
+import { describe, expect, it, vi } from 'vitest';
+import disambiguate from './index.js';
+
+vi.mock('../../lib/chatgpt/index.js', () => ({
+  default: vi.fn(async (prompt) => {
+    if (/List all distinct dictionary meanings/.test(prompt)) {
+      return '["financial institution","edge of a river"]';
+    }
+    return '[]';
+  }),
+}));
+
+vi.mock('../../verblets/list-filter/index.js', () => ({
+  default: vi.fn(async (list) => {
+    return [list[0]];
+  }),
+}));
+
+describe('disambiguate chain', () => {
+  it('selects meaning based on context', async () => {
+    const result = await disambiguate({ term: 'bank', context: 'withdraw money' });
+    expect(result.meaning).toBe('financial institution');
+    expect(result.meanings).toStrictEqual(['financial institution', 'edge of a river']);
+  });
+});

package/src/chains/dismantle/README.md

@@ -0,0 +1,67 @@
+# dismantle
+
+Break down complex systems into a tree of components using an LLM. `dismantle` creates a `ChainTree` that you can grow to any depth or inspect piece by piece.
+
+## Example
+
+```javascript
+import { dismantle, simplifyTree } from './index.js';
+
+// Break down complex systems into components
+const chain = dismantle('AirPods Pro');
+await chain.makeSubtree({ depth: 1 });
+console.log(simplifyTree(chain.getTree()));
+/* Returns:
+{
+  id: '...',
+  name: 'AirPods Pro',
+  parts: [
+    { id: '...', name: 'H2 Chip' },
+    { id: '...', name: 'Drivers: custom_dynamic' },
+    { id: '...', name: 'Noise Sensors' },
+    { id: '...', name: 'Battery: lithium_ion' }
+  ]
+}
+*/
+```
+
+## API
+
+### `dismantle(name, [options])`
+Returns a `ChainTree` for `name`. `options` lets you override how components are discovered:
+
+- `decompose(component)` – return an array of subcomponent names.
+  ```javascript
+  const chain = dismantle('Web App', {
+    decompose: async ({ name }) => fetch(`/api/parts?for=${name}`).then(r => r.json())
+  });
+  ```
+- `enhance(component)` – enrich a node with metadata or known variants.
+  ```javascript
+  const chain = dismantle('Server', {
+    enhance: async ({ name }) => ({ name, options: await lookUpVariants(name) })
+  });
+  ```
+- `makeId()` – create unique IDs (defaults to `uuid.v4`).
+- `enhanceFixes` – text appended to LLM prompts used by `enhance`.
+  ```javascript
+  const chain = dismantle('Bike', { enhanceFixes: 'Prefer standard part names.' });
+  ```
+- `decomposeFixes` – text appended to prompts used by `decompose`.
+
+### `ChainTree`
+Returned by `dismantle`. Methods:
+
+- `makeSubtree({ depth })` – grow the tree from the root for a number of levels.
+  ```javascript
+  await chain.makeSubtree({ depth: 2 });
+  ```
+- `attachSubtree({ find, depth })` – expand a node that matches a predicate.
+  ```javascript
+  await chain.attachSubtree({ find: (n) => n.name === 'Battery', depth: 1 });
+  ```
+- `getTree()` – access the internal detailed tree.
+
+### `simplifyTree(node)`
+Convert a detailed node or tree into `{ id, name, parts }` for easy consumption.
+

package/src/chains/dismantle/dismantle.examples.js

@@ -0,0 +1,27 @@
+import { describe, expect, it } from 'vitest';
+import { longTestTimeout } from '../../constants/common.js';
+import { dismantle } from './index.js';
+
+const examples = [
+  {
+    name: 'Basic usage',
+    inputs: { text: 'test' },
+    want: { result: {} },
+  },
+];
+
+describe('Dismantle chain', () => {
+  examples.forEach((example) => {
+    it(
+      example.name,
+      async () => {
+        const result = await dismantle(example.inputs.text);
+
+        if (example.want.typeOfResult) {
+          expect(JSON.stringify(result.tree)).toStrictEqual(JSON.stringify(example.want.result));
+        }
+      },
+      longTestTimeout
+    );
+  });
+});

package/src/chains/dismantle/index.js

@@ -1,12 +1,7 @@
-/* eslint-disable no-await-in-loop */
-
 import { v4 as uuid } from 'uuid';
 
 import chatGPT from '../../lib/chatgpt/index.js';
-import {
-  constants as promptConstants,
-  outputSuccinctNames,
-} from '../../prompts/index.js';
+import { outputSuccinctNames, constants as promptConstants } from '../../prompts/index.js';
 import modelService from '../../services/llm-model/index.js';
 import toObject from '../../verblets/to-object/index.js';
 
@@ -82,15 +77,11 @@ const defaultDecompose = async ({
   focus,
   rootName,
   fixes,
-  model = modelService.getBestAvailableModel(),
+  model = modelService.getBestPublicModel(),
 } = {}) => {
   const focusFormatted = focus ? `: ${focus}` : '';
 
-  const promptCreated = subComponentsPrompt(
-    `${name}${focusFormatted}`,
-    rootName,
-    fixes
-  );
+  const promptCreated = subComponentsPrompt(`${name}${focusFormatted}`, rootName, fixes);
   const budget = model.budgetTokens(promptCreated);
   return toObject(
     await chatGPT(promptCreated, {
@@ -107,7 +98,7 @@ const defaultEnhance = async ({
   name,
   rootName,
   fixes,
-  model = modelService.getBestAvailableModel(),
+  model = modelService.getBestPublicModel(),
 } = {}) => {
   const promptCreated = componentOptionsPrompt(name, rootName, fixes);
   const budget = model.budgetTokens(promptCreated);
@@ -207,6 +198,7 @@ const makeSubtree = async ({
 
   const children = [];
   for (const child of tree.children) {
+    // eslint-disable-next-line no-await-in-loop
     const subtree = await makeSubtree({
       tree: child,
       rootName,
@@ -245,10 +237,7 @@ export const simplifyTree = (node) => {
 };
 
 class ChainTree {
-  constructor(
-    name,
-    { decompose, enhance, makeId, enhanceFixes, decomposeFixes } = {}
-  ) {
+  constructor(name, { decompose, enhance, makeId, enhanceFixes, decomposeFixes } = {}) {
     this.rootName = name;
     this.tree = {};
     this.decompose = decompose;

package/src/chains/dismantle/index.spec.js

@@ -25,8 +25,7 @@ describe('Dismantle chain', () => {
       const result = await dismantle(example.inputs.text);
 
       if (example.want.typeOfResult) {
-        expect(JSON.stringify(result.tree))
-          .toStrictEqual(JSON.stringify(example.want.result));
+        expect(JSON.stringify(result.tree)).toStrictEqual(JSON.stringify(example.want.result));
       }
     });
   });

package/src/chains/expect/README.md

@@ -0,0 +1,171 @@
+# LLM Expect Chain
+
+Advanced intelligent assertions with debugging features, environment variable modes, and structured results. This chain provides enhanced functionality beyond the basic [expect verblet](../../verblets/expect/).
+
+## Why Use the Chain?
+
+While the verblet provides simple pass/fail assertions, the chain offers:
+
+- **Advanced Debugging**: Automatic code context analysis and intelligent advice
+- **Environment Modes**: Different behaviors for development, testing, and CI
+- **Structured Results**: Detailed information about assertions and failures
+- **Stack Trace Integration**: Automatic detection of calling file and line
+
+## Environment Variable Modes
+
+Control behavior with the `LLM_EXPECT_MODE` environment variable:
+
+### `none` (default)
+Silent mode - returns structured results without throwing or logging
+```bash
+export LLM_EXPECT_MODE=none
+```
+
+### `info` 
+Development mode - logs failures with context and advice to console
+```bash
+export LLM_EXPECT_MODE=info
+```
+
+### `error`
+CI/Testing mode - throws detailed errors on assertion failures
+```bash
+export LLM_EXPECT_MODE=error
+```
+
+## API Reference
+
+### Enhanced API: `expect(actual, expected?, constraint?)`
+
+Returns a tuple `[passed, details]` with structured results:
+
+```javascript
+import { expect } from './index.js';
+
+const [passed, details] = await expect(
+  actualValue,
+  "Does this meet our quality standards?"
+);
+
+console.log(details);
+/* Returns:
+{
+  passed: false,
+  advice: "ISSUE: Content lacks specific examples...",
+  file: "/path/to/test.js",
+  line: 42
+}
+*/
+```
+
+### Simple API: `expect(actual, expected?, constraint?)`
+
+Backward compatible with the verblet - returns boolean:
+
+```javascript
+import expect from './index.js';
+
+const result = await expect("hello", "hello");
+// Returns: true
+```
+
+## Real-World Use Cases
+
+### Content Quality Assurance with Debugging
+
+```javascript
+import { expect } from './index.js';
+
+// Set development mode for detailed feedback
+process.env.LLM_EXPECT_MODE = 'info';
+
+const [passed, details] = await expect(
+  generatedCopy,
+  "Is this marketing copy professional, engaging, and free of grammatical errors?"
+);
+
+if (!passed) {
+  console.log(`Failed at ${details.file}:${details.line}`);
+  console.log(details.advice);
+  // ISSUE: Copy contains informal language and lacks call-to-action
+  // FIX: Replace casual phrases with professional alternatives and add clear CTA
+  // CONTEXT: Marketing copy should maintain professional tone while being engaging
+}
+```
+
+## Advanced Features
+
+### Automatic Code Context Analysis
+
+The chain automatically:
+- Detects the calling file and line number
+- Reads 400 lines before and 100 lines after the assertion
+- Provides this context to the LLM for better debugging advice
+
+### Intelligent Advice Generation
+
+Failed assertions generate structured advice:
+- **ISSUE**: Brief description of why the assertion failed
+- **FIX**: Specific actionable steps to resolve the issue  
+- **CONTEXT**: Additional context about the problem and root causes
+
+### Environment-Aware Behavior
+
+Different modes for different environments:
+- **Development**: Rich console output with advice
+- **Testing**: Detailed error throwing for CI/CD
+- **Production**: Silent operation with structured results
+
+## Best Practices
+
+### **Write Specific Constraints**
+```javascript
+// ❌ Vague
+await expect(text, "Is this good?");
+
+// ✅ Specific  
+await expect(text, "Is this text grammatically correct, under 100 words, and written in a professional tone?");
+```
+
+### **Use Structured Results**
+```javascript
+const [passed, details] = await expect(content, constraint);
+
+// Access rich debugging information
+console.log('File:', details.file);
+console.log('Line:', details.line);
+console.log('Advice:', details.advice);
+```
+
+### **Combine with Traditional Tests**
+```javascript
+// Traditional assertion for structure
+expect(response).toHaveProperty('status');
+expect(response.status).toBe(200);
+
+// LLM assertion for content quality
+const [passed] = await expect(
+  response.message,
+  "Is this error message helpful and user-friendly?"
+);
+```
+
+### Environment Modes
+
+```bash
+# Silent operation (default)
+export LLM_EXPECT_MODE=none
+
+# Log debugging advice on failures
+export LLM_EXPECT_MODE=info
+
+# Throw with detailed debugging advice
+export LLM_EXPECT_MODE=error
+```
+
+## Best Practices
+
+- **Be specific**: Use clear, detailed constraints
+- **Test qualitatively**: Verify qualitative details with a clear yes/no answer
+- **Use robust constraints**: Write criteria to pass under a wide range of input hallucinations. Assert cases that classical software can't. Tune the level of rigorousness to the model performing the eval.
+- **Performance**: Remember this makes an LLM call - use judiciously

package/src/chains/expect/index.examples.js

@@ -0,0 +1,146 @@
+import { describe, expect, it, beforeAll, afterAll } from 'vitest';
+
+import { expect as aiExpect } from './index.js';
+import { longTestTimeout } from '../../constants/common.js';
+
+const examples = [
+  {
+    inputs: {
+      actual: 'Hello world!',
+      constraint: 'Is this a greeting?',
+    },
+    want: { result: true },
+  },
+  {
+    inputs: {
+      actual: 'Goodbye cruel world',
+      constraint: 'Is this a greeting?',
+    },
+    want: { result: false },
+  },
+  {
+    inputs: {
+      actual: 'hello',
+      expected: 'hello',
+    },
+    want: { result: true },
+  },
+  {
+    inputs: {
+      actual: 'hello',
+      expected: 'goodbye',
+    },
+    want: { result: false },
+  },
+  {
+    inputs: {
+      actual: 'This is a well-written, professional email with proper grammar and clear intent.',
+      constraint: 'Is this text professional and grammatically correct?',
+    },
+    want: { result: true },
+  },
+  {
+    inputs: {
+      actual: { name: 'John Doe', age: 30, city: 'New York' },
+      constraint: 'Does this person data look realistic?',
+    },
+    want: { result: true },
+  },
+];
+
+describe('LLM Expect Chain', () => {
+  // Set environment mode to 'none' for all tests to avoid throwing
+  const originalMode = process.env.LLM_EXPECT_MODE;
+
+  beforeAll(() => {
+    process.env.LLM_EXPECT_MODE = 'none';
+  });
+
+  afterAll(() => {
+    if (originalMode !== undefined) {
+      process.env.LLM_EXPECT_MODE = originalMode;
+    } else {
+      delete process.env.LLM_EXPECT_MODE;
+    }
+  });
+
+  examples.forEach((example) => {
+    const description = example.inputs.constraint
+      ? `${JSON.stringify(example.inputs.actual).slice(0, 30)}... - ${example.inputs.constraint}`
+      : `${JSON.stringify(example.inputs.actual)} === ${JSON.stringify(example.inputs.expected)}`;
+
+    it(
+      description,
+      async () => {
+        const [result, details] = await aiExpect(
+          example.inputs.actual,
+          example.inputs.expected,
+          example.inputs.constraint
+        );
+
+        expect(result).toBe(example.want.result);
+        expect(details).toHaveProperty('passed', example.want.result);
+        expect(details).toHaveProperty('file');
+        expect(details).toHaveProperty('line');
+        expect(typeof details.line).toBe('number');
+      },
+      longTestTimeout
+    );
+  });
+
+  it(
+    'should provide detailed debugging information on failure',
+    async () => {
+      const [result, details] = await aiExpect(
+        'This is clearly wrong content',
+        undefined,
+        'Is this a professional business email?'
+      );
+
+      expect(result).toBe(false);
+      expect(details.passed).toBe(false);
+      expect(details.file).toBeDefined();
+      expect(details.line).toBeGreaterThan(0);
+
+      // In none mode, advice should be null for failures
+      expect(details.advice).toBeNull();
+    },
+    longTestTimeout
+  );
+
+  it(
+    'should handle complex business logic validation',
+    async () => {
+      const businessRecommendation =
+        'Increase marketing budget by 20% for Q4 to boost holiday sales and target demographics aged 25-45 through social media campaigns';
+
+      const [result, details] = await aiExpect(
+        businessRecommendation,
+        undefined,
+        'Is this recommendation specific, actionable, and includes measurable targets?'
+      );
+
+      expect(result).toBe(true);
+      expect(details.passed).toBe(true);
+      expect(details.file).toBeDefined();
+    },
+    longTestTimeout
+  );
+
+  it(
+    'should validate creative content quality',
+    async () => {
+      const storyOpening =
+        'Once upon a time, in a land far away, there lived a brave knight who embarked on a quest to save the kingdom from an ancient curse that had plagued the realm for centuries.';
+
+      const [result] = await aiExpect(
+        storyOpening,
+        undefined,
+        'Is this story opening engaging, sets up clear conflict, and follows good narrative structure?'
+      );
+
+      expect(result).toBe(true);
+    },
+    longTestTimeout
+  );
+});