@far-world-labs/verblets 0.1.7 → 0.3.2

package/src/verblets/central-tendency/index.js

@@ -1,171 +0,0 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-import chatGPT from '../../lib/chatgpt/index.js';
-import { onlyJSON, strictFormat } from '../../prompts/constants.js';
-
-// Get the directory of this module
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-/**
- * Load the JSON schema for central tendency results
- * @returns {Promise<Object>} JSON schema for validation
- */
-async function getCentralTendencySchema() {
-  const schemaPath = path.join(__dirname, 'central-tendency-result.json');
-  return JSON.parse(await fs.readFile(schemaPath, 'utf8'));
-}
-
-/**
- * Core prompt template for central tendency evaluation using cognitive science principles.
- * Suitable for both individual and bulk processing.
- */
-export const CENTRAL_TENDENCY_PROMPT = `Use cognitive science principles of prototype theory and family resemblance:
-
-{context}
-{coreFeatures}
-
-COGNITIVE PRINCIPLES:
-1. Prototype Theory: Categories have graded structure with central and peripheral members
-2. Family Resemblance: Members share overlapping features without identical characteristics
-3. Feature Analysis: Consider both core (definitional) and characteristic (typical) features
-4. Functional Centrality: Assess how well the item serves the category's purpose
-
-ASSESSMENT CRITERIA:
-- Feature overlap with seed items
-- Possession of core definitional features
-- Functional alignment with category purpose
-- Typicality relative to category prototype
-
-CENTRALITY SCORING GUIDE (use precise decimals):
-• 0.0-0.1: Not a category member (completely unrelated)
-• 0.1-0.2: Extremely peripheral (minimal connection, metaphorical only)
-• 0.2-0.3: Peripheral member (weak connection, few shared features)
-• 0.3-0.4: Marginal member (some shared features but atypical)
-• 0.4-0.5: Borderline member (mixed typical/atypical features)
-• 0.5-0.6: Atypical but clear member (definitionally belongs but unusual)
-• 0.6-0.7: Moderately typical (good example with some variations)
-• 0.7-0.8: Highly typical (strong example, most expected features)
-• 0.8-0.9: Very prototypical (excellent example, nearly all expected features)
-• 0.9-1.0: Maximally prototypical (perfect/ideal example of the category)
-
-{outputRequirements}`;
-
-/**
- * Build a prompt for evaluating centrality
- * @param {string} item - The item to evaluate (for single) or placeholder for bulk
- * @param {string[]} seedItems - Array of seed items for comparison
- * @param {Object} config - Configuration options
- * @returns {string} Complete prompt
- */
-export function buildCentralTendencyPrompt(
-  item,
-  seedItems,
-  { context = '', coreFeatures = [], outputRequirements = null } = {}
-) {
-  const contextLine = context ? `Context: ${context}` : '';
-  const coreFeaturesLine =
-    coreFeatures.length > 0 ? `Core Features: ${coreFeatures.join(', ')}` : '';
-
-  // Default structured output requirements for individual verblet use
-  const defaultOutputRequirements = `OUTPUT REQUIREMENTS:
-${onlyJSON}
-${strictFormat}
-
-Required JSON structure:
-{
-  "score": <number between 0.0 and 1.0>,
-  "reason": "<brief explanation of the centrality assessment>",
-  "confidence": <number between 0.0 and 1.0 indicating confidence in the assessment>
-}
-
-The "reason" should briefly explain why the item received its centrality score based on feature overlap, typicality, and functional alignment with the seed items.
-The "confidence" should reflect how certain you are about the assessment (higher for clear cases, lower for borderline cases).`;
-
-  const outputRequirementsLine = outputRequirements || defaultOutputRequirements;
-
-  const prompt = CENTRAL_TENDENCY_PROMPT.replace('{context}', contextLine)
-    .replace('{coreFeatures}', coreFeaturesLine)
-    .replace('{outputRequirements}', outputRequirementsLine);
-
-  return `Evaluate how central "${item}" is among these category members: ${seedItems.join(', ')}
-
-${prompt}`;
-}
-
-/**
- * Create model options for structured outputs
- * @param {string|Object} llm - LLM model name or configuration object
- * @param {string} schemaName - Name for the JSON schema
- * @param {Object} [customSchema] - Custom schema to use instead of default
- * @returns {Promise<Object>} Model options for chatGPT
- */
-async function createModelOptions(
-  llm = 'fastGoodCheap',
-  schemaName = 'central_tendency_result',
-  customSchema = null
-) {
-  const schema = customSchema || (await getCentralTendencySchema());
-
-  const responseFormat = {
-    type: 'json_schema',
-    json_schema: {
-      name: schemaName,
-      schema,
-    },
-  };
-
-  if (typeof llm === 'string') {
-    return {
-      modelName: llm,
-      response_format: responseFormat,
-    };
-  } else {
-    return {
-      ...llm,
-      response_format: responseFormat,
-    };
-  }
-}
-
-/**
- * Parse response from chatGPT
- * @param {string|Object} response - Response from chatGPT
- * @returns {Object} Parsed response object
- */
-function parseResponse(response) {
-  return typeof response === 'string' ? JSON.parse(response) : response;
-}
-
-/**
- * Evaluate how central an item is among category members using cognitive science principles.
- *
- * Based on prototype theory and family resemblance, this function assesses graded typicality
- * by analyzing feature overlap, core characteristics, and functional alignment with seed items.
- *
- * @param {string} item - The item to evaluate for centrality
- * @param {string[]} seedItems - Array of known category members for comparison
- * @param {Object} [config={}] - Configuration options
- * @param {string} [config.context=''] - Context description for evaluation
- * @param {string[]} [config.coreFeatures=[]] - Known core/definitional features of the category
- * @param {string} [config.llm='fastGoodCheap'] - LLM model to use
- * @returns {Promise<{score: number, reason: string, confidence: number}>}
- */
-export default async function centralTendency(item, seedItems, config = {}) {
-  if (!item || typeof item !== 'string') {
-    throw new Error('Item must be a non-empty string');
-  }
-
-  if (!Array.isArray(seedItems) || seedItems.length === 0) {
-    throw new Error('seedItems must be a non-empty array');
-  }
-
-  const { context = '', coreFeatures = [], llm = 'fastGoodCheap' } = config;
-
-  const prompt = buildCentralTendencyPrompt(item, seedItems, { context, coreFeatures });
-  const modelOptions = await createModelOptions(llm, 'central_tendency_result');
-
-  const response = await chatGPT(prompt, { modelOptions });
-  return parseResponse(response);
-}

package/src/verblets/central-tendency/index.spec.js

@@ -1,148 +0,0 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
-import centralTendency from './index.js';
-
-// Mock the LLM service
-vi.mock('../../lib/chatgpt/index.js', () => ({
-  default: vi.fn(),
-}));
-
-import chatGPT from '../../lib/chatgpt/index.js';
-
-describe('centralTendency', () => {
-  beforeEach(() => {
-    vi.clearAllMocks();
-  });
-
-  it('evaluates centrality with config object', async () => {
-    const mockResponse = {
-      score: 0.85,
-      reason: 'High feature overlap with seed items',
-      confidence: 0.9,
-    };
-
-    chatGPT.mockResolvedValue(mockResponse);
-
-    const result = await centralTendency('robin', ['sparrow', 'bluejay', 'cardinal'], {
-      context: 'Evaluate based on typical bird characteristics',
-      coreFeatures: ['feathers', 'beak', 'lays eggs'],
-      llm: 'fastGoodCheap',
-    });
-
-    expect(result).toEqual({
-      score: 0.85,
-      reason: 'High feature overlap with seed items',
-      confidence: 0.9,
-    });
-
-    expect(chatGPT).toHaveBeenCalledWith(
-      expect.stringContaining('Evaluate how central "robin" is among these category members'),
-      { modelOptions: expect.objectContaining({ modelName: 'fastGoodCheap' }) }
-    );
-  });
-
-  it('uses default LLM model when not specified', async () => {
-    const mockResponse = {
-      score: 0.75,
-      reason: 'Good match',
-      confidence: 0.8,
-    };
-
-    chatGPT.mockResolvedValue(mockResponse);
-
-    const result = await centralTendency('penguin', ['robin', 'sparrow'], {
-      context: 'Bird evaluation',
-    });
-
-    expect(result.score).toBe(0.75);
-    expect(chatGPT).toHaveBeenCalledWith(expect.any(String), {
-      modelOptions: expect.objectContaining({ modelName: 'fastGoodCheap' }),
-    });
-  });
-
-  it('throws error for invalid item', async () => {
-    await expect(centralTendency('', ['seed1', 'seed2'])).rejects.toThrow(
-      'Item must be a non-empty string'
-    );
-    await expect(centralTendency(null, ['seed1', 'seed2'])).rejects.toThrow(
-      'Item must be a non-empty string'
-    );
-  });
-
-  it('throws error for invalid seedItems', async () => {
-    await expect(centralTendency('item', [])).rejects.toThrow(
-      'seedItems must be a non-empty array'
-    );
-    await expect(centralTendency('item', null)).rejects.toThrow(
-      'seedItems must be a non-empty array'
-    );
-  });
-
-  it('handles JSON string response', async () => {
-    const mockResponse = JSON.stringify({
-      score: 0.6,
-      reason: 'Moderate centrality',
-      confidence: 0.7,
-    });
-
-    chatGPT.mockResolvedValue(mockResponse);
-
-    const result = await centralTendency('item', ['seed1', 'seed2']);
-
-    expect(result).toEqual({
-      score: 0.6,
-      reason: 'Moderate centrality',
-      confidence: 0.7,
-    });
-  });
-
-  it('handles object response directly', async () => {
-    const mockResponse = {
-      score: 0.8,
-      reason: 'High centrality',
-      confidence: 0.9,
-    };
-
-    chatGPT.mockResolvedValue(mockResponse);
-
-    const result = await centralTendency('item', ['seed1', 'seed2']);
-
-    expect(result).toEqual(mockResponse);
-  });
-
-  it('builds correct prompt with context and core features', async () => {
-    const mockResponse = {
-      score: 0.7,
-      reason: 'Test result',
-      confidence: 0.8,
-    };
-
-    chatGPT.mockResolvedValue(mockResponse);
-
-    await centralTendency('robin', ['sparrow', 'bluejay'], {
-      context: 'Bird evaluation context',
-      coreFeatures: ['feathers', 'beak', 'flight'],
-    });
-
-    const calledPrompt = chatGPT.mock.calls[0][0];
-    expect(calledPrompt).toContain('Context: Bird evaluation context');
-    expect(calledPrompt).toContain('Core Features: feathers, beak, flight');
-    expect(calledPrompt).toContain('sparrow, bluejay');
-  });
-
-  it('uses JSON schema validation', async () => {
-    const mockResponse = {
-      score: 0.6,
-      reason: 'Assessment with schema',
-      confidence: 0.8,
-    };
-
-    chatGPT.mockResolvedValue(mockResponse);
-
-    await centralTendency('item', ['seed1', 'seed2'], {});
-
-    const modelOptions = chatGPT.mock.calls[0][1].modelOptions;
-    expect(modelOptions).toHaveProperty('response_format');
-    expect(modelOptions.response_format.type).toBe('json_schema');
-    expect(modelOptions.response_format.json_schema.name).toBe('central_tendency_result');
-  });
-});

package/src/verblets/enum/index.examples.js

@@ -1,30 +0,0 @@
-import { describe, expect, it } from 'vitest';
-
-import enumValue from './index.js';
-import { longTestTimeout } from '../../constants/common.js';
-
-const examples = [
-  {
-    inputs: {
-      text: 'What is the top color on a traffic light',
-      enum: { green: 1, yellow: 1, red: 1, purple: 1 },
-    },
-    want: { result: 'red' },
-  },
-];
-
-describe('Enum verblet', () => {
-  examples.forEach((example) => {
-    it(
-      example.inputs.text,
-      async () => {
-        const result = await enumValue(example.inputs.text, example.inputs.enum);
-
-        if (example.want.result) {
-          expect(result).toStrictEqual(example.want.result);
-        }
-      },
-      longTestTimeout
-    );
-  });
-});

package/src/verblets/enum/index.js

@@ -1,18 +0,0 @@
-import chatGPT from '../../lib/chatgpt/index.js';
-import stripResponse from '../../lib/strip-response/index.js';
-import toEnum from '../../lib/to-enum/index.js';
-import { asEnum, constants } from '../../prompts/index.js';
-
-const { asUndefinedByDefault, contentIsQuestion, explainAndSeparate } = constants;
-
-export default async (text, enumVal, config = {}) => {
-  const { llm, ...options } = config;
-  const enumText = `${contentIsQuestion} ${text}\n\n${explainAndSeparate}
-
-${asEnum(enumVal)} ${asUndefinedByDefault}`;
-
-  return toEnum(
-    stripResponse(await chatGPT(enumText, { modelOptions: { ...llm }, ...options })),
-    enumVal
-  );
-};

package/src/verblets/enum/index.spec.js

@@ -1,35 +0,0 @@
-import { describe, expect, it, vi } from 'vitest';
-
-import enumValue from './index.js';
-
-vi.mock('../../lib/chatgpt/index.js', () => ({
-  default: vi.fn().mockImplementation((text) => {
-    if (/traffic light/.test(text)) {
-      return 'red';
-    }
-    return 'undefined';
-  }),
-}));
-
-const examples = [
-  {
-    name: 'Basic usage',
-    inputs: {
-      text: 'What is the top color on a traffic light',
-      enum: { green: 1, yellow: 1, red: 1, purple: 1 },
-    },
-    want: { result: 'red' },
-  },
-];
-
-describe('Enum verblet', () => {
-  examples.forEach((example) => {
-    it(example.name, async () => {
-      const result = await enumValue(example.inputs.text, example.inputs.enum);
-
-      if (example.want.result) {
-        expect(result).toStrictEqual(example.want.result);
-      }
-    });
-  });
-});

package/src/verblets/expect/README.md

@@ -1,64 +0,0 @@
-# LLM Expect
-
-Make intelligent assertions using natural language. A single LLM call that validates content based on meaning, intent, and context.
-
-## Quick Start
-
-```javascript
-import expect, { llmAssert } from './index.js';
-
-// Simple equality check (throws on failure)
-await expect("hello").toEqual("hello");
-// ✅ Passes silently
-
-// Constraint-based validation
-await expect("Hello world!").toSatisfy("Is this a greeting?");
-// ✅ Passes silently
-
-// Failed assertion throws
-try {
-  await expect("goodbye").toEqual("hello");
-} catch (error) {
-  console.log(error.message);
-  // "LLM assertion failed: Does the actual value strictly equal the expected value?"
-}
-
-// Direct helper with custom options
-const passed = await llmAssert({
-  actual: "hello",
-  equals: "hello",
-  llm: { temperature: 0 },
-  throws: false,
-});
-
-// Custom message when the assertion fails
-await expect("bad").toEqual("good", {
-  message: ({ actual, equals }) => `Expected ${equals} but got ${actual}`,
-});
-```
-
-Use `throws: false` to return a boolean instead of throwing when the assertion fails.
-
-## Enhanced Chain Implementation
-
-For advanced debugging, detailed error analysis, and enhanced features, use the **[expect chain](../../chains/expect/)**:
-
-### Environment Modes
-
-```bash
-# Silent operation (default)
-export LLM_EXPECT_MODE=none
-
-# Log debugging advice on failures
-export LLM_EXPECT_MODE=info
-
-# Throw with detailed debugging advice
-export LLM_EXPECT_MODE=error
-```
-
-## Best Practices
-
-- **Be specific**: Use clear, detailed constraints
-- **Test qualitatively**: Verify qualitative details with a clear yes/no answer
-- **Use robust constraints**: Write criteria to pass under a wide range of input hallucinations. Assert cases that classical software can't. Tune the level of rigorousness to the model performing the eval.
-- **Performance**: Remember this makes an LLM call - use judiciously

package/src/verblets/expect/index.examples.js

@@ -1,109 +0,0 @@
-import { describe, expect, it } from 'vitest';
-
-import aiExpect from './index.js';
-import { longTestTimeout } from '../../constants/common.js';
-
-const examples = [
-  {
-    inputs: {
-      actual: 'hello',
-      expected: 'hello',
-    },
-    want: { result: true },
-  },
-  {
-    inputs: {
-      actual: 'hello',
-      expected: 'goodbye',
-    },
-    want: { result: false },
-  },
-  {
-    inputs: {
-      actual: 'Hello world!',
-      constraint: 'Is this a greeting?',
-    },
-    want: { result: true },
-  },
-  {
-    inputs: {
-      actual: 'Goodbye cruel world',
-      constraint: 'Is this a greeting?',
-    },
-    want: { result: false },
-  },
-  {
-    inputs: {
-      actual: 'This is a well-written, professional email with proper grammar and clear intent.',
-      constraint: 'Is this text professional and grammatically correct?',
-    },
-    want: { result: true },
-  },
-  {
-    inputs: {
-      actual: { name: 'John Doe', age: 30, city: 'New York' },
-      constraint: 'Does this person data look realistic?',
-    },
-    want: { result: true },
-  },
-];
-
-describe('LLM Expect Verblet', () => {
-  examples.forEach((example) => {
-    const description = example.inputs.constraint
-      ? `${JSON.stringify(example.inputs.actual).slice(0, 30)}... - ${example.inputs.constraint}`
-      : `${JSON.stringify(example.inputs.actual)} === ${JSON.stringify(example.inputs.expected)}`;
-
-    it(
-      description,
-      async () => {
-        const result =
-          example.inputs.expected !== undefined
-            ? await aiExpect(example.inputs.actual).toEqual(example.inputs.expected, {
-                throws: false,
-              })
-            : await aiExpect(example.inputs.actual).toSatisfy(example.inputs.constraint, {
-                throws: false,
-              });
-
-        expect(result).toBe(example.want.result);
-      },
-      longTestTimeout
-    );
-  });
-
-  it(
-    'should throw by default on failure',
-    async () => {
-      await expect(async () => {
-        await aiExpect('hello').toEqual('goodbye');
-      }).rejects.toThrow('LLM assertion failed');
-    },
-    longTestTimeout
-  );
-
-  it(
-    'should not throw when throws option is false',
-    async () => {
-      const result = await aiExpect('hello').toEqual('goodbye', { throws: false });
-      expect(result).toBe(false);
-    },
-    longTestTimeout
-  );
-
-  it(
-    'should handle business logic validation',
-    async () => {
-      const businessRecommendation =
-        'Increase marketing budget by 20% for Q4 to boost holiday sales and target demographics aged 25-45 through social media campaigns';
-
-      const result = await aiExpect(businessRecommendation).toSatisfy(
-        'Is this recommendation specific, actionable, and includes measurable targets?',
-        { throws: false }
-      );
-
-      expect(result).toBe(true);
-    },
-    longTestTimeout
-  );
-});

package/src/verblets/expect/index.js

@@ -1,75 +0,0 @@
-import chatgpt from '../../lib/chatgpt/index.js';
-
-function buildEqualityPrompt({ actual, expected, context }) {
-  return `Does the actual value strictly equal the expected value?\n\nActual: ${JSON.stringify(
-    actual,
-    null,
-    2
-  )}\nExpected: ${JSON.stringify(expected, null, 2)}\n\n${
-    context ? `Context: ${JSON.stringify(context, null, 2)}\n` : ''
-  }Answer only "True" or "False".`;
-}
-
-function buildConstraintPrompt({ actual, constraint, context }) {
-  return `Given this constraint: "${constraint}"\n\nActual value: ${JSON.stringify(
-    actual,
-    null,
-    2
-  )}\n\n${
-    context ? `Additional context: ${JSON.stringify(context, null, 2)}\n` : ''
-  }Does the actual value satisfy the constraint? Answer only "True" or "False".`;
-}
-
-export async function llmAssert({
-  actual,
-  equals,
-  constraint,
-  context,
-  throws = true,
-  message,
-  llm = {},
-}) {
-  if (equals === undefined && !constraint)
-    throw new TypeError('Provide either "equals" or "constraint".');
-
-  const prompt =
-    equals !== undefined
-      ? buildEqualityPrompt({ actual, expected: equals, context })
-      : buildConstraintPrompt({ actual, constraint, context });
-
-  const answer = await chatgpt(prompt, { modelOptions: llm });
-  const text = typeof answer === 'string' ? answer : answer.content;
-  const passed = /^true$/i.test(text.trim());
-
-  if (!passed && throws) {
-    let msg;
-    if (typeof message === 'function') {
-      msg = message({ actual, equals, constraint });
-    } else {
-      msg = message;
-    }
-
-    if (!msg) {
-      msg =
-        equals !== undefined
-          ? 'LLM assertion failed: Does the actual value strictly equal the expected value?'
-          : `LLM assertion failed: ${constraint}`;
-    }
-
-    throw new Error(msg);
-  }
-
-  return passed;
-}
-
-export default function expect(actual, shared = {}) {
-  const run = (payload, opts) => llmAssert({ actual, ...payload, ...shared, ...opts });
-  return {
-    toEqual(expected, opts) {
-      return run({ equals: expected }, opts);
-    },
-    toSatisfy(constraint, opts) {
-      return run({ constraint }, opts);
-    },
-  };
-}