@far-world-labs/verblets 0.2.0 → 0.3.2

package/src/verblets/expect/index.js

@@ -1,75 +0,0 @@
-import chatgpt from '../../lib/chatgpt/index.js';
-
-function buildEqualityPrompt({ actual, expected, context }) {
-  return `Does the actual value strictly equal the expected value?\n\nActual: ${JSON.stringify(
-    actual,
-    null,
-    2
-  )}\nExpected: ${JSON.stringify(expected, null, 2)}\n\n${
-    context ? `Context: ${JSON.stringify(context, null, 2)}\n` : ''
-  }Answer only "True" or "False".`;
-}
-
-function buildConstraintPrompt({ actual, constraint, context }) {
-  return `Given this constraint: "${constraint}"\n\nActual value: ${JSON.stringify(
-    actual,
-    null,
-    2
-  )}\n\n${
-    context ? `Additional context: ${JSON.stringify(context, null, 2)}\n` : ''
-  }Does the actual value satisfy the constraint? Answer only "True" or "False".`;
-}
-
-export async function llmAssert({
-  actual,
-  equals,
-  constraint,
-  context,
-  throws = true,
-  message,
-  llm = {},
-}) {
-  if (equals === undefined && !constraint)
-    throw new TypeError('Provide either "equals" or "constraint".');
-
-  const prompt =
-    equals !== undefined
-      ? buildEqualityPrompt({ actual, expected: equals, context })
-      : buildConstraintPrompt({ actual, constraint, context });
-
-  const answer = await chatgpt(prompt, { modelOptions: llm });
-  const text = typeof answer === 'string' ? answer : answer.content;
-  const passed = /^true$/i.test(text.trim());
-
-  if (!passed && throws) {
-    let msg;
-    if (typeof message === 'function') {
-      msg = message({ actual, equals, constraint });
-    } else {
-      msg = message;
-    }
-
-    if (!msg) {
-      msg =
-        equals !== undefined
-          ? 'LLM assertion failed: Does the actual value strictly equal the expected value?'
-          : `LLM assertion failed: ${constraint}`;
-    }
-
-    throw new Error(msg);
-  }
-
-  return passed;
-}
-
-export default function expect(actual, shared = {}) {
-  const run = (payload, opts) => llmAssert({ actual, ...payload, ...shared, ...opts });
-  return {
-    toEqual(expected, opts) {
-      return run({ equals: expected }, opts);
-    },
-    toSatisfy(constraint, opts) {
-      return run({ constraint }, opts);
-    },
-  };
-}

package/src/verblets/expect/index.spec.js

@@ -1,127 +0,0 @@
-import { describe, expect, it, vi } from 'vitest';
-import aiExpect from './index.js';
-import { longTestTimeout } from '../../constants/common.js';
-
-// Mock the chatgpt function to avoid actual API calls
-vi.mock('../../lib/chatgpt/index.js', () => ({
-  default: vi.fn().mockImplementation((prompt) => {
-    // Handle exact equality checks
-    if (prompt.includes('Does the actual value strictly equal the expected value?')) {
-      if (prompt.includes('Actual: "hello"') && prompt.includes('Expected: "hello"')) {
-        return 'True';
-      }
-      if (prompt.includes('Actual: "goodbye"') && prompt.includes('Expected: "hello"')) {
-        return 'False';
-      }
-    }
-
-    // Handle constraint-based validations (format: "Given this constraint:")
-    if (prompt.includes('Given this constraint:')) {
-      if (prompt.includes('Is this a greeting?') && prompt.includes('Hello world!')) {
-        return 'True';
-      }
-
-      if (prompt.includes('Is this text professional and grammatically correct?')) {
-        if (prompt.includes('well-written, professional email')) {
-          return 'True';
-        }
-      }
-
-      if (prompt.includes('Does this person data look realistic?')) {
-        if (prompt.includes('John Doe') && prompt.includes('"age": 30')) {
-          return 'True';
-        }
-      }
-
-      if (prompt.includes('Is this recommendation specific and actionable?')) {
-        if (prompt.includes('Increase marketing budget by 20%')) {
-          return 'True';
-        }
-      }
-    }
-
-    // Default to False for unmatched cases
-    return 'False';
-  }),
-}));
-
-describe('expect verblet', () => {
-  it(
-    'should pass for exact equality',
-    async () => {
-      const result = await aiExpect('hello').toEqual('hello', { throws: false });
-      expect(result).toBe(true);
-    },
-    longTestTimeout
-  );
-
-  it(
-    'should pass for constraint-based validation',
-    async () => {
-      const result = await aiExpect('Hello world!').toSatisfy('Is this a greeting?', {
-        throws: false,
-      });
-      expect(result).toBe(true);
-    },
-    longTestTimeout
-  );
-
-  it(
-    'should fail for non-matching values',
-    async () => {
-      const result = await aiExpect('goodbye').toEqual('hello', {
-        throws: false,
-      });
-      expect(result).toBe(false);
-    },
-    longTestTimeout
-  );
-
-  it(
-    'should validate content quality',
-    async () => {
-      const result = await aiExpect(
-        'This is a well-written, professional email with proper grammar.'
-      ).toSatisfy('Is this text professional and grammatically correct?', {
-        throws: false,
-      });
-      expect(result).toBe(true);
-    },
-    longTestTimeout
-  );
-
-  it(
-    'should validate data structures',
-    async () => {
-      const result = await aiExpect({ name: 'John Doe', age: 30, city: 'New York' }).toSatisfy(
-        'Does this person data look realistic?',
-        { throws: false }
-      );
-      expect(result).toBe(true);
-    },
-    longTestTimeout
-  );
-
-  it(
-    'should handle business logic validation',
-    async () => {
-      const result = await aiExpect(
-        'Increase marketing budget by 20% for Q4 to boost holiday sales'
-      ).toSatisfy('Is this recommendation specific and actionable?', {
-        throws: false,
-      });
-      expect(result).toBe(true);
-    },
-    longTestTimeout
-  );
-
-  it(
-    'should throw by default on failure',
-    async () => {
-      await expect(async () => {
-        await aiExpect('hello').toEqual('goodbye');
-      }).rejects.toThrow('LLM assertion failed');
-    },
-    longTestTimeout
-  );
-});

package/src/verblets/intent/index.examples.js

@@ -1,139 +0,0 @@
-import Ajv from 'ajv';
-import fs from 'node:fs/promises';
-import { describe, expect, it, beforeAll, afterAll } from 'vitest';
-import { aiExpect } from '../../chains/expect/index.js';
-import { longTestTimeout } from '../../constants/common.js';
-import { fileURLToPath } from 'url';
-import { dirname, join } from 'path';
-
-import intent from './index.js';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-
-async function getIntentSchema() {
-  return JSON.parse(await fs.readFile(join(__dirname, '../../json-schemas/intent.json')));
-}
-
-const examples = [
-  {
-    inputs: { text: 'Give me a flight to Burgas' },
-    want: { resultSchema: getIntentSchema },
-  },
-  {
-    inputs: {
-      text: 'Lookup a song by the quote \
-"I just gotta tell you how I\'m feeling"',
-    },
-    want: { resultSchema: getIntentSchema },
-  },
-];
-
-describe('Intent verblet', () => {
-  // Set environment mode to 'none' for all tests to avoid throwing
-  const originalMode = process.env.LLM_EXPECT_MODE;
-
-  beforeAll(() => {
-    process.env.LLM_EXPECT_MODE = 'none';
-  });
-
-  afterAll(() => {
-    if (originalMode !== undefined) {
-      process.env.LLM_EXPECT_MODE = originalMode;
-    } else {
-      delete process.env.LLM_EXPECT_MODE;
-    }
-  });
-
-  examples.forEach((example) => {
-    it(
-      example.inputs.text,
-      async () => {
-        const result = await intent({ text: example.inputs.text });
-
-        if (example.want.resultSchema) {
-          const schema = await example.want.resultSchema();
-          const ajv = new Ajv();
-          const validate = ajv.compile(schema);
-
-          const isValid = validate(result);
-          if (!isValid) {
-            console.error('Validation errors:');
-            console.error(validate.errors);
-            console.error('Returned result:');
-            console.error(JSON.stringify(result, null, 2));
-          }
-          expect(isValid).toStrictEqual(true);
-
-          // LLM assertion to validate intent extraction quality
-          const intentMakesSense = await aiExpect(
-            `Original text: "${example.inputs.text}" was parsed into an intent object`
-          ).toSatisfy('Does this seem like a reasonable intent extraction?');
-          expect(intentMakesSense).toBe(true);
-
-          // Additional assertion for intent completeness
-          const hasBasicInfo = await aiExpect(JSON.stringify(result)).toSatisfy(
-            'Does this intent object contain some useful information?'
-          );
-          expect(hasBasicInfo).toBe(true);
-        }
-      },
-      longTestTimeout
-    );
-  });
-
-  it(
-    'should extract travel booking intent correctly',
-    async () => {
-      const travelRequest =
-        'Book me a round-trip flight from New York to Tokyo for next month, preferably business class';
-      const result = await intent({ text: travelRequest });
-
-      // Traditional schema validation
-      const schema = await getIntentSchema();
-      const ajv = new Ajv();
-      const validate = ajv.compile(schema);
-      expect(validate(result)).toBe(true);
-
-      // LLM assertions for travel-specific validation
-      const isTravelRelated = await aiExpect(`Intent extracted from: "${travelRequest}"`).toSatisfy(
-        'Is this request related to travel or transportation?'
-      );
-      expect(isTravelRelated).toBe(true);
-
-      const hasLocationInfo = await aiExpect(JSON.stringify(result)).toSatisfy(
-        'Does this intent mention any locations or destinations?'
-      );
-      expect(hasLocationInfo).toBe(true);
-    },
-    longTestTimeout
-  );
-
-  it(
-    'should handle entertainment search intent',
-    async () => {
-      const musicQuery =
-        'Find that song that goes "Never gonna give you up, never gonna let you down"';
-      const result = await intent({ text: musicQuery });
-
-      // Schema validation
-      const schema = await getIntentSchema();
-      const ajv = new Ajv();
-      const validate = ajv.compile(schema);
-      expect(validate(result)).toBe(true);
-
-      // LLM assertion for entertainment intent
-      const isEntertainmentRelated = await aiExpect(
-        `Intent extracted from: "${musicQuery}"`
-      ).toSatisfy('Is this request related to music or entertainment?');
-      expect(isEntertainmentRelated).toBe(true);
-
-      // Validate that the intent captures the search criteria
-      const mentionsLyrics = await aiExpect(JSON.stringify(result)).toSatisfy(
-        'Does this intent mention song lyrics or music search?'
-      );
-      expect(mentionsLyrics).toBe(true);
-    },
-    longTestTimeout
-  );
-});

package/src/verblets/intent/index.js

@@ -1,60 +0,0 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-import chatGPT from '../../lib/chatgpt/index.js';
-import { constants as promptConstants } from '../../prompts/index.js';
-
-const { contentIsQuestion } = promptConstants;
-
-// Get the directory of this module
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-/**
- * Load the JSON schema for intent results
- * @returns {Promise<Object>} JSON schema for validation
- */
-async function getIntentSchema() {
-  const schemaPath = path.resolve(__dirname, '../../json-schemas/intent.json');
-  return JSON.parse(await fs.readFile(schemaPath, 'utf8'));
-}
-
-/**
- * Create model options for structured outputs
- * @param {string|Object} llm - LLM model name or configuration object
- * @returns {Promise<Object>} Model options for chatGPT
- */
-async function createModelOptions(llm = 'fastGoodCheap') {
-  const schema = await getIntentSchema();
-
-  const responseFormat = {
-    type: 'json_schema',
-    json_schema: {
-      name: 'intent_result',
-      schema,
-    },
-  };
-
-  if (typeof llm === 'string') {
-    return {
-      modelName: llm,
-      response_format: responseFormat,
-    };
-  } else {
-    return {
-      ...llm,
-      response_format: responseFormat,
-    };
-  }
-}
-
-export default async function intent({ text, config = {} } = {}) {
-  const { llm, ...options } = config;
-  const prompt = `${contentIsQuestion} What is the intent of this text?\n\n${text}`;
-
-  const modelOptions = await createModelOptions(llm);
-  const response = await chatGPT(prompt, { modelOptions, ...options });
-
-  // With structured outputs, response should already be parsed
-  return typeof response === 'string' ? JSON.parse(response) : response;
-}

package/src/verblets/intent/index.spec.js

@@ -1,31 +0,0 @@
-import { describe, expect, it, vi } from 'vitest';
-
-import intent from './index.js';
-
-vi.mock('../../lib/chatgpt/index.js', () => ({
-  default: vi.fn().mockImplementation((text) => {
-    if (/a flight to/.test(text)) {
-      return '{}';
-    }
-    return 'undefined';
-  }),
-}));
-
-const examples = [
-  {
-    name: 'Basic usage',
-    inputs: { text: 'Give me a flight to Burgas' },
-    want: { typeOfResult: 'object' },
-  },
-];
-
-describe('Intent verblet', () => {
-  examples.forEach((example) => {
-    it(example.name, async () => {
-      const result = await intent({ text: example.inputs.text });
-      if (example.want.typeOfResult) {
-        expect(typeof result).toStrictEqual(example.want.typeOfResult);
-      }
-    });
-  });
-});

package/src/verblets/intersection/README.md

@@ -1,16 +0,0 @@
-# intersection
-
-Find common threads between multiple items using an LLM. The verblet checks every combination from pairs up to the full set. If no relationship is obvious, an empty array is returned.
-
-```javascript
-import intersection from './index.js';
-
-await intersection(['smartphone', 'tablet', 'laptop']);
-// => ['Portable electronics', 'Portable computers']
-
-// Provide custom instructions for how to find intersections
-await intersection(['car', 'bicycle', 'train'], {
-  instructions: 'focus on transportation methods available in a city',
-});
-// => ['Wheeled vehicles', 'Public transit']
-```

package/src/verblets/intersection/index.examples.js

@@ -1,89 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import intersection from './index.js';
-import { longTestTimeout } from '../../constants/common.js';
-import aiExpect from '../expect/index.js';
-
-describe('intersection examples', () => {
-  it(
-    'finds commonalities among devices',
-    async () => {
-      const result = await intersection(['smartphone', 'laptop', 'tablet']);
-      expect(Array.isArray(result), `Saw: ${JSON.stringify(result)}`).toBe(true);
-
-      // LLM assertion to verify the intersection contains meaningful commonalities
-      await aiExpect(result).toSatisfy(
-        'should be an array of strings that could reasonably represent commonalities between technology devices',
-        {
-          context: 'Testing intersection verblet with electronic devices',
-        }
-      );
-    },
-    longTestTimeout
-  );
-
-  it(
-    'finds commonalities among animals',
-    async () => {
-      const result = await intersection(['dog', 'cat', 'bird']);
-      expect(Array.isArray(result), `Saw: ${JSON.stringify(result)}`).toBe(true);
-
-      // LLM assertion for animal traits - be more lenient
-      await aiExpect(result).toSatisfy(
-        'should be an array that represents some form of analysis or commonalities related to animals',
-        {
-          context: 'Testing intersection verblet with animals',
-        }
-      );
-
-      // Just check that it's an array - don't require specific content
-      expect(Array.isArray(result)).toBe(true);
-    },
-    longTestTimeout
-  );
-
-  it(
-    'handles abstract concepts',
-    async () => {
-      const result = await intersection(['love', 'friendship', 'trust']);
-      expect(Array.isArray(result), `Saw: ${JSON.stringify(result)}`).toBe(true);
-
-      // LLM assertion for abstract concept intersections - be more specific
-      await aiExpect(result).toSatisfy(
-        'should be an array of strings representing common emotional or relational concepts that love, friendship, and trust share (like emotional connection, mutual respect, care, etc.)',
-        {
-          context: 'Testing intersection verblet with abstract concepts: love, friendship, trust',
-        }
-      );
-
-      // Verify it's a non-empty array with string elements
-      expect(Array.isArray(result)).toBe(true);
-      expect(result.length).toBeGreaterThan(0);
-      expect(result.every((item) => typeof item === 'string')).toBe(true);
-    },
-    longTestTimeout
-  );
-
-  it(
-    'works with single item',
-    async () => {
-      const result = await intersection(['bicycle']);
-      expect(Array.isArray(result), `Saw: ${JSON.stringify(result)}`).toBe(true);
-
-      // Single items should return empty array based on the implementation
-      expect(result.length).toBe(0);
-    },
-    longTestTimeout
-  );
-
-  it(
-    'handles empty input gracefully',
-    async () => {
-      const result = await intersection([]);
-      expect(Array.isArray(result), `Saw: ${JSON.stringify(result)}`).toBe(true);
-
-      // Empty input should return empty array
-      expect(result.length).toBe(0);
-    },
-    longTestTimeout
-  );
-});

package/src/verblets/intersection/index.js

@@ -1,125 +0,0 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import { fileURLToPath } from 'node:url';
-import chatGPT from '../../lib/chatgpt/index.js';
-import wrapVariable from '../../prompts/wrap-variable.js';
-import { constants as promptConstants } from '../../prompts/index.js';
-
-const { contentIsQuestion, tryCompleteData, onlyJSONStringArray } = promptConstants;
-
-// Get the directory of this module
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-/**
- * Load the JSON schema for intersection results
- * @returns {Promise<Object>} JSON schema for validation
- */
-async function getIntersectionSchema() {
-  const schemaPath = path.join(__dirname, 'intersection-result.json');
-  return JSON.parse(await fs.readFile(schemaPath, 'utf8'));
-}
-
-/**
- * Create model options for structured outputs
- * @param {string|Object} llm - LLM model name or configuration object
- * @returns {Promise<Object>} Model options for chatGPT
- */
-async function createModelOptions(llm = 'fastGoodCheap') {
-  const schema = await getIntersectionSchema();
-
-  const responseFormat = {
-    type: 'json_schema',
-    json_schema: {
-      name: 'intersection_result',
-      schema,
-    },
-  };
-
-  if (typeof llm === 'string') {
-    return {
-      modelName: llm,
-      response_format: responseFormat,
-    };
-  } else {
-    return {
-      ...llm,
-      response_format: responseFormat,
-    };
-  }
-}
-
-export const buildPrompt = (items, { instructions } = {}) => {
-  const itemsList = items.join(' | ');
-  const itemsBlock = wrapVariable(itemsList, { tag: 'items' });
-  const intro =
-    instructions ||
-    'Identify the common elements, shared features, or overlapping aspects that connect all the given items.';
-
-  return `${contentIsQuestion} ${intro}
-
-${itemsBlock}
-
-Provide a clear, focused list of items that represent the intersection or commonality between all the given categories.
-
-${tryCompleteData} ${onlyJSONStringArray}`;
-};
-
-export default async function intersection(items, config = {}) {
-  if (!Array.isArray(items) || items.length === 0) {
-    return [];
-  }
-
-  // Intersection requires at least 2 items
-  if (items.length < 2) {
-    return [];
-  }
-
-  const { llm, ...options } = config;
-  const modelOptions = await createModelOptions(llm);
-
-  const output = await chatGPT(buildPrompt(items, options), {
-    modelOptions,
-  });
-
-  // Handle JSON parsing with robust error handling
-  let parsed;
-  try {
-    if (typeof output === 'string') {
-      // Try to clean up common JSON issues
-      let cleanedOutput = output.trim();
-
-      // Remove any text before the first { or [
-      const jsonStart = Math.min(
-        cleanedOutput.indexOf('{') !== -1 ? cleanedOutput.indexOf('{') : Infinity,
-        cleanedOutput.indexOf('[') !== -1 ? cleanedOutput.indexOf('[') : Infinity
-      );
-
-      if (jsonStart !== Infinity && jsonStart > 0) {
-        cleanedOutput = cleanedOutput.substring(jsonStart);
-      }
-
-      // Remove any text after the last } or ]
-      const lastBrace = cleanedOutput.lastIndexOf('}');
-      const lastBracket = cleanedOutput.lastIndexOf(']');
-      const jsonEnd = Math.max(lastBrace, lastBracket);
-
-      if (jsonEnd !== -1 && jsonEnd < cleanedOutput.length - 1) {
-        cleanedOutput = cleanedOutput.substring(0, jsonEnd + 1);
-      }
-
-      parsed = JSON.parse(cleanedOutput);
-    } else {
-      parsed = output;
-    }
-  } catch (error) {
-    console.error('Failed to parse JSON response from LLM:', error.message);
-    console.error('Raw output:', output);
-
-    return [];
-  }
-
-  // Extract the items array from the object structure
-  const resultArray = parsed?.items || parsed;
-  return Array.isArray(resultArray) ? resultArray.filter(Boolean) : [];
-}