@far-world-labs/verblets 0.1.1 → 0.1.2

package/src/chains/anonymize/index.spec.js

@@ -0,0 +1,78 @@
+import { describe, expect, it, vi } from 'vitest';
+import { anonymize, anonymizeMethod } from './index.js';
+
+vi.mock('./index.js', () => {
+  return {
+    anonymize: vi.fn(async (input) => {
+      if (!input || typeof input.text !== 'string' || !input.text.trim()) {
+        throw new Error('Text is required');
+      }
+      if (!input.method) {
+        throw new Error('Method is required');
+      }
+      if (!['STRICT', 'BALANCED', 'LIGHT'].includes(input.method)) {
+        throw new Error('Invalid method');
+      }
+      return {
+        text: 'anonymized',
+        stages: {
+          distinctiveContentRemoved: true,
+          structureNormalized: true,
+          patternsSuppressed: true,
+        },
+      };
+    }),
+    anonymizeMethod: { STRICT: 'STRICT', BALANCED: 'BALANCED', LIGHT: 'LIGHT' },
+  };
+});
+
+describe('anonymize', () => {
+  it('should return an object with text and stages properties', async () => {
+    const input = {
+      text: 'Test input',
+      method: anonymizeMethod.LIGHT,
+    };
+
+    const result = await anonymize(input);
+
+    expect(result).to.have.property('text');
+    expect(result).to.have.property('stages');
+    expect(result.stages).to.have.property('distinctiveContentRemoved');
+    expect(result.stages).to.have.property('structureNormalized');
+    expect(result.stages).to.have.property('patternsSuppressed');
+  });
+
+  it('should throw an error if method is not provided', async () => {
+    const input = {
+      text: 'Test input',
+    };
+
+    await expect(anonymize(input)).rejects.toThrow('Method is required');
+  });
+
+  it('should throw an error if method is invalid', async () => {
+    const input = {
+      text: 'Test input',
+      method: 'INVALID',
+    };
+
+    await expect(anonymize(input)).rejects.toThrow('Invalid method');
+  });
+
+  it('should throw an error if text is not provided', async () => {
+    const input = {
+      method: anonymizeMethod.LIGHT,
+    };
+
+    await expect(anonymize(input)).rejects.toThrow('Text is required');
+  });
+
+  it('should throw an error if text is empty', async () => {
+    const input = {
+      text: '',
+      method: anonymizeMethod.LIGHT,
+    };
+
+    await expect(anonymize(input)).rejects.toThrow('Text is required');
+  });
+});

package/src/chains/bulk-central-tendency/index.examples.js

@@ -0,0 +1,138 @@
+import { describe, expect, it } from 'vitest';
+import { longTestTimeout } from '../../constants/common.js';
+import { expect as aiExpect } from '../expect/index.js';
+import bulkCentralTendency from './index.js';
+
+describe('Bulk Central Tendency Chain', () => {
+  it(
+    'processes multiple fruit items with consistent results',
+    async () => {
+      const items = ['apple', 'orange', 'durian', 'jackfruit', 'banana'];
+      const seedItems = ['apple', 'orange', 'banana', 'grape', 'strawberry'];
+
+      const results = await bulkCentralTendency(items, seedItems, {
+        context: 'Common fruits found in grocery stores',
+      });
+
+      expect(results).toHaveLength(5);
+      expect(results.every((r) => r && typeof r.score === 'number')).toBe(true);
+      expect(results.every((r) => r && r.score >= 0 && r.score <= 1)).toBe(true);
+      expect(results.every((r) => r && typeof r.reason === 'string')).toBe(true);
+      expect(results.every((r) => r && typeof r.confidence === 'number')).toBe(true);
+
+      // Use expect-chain for loose verification
+      const [isValidCentralityScoring] = await aiExpect(
+        results,
+        undefined,
+        'Do these centrality scores make sense? Common fruits like apple, orange, banana should have higher scores than exotic fruits like durian and jackfruit.'
+      );
+      expect(isValidCentralityScoring).toBe(true);
+    },
+    longTestTimeout
+  );
+
+  it(
+    'handles tool centrality with core features',
+    async () => {
+      const items = ['hammer', 'screwdriver', 'wrench', 'pliers', 'chainsaw'];
+      const seedItems = ['hammer', 'screwdriver', 'wrench', 'saw', 'drill'];
+
+      const results = await bulkCentralTendency(items, seedItems, {
+        context: 'Hand tools for construction and repair',
+        coreFeatures: ['handheld', 'mechanical', 'durable'],
+      });
+
+      expect(results).toHaveLength(5);
+      expect(results.every((r) => r && typeof r.score === 'number')).toBe(true);
+      expect(results.every((r) => r && r.score >= 0 && r.score <= 1)).toBe(true);
+
+      // Use expect-chain for loose verification
+      const [isValidToolScoring] = await aiExpect(
+        results,
+        undefined,
+        'Do these tool centrality scores make sense? Basic hand tools like hammer, screwdriver, wrench should have high scores, while chainsaw (power tool) should have a lower score.'
+      );
+      expect(isValidToolScoring).toBe(true);
+    },
+    longTestTimeout
+  );
+
+  it(
+    'demonstrates context effects on centrality',
+    async () => {
+      const items = ['robin', 'eagle', 'penguin', 'ostrich'];
+      const seedItems = ['robin', 'sparrow', 'cardinal', 'blue jay'];
+
+      const results = await bulkCentralTendency(items, seedItems, {
+        context: 'Small songbirds commonly seen in backyards',
+      });
+
+      expect(results).toHaveLength(4);
+      expect(results.every((r) => r && typeof r.score === 'number')).toBe(true);
+
+      // Use expect-chain for loose verification
+      const [isValidBirdScoring] = await aiExpect(
+        results,
+        undefined,
+        'Given the context of "small songbirds commonly seen in backyards", does robin have the highest centrality score, while penguin and ostrich have much lower scores?'
+      );
+      expect(isValidBirdScoring).toBe(true);
+    },
+    longTestTimeout
+  );
+
+  it(
+    'manages retry logic for failed items',
+    async () => {
+      const items = ['cat', 'dog', 'elephant'];
+      const seedItems = ['cat', 'dog', 'rabbit', 'hamster'];
+
+      const results = await bulkCentralTendency(items, seedItems, {
+        maxAttempts: 2,
+      });
+
+      expect(results).toHaveLength(3);
+      expect(results.every((r) => r && typeof r.score === 'number')).toBe(true);
+
+      // Use expect-chain for loose verification
+      const [isValidPetScoring] = await aiExpect(
+        results,
+        undefined,
+        'Are these reasonable centrality scores for pets, with cat and dog having higher scores than elephant?'
+      );
+      expect(isValidPetScoring).toBe(true);
+    },
+    longTestTimeout
+  );
+
+  it('handles empty input', async () => {
+    const results = await bulkCentralTendency([], ['apple', 'orange']);
+    expect(results).toEqual([]);
+  });
+
+  it('throws error for invalid seed items', async () => {
+    await expect(bulkCentralTendency(['apple'], [])).rejects.toThrow(
+      'seedItems must be a non-empty array'
+    );
+    await expect(bulkCentralTendency(['apple'], null)).rejects.toThrow(
+      'seedItems must be a non-empty array'
+    );
+  });
+
+  it(
+    'processes large batches efficiently',
+    async () => {
+      const items = Array.from({ length: 15 }, (_, i) => `item${i + 1}`);
+      const seedItems = ['item1', 'item2', 'item3', 'item4', 'item5'];
+
+      const results = await bulkCentralTendency(items, seedItems, {
+        chunkSize: 3,
+      });
+
+      expect(results).toHaveLength(15);
+      expect(results.every((r) => r && typeof r.score === 'number')).toBe(true);
+      expect(results.every((r) => r && r.score >= 0 && r.score <= 1)).toBe(true);
+    },
+    longTestTimeout
+  );
+});

package/src/chains/bulk-central-tendency/index.js

@@ -0,0 +1,91 @@
+import { bulkMapRetry } from '../bulk-map/index.js';
+import { CENTRAL_TENDENCY_PROMPT } from '../../verblets/central-tendency/index.js';
+
+/**
+ * Build instructions for bulk central tendency evaluation using the core verblet prompt
+ * @param {string[]} seedItems - Array of seed items for comparison
+ * @param {Object} config - Configuration options
+ * @returns {string} Instructions for the bulk mapper
+ */
+function buildBulkCentralTendencyInstructions(seedItems, { context = '', coreFeatures = [] } = {}) {
+  const contextLine = context ? `Context: ${context}` : '';
+  const coreFeaturesLine =
+    coreFeatures.length > 0 ? `Core Features: ${coreFeatures.join(', ')}` : '';
+  const outputRequirementsLine = `OUTPUT FORMAT: Return exactly one compact JSON object per line (no line breaks within the JSON):
+{"score": <number>, "reason": "<brief explanation>", "confidence": <number>}`;
+
+  // Use the core prompt with all variables replaced
+  const corePrompt = CENTRAL_TENDENCY_PROMPT.replace('{context}', contextLine)
+    .replace('{coreFeatures}', coreFeaturesLine)
+    .replace('{outputRequirements}', outputRequirementsLine);
+
+  return `For each item, evaluate its centrality among these category members: ${seedItems.join(
+    ', '
+  )}
+
+${corePrompt}`;
+}
+
+/**
+ * Process multiple items for central tendency evaluation in bulk with retry support.
+ * Uses the existing bulkMapRetry infrastructure for efficiency and reliability.
+ *
+ * @param {string[]} items - Array of items to evaluate
+ * @param {string[]} seedItems - Array of seed items for comparison
+ * @param {Object} [config={}] - Configuration options
+ * @param {string} [config.context=''] - Context description for evaluation
+ * @param {string[]} [config.coreFeatures=[]] - Known core/definitional features
+ * @param {string|Object} [config.llm='fastGoodCheap'] - LLM model to use
+ * @param {number} [config.chunkSize=5] - Batch size for processing
+ * @param {number} [config.maxAttempts=3] - Max retry attempts for failed items
+ * @returns {Promise<Array>} Array of central tendency results
+ */
+export default async function bulkCentralTendency(items, seedItems, config = {}) {
+  if (!Array.isArray(items)) {
+    throw new Error('Items must be an array');
+  }
+
+  if (items.length === 0) {
+    return [];
+  }
+
+  if (!Array.isArray(seedItems) || seedItems.length === 0) {
+    throw new Error('seedItems must be a non-empty array');
+  }
+
+  const { chunkSize = 5, maxAttempts = 3, ...otherConfig } = config;
+
+  // Build instructions for the bulk mapper
+  const instructions = buildBulkCentralTendencyInstructions(seedItems, otherConfig);
+
+  // Use bulkMapRetry to handle all the complexity
+  const results = await bulkMapRetry(items, instructions, { chunkSize, maxAttempts });
+
+  // Parse JSON responses and handle any parsing errors
+  return results.map((result, _index) => {
+    if (result === undefined) {
+      return undefined;
+    }
+
+    try {
+      const parsed = JSON.parse(result);
+
+      // Validate the structure
+      if (
+        parsed &&
+        typeof parsed.score === 'number' &&
+        typeof parsed.reason === 'string' &&
+        typeof parsed.confidence === 'number'
+      ) {
+        return parsed;
+      } else {
+        return undefined;
+      }
+    } catch {
+      return undefined;
+    }
+  });
+}
+
+// Export the retry version as well for consistency with other bulk processors
+export const bulkCentralTendencyRetry = bulkCentralTendency;

package/src/chains/bulk-filter/README.md

@@ -0,0 +1,21 @@
+# bulk-filter
+
+Filter very long lists in manageable chunks using `listFilter`. Failed batches can be retried.
+
+```javascript
+import bulkFilter from './index.js';
+
+const diary = [
+  'Walked the dog and bought milk.',
+  'One day I hope to sail across the Atlantic.',
+  'Cleaned out the garage.',
+  "Maybe I'll start that bakery I keep dreaming about.",
+];
+
+const aspirations = await bulkFilter(
+  diary,
+  'Keep only lines about hopes or big dreams',
+  { chunkSize: 2 }
+);
+// => ['One day I hope to sail across the Atlantic.', "Maybe I'll start that bakery I keep dreaming about."]
+```

package/src/chains/bulk-filter/index.examples.js

@@ -0,0 +1,22 @@
+import { describe, expect, it } from 'vitest';
+import bulkFilter from './index.js';
+import { longTestTimeout } from '../../constants/common.js';
+
+describe('bulk-filter examples', () => {
+  it(
+    'filters with listFilter',
+    async () => {
+      const notes = [
+        'Saw a dolphin while surfing',
+        'Finished laundry',
+        'Dream of traveling to Iceland',
+        'Paid the electricity bill',
+      ];
+      const dreams = await bulkFilter(notes, 'keep only lines about aspirations or dreams', {
+        chunkSize: 2,
+      });
+      expect(dreams.length).toBeGreaterThan(0);
+    },
+    longTestTimeout
+  );
+});

package/src/chains/bulk-filter/index.js

@@ -0,0 +1,58 @@
+import listFilter from '../../verblets/list-filter/index.js';
+
+const buildMask = async (list, instructions, chunkSize, config = {}) => {
+  const mask = new Array(list.length);
+  for (let i = 0; i < list.length; i += chunkSize) {
+    const batch = list.slice(i, i + chunkSize);
+    try {
+      // eslint-disable-next-line no-await-in-loop
+      const result = await listFilter(batch, instructions, config);
+      const valid = result.every((item) => batch.includes(item));
+      if (!valid) {
+        for (let j = 0; j < batch.length; j += 1) {
+          mask[i + j] = undefined;
+        }
+        continue;
+      }
+      for (let j = 0; j < batch.length; j += 1) {
+        mask[i + j] = result.includes(batch[j]);
+      }
+    } catch {
+      for (let j = 0; j < batch.length; j += 1) {
+        mask[i + j] = undefined;
+      }
+    }
+  }
+  return mask;
+};
+
+export const bulkFilterRetry = async (list, instructions, config = {}) => {
+  const { chunkSize = 10, maxAttempts = 3, llm, ...options } = config;
+  const filterConfig = { llm, ...options };
+  let mask = await buildMask(list, instructions, chunkSize, filterConfig);
+  for (let attempt = 1; attempt < maxAttempts; attempt += 1) {
+    const missingIdx = [];
+    const missingItems = [];
+    mask.forEach((val, idx) => {
+      if (val === undefined) {
+        missingIdx.push(idx);
+        missingItems.push(list[idx]);
+      }
+    });
+    if (missingItems.length === 0) break;
+    // eslint-disable-next-line no-await-in-loop
+    const retryMask = await buildMask(missingItems, instructions, chunkSize, filterConfig);
+    retryMask.forEach((val, i) => {
+      if (val !== undefined) {
+        mask[missingIdx[i]] = val;
+      }
+    });
+  }
+  return list.filter((_, idx) => mask[idx]);
+};
+
+export default async function bulkFilter(list, instructions, config = {}) {
+  const { chunkSize = 10, llm, ...options } = config;
+  const mask = await buildMask(list, instructions, chunkSize, { llm, ...options });
+  return list.filter((_, idx) => mask[idx]);
+}

package/src/chains/bulk-filter/index.spec.js

@@ -0,0 +1,38 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import bulkFilter, { bulkFilterRetry } from './index.js';
+import listFilter from '../../verblets/list-filter/index.js';
+
+vi.mock('../../verblets/list-filter/index.js', () => ({
+  default: vi.fn(async (items, instructions) => {
+    if (items.includes('FAIL')) throw new Error('fail');
+    return items.filter((l) => l.includes(instructions));
+  }),
+}));
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe('bulk-filter', () => {
+  it('filters items in batches', async () => {
+    const result = await bulkFilter(['a', 'b', 'c'], 'a', { chunkSize: 2 });
+    expect(result).toStrictEqual(['a']);
+    expect(listFilter).toHaveBeenCalledTimes(2);
+  });
+
+  it('retries failed batches', async () => {
+    let call = 0;
+    listFilter.mockImplementation(async (items) => {
+      call += 1;
+      if (call === 1) throw new Error('fail');
+      return items.filter((l) => l.includes('a'));
+    });
+
+    const result = await bulkFilterRetry(['FAIL', 'a', 'b'], 'a', {
+      chunkSize: 2,
+      maxAttempts: 2,
+    });
+    expect(result).toStrictEqual(['a']);
+    expect(listFilter).toHaveBeenCalledTimes(3);
+  });
+});

package/src/chains/bulk-find/README.md

@@ -0,0 +1,16 @@
+# bulk-find
+
+Scan long lists in manageable batches to locate the item that best matches your instructions.
+
+```javascript
+import bulkFind from './index.js';
+
+const emails = [
+  'update from accounting',
+  'party invitation',
+  'weekly newsletter',
+  // ... potentially thousands more
+];
+const best = await bulkFind(emails, 'Which email is most urgent?');
+// => 'update from accounting'
+```

package/src/chains/bulk-find/index.examples.js

@@ -0,0 +1,20 @@
+import { describe, expect, it } from 'vitest';
+import bulkFind from './index.js';
+import { longTestTimeout } from '../../constants/common.js';
+
+describe('bulk-find examples', () => {
+  it(
+    'finds the best match across batches',
+    async () => {
+      const titles = [
+        'ancient mystery',
+        'space odyssey',
+        'underwater adventure',
+        'future tech thriller',
+      ];
+      const result = await bulkFind(titles, 'Which title feels most futuristic?', { chunkSize: 2 });
+      expect(result).toBeDefined();
+    },
+    longTestTimeout
+  );
+});

package/src/chains/bulk-find/index.js

@@ -0,0 +1,30 @@
+import listFind from '../../verblets/list-find/index.js';
+
+export const bulkFind = async function (list, instructions, config = {}) {
+  const { chunkSize = 10, llm, ...options } = config;
+  let candidate = '';
+  for (let i = 0; i < list.length; i += chunkSize) {
+    const batch = list.slice(i, i + chunkSize);
+    const combined = candidate ? [candidate, ...batch] : batch;
+    // eslint-disable-next-line no-await-in-loop
+    candidate = await listFind(combined, instructions, { llm, ...options });
+  }
+  return candidate;
+};
+
+export const bulkFindRetry = async function (list, instructions, config = {}) {
+  const { chunkSize = 10, maxAttempts = 3, llm, ...options } = config;
+  let result;
+  for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+    try {
+      // eslint-disable-next-line no-await-in-loop
+      result = await bulkFind(list, instructions, { chunkSize, llm, ...options });
+      if (result) break;
+    } catch {
+      // continue
+    }
+  }
+  return result;
+};
+
+export default bulkFind;

package/src/chains/bulk-find/index.spec.js

@@ -0,0 +1,26 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import bulkFind, { bulkFindRetry } from './index.js';
+import listFind from '../../verblets/list-find/index.js';
+
+vi.mock('../../verblets/list-find/index.js', () => ({
+  default: vi.fn(async (items) => items[items.length - 1]),
+}));
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe('bulk-find chain', () => {
+  it('scans batches to find best item', async () => {
+    const result = await bulkFind(['a', 'b', 'c', 'd'], 'find', { chunkSize: 2 });
+    expect(result).toBe('d');
+    expect(listFind).toHaveBeenCalledTimes(2);
+  });
+
+  it('retries on failure', async () => {
+    listFind.mockRejectedValueOnce(new Error('fail'));
+    const result = await bulkFindRetry(['x', 'y'], 'find', { chunkSize: 2, maxAttempts: 2 });
+    expect(result).toBe('y');
+    expect(listFind).toHaveBeenCalledTimes(2);
+  });
+});

package/src/chains/bulk-group/README.md

@@ -0,0 +1,23 @@
+# bulk-group
+
+Group long lists by first discovering the best categories and then grouping
+items into those categories in smaller batches.
+
+```javascript
+import bulkGroup from './index.js';
+
+const feedback = [
+  'Great interface and onboarding',
+  'Price is a bit steep',
+  'Love the mobile app',
+  'Needs more integrations',
+];
+const result = await bulkGroup(
+  feedback,
+  'Is each line praise, criticism, or a feature request?',
+  { chunkSize: 2, topN: 3 }
+);
+// => { praise: ['Great interface and onboarding', 'Love the mobile app'],
+//      criticism: ['Price is a bit steep'],
+//      'feature request': ['Needs more integrations'] }
+```

package/src/chains/bulk-group/index.examples.js

@@ -0,0 +1,18 @@
+import bulkGroup from './index.js';
+import { describe, it, expect } from 'vitest';
+import { longTestTimeout } from '../../constants/common.js';
+
+describe('bulk-group examples', () => {
+  it(
+    'groups a long list',
+    async () => {
+      const items = ['dog', 'fish', 'cat', 'whale', 'bird', 'shark', 'horse', 'dolphin'];
+      const result = await bulkGroup(items, 'Is each creature terrestrial or aquatic?', {
+        chunkSize: 4,
+      });
+      expect(typeof result).toBe('object');
+      expect(Object.keys(result).length).toBeGreaterThan(0);
+    },
+    longTestTimeout
+  );
+});

package/src/chains/bulk-group/index.js

@@ -0,0 +1,34 @@
+import listGroup from '../../verblets/list-group/index.js';
+
+export default async function bulkGroup(list, instructions, config = {}) {
+  const { chunkSize = 10, topN, llm, ...options } = config;
+  let categories;
+  const groups = {};
+
+  for (let i = 0; i < list.length; i += chunkSize) {
+    const batch = list.slice(i, i + chunkSize);
+
+    // eslint-disable-next-line no-await-in-loop
+    const result = await listGroup(batch, instructions, categories, { llm, ...options });
+
+    // Use categories from first batch for consistency
+    if (!categories) {
+      categories = Object.keys(result);
+    }
+
+    for (const [key, items] of Object.entries(result)) {
+      if (!groups[key]) groups[key] = [];
+      groups[key].push(...items);
+    }
+  }
+
+  // Apply topN filtering if specified
+  if (topN) {
+    const sortedEntries = Object.entries(groups)
+      .sort(([, a], [, b]) => b.length - a.length)
+      .slice(0, topN);
+    return Object.fromEntries(sortedEntries);
+  }
+
+  return groups;
+}

package/src/chains/bulk-group/index.spec.js

@@ -0,0 +1,41 @@
+import bulkGroup from './index.js';
+import listGroup from '../../verblets/list-group/index.js';
+import { vi, describe, it, expect, beforeEach } from 'vitest';
+
+vi.mock('../../verblets/list-group/index.js', () => ({
+  default: vi.fn(),
+}));
+
+describe('bulk-group chain', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it('groups in batches', async () => {
+    const items = ['a', 'bb', 'ccc', 'dddd', 'eeeee'];
+
+    // Mock the calls in order - with chunkSize=2, we'll have 3 batches: [a,bb], [ccc,dddd], [eeeee]
+    listGroup
+      .mockResolvedValueOnce({ odd: ['a'], even: ['bb'] }) // First batch
+      .mockResolvedValueOnce({ odd: ['ccc'], even: ['dddd'] }) // Second batch
+      .mockResolvedValueOnce({ odd: ['eeeee'] }); // Third batch
+
+    const result = await bulkGroup(items, 'odd or even', {
+      chunkSize: 2,
+    });
+
+    expect(result).toStrictEqual({ odd: ['a', 'ccc', 'eeeee'], even: ['bb', 'dddd'] });
+    expect(listGroup).toHaveBeenCalledTimes(3);
+
+    // Verify the calls were made with the right parameters
+    expect(listGroup).toHaveBeenNthCalledWith(1, ['a', 'bb'], 'odd or even', undefined, {
+      llm: undefined,
+    });
+    expect(listGroup).toHaveBeenNthCalledWith(2, ['ccc', 'dddd'], 'odd or even', ['odd', 'even'], {
+      llm: undefined,
+    });
+    expect(listGroup).toHaveBeenNthCalledWith(3, ['eeeee'], 'odd or even', ['odd', 'even'], {
+      llm: undefined,
+    });
+  });
+});