@far-world-labs/verblets 0.1.1 → 0.1.3

package/src/chains/anonymize/index.examples.js

@@ -0,0 +1,75 @@
+import { describe, it } from 'vitest';
+import { expect } from 'chai';
+import { anonymize, anonymizeMethod } from './index.js';
+
+const sampleText = `As a seasoned engineer from Silicon Valley, I've found that React's 
+component lifecycle is like a well-oiled machine - understanding the mounting 
+phase is crucial, especially with those pesky useEffect hooks. Trust me, after 
+10 years of experience, proper cleanup is key to avoiding memory leaks!`;
+
+describe('anonymize examples', () => {
+  it.only('should anonymize text using strict method', { timeout: 60_000 }, async () => {
+    const input = {
+      text: sampleText,
+      method: anonymizeMethod.STRICT,
+    };
+
+    const result = await anonymize(input);
+
+    expect(result).to.have.property('text');
+    expect(result).to.have.property('stages');
+    expect(result.stages).to.have.property('distinctiveContentRemoved');
+    expect(result.stages).to.have.property('structureNormalized');
+    expect(result.stages).to.have.property('patternsSuppressed');
+
+    // Verify anonymization removed personal markers
+    expect(result.text).to.not.include('Silicon Valley');
+    expect(result.text).to.not.include('10 years of experience');
+    expect(result.text).to.not.include('Trust me');
+
+    // Verify metaphors and idioms are removed
+    expect(result.text).to.not.include('well-oiled machine');
+    expect(result.text).to.not.include('pesky');
+
+    // Verify the text has been transformed
+    expect(result.text).to.not.equal(sampleText);
+    expect(result.text.length).to.be.lessThan(sampleText.length);
+  });
+
+  it('should preserve more content with balanced method', { timeout: 60_000 }, async () => {
+    const input = {
+      text: sampleText,
+      method: anonymizeMethod.BALANCED,
+    };
+
+    const result = await anonymize(input);
+
+    // Verify some personal markers are still removed
+    expect(result.text).to.not.include('Silicon Valley');
+    expect(result.text).to.not.include('Trust me');
+
+    // But technical content is more preserved
+    expect(result.text.length).to.be.greaterThan(
+      (await anonymize({ text: sampleText, method: anonymizeMethod.STRICT })).text.length
+    );
+  });
+
+  it('should minimally transform text with light method', { timeout: 60_000 }, async () => {
+    const input = {
+      text: sampleText,
+      method: anonymizeMethod.LIGHT,
+    };
+
+    const result = await anonymize(input);
+
+    // Verify minimal transformation
+    expect(result.text.length).to.be.greaterThan(
+      (await anonymize({ text: sampleText, method: anonymizeMethod.BALANCED })).text.length
+    );
+
+    // Only the most obvious personal markers should be removed
+    expect(result.text).to.not.include('Trust me');
+
+    expect(result.text).to.not.include("I've found");
+  });
+});

package/src/chains/anonymize/index.js

@@ -0,0 +1,121 @@
+import { run } from '../../lib/chatgpt/index.js';
+
+export const anonymizeMethod = {
+  STRICT: 'strict',
+  BALANCED: 'balanced',
+  LIGHT: 'light',
+};
+
+const METHODS = Object.values(anonymizeMethod);
+
+const validateInput = (input) => {
+  if (!input || typeof input !== 'object') {
+    throw new Error('Input must be an object');
+  }
+
+  const { text, method, context } = input;
+
+  if (!text || typeof text !== 'string') {
+    throw new Error('Input must include a text string');
+  }
+
+  if (!method || !METHODS.includes(method)) {
+    throw new Error(`Method must be one of: ${METHODS.join(', ')}`);
+  }
+
+  if (context !== undefined && typeof context !== 'string') {
+    throw new Error('Context must be a string if provided');
+  }
+
+  return { text, method, context };
+};
+
+const stage1Prompt = (text, context) => `
+Remove Distinctive Content and Markers
+ - Identify and replace every distinctive or uncommon word, phrase, or sentence structure with the most widely used, nondescript alternative.
+ - Remove all idioms, metaphors, analogies, cultural references, personal perspectives, and subjective tones.
+ - Eliminate any explicit or implicit references to the author's identity, background, education, expertise, region, or intent.
+
+${context ? `Context: ${context}\n` : ''}
+Text to process:
+${text}
+
+Return ONLY the processed text, with no explanations or additional content.`;
+
+const stage2Prompt = (text, context) => `
+Normalize Structure, Formatting, and Tone
+ - Restructure sentences and paragraphs to strictly follow standard, average patterns in length, order, and construction. Avoid any distinctive rhythm, complexity, or flow.
+ - Uniformly normalize punctuation, formatting, and paragraphing; avoid any variation or emphasis that could signal style.
+ - Strip out all emotional, evaluative, or expressive language, enforcing a neutral, impersonal, and objective tone.
+
+ ${context ? `Context: ${context}\n` : ''}
+ Text to process:
+${text}
+
+Return ONLY the normalized text, with no explanations or additional content.`;
+
+const stage3Prompt = (text, context) => `
+Stage 3: Suppress Latent Stylistic Patterns
+ - Review for and suppress any recurring linguistic patterns, syntactic habits, or structural quirks—even if they appear common.
+ - For all possible ways to phrase content, always select the plainest, most generic, and least distinctive form.
+ - Ensure the final text reads as if generated by an automated system, with no evidence of personality, emotion, region, or any unique authorial traits.
+
+ ${context ? `Context: ${context}\n` : ''}
+Text to process:
+${text}
+
+Return ONLY the final anonymized text, with no explanations or additional content.`;
+
+const anonymize = async (input, config = {}) => {
+  const { text, method, context } = validateInput(input);
+  const { llm, ...options } = config;
+
+  // Stage 1: Remove distinctive content
+  const stage1Result = await run(stage1Prompt(text, method, context), {
+    modelOptions: { modelName: 'privacy', ...llm },
+    ...options,
+  });
+
+  if (method === anonymizeMethod.LIGHT) {
+    return {
+      text: stage1Result,
+      stages: {
+        distinctiveContentRemoved: stage1Result,
+      },
+    };
+  }
+
+  // Stage 2: Normalize structure and tone
+  const stage2Result = await run(stage2Prompt(stage1Result, method), {
+    modelOptions: { modelName: 'privacy', ...llm },
+    ...options,
+  });
+
+  if (method === anonymizeMethod.BALANCED) {
+    return {
+      text: stage2Result,
+      stages: {
+        distinctiveContentRemoved: stage1Result,
+        structureNormalized: stage2Result,
+      },
+    };
+  }
+
+  // Stage 3: Suppress stylistic patterns
+  const stage3Result = await run(stage3Prompt(stage2Result, method), {
+    modelOptions: { modelName: 'privacy', ...llm },
+    ...options,
+  });
+
+  return {
+    text: stage3Result,
+    stages: {
+      distinctiveContentRemoved: stage1Result,
+      structureNormalized: stage2Result,
+      patternsSuppressed: stage3Result,
+    },
+  };
+};
+
+export { anonymize };
+export default anonymize;

package/src/chains/anonymize/index.spec.js

@@ -0,0 +1,78 @@
+import { describe, expect, it, vi } from 'vitest';
+import { anonymize, anonymizeMethod } from './index.js';
+
+vi.mock('./index.js', () => {
+  return {
+    anonymize: vi.fn(async (input) => {
+      if (!input || typeof input.text !== 'string' || !input.text.trim()) {
+        throw new Error('Text is required');
+      }
+      if (!input.method) {
+        throw new Error('Method is required');
+      }
+      if (!['STRICT', 'BALANCED', 'LIGHT'].includes(input.method)) {
+        throw new Error('Invalid method');
+      }
+      return {
+        text: 'anonymized',
+        stages: {
+          distinctiveContentRemoved: true,
+          structureNormalized: true,
+          patternsSuppressed: true,
+        },
+      };
+    }),
+    anonymizeMethod: { STRICT: 'STRICT', BALANCED: 'BALANCED', LIGHT: 'LIGHT' },
+  };
+});
+
+describe('anonymize', () => {
+  it('should return an object with text and stages properties', async () => {
+    const input = {
+      text: 'Test input',
+      method: anonymizeMethod.LIGHT,
+    };
+
+    const result = await anonymize(input);
+
+    expect(result).to.have.property('text');
+    expect(result).to.have.property('stages');
+    expect(result.stages).to.have.property('distinctiveContentRemoved');
+    expect(result.stages).to.have.property('structureNormalized');
+    expect(result.stages).to.have.property('patternsSuppressed');
+  });
+
+  it('should throw an error if method is not provided', async () => {
+    const input = {
+      text: 'Test input',
+    };
+
+    await expect(anonymize(input)).rejects.toThrow('Method is required');
+  });
+
+  it('should throw an error if method is invalid', async () => {
+    const input = {
+      text: 'Test input',
+      method: 'INVALID',
+    };
+
+    await expect(anonymize(input)).rejects.toThrow('Invalid method');
+  });
+
+  it('should throw an error if text is not provided', async () => {
+    const input = {
+      method: anonymizeMethod.LIGHT,
+    };
+
+    await expect(anonymize(input)).rejects.toThrow('Text is required');
+  });
+
+  it('should throw an error if text is empty', async () => {
+    const input = {
+      text: '',
+      method: anonymizeMethod.LIGHT,
+    };
+
+    await expect(anonymize(input)).rejects.toThrow('Text is required');
+  });
+});

package/src/chains/bulk-central-tendency/index.examples.js

@@ -0,0 +1,138 @@
+import { describe, expect, it } from 'vitest';
+import { longTestTimeout } from '../../constants/common.js';
+import { expect as aiExpect } from '../expect/index.js';
+import bulkCentralTendency from './index.js';
+
+describe('Bulk Central Tendency Chain', () => {
+  it(
+    'processes multiple fruit items with consistent results',
+    async () => {
+      const items = ['apple', 'orange', 'durian', 'jackfruit', 'banana'];
+      const seedItems = ['apple', 'orange', 'banana', 'grape', 'strawberry'];
+
+      const results = await bulkCentralTendency(items, seedItems, {
+        context: 'Common fruits found in grocery stores',
+      });
+
+      expect(results).toHaveLength(5);
+      expect(results.every((r) => r && typeof r.score === 'number')).toBe(true);
+      expect(results.every((r) => r && r.score >= 0 && r.score <= 1)).toBe(true);
+      expect(results.every((r) => r && typeof r.reason === 'string')).toBe(true);
+      expect(results.every((r) => r && typeof r.confidence === 'number')).toBe(true);
+
+      // Use expect-chain for loose verification
+      const [isValidCentralityScoring] = await aiExpect(
+        results,
+        undefined,
+        'Do these centrality scores make sense? Common fruits like apple, orange, banana should have higher scores than exotic fruits like durian and jackfruit.'
+      );
+      expect(isValidCentralityScoring).toBe(true);
+    },
+    longTestTimeout
+  );
+
+  it(
+    'handles tool centrality with core features',
+    async () => {
+      const items = ['hammer', 'screwdriver', 'wrench', 'pliers', 'chainsaw'];
+      const seedItems = ['hammer', 'screwdriver', 'wrench', 'saw', 'drill'];
+
+      const results = await bulkCentralTendency(items, seedItems, {
+        context: 'Hand tools for construction and repair',
+        coreFeatures: ['handheld', 'mechanical', 'durable'],
+      });
+
+      expect(results).toHaveLength(5);
+      expect(results.every((r) => r && typeof r.score === 'number')).toBe(true);
+      expect(results.every((r) => r && r.score >= 0 && r.score <= 1)).toBe(true);
+
+      // Use expect-chain for loose verification
+      const [isValidToolScoring] = await aiExpect(
+        results,
+        undefined,
+        'Do these tool centrality scores make sense? Basic hand tools like hammer, screwdriver, wrench should have high scores, while chainsaw (power tool) should have a lower score.'
+      );
+      expect(isValidToolScoring).toBe(true);
+    },
+    longTestTimeout
+  );
+
+  it(
+    'demonstrates context effects on centrality',
+    async () => {
+      const items = ['robin', 'eagle', 'penguin', 'ostrich'];
+      const seedItems = ['robin', 'sparrow', 'cardinal', 'blue jay'];
+
+      const results = await bulkCentralTendency(items, seedItems, {
+        context: 'Small songbirds commonly seen in backyards',
+      });
+
+      expect(results).toHaveLength(4);
+      expect(results.every((r) => r && typeof r.score === 'number')).toBe(true);
+
+      // Use expect-chain for loose verification
+      const [isValidBirdScoring] = await aiExpect(
+        results,
+        undefined,
+        'Given the context of "small songbirds commonly seen in backyards", does robin have the highest centrality score, while penguin and ostrich have much lower scores?'
+      );
+      expect(isValidBirdScoring).toBe(true);
+    },
+    longTestTimeout
+  );
+
+  it(
+    'manages retry logic for failed items',
+    async () => {
+      const items = ['cat', 'dog', 'elephant'];
+      const seedItems = ['cat', 'dog', 'rabbit', 'hamster'];
+
+      const results = await bulkCentralTendency(items, seedItems, {
+        maxAttempts: 2,
+      });
+
+      expect(results).toHaveLength(3);
+      expect(results.every((r) => r && typeof r.score === 'number')).toBe(true);
+
+      // Use expect-chain for loose verification
+      const [isValidPetScoring] = await aiExpect(
+        results,
+        undefined,
+        'Are these reasonable centrality scores for pets, with cat and dog having higher scores than elephant?'
+      );
+      expect(isValidPetScoring).toBe(true);
+    },
+    longTestTimeout
+  );
+
+  it('handles empty input', async () => {
+    const results = await bulkCentralTendency([], ['apple', 'orange']);
+    expect(results).toEqual([]);
+  });
+
+  it('throws error for invalid seed items', async () => {
+    await expect(bulkCentralTendency(['apple'], [])).rejects.toThrow(
+      'seedItems must be a non-empty array'
+    );
+    await expect(bulkCentralTendency(['apple'], null)).rejects.toThrow(
+      'seedItems must be a non-empty array'
+    );
+  });
+
+  it(
+    'processes large batches efficiently',
+    async () => {
+      const items = Array.from({ length: 15 }, (_, i) => `item${i + 1}`);
+      const seedItems = ['item1', 'item2', 'item3', 'item4', 'item5'];
+
+      const results = await bulkCentralTendency(items, seedItems, {
+        chunkSize: 3,
+      });
+
+      expect(results).toHaveLength(15);
+      expect(results.every((r) => r && typeof r.score === 'number')).toBe(true);
+      expect(results.every((r) => r && r.score >= 0 && r.score <= 1)).toBe(true);
+    },
+    longTestTimeout
+  );
+});

package/src/chains/bulk-central-tendency/index.js

@@ -0,0 +1,91 @@
+import { bulkMapRetry } from '../bulk-map/index.js';
+import { CENTRAL_TENDENCY_PROMPT } from '../../verblets/central-tendency/index.js';
+
+/**
+ * Build instructions for bulk central tendency evaluation using the core verblet prompt
+ * @param {string[]} seedItems - Array of seed items for comparison
+ * @param {Object} config - Configuration options
+ * @returns {string} Instructions for the bulk mapper
+ */
+function buildBulkCentralTendencyInstructions(seedItems, { context = '', coreFeatures = [] } = {}) {
+  const contextLine = context ? `Context: ${context}` : '';
+  const coreFeaturesLine =
+    coreFeatures.length > 0 ? `Core Features: ${coreFeatures.join(', ')}` : '';
+  const outputRequirementsLine = `OUTPUT FORMAT: Return exactly one compact JSON object per line (no line breaks within the JSON):
+{"score": <number>, "reason": "<brief explanation>", "confidence": <number>}`;
+
+  // Use the core prompt with all variables replaced
+  const corePrompt = CENTRAL_TENDENCY_PROMPT.replace('{context}', contextLine)
+    .replace('{coreFeatures}', coreFeaturesLine)
+    .replace('{outputRequirements}', outputRequirementsLine);
+
+  return `For each item, evaluate its centrality among these category members: ${seedItems.join(
+    ', '
+  )}
+
+${corePrompt}`;
+}
+
+/**
+ * Process multiple items for central tendency evaluation in bulk with retry support.
+ * Uses the existing bulkMapRetry infrastructure for efficiency and reliability.
+ *
+ * @param {string[]} items - Array of items to evaluate
+ * @param {string[]} seedItems - Array of seed items for comparison
+ * @param {Object} [config={}] - Configuration options
+ * @param {string} [config.context=''] - Context description for evaluation
+ * @param {string[]} [config.coreFeatures=[]] - Known core/definitional features
+ * @param {string|Object} [config.llm='fastGoodCheap'] - LLM model to use
+ * @param {number} [config.chunkSize=5] - Batch size for processing
+ * @param {number} [config.maxAttempts=3] - Max retry attempts for failed items
+ * @returns {Promise<Array>} Array of central tendency results
+ */
+export default async function bulkCentralTendency(items, seedItems, config = {}) {
+  if (!Array.isArray(items)) {
+    throw new Error('Items must be an array');
+  }
+
+  if (items.length === 0) {
+    return [];
+  }
+
+  if (!Array.isArray(seedItems) || seedItems.length === 0) {
+    throw new Error('seedItems must be a non-empty array');
+  }
+
+  const { chunkSize = 5, maxAttempts = 3, ...otherConfig } = config;
+
+  // Build instructions for the bulk mapper
+  const instructions = buildBulkCentralTendencyInstructions(seedItems, otherConfig);
+
+  // Use bulkMapRetry to handle all the complexity
+  const results = await bulkMapRetry(items, instructions, { chunkSize, maxAttempts });
+
+  // Parse JSON responses and handle any parsing errors
+  return results.map((result, _index) => {
+    if (result === undefined) {
+      return undefined;
+    }
+
+    try {
+      const parsed = JSON.parse(result);
+
+      // Validate the structure
+      if (
+        parsed &&
+        typeof parsed.score === 'number' &&
+        typeof parsed.reason === 'string' &&
+        typeof parsed.confidence === 'number'
+      ) {
+        return parsed;
+      } else {
+        return undefined;
+      }
+    } catch {
+      return undefined;
+    }
+  });
+}
+
+// Export the retry version as well for consistency with other bulk processors
+export const bulkCentralTendencyRetry = bulkCentralTendency;

package/src/chains/bulk-filter/README.md

@@ -0,0 +1,21 @@
+# bulk-filter
+
+Filter very long lists in manageable chunks using `listFilter`. Failed batches can be retried.
+
+```javascript
+import bulkFilter from './index.js';
+
+const diary = [
+  'Walked the dog and bought milk.',
+  'One day I hope to sail across the Atlantic.',
+  'Cleaned out the garage.',
+  "Maybe I'll start that bakery I keep dreaming about.",
+];
+
+const aspirations = await bulkFilter(
+  diary,
+  'Keep only lines about hopes or big dreams',
+  { chunkSize: 2 }
+);
+// => ['One day I hope to sail across the Atlantic.', "Maybe I'll start that bakery I keep dreaming about."]
+```

package/src/chains/bulk-filter/index.examples.js

@@ -0,0 +1,22 @@
+import { describe, expect, it } from 'vitest';
+import bulkFilter from './index.js';
+import { longTestTimeout } from '../../constants/common.js';
+
+describe('bulk-filter examples', () => {
+  it(
+    'filters with listFilter',
+    async () => {
+      const notes = [
+        'Saw a dolphin while surfing',
+        'Finished laundry',
+        'Dream of traveling to Iceland',
+        'Paid the electricity bill',
+      ];
+      const dreams = await bulkFilter(notes, 'keep only lines about aspirations or dreams', {
+        chunkSize: 2,
+      });
+      expect(dreams.length).toBeGreaterThan(0);
+    },
+    longTestTimeout
+  );
+});

package/src/chains/bulk-filter/index.js

@@ -0,0 +1,58 @@
+import listFilter from '../../verblets/list-filter/index.js';
+
+const buildMask = async (list, instructions, chunkSize, config = {}) => {
+  const mask = new Array(list.length);
+  for (let i = 0; i < list.length; i += chunkSize) {
+    const batch = list.slice(i, i + chunkSize);
+    try {
+      // eslint-disable-next-line no-await-in-loop
+      const result = await listFilter(batch, instructions, config);
+      const valid = result.every((item) => batch.includes(item));
+      if (!valid) {
+        for (let j = 0; j < batch.length; j += 1) {
+          mask[i + j] = undefined;
+        }
+        continue;
+      }
+      for (let j = 0; j < batch.length; j += 1) {
+        mask[i + j] = result.includes(batch[j]);
+      }
+    } catch {
+      for (let j = 0; j < batch.length; j += 1) {
+        mask[i + j] = undefined;
+      }
+    }
+  }
+  return mask;
+};
+
+export const bulkFilterRetry = async (list, instructions, config = {}) => {
+  const { chunkSize = 10, maxAttempts = 3, llm, ...options } = config;
+  const filterConfig = { llm, ...options };
+  let mask = await buildMask(list, instructions, chunkSize, filterConfig);
+  for (let attempt = 1; attempt < maxAttempts; attempt += 1) {
+    const missingIdx = [];
+    const missingItems = [];
+    mask.forEach((val, idx) => {
+      if (val === undefined) {
+        missingIdx.push(idx);
+        missingItems.push(list[idx]);
+      }
+    });
+    if (missingItems.length === 0) break;
+    // eslint-disable-next-line no-await-in-loop
+    const retryMask = await buildMask(missingItems, instructions, chunkSize, filterConfig);
+    retryMask.forEach((val, i) => {
+      if (val !== undefined) {
+        mask[missingIdx[i]] = val;
+      }
+    });
+  }
+  return list.filter((_, idx) => mask[idx]);
+};
+
+export default async function bulkFilter(list, instructions, config = {}) {
+  const { chunkSize = 10, llm, ...options } = config;
+  const mask = await buildMask(list, instructions, chunkSize, { llm, ...options });
+  return list.filter((_, idx) => mask[idx]);
+}

package/src/chains/bulk-filter/index.spec.js

@@ -0,0 +1,38 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import bulkFilter, { bulkFilterRetry } from './index.js';
+import listFilter from '../../verblets/list-filter/index.js';
+
+vi.mock('../../verblets/list-filter/index.js', () => ({
+  default: vi.fn(async (items, instructions) => {
+    if (items.includes('FAIL')) throw new Error('fail');
+    return items.filter((l) => l.includes(instructions));
+  }),
+}));
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe('bulk-filter', () => {
+  it('filters items in batches', async () => {
+    const result = await bulkFilter(['a', 'b', 'c'], 'a', { chunkSize: 2 });
+    expect(result).toStrictEqual(['a']);
+    expect(listFilter).toHaveBeenCalledTimes(2);
+  });
+
+  it('retries failed batches', async () => {
+    let call = 0;
+    listFilter.mockImplementation(async (items) => {
+      call += 1;
+      if (call === 1) throw new Error('fail');
+      return items.filter((l) => l.includes('a'));
+    });
+
+    const result = await bulkFilterRetry(['FAIL', 'a', 'b'], 'a', {
+      chunkSize: 2,
+      maxAttempts: 2,
+    });
+    expect(result).toStrictEqual(['a']);
+    expect(listFilter).toHaveBeenCalledTimes(3);
+  });
+});

package/src/chains/bulk-find/README.md

@@ -0,0 +1,16 @@
+# bulk-find
+
+Scan long lists in manageable batches to locate the item that best matches your instructions.
+
+```javascript
+import bulkFind from './index.js';
+
+const emails = [
+  'update from accounting',
+  'party invitation',
+  'weekly newsletter',
+  // ... potentially thousands more
+];
+const best = await bulkFind(emails, 'Which email is most urgent?');
+// => 'update from accounting'
+```

package/src/chains/bulk-find/index.examples.js

@@ -0,0 +1,20 @@
+import { describe, expect, it } from 'vitest';
+import bulkFind from './index.js';
+import { longTestTimeout } from '../../constants/common.js';
+
+describe('bulk-find examples', () => {
+  it(
+    'finds the best match across batches',
+    async () => {
+      const titles = [
+        'ancient mystery',
+        'space odyssey',
+        'underwater adventure',
+        'future tech thriller',
+      ];
+      const result = await bulkFind(titles, 'Which title feels most futuristic?', { chunkSize: 2 });
+      expect(result).toBeDefined();
+    },
+    longTestTimeout
+  );
+});