@far-world-labs/verblets 0.1.1 → 0.1.2

package/src/lib/bulk-filter/index.examples.js

@@ -0,0 +1,27 @@
+import { describe, it, expect } from 'vitest';
+import bulkFilter from './index.js';
+import { longTestTimeout } from '../../constants/common.js';
+
+describe('bulk-filter examples', () => {
+  it(
+    'filters items in batches',
+    async () => {
+      const entries = [
+        'Losing that match taught me the value of persistence.',
+        "I hate losing and it proves I'm worthless.",
+        'After failing my exam, I studied harder and passed the retake.',
+        "No matter what I do, I'll never succeed.",
+      ];
+      const result = await bulkFilter(
+        entries,
+        'keep only reflections that show personal growth or learning from mistakes',
+        2
+      );
+      expect(result).toStrictEqual([
+        'Losing that match taught me the value of persistence.',
+        'After failing my exam, I studied harder and passed the retake.',
+      ]);
+    },
+    longTestTimeout
+  );
+});

package/src/lib/bulk-filter/index.js

@@ -0,0 +1,63 @@
+import listFilter from '../../verblets/list-filter/index.js';
+
+/**
+ * Filter a list by processing newline-delimited batches with `listFilter`.
+ *
+ * @param {string[]} list - array of items to evaluate
+ * @param {string} instructions - filter instructions for `listFilter`
+ * @param {number} [chunkSize=10] - how many items per batch
+ * @returns {Promise<string[]>} filtered items preserving order
+ */
+export default async function bulkFilter(list, instructions, chunkSize = 10) {
+  const batches = [];
+  for (let i = 0; i < list.length; i += chunkSize) {
+    const batch = list.slice(i, i + chunkSize);
+    const filtered = await listFilter(batch, instructions);
+    batches.push(filtered);
+  }
+  return batches.flat();
+}
+
+/**
+ * Retry filtering failed batches until `maxAttempts` is reached.
+ *
+ * @param {string[]} list
+ * @param {string} instructions
+ * @param {object} [options]
+ * @param {number} [options.chunkSize=10]
+ * @param {number} [options.maxAttempts=3]
+ * @returns {Promise<string[]>}
+ */
+export async function bulkFilterRetry(
+  list,
+  instructions,
+  { chunkSize = 10, maxAttempts = 3 } = {}
+) {
+  const batches = [];
+  for (let i = 0; i < list.length; i += chunkSize) {
+    batches.push(list.slice(i, i + chunkSize));
+  }
+  const results = new Array(batches.length).fill(null);
+  let pending = batches.map((_, idx) => idx);
+
+  for (let attempt = 0; attempt < maxAttempts && pending.length > 0; attempt += 1) {
+    const newPending = [];
+    for (const idx of pending) {
+      const batch = batches[idx];
+      try {
+        const filtered = await listFilter(batch, instructions);
+        const valid = filtered.every((line) => batch.includes(line));
+        if (valid) {
+          results[idx] = filtered;
+        } else {
+          newPending.push(idx);
+        }
+      } catch {
+        newPending.push(idx);
+      }
+    }
+    pending = newPending;
+  }
+
+  return results.flat().filter(Boolean);
+}

package/src/lib/bulk-filter/index.spec.js

@@ -0,0 +1,38 @@
+import { describe, expect, it, vi, beforeEach } from 'vitest';
+import bulkFilter, { bulkFilterRetry } from './index.js';
+import listFilter from '../../verblets/list-filter/index.js';
+
+vi.mock('../../verblets/list-filter/index.js', () => ({
+  default: vi.fn(async (items, instructions) => {
+    if (items.includes('FAIL')) throw new Error('fail');
+    return items.filter((l) => l.includes(instructions));
+  }),
+}));
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe('bulk-filter', () => {
+  it('filters fragments in batches', async () => {
+    const result = await bulkFilter(['aa', 'b', 'ca', 'd'], 'a', 2);
+    expect(result).toStrictEqual(['aa', 'ca']);
+    expect(listFilter).toHaveBeenCalledTimes(2);
+  });
+
+  it('retries failed batches', async () => {
+    let call = 0;
+    listFilter.mockImplementation(async (items) => {
+      call += 1;
+      if (call === 1) throw new Error('fail');
+      return items.filter((l) => l.includes('a'));
+    });
+
+    const result = await bulkFilterRetry(['aa', 'b', 'ca'], 'a', {
+      chunkSize: 2,
+      maxAttempts: 2,
+    });
+    expect(result).toStrictEqual(['aa', 'ca']);
+    expect(listFilter).toHaveBeenCalledTimes(3);
+  });
+});

package/src/lib/bulk-find/README.md

@@ -0,0 +1,18 @@
+# bulk-find
+
+Search large lists in batches using `listFind` and return the first matching item.
+Failed batches can be retried with `bulkFindRetry`.
+
+```javascript
+import bulkFind, { bulkFindRetry } from './index.js';
+
+const diaryEntries = [
+  'Hiked up the mountains today and saw breathtaking views',
+  'Visited the local market and tried a spicy stew',
+  'Spotted penguins playing on the beach this morning'
+];
+const penguinMoment = await bulkFindRetry(diaryEntries, 'mentions penguins', {
+  chunkSize: 2
+});
+// penguinMoment === 'Spotted penguins playing on the beach this morning'
+```

package/src/lib/bulk-find/index.examples.js

@@ -0,0 +1,19 @@
+import { describe, it, expect } from 'vitest';
+import { bulkFindRetry } from './index.js';
+import { longTestTimeout } from '../../constants/common.js';
+
+describe('bulk-find examples', () => {
+  it(
+    'finds an item across batches',
+    async () => {
+      const diaryEntries = [
+        'Hiked up the mountains today and saw breathtaking views',
+        'Visited the local market and tried a spicy stew',
+        'Spotted penguins playing on the beach this morning',
+      ];
+      const result = await bulkFindRetry(diaryEntries, 'mentions penguins');
+      expect(result).toBeDefined();
+    },
+    longTestTimeout
+  );
+});

package/src/lib/bulk-find/index.js

@@ -0,0 +1,30 @@
+import listFind from '../../verblets/list-find/index.js';
+
+/**
+ * Search a list in batches using `listFind`.
+ *
+ * @param { string[] } list - array of items to search
+ * @param { string } instructions - criteria passed to `listFind`
+ * @param { number } [chunkSize=10] - how many items per batch
+ * @returns { Promise<string|undefined> } first match or undefined
+ */
+export default async function bulkFind(list, instructions, chunkSize = 10) {
+  for (let i = 0; i < list.length; i += chunkSize) {
+    const batch = list.slice(i, i + chunkSize);
+    try {
+      const result = await listFind(batch, instructions);
+      if (result) return result;
+    } catch {
+      // ignore and continue to next batch
+    }
+  }
+  return undefined;
+}
+
+export async function bulkFindRetry(list, instructions, { chunkSize = 10, maxAttempts = 3 } = {}) {
+  for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+    const result = await bulkFind(list, instructions, chunkSize);
+    if (result) return result;
+  }
+  return undefined;
+}

package/src/lib/bulk-find/index.spec.js

@@ -0,0 +1,41 @@
+import { describe, expect, it, vi, beforeEach } from 'vitest';
+import bulkFind, { bulkFindRetry } from './index.js';
+import listFind from '../../verblets/list-find/index.js';
+
+vi.mock('../../verblets/list-find/index.js', () => ({
+  default: vi.fn(async (list, instructions) => list.find((l) => l.includes(instructions)) || ''),
+}));
+
+beforeEach(() => {
+  vi.clearAllMocks();
+});
+
+describe('bulk-find', () => {
+  it('returns first match across batches', async () => {
+    const result = await bulkFind(['a', 'b', 'c', 'd'], 'c', 2);
+    expect(result).toBe('c');
+    expect(listFind).toHaveBeenCalledTimes(2);
+  });
+
+  it('returns undefined when not found', async () => {
+    listFind.mockResolvedValueOnce('');
+    listFind.mockResolvedValueOnce('');
+    const result = await bulkFind(['a', 'b'], 'x', 1);
+    expect(result).toBeUndefined();
+  });
+
+  it('retries until a match is found', async () => {
+    listFind.mockResolvedValueOnce('');
+    listFind.mockResolvedValueOnce('c');
+    const result = await bulkFindRetry(['a', 'b', 'c'], 'c', { chunkSize: 2, maxAttempts: 2 });
+    expect(result).toBe('c');
+    expect(listFind).toHaveBeenCalledTimes(2);
+  });
+
+  it('returns undefined after maxAttempts', async () => {
+    listFind.mockResolvedValue('');
+    const result = await bulkFindRetry(['a', 'b'], 'x', { chunkSize: 1, maxAttempts: 2 });
+    expect(result).toBeUndefined();
+    expect(listFind).toHaveBeenCalledTimes(4);
+  });
+});

package/src/lib/chatgpt/index.js

@@ -1,18 +1,14 @@
 import fetch from 'node-fetch';
 
 import {
-  apiKey,
-  apiUrl,
   debugPromptGlobally,
   debugPromptGloballyIfChanged,
   debugResultGlobally,
   debugResultGloballyIfChanged,
-} from '../../constants/openai.js';
+  models,
+} from '../../constants/models.js';
 import anySignal from '../any-signal/index.js';
-import {
-  get as getPromptResult,
-  set as setPromptResult,
-} from '../prompt-cache/index.js';
+import { get as getPromptResult, set as setPromptResult } from '../prompt-cache/index.js';
 import TimedAbortController from '../timed-abort-controller/index.js';
 import modelService from '../../services/llm-model/index.js';
 import { getClient as getRedis } from '../../services/redis/index.js';
@@ -20,13 +16,11 @@ import { getClient as getRedis } from '../../services/redis/index.js';
 const shapeOutputDefault = (result) => {
   // GPT-4
   if (result.choices[0].message.tool_calls?.length) {
-    const toolCall = result.choices[0].message.tool_calls[0]
+    const toolCall = result.choices[0].message.tool_calls[0];
     return {
-      name: toolCall['function'].name,
-      arguments: JSON.parse(
-	toolCall['function'].arguments
-      ),
-      result: result.choices[0].message.tool_calls[0].function
+      name: toolCall.function.name,
+      arguments: JSON.parse(toolCall.function.arguments),
+      result: result.choices[0].message.tool_calls[0].function,
     };
   }
   if (result.choices[0].message) {
@@ -36,30 +30,30 @@ const shapeOutputDefault = (result) => {
 };
 
 const onBeforeRequestDefault = ({ debugPrompt, isCached, prompt }) => {
-  if (
-    debugPrompt ||
-    debugPromptGlobally ||
-    (debugPromptGloballyIfChanged && !isCached)
-  ) {
-    console.error(`+++ DEBUG PROMPT +++`);
+  if (debugPrompt || debugPromptGlobally || (debugPromptGloballyIfChanged && !isCached)) {
+    console.error('+++ DEBUG PROMPT +++');
     console.error(prompt);
     console.error('+++ DEBUG PROMPT END +++');
   }
 };
 
 const onAfterRequestDefault = ({ debugResult, isCached, resultShaped }) => {
-  if (
-    debugResult ||
-      debugResultGlobally ||
-      (debugResultGloballyIfChanged && !isCached)
-  ) {
-    console.error(`+++ DEBUG RESULT +++`);
+  if (debugResult || debugResultGlobally || (debugResultGloballyIfChanged && !isCached)) {
+    console.error('+++ DEBUG RESULT +++');
     console.error(resultShaped);
     console.error('+++ DEBUG RESULT END +++');
   }
 };
 
-export const run = async (prompt, options = {}) => {
+export const run = async (prompt, config = {}) => {
+  // Handle config parameter - can be string (model name) or object (full options)
+  let options;
+  if (typeof config === 'string') {
+    options = { modelOptions: { modelName: config } };
+  } else {
+    options = config;
+  }
+
   const {
     abortSignal,
     debugPrompt,
@@ -71,17 +65,43 @@ export const run = async (prompt, options = {}) => {
     shapeOutput = shapeOutputDefault,
   } = options;
 
-  debugger;
+  // Apply global overrides to model options
+  const modelOptionsWithOverrides = modelService.applyGlobalOverrides(modelOptions);
+
+  // Check if negotiation was applied via global override
+  const negotiationFromGlobalOverride = modelService.getGlobalOverride('negotiate');
+
+  const modelNameNegotiated = modelOptionsWithOverrides.negotiate
+    ? modelService.negotiateModel(
+        // If negotiation came from global override, don't use preferred model
+        negotiationFromGlobalOverride ? null : modelOptionsWithOverrides.modelName,
+        modelOptionsWithOverrides.negotiate
+      )
+    : modelOptionsWithOverrides.modelName;
+
+  const modelFound = modelService.getModel(modelNameNegotiated);
 
-  const modelFound = modelService.getModel(modelOptions.modelName);
+  // Use model-specific API URL and key if defined, otherwise fall back to defaults
+  const apiUrl = modelFound?.apiUrl || models.fastGood.apiUrl;
+  const apiKey = modelFound?.apiKey || models.fastGood.apiKey;
 
   const requestConfig = modelService.getRequestConfig({
     prompt,
-    ...modelOptions,
+    ...modelOptionsWithOverrides,
+    modelName: modelNameNegotiated,
   });
 
-  const cache = await getRedis();
-  const { result: cacheResult } = await getPromptResult(cache, requestConfig);
+  // Check if caching is disabled via environment variable
+  const cachingDisabled = process.env.DISABLE_CACHE === 'true';
+
+  let cacheResult = null;
+  let cache = null;
+
+  if (!cachingDisabled) {
+    cache = await getRedis();
+    const { result } = await getPromptResult(cache, requestConfig);
+    cacheResult = result;
+  }
 
   onBeforeRequest({
     isCached: !!cacheResult,
@@ -92,9 +112,12 @@ export const run = async (prompt, options = {}) => {
 
   let result = cacheResult;
   if (!cacheResult || forceQuery) {
-    const timeoutController = new TimedAbortController(
-      modelService.getModel(modelOptions.modelName).requestTimeout
-    );
+    // Use custom requestTimeout from modelOptions if provided, otherwise use model default
+    const requestTimeout =
+      modelOptionsWithOverrides.requestTimeout ||
+      modelService.getModel(modelNameNegotiated).requestTimeout;
+
+    const timeoutController = new TimedAbortController(requestTimeout);
 
     // console.log(requestConfig, `${apiUrl}${modelFound.endpoint}`)
 
@@ -111,21 +134,18 @@ export const run = async (prompt, options = {}) => {
     result = await response.json();
 
     if (!response.ok) {
-      const vars = [
-        `status: ${response?.status}`,
-        `type: ${result?.error?.type}`,
-      ].join(', ');
-      throw new Error(
-        `Completions request [error]: ${result?.error?.message} (${vars})`
-      );
+      const vars = [`status: ${response?.status}`, `type: ${result?.error?.type}`].join(', ');
+      throw new Error(`Completions request [error]: ${result?.error?.message} (${vars})`);
     }
 
     timeoutController.clearTimeout();
 
-    await setPromptResult(cache, requestConfig, result);
+    // Only cache the result if caching is not disabled
+    if (!cachingDisabled && cache) {
+      await setPromptResult(cache, requestConfig, result);
+    }
   }
 
-  debugger;
   const resultShaped = shapeOutput(result);
 
   onAfterRequest({

package/src/lib/combinations/index.js

@@ -0,0 +1,30 @@
+export default function combinations(items, size = 2) {
+  const result = [];
+  if (!Array.isArray(items) || size < 1) return result;
+
+  const combo = [];
+  const generate = (start) => {
+    if (combo.length === size) {
+      result.push([...combo]);
+      return;
+    }
+    for (let i = start; i < items.length; i++) {
+      combo.push(items[i]);
+      generate(i + 1);
+      combo.pop();
+    }
+  };
+
+  generate(0);
+  return result;
+}
+
+export function rangeCombinations(items, minSize = 2, maxSize = items.length) {
+  const sets = [];
+  if (!Array.isArray(items)) return sets;
+  const upper = Math.min(items.length, maxSize);
+  for (let s = minSize; s <= upper; s++) {
+    sets.push(...combinations(items, s));
+  }
+  return sets;
+}

package/src/lib/combinations/index.spec.js

@@ -0,0 +1,23 @@
+import { describe, it, expect } from 'vitest';
+import combinations, { rangeCombinations } from './index.js';
+
+describe('combinations helper', () => {
+  it('generates pairwise combinations', () => {
+    const result = combinations(['a', 'b', 'c'], 2);
+    expect(result).toStrictEqual([
+      ['a', 'b'],
+      ['a', 'c'],
+      ['b', 'c'],
+    ]);
+  });
+
+  it('generates combinations of varying sizes', () => {
+    const result = rangeCombinations(['a', 'b', 'c']);
+    expect(result).toStrictEqual([
+      ['a', 'b'],
+      ['a', 'c'],
+      ['b', 'c'],
+      ['a', 'b', 'c'],
+    ]);
+  });
+});

package/src/lib/functional/index.js

@@ -0,0 +1,28 @@
+export const hook = (list, f, returnVal) => {
+  list.push(f);
+  return returnVal;
+};
+
+export const hookOnce = (list, f, returnVal) => {
+  if (list.includes(f)) return returnVal;
+  list.push(f);
+  return returnVal;
+};
+
+export const hookFn = (list, returnVal) => (f) => hook(list, f, returnVal);
+
+export const hookOnceFn = (list, returnVal) => (f) => hookOnce(list, f, returnVal);
+
+export const unhook = (list, f, returnVal) => {
+  const idx = list.indexOf(f);
+  if (idx === -1) return returnVal;
+  list.splice(idx, 1);
+  return returnVal;
+};
+
+export const unhookFn = (list, returnVal) => (f) => unhook(list, f, returnVal);
+
+export const unhookAll = (list, returnVal) => {
+  list.splice(0);
+  return returnVal;
+};

package/src/lib/logger-service/index.js

@@ -0,0 +1,32 @@
+// Global logger service
+let globalLogger = null;
+
+/**
+ * Set the global logger instance
+ * @param {Object} logger - Logger instance to use globally
+ */
+export const setLogger = (logger) => {
+  globalLogger = logger;
+};
+
+/**
+ * Reset the global logger to null
+ */
+export const resetLogger = () => {
+  globalLogger = null;
+};
+
+/**
+ * Get the current global logger instance
+ * @returns {Object|null} Current logger instance or null if not set
+ */
+export const getLogger = () => globalLogger;
+
+// Export convenience methods that use the global logger
+export const log = (...args) => globalLogger?.log(...args);
+export const info = (...args) => globalLogger?.info(...args);
+export const warn = (...args) => globalLogger?.warn(...args);
+export const error = (...args) => globalLogger?.error(...args);
+export const debug = (...args) => globalLogger?.debug(...args);
+export const trace = (...args) => globalLogger?.trace(...args);
+export const fatal = (...args) => globalLogger?.fatal(...args);

package/src/lib/parse-js-parts/index.js

@@ -44,9 +44,7 @@ const scanFile = (file, code) => {
           return { ...s.imported }.name; // also has local name
         });
       const source = importNode.source.value;
-      const importKey = source.startsWith('.')
-        ? convertImport(file, source)
-        : source;
+      const importKey = source.startsWith('.') ? convertImport(file, source) : source;
       importsMap[importKey] = {
         start: importNode.start,
         end: importNode.end,
@@ -60,9 +58,7 @@ const scanFile = (file, code) => {
       if (expNode.source) {
         // Handle re-exports
         const source = expNode.source.value;
-        const importKey = source.startsWith('.')
-          ? convertImport(file, source)
-          : source;
+        const importKey = source.startsWith('.') ? convertImport(file, source) : source;
 
         // Named exports
         if (expNode.specifiers) {
@@ -128,9 +124,7 @@ const scanFile = (file, code) => {
     ExportAllDeclaration(expNode) {
       if (expNode.source) {
         const source = expNode.source.value;
-        const importKey = source.startsWith('.')
-          ? convertImport(file, source)
-          : source;
+        const importKey = source.startsWith('.') ? convertImport(file, source) : source;
         exportsMap[expNode.exported.name] = {
           start: expNode.start,
           end: expNode.end,
@@ -161,7 +155,7 @@ const scanFile = (file, code) => {
           end: expNode.end,
         };
 
-        functionsMap[`ArrowFunctionExpression:default`] = {
+        functionsMap['ArrowFunctionExpression:default'] = {
           start: expNode.declaration.start,
           end: expNode.declaration.end,
           functionName: '<default-export>',
@@ -195,9 +189,7 @@ const scanFile = (file, code) => {
       });
 
       arrowDeclarations.forEach((arrowFnNode) => {
-        functionsSeen[
-          `${arrowFnNode.init.start}:${arrowFnNode.init.end}`
-        ] = true;
+        functionsSeen[`${arrowFnNode.init.start}:${arrowFnNode.init.end}`] = true;
 
         functionsMap[`ArrowFunctionExpression:${arrowFnNode.id.name}`] = {
           start: arrowFnNode.start,
@@ -283,16 +275,12 @@ const scanFile = (file, code) => {
       const className = node.id.name;
       node.body.body.forEach((classElement) => {
         if (classElement.type === 'MethodDefinition') {
-          functionsSeen[
-            `${classElement.value.start}:${classElement.value.end}`
-          ] = true;
+          functionsSeen[`${classElement.value.start}:${classElement.value.end}`] = true;
 
-          functionsMap[
-            `MethodDefinition:${className}.${classElement.key.name}`
-          ] = {
+          functionsMap[`MethodDefinition:${className}.${classElement.key.name}`] = {
             start: node.start,
             end: node.end,
-            className: className,
+            className,
             functionName: `${className}.${classElement.key.name}`,
             name: classElement.key.name,
             async: classElement.value.async,
@@ -310,7 +298,7 @@ const scanFile = (file, code) => {
       if (!functionsSeen[`${node.start}:${node.end}`]) {
         functionsMap[`${node.type}:${node.start}`] = {
           start: node.start,
-          functionName: `<exp>`,
+          functionName: '<exp>',
           end: node.end,
           type: node.type,
           async: node.async,

package/src/lib/parse-llm-list/README.md

@@ -0,0 +1,39 @@
+# Parse LLM List
+
+A utility function for parsing lists from LLM responses that may be in JSON array or CSV format.
+
+## Usage
+
+```javascript
+import parseLLMList from './src/lib/parse-llm-list/index.js';
+
+// Parse JSON array response
+const jsonResponse = '["term1", "term2", "term3"]';
+const terms1 = parseLLMList(jsonResponse);
+// Returns: ["term1", "term2", "term3"]
+
+// Parse CSV response
+const csvResponse = 'term1, term2, term3';
+const terms2 = parseLLMList(csvResponse);
+// Returns: ["term1", "term2", "term3"]
+
+// With custom options
+const terms3 = parseLLMList(response, {
+  excludeValues: ['none', 'null', 'n/a'],
+  trimItems: true,
+  filterEmpty: true
+});
+```
+
+## Options
+
+- `excludeValues` (string[]): Values to exclude from results (case-insensitive). Default: `['none', 'null', 'undefined']`
+- `trimItems` (boolean): Whether to trim whitespace from items. Default: `true`
+- `filterEmpty` (boolean): Whether to filter out empty strings. Default: `true`
+
+## Features
+
+- Automatically detects JSON array vs CSV format
+- Handles common LLM response patterns like `<note>` tags
+- Filters out excluded values and empty responses
+- Robust error handling with fallback parsing