@far-world-labs/verblets 0.1.1 → 0.1.2

package/eslint.config.js

@@ -0,0 +1,80 @@
+import js from '@eslint/js';
+import prettierConfig from 'eslint-config-prettier';
+import prettierPlugin from 'eslint-plugin-prettier';
+
+export default [
+  js.configs.recommended,
+  {
+    files: ['src/**/*.js'],
+    languageOptions: {
+      ecmaVersion: 2022,
+      sourceType: 'module',
+      globals: {
+        console: 'readonly',
+        process: 'readonly',
+        Buffer: 'readonly',
+        __dirname: 'readonly',
+        __filename: 'readonly',
+        URL: 'readonly',
+        AbortController: 'readonly',
+        fetch: 'readonly',
+        setTimeout: 'readonly',
+        clearTimeout: 'readonly',
+        setInterval: 'readonly',
+        clearInterval: 'readonly',
+        // Vitest
+        beforeEach: 'readonly',
+        afterEach: 'readonly',
+        describe: 'readonly',
+        it: 'readonly',
+        expect: 'readonly',
+        vi: 'readonly',
+      },
+    },
+    linterOptions: {
+      reportUnusedDisableDirectives: false,
+    },
+    plugins: {
+      prettier: prettierPlugin,
+    },
+    rules: {
+      'prettier/prettier': 'error',
+      
+      // TODO: Remove console statements and replace with proper logging
+      'no-console': 'off',
+      
+      // TODO: Refactor await-in-loop patterns to use Promise.all()
+      'no-await-in-loop': 'off',
+
+      // Customize recommended rules
+      'no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
+
+      // Additional quality rules not in recommended
+      'require-await': 'error',
+      'prefer-arrow-callback': 'error',
+      'prefer-template': 'error',
+      'object-shorthand': 'error',
+      'array-callback-return': 'error',
+
+      // Syntax restrictions
+      'no-restricted-syntax': [
+        'error',
+        {
+          selector: 'ForInStatement',
+          message: 'Use Object.{keys,values,entries} instead of for..in',
+        },
+        {
+          selector: 'WithStatement',
+          message: '`with` is not allowed',
+        },
+      ],
+    },
+  },
+  {
+    files: ['**/*.spec.js', '**/*.test.js'],
+    rules: {
+      'require-await': 'off', // Test functions often don't need await
+    },
+  },
+  prettierConfig,
+]; 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@far-world-labs/verblets",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "OpenAI Client",
   "main": "index.js",
   "type": "module",
@@ -15,56 +15,68 @@
     "--": "npm run script -- generate-verblet foo",
     "script": "./scripts/run.sh",
     "test": "vitest",
-    "examples": "EXAMPLES=true vitest --config .vite.config.examples.js",
+    "examples:warn": "source .env && LLM_EXPECT_MODE=info EXAMPLES=true vitest --config .vitest.config.examples.js",
+    "examples": "source .env && LLM_EXPECT_MODE=error EXAMPLES=true vitest --config .vitest.config.examples.js",
     "lint": "eslint 'src/**/*.{js,jsx}'",
     "lint:fix": "eslint 'src/**/*.{js,jsx}' --fix",
+    "check:deps": "npx npm-deprecated-check current",
     "husky:install": "husky install",
     "husky:uninstall": "husky uninstall",
-    "prepare": "npx husky install"
+    "prepare": "npx husky install",
+    "version:patch": "npm version patch",
+    "version:minor": "npm version minor",
+    "version:major": "npm version major"
   },
   "config": {
     "gen-script": "./scripts/run.sh gen-$1"
   },
   "author": "Far World Labs",
   "license": "All Rights Reserved",
+  "publishConfig": {
+    "access": "public"
+  },
   "dependencies": {
     "acorn": "^8.8.2",
     "acorn-walk": "^8.2.0",
     "ajv": "^8.12.0",
     "chai": "^4.3.7",
     "change-case": "^4.1.2",
+    "commander": "^11.0.0",
+    "compromise": "^14.14.4",
     "dependency-tree": "^10.0.1",
     "dotenv": "^16.0.3",
     "gpt-tokenizer": "^2.1.2",
     "gpt4-tokenizer": "^1.3.0",
+    "lodash": "^4.17.21",
     "mocha": "^10.2.0",
     "node-fetch": "^3.3.0",
-    "nodemon": "^2.0.22",
+    "node-record-lpcm16": "^1.0.1",
     "ramda": "^0.29.0",
     "redis": "^4.6.5",
     "uuid": "^9.0.0",
-    "vitest": "^0.30.1",
+    "vitest": "^3.1.3",
+    "whisper-node": "^1.1.1",
     "yargs": "^17.7.1"
   },
   "devDependencies": {
+    "@eslint/js": "^9.28.0",
+    "@stylistic/eslint-plugin": "^4.4.0",
     "@types/node": "^18.16.3",
-    "eslint": "^8.39.0",
-    "eslint-config-airbnb": "^19.0.4",
+    "@vitest/ui": "^3.1.3",
+    "eslint": "^9.28.0",
     "eslint-config-prettier": "^8.8.0",
-    "eslint-plugin-import": "^2.27.5",
-    "eslint-plugin-jest": "^27.2.1",
-    "eslint-plugin-jsx-a11y": "^6.7.1",
+    "eslint-plugin-import": "^2.31.0",
+    "eslint-plugin-jest": "^28.12.0",
     "eslint-plugin-prettier": "^4.2.1",
-    "eslint-plugin-react": "^7.32.2",
-    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-unicorn": "^59.0.1",
+    "eslint-plugin-vitest": "^0.5.4",
     "husky": "^8.0.3",
     "install-peerdeps": "^3.0.3",
     "lint-staged": "^13.2.2",
-    "node-record-lpcm16": "^1.0.1",
-    "npm-check": "^6.0.1",
+    "nodemon": "^3.1.10",
+    "npm-deprecated-check": "^1.5.0",
     "prettier": "^2.8.8",
-    "release-it": "^15.10.1",
-    "whisper-node": "^1.1.1",
+    "release-it": "^19.0.3",
     "why-is-node-running": "^2.2.2"
   },
   "lint-staged": {

package/scripts/generate-test/index.js

@@ -8,9 +8,35 @@ chatGPT,
   getRedis,
   SummaryMap
 } from '../../src/index.js';
-
-const modulePath = process.argv[2];
-const functionName = process.argv[3];
+import modelService from '../../src/services/llm-model/index.js';
+import { Command } from 'commander';
+
+const program = new Command();
+program
+  .argument('<modulePath>', 'Module to test')
+  .argument('[functionName]', 'Specific function to test')
+  .option('-p, --privacy', 'Use privacy model if configured')
+  .option('-m, --model <modelName>', 'Specify model name to use');
+
+program.parse(process.argv);
+
+const options = program.opts();
+const [modulePath, functionName] = program.args;
+
+if (options.privacy) {
+  try {
+    modelService.setGlobalOverride('modelName', 'privacy');
+  } catch (err) {
+    console.error(`Privacy model error: ${err.message}`);
+  }
+}
+if (options.model) {
+  try {
+    modelService.setGlobalOverride('modelName', options.model);
+  } catch (err) {
+    console.error(`Model override error: ${err.message}`);
+  }
+}
 
 if (!modulePath) {
   console.error('Please specify a module to test.');

package/scripts/runner/index.js

@@ -5,6 +5,32 @@ import chatGPT, {
   retry as run,
   schemas,
 } from '../../src/index.js';
+import modelService from '../../src/services/llm-model/index.js';
+import { Command } from 'commander';
+
+const program = new Command();
+program
+  .option('-p, --privacy', 'Use privacy model if configured')
+  .option('-m, --model <modelName>', 'Specify model name to use');
+
+program.parse(process.argv);
+
+const options = program.opts();
+
+if (options.privacy) {
+  try {
+    modelService.setGlobalOverride('modelName', 'privacy');
+  } catch (err) {
+    console.error(`Privacy model error: ${err.message}`);
+  }
+}
+if (options.model) {
+  try {
+    modelService.setGlobalOverride('modelName', options.model);
+  } catch (err) {
+    console.error(`Model override error: ${err.message}`);
+  }
+}
 
 await run(async () => {
   const results = await chatGPT('make a list of nintendo games with a schema that includes a title, year, and maybe a couple others of your choice', {

package/scripts/simple-editor/index.js

@@ -1,25 +1,36 @@
 import dotenv from 'dotenv/config';
-import yargs from 'yargs';
-import { hideBin } from 'yargs/helpers';
+import { Command } from 'commander';
 
 import chatGPT, { getRedis, auto, bool } from '../../src/index.js';
+import modelService from '../../src/services/llm-model/index.js';
 import edit from '../../src/lib/editor/index.js';
 import Transcriber from '../../src/lib/transcribe/index.js';
 
-const argv = yargs(hideBin(process.argv))
-  .option('use-intent', {
-    alias: 'i',
-    default: 'true',
-    type: 'string',
-    description: '"false" to disable intent parsing and just use chatGPT',
-  })
-  .option('transcribe', {
-    alias: 't',
-    default: 'false',
-    type: 'string',
-    description: '"true" to enable audio transcription',
-  })
-  .argv;
+const program = new Command();
+program
+  .option('-t, --transcribe', 'Enable audio transcription')
+  .option('--no-use-intent', 'Disable intent parsing')
+  .option('-p, --privacy', 'Use privacy model if configured')
+  .option('-m, --model <modelName>', 'Specify model name to use');
+
+program.parse(process.argv);
+
+const argv = program.opts();
+
+if (argv.privacy) {
+  try {
+    modelService.setGlobalOverride('modelName', 'privacy');
+  } catch (err) {
+    console.error(`Privacy model error: ${err.message}`);
+  }
+}
+if (argv.model) {
+  try {
+    modelService.setGlobalOverride('modelName', argv.model);
+  } catch (err) {
+    console.error(`Model override error: ${err.message}`);
+  }
+}
 
 const operations = [
   {
@@ -32,7 +43,7 @@ const operations = [
 
 
 let userInput;
-const useTranscribe = ['true', '1'].includes(argv.transcribe)
+const useTranscribe = !!argv.transcribe;
 if (useTranscribe) {
   const transcriber = new Transcriber("stopword"); // Replace "stopword" with the word you want to trigger the stop
   userInput = await transcriber.startRecording()
@@ -40,7 +51,7 @@ if (useTranscribe) {
   userInput = await edit();
 }
 
-const useIntent = !!['true', '1'].includes(argv.useIntent.toLowerCase())
+const useIntent = argv.useIntent !== false;
 
 const commandType = useIntent ? 'Tool selection' : 'Direct ChatGPT';
 console.error(`Command: ${commandType}`);

package/scripts/summarize-files/index.js

@@ -1,12 +1,36 @@
 import glob from 'glob';
 import { readFile } from 'fs/promises';
 import SummaryMap from '../../src/chains/summary-map/index.js';
+import modelService from '../../src/services/llm-model/index.js';
+import { Command } from 'commander';
 
-// Define your glob pattern here or pass it as a command line argument
-const globPattern = process.argv[2] || './src/**/*.js';
+const program = new Command();
+program
+  .argument('[globPattern]', 'Glob pattern to summarize', './src/**/*.js')
+  .argument('[targetTokens]', 'Target token count', '4097')
+  .option('-p, --privacy', 'Use privacy model if configured')
+  .option('-m, --model <modelName>', 'Specify model name to use');
 
-// Define your target tokens here or pass it as a command line argument
-const targetTokens = process.argv[3] || 4097;
+program.parse(process.argv);
+
+const options = program.opts();
+const [globPattern, targetTokensInput] = program.args;
+const targetTokens = Number(targetTokensInput);
+
+if (options.privacy) {
+  try {
+    modelService.setGlobalOverride('modelName', 'privacy');
+  } catch (err) {
+    console.error(`Privacy model error: ${err.message}`);
+  }
+}
+if (options.model) {
+  try {
+    modelService.setGlobalOverride('modelName', options.model);
+  } catch (err) {
+    console.error(`Model override error: ${err.message}`);
+  }
+}
 
 // Initialize the SummaryMap with the target tokens
 const map = new SummaryMap({

package/src/chains/README.md

@@ -0,0 +1,30 @@
+# Chains
+
+Chains orchestrate multiple verblets or helper functions to perform more complex tasks. Each subdirectory exposes a specific workflow that can be imported as a single function.
+
+Available chains:
+
+- [anonymize](./anonymize)
+- [bulk-map](./bulk-map)
+- [bulk-reduce](./bulk-reduce)
+- [bulk-filter](./bulk-filter)
+- [bulk-group](./bulk-group)
+- [dismantle](./dismantle)
+- [disambiguate](./disambiguate)
+- [intersections](./intersections)
+- [list](./list)
+- [questions](./questions)
+- [socratic](./socratic)
+- [glossary](./glossary)
+- [scan-js](./scan-js)
+- [sort](./sort)
+- [summary-map](./summary-map)
+- [themes](./themes)
+- [set-interval](./set-interval)
+- [test](./test)
+- [test-advice](./test-advice)
+- [veiled-variants](./veiled-variants)
+- [collect-terms](./collect-terms) - gather complex vocabulary
+
+Chains are free to use any utilities from [`../lib`](../lib/README.md) and often rely on one or more verblets from [`../verblets`](../verblets/README.md).
+

package/src/chains/anonymize/README.md

@@ -0,0 +1,21 @@
+# anonymize
+
+Remove personal style, references, and formatting from text to conceal the original author. The chain runs through your configured LLM models, so it works with fully private or self‑hosted LLMs.
+
+Supported methods: `STRICT`, `BALANCED`, and `LIGHT` to control how aggressively style is removed.
+
+
+```javascript
+import anonymize, { anonymizeMethod } from './index.js';
+
+const message = `As a software lead in Chicago, I've found our new UI framework helps junior devs ramp up fast.`;
+
+const { text } = await anonymize({
+  text: message,
+  method: anonymizeMethod.STRICT,
+});
+
+console.log(text);
+// => "The new UI framework shortens the learning curve for new developers."
+```
+

package/src/chains/anonymize/index.examples.js

@@ -0,0 +1,75 @@
+import { describe, it } from 'vitest';
+import { expect } from 'chai';
+import { anonymize, anonymizeMethod } from './index.js';
+
+const sampleText = `As a seasoned engineer from Silicon Valley, I've found that React's 
+component lifecycle is like a well-oiled machine - understanding the mounting 
+phase is crucial, especially with those pesky useEffect hooks. Trust me, after 
+10 years of experience, proper cleanup is key to avoiding memory leaks!`;
+
+describe('anonymize examples', () => {
+  it.only('should anonymize text using strict method', { timeout: 60_000 }, async () => {
+    const input = {
+      text: sampleText,
+      method: anonymizeMethod.STRICT,
+    };
+
+    const result = await anonymize(input);
+
+    expect(result).to.have.property('text');
+    expect(result).to.have.property('stages');
+    expect(result.stages).to.have.property('distinctiveContentRemoved');
+    expect(result.stages).to.have.property('structureNormalized');
+    expect(result.stages).to.have.property('patternsSuppressed');
+
+    // Verify anonymization removed personal markers
+    expect(result.text).to.not.include('Silicon Valley');
+    expect(result.text).to.not.include('10 years of experience');
+    expect(result.text).to.not.include('Trust me');
+
+    // Verify metaphors and idioms are removed
+    expect(result.text).to.not.include('well-oiled machine');
+    expect(result.text).to.not.include('pesky');
+
+    // Verify the text has been transformed
+    expect(result.text).to.not.equal(sampleText);
+    expect(result.text.length).to.be.lessThan(sampleText.length);
+  });
+
+  it('should preserve more content with balanced method', { timeout: 60_000 }, async () => {
+    const input = {
+      text: sampleText,
+      method: anonymizeMethod.BALANCED,
+    };
+
+    const result = await anonymize(input);
+
+    // Verify some personal markers are still removed
+    expect(result.text).to.not.include('Silicon Valley');
+    expect(result.text).to.not.include('Trust me');
+
+    // But technical content is more preserved
+    expect(result.text.length).to.be.greaterThan(
+      (await anonymize({ text: sampleText, method: anonymizeMethod.STRICT })).text.length
+    );
+  });
+
+  it('should minimally transform text with light method', { timeout: 60_000 }, async () => {
+    const input = {
+      text: sampleText,
+      method: anonymizeMethod.LIGHT,
+    };
+
+    const result = await anonymize(input);
+
+    // Verify minimal transformation
+    expect(result.text.length).to.be.greaterThan(
+      (await anonymize({ text: sampleText, method: anonymizeMethod.BALANCED })).text.length
+    );
+
+    // Only the most obvious personal markers should be removed
+    expect(result.text).to.not.include('Trust me');
+
+    expect(result.text).to.not.include("I've found");
+  });
+});

package/src/chains/anonymize/index.js

@@ -0,0 +1,121 @@
+import { run } from '../../lib/chatgpt/index.js';
+
+export const anonymizeMethod = {
+  STRICT: 'strict',
+  BALANCED: 'balanced',
+  LIGHT: 'light',
+};
+
+const METHODS = Object.values(anonymizeMethod);
+
+const validateInput = (input) => {
+  if (!input || typeof input !== 'object') {
+    throw new Error('Input must be an object');
+  }
+
+  const { text, method, context } = input;
+
+  if (!text || typeof text !== 'string') {
+    throw new Error('Input must include a text string');
+  }
+
+  if (!method || !METHODS.includes(method)) {
+    throw new Error(`Method must be one of: ${METHODS.join(', ')}`);
+  }
+
+  if (context !== undefined && typeof context !== 'string') {
+    throw new Error('Context must be a string if provided');
+  }
+
+  return { text, method, context };
+};
+
+const stage1Prompt = (text, context) => `
+Remove Distinctive Content and Markers
+ - Identify and replace every distinctive or uncommon word, phrase, or sentence structure with the most widely used, nondescript alternative.
+ - Remove all idioms, metaphors, analogies, cultural references, personal perspectives, and subjective tones.
+ - Eliminate any explicit or implicit references to the author's identity, background, education, expertise, region, or intent.
+
+${context ? `Context: ${context}\n` : ''}
+Text to process:
+${text}
+
+Return ONLY the processed text, with no explanations or additional content.`;
+
+const stage2Prompt = (text, context) => `
+Normalize Structure, Formatting, and Tone
+ - Restructure sentences and paragraphs to strictly follow standard, average patterns in length, order, and construction. Avoid any distinctive rhythm, complexity, or flow.
+ - Uniformly normalize punctuation, formatting, and paragraphing; avoid any variation or emphasis that could signal style.
+ - Strip out all emotional, evaluative, or expressive language, enforcing a neutral, impersonal, and objective tone.
+
+ ${context ? `Context: ${context}\n` : ''}
+ Text to process:
+${text}
+
+Return ONLY the normalized text, with no explanations or additional content.`;
+
+const stage3Prompt = (text, context) => `
+Stage 3: Suppress Latent Stylistic Patterns
+ - Review for and suppress any recurring linguistic patterns, syntactic habits, or structural quirks—even if they appear common.
+ - For all possible ways to phrase content, always select the plainest, most generic, and least distinctive form.
+ - Ensure the final text reads as if generated by an automated system, with no evidence of personality, emotion, region, or any unique authorial traits.
+
+ ${context ? `Context: ${context}\n` : ''}
+Text to process:
+${text}
+
+Return ONLY the final anonymized text, with no explanations or additional content.`;
+
+const anonymize = async (input, config = {}) => {
+  const { text, method, context } = validateInput(input);
+  const { llm, ...options } = config;
+
+  // Stage 1: Remove distinctive content
+  const stage1Result = await run(stage1Prompt(text, method, context), {
+    modelOptions: { modelName: 'privacy', ...llm },
+    ...options,
+  });
+
+  if (method === anonymizeMethod.LIGHT) {
+    return {
+      text: stage1Result,
+      stages: {
+        distinctiveContentRemoved: stage1Result,
+      },
+    };
+  }
+
+  // Stage 2: Normalize structure and tone
+  const stage2Result = await run(stage2Prompt(stage1Result, method), {
+    modelOptions: { modelName: 'privacy', ...llm },
+    ...options,
+  });
+
+  if (method === anonymizeMethod.BALANCED) {
+    return {
+      text: stage2Result,
+      stages: {
+        distinctiveContentRemoved: stage1Result,
+        structureNormalized: stage2Result,
+      },
+    };
+  }
+
+  // Stage 3: Suppress stylistic patterns
+  const stage3Result = await run(stage3Prompt(stage2Result, method), {
+    modelOptions: { modelName: 'privacy', ...llm },
+    ...options,
+  });
+
+  return {
+    text: stage3Result,
+    stages: {
+      distinctiveContentRemoved: stage1Result,
+      structureNormalized: stage2Result,
+      patternsSuppressed: stage3Result,
+    },
+  };
+};
+
+export { anonymize };
+export default anonymize;