@far-world-labs/verblets 0.1.1

package/src/chains/questions/index.examples.js

@@ -0,0 +1,68 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { describe, expect, it } from 'vitest';
+
+import { longTestTimeout } from '../../constants/common.js';
+import questions from './index.js';
+
+const ensureDirectoryExists = async (directoryPath) => {
+  try {
+    await fs.access(directoryPath);
+  } catch (error) {
+    await fs.mkdir(directoryPath, { recursive: true });
+  }
+};
+
+const readFileOrUndefined = async (filePath) => {
+  let result;
+  try {
+    result = (await fs.readFile(filePath)).toString();
+    // eslint-disable-next-line no-empty
+  } catch (error) {
+    // do nothing
+  }
+  return result;
+};
+
+const cacheDir = path.join(process.env.HOME, '.cache', 'puck');
+const cacheFile = `${cacheDir}/questions-verblet-test-cache-1.json`;
+
+const examples = [
+  {
+    inputs: {
+      text: 'Writing a prompt toolkit for ChatGPT',
+      searchBreadth: 0.5,
+    },
+    want: { minLength: 10 },
+  },
+];
+
+describe('Questions verblet', () => {
+  examples.forEach((example) => {
+    it(
+      example.inputs.text,
+      async () => {
+        const canUseCache = process.env.RUN_TESTS_WITH_RANDOMNESS_ONCE;
+
+        const cache = await readFileOrUndefined(cacheFile);
+
+        let result;
+        if (canUseCache && cache) {
+          result = JSON.parse(cache);
+        } else {
+          result = await questions(example.inputs.text);
+        }
+
+        if (canUseCache) {
+          await ensureDirectoryExists(cacheDir);
+          await fs.writeFile(cacheFile, JSON.stringify(result));
+        }
+
+        if (example.want.minLength) {
+          expect(result.length).gt(example.want.minLength);
+        }
+      },
+      longTestTimeout
+    );
+  });
+});

package/src/chains/questions/index.js

@@ -0,0 +1,136 @@
+/* eslint-disable no-await-in-loop */
+
+import * as R from 'ramda';
+
+import chatGPT from '../../lib/chatgpt/index.js';
+import {
+  constants as promptConstants,
+  generateQuestions as generateQuestionsPrompt,
+} from '../../prompts/index.js';
+import modelService from '../../services/llm-model/index.js';
+import toObject from '../../verblets/to-object/index.js';
+
+const { asSplitIntoJSONArray, contentIsChoices, onlyJSON } = promptConstants;
+
+// Returns a random subset of a list with length between 1 and the length of the list
+// based on an input value between 0 and 1
+const getRandomSubset = (list, value) => {
+  const numItems = Math.round(value * list.length) + 1;
+  const listShuffled = R.sort(() => Math.random() - 0.5, list);
+  return listShuffled.slice(0, numItems);
+};
+
+const pickInterestingQuestion = (originalQuestion, { existing = [] }) => {
+  const existingJoined = existing.map((item) => ` - ${item}`).join('\n');
+
+  return `Choose one interesting question from the following list of questions. The main goal is to determine "${originalQuestion}".
+
+${contentIsChoices}
+\`\`\`
+${existingJoined}
+\`\`\`
+`;
+};
+
+const shouldSkipNull = (result, resultsAll) => {
+  return resultsAll.includes(result);
+};
+
+const shouldStopNull = (result, resultsAll, resultsNew, attempts = 0) => {
+  return resultsAll.length > 50 || attempts > 5;
+};
+
+const generateQuestions = async function* generateQuestionsGenerator(
+  text,
+  options = {}
+) {
+  const resultsAll = [];
+  const resultsAllMap = {};
+  const drilldownResults = [];
+  let isDone = false;
+  let textSelected = text;
+
+  const {
+    searchBreadth = 0.5,
+    shouldSkip = shouldSkipNull,
+    shouldStop = shouldStopNull,
+    model = modelService.getBestAvailableModel(),
+  } = options;
+
+  let attempts = 0;
+  while (!isDone) {
+    if (resultsAll.length) {
+      const choices = resultsAll.filter((item) => {
+        return !drilldownResults.includes(item);
+      });
+      const pickInterestingQuestionPrompt = pickInterestingQuestion(
+        textSelected,
+        { existing: choices }
+      );
+      textSelected = await chatGPT(pickInterestingQuestionPrompt);
+      drilldownResults.push(textSelected);
+    }
+
+    const promptCreated = generateQuestionsPrompt(textSelected, {
+      existing: resultsAll,
+    });
+    const budget = model.budgetTokens(promptCreated);
+    const chatGPTConfig = {
+      modelOptions: {
+        maxTokens: budget.completion,
+        temperature: 1,
+      },
+    };
+
+    const results = await chatGPT(`${promptCreated}`, chatGPTConfig);
+    let resultsParsed;
+    try {
+      resultsParsed = await toObject(results);
+    } catch (error) {
+      if (/Unexpected string in JSON/.test(error.message)) {
+        const resultsUpdated = await chatGPT(
+          `${asSplitIntoJSONArray}${onlyJSON} \`\`\`${results}\`\`\``,
+          chatGPTConfig
+        );
+        resultsParsed = await toObject(resultsUpdated);
+      }
+    }
+    const resultsNew = getRandomSubset(resultsParsed, searchBreadth);
+    if (searchBreadth < 0.5) {
+      const randomIndex = Math.floor(Math.random() * resultsNew.length);
+      textSelected = resultsNew[randomIndex];
+    }
+    const resultsNewUnique = resultsNew.filter(
+      (item) => !(item in resultsAllMap)
+    );
+
+    attempts += 1;
+
+    for (const result of resultsNewUnique) {
+      if (await shouldStop(result, resultsAll, resultsNew, attempts)) {
+        isDone = true;
+        break;
+      }
+      if (!(await shouldSkip(result, resultsAll))) {
+        resultsAllMap[result] = true;
+        resultsAll.push(result);
+        yield result;
+      }
+    }
+  }
+};
+
+export default async (text, options) => {
+  const generator = generateQuestions(text, options);
+
+  const results = [];
+  for await (const result of generator) {
+    if (!results.includes(result)) {
+      results.push(result);
+    }
+  }
+
+  const resultsSorted = R.sort((a, b) => a.localeCompare(b), results);
+
+  return resultsSorted;
+};

package/src/chains/questions/index.spec.js

@@ -0,0 +1,29 @@
+// Testing TBD. Randomness makes it challenging.
+
+import { describe } from 'vitest';
+
+/*
+import questions from "./index.js";
+
+vi.mock("../../lib/chatgpt/index.js", () => ({
+  default: vi.fn().mockImplementation((text) => {
+    if (/a prompt toolkit/.test(text)) {
+      return "{}";
+    }
+    return "undefined";
+  }),
+}));
+
+const examples = [
+  {
+    name: "Basic usage",
+    inputs: {
+      text: "Writing a prompt toolkit for ChatGPT",
+      searchBreadth: 0.5,
+    },
+    want: {},
+  },
+];
+
+*/
+describe.skip('Questions verblet', () => {});

package/src/chains/scan-js/index.js

@@ -0,0 +1,119 @@
+import fs from 'node:fs/promises';
+import * as R from 'ramda';
+
+import sort from '../sort/index.js';
+import chatGPT from '../../lib/chatgpt/index.js';
+import pathAliases from '../../lib/path-aliases/index.js';
+import retry from '../../lib/retry/index.js';
+import search from '../../lib/search-js-files/index.js';
+import stripResponse from '../../lib/strip-response/index.js';
+import codeFeaturesPrompt from '../../prompts/code-features.js';
+import makeJSONSchema from '../../prompts/features-json-schema.js';
+import modelService from '../../services/llm-model/index.js';
+import toObject from '../../verblets/to-object/index.js';
+
+const codeFeatureDefinitions = JSON.parse(
+  await fs.readFile(
+    new URL(
+      '../../lib/search-js-files/code-features-property-definitions.json',
+      import.meta.url
+    ),
+    'utf-8'
+  )
+);
+
+const organizeResult = (result) => {
+  return R.sort(
+    (a, b) => a.localeCompare(b),
+    Object.entries(result).map(([key, value]) => `${key}: ${value}`)
+  );
+};
+
+const visit = async ({
+  node,
+  state: stateInitial,
+  features: featuresInitial = 'maintainability',
+}) => {
+  if (!node.functionName) {
+    // skipping
+    return stateInitial;
+  }
+
+  const sortResults = await sort(
+    {
+      by: `best criteria for looking at "${featuresInitial}" within code`,
+      chunkSize: 4,
+      extremeK: 4,
+    },
+    codeFeatureDefinitions.map((d) => d.criteria),
+    modelService.getModel('textDavinci003')
+  );
+  const sortCriteria = sortResults.slice(0, 5);
+  const features = codeFeatureDefinitions.filter((def) => {
+    return sortCriteria.includes(def.criteria);
+  });
+
+  if (!features?.length) {
+    throw new Error('Visit [error]: Features list not defined');
+  }
+  const schema = makeJSONSchema(features);
+
+  const fileText = await fs.readFile(node.filename, 'utf-8');
+  const functionText = fileText.slice(node.start, node.end);
+
+  const state = { ...stateInitial };
+
+  const visitPrompt = codeFeaturesPrompt({
+    text: functionText,
+    schema,
+  });
+
+  await retry(async () => {
+    const results = await chatGPT(visitPrompt, {
+      modelOptions: {
+        modelName: 'gpt35Turbo',
+      },
+    });
+
+    const resultParsed = await toObject(stripResponse(results), schema);
+
+    const id = `${node.filename}:::${node.functionName}`;
+
+    state[id] = resultParsed;
+    state.nodesFound = (state.nodesFound ?? 0) + 1;
+    state.abbreviations = state.abbreviations ?? {};
+    state.abbreviations[id] = state.abbreviations[id] ?? state.nodesFound;
+
+    const idDisplay = (state.pathAliases[id] ?? id).slice(-50).padStart(50);
+
+    console.error(
+      `${`${state.nodesFound}`.padEnd(3, ' ')} ${idDisplay}: ${organizeResult(
+        resultParsed
+      ).join(', ')}`
+    );
+  });
+
+  return state;
+};
+
+// node: { filename: './src/index.js' },
+export default async (moduleOptions) => {
+  const state = await search({
+    ...moduleOptions,
+  });
+
+  const preState = {
+    visited: new Set(),
+    pathAliases: pathAliases([...state.visited]),
+  };
+
+  return search({
+    ...moduleOptions,
+    state: preState,
+    visit: (options) =>
+      visit({
+        ...options,
+        features: moduleOptions.features,
+      }),
+  });
+};

package/src/chains/sort/index.examples.js

@@ -0,0 +1,40 @@
+import { describe, expect, it } from 'vitest';
+
+import { longTestTimeout } from '../../constants/common.js';
+import list from '../list/index.js';
+import sort from './index.js';
+
+const examples = [
+  {
+    name: '"The Office" episodes',
+    inputs: {
+      listText: '"The Office" most famous episodes',
+      sortText: 'have scenes that became memes',
+    },
+    want: { result: true },
+  },
+];
+
+describe('Sort chain', () => {
+  examples.forEach((example) => {
+    it(
+      example.name,
+      async () => {
+        const listResults = await list(example.inputs.listText);
+
+        const result = await sort(
+          {
+            by: example.inputs.sortText,
+            iterations: 1,
+          },
+          listResults
+        );
+
+        // console.error(result);
+
+        expect(result).toStrictEqual(result);
+      },
+      longTestTimeout
+    );
+  });
+});

package/src/chains/sort/index.js

@@ -0,0 +1,113 @@
+import * as R from 'ramda';
+
+import chatGPT from '../../lib/chatgpt/index.js';
+import toObject from '../../verblets/to-object/index.js';
+import { sort as sortPromptInitial } from '../../prompts/index.js';
+import modelService from '../../services/llm-model/index.js';
+
+// redeclared so it's clearer how tests can override the sorter
+let sortPrompt = sortPromptInitial;
+
+export const defaultSortChunkSize = 10;
+export const defaultSortExtremeK = 10;
+export const defaultSortIterations = 1;
+
+// Keeping this here because it's useful for internal debugging
+// eslint-disable-next-line no-unused-vars
+const assertSorted = (list) => {
+  const pairwiseComparisons = R.aperture(2, list);
+  if (!R.all(([a, b]) => b.localeCompare(a) <= 0, pairwiseComparisons)) {
+    throw new Error(JSON.stringify(list, null, 2));
+  }
+};
+
+export const useTestSortPrompt = () => {
+  sortPrompt = (options, list) => ({ options, list });
+};
+
+const sanitizeList = (list) => {
+  return [...new Set(list.filter((item) => item.trim() !== ''))];
+};
+
+const sort = async (
+  options,
+  listInitial,
+  model = modelService.getBestAvailableModel()
+) => {
+  const {
+    by,
+    chunkSize = defaultSortChunkSize,
+    extremeK = defaultSortExtremeK,
+    iterations = defaultSortIterations,
+  } = options;
+
+  const list = sanitizeList(listInitial);
+  let i = iterations;
+  let top = [];
+  let bottom = [];
+  let middle = list;
+
+  while (i > 0) {
+    let newTop = [];
+    let newBottom = [];
+    let discardedTop = [];
+    let discardedBottom = [];
+
+    for (let j = 0; j < middle.length; j += chunkSize) {
+      const batch = middle.slice(j, j + chunkSize);
+      const prompt = sortPrompt(
+        {
+          description: by,
+        },
+        [...batch, ...newTop, ...newBottom]
+      );
+
+      const budget = model.budgetTokens(prompt);
+
+      // eslint-disable-next-line no-await-in-loop
+      const result = await chatGPT(prompt, {
+        modelOptions: {
+          maxTokens: budget.completion,
+          requestTimeout: model.requestTimeout * 1.5,
+        },
+      });
+
+      // eslint-disable-next-line no-await-in-loop
+      const batchSorted = await toObject(result);
+
+      const batchTop = batchSorted.slice(0, extremeK);
+      const batchBottom = batchSorted.slice(-extremeK);
+
+      discardedTop = [
+        ...newTop.filter((x) => !batchTop.includes(x)),
+        ...discardedTop,
+      ];
+
+      discardedBottom = [
+        ...discardedBottom,
+        ...newBottom.filter((x) => !batchBottom.includes(x)),
+      ];
+
+      newTop = batchTop;
+      newBottom = batchBottom;
+    }
+    top = [...top, ...newTop];
+    bottom = [...newBottom, ...bottom];
+
+    const middleOld = middle.filter((x) => {
+      return (
+        !newTop.includes(x) &&
+        !discardedTop.includes(x) &&
+        !discardedBottom.includes(x) &&
+        !newBottom.includes(x)
+      );
+    });
+    middle = [...discardedTop, ...middleOld, ...discardedBottom];
+    i -= 1;
+  }
+
+  const finalList = [...top, ...middle, ...bottom];
+  return finalList;
+};
+
+export default sort;

package/src/chains/sort/index.spec.js

@@ -0,0 +1,115 @@
+import { describe, expect, it, vi } from 'vitest';
+import * as R from 'ramda';
+
+import sort, { useTestSortPrompt } from './index.js';
+
+useTestSortPrompt();
+
+const extremeK = 4;
+const chunkSize = 12;
+const byAB = (a, b) => b.localeCompare(a);
+
+const unsortedStrings = [
+  'zebra',
+  'apple',
+  'quail',
+  'mango',
+  'giraffe',
+  'banana',
+  'dog',
+  'lion',
+  'tiger',
+  'elephant',
+  'kiwi',
+  'raspberry',
+  'grape',
+  'apricot',
+  'kangaroo',
+  'owl',
+  'peacock',
+  'xenon',
+  'uranium',
+  'platinum',
+  'walrus',
+  'fox',
+  'capybara',
+  'iguana',
+  'jaguar',
+  'koi',
+  'lobster',
+  'moose',
+  'nugget',
+  'octopus',
+  'python',
+  'quokka',
+  'raccoon',
+  'starfish',
+  'tortoise',
+  'umbrella',
+  'vulture',
+  'wombat',
+  'xerus',
+  'yak',
+  'zeppelin',
+  'ant',
+  'beaver',
+  'cat',
+  'dolphin',
+  'echidna',
+  'frog',
+  'hamster',
+  'impala',
+  'jellyfish',
+];
+
+vi.mock('../../lib/chatgpt/index.js', () => ({
+  default: vi.fn().mockImplementation((text) => {
+    if (text.options.description === 'alphabetically') {
+      const sorted = R.sort((a, b) => b.localeCompare(a), text.list);
+      return JSON.stringify(sorted);
+    }
+    return '[]';
+  }),
+}));
+
+const examples = [
+  {
+    name: 'Basic usage',
+    inputs: {
+      options: { by: 'alphabetically', iterations: 3, extremeK, chunkSize },
+      list: [...unsortedStrings],
+    },
+    want: {
+      highest: R.sort(byAB, [...unsortedStrings]).slice(0, extremeK * 3),
+      lowest: R.sort(byAB, [...unsortedStrings]).slice(-(extremeK * 3)),
+    },
+  },
+  {
+    name: 'Empty list',
+    inputs: {
+      options: { by: 'alphabetically', extremeK, chunkSize },
+      list: [],
+    },
+    want: {
+      highest: [],
+      lowest: [],
+    },
+  },
+];
+
+describe('Sort', () => {
+  examples.forEach((example) => {
+    it(example.name, async () => {
+      const iterations = example.inputs.options.iterations ?? 1;
+      const result = await sort(example.inputs.options, example.inputs.list, {
+        budgetTokens: () => ({ completion: 0 }),
+      });
+      expect(result.slice(0, extremeK * iterations)).toStrictEqual(
+        example.want.highest
+      );
+      expect(result.slice(-(extremeK * iterations))).toStrictEqual(
+        example.want.lowest
+      );
+    });
+  });
+});

package/src/chains/summary-map/README.md

@@ -0,0 +1,33 @@
+# SummaryMap
+
+SummaryMap is a utility class designed to help manage a collection of data elements for AI prompts. It allows you to compress incoming variables based on weights relative to an overall target budget for the variables.
+
+## Usage
+
+```javascript
+import SummaryMap from '../lib/summary-map/index.js';
+import chatGPT from '../../lib/chatgpt/index.js';
+import pave from '../../lib/pave/index.js';
+import modelService from '../../services/llm-model/index.js';
+
+const promptFunction = (data) => {
+  return `Please solve a problem for me with the following input data:
+  ${data.example.text}
+
+Reference the following code:
+${data.example.code}`;
+};
+
+const variableTokens = 100;
+const promptTokens = modelService.getBestAvailableModel().toTokens(promptFunction).length;
+const solutionTokens = 200;
+const maxTokens = promptTokens + variableTokens + solutionTokens
+
+const summaryMap = new SummaryMap({ targetTokens: variableTokens });
+summaryMap.set('example.text', { value: 'Long text data...', weight: 1, type: 'text' });
+summaryMap.set('example.code', { value: 'Long code data...', weight: 0.5, type: 'code' });
+
+const promptInputs = await summaryMap.pavedSummaryResult();
+const prompt = promptFunction(promptInputs);
+const response = await chatGPT(prompt, { modelOptions: { maxTokens }});
+```