@far-world-labs/verblets 0.1.1

package/src/lib/search-js-files/scan-file.js

@@ -0,0 +1,253 @@
+/* eslint-disable no-await-in-loop */
+
+import path from 'path';
+import fs from 'fs';
+import { parse } from 'acorn';
+import * as walk from 'acorn-walk';
+import dependencyTree from 'dependency-tree';
+
+const stripRootDir = (pathStr, root = process.cwd()) => {
+  return pathStr.replace(new RegExp(`^${root}`), '');
+};
+
+const convertImport = (filePath, importPath) => {
+  return stripRootDir(path.resolve(path.dirname(filePath), importPath));
+};
+
+const flatten = (obj) => {
+  let result = Object.keys(obj).reduce(
+    (memo, key) => ({
+      ...memo,
+      [key]: 1,
+    }),
+    {}
+  );
+
+  result = Object.keys(obj).reduce((acc, key) => {
+    if (typeof obj[key] === 'object') {
+      const flattenedValue = flatten(obj[key]);
+      return Object.entries(flattenedValue).reduce(
+        (innerAcc, [innerKey, innerVal]) => {
+          return {
+            ...innerAcc,
+            [innerKey]: (innerAcc[innerKey] ?? 0) + innerVal,
+          };
+        },
+        acc
+      );
+    }
+    return acc;
+  }, result);
+
+  return result;
+};
+
+const listFiles = (file, cwd) => {
+  const config = {
+    filename: file,
+    directory: cwd,
+    nodeModulesConfig: { entry: 'module' },
+    nonExistent: [],
+    filter: (p) => p.indexOf('node_modules') === -1,
+  };
+
+  return dependencyTree(config);
+};
+
+const parseFiles = (files) => {
+  const filesMap = {};
+  for (const file of files) {
+    const functionsMap = {};
+    const functionsSeen = {};
+    const variablesMap = {};
+    const importsMap = {};
+
+    const code = fs.readFileSync(file, 'utf-8');
+
+    const comments = [];
+
+    const ast = parse(code, {
+      sourceType: 'module',
+      ecmaVersion: 'latest',
+      onComment: comments,
+    });
+
+    const commentsMap = comments.reduce((acc, comment) => {
+      return { ...acc, [comment.start]: comment };
+    }, {});
+
+    filesMap[stripRootDir(file)] = {
+      functionsMap,
+      variablesMap,
+      importsMap,
+      commentsMap,
+    };
+
+    walk.simple(ast, {
+      ImportDeclaration(node) {
+        const declaration = node.specifiers
+          .filter((s) => s.type === 'ImportDefaultSpecifier')
+          .map((s) => {
+            return s.local.name;
+          })?.[0];
+        const specifiers = node.specifiers
+          .filter((s) => s.type === 'ImportSpecifier')
+          .map((s) => {
+            return { ...s.imported }.name; // also has local name
+          });
+        const source = node.source.value;
+        const importKey = source.startsWith('.')
+          ? convertImport(file, source)
+          : source;
+        importsMap[importKey] = {
+          start: node.start,
+          end: node.end,
+          declaration,
+          specifiers,
+          source,
+        };
+      },
+
+      FunctionDeclaration(node) {
+        functionsMap[`${node.type}:${node.id.name}`] = {
+          start: node.start,
+          end: node.end,
+          name: node.id.name,
+          type: node.type,
+          async: node.async,
+          generator: node.generator,
+        };
+      },
+
+      VariableDeclaration(node) {
+        const arrowDeclarations = node.declarations.filter((d) => {
+          return (
+            d.id.type === 'Identifier' &&
+            d.init?.type === 'ArrowFunctionExpression' &&
+            d.init?.body.type === 'BlockStatement'
+          );
+        });
+
+        arrowDeclarations.forEach((d) => {
+          functionsSeen[`${d.init.start}:${d.init.end}`] = true;
+
+          functionsMap[`ArrowFunctionExpression:${d.id.name}`] = {
+            start: d.start,
+            end: d.end,
+            name: d.id.name,
+            async: d.init.async,
+            generator: d.init.generator,
+            type: 'ArrowFunctionExpression',
+          };
+        });
+
+        const fnExpDeclarations = node.declarations.filter((d) => {
+          return (
+            d.id.type === 'Identifier' &&
+            d.init?.type === 'FunctionExpression' &&
+            d.init?.body.type === 'BlockStatement'
+          );
+        });
+
+        fnExpDeclarations.forEach((d) => {
+          functionsSeen[`${d.init.start}:${d.init.end}`] = true;
+
+          functionsMap[`FunctionExpression:${d.id.name}`] = {
+            start: d.start,
+            end: d.end,
+            name: d.id.name,
+            async: d.init.async,
+            generator: d.init.generator,
+            type: 'FunctionExpression',
+          };
+        });
+      },
+
+      // Property Assignment
+      AssignmentExpression(node) {
+        if (
+          (node.right.type === 'FunctionExpression' ||
+            node.right.type === 'ArrowFunctionExpression') &&
+          node.value
+        ) {
+          functionsSeen[`${node.right.start}:${node.right.end}`] = true;
+
+          functionsMap[`${node.value.type}:${node.left.property.name}`] = {
+            start: node.start,
+            end: node.end,
+            name: node.left.property.name,
+            async: node.right.async,
+            generator: node.right.generator,
+            type: node.right.type,
+          };
+        }
+      },
+
+      // Object Definition
+      Property(node) {
+        if (
+          node.value.type === 'FunctionExpression' ||
+          node.value.type === 'ArrowFunctionExpression'
+        ) {
+          functionsSeen[`${node.value.start}:${node.value.end}`] = true;
+
+          functionsMap[`Property:${node.key.name}`] = {
+            start: node.start,
+            end: node.end,
+            name: node.key.name,
+            async: node.value.async,
+            generator: node.value.generator,
+            type: node.value.type,
+          };
+        }
+      },
+
+      // Class Definition
+      ClassDeclaration(node) {
+        const className = node.id.name;
+        node.body.body.forEach((classElement) => {
+          if (classElement.type === 'MethodDefinition') {
+            functionsSeen[
+              `${classElement.value.start}:${classElement.value.end}`
+            ] = true;
+
+            functionsMap[
+              `MethodDefinition:${className}.${classElement.key.name}`
+            ] = {
+              start: node.start,
+              end: node.end,
+              className: className,
+              name: classElement.key.name,
+              async: classElement.value.async,
+              generator: classElement.value.generator,
+              type: 'MethodDefinition',
+            };
+          }
+        });
+      },
+    });
+
+    walk.simple(ast, {
+      FunctionExpression(node) {
+        if (!functionsSeen[`${node.start}:${node.end}`]) {
+          functionsMap[`${node.type}:${node.start}`] = {
+            start: node.start,
+            end: node.end,
+            type: node.type,
+            async: node.async,
+            generator: node.generator,
+          };
+        }
+      },
+    });
+  }
+
+  return filesMap;
+};
+
+export default (entryFile) => {
+  const filesTree = listFiles(entryFile, process.cwd());
+  const filesList = Object.keys(flatten(filesTree));
+  const results = parseFiles(filesList);
+  return results;
+};

package/src/lib/shorten-text/index.js

@@ -0,0 +1,30 @@
+import modelService from '../../services/llm-model/index.js';
+
+export default (
+  text,
+  {
+    minCharsToRemove = 10,
+    model = modelService.getBestAvailableModel(),
+    targetTokenCount,
+  }
+) => {
+  const ellipsis = '...';
+  const textToTokenRatio = text.length / model.toTokens(text).length;
+  let trimmedText = text;
+  let tokenCount = model.toTokens(trimmedText).length;
+
+  while (tokenCount > targetTokenCount) {
+    const charsToRemove = Math.max(
+      Math.ceil((tokenCount - targetTokenCount) * textToTokenRatio),
+      minCharsToRemove
+    );
+    const middleIndex = Math.floor(trimmedText.length / 2);
+    const startIndex = middleIndex - Math.ceil(charsToRemove / 2);
+    const endIndex = middleIndex + Math.floor(charsToRemove / 2);
+    trimmedText =
+      trimmedText.slice(0, startIndex) + ellipsis + trimmedText.slice(endIndex);
+    tokenCount = model.toTokens(trimmedText).length;
+  }
+
+  return trimmedText;
+};

package/src/lib/shorten-text/index.spec.js

@@ -0,0 +1,68 @@
+import { describe, expect, it } from 'vitest';
+import modelService from '../../services/llm-model/index.js';
+
+import shortenText from './index.js';
+
+const examples = [
+  {
+    name: 'Basic usage',
+    inputs: {
+      text: 'Hello, world! This is a long text for testing the shortenText function.',
+      targetTokenCount: 10,
+    },
+    want: {
+      start: /^Hello, world!/,
+      end: /Text function\.$/,
+      maxLength: 40,
+    },
+  },
+  {
+    name: 'No trimming needed',
+    inputs: {
+      text: 'This text is short enough.',
+      targetTokenCount: 8,
+    },
+    want: {
+      result: 'This text is short enough.',
+    },
+  },
+  {
+    name: 'Minimum characters removal',
+    inputs: {
+      text: 'This is another test to check the minimum characters removal feature.',
+      targetTokenCount: 6,
+      minCharsToRemove: 5,
+    },
+    want: {
+      start: /^This is/,
+      end: /feature\.$/,
+      maxLength: 25,
+    },
+  },
+];
+
+describe('Shorten text', () => {
+  examples.forEach((example) => {
+    it(example.name, () => {
+      const got = shortenText(example.inputs.text, {
+        targetTokenCount: example.inputs.targetTokenCount,
+        minCharsToRemove: example.inputs.minCharsToRemove,
+      });
+
+      if (example.want.result) {
+        expect(got).toEqual(example.want.result);
+      }
+      if (example.want.start) {
+        expect(example.want.start.test(got)).toBe(true);
+      }
+      if (example.want.start) {
+        expect(example.want.end.test(got)).toBe(true);
+      }
+      if (example.want.maxLength) {
+        expect(
+          modelService.getBestAvailableModel().toTokens(got).length
+        ).toBeLessThanOrEqual(example.want.maxLength);
+      }
+    });
+  });
+});

package/src/lib/strip-numeric/index.js

@@ -0,0 +1,5 @@
+export default (val) => {
+  const noAnswer = val.replace(/[aA]nswer:?/, '').trim();
+  const onlyNumberParts = noAnswer.replace(/[^0-9.]/g, '').trim();
+  return onlyNumberParts;
+};

package/src/lib/strip-response/index.js

@@ -0,0 +1,35 @@
+/*
+ * This regex pattern looks for combinations of:
+ *
+ *   { followed by optional whitespace and ",
+ *   [ followed by optional whitespace, {, ", a digit [0-9], or [.
+ */
+const jsonStartRegex = /(?:\s*[{[]|[{[]+\s*[{[])/;
+
+export default (val) => {
+  const [questionPart = '', , answerPart = ''] = val.split(/(=){11,29}/);
+
+  const answerPartTrimmed = answerPart.trim() ?? '';
+
+  const answerSection = answerPartTrimmed.length
+    ? answerPart.trim()
+    : questionPart.trim();
+
+  const answerNoPrefix = answerSection.replace(/[aA]nswer:?/, '').trim();
+
+  const answerNoPunctuation = answerNoPrefix.replace(/[., ]+$/g, '').trim();
+
+  const answerNoQuotes = answerNoPunctuation
+    .replace(/^['"]/, '')
+    .replace(/['"]$/, '')
+    .trim();
+
+  if (answerNoQuotes.startsWith('{') || answerNoQuotes.startsWith('[')) {
+    return answerNoQuotes;
+  }
+
+  const match = answerNoQuotes.match(jsonStartRegex);
+  const answerJSONStart = match ? answerNoQuotes.slice(match.index) : undefined;
+
+  return answerJSONStart ?? answerNoQuotes;
+};

package/src/lib/timed-abort-controller/index.js

@@ -0,0 +1,41 @@
+/**
+ * A class that extends AbortController to include a timeout.
+ *
+ * @class
+ * @extends AbortController
+ *
+ * @example
+ * const timeoutController = new TimedAbortController(3000); // Set a 3 second timeout
+ * const signal = timeoutController.signal;
+ * fetch('https://example.com', { signal })
+ *   .then(response => response.text())
+ *   .then(text => console.error(text))
+ *   .catch(err => {
+ *     if (err.name === 'AbortError') {
+ *       console.error('Fetch operation was aborted');
+ *     } else {
+ *       console.error('Fetch operation failed', err);
+ *     }
+ *   });
+ * timeoutController.clearTimeout(); // Manually clear the timeout
+ */
+export default class TimedAbortController extends AbortController {
+  /**
+   * Creates a new TimedAbortController instance.
+   *
+   * @param {number} timeout - The time in milliseconds after which to automatically trigger the abort signal.
+   */
+  constructor(timeout) {
+    super(`AbortError: The operation timed out after ${timeout}ms`);
+    this.timeoutId = setTimeout(() => {
+      this.abort();
+    }, timeout);
+  }
+
+  /**
+   * Clears the timeout, preventing the abort signal from being automatically triggered.
+   */
+  clearTimeout() {
+    clearTimeout(this.timeoutId);
+  }
+}

package/src/lib/to-bool/index.js

@@ -0,0 +1,8 @@
+import stripResponse from '../strip-response/index.js';
+
+export default (val) => {
+  const valLower = stripResponse(val.toLowerCase());
+  if (valLower === 'true') return true;
+  if (valLower === 'false') return false;
+  return undefined;
+};

package/src/lib/to-enum/index.js

@@ -0,0 +1,14 @@
+import stripResponse from '../strip-response/index.js';
+
+export default (value, enumValue) => {
+  // Clean up the input by removing whitespace and punctuation
+  const valueStripped = stripResponse(value);
+  const valueCleaned = valueStripped.replace(/[^\w\s-_/\\]/gi, '').trim();
+
+  // Map the cleaned input to an enum value
+  const foundKey = Object.keys(enumValue).find(
+    (key) => key.toLowerCase() === valueCleaned.toLowerCase()
+  );
+
+  return foundKey || undefined;
+};

package/src/lib/to-number/index.js

@@ -0,0 +1,12 @@
+import stripResponse from '../strip-response/index.js';
+import stripNumeric from '../strip-numeric/index.js';
+
+export default (val) => {
+  const valLower = stripResponse(val.toLowerCase());
+  if (valLower === 'undefined') return undefined;
+  const valParsed = +stripNumeric(val);
+  if (Number.isNaN(valParsed)) {
+    throw new Error(`ChatGPT output [error]`);
+  }
+  return valParsed;
+};

package/src/lib/to-number-with-units/index.js

@@ -0,0 +1,51 @@
+import stripResponse from '../strip-response/index.js';
+import stripNumeric from '../strip-numeric/index.js';
+
+const badDatatypeError = 'Bad datatype returned for number query.';
+const valueNotANumberError = 'Value is not a number.';
+
+export default (envelope) => {
+  const envelopeStripped = stripResponse(envelope);
+  let valueExtracted;
+  let unitExtracted;
+
+  if (envelopeStripped === 'undefined') {
+    return undefined;
+  }
+
+  try {
+    const envelopeParsed = JSON.parse(envelopeStripped);
+    const { value, unit } = envelopeParsed;
+    valueExtracted = value;
+    unitExtracted = unit;
+  } catch (error) {
+    throw new Error(`ChatGPT output [error]: ${error.message}`);
+  }
+
+  let unitParsed = unitExtracted;
+  if (unitExtracted === 'undefined') {
+    unitParsed = undefined;
+  } else if (typeof unitExtracted === 'undefined') {
+    unitParsed = undefined;
+  }
+
+  let valueParsed;
+  if (typeof valueExtracted === 'string') {
+    const valueLower = valueExtracted.toLowerCase();
+    valueParsed = +stripNumeric(valueLower);
+  } else if (typeof valueExtracted === 'number') {
+    valueParsed = valueExtracted;
+  } else if (valueExtracted === 'undefined') {
+    valueParsed = undefined;
+  } else if (typeof valueExtracted === 'undefined' || valueExtracted === null) {
+    valueParsed = undefined;
+  } else {
+    throw new Error(`ChatGPT output [error]: ${badDatatypeError}`);
+  }
+
+  if (typeof valueParsed !== 'undefined' && Number.isNaN(valueParsed)) {
+    throw new Error(`ChatGPT output [error]: ${valueNotANumberError}`);
+  }
+
+  return { value: valueParsed, unit: unitParsed };
+};

package/src/lib/transcribe/index.js

@@ -0,0 +1,61 @@
+import whisper from 'whisper-node';
+import record from 'node-record-lpcm16';
+
+export default class Transcriber {
+
+  constructor(targetWord, silenceDuration = 5000, wordPauseDuration = 2000) {
+    this.targetWord = targetWord;
+    this.silenceDuration = silenceDuration;
+    this.wordPauseDuration = wordPauseDuration;
+    this.transcription = '';
+    this.lastTranscribedTime = Date.now();
+    this.recording = null;
+  }
+
+  startRecording() {
+    this.recording = record.record({
+      sampleRateHertz: 16000,
+      threshold: 0,
+      recordProgram: 'rec',
+      silence: '5.0',
+    });
+
+    const audioStream = this.recording.stream();
+    this.transcribe(audioStream);
+  }
+
+  stopRecording() {
+    if (this.recording) {
+      this.recording.stop();
+    }
+  }
+
+  transcribe(stream) {
+    whisper.transcribeStream(stream, { streaming: true })
+      .then(transcription => {
+        this.handleTranscription(transcription);
+      })
+      .catch(error => {
+        console.error(error);
+      });
+  }
+
+  handleTranscription(transcription) {
+    this.transcription += transcription;
+    this.lastTranscribedTime = Date.now();
+
+    if (transcription.includes(this.targetWord)) {
+      setTimeout(() => {
+        this.stopRecording();
+      }, this.wordPauseDuration);
+    }
+
+    if (Date.now() - this.lastTranscribedTime >= this.silenceDuration) {
+      this.stopRecording();
+    }
+  }
+
+  getText() {
+    return this.transcription;
+  }
+}

package/src/prompts/README.md

@@ -0,0 +1,15 @@
+# Prompts
+
+Prompts are functions that produce a single LLM prompt. They can be parameterized in a number of ways in order to fit their problem as closely as possible.
+
+## Prompt Functions
+
+Currently there isn't much to the structure of these functions, and they're not currently tested.
+
+Going forward, functions should output a structured representation whose contents can be validated. An obvious candidate for this representation is HTML.
+
+The functions can generate HTML from their inputs and this structure can be transformed into text before sending to ChatGPT. This approach has many benefits. We can use all the tools available for working with HTML. The function outputs can be tested. All varieties of markup problems, like nested content, are trivially solved as well.
+
+# Constants
+
+Currently it's reasonable to keep the text fragments needed for building prompts in a single constants.js file. It's forseable that the number of text fragments will grow. At that point, it would make sense to maintain them with a CMS such as Strapi.

package/src/prompts/as-enum.js

@@ -0,0 +1,5 @@
+export default (enumVal) => {
+  const keys = Object.keys(enumVal);
+  const options = keys.map((k) => `"${k}"`).join(', ');
+  return `Choose from one of the following options: ${options}.`;
+};

package/src/prompts/as-json-schema.js

@@ -0,0 +1,9 @@
+import { onlyJSON } from './constants.js';
+
+export default (propertiesDescription) => {
+  return `Give me a JSONSchema definition for the following properties: ${propertiesDescription}
+
+Include per-property metadata as JSON comments.
+
+${onlyJSON}`;
+};

package/src/prompts/as-object-with-schema.js

@@ -0,0 +1,31 @@
+import { contentIsExampleObject, onlyJSON } from './constants.js';
+
+const jsonSchemaDefault = {
+  type: 'object',
+  properties: {
+    name: {
+      type: 'string',
+    },
+  },
+};
+
+export default (jsonSchema = jsonSchemaDefault) => {
+  const propertiesJoined = Object.entries(jsonSchema.properties)
+    .map(([key, val]) => {
+      const annotations = Object.entries(val).filter(
+        ([annKey, annVal]) =>
+          ['format', 'description'].includes(annKey) && !!annVal
+      );
+      const annotationsFormatted = annotations
+        .map(([k, v]) => `${k}: ${v}`)
+        .join(', ');
+      const annotationsWrapped = annotations.length
+        ? ` (${annotationsFormatted})`
+        : '';
+      return `"${key}": "<${val.type ?? ''}${annotationsWrapped}>"`;
+    })
+    .join(', ');
+
+  return `${contentIsExampleObject} \`{ ${propertiesJoined} }\`. ${onlyJSON}.
+`;
+};

package/src/prompts/as-schema-org-text.js

@@ -0,0 +1,17 @@
+import { onlyJSON } from './constants.js';
+import asSchemaOrgType from './as-schema-org-type.js';
+
+const ensureNumbers = 'ensure values meant to be numbers are numbers';
+const ensureSchemaOrgType = 'ensure the type is a real schema.org type';
+const ensureProperties = 'ensure the returned object has @context, name';
+
+export default (object, type) => {
+  const typeText = `${asSchemaOrgType(type)}`;
+  return `Give me "${object}" in schema.org JSON format with a full set of properties. ${
+    typeText ? `${typeText}.` : ''
+  }
+- ${ensureNumbers}
+- ${ensureSchemaOrgType}
+- ${ensureProperties}
+${onlyJSON}`;
+};

package/src/prompts/as-schema-org-type.js

@@ -0,0 +1 @@
+export default (type) => (type ? `Ensure the type is ${type}. ` : '');

package/src/prompts/blog-post.js

@@ -0,0 +1,7 @@
+export default (
+  text
+) => `Use the following information to create a blog post. Don't necessarily use all of the information. Keep your sentences varied, but nothing too long. Focus on writing an interesting, engaging post:
+\`\`\`
+${text}
+\`\`\`
+`;