@forwardimpact/libutil 0.1.60

package/test/tokenizer.test.js

@@ -0,0 +1,123 @@
+import { test, describe, beforeEach } from "node:test";
+import assert from "node:assert";
+
+// Module under test
+import { Tokenizer, ranks } from "../tokenizer.js";
+import { countTokens, createTokenizer } from "../index.js";
+
+describe("Tokenizer", () => {
+  describe("constructor", () => {
+    test("creates instance with ranks parameter", () => {
+      const tokenizer = new Tokenizer(ranks);
+      assert.ok(tokenizer instanceof Tokenizer);
+    });
+
+    test("creates instance without ranks parameter", () => {
+      const tokenizer = new Tokenizer();
+      assert.ok(tokenizer instanceof Tokenizer);
+    });
+  });
+
+  describe("encode", () => {
+    let tokenizer;
+
+    beforeEach(() => {
+      tokenizer = new Tokenizer(ranks);
+    });
+
+    test("handles empty string", () => {
+      const result = tokenizer.encode("");
+      assert.strictEqual(result.length, 0);
+    });
+
+    test("handles non-string input", () => {
+      const result = tokenizer.encode(null);
+      assert.strictEqual(result.length, 0);
+    });
+
+    test("encodes simple word", () => {
+      const result = tokenizer.encode("hello");
+      assert.ok(result.length >= 1);
+      assert.ok(Array.isArray(result));
+    });
+
+    test("encodes longer text", () => {
+      const shortText = "hello";
+      const longText = "hello world this is a longer piece of text";
+
+      const shortResult = tokenizer.encode(shortText);
+      const longResult = tokenizer.encode(longText);
+
+      assert.ok(longResult.length > shortResult.length);
+    });
+
+    test("handles whitespace-only text", () => {
+      const result = tokenizer.encode("   ");
+      assert.strictEqual(result.length, 0);
+    });
+
+    test("handles punctuation", () => {
+      const result = tokenizer.encode("Hello, world!");
+      assert.ok(result.length >= 3); // At least hello + comma + world + exclamation
+    });
+
+    test("handles mixed content", () => {
+      const result = tokenizer.encode("The year 2024 was great!");
+      assert.ok(result.length >= 5); // Multiple words and punctuation
+    });
+
+    test("provides reasonable approximation", () => {
+      // Test that the approximation is in a reasonable range
+      const text = "This is a test sentence with about ten words here.";
+      const result = tokenizer.encode(text);
+
+      // Should be roughly 10-15 tokens for this sentence
+      assert.ok(result.length >= 8);
+      assert.ok(result.length <= 20);
+    });
+  });
+
+  describe("decode", () => {
+    test("throws error when called", () => {
+      const tokenizer = new Tokenizer(ranks);
+      assert.throws(() => tokenizer.decode([1, 2, 3]), {
+        message: /decode\(\) not implemented/,
+      });
+    });
+  });
+});
+
+describe("Integration with libutil functions", () => {
+  describe("tokenizerFactory", () => {
+    test("creates Tokenizer instance", () => {
+      const tokenizer = createTokenizer();
+      assert.ok(tokenizer instanceof Tokenizer);
+    });
+  });
+
+  describe("countTokens", () => {
+    test("returns token count for text", () => {
+      const count = countTokens("hello world");
+      assert.ok(typeof count === "number");
+      assert.ok(count >= 1);
+    });
+
+    test("handles empty text", () => {
+      const count = countTokens("");
+      assert.strictEqual(count, 0);
+    });
+
+    test("uses provided tokenizer", () => {
+      const customTokenizer = new Tokenizer(ranks);
+      const count = countTokens("test", customTokenizer);
+      assert.ok(typeof count === "number");
+      assert.ok(count >= 1);
+    });
+
+    test("uses default tokenizer when none provided", () => {
+      const count = countTokens("test");
+      assert.ok(typeof count === "number");
+      assert.ok(count >= 1);
+    });
+  });
+});

package/test/wait.test.js

@@ -0,0 +1,89 @@
+import { describe, test } from "node:test";
+import assert from "node:assert";
+
+import { waitFor } from "../wait.js";
+
+describe("waitFor", () => {
+  test("resolves immediately when condition is true", async () => {
+    let callCount = 0;
+    await waitFor(() => {
+      callCount++;
+      return Promise.resolve(true);
+    });
+
+    assert.strictEqual(callCount, 1);
+  });
+
+  test("retries until condition becomes true", async () => {
+    let callCount = 0;
+    await waitFor(
+      () => {
+        callCount++;
+        return Promise.resolve(callCount >= 3);
+      },
+      { interval: 10, timeout: 5000 },
+    );
+
+    assert.strictEqual(callCount, 3);
+  });
+
+  test("throws error on timeout", async () => {
+    await assert.rejects(
+      () =>
+        waitFor(() => Promise.resolve(false), {
+          timeout: 50,
+          interval: 10,
+        }),
+      { message: "Timeout waiting for condition after 50ms" },
+    );
+  });
+
+  test("ignores errors during polling", async () => {
+    let callCount = 0;
+    await waitFor(
+      () => {
+        callCount++;
+        if (callCount < 3) {
+          throw new Error("Service not ready");
+        }
+        return Promise.resolve(true);
+      },
+      { interval: 10, timeout: 5000 },
+    );
+
+    assert.strictEqual(callCount, 3);
+  });
+
+  test("uses default options when not provided", async () => {
+    // Just verify it doesn't throw with defaults
+    let called = false;
+    await waitFor(() => {
+      called = true;
+      return Promise.resolve(true);
+    });
+
+    assert.strictEqual(called, true);
+  });
+
+  test("increases interval with exponential backoff", async () => {
+    const intervals = [];
+    let lastTime = Date.now();
+    let callCount = 0;
+
+    await waitFor(
+      () => {
+        const now = Date.now();
+        if (callCount > 0) {
+          intervals.push(now - lastTime);
+        }
+        lastTime = now;
+        callCount++;
+        return Promise.resolve(callCount >= 4);
+      },
+      { interval: 20, maxInterval: 100, timeout: 5000 },
+    );
+
+    // Intervals should generally increase (with some tolerance for timing)
+    assert.ok(intervals.length >= 2);
+  });
+});

package/tokenizer.js

@@ -0,0 +1,89 @@
+/**
+ * Simple tokenizer class that provides API compatibility with js-tiktoken
+ * Uses basic approximation logic for token counting
+ */
+export class Tokenizer {
+  /**
+   * Creates a new Tokenizer instance
+   * @param {object} _ranks - Ranking data (unused in simple implementation)
+   */
+  constructor(_ranks) {
+    // Ranks parameter is ignored in this simple implementation
+  }
+
+  /**
+   * Encodes text into tokens using simple approximation
+   * @param {string} text - Text to encode
+   * @returns {number[]} Array of token IDs (approximated)
+   */
+  encode(text) {
+    if (typeof text !== "string") {
+      return [];
+    }
+
+    if (text.length === 0) {
+      return [];
+    }
+
+    // Simple approximation logic:
+    // 1. Split on whitespace and punctuation
+    // 2. Count characters in a way that approximates GPT tokenization
+    // 3. Return array with length approximating actual token count
+
+    // Remove extra whitespace and normalize
+    const normalized = text.trim().replace(/\s+/g, " ");
+
+    if (normalized.length === 0) {
+      return [];
+    }
+
+    // Simple heuristic for token counting:
+    // - Average English word is ~4 characters = 1 token
+    // - Punctuation and special chars often = 1 token each
+    // - Numbers and code can be more dense
+
+    let tokenCount = 0;
+
+    // Count words (sequences of letters/numbers)
+    const words = normalized.match(/\b\w+\b/g) || [];
+    for (const word of words) {
+      // Short words (1-4 chars) = 1 token
+      // Longer words = roughly chars/4 tokens
+      if (word.length <= 4) {
+        tokenCount += 1;
+      } else {
+        tokenCount += Math.ceil(word.length / 4);
+      }
+    }
+
+    // Count punctuation and special characters
+    const punctuation = normalized.match(/[^\w\s]/g) || [];
+    tokenCount += punctuation.length;
+
+    // Count whitespace as minimal tokens (spaces between words)
+    const spaces = normalized.match(/\s/g) || [];
+    tokenCount += Math.ceil(spaces.length / 2);
+
+    // Ensure minimum of 1 token for non-empty text
+    tokenCount = Math.max(1, tokenCount);
+
+    // Return array with dummy token IDs
+    // The actual values don't matter since only .length is used
+    return new Array(tokenCount).fill(0).map((_, i) => i);
+  }
+
+  /**
+   * Decodes tokens back to text (not implemented - not used in codebase)
+   * @param {number[]} _tokens - Token IDs to decode
+   * @throws {Error} Always throws - not implemented
+   */
+  decode(_tokens) {
+    throw new Error("decode() not implemented in Tokenizer");
+  }
+}
+
+/**
+ * Dummy ranks object for compatibility
+ * Not used in the simple implementation but needed for API compatibility
+ */
+export const ranks = {};

package/wait.js

@@ -0,0 +1,28 @@
+/**
+ * Poll until a condition returns true with exponential backoff
+ * @param {() => Promise<boolean>} checkFn - Function that returns true when ready
+ * @param {object} [options] - Configuration options
+ * @param {number} [options.timeout] - Maximum time to wait in ms
+ * @param {number} [options.interval] - Initial polling interval in ms
+ * @param {number} [options.maxInterval] - Maximum polling interval in ms
+ * @returns {Promise<void>}
+ * @throws {Error} When timeout is reached
+ */
+export async function waitFor(checkFn, options = {}) {
+  const { timeout = 30000, interval = 1000, maxInterval = 10000 } = options;
+  const startTime = Date.now();
+  let currentInterval = interval;
+
+  while (Date.now() - startTime < timeout) {
+    try {
+      if (await checkFn()) return;
+    } catch {
+      // Ignore errors during polling - service may not be up yet
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, currentInterval));
+    currentInterval = Math.min(currentInterval * 1.5, maxInterval);
+  }
+
+  throw new Error(`Timeout waiting for condition after ${timeout}ms`);
+}