@hyvmind/tiktoken-ts 0.0.1

package/dist/utils.d.ts

@@ -0,0 +1,152 @@
+/**
+ * Tokenizer Utilities
+ * Helper functions for tokenization
+ */
+/**
+ * Convert a string to UTF-8 bytes
+ *
+ * @param text - Input text
+ * @returns Uint8Array of UTF-8 bytes
+ */
+export declare function stringToBytes(text: string): Uint8Array;
+/**
+ * Convert UTF-8 bytes to string
+ *
+ * @param bytes - UTF-8 bytes
+ * @returns Decoded string
+ */
+export declare function bytesToString(bytes: Uint8Array | number[]): string;
+/**
+ * Convert a byte to its hexadecimal representation
+ *
+ * @param byte - Single byte value (0-255)
+ * @returns Two-character hex string
+ */
+export declare function byteToHex(byte: number): string;
+/**
+ * Convert hexadecimal string to byte
+ *
+ * @param hex - Two-character hex string
+ * @returns Byte value (0-255)
+ */
+export declare function hexToByte(hex: string): number;
+/**
+ * Check if a character is a whitespace character
+ *
+ * @param char - Single character
+ * @returns True if whitespace
+ */
+export declare function isWhitespace(char: string): boolean;
+/**
+ * Check if a character is a letter
+ *
+ * @param char - Single character
+ * @returns True if letter
+ */
+export declare function isLetter(char: string): boolean;
+/**
+ * Check if a character is a digit
+ *
+ * @param char - Single character
+ * @returns True if digit
+ */
+export declare function isDigit(char: string): boolean;
+/**
+ * Check if a character is punctuation
+ *
+ * @param char - Single character
+ * @returns True if punctuation
+ */
+export declare function isPunctuation(char: string): boolean;
+/**
+ * Check if a string contains only ASCII characters
+ *
+ * @param text - Input text
+ * @returns True if ASCII only
+ */
+export declare function isAscii(text: string): boolean;
+/**
+ * Count the number of Unicode code points in a string
+ * Handles surrogate pairs correctly
+ *
+ * @param text - Input text
+ * @returns Number of code points
+ */
+export declare function countCodePoints(text: string): number;
+/**
+ * Split text into words using Unicode-aware boundaries
+ *
+ * @param text - Input text
+ * @returns Array of words and whitespace
+ */
+export declare function splitIntoWords(text: string): string[];
+/**
+ * Escape special regex characters in a string
+ *
+ * @param text - Input text
+ * @returns Escaped string safe for regex
+ */
+export declare function escapeRegex(text: string): string;
+/**
+ * Clamp a number between min and max values
+ *
+ * @param value - Value to clamp
+ * @param min - Minimum value
+ * @param max - Maximum value
+ * @returns Clamped value
+ */
+export declare function clamp(value: number, min: number, max: number): number;
+/**
+ * Calculate the percentage of a value
+ *
+ * @param value - Base value
+ * @param percentage - Percentage (0-1)
+ * @returns Calculated percentage value
+ */
+export declare function percentage(value: number, pct: number): number;
+/**
+ * Create a hash from a string (for vocabulary lookup)
+ * Simple FNV-1a hash for performance
+ *
+ * @param text - Input text
+ * @returns 32-bit hash value
+ */
+export declare function hashString(text: string): number;
+/**
+ * Count occurrences of a substring in text
+ *
+ * @param text - Text to search in
+ * @param substring - Substring to count
+ * @returns Number of occurrences
+ */
+export declare function countOccurrences(text: string, substring: string): number;
+/**
+ * Normalize whitespace in text (collapse multiple spaces)
+ *
+ * @param text - Input text
+ * @returns Text with normalized whitespace
+ */
+export declare function normalizeWhitespace(text: string): string;
+/**
+ * Check if text contains CJK (Chinese/Japanese/Korean) characters
+ *
+ * @param text - Input text
+ * @returns True if contains CJK characters
+ */
+export declare function containsCJK(text: string): boolean;
+/**
+ * Check if text contains emoji
+ *
+ * @param text - Input text
+ * @returns True if contains emoji
+ */
+export declare function containsEmoji(text: string): boolean;
+/**
+ * Estimate token multiplier based on text characteristics
+ * Used for non-English text where token count can vary significantly
+ *
+ * @param text - Input text
+ * @returns Multiplier to apply to base token estimate
+ */
+export declare function getTextComplexityMultiplier(text: string): number;
+//# sourceMappingURL=utils.d.ts.map

package/dist/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,CAGtD;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,UAAU,GAAG,MAAM,EAAE,GAAG,MAAM,CAKlE;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE9C;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAE7C;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAElD;AAED;;;;;GAKG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE9C;AAED;;;;;GAKG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE7C;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEnD;AAED;;;;;GAKG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAG7C;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGpD;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAyBrD;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEhD;AAED;;;;;;;GAOG;AACH,wBAAgB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAErE;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAE7D;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAU/C;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAYxE;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAExD;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAKjD;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEnD;AAED;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAmBhE"}

package/dist/utils.js

@@ -0,0 +1,244 @@
+/**
+ * Tokenizer Utilities
+ * Helper functions for tokenization
+ */
+/**
+ * Convert a string to UTF-8 bytes
+ *
+ * @param text - Input text
+ * @returns Uint8Array of UTF-8 bytes
+ */
+export function stringToBytes(text) {
+    const encoder = new TextEncoder();
+    return encoder.encode(text);
+}
+/**
+ * Convert UTF-8 bytes to string
+ *
+ * @param bytes - UTF-8 bytes
+ * @returns Decoded string
+ */
+export function bytesToString(bytes) {
+    const decoder = new TextDecoder("utf-8", { fatal: false });
+    return decoder.decode(bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes));
+}
+/**
+ * Convert a byte to its hexadecimal representation
+ *
+ * @param byte - Single byte value (0-255)
+ * @returns Two-character hex string
+ */
+export function byteToHex(byte) {
+    return byte.toString(16).padStart(2, "0");
+}
+/**
+ * Convert hexadecimal string to byte
+ *
+ * @param hex - Two-character hex string
+ * @returns Byte value (0-255)
+ */
+export function hexToByte(hex) {
+    return parseInt(hex, 16);
+}
+/**
+ * Check if a character is a whitespace character
+ *
+ * @param char - Single character
+ * @returns True if whitespace
+ */
+export function isWhitespace(char) {
+    return /\s/.test(char);
+}
+/**
+ * Check if a character is a letter
+ *
+ * @param char - Single character
+ * @returns True if letter
+ */
+export function isLetter(char) {
+    return /\p{L}/u.test(char);
+}
+/**
+ * Check if a character is a digit
+ *
+ * @param char - Single character
+ * @returns True if digit
+ */
+export function isDigit(char) {
+    return /\p{N}/u.test(char);
+}
+/**
+ * Check if a character is punctuation
+ *
+ * @param char - Single character
+ * @returns True if punctuation
+ */
+export function isPunctuation(char) {
+    return /\p{P}/u.test(char);
+}
+/**
+ * Check if a string contains only ASCII characters
+ *
+ * @param text - Input text
+ * @returns True if ASCII only
+ */
+export function isAscii(text) {
+    // eslint-disable-next-line no-control-regex
+    return /^[\x00-\x7F]*$/.test(text);
+}
+/**
+ * Count the number of Unicode code points in a string
+ * Handles surrogate pairs correctly
+ *
+ * @param text - Input text
+ * @returns Number of code points
+ */
+export function countCodePoints(text) {
+    // Using the spread operator to handle surrogate pairs
+    return [...text].length;
+}
+/**
+ * Split text into words using Unicode-aware boundaries
+ *
+ * @param text - Input text
+ * @returns Array of words and whitespace
+ */
+export function splitIntoWords(text) {
+    // Split on word boundaries while preserving whitespace
+    const segments = [];
+    let current = "";
+    let inWord = false;
+    for (const char of text) {
+        const charIsWord = isLetter(char) || isDigit(char) || char === "'";
+        if (charIsWord !== inWord) {
+            if (current) {
+                segments.push(current);
+            }
+            current = char;
+            inWord = charIsWord;
+        }
+        else {
+            current += char;
+        }
+    }
+    if (current) {
+        segments.push(current);
+    }
+    return segments;
+}
+/**
+ * Escape special regex characters in a string
+ *
+ * @param text - Input text
+ * @returns Escaped string safe for regex
+ */
+export function escapeRegex(text) {
+    return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+/**
+ * Clamp a number between min and max values
+ *
+ * @param value - Value to clamp
+ * @param min - Minimum value
+ * @param max - Maximum value
+ * @returns Clamped value
+ */
+export function clamp(value, min, max) {
+    return Math.min(Math.max(value, min), max);
+}
+/**
+ * Calculate the percentage of a value
+ *
+ * @param value - Base value
+ * @param percentage - Percentage (0-1)
+ * @returns Calculated percentage value
+ */
+export function percentage(value, pct) {
+    return Math.round(value * pct);
+}
+/**
+ * Create a hash from a string (for vocabulary lookup)
+ * Simple FNV-1a hash for performance
+ *
+ * @param text - Input text
+ * @returns 32-bit hash value
+ */
+export function hashString(text) {
+    let hash = 0x811c9dc5; // FNV offset basis
+    const bytes = stringToBytes(text);
+    for (const byte of bytes) {
+        hash ^= byte;
+        hash = Math.imul(hash, 0x01000193); // FNV prime
+    }
+    return hash >>> 0; // Convert to unsigned
+}
+/**
+ * Count occurrences of a substring in text
+ *
+ * @param text - Text to search in
+ * @param substring - Substring to count
+ * @returns Number of occurrences
+ */
+export function countOccurrences(text, substring) {
+    if (!substring)
+        return 0;
+    let count = 0;
+    let position = 0;
+    while ((position = text.indexOf(substring, position)) !== -1) {
+        count++;
+        position += substring.length;
+    }
+    return count;
+}
+/**
+ * Normalize whitespace in text (collapse multiple spaces)
+ *
+ * @param text - Input text
+ * @returns Text with normalized whitespace
+ */
+export function normalizeWhitespace(text) {
+    return text.replace(/\s+/g, " ").trim();
+}
+/**
+ * Check if text contains CJK (Chinese/Japanese/Korean) characters
+ *
+ * @param text - Input text
+ * @returns True if contains CJK characters
+ */
+export function containsCJK(text) {
+    // CJK Unified Ideographs and related blocks
+    return /[\u4e00-\u9fff\u3400-\u4dbf\u{20000}-\u{2a6df}\u{2a700}-\u{2b73f}\u{2b740}-\u{2b81f}\u{2b820}-\u{2ceaf}\u{2ceb0}-\u{2ebef}\u{30000}-\u{3134f}\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/u.test(text);
+}
+/**
+ * Check if text contains emoji
+ *
+ * @param text - Input text
+ * @returns True if contains emoji
+ */
+export function containsEmoji(text) {
+    return /\p{Emoji}/u.test(text);
+}
+/**
+ * Estimate token multiplier based on text characteristics
+ * Used for non-English text where token count can vary significantly
+ *
+ * @param text - Input text
+ * @returns Multiplier to apply to base token estimate
+ */
+export function getTextComplexityMultiplier(text) {
+    let multiplier = 1.0;
+    // CJK characters typically use more tokens per character
+    if (containsCJK(text)) {
+        multiplier *= 1.5;
+    }
+    // Emoji can use multiple tokens
+    if (containsEmoji(text)) {
+        multiplier *= 1.2;
+    }
+    // Non-ASCII text generally uses more tokens
+    if (!isAscii(text)) {
+        multiplier *= 1.1;
+    }
+    return multiplier;
+}
+//# sourceMappingURL=utils.js.map

package/dist/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../src/utils.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,KAA4B;IACxD,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3D,OAAO,OAAO,CAAC,MAAM,CACnB,KAAK,YAAY,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,CAC5D,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,GAAW;IACnC,OAAO,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;AAC3B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,4CAA4C;IAC5C,OAAO,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,sDAAsD;IACtD,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC;AAC1B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,uDAAuD;IACvD,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,GAAG,CAAC;QAEnE,IAAI,UAAU,KAAK,MAAM,EAAE,CAAC;YAC1B,IAAI,OAAO,EAAE,CAAC;gBACZ,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzB,CAAC;YACD,OAAO,GAAG,IAAI,CAAC;YACf,MAAM,GAAG,UAAU,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,IAAI,CAAC;QAClB,CAAC;IACH,CAAC;IAED,IAAI,OAAO,EAAE,CAAC;QACZ,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACzB,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,OAAO,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,KAAK,CAAC,KAAa,EAAE,GAAW,EAAE,GAAW;IAC3D,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;AAC7C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,KAAa,EAAE,GAAW;IACnD,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,IAAI,IAAI,GAAG,UAAU,CAAC,CAAC,mBAAmB;IAC1C,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,IAAI,CAAC;QACb,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,YAAY;IAClD,CAAC;IAED,OAAO,IAAI,KAAK,CAAC,CAAC,CAAC,sBAAsB;AAC3C,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,SAAiB;IAC9D,IAAI,CAAC,SAAS;QAAE,OAAO,CAAC,CAAC;IAEzB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QAC7D,KAAK,EAAE,CAAC;QACR,QAAQ,IAAI,SAAS,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC9C,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC1C,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,4CAA4C;IAC5C,OAAO,wLAAwL,CAAC,IAAI,CAClM,IAAI,CACL,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,2BAA2B,CAAC,IAAY;IACtD,IAAI,UAAU,GAAG,GAAG,CAAC;IAErB,yDAAyD;IACzD,IAAI,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,UAAU,IAAI,GAAG,CAAC;IACpB,CAAC;IAED,gCAAgC;IAChC,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,UAAU,IAAI,GAAG,CAAC;IACpB,CAAC;IAED,4CAA4C;IAC5C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACnB,UAAU,IAAI,GAAG,CAAC;IACpB,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/package.json

@@ -0,0 +1,78 @@
+{
+  "name": "@hyvmind/tiktoken-ts",
+  "version": "0.0.1",
+  "sideEffects": false,
+  "description": "A pure TypeScript implementation of OpenAI's tiktoken tokenizer, compatible with tiktoken-rs",
+  "author": "HyvMind",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./encodings": {
+      "import": "./dist/encodings/index.js",
+      "types": "./dist/encodings/index.d.ts"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "keywords": [
+    "tiktoken",
+    "tokenizer",
+    "bpe",
+    "openai",
+    "gpt",
+    "gpt-4",
+    "gpt-4o",
+    "claude",
+    "llm",
+    "tokens",
+    "encoding"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/hyvmind-io/tiktoken-ts.git"
+  },
+  "bugs": {
+    "url": "https://github.com/hyvmind-io/tiktoken-ts/issues"
+  },
+  "homepage": "https://github.com/hyvmind-io/tiktoken-ts#readme",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.39.2",
+    "@types/node": "^22.0.0",
+    "@typescript-eslint/eslint-plugin": "^8.0.0",
+    "@typescript-eslint/parser": "^8.0.0",
+    "@vitest/coverage-v8": "^2.0.0",
+    "eslint": "^9.0.0",
+    "eslint-config-prettier": "^9.0.0",
+    "prettier": "^3.0.0",
+    "typescript": "^5.6.0",
+    "typescript-eslint": "^8.54.0",
+    "vitest": "^2.0.0"
+  },
+  "scripts": {
+    "clean": "rm -rf dist",
+    "prebuild": "pnpm run clean",
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "lint": "eslint src --ext .ts",
+    "lint:fix": "eslint src --ext .ts --fix",
+    "format": "prettier --write \"src/**/*.ts\"",
+    "format:check": "prettier --check \"src/**/*.ts\""
+  }
+}