@orion-agents/orion-code 0.1.0

package/dist/utils/token-estimate.js

@@ -0,0 +1,150 @@
+"use strict";
+/**
+ * orion code - Token estimation utilities
+ *
+ * Accurate token estimation that accounts for CJK characters,
+ * which have significantly different token ratios than ASCII text.
+ *
+ * CJK characters: ~1.5 tokens/char (vs ~0.25 for ASCII words)
+ * This is critical for bilingual (CN/EN) coding agents.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.estimateTokens = estimateTokens;
+exports.estimateMessagesTokens = estimateMessagesTokens;
+exports.quickTokenEstimate = quickTokenEstimate;
+/**
+ * Check if a character is a CJK (Chinese/Japanese/Korean) character
+ * These characters typically consume ~1.5 tokens each in most tokenizers
+ */
+function isCJK(char) {
+    const cp = char.codePointAt(0) || 0;
+    return (
+    // CJK Unified Ideographs
+    (cp >= 0x4E00 && cp <= 0x9FFF) ||
+        // CJK Extension A
+        (cp >= 0x3400 && cp <= 0x4DBF) ||
+        // CJK Extension B
+        (cp >= 0x20000 && cp <= 0x2A6DF) ||
+        // CJK Extension C
+        (cp >= 0x2A700 && cp <= 0x2B73F) ||
+        // CJK Extension D
+        (cp >= 0x2B740 && cp <= 0x2B81F) ||
+        // CJK Compatibility Ideographs
+        (cp >= 0xF900 && cp <= 0xFAFF) ||
+        // CJK Compatibility Ideographs Supplement
+        (cp >= 0x2F800 && cp <= 0x2FA1F) ||
+        // Hiragana (Japanese)
+        (cp >= 0x3040 && cp <= 0x309F) ||
+        // Katakana (Japanese)
+        (cp >= 0x30A0 && cp <= 0x30FF) ||
+        // Katakana Phonetic Extensions
+        (cp >= 0x31F0 && cp <= 0x31FF) ||
+        // Hangul Syllables (Korean)
+        (cp >= 0xAC00 && cp <= 0xD7AF) ||
+        // Hangul Jamo
+        (cp >= 0x1100 && cp <= 0x11FF) ||
+        // Hangul Compatibility Jamo
+        (cp >= 0x3130 && cp <= 0x318F) ||
+        // Fullwidth Forms (CJK punctuation, etc.)
+        (cp >= 0xFF00 && cp <= 0xFFEF));
+}
+/**
+ * Check if a character is a whitespace/newline
+ */
+function isWhitespace(char) {
+    return char === ' ' || char === '\t' || char === '\n' || char === '\r';
+}
+/**
+ * Estimate token count for a given text.
+ *
+ * Token ratios by character type (approximate, based on common tokenizers):
+ * - CJK characters: ~1.5 tokens/char (each CJK char is typically 1-2 tokens)
+ * - ASCII letters/digits: ~0.25 tokens/char (4 chars per token on average)
+ * - Punctuation: ~1 token/char (each punctuation mark is often its own token)
+ * - Whitespace: ~0.5 tokens/char (spaces often grouped, newlines are separate)
+ *
+ * @param text - The text to estimate tokens for
+ * @returns Estimated token count (integer)
+ */
+function estimateTokens(text) {
+    if (!text)
+        return 0;
+    let tokens = 0;
+    let asciiRun = 0;
+    for (const char of text) {
+        if (isCJK(char)) {
+            // Flush any accumulated ASCII run
+            if (asciiRun > 0) {
+                tokens += Math.ceil(asciiRun / 4);
+                asciiRun = 0;
+            }
+            // CJK characters: ~1.5 tokens each
+            tokens += 1.5;
+        }
+        else if (isWhitespace(char)) {
+            // Flush ASCII run
+            if (asciiRun > 0) {
+                tokens += Math.ceil(asciiRun / 4);
+                asciiRun = 0;
+            }
+            // Newlines are typically their own token
+            if (char === '\n') {
+                tokens += 1;
+            }
+            else {
+                tokens += 0.5;
+            }
+        }
+        else if (char.codePointAt(0) < 128) {
+            // ASCII letter/digit - accumulate for word-level estimation
+            asciiRun++;
+        }
+        else {
+            // Non-ASCII punctuation/symbols - flush ASCII run first
+            if (asciiRun > 0) {
+                tokens += Math.ceil(asciiRun / 4);
+                asciiRun = 0;
+            }
+            // Punctuation and symbols: ~1 token each
+            tokens += 1;
+        }
+    }
+    // Flush remaining ASCII run
+    if (asciiRun > 0) {
+        tokens += Math.ceil(asciiRun / 4);
+    }
+    return Math.ceil(tokens);
+}
+/**
+ * Estimate tokens for an array of messages (for compact/autocompact fallback).
+ *
+ * @param messages - Array of message objects with content
+ * @returns Estimated total token count
+ */
+function estimateMessagesTokens(messages) {
+    let total = 0;
+    for (const msg of messages) {
+        if (msg.content) {
+            // Add overhead for message framing (~4 tokens per message)
+            total += estimateTokens(msg.content) + 4;
+        }
+    }
+    return total;
+}
+/**
+ * Quick token estimate using character count as fallback.
+ * Uses a more conservative ratio than text.length / 4.
+ *
+ * @param textLength - Character count of the text
+ * @param hasCJK - Whether the text contains CJK characters
+ * @returns Estimated token count
+ */
+function quickTokenEstimate(textLength, hasCJK = false) {
+    if (hasCJK) {
+        // CJK-heavy text: ~1.5 tokens per char
+        return Math.ceil(textLength * 1.5);
+    }
+    // ASCII-heavy text: ~0.25 tokens per char
+    return Math.ceil(textLength / 4);
+}
+//# sourceMappingURL=token-estimate.js.map

package/dist/utils/token-estimate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-estimate.js","sourceRoot":"","sources":["../../src/utils/token-estimate.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;AA2DH,wCA+CC;AAQD,wDAWC;AAUD,gDAOC;AA5ID;;;GAGG;AACH,SAAS,KAAK,CAAC,IAAY;IACzB,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACpC,OAAO;IACL,yBAAyB;IACzB,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,kBAAkB;QAClB,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,kBAAkB;QAClB,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC;QAChC,kBAAkB;QAClB,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC;QAChC,kBAAkB;QAClB,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC;QAChC,+BAA+B;QAC/B,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,0CAA0C;QAC1C,CAAC,EAAE,IAAI,OAAO,IAAI,EAAE,IAAI,OAAO,CAAC;QAChC,sBAAsB;QACtB,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,sBAAsB;QACtB,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,+BAA+B;QAC/B,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,4BAA4B;QAC5B,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,cAAc;QACd,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,4BAA4B;QAC5B,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC;QAC9B,0CAA0C;QAC1C,CAAC,EAAE,IAAI,MAAM,IAAI,EAAE,IAAI,MAAM,CAAC,CAC/B,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC;AACzE,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAgB,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAChB,kCAAkC;YAClC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;gBACjB,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;gBAClC,QAAQ,GAAG,CAAC,CAAC;YACf,CAAC;YACD,mCAAmC;YACnC,MAAM,IAAI,GAAG,CAAC;QAChB,CAAC;aAAM,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9B,kBAAkB;YAClB,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;gBACjB,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;gBAClC,QAAQ,GAAG,CAAC,CAAC;YACf,CAAC;YACD,yCAAyC;YACzC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBAClB,MAAM,IAAI,CAAC,CAAC;YACd,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,GAAG,CAAC;YAChB,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAE,GAAG,GAAG,EAAE,CAAC;YACtC,4DAA4D;YAC5D,QAAQ,EAAE,CAAC;QACb,CAAC;aAAM,CAAC;YACN,wDAAwD;YACxD,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;gBACjB,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;gBAClC,QAAQ,GAAG,CAAC,CAAC;YACf,CAAC;YACD,yCAAyC;YACzC,MAAM,IAAI,CAAC,CAAC;QACd,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QACjB,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC3B,CAAC;AAED;;;;;GAKG;AACH,SAAgB,sBAAsB,CACpC,QAA2D;IAE3D,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAChB,2DAA2D;YAC3D,KAAK,IAAI,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,kBAAkB,CAAC,UAAkB,EAAE,MAAM,GAAG,KAAK;IACnE,IAAI,MAAM,EAAE,CAAC;QACX,uCAAuC;QACvC,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC;IACrC,CAAC;IACD,0CAA0C;IAC1C,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;AACnC,CAAC"}

package/package.json

@@ -0,0 +1,100 @@
+{
+  "name": "@orion-agents/orion-code",
+  "version": "0.1.0",
+  "description": "Orion Code - goal-driven coding agent for the terminal.",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "orion": "bin/orion"
+  },
+  "files": [
+    "dist/",
+    "bin/",
+    "assets/",
+    "LICENSE",
+    "README.md"
+  ],
+  "scripts": {
+    "start": "ts-node src/cli.ts",
+    "dev": "nodemon src/cli.ts",
+    "cli": "ts-node src/cli.ts",
+    "build": "tsc",
+    "test": "jest",
+    "lint": "eslint src/",
+    "format": "prettier --write src/",
+    "prepublishOnly": "npm run build && npm test -- --runInBand --testPathIgnorePatterns='session-consistency|ink-ui-pty'"
+  },
+  "keywords": [
+    "agent",
+    "ai",
+    "coding-agent",
+    "cli-agent",
+    "llm",
+    "openai",
+    "claude",
+    "mcp",
+    "harness",
+    "automation",
+    "coding-assistant",
+    "developer-tools"
+  ],
+  "author": "Linux2010",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/orion-agents/orion-code.git"
+  },
+  "homepage": "https://github.com/orion-agents/orion-code#readme",
+  "bugs": {
+    "url": "https://github.com/orion-agents/orion-code/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@clack/prompts": "^0.7.0",
+    "@types/better-sqlite3": "^7.6.13",
+    "auto-bind": "^5.0.1",
+    "axios": "^1.6.0",
+    "better-sqlite3": "^12.10.0",
+    "chalk": "^4.1.2",
+    "commander": "^12.0.0",
+    "dotenv": "^16.6.1",
+    "eventemitter3": "^5.0.1",
+    "figlet": "^1.11.0",
+    "ink": "^3.2.0",
+    "js-yaml": "^4.1.1",
+    "lodash-es": "^4.18.1",
+    "marked": "^11.2.0",
+    "marked-terminal": "^6.2.0",
+    "openai": "^4.0.0",
+    "react": "^17.0.2",
+    "signal-exit": "^4.1.0",
+    "sqlite-vec": "^0.1.9",
+    "string-width": "^4.2.3",
+    "throttle": "^1.0.3",
+    "type-fest": "^5.6.0",
+    "uuid": "^9.0.0"
+  },
+  "devDependencies": {
+    "@types/figlet": "^1.5.8",
+    "@types/jest": "^30.0.0",
+    "@types/js-yaml": "^4.0.9",
+    "@types/node": "^20.0.0",
+    "@types/react": "^17.0.83",
+    "@types/uuid": "^9.0.0",
+    "@typescript-eslint/eslint-plugin": "^7.18.0",
+    "@typescript-eslint/parser": "^7.18.0",
+    "eslint": "^8.56.0",
+    "eslint-config-prettier": "^9.1.2",
+    "jest": "^29.7.0",
+    "nodemon": "^3.0.0",
+    "prettier": "^3.1.0",
+    "ts-jest": "^29.4.9",
+    "ts-node": "^10.9.0",
+    "typescript": "^5.3.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}