kodu 1.1.12 → 1.1.13

package/src/shared/constants.ts

@@ -0,0 +1,121 @@
+export const MAX_FILE_SIZE_BYTES = 1024 * 1024; // 1 MB
+export const WARNING_TOKEN_THRESHOLD = 12_000;
+export const DEFAULT_COMMIT_TOKENS = 1_500;
+export const DEFAULT_REVIEW_TOKENS = 5_000;
+export const DEFAULT_LLM_MODEL = 'gpt-4o';
+
+const BINARY_EXTENSION_LIST = [
+  '.png',
+  '.jpg',
+  '.jpeg',
+  '.webp',
+  '.gif',
+  '.bmp',
+  '.ico',
+  '.tif',
+  '.tiff',
+  '.psd',
+  '.ai',
+  '.sketch',
+  '.heic',
+  '.heif',
+  '.mp3',
+  '.wav',
+  '.flac',
+  '.ogg',
+  '.m4a',
+  '.mp4',
+  '.mkv',
+  '.mov',
+  '.avi',
+  '.webm',
+  '.wmv',
+  '.flv',
+  '.mpg',
+  '.mpeg',
+  '.ogv',
+  '.zip',
+  '.gz',
+  '.tgz',
+  '.bz2',
+  '.xz',
+  '.rar',
+  '.7z',
+  '.tar',
+  '.pdf',
+  '.exe',
+  '.dll',
+  '.so',
+  '.dylib',
+  '.class',
+  '.jar',
+  '.war',
+  '.ear',
+  '.ttf',
+  '.otf',
+  '.woff',
+  '.woff2',
+  '.eot',
+  '.bin',
+  '.pak',
+  '.dat',
+];
+
+const KNOWN_TEXT_LIST = [
+  // Web / JS
+  '.js',
+  '.jsx',
+  '.ts',
+  '.tsx',
+  '.mjs',
+  '.cjs',
+  '.json',
+  '.html',
+  '.css',
+  '.scss',
+  '.less',
+  '.vue',
+  '.svelte',
+  // Backend / System
+  '.java',
+  '.py',
+  '.c',
+  '.cpp',
+  '.h',
+  '.hpp',
+  '.cs',
+  '.go',
+  '.rs',
+  '.php',
+  '.rb',
+  '.swift',
+  '.kt',
+  '.dart',
+  '.scala',
+  '.pl',
+  '.lua',
+  '.sh',
+  '.bat',
+  // Data / Docs
+  '.md',
+  '.txt',
+  '.xml',
+  '.yaml',
+  '.yml',
+  '.sql',
+  '.graphql',
+  '.toml',
+  '.ini',
+  '.env',
+  // Config
+  '.gitignore',
+  '.dockerignore',
+  'dockerfile',
+  '.editorconfig',
+];
+
+const createLowercaseSet = (values: string[]): ReadonlySet<string> =>
+  new Set(values.map((value) => value.toLowerCase()));
+
+export const BINARY_EXTENSIONS = createLowercaseSet(BINARY_EXTENSION_LIST);
+export const KNOWN_TEXT_EXTENSIONS = createLowercaseSet(KNOWN_TEXT_LIST);

package/src/shared/tokenizer/tokenizer.service.ts

@@ -1,6 +1,7 @@
 import { Injectable } from '@nestjs/common';
-import { encodingForModel, type TiktokenModel } from 'js-tiktoken';
+import { encodingForModel, getEncoding, type TiktokenModel } from 'js-tiktoken';
 import { ConfigService } from '../../core/config/config.service';
+import { DEFAULT_LLM_MODEL } from '../constants';
 
 type TokenEstimate = {
   tokens: number;
@@ -8,6 +9,7 @@ type TokenEstimate = {
 };
 
 const PRICE_PER_MILLION: Record<string, number> = {
+  'gpt-4o': 10,
   'gpt-5-mini': 2.5,
 };
 
@@ -27,19 +29,16 @@ export class TokenizerService {
 
   private getPricePerMillion(): number {
     const config = this.configService.getConfig();
-    const model = config.llm?.model ?? 'gpt-5-mini';
-    const key = model.toLowerCase();
+    const model = config.llm?.model ?? `openai/${DEFAULT_LLM_MODEL}`;
+    const key = this.normalizeModelKey(model);
     return PRICE_PER_MILLION[key] ?? 0;
   }
 
   private createEncoder() {
-    const config = this.configService.getConfig();
-    const model = config.llm?.model ?? 'gpt-5-mini';
-
     try {
-      return encodingForModel(model as TiktokenModel);
+      return encodingForModel(DEFAULT_LLM_MODEL as TiktokenModel);
     } catch {
-      return encodingForModel('gpt-5-mini');
+      return getEncoding('o200k_base');
     }
   }
 
@@ -50,4 +49,12 @@ export class TokenizerService {
 
     return this.encoder;
   }
+
+  private normalizeModelKey(model: string): string {
+    const lower = model.toLowerCase();
+    if (lower.includes('/')) {
+      return lower.split('/').pop() ?? lower;
+    }
+    return lower;
+  }
 }