@character-foundry/character-foundry 0.1.3 → 0.1.5

package/dist/tokenizers.js

@@ -1,2 +1,130 @@
-export * from '@character-foundry/tokenizers';
+// ../tokenizers/dist/index.js
+import { encode } from "gpt-tokenizer";
+var GptTokenizer = class {
+  id = "gpt-4";
+  name = "GPT-4 (cl100k_base)";
+  count(text) {
+    if (!text) return 0;
+    try {
+      return encode(text).length;
+    } catch (error) {
+      console.warn("GPT tokenization failed, falling back to approximation", error);
+      return Math.ceil(text.length / 4);
+    }
+  }
+  countMany(texts) {
+    return texts.map((t) => this.count(t));
+  }
+  encode(text) {
+    return encode(text);
+  }
+};
+var SimpleTokenizer = class {
+  id = "simple";
+  name = "Simple Approximation";
+  count(text) {
+    if (!text) return 0;
+    const words = text.split(/\s+/).filter((w) => w.length > 0);
+    const charCount = text.length;
+    return Math.ceil(charCount / 4 + words.length * 0.3);
+  }
+  countMany(texts) {
+    return texts.map((t) => this.count(t));
+  }
+};
+var LlamaTokenizer = class {
+  id = "llama";
+  name = "LLaMA Approximation";
+  count(text) {
+    if (!text) return 0;
+    return Math.ceil(text.length / 4.5);
+  }
+  countMany(texts) {
+    return texts.map((t) => this.count(t));
+  }
+};
+var TokenizerRegistry = class {
+  tokenizers = /* @__PURE__ */ new Map();
+  defaultId = "gpt-4";
+  constructor() {
+    this.register(new GptTokenizer());
+    this.register(new SimpleTokenizer());
+    this.register(new LlamaTokenizer());
+  }
+  register(tokenizer) {
+    this.tokenizers.set(tokenizer.id, tokenizer);
+  }
+  get(id) {
+    const targetId = id || this.defaultId;
+    const tokenizer = this.tokenizers.get(targetId);
+    if (!tokenizer) {
+      return this.tokenizers.get("simple") || new SimpleTokenizer();
+    }
+    return tokenizer;
+  }
+  list() {
+    return Array.from(this.tokenizers.values());
+  }
+};
+var registry = new TokenizerRegistry();
+function countTokens(text, tokenizerId) {
+  return registry.get(tokenizerId).count(text);
+}
+function getTokenizer(tokenizerId) {
+  return registry.get(tokenizerId);
+}
+function countCardTokens(card, options = {}) {
+  const tokenizer = getTokenizer(options.tokenizer ?? "gpt-4");
+  const onlyEnabled = options.onlyEnabledLorebook ?? true;
+  const data = card.data ?? card;
+  const description = tokenizer.count(data.description ?? "");
+  const personality = tokenizer.count(data.personality ?? "");
+  const scenario = tokenizer.count(data.scenario ?? "");
+  const firstMes = tokenizer.count(data.first_mes ?? "");
+  const mesExample = tokenizer.count(data.mes_example ?? "");
+  const systemPrompt = tokenizer.count(data.system_prompt ?? "");
+  const postHistoryInstructions = tokenizer.count(data.post_history_instructions ?? "");
+  const creatorNotes = tokenizer.count(data.creator_notes ?? "");
+  const altGreetings = data.alternate_greetings ?? [];
+  const alternateGreetings = altGreetings.reduce(
+    (sum, greeting) => sum + tokenizer.count(greeting ?? ""),
+    0
+  );
+  const entries = data.character_book?.entries ?? [];
+  const lorebook = entries.reduce((sum, entry) => {
+    if (onlyEnabled && entry.enabled === false) {
+      return sum;
+    }
+    return sum + tokenizer.count(entry.content ?? "");
+  }, 0);
+  const total = description + personality + scenario + firstMes + mesExample + systemPrompt + postHistoryInstructions + alternateGreetings + lorebook + creatorNotes;
+  return {
+    description,
+    personality,
+    scenario,
+    firstMes,
+    mesExample,
+    systemPrompt,
+    postHistoryInstructions,
+    alternateGreetings,
+    lorebook,
+    creatorNotes,
+    total
+  };
+}
+function countText(text, tokenizerId = "gpt-4") {
+  const tokenizer = getTokenizer(tokenizerId);
+  return tokenizer.count(text);
+}
+export {
+  GptTokenizer,
+  LlamaTokenizer,
+  SimpleTokenizer,
+  TokenizerRegistry,
+  countCardTokens,
+  countText,
+  countTokens,
+  getTokenizer,
+  registry
+};
 //# sourceMappingURL=tokenizers.js.map

package/dist/tokenizers.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizers.js","sourceRoot":"","sources":["../src/tokenizers.ts"],"names":[],"mappings":"AAAA,cAAc,+BAA+B,CAAC"}
+{"version":3,"sources":["../../tokenizers/src/gpt.ts","../../tokenizers/src/simple.ts","../../tokenizers/src/registry.ts","../../tokenizers/src/card-counter.ts"],"sourcesContent":["import { encode } from 'gpt-tokenizer';\nimport type { TokenizerAdapter } from './types.js';\n\n/**\n * GPT-3.5/4 compatible tokenizer using cl100k_base encoding\n * Uses gpt-tokenizer (pure JS)\n */\nexport class GptTokenizer implements TokenizerAdapter {\n  readonly id = 'gpt-4';\n  readonly name = 'GPT-4 (cl100k_base)';\n\n  count(text: string): number {\n    if (!text) return 0;\n    try {\n      return encode(text).length;\n    } catch (error) {\n      console.warn('GPT tokenization failed, falling back to approximation', error);\n      return Math.ceil(text.length / 4);\n    }\n  }\n\n  countMany(texts: string[]): number[] {\n    return texts.map(t => this.count(t));\n  }\n\n  encode(text: string): number[] {\n    return encode(text);\n  }\n}\n","import type { TokenizerAdapter } from './types.js';\n\n/**\n * Simple BPE-style tokenizer approximation (roughly GPT-2)\n * Useful when exact counts aren't critical or for performance\n */\nexport class SimpleTokenizer implements TokenizerAdapter {\n  readonly id = 'simple';\n  readonly name = 'Simple Approximation';\n\n  count(text: string): number {\n    if (!text) return 0;\n    // Rough approximation: ~4 chars per token, but words count as tokens\n    const words = text.split(/\\s+/).filter((w) => w.length > 0);\n    const charCount = text.length;\n    return Math.ceil(charCount / 4 + words.length * 0.3);\n  }\n\n  countMany(texts: string[]): number[] {\n    return texts.map(t => this.count(t));\n  }\n}\n\n/**\n * LLaMA-style tokenizer approximation\n */\nexport class LlamaTokenizer implements TokenizerAdapter {\n  readonly id = 'llama';\n  readonly name = 'LLaMA Approximation';\n\n  count(text: string): number {\n    if (!text) return 0;\n    return Math.ceil(text.length / 4.5);\n  }\n\n  countMany(texts: string[]): number[] {\n    return texts.map(t => this.count(t));\n  }\n}\n","import type { TokenizerAdapter } from './types.js';\nimport { GptTokenizer } from './gpt.js';\nimport { SimpleTokenizer, LlamaTokenizer } from './simple.js';\n\nexport class TokenizerRegistry {\n  private tokenizers = new Map<string, TokenizerAdapter>();\n  private defaultId: string = 'gpt-4';\n\n  constructor() {\n    this.register(new GptTokenizer());\n    this.register(new SimpleTokenizer());\n    this.register(new LlamaTokenizer());\n  }\n\n  register(tokenizer: TokenizerAdapter): void {\n    this.tokenizers.set(tokenizer.id, tokenizer);\n  }\n\n  get(id?: string): TokenizerAdapter {\n    const targetId = id || this.defaultId;\n    const tokenizer = this.tokenizers.get(targetId);\n    if (!tokenizer) {\n      // Fallback to simple if requested ID not found\n      return this.tokenizers.get('simple') || new SimpleTokenizer();\n    }\n    return tokenizer;\n  }\n\n  list(): TokenizerAdapter[] {\n    return Array.from(this.tokenizers.values());\n  }\n}\n\nexport const registry = new TokenizerRegistry();\n\n// Helper for quick usage\nexport function countTokens(text: string, tokenizerId?: string): number {\n  return registry.get(tokenizerId).count(text);\n}\n\n// Helper to get a tokenizer by ID\nexport function getTokenizer(tokenizerId?: string): TokenizerAdapter {\n  return registry.get(tokenizerId);\n}\n","/**\n * Card Token Counter\n *\n * Count tokens in character card fields for consistent\n * cross-platform token reporting.\n */\n\nimport type { TokenizerAdapter } from './types.js';\nimport { getTokenizer } from './registry.js';\n\n/**\n * Token counts for a character card\n */\nexport interface CardTokenCounts {\n  description: number;\n  personality: number;\n  scenario: number;\n  firstMes: number;\n  mesExample: number;\n  systemPrompt: number;\n  postHistoryInstructions: number;\n  alternateGreetings: number;\n  lorebook: number;\n  creatorNotes: number;\n  total: number;\n}\n\n/**\n * Card data structure for token counting\n * Accepts both CCv2 and CCv3 card structures\n */\nexport interface CardForCounting {\n  data?: {\n    name?: string;\n    description?: string;\n    personality?: string | null;\n    scenario?: string;\n    first_mes?: string;\n    mes_example?: string | null;\n    system_prompt?: string;\n    post_history_instructions?: string;\n    alternate_greetings?: string[];\n    creator_notes?: string;\n    character_book?: {\n      entries?: Array<{\n        content?: string;\n        enabled?: boolean;\n      }>;\n    } | null;\n  };\n  // Support for unwrapped cards\n  name?: string;\n  description?: string;\n  personality?: string | null;\n  scenario?: string;\n  first_mes?: string;\n  mes_example?: string | null;\n  system_prompt?: string;\n  post_history_instructions?: string;\n  alternate_greetings?: string[];\n  creator_notes?: string;\n  character_book?: {\n    entries?: Array<{\n      content?: string;\n      enabled?: boolean;\n    }>;\n  } | null;\n}\n\n/**\n * Token counting options\n */\nexport interface TokenCountOptions {\n  /** Tokenizer ID to use. Default: 'gpt-4' */\n  tokenizer?: string;\n  /** Only count enabled lorebook entries. Default: true */\n  onlyEnabledLorebook?: boolean;\n}\n\n/**\n * Count tokens in a character card's fields.\n *\n * @param card - Character card data (CCv2 or CCv3)\n * @param options - Counting options\n * @returns Token counts per field and total\n */\nexport function countCardTokens(\n  card: CardForCounting,\n  options: TokenCountOptions = {}\n): CardTokenCounts {\n  const tokenizer = getTokenizer(options.tokenizer ?? 'gpt-4');\n  const onlyEnabled = options.onlyEnabledLorebook ?? true;\n\n  // Handle both wrapped and unwrapped formats\n  const data = card.data ?? card;\n\n  // Count each field\n  const description = tokenizer.count(data.description ?? '');\n  const personality = tokenizer.count(data.personality ?? '');\n  const scenario = tokenizer.count(data.scenario ?? '');\n  const firstMes = tokenizer.count(data.first_mes ?? '');\n  const mesExample = tokenizer.count(data.mes_example ?? '');\n  const systemPrompt = tokenizer.count(data.system_prompt ?? '');\n  const postHistoryInstructions = tokenizer.count(data.post_history_instructions ?? '');\n  const creatorNotes = tokenizer.count(data.creator_notes ?? '');\n\n  // Count alternate greetings\n  const altGreetings = data.alternate_greetings ?? [];\n  const alternateGreetings = altGreetings.reduce(\n    (sum, greeting) => sum + tokenizer.count(greeting ?? ''),\n    0\n  );\n\n  // Count lorebook entries\n  const entries = data.character_book?.entries ?? [];\n  const lorebook = entries.reduce((sum, entry) => {\n    if (onlyEnabled && entry.enabled === false) {\n      return sum;\n    }\n    return sum + tokenizer.count(entry.content ?? '');\n  }, 0);\n\n  // Calculate total\n  const total =\n    description +\n    personality +\n    scenario +\n    firstMes +\n    mesExample +\n    systemPrompt +\n    postHistoryInstructions +\n    alternateGreetings +\n    lorebook +\n    creatorNotes;\n\n  return {\n    description,\n    personality,\n    scenario,\n    firstMes,\n    mesExample,\n    systemPrompt,\n    postHistoryInstructions,\n    alternateGreetings,\n    lorebook,\n    creatorNotes,\n    total,\n  };\n}\n\n/**\n * Count tokens in a text string using the specified tokenizer.\n *\n * @param text - Text to count tokens in\n * @param tokenizerId - Tokenizer ID to use. Default: 'gpt-4'\n * @returns Token count\n */\nexport function countText(text: string, tokenizerId = 'gpt-4'): number {\n  const tokenizer = getTokenizer(tokenizerId);\n  return tokenizer.count(text);\n}\n"],"mappings":";AAAA,SAAS,cAAc;AAOhB,IAAM,eAAN,MAA+C;EAC3C,KAAK;EACL,OAAO;EAEhB,MAAM,MAAsB;AAC1B,QAAI,CAAC,KAAM,QAAO;AAClB,QAAI;AACF,aAAO,OAAO,IAAI,EAAE;IACtB,SAAS,OAAO;AACd,cAAQ,KAAK,0DAA0D,KAAK;AAC5E,aAAO,KAAK,KAAK,KAAK,SAAS,CAAC;IAClC;EACF;EAEA,UAAU,OAA2B;AACnC,WAAO,MAAM,IAAI,CAAA,MAAK,KAAK,MAAM,CAAC,CAAC;EACrC;EAEA,OAAO,MAAwB;AAC7B,WAAO,OAAO,IAAI;EACpB;AACF;ACtBO,IAAM,kBAAN,MAAkD;EAC9C,KAAK;EACL,OAAO;EAEhB,MAAM,MAAsB;AAC1B,QAAI,CAAC,KAAM,QAAO;AAElB,UAAM,QAAQ,KAAK,MAAM,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AAC1D,UAAM,YAAY,KAAK;AACvB,WAAO,KAAK,KAAK,YAAY,IAAI,MAAM,SAAS,GAAG;EACrD;EAEA,UAAU,OAA2B;AACnC,WAAO,MAAM,IAAI,CAAA,MAAK,KAAK,MAAM,CAAC,CAAC;EACrC;AACF;AAKO,IAAM,iBAAN,MAAiD;EAC7C,KAAK;EACL,OAAO;EAEhB,MAAM,MAAsB;AAC1B,QAAI,CAAC,KAAM,QAAO;AAClB,WAAO,KAAK,KAAK,KAAK,SAAS,GAAG;EACpC;EAEA,UAAU,OAA2B;AACnC,WAAO,MAAM,IAAI,CAAA,MAAK,KAAK,MAAM,CAAC,CAAC;EACrC;AACF;AClCO,IAAM,oBAAN,MAAwB;EACrB,aAAa,oBAAI,IAA8B;EAC/C,YAAoB;EAE5B,cAAc;AACZ,SAAK,SAAS,IAAI,aAAa,CAAC;AAChC,SAAK,SAAS,IAAI,gBAAgB,CAAC;AACnC,SAAK,SAAS,IAAI,eAAe,CAAC;EACpC;EAEA,SAAS,WAAmC;AAC1C,SAAK,WAAW,IAAI,UAAU,IAAI,SAAS;EAC7C;EAEA,IAAI,IAA+B;AACjC,UAAM,WAAW,MAAM,KAAK;AAC5B,UAAM,YAAY,KAAK,WAAW,IAAI,QAAQ;AAC9C,QAAI,CAAC,WAAW;AAEd,aAAO,KAAK,WAAW,IAAI,QAAQ,KAAK,IAAI,gBAAgB;IAC9D;AACA,WAAO;EACT;EAEA,OAA2B;AACzB,WAAO,MAAM,KAAK,KAAK,WAAW,OAAO,CAAC;EAC5C;AACF;AAEO,IAAM,WAAW,IAAI,kBAAkB;AAGvC,SAAS,YAAY,MAAc,aAA8B;AACtE,SAAO,SAAS,IAAI,WAAW,EAAE,MAAM,IAAI;AAC7C;AAGO,SAAS,aAAa,aAAwC;AACnE,SAAO,SAAS,IAAI,WAAW;AACjC;AC2CO,SAAS,gBACd,MACA,UAA6B,CAAC,GACb;AACjB,QAAM,YAAY,aAAa,QAAQ,aAAa,OAAO;AAC3D,QAAM,cAAc,QAAQ,uBAAuB;AAGnD,QAAM,OAAO,KAAK,QAAQ;AAG1B,QAAM,cAAc,UAAU,MAAM,KAAK,eAAe,EAAE;AAC1D,QAAM,cAAc,UAAU,MAAM,KAAK,eAAe,EAAE;AAC1D,QAAM,WAAW,UAAU,MAAM,KAAK,YAAY,EAAE;AACpD,QAAM,WAAW,UAAU,MAAM,KAAK,aAAa,EAAE;AACrD,QAAM,aAAa,UAAU,MAAM,KAAK,eAAe,EAAE;AACzD,QAAM,eAAe,UAAU,MAAM,KAAK,iBAAiB,EAAE;AAC7D,QAAM,0BAA0B,UAAU,MAAM,KAAK,6BAA6B,EAAE;AACpF,QAAM,eAAe,UAAU,MAAM,KAAK,iBAAiB,EAAE;AAG7D,QAAM,eAAe,KAAK,uBAAuB,CAAC;AAClD,QAAM,qBAAqB,aAAa;IACtC,CAAC,KAAK,aAAa,MAAM,UAAU,MAAM,YAAY,EAAE;IACvD;EACF;AAGA,QAAM,UAAU,KAAK,gBAAgB,WAAW,CAAC;AACjD,QAAM,WAAW,QAAQ,OAAO,CAAC,KAAK,UAAU;AAC9C,QAAI,eAAe,MAAM,YAAY,OAAO;AAC1C,aAAO;IACT;AACA,WAAO,MAAM,UAAU,MAAM,MAAM,WAAW,EAAE;EAClD,GAAG,CAAC;AAGJ,QAAM,QACJ,cACA,cACA,WACA,WACA,aACA,eACA,0BACA,qBACA,WACA;AAEF,SAAO;IACL;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;EACF;AACF;AASO,SAAS,UAAU,MAAc,cAAc,SAAiB;AACrE,QAAM,YAAY,aAAa,WAAW;AAC1C,SAAO,UAAU,MAAM,IAAI;AAC7B;","names":[]}