foliko 1.0.87 → 1.1.1

package/calc_tokens_weixin.js

@@ -0,0 +1,81 @@
+const fs = require('fs');
+
+function encode(text) {
+  if (!text || typeof text !== 'string') return 0;
+  const cleanText = text.replace(/\0+/g, '').replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '').slice(0, 100000);
+  if (!cleanText) return 0;
+  const chineseChars = (cleanText.match(/[\u4e00-\u9fff\u3400-\u4dbf]/g) || []).length;
+  const englishChars = cleanText.length - chineseChars;
+  const chineseTokens = chineseChars / 2;
+  const englishTokens = englishChars / 4;
+  return Math.ceil(chineseTokens + englishTokens);
+}
+
+const filePath = 'D:/code/vb-agent/.agent/sessions/weixin_o9cq80zgZqKPA2-s59PN43GdDy1w@im.wechat.json';
+const data = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+
+let totalTokens = 0;
+let totalChars = 0;
+let largeMessages = [];
+const messages = data.messages || [];
+
+for (let i = 0; i < messages.length; i++) {
+  const msg = messages[i];
+  let msgTokens = 4;
+  let msgChars = 0;
+
+  if (typeof msg.content === 'string') {
+    msgTokens += encode(msg.content);
+    msgChars += msg.content.length;
+  } else if (Array.isArray(msg.content)) {
+    for (const part of msg.content) {
+      if (!part || typeof part !== 'object') continue;
+      if (part.text) {
+        const text = String(part.text);
+        msgTokens += encode(text);
+        msgChars += text.length;
+      }
+      if (part.type === 'tool-result' && part.output) {
+        let outputText = '';
+        if (typeof part.output === 'string') {
+          outputText = part.output;
+        } else if (part.output && part.output.value) {
+          outputText = String(part.output.value);
+        } else if (part.output && part.output.html) {
+          outputText = part.output.html;
+        } else if (part.output) {
+          outputText = JSON.stringify(part.output);
+        }
+        msgTokens += encode(outputText);
+        msgChars += outputText.length;
+      }
+      if (part.type === 'tool_use' && part.input) {
+        const inputStr = JSON.stringify(part.input);
+        msgTokens += encode(inputStr);
+        msgChars += inputStr.length;
+      }
+    }
+  } else if (msg.content && typeof msg.content === 'object') {
+    try {
+      const str = JSON.stringify(msg.content);
+      msgTokens += encode(str);
+      msgChars += str.length;
+    } catch (e) {}
+  }
+
+  totalTokens += msgTokens;
+  totalChars += msgChars;
+
+  if (msgTokens > 3000) {
+    largeMessages.push({ index: i, role: msg.role, tokens: msgTokens, chars: msgChars });
+  }
+}
+
+console.log('Messages:', messages.length);
+console.log('Total chars:', totalChars);
+console.log('Total tokens (approx):', totalTokens);
+console.log('File size:', fs.statSync(filePath).size, 'bytes');
+console.log('\nLarge messages (>3000 tokens):');
+largeMessages.forEach(m => {
+  console.log(`  [${m.index}] ${m.role}: ${m.tokens} tokens, ${m.chars} chars`);
+});

package/cli/src/commands/chat.js

@@ -94,7 +94,8 @@ async function chatCommand(args) {
       baseURL: options.baseURL,
       apiKey: options.apiKey,
       providerOptions: {
-        maxOutputTokens: 8192,
+        maxOutputTokens: 16384,
+        reasoning_split: true,
       },
     },
   });