@ghenya/clinn 0.7.0

package/Tools/tokenizer.js

@@ -0,0 +1,150 @@
+const fs = require("fs");
+const path = require("path");
+
+let _wordFreq = {};
+let _totalFreq = 0;
+const _MIN_FREQ = 1;
+let _maxWordLen = 6;
+
+function _loadConfigWords() {
+  try {
+    const configPath = path.join(__dirname, "..", "config.json");
+    const cfg = JSON.parse(fs.readFileSync(configPath, "utf-8"));
+    const tz = cfg.tokenizer || {};
+    _maxWordLen = tz.maxWordLen || 6;
+    return tz.commonWords || [];
+  } catch (_) {
+    return [];
+  }
+}
+
+let _commonWords = _loadConfigWords();
+
+function _initBaseDict() {
+  _wordFreq = {};
+  for (const w of _commonWords) {
+    _wordFreq[w] = (_wordFreq[w] || 0) + 3;
+  }
+  _totalFreq = Object.values(_wordFreq).reduce((a, b) => a + b, 0);
+}
+
+_initBaseDict();
+
+function reloadConfig() {
+  _commonWords = _loadConfigWords();
+  _initBaseDict();
+}
+
+function buildDictFromKeywords(keywordsList) {
+  _initBaseDict();
+  for (const kws of keywordsList) {
+    for (const kw of kws) {
+      if (kw.length < 2) continue;
+      _wordFreq[kw] = (_wordFreq[kw] || 0) + 10;
+    }
+  }
+  _totalFreq = Object.values(_wordFreq).reduce((a, b) => a + b, 0);
+}
+
+function _getFreq(word) {
+  return _wordFreq[word] || _MIN_FREQ;
+}
+
+function _buildDAG(text) {
+  const dag = {};
+  const n = text.length;
+  for (let i = 0; i < n; i++) {
+    dag[i] = [];
+    let candidate = "";
+    for (let j = i; j < n && j - i < _maxWordLen; j++) {
+      candidate += text[j];
+      if (_wordFreq[candidate]) {
+        dag[i].push(j);
+      }
+    }
+    if (dag[i].length === 0) {
+      dag[i].push(i);
+    }
+  }
+  return dag;
+}
+
+function _calcRoute(text, dag) {
+  const n = text.length;
+  const route = {};
+  route[n] = { freq: 0, next: null };
+
+  function logProb(word) {
+    const f = _getFreq(word);
+    return Math.log(f > 0 ? f : _MIN_FREQ) - Math.log(_totalFreq);
+  }
+
+  for (let i = n - 1; i >= 0; i--) {
+    let bestFreq = -Infinity;
+    let bestEnd = i;
+    for (const j of dag[i]) {
+      const word = text.slice(i, j + 1);
+      const score = logProb(word) + route[j + 1].freq;
+      if (score > bestFreq) {
+        bestFreq = score;
+        bestEnd = j;
+      }
+    }
+    route[i] = { freq: bestFreq, next: bestEnd };
+  }
+
+  return route;
+}
+
+function _cutByRoute(text, route) {
+  const words = [];
+  let i = 0;
+  while (i < text.length) {
+    const end = route[i].next;
+    words.push(text.slice(i, end + 1));
+    i = end + 1;
+  }
+  return words;
+}
+
+function segment(text) {
+  if (!text) return [];
+
+  const tokens = [];
+  let buf = "";
+  let isChinese = null;
+
+  for (const ch of text) {
+    const isCh = /[\u4e00-\u9fa5]/.test(ch);
+    if (isChinese === null) {
+      isChinese = isCh;
+      buf = ch;
+    } else if (isCh === isChinese) {
+      buf += ch;
+    } else {
+      if (isChinese) {
+        const dag = _buildDAG(buf);
+        const route = _calcRoute(buf, dag);
+        tokens.push(..._cutByRoute(buf, route));
+      } else {
+        tokens.push(...buf.split(/[\s,，。.!！?？;；:：、\-\–—/\\|]+/).filter(Boolean));
+      }
+      buf = ch;
+      isChinese = isCh;
+    }
+  }
+
+  if (buf) {
+    if (isChinese) {
+      const dag = _buildDAG(buf);
+      const route = _calcRoute(buf, dag);
+      tokens.push(..._cutByRoute(buf, route));
+    } else {
+      tokens.push(...buf.split(/[\s,，。.!！?？;；:：、\-\–—/\\|]+/).filter(Boolean));
+    }
+  }
+
+  return tokens.filter((t) => t.length >= 1);
+}
+
+module.exports = { segment, buildDictFromKeywords, reloadConfig };

package/config.json

@@ -0,0 +1,251 @@
+{
+  "agent": {
+    "name": "Clinn",
+    "version": "0.7.0",
+    "description": "控制台智能体助手"
+  },
+  "llm": {
+    "provider": "deepseek",
+    "apiKey": "YOUR_DEEPSEEK_API_KEY_HERE",
+    "baseURL": "https://api.deepseek.com/v1",
+    "model": "deepseek-v4-pro",
+    "maxTokens": 8000,
+    "temperature": 0.7,
+    "topP": 0.9
+  },
+  "systemPrompt": "你是 Clinn，一个接管控制台的智能编程助手。核心铁则: 1) 每个任务完成后必须验证结果(用工具检查文件/命令输出), 2) 验证通过后明确告知用户完成情况。3) 绝不为节约token而偷工减料, 宁可多调用几轮工具也要确保任务完整正确。4) 每次完成任务后, 将用户偏好/操作习惯自动保存到记忆(save_memory, tags用 'habit')。5) 编写的可复用工具代码务必用 save_tool 持久化, 避免重复造轮子。6) 调用工具前后都精简上下文: 旧结果超过3轮的自动概括。7) 上下文即将超限时, 主动调用 forget_conversation 遗忘旧对话(摘要自动保存)或 restart_session 重新开始。8) 复杂任务用 todo_write 规划步骤。你可以: 文件操作(read/write/delete_file/move_file/copy_file/file_info/search_replace/edit_lines/read_lines)、目录(list_dir/ls/tree/glob — 优先用 tree 一次看完项目结构而非逐目录 ls)、搜索(grep/search_in_files/search_in_range/find_files)、终端(exec_console/wait_command/check_command_status)、网络(web_fetch/web_search/open_preview)、诊断(get_diagnostics)、记忆(search_memory/save_memory/list_memory/delete_memory/compress_context/forget_conversation/restart_session)、任务管理(todo_write/set_timer/skill)、工具管理(save_tool/delete_tool_file/list_saved_tools)。编写工具时可放心使用 node 内置模块(fs/path/https/http/child_process/crypto/os)。工具模块格式为 CommonJS, 持久化保存在 Tools/custom/ 目录下。面对任务时多轮迭代工具调用, 每次验证结果。需要查看全部工具时输入 'more' 或 '更多工具'。",
+  "tools": {
+    "enabled": true,
+    "permissionMode": "ask",
+    "trustedTools": [
+      "exec_console",
+      "edit_lines"
+    ]
+  },
+  "memory": {
+    "maxHistory": 100,
+    "maxEntries": 800,
+    "maxEntryChars": 200,
+    "autoCompressThreshold": 5000
+  },
+  "ui": {
+    "showLogo": true,
+    "logoPath": "./Logos/StartLogo.txt",
+    "dividerChar": "-"
+  },
+  "tokenizer": {
+    "maxWordLen": 6,
+    "commonWords": [
+      "的",
+      "了",
+      "在",
+      "是",
+      "我",
+      "有",
+      "和",
+      "就",
+      "不",
+      "人",
+      "都",
+      "一",
+      "一个",
+      "上",
+      "也",
+      "很",
+      "到",
+      "说",
+      "要",
+      "去",
+      "你",
+      "会",
+      "着",
+      "没有",
+      "看",
+      "好",
+      "自己",
+      "这",
+      "他",
+      "她",
+      "它",
+      "们",
+      "那",
+      "些",
+      "什么",
+      "怎么",
+      "如何",
+      "哪个",
+      "为什么",
+      "可以",
+      "需要",
+      "应该",
+      "能够",
+      "已经",
+      "这个",
+      "那个",
+      "请",
+      "帮",
+      "用",
+      "把",
+      "被",
+      "从",
+      "对",
+      "与",
+      "或",
+      "但",
+      "而",
+      "所以",
+      "因为",
+      "如果",
+      "虽然",
+      "然后",
+      "之后",
+      "之前",
+      "现在",
+      "以后",
+      "以上",
+      "以下",
+      "等等",
+      "文件",
+      "代码",
+      "搜索",
+      "读取",
+      "写入",
+      "编辑",
+      "删除",
+      "创建",
+      "执行",
+      "运行",
+      "命令",
+      "查找",
+      "复制",
+      "移动",
+      "保存",
+      "打开",
+      "关闭",
+      "工具",
+      "模块",
+      "目录",
+      "路径",
+      "内容",
+      "结果",
+      "输出",
+      "输入",
+      "错误",
+      "配置",
+      "设置",
+      "修改",
+      "检查",
+      "验证",
+      "测试",
+      "安装",
+      "参数",
+      "函数",
+      "方法",
+      "数据",
+      "项目",
+      "仓库",
+      "终端",
+      "网页",
+      "记忆",
+      "定时器",
+      "抓取",
+      "持久化",
+      "系统",
+      "环境",
+      "查看",
+      "当前",
+      "所有",
+      "全部",
+      "临时",
+      "清理",
+      "缓存",
+      "构建",
+      "编译",
+      "更新",
+      "下载",
+      "上传",
+      "备份",
+      "恢复",
+      "重启",
+      "停止",
+      "启动",
+      "生成",
+      "导出",
+      "导入",
+      "解析",
+      "处理",
+      "统计",
+      "分析",
+      "替换",
+      "追加",
+      "插入",
+      "重命名",
+      "新建",
+      "初始化",
+      "格式化",
+      "遍历",
+      "递归",
+      "匹配",
+      "过滤",
+      "排序",
+      "合并",
+      "拆分",
+      "转换",
+      "连接",
+      "断开",
+      "同步",
+      "异步",
+      "等待",
+      "监听",
+      "响应",
+      "请求",
+      "类型",
+      "结构",
+      "对象",
+      "数组",
+      "字符串",
+      "数字",
+      "布尔",
+      "空值",
+      "属性",
+      "索引",
+      "长度",
+      "版本",
+      "名称",
+      "作者",
+      "日期",
+      "大小",
+      "窗口",
+      "进程",
+      "线程",
+      "服务",
+      "端口",
+      "地址",
+      "日志",
+      "调试",
+      "提示",
+      "警告",
+      "信息",
+      "帮助",
+      "文档",
+      "说明",
+      "示例",
+      "注释",
+      "状态",
+      "进度",
+      "完成",
+      "失败",
+      "成功",
+      "冲突",
+      "忽略",
+      "跳过",
+      "继续",
+      "终止",
+      "取消",
+      "确认",
+      "选择",
+      "显示",
+      "隐藏"
+    ]
+  }
+}

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@ghenya/clinn",
+  "version": "0.7.0",
+  "description": "终端原生 AI 编程助手 — DeepSeek 驱动，50+ 工具，对话记忆，虚拟浏览器",
+  "main": "Src/index.js",
+  "bin": {
+    "clinn": "Src/index.js"
+  },
+  "files": [
+    "Src/",
+    "Tools/",
+    "Mem/",
+    "Logos/",
+    "config.json",
+    "README.md"
+  ],
+  "scripts": {
+    "start": "node Src/index.js",
+    "lint": "node --check Src/index.js && node --check Src/agent.js && node --check Src/llm.js && node --check Tools/index.js && node --check Tools/extended_tools.js && node --check Tools/search_tools.js && node --check Tools/browser.js && node --check Mem/index.js && node --check Mem/history.js"
+  },
+  "keywords": [
+    "ai",
+    "agent",
+    "cli",
+    "deepseek",
+    "terminal",
+    "coding-assistant",
+    "llm",
+    "chatbot"
+  ],
+  "author": "PillowBots",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/PillowBots/clinn"
+  },
+  "homepage": "https://github.com/PillowBots/clinn#readme",
+  "bugs": {
+    "url": "https://github.com/PillowBots/clinn/issues"
+  },
+  "type": "commonjs",
+  "engines": {
+    "node": ">=16.0.0"
+  },
+  "dependencies": {
+    "puppeteer-core": "^25.1.0"
+  }
+}