@hupan56/wlkj 2.6.0 → 2.7.0

package/templates/qoder/scripts/learn.py

@@ -1,67 +1,146 @@
-# learn.py - Learning engine for QODER evolution
-import os, json
+# -*- coding: utf-8 -*-
+"""learn.py - Learning engine: 个人埋点 (workspace/members/{dev}/journal/)
+
+数据归属 (和 kg.duckdb 对称的设计):
+  - 原始事件流 → workspace/members/{dev}/journal/feedback.jsonl  (个人, 进git, 按人分不冲突)
+  - 统计计数   → workspace/members/{dev}/journal/learning.yaml    (个人, 进git)
+  - 团队聚合   → data/index/kg.duckdb 的 learning_events 表       (管理员跑 learn_aggregate.py 聚合)
+
+为什么按人分文件:
+  team_sync 用单分支 rebase 协作, 所有人 append 同一个 feedback.jsonl 会冲突。
+  每人写自己的文件 → rebase 时改的是不同文件 → git 自动合并, 零冲突。
+  (旧版把 feedback 放 .qoder/learning/ 并 gitignore, 导致数据不随团队同步 = 丢失)
+
+用法:
+  python learn.py record prd_accepted '{"req_id":"REQ-2026-009","score":87}'
+  python learn.py status
+  python learn.py aggregate          # 管理员: 扫所有人 → kg.duckdb
+"""
+import os
+import sys
+import json
 from datetime import datetime
 
-BASE = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-LEARNING_DIR = os.path.join(BASE, '.qoder', 'learning')
-FEEDBACK_FILE = os.path.join(LEARNING_DIR, 'feedback.jsonl')
-PREFS_FILE = os.path.join(LEARNING_DIR, 'preferences.yaml')
+SCRIPTS = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, SCRIPTS)
+
+from common.paths import get_developer, get_developer_journal_dir
+
 
-def record_feedback(event, data):
+def _feedback_path():
+    """当前开发者的 feedback 文件 (workspace/members/{dev}/journal/feedback.jsonl)。"""
+    dev = get_developer()
+    if not dev:
+        raise RuntimeError('未初始化开发者身份, 请先运行 /wl-init')
+    return get_developer_journal_dir(dev) / 'feedback.jsonl'
+
+
+def _prefs_path():
+    """当前开发者的统计文件 (workspace/members/{dev}/journal/learning.yaml)。"""
+    dev = get_developer()
+    if not dev:
+        raise RuntimeError('未初始化开发者身份, 请先运行 /wl-init')
+    return get_developer_journal_dir(dev) / 'learning.yaml'
+
+
+def record_feedback(event, data=None):
+    """记录一条事件到当前开发者的 feedback.jsonl。
+    每条带 dev 字段 (聚合时区分来源)。
+    """
+    dev = get_developer()
+    if not dev:
+        raise RuntimeError('未初始化开发者身份, 请先运行 /wl-init')
+    if data is None:
+        data = {}
     entry = {
-        'ts': datetime.now().isoformat(),
+        'ts': datetime.now().isoformat(timespec='seconds'),
+        'dev': dev,
         'event': event,
-        'data': data
+        'data': data,
     }
-    with open(FEEDBACK_FILE, 'a', encoding='utf-8') as f:
-        f.write(json.dumps(entry, ensure_ascii=False) + chr(10))
+    fb_path = _feedback_path()
+    fb_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(fb_path, 'a', encoding='utf-8') as f:
+        f.write(json.dumps(entry, ensure_ascii=False) + '\n')
     _update_stats(event)
-    print('Recorded: ' + event)
+    print('Recorded: %s → %s' % (event, fb_path.name))
+
 
 def _update_stats(event):
-    import yaml
+    """更新当前开发者的统计计数 (learning.yaml)。
+    事件→列名映射来自 common/events.py (SSOT), 不再硬编码。
+    """
+    try:
+        import yaml
+    except ImportError:
+        return  # 无 yaml 库则跳过统计, 不阻塞埋点
+    from common.events import stats_event_map
+    key = stats_event_map().get(event)
+    if not key:
+        return
     prefs = _load_prefs()
-    stats = {
-        'prd_accepted': 'prds_generated',
-        'prd_rejected': 'rejections',
-        'review_done': 'reviews_done',
-        'task_completed': 'tasks_completed',
-        'handoff_done': 'handoffs_completed',
-    }
-    key = stats.get(event)
-    if key:
-        prefs[key] = prefs.get(key, 0) + 1
-        _save_prefs(prefs)
+    prefs[key] = prefs.get(key, 0) + 1
+    _save_prefs(prefs)
+
 
 def _load_prefs():
-    import yaml
-    if not os.path.isfile(PREFS_FILE): return {}
-    try: return yaml.safe_load(open(PREFS_FILE, encoding='utf-8')) or {}
-    except: return {}
+    try:
+        import yaml
+    except ImportError:
+        return {}
+    p = _prefs_path()
+    if not p.is_file():
+        return {}
+    try:
+        return yaml.safe_load(p.read_text(encoding='utf-8')) or {}
+    except Exception:
+        return {}
+
 
 def _save_prefs(prefs):
-    import yaml
-    with open(PREFS_FILE, 'w', encoding='utf-8') as f:
+    try:
+        import yaml
+    except ImportError:
+        return
+    p = _prefs_path()
+    p.parent.mkdir(parents=True, exist_ok=True)
+    with open(p, 'w', encoding='utf-8') as f:
         yaml.dump(prefs, f, allow_unicode=True, default_flow_style=False)
 
+
 def get_prefs():
     return _load_prefs()
 
+
 def should_evolve():
-    prefs = _load_prefs()
-    tc = prefs.get('tasks_completed', 0)
-    return tc > 0 and tc % 10 == 0
+    """是否到了该提炼模式的阈值 (每完成 10 个任务)。"""
+    return _load_prefs().get('tasks_completed', 0) % 10 == 0
+
+
+def aggregate_all():
+    """管理员聚合: 扫所有人的 feedback.jsonl → 写入 kg.duckdb。
+    委托给 learn_aggregate.py (同构 kg_build.py)。
+    """
+    import learn_aggregate
+    return learn_aggregate.aggregate()
+
 
 if __name__ == '__main__':
-    import sys
     if len(sys.argv) < 2:
-        print('Usage: learn.py record <event> [json] | learn.py status')
+        print(__doc__)
+        print('用法: learn.py record <event> [json] | learn.py status | learn.py aggregate')
     elif sys.argv[1] == 'status':
         prefs = get_prefs()
-        print('Learning Stats:')
+        print('Learning Stats (%s):' % (get_developer() or 'unknown'))
         for k, v in prefs.items():
-            if isinstance(v, int): print('  ' + k + ': ' + str(v))
-        print('  Evolve ready: ' + str(should_evolve()))
+            if isinstance(v, int):
+                print('  %s: %d' % (k, v))
+        print('  Evolve ready: %s' % should_evolve())
+    elif sys.argv[1] == 'aggregate':
+        n = aggregate_all()
+        print('聚合完成: %d 条事件 → kg.duckdb' % n)
     elif sys.argv[1] == 'record' and len(sys.argv) >= 3:
         data = json.loads(sys.argv[3]) if len(sys.argv) > 3 else {}
-        record_feedback(sys.argv[2], data)
+        record_feedback(sys.argv[2], data)
+    else:
+        print('未知子命令: %s' % sys.argv[1])

package/templates/qoder/scripts/learn_aggregate.py

@@ -0,0 +1,201 @@
+# -*- coding: utf-8 -*-
+"""learn_aggregate.py - 团队学习聚合 (同构 kg_build.py)
+
+扫所有人的 workspace/members/{dev}/journal/feedback.jsonl
+→ 增量写入 kg.duckdb 的 learning_events 表 (团队单一真源)
+
+和 kg_build.py 对称:
+  kg_build.py:      扫 data/code/* → 写 kg.duckdb (代码图谱)
+  learn_aggregate.py: 扫 workspace/members/*/journal → 写 kg.duckdb (学习图谱)
+
+增量策略 (复用 build_meta):
+  记录上次聚合时间, 只处理 mtime > last 的文件。全量重建用 --rebuild。
+
+用法:
+  python learn_aggregate.py            # 增量
+  python learn_aggregate.py --rebuild  # 全量重建
+"""
+import os
+import sys
+import json
+import hashlib
+import time
+from datetime import datetime
+
+SCRIPTS = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, SCRIPTS)
+
+from common.paths import MEMBERS_DIR
+from kg_duckdb import get_db, init_schema
+
+BUILD_META_KEY = 'learning_last_aggregate'
+
+
+def _collect_journal_files():
+    """扫 workspace/members/*/journal/feedback.jsonl, 返回 [(dev, path)]。"""
+    results = []
+    if not MEMBERS_DIR.is_dir():
+        return results
+    for dev_dir in sorted(MEMBERS_DIR.iterdir()):
+        if not dev_dir.is_dir():
+            continue
+        fb = dev_dir / 'journal' / 'feedback.jsonl'
+        if fb.is_file():
+            results.append((dev_dir.name, fb))
+    return results
+
+
+def _event_id(entry):
+    """生成事件唯一 ID (去重用): hash(ts+dev+event+data)。"""
+    raw = entry.get('ts', '') + '|' + entry.get('dev', '') + '|' + \
+          entry.get('event', '') + '|' + json.dumps(entry.get('data', {}), sort_keys=True)
+    return hashlib.md5(raw.encode('utf-8')).hexdigest()[:16]
+
+
+def _read_events(dev, path):
+    """读一个开发者的 feedback.jsonl, 返回事件列表。容错: 跳过坏行。"""
+    events = []
+    try:
+        with open(path, encoding='utf-8') as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    e = json.loads(line)
+                    e.setdefault('dev', dev)
+                    events.append(e)
+                except json.JSONDecodeError:
+                    continue  # 跳过坏行, 不中断
+    except (OSError, IOError):
+        pass
+    return events
+
+
+def aggregate(rebuild=False):
+    """聚合所有开发者的 feedback → kg.duckdb。
+    返回写入的事件数。
+    增量: 只处理 mtime > last_aggregate 的文件。
+    """
+    con = get_db()
+    init_schema(con)
+
+    # 读上次聚合时间 (增量依据)
+    last_ts = None
+    if not rebuild:
+        row = con.execute(
+            "SELECT value FROM build_meta WHERE key = ?", [BUILD_META_KEY]
+        ).fetchone()
+        if row:
+            try:
+                last_ts = float(row[0])
+            except (ValueError, TypeError):
+                last_ts = None
+
+    files = _collect_journal_files()
+    if rebuild:
+        con.execute("DELETE FROM learning_events")
+        to_process = files
+    else:
+        to_process = [(d, p) for d, p in files
+                      if last_ts is None or p.stat().st_mtime > last_ts]
+
+    if not to_process:
+        print('无新增事件 (上次聚合后无变化)。')
+        _refresh_stats(con)
+        _update_meta(con)
+        return 0
+
+    # 收集所有事件 + 去重 (同 event_id 只写一次)
+    seen_ids = set()
+    if not rebuild:
+        existing = con.execute("SELECT id FROM learning_events").fetchall()
+        seen_ids = {r[0] for r in existing}
+
+    rows = []
+    total_events = 0
+    for dev, path in to_process:
+        for e in _read_events(dev, path):
+            eid = _event_id(e)
+            if eid in seen_ids:
+                continue
+            seen_ids.add(eid)
+            rows.append((
+                eid,
+                e.get('ts', ''),
+                e.get('dev', dev),
+                e.get('event', ''),
+                json.dumps(e.get('data', {}), ensure_ascii=False),
+            ))
+            total_events += 1
+
+    # 批量写入 (DuckDB executemany)
+    if rows:
+        con.executemany(
+            "INSERT OR IGNORE INTO learning_events(id, ts, dev, event, data) VALUES (?,?,?,?,?)",
+            rows,
+        )
+
+    _refresh_stats(con)
+    _update_meta(con)
+    print('聚合完成: %d 条事件, 来自 %d 个开发者 (%d 个文件)。'
+          % (total_events, len({d for d, _ in to_process}), len(to_process)))
+    return total_events
+
+
+def _refresh_stats(con):
+    """刷新 learning_stats 表 (从 learning_events 重新统计每人计数)。
+
+    事件→列映射来自 common/events.py (SSOT)。新增埋点只要在 events.py 注册,
+    这里自动生效, 不用改 SQL (修旧版硬编码漂移 bug)。
+    """
+    from common.events import event_to_sql_filter
+    con.execute("DELETE FROM learning_stats")
+    # 列名 = events.py 的 stat 值; 动态拼 FILTER 子句
+    cols = event_to_sql_filter()  # {stat_column: event}
+    col_names = list(cols.keys())
+    select_cols = ', '.join(
+        "COUNT(*) FILTER (WHERE event = '%s') AS %s" % (cols[c], c)
+        for c in col_names
+    )
+    col_list = ', '.join(col_names)
+    con.execute("""
+        INSERT INTO learning_stats (dev, %s, updated_at)
+        SELECT dev, %s, ?
+        FROM learning_events
+        GROUP BY dev
+    """ % (col_list, select_cols), [datetime.now().isoformat(timespec='seconds')])
+
+
+def _update_meta(con):
+    """记录本次聚合时间。"""
+    con.execute(
+        "INSERT OR REPLACE INTO build_meta(key, value) VALUES (?, ?)",
+        [BUILD_META_KEY, str(time.time())],
+    )
+
+
+def query_summary():
+    """查询团队学习摘要 (给 /wl-status 用)。"""
+    con = get_db()
+    init_schema(con)
+    total = con.execute("SELECT COUNT(*) FROM learning_events").fetchone()[0]
+    by_event = con.execute("""
+        SELECT event, COUNT(*) FROM learning_events GROUP BY event ORDER BY COUNT(*) DESC
+    """).fetchall()
+    by_dev = con.execute("""
+        SELECT dev, COUNT(*) FROM learning_events GROUP BY dev ORDER BY COUNT(*) DESC
+    """).fetchall()
+    return {'total': total, 'by_event': by_event, 'by_dev': by_dev}
+
+
+if __name__ == '__main__':
+    rebuild = '--rebuild' in sys.argv
+    n = aggregate(rebuild=rebuild)
+    s = query_summary()
+    print('\n团队学习摘要:')
+    print('  总事件: %d' % s['total'])
+    for ev, cnt in s['by_event']:
+        print('    %s: %d' % (ev, cnt))
+    for dev, cnt in s['by_dev']:
+        print('  %s: %d 条' % (dev, cnt))