@hupan56/wlkj 2.7.12 → 3.1.2

package/templates/qoder/scripts/{learn_aggregate.py → domain/kg/build/learn_aggregate.py}

@@ -1,201 +1,212 @@
-# -*- coding: utf-8 -*-
-"""learn_aggregate.py - 团队学习聚合 (同构 kg_build.py)
-
-扫所有人的 workspace/members/{dev}/journal/feedback.jsonl
-→ 增量写入 kg.duckdb 的 learning_events 表 (团队单一真源)
-
-和 kg_build.py 对称:
-  kg_build.py:      扫 data/code/* → 写 kg.duckdb (代码图谱)
-  learn_aggregate.py: 扫 workspace/members/*/journal → 写 kg.duckdb (学习图谱)
-
-增量策略 (复用 build_meta):
-  记录上次聚合时间, 只处理 mtime > last 的文件。全量重建用 --rebuild。
-
-用法:
-  python learn_aggregate.py            # 增量
-  python learn_aggregate.py --rebuild  # 全量重建
-"""
-import os
-import sys
-import json
-import hashlib
-import time
-from datetime import datetime
-
-SCRIPTS = os.path.dirname(os.path.abspath(__file__))
-sys.path.insert(0, SCRIPTS)
-
-from common.paths import MEMBERS_DIR
-from kg_duckdb import get_db, init_schema
-
-BUILD_META_KEY = 'learning_last_aggregate'
-
-
-def _collect_journal_files():
-    """扫 workspace/members/*/journal/feedback.jsonl, 返回 [(dev, path)]。"""
-    results = []
-    if not MEMBERS_DIR.is_dir():
-        return results
-    for dev_dir in sorted(MEMBERS_DIR.iterdir()):
-        if not dev_dir.is_dir():
-            continue
-        fb = dev_dir / 'journal' / 'feedback.jsonl'
-        if fb.is_file():
-            results.append((dev_dir.name, fb))
-    return results
-
-
-def _event_id(entry):
-    """生成事件唯一 ID (去重用): hash(ts+dev+event+data)。"""
-    raw = entry.get('ts', '') + '|' + entry.get('dev', '') + '|' + \
-          entry.get('event', '') + '|' + json.dumps(entry.get('data', {}), sort_keys=True)
-    return hashlib.md5(raw.encode('utf-8')).hexdigest()[:16]
-
-
-def _read_events(dev, path):
-    """读一个开发者的 feedback.jsonl, 返回事件列表。容错: 跳过坏行。"""
-    events = []
-    try:
-        with open(path, encoding='utf-8') as f:
-            for line in f:
-                line = line.strip()
-                if not line:
-                    continue
-                try:
-                    e = json.loads(line)
-                    e.setdefault('dev', dev)
-                    events.append(e)
-                except json.JSONDecodeError:
-                    continue  # 跳过坏行, 不中断
-    except (OSError, IOError):
-        pass
-    return events
-
-
-def aggregate(rebuild=False):
-    """聚合所有开发者的 feedback → kg.duckdb。
-    返回写入的事件数。
-    增量: 只处理 mtime > last_aggregate 的文件。
-    """
-    con = get_db()
-    init_schema(con)
-
-    # 读上次聚合时间 (增量依据)
-    last_ts = None
-    if not rebuild:
-        row = con.execute(
-            "SELECT value FROM build_meta WHERE key = ?", [BUILD_META_KEY]
-        ).fetchone()
-        if row:
-            try:
-                last_ts = float(row[0])
-            except (ValueError, TypeError):
-                last_ts = None
-
-    files = _collect_journal_files()
-    if rebuild:
-        con.execute("DELETE FROM learning_events")
-        to_process = files
-    else:
-        to_process = [(d, p) for d, p in files
-                      if last_ts is None or p.stat().st_mtime > last_ts]
-
-    if not to_process:
-        print('无新增事件 (上次聚合后无变化)。')
-        _refresh_stats(con)
-        _update_meta(con)
-        return 0
-
-    # 收集所有事件 + 去重 (同 event_id 只写一次)
-    seen_ids = set()
-    if not rebuild:
-        existing = con.execute("SELECT id FROM learning_events").fetchall()
-        seen_ids = {r[0] for r in existing}
-
-    rows = []
-    total_events = 0
-    for dev, path in to_process:
-        for e in _read_events(dev, path):
-            eid = _event_id(e)
-            if eid in seen_ids:
-                continue
-            seen_ids.add(eid)
-            rows.append((
-                eid,
-                e.get('ts', ''),
-                e.get('dev', dev),
-                e.get('event', ''),
-                json.dumps(e.get('data', {}), ensure_ascii=False),
-            ))
-            total_events += 1
-
-    # 批量写入 (DuckDB executemany)
-    if rows:
-        con.executemany(
-            "INSERT OR IGNORE INTO learning_events(id, ts, dev, event, data) VALUES (?,?,?,?,?)",
-            rows,
-        )
-
-    _refresh_stats(con)
-    _update_meta(con)
-    print('聚合完成: %d 条事件, 来自 %d 个开发者 (%d 个文件)。'
-          % (total_events, len({d for d, _ in to_process}), len(to_process)))
-    return total_events
-
-
-def _refresh_stats(con):
-    """刷新 learning_stats 表 (从 learning_events 重新统计每人计数)。
-
-    事件→列映射来自 common/events.py (SSOT)。新增埋点只要在 events.py 注册,
-    这里自动生效, 不用改 SQL (修旧版硬编码漂移 bug)。
-    """
-    from common.events import event_to_sql_filter
-    con.execute("DELETE FROM learning_stats")
-    # 列名 = events.py 的 stat 值; 动态拼 FILTER 子句
-    cols = event_to_sql_filter()  # {stat_column: event}
-    col_names = list(cols.keys())
-    select_cols = ', '.join(
-        "COUNT(*) FILTER (WHERE event = '%s') AS %s" % (cols[c], c)
-        for c in col_names
-    )
-    col_list = ', '.join(col_names)
-    con.execute("""
-        INSERT INTO learning_stats (dev, %s, updated_at)
-        SELECT dev, %s, ?
-        FROM learning_events
-        GROUP BY dev
-    """ % (col_list, select_cols), [datetime.now().isoformat(timespec='seconds')])
-
-
-def _update_meta(con):
-    """记录本次聚合时间。"""
-    con.execute(
-        "INSERT OR REPLACE INTO build_meta(key, value) VALUES (?, ?)",
-        [BUILD_META_KEY, str(time.time())],
-    )
-
-
-def query_summary():
-    """查询团队学习摘要 (给 /wl-status 用)。"""
-    con = get_db()
-    init_schema(con)
-    total = con.execute("SELECT COUNT(*) FROM learning_events").fetchone()[0]
-    by_event = con.execute("""
-        SELECT event, COUNT(*) FROM learning_events GROUP BY event ORDER BY COUNT(*) DESC
-    """).fetchall()
-    by_dev = con.execute("""
-        SELECT dev, COUNT(*) FROM learning_events GROUP BY dev ORDER BY COUNT(*) DESC
-    """).fetchall()
-    return {'total': total, 'by_event': by_event, 'by_dev': by_dev}
-
-
-if __name__ == '__main__':
-    rebuild = '--rebuild' in sys.argv
-    n = aggregate(rebuild=rebuild)
-    s = query_summary()
-    print('\n团队学习摘要:')
-    print('  总事件: %d' % s['total'])
-    for ev, cnt in s['by_event']:
-        print('    %s: %d' % (ev, cnt))
-    for dev, cnt in s['by_dev']:
-        print('  %s: %d 条' % (dev, cnt))
+# -*- coding: utf-8 -*-
+"""learn_aggregate.py - 团队学习聚合 (同构 kg_build.py)
+
+# v3.0 路径自举: 引导到 common/bootstrap, 统一 sys.path 逻辑
+import os as _o, sys as _s
+_f = _o.path.abspath(__file__)
+for _ in range(10):
+    _f = _o.path.dirname(_f)
+    _cp = _o.path.join(_f, 'foundation')
+    if _o.path.isfile(_o.path.join(_cp, 'bootstrap.py')):
+        break
+if _cp not in _s.path: _s.path.insert(0, _cp)
+from bootstrap import setup; setup()
+
+扫所有人的 workspace/members/{dev}/journal/feedback.jsonl
+→ 增量写入 kg.duckdb 的 learning_events 表 (团队单一真源)
+
+和 kg_build.py 对称:
+  kg_build.py:      扫 data/code/* → 写 kg.duckdb (代码图谱)
+  learn_aggregate.py: 扫 workspace/members/*/journal → 写 kg.duckdb (学习图谱)
+
+增量策略 (复用 build_meta):
+  记录上次聚合时间, 只处理 mtime > last 的文件。全量重建用 --rebuild。
+
+用法:
+  python learn_aggregate.py            # 增量
+  python learn_aggregate.py --rebuild  # 全量重建
+"""
+import os
+import sys
+import json
+import hashlib
+import time
+from datetime import datetime
+
+SCRIPTS = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, SCRIPTS)
+
+from foundation.core.paths import MEMBERS_DIR
+from domain.kg.storage.kg_duckdb import get_db, init_schema  # v3.0: kg_duckdb 已迁入 kg/ 子包
+
+BUILD_META_KEY = 'learning_last_aggregate'
+
+
+def _collect_journal_files():
+    """扫 workspace/members/*/journal/feedback.jsonl, 返回 [(dev, path)]。"""
+    results = []
+    if not MEMBERS_DIR.is_dir():
+        return results
+    for dev_dir in sorted(MEMBERS_DIR.iterdir()):
+        if not dev_dir.is_dir():
+            continue
+        fb = dev_dir / 'journal' / 'feedback.jsonl'
+        if fb.is_file():
+            results.append((dev_dir.name, fb))
+    return results
+
+
+def _event_id(entry):
+    """生成事件唯一 ID (去重用): hash(ts+dev+event+data)。"""
+    raw = entry.get('ts', '') + '|' + entry.get('dev', '') + '|' + \
+          entry.get('event', '') + '|' + json.dumps(entry.get('data', {}), sort_keys=True)
+    return hashlib.md5(raw.encode('utf-8')).hexdigest()[:16]
+
+
+def _read_events(dev, path):
+    """读一个开发者的 feedback.jsonl, 返回事件列表。容错: 跳过坏行。"""
+    events = []
+    try:
+        with open(path, encoding='utf-8') as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    e = json.loads(line)
+                    e.setdefault('dev', dev)
+                    events.append(e)
+                except json.JSONDecodeError:
+                    continue  # 跳过坏行, 不中断
+    except (OSError, IOError):
+        pass
+    return events
+
+
+def aggregate(rebuild=False):
+    """聚合所有开发者的 feedback → kg.duckdb。
+    返回写入的事件数。
+    增量: 只处理 mtime > last_aggregate 的文件。
+    """
+    con = get_db()
+    init_schema(con)
+
+    # 读上次聚合时间 (增量依据)
+    last_ts = None
+    if not rebuild:
+        row = con.execute(
+            "SELECT value FROM build_meta WHERE key = ?", [BUILD_META_KEY]
+        ).fetchone()
+        if row:
+            try:
+                last_ts = float(row[0])
+            except (ValueError, TypeError):
+                last_ts = None
+
+    files = _collect_journal_files()
+    if rebuild:
+        con.execute("DELETE FROM learning_events")
+        to_process = files
+    else:
+        to_process = [(d, p) for d, p in files
+                      if last_ts is None or p.stat().st_mtime > last_ts]
+
+    if not to_process:
+        print('无新增事件 (上次聚合后无变化)。')
+        _refresh_stats(con)
+        _update_meta(con)
+        return 0
+
+    # 收集所有事件 + 去重 (同 event_id 只写一次)
+    seen_ids = set()
+    if not rebuild:
+        existing = con.execute("SELECT id FROM learning_events").fetchall()
+        seen_ids = {r[0] for r in existing}
+
+    rows = []
+    total_events = 0
+    for dev, path in to_process:
+        for e in _read_events(dev, path):
+            eid = _event_id(e)
+            if eid in seen_ids:
+                continue
+            seen_ids.add(eid)
+            rows.append((
+                eid,
+                e.get('ts', ''),
+                e.get('dev', dev),
+                e.get('event', ''),
+                json.dumps(e.get('data', {}), ensure_ascii=False),
+            ))
+            total_events += 1
+
+    # 批量写入 (DuckDB executemany)
+    if rows:
+        con.executemany(
+            "INSERT OR IGNORE INTO learning_events(id, ts, dev, event, data) VALUES (?,?,?,?,?)",
+            rows,
+        )
+
+    _refresh_stats(con)
+    _update_meta(con)
+    print('聚合完成: %d 条事件, 来自 %d 个开发者 (%d 个文件)。'
+          % (total_events, len({d for d, _ in to_process}), len(to_process)))
+    return total_events
+
+
+def _refresh_stats(con):
+    """刷新 learning_stats 表 (从 learning_events 重新统计每人计数)。
+
+    事件→列映射来自 common/events.py (SSOT)。新增埋点只要在 events.py 注册,
+    这里自动生效, 不用改 SQL (修旧版硬编码漂移 bug)。
+    """
+    from foundation.data.events import event_to_sql_filter
+    con.execute("DELETE FROM learning_stats")
+    # 列名 = events.py 的 stat 值; 动态拼 FILTER 子句
+    cols = event_to_sql_filter()  # {stat_column: event}
+    col_names = list(cols.keys())
+    select_cols = ', '.join(
+        "COUNT(*) FILTER (WHERE event = '%s') AS %s" % (cols[c], c)
+        for c in col_names
+    )
+    col_list = ', '.join(col_names)
+    con.execute("""
+        INSERT INTO learning_stats (dev, %s, updated_at)
+        SELECT dev, %s, ?
+        FROM learning_events
+        GROUP BY dev
+    """ % (col_list, select_cols), [datetime.now().isoformat(timespec='seconds')])
+
+
+def _update_meta(con):
+    """记录本次聚合时间。"""
+    con.execute(
+        "INSERT OR REPLACE INTO build_meta(key, value) VALUES (?, ?)",
+        [BUILD_META_KEY, str(time.time())],
+    )
+
+
+def query_summary():
+    """查询团队学习摘要 (给 /wl-status 用)。"""
+    con = get_db()
+    init_schema(con)
+    total = con.execute("SELECT COUNT(*) FROM learning_events").fetchone()[0]
+    by_event = con.execute("""
+        SELECT event, COUNT(*) FROM learning_events GROUP BY event ORDER BY COUNT(*) DESC
+    """).fetchall()
+    by_dev = con.execute("""
+        SELECT dev, COUNT(*) FROM learning_events GROUP BY dev ORDER BY COUNT(*) DESC
+    """).fetchall()
+    return {'total': total, 'by_event': by_event, 'by_dev': by_dev}
+
+
+if __name__ == '__main__':
+    rebuild = '--rebuild' in sys.argv
+    n = aggregate(rebuild=rebuild)
+    s = query_summary()
+    print('\n团队学习摘要:')
+    print('  总事件: %d' % s['total'])
+    for ev, cnt in s['by_event']:
+        print('    %s: %d' % (ev, cnt))
+    for dev, cnt in s['by_dev']:
+        print('  %s: %d 条' % (dev, cnt))