@yottameta/yotta-logs 0.1.0 → 0.2.0

package/references/security.md

@@ -2,7 +2,7 @@
 
 ## 只读保证
 
-- 引擎只调用读取类操作（`open(path, "r")`），对会话目录不做任何写入 / 修改 / 删除；
+- 引擎只调用读取类操作（`open(path, "r")`、SQLite `file:<path>?mode=ro` 只读 URI），对任何日志 / 记忆文件与数据库不做写入 / 修改 / 删除；
 - 检索、提取、统计全程无网络请求；不把日志内容上传到任何服务；
 - 测试含只读回归：跑完 scan / search / session / stats / tools 后目录文件清单与大小不变。
 
@@ -26,17 +26,18 @@ URL 路径（非凭据）原文保留，方便回溯链接。
 
 ## 边界
 
-- 只检索 `*.jsonl` 会话文件与 `sessions.json` 索引；不读取、不分析其它文件；
-- 检索范围限定用户显式传入的 `--dir`（或环境变量 / 自动定位结果），不主动扫描磁盘；
-- 输出可能包含会话原文中的隐私，默认脱敏且仅用于本机回溯，请勿外传。
+- 只检索显式传入的 `--dir`（或环境变量 / discover 登记的来源），不主动扫描整盘；
+- 默认检索范围只含会话源 + 结构化记忆源；自由笔记（kind=note）与二进制日志（kind=log）默认不读，需显式 `--kind note / --kind log` 或配置 default_scope 开启；
+- discover 只登记已知根（见 agent-formats.md 第四节），命中即登记来源，不展开内容；
+- 输出可能包含会话 / 记忆原文中的隐私，默认脱敏且仅用于本机回溯，请勿外传。
 
 ## 与元忆（yotta-memory）的分工
 
 | 维度 | yotta-logs（元史） | yotta-memory（元忆） |
 |---|---|---|
-| 定位 | 原始会话日志（JSONL 事实） | 语义记忆（结构化条目） |
-| 输出 | 原文片段 + 行号 + 时间戳 | 记忆条目 + 权限边界 + 画像 |
+| 定位 | 原始会话日志 / 记忆文件（JSONL / JSON / SQLite / Markdown 事实） | 语义记忆（结构化条目 + 权限边界） |
+| 输出 | 原文片段 + 行号 + 时间戳 + 来源 | 记忆条目 + 权限边界 + 画像 |
 | 写操作 | 无（只读） | 支持（remember / forget / archive） |
-| 权限 | 目录级只读 | 类型 / 属主级权限边界 |
+| 权限 | 目录 / 来源级只读 | 类型 / 属主级权限边界 |
 
 回溯「原文」用元史；沉淀「长期知识 / 偏好 / 承诺」用元忆，二者互补。

package/scripts/test_yotta_logs.py

@@ -374,6 +374,348 @@ def test_readonly(fx):
           "before=%s after=%s" % (before, after))
 
 
+# ── v0.2.0 通用化测试 ────────────────────────────────────────────────────
+
+def build_json_fixture(base):
+    """单文件 JSON 会话：一个数组文件 + 一个 dict-of-lists 文件。"""
+    d = base / "jsons"
+    d.mkdir(parents=True, exist_ok=True)
+    arr = [
+        {"role": "user", "content": "你好，单文件 JSON 测试。", "ts": "2026-08-26T09:00:00+08:00"},
+        {"role": "assistant", "content": "收到。", "ts": "2026-08-26T09:01:00+08:00"},
+    ]
+    (d / "convo.json").write_text(json.dumps(arr, ensure_ascii=False),
+                                  encoding="utf-8")
+    multi = {
+        "s1": [{"role": "user", "content": "s1 的第一个问题"},
+               {"role": "assistant", "content": "s1 的回复"}],
+        "s2": [{"role": "user", "content": "s2 的问题"}],
+    }
+    (d / "multi.json").write_text(json.dumps(multi, ensure_ascii=False),
+                                  encoding="utf-8")
+    return d
+
+
+def build_sqlite_opencode(p):
+    import sqlite3 as _sq
+    con = _sq.connect(str(p))
+    con.execute("CREATE TABLE session (id TEXT, title TEXT, time_created INTEGER,"
+                " cost REAL, tokens_input INTEGER, tokens_output INTEGER)")
+    con.execute("CREATE TABLE message (id TEXT, session_id TEXT, time_created INTEGER, data TEXT)")
+    con.execute("CREATE TABLE part (id TEXT, message_id TEXT, session_id TEXT,"
+                " time_created INTEGER, data TEXT)")
+    con.execute("INSERT INTO session VALUES ('ses_a','部署讨论',1785834902916,0.05,100,50)")
+    con.execute("INSERT INTO session VALUES ('ses_b','CI 排查',1785835000000,0.0,0,0)")
+    con.execute("INSERT INTO message VALUES ('msg_1','ses_a',1785834903000,'{\"role\":\"user\"}')")
+    con.execute("INSERT INTO part VALUES ('prt_1','msg_1','ses_a',1785834903001,"
+                "'{\"type\":\"text\",\"text\":\"你好，部署方案定了吗？\"}')")
+    con.execute("INSERT INTO message VALUES ('msg_2','ses_a',1785834904000,'{\"role\":\"assistant\"}')")
+    con.execute("INSERT INTO part VALUES ('prt_2','msg_2','ses_a',1785834904001,"
+                "'{\"type\":\"text\",\"text\":\"定了，按灰度发布执行。\"}')")
+    con.execute("INSERT INTO part VALUES ('prt_3','msg_2','ses_a',1785834904002,"
+                "'{\"type\":\"tool\",\"tool\":\"read_file\"}')")
+    con.execute("INSERT INTO part VALUES ('prt_4','msg_2','ses_a',1785834904003,"
+                "'{\"type\":\"reasoning\",\"text\":\"隐藏推理不输出\"}')")
+    con.execute("INSERT INTO message VALUES ('msg_3','ses_b',1785835001000,'{\"role\":\"user\"}')")
+    con.execute("INSERT INTO part VALUES ('prt_5','msg_3','ses_b',1785835001001,"
+                "'{\"type\":\"text\",\"text\":\"CI 又失败了，看下日志。\"}')")
+    con.commit()
+    con.close()
+
+
+def build_sqlite_generic(p):
+    import sqlite3 as _sq
+    con = _sq.connect(str(p))
+    con.execute("CREATE TABLE messages (id INTEGER, session_id TEXT, role TEXT,"
+                " content TEXT, created_at TEXT)")
+    con.execute("INSERT INTO messages VALUES (1,'g1','user','泛型表问题甲','2026-08-26T03:00:00+08:00')")
+    con.execute("INSERT INTO messages VALUES (2,'g1','assistant','泛型表回复甲','2026-08-26T03:01:00+08:00')")
+    con.execute("INSERT INTO messages VALUES (3,'g2','user','乙的问题','2026-08-27T10:00:00+08:00')")
+    con.commit()
+    con.close()
+
+
+def build_md_fixture(base):
+    facts = base / ".yottamemory" / "facts"
+    facts.mkdir(parents=True, exist_ok=True)
+    (facts / "2026-08-25-0002.md").write_text(
+        "---\ntype: FACT\nsubject: 共享记忆引擎接入指南\n"
+        "statement: 本机运行 yotta-memory 记忆引擎，接入方式见正文。\n"
+        "confidence: 1\ncreated: 2026-08-25\ntags: [memory, guide]\n---\n"
+        "正文补充。\n", encoding="utf-8")
+    notes = base / ".CodexData" / "memories"
+    notes.mkdir(parents=True, exist_ok=True)
+    (notes / "note.md").write_text(
+        "# 推送闸门红线\n\n规则：测试通过才能推。\n", encoding="utf-8")
+    return facts, notes
+
+
+def test_norm_time():
+    check("毫秒转 ISO", YL._norm_time(1785834903000)
+          .startswith("20") and "T" in YL._norm_time(1785834903000),
+          YL._norm_time(1785834903000))
+    check("秒时间戳", YL._norm_time(1785834903).startswith("20"),
+          YL._norm_time(1785834903))
+    check("ISO 原样", YL._norm_time("2026-08-26T03:00:00+08:00")
+          == "2026-08-26T03:00:00+08:00")
+    check("日期原样", YL._norm_time("2026-08-25") == "2026-08-25")
+    check("Z 归一", YL._norm_time("2026-08-26T03:00:00Z")
+          == "2026-08-26T03:00:00+00:00")
+
+
+def test_json_reader():
+    with tempfile.TemporaryDirectory() as td:
+        d = build_json_fixture(Path(td))
+        src = YL.sniff_source(str(d))
+        check("JSON 目录嗅探", src["format"] == "json" and src["kind"] == "session",
+              str(src))
+        reader = YL.JSONReader()
+        rows, tm, inv = reader.iter_sessions(src)
+        check("JSON scan 3 会话", len(rows) == 3 and tm == 5, str(rows))
+        r = YL.search_all([src], "单文件")
+        check("JSON search 命中", len(r["matches"]) == 1
+              and r["matches"][0]["session"] == "convo", str(r))
+        r2 = YL.search_all([src], "s2")
+        check("JSON dict 会话检索", len(r2["matches"]) == 1
+              and r2["matches"][0]["session"] == "s2", str(r2))
+        ex = YL.extract_all([src], "convo")
+        check("JSON extract", len(ex["messages"]) == 2
+              and ex["messages"][0]["role"] == "user", str(ex))
+        st = YL.stats_all([src])
+        check("JSON stats 消息 5", st["messages"] == 5, str(st["messages"]))
+        check("JSON stats 角色", st["roles"].get("user") == 3, str(st["roles"]))
+
+
+def test_sqlite_opencode_reader():
+    with tempfile.TemporaryDirectory() as td:
+        p = Path(td) / "opencode.db"
+        build_sqlite_opencode(p)
+        src = YL.sniff_source(str(p))
+        check("SQLite 文件嗅探", src["format"] == "sqlite", str(src))
+        reader = YL.SQLiteReader()
+        rows, tm, inv = reader.iter_sessions(src)
+        check("opencode scan 2 会话", len(rows) == 2, str(rows))
+        by = {r["session"]: r for r in rows}
+        check("opencode ses_a 消息 2", by["ses_a"]["messages"] == 2,
+              str(by["ses_a"]))
+        check("opencode 毫秒日期", by["ses_a"]["date"].startswith("20"),
+              by["ses_a"]["date"])
+        r = YL.search_all([src], "灰度")
+        check("opencode search 命中", len(r["matches"]) == 1
+              and r["matches"][0]["session"] == "ses_a", str(r))
+        r2 = YL.search_all([src], "隐藏推理")
+        check("opencode reasoning 不进文本", r2["matches"] == [], str(r2))
+        ex = YL.extract_all([src], "ses_a")
+        check("opencode extract 消息 2", len(ex["messages"]) == 2, str(ex))
+        msg2 = ex["messages"][1]
+        check("opencode 工具标注", msg2["tools"] == ["read_file"], str(msg2))
+        check("opencode role assistant", msg2["role"] == "assistant", str(msg2))
+        st = YL.stats_all([src])
+        check("opencode stats 消息 3", st["messages"] == 3, str(st["messages"]))
+        check("opencode stats 角色", st["roles"] == {"user": 2, "assistant": 1},
+              str(st["roles"]))
+        items = YL.tools_all([src])
+        check("opencode tools read_file 1", dict(items).get("read_file") == 1,
+              str(items))
+
+
+def test_sqlite_generic_reader():
+    with tempfile.TemporaryDirectory() as td:
+        p = Path(td) / "app.db"
+        build_sqlite_generic(p)
+        src = YL._mk_source("generic-test", "session", "sqlite", p,
+                            extra={"table": "messages"})
+        reader = YL.SQLiteReader()
+        rows, tm, inv = reader.iter_sessions(src)
+        by = {r["session"]: r for r in rows}
+        check("generic scan 2 会话", len(rows) == 2 and tm == 3, str(rows))
+        check("generic g1 消息 2", by["g1"]["messages"] == 2, str(by["g1"]))
+        r = YL.search_all([src], "甲")
+        check("generic search 命中 2", len(r["matches"]) == 2, str(r))
+        ex = YL.extract_all([src], "g1")
+        check("generic extract 2 条", len(ex["messages"]) == 2, str(ex))
+        st = YL.stats_all([src])
+        check("generic stats 角色", st["roles"] == {"user": 2, "assistant": 1},
+              str(st["roles"]))
+
+
+def test_markdown_reader():
+    with tempfile.TemporaryDirectory() as td:
+        facts, notes = build_md_fixture(Path(td))
+        fsrc = YL._mk_source("yottamemory-facts", "memory", "markdown", facts)
+        nsrc = YL._mk_source("codex-notes", "note", "markdown", notes)
+        fr = list(YL.MarkdownReader().iter_records(fsrc))
+        check("md memory 1 条", len(fr) == 1, str(fr))
+        rec = fr[0]
+        check("md role FACT", rec["role"] == "FACT", str(rec["role"]))
+        check("md title subject", rec["meta"].get("title") == "共享记忆引擎接入指南",
+              str(rec["meta"]))
+        check("md text statement", "yotta-memory" in rec["text"], rec["text"][:50])
+        check("md created 时间", rec["time"] == "2026-08-25", rec["time"])
+        nr = list(YL.MarkdownReader().iter_records(nsrc))
+        check("md note 1 条", len(nr) == 1 and nr[0]["kind"] == "note", str(nr))
+        check("md note 标题", nr[0]["meta"].get("title") == "推送闸门红线",
+              str(nr[0]["meta"]))
+        check("md note 正文", "测试通过" in nr[0]["text"], nr[0]["text"][:40])
+        rows, tm, inv = YL.MarkdownReader().iter_sessions(fsrc)
+        check("md memory scan", len(rows) == 1 and tm == 1, str(rows))
+        # 结构化 md 文件单独 --dir → kind memory
+        fp = facts / "2026-08-25-0002.md"
+        s = YL.sniff_source(str(fp))
+        check("md 文件嗅探 memory", s["format"] == "markdown" and s["kind"] == "memory",
+              str(s))
+        np = notes / "note.md"
+        s2 = YL.sniff_source(str(np))
+        check("md 文件嗅探 note", s2["kind"] == "note", str(s2))
+
+
+def test_binary_reader():
+    with tempfile.TemporaryDirectory() as td:
+        p = Path(td) / "conv.pbtxt"
+        p.write_bytes(b"\x00\x01Conversation WindSurf Title\x00\x02more")
+        src = YL._mk_source("windsurf-conv", "log", "binary", p, default_on=False)
+        recs = list(YL.BinaryReader().iter_records(src))
+        check("binary 1 条且不崩", len(recs) == 1, str(recs))
+        check("binary title 提取", "WindSurf" in recs[0]["text"], recs[0]["text"])
+        check("binary kind log", recs[0]["kind"] == "log", str(recs[0]["kind"]))
+        rows, tm, inv = YL.BinaryReader().iter_sessions(src)
+        check("binary scan", len(rows) == 1 and tm == 1, str(rows))
+
+
+def test_sniff_and_discover():
+    with tempfile.TemporaryDirectory() as td:
+        base = Path(td)
+        # discover：JSONL + SQLite(opencode) + 记忆 md + 自由笔记
+        jd = base / ".codex" / "sessions"
+        jd.mkdir(parents=True, exist_ok=True)
+        (jd / "a.jsonl").write_text('{"type":"message","message":{"role":"user","content":"hi"}}\n',
+                                    encoding="utf-8")
+        db = base / ".local" / "share" / "opencode" / "opencode.db"
+        db.parent.mkdir(parents=True, exist_ok=True)
+        build_sqlite_opencode(db)
+        facts, notes = build_md_fixture(base)
+        jsrcs = YL.JSONLReader.discover(base)
+        check("discover JSONL 命中 codex", any(s["name"] == "codex-sessions"
+              for s in jsrcs), str(jsrcs))
+        ssrcs = YL.SQLiteReader.discover(base)
+        check("discover SQLite 命中 opencode", any(s["name"] == "opencode-db"
+              for s in ssrcs), str(ssrcs))
+        msrcs = YL.MarkdownReader.discover(base)
+        names = {s["name"] for s in msrcs}
+        check("discover md 命中 yottamemory-facts", "yottamemory-facts" in names,
+              str(msrcs))
+        check("discover md 命中 codex-notes", "codex-notes" in names, str(msrcs))
+        for s in msrcs:
+            if s["name"] == "codex-notes":
+                check("自由笔记默认关", s["default_on"] is False, str(s))
+            if s["name"] == "yottamemory-facts":
+                check("结构化记忆默认开", s["default_on"] is True, str(s))
+        # 配置兜底
+        cfg = {"sources": [{"name": "myapp", "path": str(base / "app.db"),
+                            "format": "sqlite", "kind": "session",
+                            "table": "messages", "col_text": "content"}]}
+        srcs = YL.discover_sources(cfg)
+        check("配置源登记", any(s["name"] == "myapp" for s in srcs), str(srcs))
+
+
+def test_filters_and_scope():
+    srcs = [
+        YL._mk_source("sess", "session", "jsonl", "/x", True),
+        YL._mk_source("mem", "memory", "markdown", "/y", True),
+        YL._mk_source("note", "note", "markdown", "/z", False),
+    ]
+
+    class A:
+        source = None
+        kind = None
+        format = None
+
+    a = A()
+    out = YL.filter_sources(srcs, a)
+    check("默认范围排除 note", [s["name"] for s in out] == ["sess", "mem"],
+          str(out))
+    a.kind = "note"
+    out = YL.filter_sources(srcs, a)
+    check("--kind note 显式开", [s["name"] for s in out] == ["note"], str(out))
+    a.kind = None
+    a.format = "markdown"
+    out = YL.filter_sources(srcs, a)
+    check("--format markdown 显式含 note", {s["name"] for s in out} == {"mem", "note"},
+          str(out))
+    a.format = None
+    a.source = ["sess"]
+    out = YL.filter_sources(srcs, a)
+    check("--source 过滤", [s["name"] for s in out] == ["sess"], str(out))
+
+
+def test_cross_source_scope():
+    with tempfile.TemporaryDirectory() as td:
+        base = Path(td)
+        # 用自己的 fixture：jsonl 会话 + 记忆 md + 自由笔记 都含关键词
+        jd = base / "sessions"
+        jd.mkdir(parents=True, exist_ok=True)
+        (jd / "s1.jsonl").write_text(
+            '{"type":"message","timestamp":"2026-08-26T03:00:00+08:00","message":{"role":"user","content":"部署方案定了吗"}}\n',
+            encoding="utf-8")
+        facts, notes = build_md_fixture(base)
+        (facts / "m1.md").write_text(
+            "---\ntype: FACT\nsubject: 部署\nstatement: 部署方案已拍板。\ncreated: 2026-08-25\n---\n",
+            encoding="utf-8")
+        (notes / "n1.md").write_text("# 部署笔记\n\n部署草稿。\n", encoding="utf-8")
+        jsrc = YL.sniff_source(str(jd))
+        fsrc = YL._mk_source("facts", "memory", "markdown", facts)
+        nsrc = YL._mk_source("notes", "note", "markdown", notes, default_on=False)
+        srcs = [jsrc, fsrc, nsrc]
+        # 默认：会话 + 记忆，排除自由笔记
+        default = YL.filter_sources(srcs, type("A", (), {"source": None,
+                                                         "kind": None,
+                                                         "format": None})())
+        res = YL.search_all(default, "部署")
+        names = {(m["source"], m["session"]) for m in res["matches"]}
+        check("默认范围不含自由笔记", ("notes", "n1") not in names, str(names))
+        check("默认范围含会话与记忆",
+              ("sessions", "s1") in names and ("facts", "m1") in names, str(names))
+        # 显式开 note
+        res2 = YL.search_all([nsrc], "部署")
+        check("显式开 note 可检索", ("notes", "n1") in
+              {(m["source"], m["session"]) for m in res2["matches"]}, str(res2))
+
+
+def test_cli_v020(fx):
+    # --kind / --format / --source 参数存在且 --dir 行为不变
+    r = _run(["search", "部署", "--dir", str(fx), "--format", "jsonl"])
+    check("CLI --format jsonl", r.returncode == 0 and "部署方案" in r.stdout,
+          "rc=%d" % r.returncode)
+    r = _run(["search", "部署", "--dir", str(fx), "--kind", "session"])
+    check("CLI --kind session", r.returncode == 0, "rc=%d" % r.returncode)
+    r = _run(["scan", "--dir", str(fx), "--source", "nope"])
+    check("CLI --source 无命中退出码 1", r.returncode == 1,
+          "rc=%d" % r.returncode)
+    # locate --json 结构
+    r = _run(["locate", "--json"])
+    try:
+        obj = json.loads(r.stdout)
+        check("CLI locate --json", r.returncode == 0
+              and "sources" in obj and "default_scope" in obj, r.stdout[:120])
+    except Exception as e:  # noqa: BLE001
+        check("CLI locate --json", False, str(e))
+    # 单文件 md --dir（自由笔记显式可查）
+    with tempfile.TemporaryDirectory() as td:
+        np = Path(td) / "note.md"
+        np.write_text("# 标题甲\n\n正文含关键词乙。\n", encoding="utf-8")
+        r = _run(["search", "关键词乙", "--dir", str(np)])
+        check("CLI 单 md 文件检索", r.returncode == 0 and "关键词乙" in r.stdout,
+              "rc=%d out=%s" % (r.returncode, r.stdout[:80]))
+        r = _run(["scan", "--dir", str(np), "--json"])
+        try:
+            obj = json.loads(r.stdout)
+            check("CLI 单 md scan", r.returncode == 0
+                  and obj["total_sessions"] == 1, r.stdout[:120])
+        except Exception as e:  # noqa: BLE001
+            check("CLI 单 md scan", False, str(e))
+
+
+
 def main():
     print("元史（yotta-logs）测试开始…")
     with tempfile.TemporaryDirectory() as td:
@@ -391,6 +733,16 @@ def main():
         test_cli(fx)
         test_gbk_console(fx)
         test_readonly(fx)
+        test_norm_time()
+        test_json_reader()
+        test_sqlite_opencode_reader()
+        test_sqlite_generic_reader()
+        test_markdown_reader()
+        test_binary_reader()
+        test_sniff_and_discover()
+        test_filters_and_scope()
+        test_cross_source_scope()
+        test_cli_v020(fx)
     print("")
     print("通过 %d 项，失败 %d 项" % (PASS, FAIL))
     if FAILED:
@@ -400,5 +752,6 @@ def main():
     sys.exit(1 if FAIL else 0)
 
 
+
 if __name__ == "__main__":
     main()