@easysociable/cli 0.1.3 → 0.1.5

package/content-engine/workbench/build.py

@@ -29,6 +29,8 @@ SWIPE_RE = re.compile(r"\bS\d+\b")
 ATOM_RE = re.compile(r"\bA-[a-z0-9-]+\b")
 CLAIM_RE = re.compile(r"\bC-(?!\d{8})[a-z0-9-]+\b")
 RUN_RE = re.compile(r"RUN-[A-Za-z0-9-]+")
+PRODUCT_RE = re.compile(r"\bP-[A-Za-z0-9-]+\b")
+RECOMMENDATION_RE = re.compile(r"\bR-[A-Za-z0-9-]+\b")
 WIKI = re.compile(r"\[\[([^\]|#]+)(?:[|#][^\]]*)?\]\]")
 
 
@@ -165,7 +167,8 @@ HTTP_RE = re.compile(r"https?://[^\s\]\)>'\"|]+")
 def wiki_path(cell: str) -> str:
     m = WIKI.search(cell or "")
     if not m:
-        return ""
+        raw = (cell or "").strip().strip("`")
+        return raw if raw.endswith(".md") and "://" not in raw else ""
     p = m.group(1).strip()
     if p.endswith(".md"):
         return p
@@ -304,44 +307,128 @@ def parse_feedback_metrics(text: str) -> dict:
 
 
 ENGAGEMENT_SNAP_RE = re.compile(
-    r"engagement\s+snapshot:\s*(.+?)(?:\n|$)",
+    r"(?:engagement\s+snapshot|snapshot\s+engagement)[^\n:]*:\s*(.+?)(?:\n|$)",
+    re.I,
+)
+ENGAGEMENT_SECTION_RE = re.compile(
+    r"^##\s+(?:Engagement\s+snapshot|Snapshot\s+metrics(?:\s*\([^)]*\))?)\s*\n+(.*?)(?=^##\s|\Z)",
+    re.I | re.M | re.S,
+)
+ENGAGEMENT_KIND = r"likes?|replies|reply|bookmarks?|views?"
+ENGAGEMENT_NUM = r"~?\d[\d,]*(?:\.\d+)?[kKmMbB]?"
+# kind-first: likes 113 / likes: 3
+ENGAGEMENT_KIND_NUM_RE = re.compile(
+    rf"\b({ENGAGEMENT_KIND})\s*[:=]?\s*({ENGAGEMENT_NUM})\b",
     re.I,
 )
-ENGAGEMENT_PART_RE = re.compile(
-    r"(likes|replies|bookmarks|views)\s+(\d[\d,]*)",
+# num-first: 1086 likes / ~92.9k views
+ENGAGEMENT_NUM_KIND_RE = re.compile(
+    rf"({ENGAGEMENT_NUM})\s+({ENGAGEMENT_KIND})\b",
     re.I,
 )
 ENGAGEMENT_AT_RE = re.compile(r"@\s*(\d{4}-\d{2}-\d{2})")
+ENGAGEMENT_FETCH_AT_RE = re.compile(
+    r"(?:fetch|at\s+capture|snapshot_at)\s*[:=]?\s*(\d{4}-\d{2}-\d{2})",
+    re.I,
+)
 
 
-def parse_engagement_snapshot(text: str) -> dict[str, str]:
-    """Parse Discovery `engagement snapshot` line into src_* fields.
-
-    Only the primary snapshot counts. Parenthetical notes like
-    `(quoted earlier demo: likes 67 / bookmarks 115)` are ignored.
-    """
-    out = {
+def _empty_engagement() -> dict[str, str]:
+    return {
         "src_likes": "",
         "src_replies": "",
         "src_bookmarks": "",
         "src_views": "",
         "engagement_at": "",
     }
+
+
+def _normalize_metric_kind(kind: str) -> str:
+    k = kind.lower()
+    if k.startswith("like"):
+        return "likes"
+    if k.startswith("repl"):
+        return "replies"
+    if k.startswith("bookmark"):
+        return "bookmarks"
+    if k.startswith("view"):
+        return "views"
+    return k
+
+
+def _parse_compact_count(raw: str) -> str:
+    """'92.9k' → '92900'; '1.1M' → '1100000'; '9,574' → '9574'."""
+    s = raw.strip().lstrip("~").replace(",", "")
+    if not s:
+        return ""
+    m = re.fullmatch(r"(\d+(?:\.\d+)?)([kKmMbB])?", s)
+    if not m:
+        digits = re.sub(r"[^\d]", "", s)
+        return digits
+    n = float(m.group(1))
+    suf = (m.group(2) or "").lower()
+    if suf == "k":
+        n *= 1_000
+    elif suf == "m":
+        n *= 1_000_000
+    elif suf == "b":
+        n *= 1_000_000_000
+    return str(int(round(n)))
+
+
+def _fill_engagement_from_blob(out: dict[str, str], blob: str) -> None:
+    if not blob:
+        return
+    # Prefer kind-first matches (list form), then num-first (capture prose).
+    for kind, num in ENGAGEMENT_KIND_NUM_RE.findall(blob):
+        key = f"src_{_normalize_metric_kind(kind)}"
+        if key in out and not out[key]:
+            out[key] = _parse_compact_count(num)
+    for num, kind in ENGAGEMENT_NUM_KIND_RE.findall(blob):
+        key = f"src_{_normalize_metric_kind(kind)}"
+        if key in out and not out[key]:
+            out[key] = _parse_compact_count(num)
+    if not out["engagement_at"]:
+        at = ENGAGEMENT_AT_RE.search(blob) or ENGAGEMENT_FETCH_AT_RE.search(blob)
+        if at:
+            out["engagement_at"] = at.group(1)
+
+
+def parse_engagement_snapshot(text: str) -> dict[str, str]:
+    """Parse engagement into src_* fields from Capture / Raw notes.
+
+    Supports:
+    - Compact: `engagement snapshot: likes 113 / replies 18 / ... @ 2026-08-30`
+    - Capture prose: `Engagement snapshot (fetch DATE): ~92.9k views · 1086 likes · ...`
+    - Raw list under `## Engagement snapshot` with `- likes: 3`
+    Parenthetical secondary demos after the primary snapshot are ignored.
+    """
+    out = _empty_engagement()
     if not text:
         return out
     m = ENGAGEMENT_SNAP_RE.search(text)
-    if not m:
+    if m:
+        blob = m.group(1).split("(", 1)[0]
+        # Date often sits in the label: Engagement snapshot (fetch 2026-08-25)
+        label = text[m.start() : m.start(1)]
+        _fill_engagement_from_blob(out, blob)
+        if not out["engagement_at"]:
+            at = ENGAGEMENT_FETCH_AT_RE.search(label) or ENGAGEMENT_AT_RE.search(label)
+            if at:
+                out["engagement_at"] = at.group(1)
         return out
-    # Keep only the primary snapshot; drop parenthetical secondary demos.
-    blob = m.group(1).split("(", 1)[0]
-    for kind, num in ENGAGEMENT_PART_RE.findall(blob):
-        key = f"src_{kind.lower()}"
-        # First occurrence wins (do not let later noise overwrite).
-        if key in out and not out[key]:
-            out[key] = num.replace(",", "")
-    at = ENGAGEMENT_AT_RE.search(blob)
-    if at:
-        out["engagement_at"] = at.group(1)
+    sec = ENGAGEMENT_SECTION_RE.search(text)
+    if sec:
+        _fill_engagement_from_blob(out, sec.group(1))
+    return out
+
+
+def merge_engagement(*parts: dict[str, str]) -> dict[str, str]:
+    out = _empty_engagement()
+    for part in parts:
+        for k, v in part.items():
+            if k in out and v and not out[k]:
+                out[k] = v
     return out
 
 def build_graph() -> dict:
@@ -359,7 +446,7 @@ def build_graph() -> dict:
     for p in profiles:
         base = p["path"].rstrip("/") + "/"
         files = {}
-        for name in ("README", "voice", "audience", "pillars"):
+        for name in ("README", "voice", "audience", "pillars", "lanes"):
             rel = f"{base}{name}.md"
             if (ROOT / rel).exists():
                 files[name.lower()] = rel
@@ -432,102 +519,142 @@ def build_graph() -> dict:
         for kind, lib_id in parse_library(row["library"]):
             edges.append({"from": nid("capture", ident), "to": nid(kind, lib_id), "rel": "ingested_as"})
 
-    for row in table_with(read("raw/_index.md"), "id", "date", "capture_id", "file", "type", "source_url", "one_liner"):
+    needs_md = read("needs/_index.md")
+    needs_rows = table_with(
+        needs_md,
+        "id",
+        "date",
+        "profile",
+        "source_type",
+        "quote",
+        "speaker",
+        "frequency",
+        "status",
+        "topic_ids",
+        "file",
+    )
+    for row in needs_rows:
         ident = row["id"]
-        fpath = wiki_path(row["file"])
-        if fpath and not fpath.endswith(".md"):
+        if not ident.startswith("N-"):
+            continue
+        fpath = wiki_path(row["file"]) or f"needs/entries/{ident}.md"
+        if not fpath.endswith(".md"):
             fpath = f"{fpath}.md"
+        topic_ids = (row.get("topic_ids") or "").strip()
+        quote = (row.get("quote") or "").strip()
         add_node(
             node(
-                "raw",
+                "need",
                 ident,
-                row["one_liner"],
+                quote or f"Need {ident}",
                 fpath,
                 {
                     "date": row["date"],
-                    "type": row["type"],
-                    "source_url": row["source_url"],
-                    "capture_id": row["capture_id"],
+                    "profile": row["profile"],
+                    "source_type": row["source_type"],
+                    "quote": quote,
+                    "speaker": row.get("speaker", "") or "",
+                    "frequency": row.get("frequency", "") or "1",
+                    "status": row["status"],
+                    "topic_ids": topic_ids,
                 },
             )
         )
-        if row["capture_id"]:
-            edge(nid("capture", row["capture_id"]), nid("raw", ident), "has_raw")
+        # Topic edges are wired after topic nodes exist (see post-pass below).
 
-    discovery_md = read("discovery/_index.md")
-    # Current minimal schema. Legacy rows may still carry relevance/ownership/candidates.
-    discovery_rows = table_with(
-        discovery_md,
-        "id",
-        "date",
-        "profile",
-        "platform",
-        "source_handle",
-        "source_url",
-        "source_claim",
-        "source_role",
-        "status",
-        "next_action",
-        "topic_ids",
-        "file",
-    )
-    if not discovery_rows:
-        discovery_rows = table_with(
-            discovery_md,
+    products_path = ROOT / "products" / "_index.md"
+    if products_path.exists():
+        product_rows = table_with(
+            products_path.read_text(encoding="utf-8"),
             "id",
-            "date",
-            "profile",
-            "platform",
-            "source_handle",
-            "source_url",
-            "source_claim",
-            "source_role",
-            "relevance",
-            "ownership",
-            "product_relevance",
+            "name",
+            "status",
+            "audience",
+            "price",
+            "effective_from",
+            "file",
+        )
+        for row in product_rows:
+            ident = row["id"]
+            if not ident.startswith("P-"):
+                continue
+            fpath = wiki_path(row["file"]) or f"products/entries/{ident}.md"
+            if not fpath.endswith(".md"):
+                fpath = f"{fpath}.md"
+            add_node(
+                node(
+                    "product",
+                    ident,
+                    row["name"] or ident,
+                    fpath,
+                    {
+                        "status": row["status"],
+                        "audience": row["audience"],
+                        "price": row["price"],
+                        "effective_from": row["effective_from"],
+                    },
+                )
+            )
+
+    recommendations_path = ROOT / "recommendations" / "_index.md"
+    if recommendations_path.exists():
+        recommendation_rows = table_with(
+            recommendations_path.read_text(encoding="utf-8"),
+            "id",
+            "name",
             "status",
-            "next_action",
-            "candidates",
-            "topic_ids",
+            "category",
+            "profiles",
+            "pillars",
+            "last_researched",
+            "relationship",
             "file",
         )
-    for row in discovery_rows:
+        for row in recommendation_rows:
+            ident = row["id"]
+            if not ident.startswith("R-"):
+                continue
+            fpath = wiki_path(row["file"]) or f"recommendations/{ident}.md"
+            if not fpath.endswith(".md"):
+                fpath = f"{fpath}.md"
+            add_node(
+                node(
+                    "recommendation",
+                    ident,
+                    row["name"] or ident,
+                    fpath,
+                    {
+                        "status": row["status"],
+                        "category": row["category"],
+                        "profiles": row["profiles"],
+                        "pillars": row["pillars"],
+                        "last_researched": row["last_researched"],
+                        "relationship": row["relationship"],
+                    },
+                )
+            )
+
+    for row in table_with(read("raw/_index.md"), "id", "date", "capture_id", "file", "type", "source_url", "one_liner"):
         ident = row["id"]
-        if not ident.startswith("D-"):
-            continue
         fpath = wiki_path(row["file"])
         if fpath and not fpath.endswith(".md"):
             fpath = f"{fpath}.md"
-        topic_ids = (row.get("topic_ids") or "").strip()
-        eng = parse_engagement_snapshot(read(fpath) if fpath and (ROOT / fpath).exists() else "")
         add_node(
             node(
-                "discovery",
+                "raw",
                 ident,
-                row["source_claim"] or f"Discovery {ident}",
+                row["one_liner"],
                 fpath,
                 {
                     "date": row["date"],
-                    "profile": row["profile"],
-                    "platform": row["platform"],
-                    "source_handle": row["source_handle"],
+                    "type": row["type"],
                     "source_url": row["source_url"],
-                    "source_claim": row["source_claim"],
-                    "source_role": row["source_role"],
-                    "relevance": row.get("relevance", "") or "",
-                    "ownership": row.get("ownership", "") or "",
-                    "product_relevance": row.get("product_relevance", "") or "",
-                    "status": row["status"],
-                    "next_action": row.get("next_action", "") or "",
-                    "candidates": row.get("candidates", "") or "",
-                    "topic_ids": topic_ids,
-                    **eng,
+                    "capture_id": row["capture_id"],
                 },
             )
         )
-        linked = re.findall(r"\bT-\d{8}-\d{2}\b", f"{topic_ids} {row.get('candidates') or ''}")
-        for tid in linked:
-            edge(nid("discovery", ident), nid("topic", tid), "candidate")
+        if row["capture_id"]:
+            edge(nid("capture", row["capture_id"]), nid("raw", ident), "has_raw")
 
     topic_md = read("topics/_index.md")
     # Current: opp + aud (5C opportunity). Legacy: craft + fit, then single total.
@@ -542,6 +669,8 @@ def build_graph() -> dict:
         "scout",
         "platform",
         "pillar",
+        "lane",
+        "generation_mode",
         "one_liner",
         "item",
         "source_type",
@@ -552,6 +681,10 @@ def build_graph() -> dict:
         "src_bookmarks",
         "engagement_at",
         "capture_id",
+        "need_ids",
+        "product_ids",
+        "product_fit",
+        "recommendation_ids",
         "run",
     )
     if topic_rows:
@@ -655,6 +788,8 @@ def build_graph() -> dict:
                     "status": row["status"],
                     "platform": row["platform"],
                     "pillar": row["pillar"],
+                    "lane": row.get("lane", "") or "",
+                    "generation_mode": row.get("generation_mode", "") or "",
                     "profile": profile,
                     "source_type": row["source_type"],
                     "src_url": row["src_url"],
@@ -664,6 +799,10 @@ def build_graph() -> dict:
                     "src_bookmarks": row["src_bookmarks"],
                     "engagement_at": row["engagement_at"],
                     "capture_id": row["capture_id"],
+                    "need_ids": row.get("need_ids", "") or "",
+                    "product_ids": row.get("product_ids", "") or "",
+                    "product_fit": row.get("product_fit", "") or "",
+                    "recommendation_ids": row.get("recommendation_ids", "") or "",
                     "run_id": row["run"].strip(),
                     "usage": usage,
                     "src_heat_note": "来源帖热度，不是你的帖",
@@ -674,10 +813,31 @@ def build_graph() -> dict:
             edge(nid("topic", ident), nid("capture", row["capture_id"]), "from_capture")
         if row["run"].strip().startswith("RUN-"):
             edge(nid("topic", ident), nid("run", row["run"].strip()), "opened_run")
+        # Index cells are enough for edges; item frontmatter may repeat the same ids.
+        for need_id in re.findall(r"\bN-\d{8}-\d{2}\b", row.get("need_ids", "") or ""):
+            edge(nid("topic", ident), nid("need", need_id), "from_need")
+        for product_id in PRODUCT_RE.findall(row.get("product_ids", "") or ""):
+            edge(nid("topic", ident), nid("product", product_id), "uses_product")
+        for recommendation_id in RECOMMENDATION_RE.findall(row.get("recommendation_ids", "") or ""):
+            edge(nid("topic", ident), nid("recommendation", recommendation_id), "reviews")
         if item and (ROOT / item).exists():
             body = read(item)
+            fm = re.match(r"^---\n(.*?)\n---", body, re.S)
+            if fm:
+                mneeds = re.search(r"^need_ids:\s*(.*)$", fm.group(1), re.M)
+                if mneeds:
+                    for need_id in re.findall(r"\bN-\d{8}-\d{2}\b", mneeds.group(1)):
+                        edge(nid("topic", ident), nid("need", need_id), "from_need")
+                mproducts = re.search(r"^product_ids:\s*(.*)$", fm.group(1), re.M)
+                if mproducts:
+                    for product_id in PRODUCT_RE.findall(mproducts.group(1)):
+                        edge(nid("topic", ident), nid("product", product_id), "uses_product")
+                mrecommendations = re.search(r"^recommendation_ids:\s*(.*)$", fm.group(1), re.M)
+                if mrecommendations:
+                    for recommendation_id in RECOMMENDATION_RE.findall(mrecommendations.group(1)):
+                        edge(nid("topic", ident), nid("recommendation", recommendation_id), "reviews")
             chunk = body
-            m = re.search(r"^## Suggested production.*?(?=^## |\Z)", body, re.M | re.S)
+            m = re.search(r"^## Suggested production.*?(?=^## |\Z)|^## Trace.*?(?=^## |\Z)", body, re.M | re.S)
             if m:
                 chunk = m.group(0)
             for line in chunk.splitlines():
@@ -697,6 +857,12 @@ def build_graph() -> dict:
                     for cid in ids_in(rest)["claim"]:
                         edges.append({"from": nid("topic", ident), "to": nid("claim", cid), "rel": "suggested"})
 
+    for n in nodes.values():
+        if n["kind"] != "need":
+            continue
+        for tid in re.findall(r"\bT-\d{8}-\d{2}\b", n.get("topic_ids") or ""):
+            edge(n["id"], nid("topic", tid), "evidence_for")
+
     for row in table_with(
         read("swipe/_index.md"),
         "id",
@@ -1004,18 +1170,35 @@ def build_graph() -> dict:
         for cid in field_ids(row["claim"], "claim"):
             edge(nid("published", ident), nid("claim", cid), "uses")
 
-    # Capture → surface source_url from linked Raw
+    # Capture → source_url from Raw + engagement from entry/raw notes
     for n in list(nodes.values()):
         if n["kind"] != "capture":
             continue
+        raw_node = None
         for e in edges:
             if e["rel"] != "has_raw" or e["from"] != n["id"]:
                 continue
-            raw = nodes.get(e["to"])
-            if raw and raw.get("source_url"):
-                n["source_url"] = raw["source_url"]
-                n["raw_id"] = raw["ident"]
-                break
+            raw_node = nodes.get(e["to"])
+            if raw_node and raw_node.get("source_url"):
+                n["source_url"] = raw_node["source_url"]
+                n["raw_id"] = raw_node["ident"]
+            break
+        entry_text = read(n["path"]) if n.get("path") and (ROOT / n["path"]).exists() else ""
+        raw_text = ""
+        raw_path = ""
+        if raw_node and raw_node.get("path"):
+            raw_path = raw_node["path"]
+        elif n.get("raw_path"):
+            raw_path = n["raw_path"] if str(n["raw_path"]).endswith(".md") else f'{n["raw_path"]}.md'
+        if raw_path and (ROOT / raw_path).exists():
+            raw_text = read(raw_path)
+        eng = merge_engagement(
+            parse_engagement_snapshot(entry_text),
+            parse_engagement_snapshot(raw_text),
+        )
+        for k, v in eng.items():
+            if v:
+                n[k] = v
 
     # drop edges to missing nodes
     edges = [e for e in edges if e["from"] in nodes and e["to"] in nodes]
@@ -1056,11 +1239,41 @@ def build_graph() -> dict:
         "accounts": accounts,
         "pillars_by_profile": pillars_by_profile,
         "pillars": flat_pillars,
+        "needs_index_present": (ROOT / "needs" / "_index.md").exists(),
+        "products_index_present": (ROOT / "products" / "_index.md").exists(),
         "nodes": list(nodes.values()),
         "edges": uniq,
     }
 
 
+def _self_check() -> None:
+    # ponytail: tiny parser asserts; expand if capture/raw formats drift again.
+    d = parse_engagement_snapshot(
+        "- **engagement snapshot:** likes 113 / replies 18 / bookmarks 17 / views 9574 @ 2026-08-30\n"
+    )
+    assert d == {
+        "src_likes": "113",
+        "src_replies": "18",
+        "src_bookmarks": "17",
+        "src_views": "9574",
+        "engagement_at": "2026-08-30",
+    }, d
+    c = parse_engagement_snapshot(
+        "6. **Engagement snapshot (fetch 2026-08-25):** ~92.9k views · 1086 likes · 1380 bookmarks · 738 replies · 70 reposts.\n"
+    )
+    assert c["src_views"] == "92900" and c["src_likes"] == "1086", c
+    assert c["src_bookmarks"] == "1380" and c["src_replies"] == "738", c
+    assert c["engagement_at"] == "2026-08-25", c
+    r = parse_engagement_snapshot(
+        "## Engagement snapshot\n\n- likes: 3\n- replies: 3\n- views: 426\n- bookmarks: 2\n- snapshot_at: 2026-09-13\n"
+    )
+    assert r["src_likes"] == "3" and r["src_views"] == "426" and r["engagement_at"] == "2026-09-13", r
+    m = parse_engagement_snapshot(
+        "## Snapshot metrics (at capture)\n\n- Views: ~8091\n- Likes: 103\n- Bookmarks: 29\n- Replies: 20\n"
+    )
+    assert m["src_views"] == "8091" and m["src_likes"] == "103", m
+    assert m["src_bookmarks"] == "29" and m["src_replies"] == "20", m
+    print("parse_engagement_snapshot self-check ok")
 def main() -> None:
     graph = build_graph()
     html = TPL.read_text(encoding="utf-8")
@@ -1071,4 +1284,9 @@ def main() -> None:
 
 
 if __name__ == "__main__":
-    main()
+    import sys
+
+    if "--self-check" in sys.argv:
+        _self_check()
+    else:
+        main()