@brainpilot/skills 0.0.6 → 0.0.8

package/skills/01_Meta-Skills/ethoclaw-daily-paper/scripts/run_pipeline.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+if str(SCRIPT_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPT_DIR))
+
+from common import load_config  # noqa: E402
+
+
+SKILL_DIR = Path(__file__).resolve().parents[1]
+SCRIPTS_DIR = SKILL_DIR / "scripts"
+DEFAULT_CONFIG = SKILL_DIR / "assets" / "config.template.yaml"
+
+
+def coerce_float(value, default):
+    if value in (None, ""):
+        return default
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def run_command(command, cwd):
+    subprocess.run(command, cwd=str(cwd), check=True)
+
+
+def join_values(values, limit=None):
+    if not values:
+        return "Unknown"
+    selected = values[:limit] if limit else values
+    return ", ".join(selected)
+
+
+def render_candidate_titles(payload):
+    lines = [
+        "# Candidate Titles Passed to Agent",
+        "",
+        f"- Generated At: {payload.get('generated_at', '')}",
+        f"- Candidate Count: {payload.get('count', 0)}",
+        "",
+    ]
+    for index, item in enumerate(payload.get("items", []), start=1):
+        source = item.get("source_label") or item.get("source", "unknown")
+        published = item.get("published", "")[:10] or "unknown"
+        lines.append(f"{index}. [{source}] {item.get('title', '').strip() or '(untitled)'} ({published})")
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def render_candidate_pool(payload):
+    lines = [
+        "# Neuroethology Candidate Pool",
+        "",
+        f"- Generated At: {payload.get('generated_at', '')}",
+        f"- Candidate Count: {payload.get('count', 0)}",
+        "",
+    ]
+    for index, item in enumerate(payload.get("items", []), start=1):
+        lines.extend(
+            [
+                f"## {index}. {item.get('title', '').strip() or '(untitled)'}",
+                "",
+                f"- Source: {item.get('source_label') or item.get('source', 'unknown')}",
+                f"- Authors: {join_values(item.get('authors', []), limit=10)}",
+                f"- Published: {item.get('published', '')[:10] or 'Unknown'}",
+                f"- Journal: {item.get('journal') or 'Unknown'}",
+                f"- Link: {item.get('url') or 'N/A'}",
+                f"- PDF Link: {item.get('pdf_url') or 'N/A'}",
+                f"- Weighted Score: {item.get('weighted_score', 'Unknown')}",
+                "",
+                "### Original Abstract",
+                "",
+                item.get("summary", "").strip() or "No abstract available",
+                "",
+            ]
+        )
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run the full neuroethology Top 5 literature workflow.")
+    parser.add_argument("--query-file", default="", help="Config file path")
+    parser.add_argument("--output-dir", required=True, help="Directory for all generated files")
+    parser.add_argument("--days", type=int, default=7, help="Recency window in days")
+    parser.add_argument("--selected-indexes", default="", help="Comma-separated candidate indexes chosen by the agent")
+    args = parser.parse_args()
+
+    config_path = Path(args.query_file).expanduser().resolve() if args.query_file else DEFAULT_CONFIG
+    config = load_config(str(config_path))
+
+    output_dir = Path(args.output_dir).expanduser().resolve()
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    arxiv_output = output_dir / "arxiv.json"
+    pubmed_output = output_dir / "pubmed.json"
+    merged_output = output_dir / "merged.json"
+    candidate_titles_output = output_dir / "candidate_titles.md"
+    candidate_pool_output = output_dir / "candidate_pool.md"
+    agent_packet_json_output = output_dir / "top5_agent_packet.json"
+    agent_packet_md_output = output_dir / "top5_agent_packet.md"
+
+    run_command(
+        [
+            sys.executable,
+            str(SCRIPTS_DIR / "search_arxiv.py"),
+            "--query-file",
+            str(config_path),
+            "--days",
+            str(args.days),
+            "--output",
+            str(arxiv_output),
+        ],
+        SKILL_DIR,
+    )
+
+    run_command(
+        [
+            sys.executable,
+            str(SCRIPTS_DIR / "search_pubmed.py"),
+            "--query-file",
+            str(config_path),
+            "--days",
+            str(args.days),
+            "--output",
+            str(pubmed_output),
+        ],
+        SKILL_DIR,
+    )
+
+    run_command(
+        [
+            sys.executable,
+            str(SCRIPTS_DIR / "merge_results.py"),
+            str(arxiv_output),
+            str(pubmed_output),
+            "--pubmed-weight",
+            str(coerce_float(config.get("source_weight_pubmed"), 1.25)),
+            "--arxiv-weight",
+            str(coerce_float(config.get("source_weight_arxiv"), 1.0)),
+            "--output",
+            str(merged_output),
+        ],
+        SKILL_DIR,
+    )
+
+    merged_payload = json.loads(merged_output.read_text(encoding="utf-8-sig"))
+    candidate_titles_output.write_text(render_candidate_titles(merged_payload), encoding="utf-8")
+    candidate_pool_output.write_text(render_candidate_pool(merged_payload), encoding="utf-8")
+
+    if args.selected_indexes:
+        run_command(
+            [
+                sys.executable,
+                str(SCRIPTS_DIR / "build_top5_digest.py"),
+                "--input",
+                str(merged_output),
+                "--paper-indexes",
+                args.selected_indexes,
+                "--agent-json-output",
+                str(agent_packet_json_output),
+                "--review-md-output",
+                str(agent_packet_md_output),
+            ],
+            SKILL_DIR,
+        )
+
+    print(output_dir)
+
+
+if __name__ == "__main__":
+    main()
+```

package/skills/01_Meta-Skills/ethoclaw-daily-paper/scripts/search_arxiv.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+import argparse
+import datetime as dt
+import json
+import sys
+import urllib.parse
+import urllib.request
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+if str(SCRIPT_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPT_DIR))
+
+from common import load_config, normalize  # noqa: E402
+
+ATOM_NS = {"atom": "http://www.w3.org/2005/Atom"}
+
+
+def fetch_feed(query: str, start: int, max_results: int):
+    params = {
+        "search_query": query,
+        "start": str(start),
+        "max_results": str(max_results),
+        "sortBy": "submittedDate",
+        "sortOrder": "descending",
+    }
+    url = "http://export.arxiv.org/api/query?" + urllib.parse.urlencode(params)
+    with urllib.request.urlopen(url, timeout=30) as resp:
+        return resp.read()
+
+
+def text(node, expr, default=""):
+    found = node.find(expr, ATOM_NS)
+    return found.text.strip() if found is not None and found.text else default
+
+
+def arxiv_id_from_entry(entry):
+    raw_id = text(entry, "atom:id")
+    return raw_id.rsplit("/", 1)[-1] if raw_id else ""
+
+
+def entry_to_doc(entry):
+    title = text(entry, "atom:title")
+    summary = text(entry, "atom:summary")
+    published = text(entry, "atom:published")
+    updated = text(entry, "atom:updated")
+    entry_id = arxiv_id_from_entry(entry)
+    authors = [a.text.strip() for a in entry.findall("atom:author/atom:name", ATOM_NS) if a.text]
+    categories = [c.attrib.get("term", "") for c in entry.findall("atom:category", ATOM_NS)]
+    links = entry.findall("atom:link", ATOM_NS)
+    pdf_url = ""
+    primary_url = text(entry, "atom:id")
+    for link in links:
+        title_attr = link.attrib.get("title", "")
+        href = link.attrib.get("href", "")
+        if title_attr == "pdf":
+            pdf_url = href
+            break
+    combined = normalize(" ".join([title, summary, " ".join(categories)]))
+    return {
+        "id": entry_id,
+        "title": title,
+        "summary": summary,
+        "published": published,
+        "updated": updated,
+        "authors": authors,
+        "categories": categories,
+        "url": primary_url,
+        "pdf_url": pdf_url,
+        "search_blob": combined,
+        "source": "arxiv",
+        "source_label": "arXiv",
+    }
+
+
+def iso_to_dt(value: str):
+    return dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
+
+
+def score_doc(doc, include_keywords):
+    blob = doc["search_blob"]
+    return sum(1 for kw in include_keywords if kw.lower() in blob)
+
+
+def category_bonus(doc):
+    categories = set(doc.get("categories", []))
+    bonus = 0
+    for cat in categories:
+        if cat.startswith("q-bio.NC"):
+            bonus += 3
+        elif cat.startswith("q-bio"):
+            bonus += 2
+        elif cat.startswith("nlin.AO") or cat.startswith("physics.bio-ph"):
+            bonus += 1
+    return bonus
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Fetch and filter recent arXiv papers.")
+    parser.add_argument("--query", help="arXiv search query")
+    parser.add_argument("--query-file", help="Simple YAML config file")
+    parser.add_argument("--days", type=int, default=3)
+    parser.add_argument("--max-results", type=int, default=15)
+    parser.add_argument("--output", help="Write JSON to this file")
+    args = parser.parse_args()
+
+    config = load_config(args.query_file)
+    query = args.query or config.get("query") or "all:neuroscience"
+    include_keywords = [s.lower() for s in config.get("include_keywords", [])]
+    exclude_keywords = [s.lower() for s in config.get("exclude_keywords", [])]
+    max_results = int(config.get("max_results", args.max_results))
+    days = args.days
+
+    feed = fetch_feed(query, start=0, max_results=max_results)
+    root = ET.fromstring(feed)
+    now = dt.datetime.now(dt.timezone.utc)
+    cutoff = now - dt.timedelta(days=days)
+
+    docs = []
+    seen = set()
+    for entry in root.findall("atom:entry", ATOM_NS):
+        doc = entry_to_doc(entry)
+        if not doc["id"] or doc["id"] in seen:
+            continue
+        try:
+            published_dt = iso_to_dt(doc["published"])
+        except Exception:
+            continue
+        if published_dt < cutoff:
+            continue
+        blob = doc["search_blob"]
+        if include_keywords and not any(kw in blob for kw in include_keywords):
+            continue
+        if exclude_keywords and any(kw in blob for kw in exclude_keywords):
+            continue
+        if "neural network" in blob and not any(term in blob for term in ["brain", "neuron", "neuronal", "cortex", "hippocampus", "spike", "synapse", "electrophysiology", "animal behavior", "behaviour", "zebrafish", "rodent", "mouse", "primate"]):
+            continue
+        doc["keyword_score"] = score_doc(doc, include_keywords)
+        doc["relevance_score"] = doc.get("keyword_score", 0) + category_bonus(doc)
+        docs.append(doc)
+        seen.add(doc["id"])
+
+    docs.sort(key=lambda d: (d.get("relevance_score", 0), d["published"]), reverse=True)
+    payload = {
+        "query": query,
+        "generated_at": now.isoformat(),
+        "days": days,
+        "count": len(docs),
+        "items": docs,
+    }
+
+    if args.output:
+        out_path = Path(args.output)
+        out_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+    else:
+        json.dump(payload, sys.stdout, ensure_ascii=False, indent=2)
+        sys.stdout.write("\n")
+
+
+if __name__ == "__main__":
+    main()

package/skills/01_Meta-Skills/ethoclaw-daily-paper/scripts/search_pubmed.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+import argparse
+import datetime as dt
+import json
+import sys
+import time
+import urllib.parse
+import urllib.request
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+if str(SCRIPT_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPT_DIR))
+
+from common import load_config, normalize  # noqa: E402
+
+BASE_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
+
+
+def http_get(url: str, params: dict):
+    full_url = url + "?" + urllib.parse.urlencode(params)
+    req = urllib.request.Request(full_url, headers={"User-Agent": "OpenClaw neuro-paper-monitor/1.0"})
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        return resp.read()
+
+
+def search_pubmed(term: str, retmax: int, days: int):
+    payload = http_get(
+        BASE_URL + "esearch.fcgi",
+        {
+            "db": "pubmed",
+            "term": term,
+            "retmax": str(retmax),
+            "retmode": "json",
+            "sort": "pub date",
+            "datetype": "pdat",
+            "reldate": str(days),
+        },
+    )
+    data = json.loads(payload.decode("utf-8"))
+    return data.get("esearchresult", {}).get("idlist", [])
+
+
+def chunked(seq, size):
+    for i in range(0, len(seq), size):
+        yield seq[i : i + size]
+
+
+def text_join(node, path_expr):
+    found = node.findall(path_expr)
+    values = []
+    for item in found:
+        txt = "".join(item.itertext()).strip()
+        if txt:
+            values.append(txt)
+    return " ".join(values).strip()
+
+
+def parse_pub_date(article):
+    pub_date = article.find(".//PubDate")
+    if pub_date is None:
+        return ""
+    year = text_join(pub_date, "Year") or text_join(pub_date, "MedlineDate")[:4]
+    month_raw = text_join(pub_date, "Month")
+    day = text_join(pub_date, "Day") or "01"
+    month_map = {
+        "Jan": "01", "Feb": "02", "Mar": "03", "Apr": "04", "May": "05", "Jun": "06",
+        "Jul": "07", "Aug": "08", "Sep": "09", "Oct": "10", "Nov": "11", "Dec": "12",
+    }
+    month = month_map.get(month_raw[:3], month_raw.zfill(2) if month_raw.isdigit() else "01")
+    if not year:
+        return ""
+    return f"{year}-{month}-{day.zfill(2)}"
+
+
+def fetch_details(pmids):
+    docs = []
+    for batch in chunked(pmids, 100):
+        payload = http_get(
+            BASE_URL + "efetch.fcgi",
+            {
+                "db": "pubmed",
+                "id": ",".join(batch),
+                "retmode": "xml",
+            },
+        )
+        root = ET.fromstring(payload)
+        for article in root.findall(".//PubmedArticle"):
+            pmid = text_join(article, ".//PMID")
+            title = text_join(article, ".//ArticleTitle")
+            abstract = text_join(article, ".//Abstract/AbstractText")
+            journal = text_join(article, ".//Journal/Title")
+            published = parse_pub_date(article)
+            doi = ""
+            for el in article.findall(".//ArticleId"):
+                if el.attrib.get("IdType") == "doi" and (el.text or "").strip():
+                    doi = el.text.strip()
+                    break
+            authors = []
+            for author in article.findall(".//Author"):
+                last = text_join(author, "LastName")
+                fore = text_join(author, "ForeName")
+                collective = text_join(author, "CollectiveName")
+                name = collective or " ".join(part for part in [fore, last] if part).strip()
+                if name:
+                    authors.append(name)
+            mesh_terms = [text_join(mh, ".") for mh in article.findall(".//MeshHeading/DescriptorName")]
+            keywords = [text_join(kw, ".") for kw in article.findall(".//Keyword")]
+            blob = normalize(" ".join([title, abstract, journal, " ".join(mesh_terms), " ".join(keywords)]))
+            docs.append(
+                {
+                    "id": pmid,
+                    "pmid": pmid,
+                    "doi": doi,
+                    "title": title,
+                    "summary": abstract,
+                    "published": published,
+                    "updated": published,
+                    "authors": authors,
+                    "journal": journal,
+                    "mesh_terms": mesh_terms,
+                    "keywords": keywords,
+                    "url": f"https://pubmed.ncbi.nlm.nih.gov/{pmid}/" if pmid else "",
+                    "pdf_url": "",
+                    "search_blob": blob,
+                    "source": "pubmed",
+                    "source_label": "PubMed",
+                }
+            )
+        time.sleep(0.34)
+    return docs
+
+
+def score_doc(doc, include_keywords):
+    blob = doc.get("search_blob", "")
+    return sum(1 for kw in include_keywords if kw.lower() in blob)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Fetch and filter recent PubMed papers.")
+    parser.add_argument("--query", help="PubMed query")
+    parser.add_argument("--query-file", help="Simple YAML config file")
+    parser.add_argument("--days", type=int, default=3)
+    parser.add_argument("--max-results", type=int, default=15)
+    parser.add_argument("--output", help="Write JSON to this file")
+    args = parser.parse_args()
+
+    config = load_config(args.query_file)
+    default_query = '(("Neurosciences"[mh] OR neuroscience[tiab] OR neuroethology[tiab]) AND ("Behavior, Animal"[mh] OR "animal behavior"[tiab] OR behaviour[tiab] OR behavior[tiab] OR electrophysiology[tiab] OR "calcium imaging"[tiab] OR hippocampus[tiab] OR cortex[tiab] OR zebrafish[tiab] OR rodent[tiab] OR mouse[tiab] OR primate[tiab]))'
+    query = args.query or config.get("pubmed_query") or default_query
+    include_keywords = [s.lower() for s in config.get("include_keywords", [])]
+    exclude_keywords = [s.lower() for s in config.get("exclude_keywords", [])]
+    max_results = int(config.get("pubmed_max_results", config.get("max_results", args.max_results)))
+    days = args.days
+
+    pmids = search_pubmed(query, retmax=max_results, days=days)
+    docs = fetch_details(pmids)
+
+    now = dt.datetime.now(dt.timezone.utc)
+    cutoff = now - dt.timedelta(days=days)
+    filtered = []
+    seen = set()
+    for doc in docs:
+        if not doc.get("id") or doc["id"] in seen:
+            continue
+        try:
+            published_dt = dt.datetime.fromisoformat(doc["published"] + "T00:00:00+00:00")
+        except Exception:
+            continue
+        if published_dt < cutoff:
+            continue
+        blob = doc["search_blob"]
+        if include_keywords and not any(kw in blob for kw in include_keywords):
+            continue
+        if exclude_keywords and any(kw in blob for kw in exclude_keywords):
+            continue
+        doc["keyword_score"] = score_doc(doc, include_keywords)
+        doc["relevance_score"] = doc["keyword_score"] + (2 if doc.get("mesh_terms") else 0)
+        filtered.append(doc)
+        seen.add(doc["id"])
+
+    filtered.sort(key=lambda d: (d.get("relevance_score", 0), d.get("published", "")), reverse=True)
+    payload = {
+        "query": query,
+        "generated_at": now.isoformat(),
+        "days": days,
+        "count": len(filtered),
+        "source": "pubmed",
+        "items": filtered,
+    }
+
+    if args.output:
+        out_path = Path(args.output)
+        out_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+    else:
+        json.dump(payload, sys.stdout, ensure_ascii=False, indent=2)
+        sys.stdout.write("\n")
+
+
+if __name__ == "__main__":
+    main()