@m13v/s4l 1.6.197-rc.10

package/scripts/update_instagram_stats.py

@@ -0,0 +1,261 @@
+#!/usr/bin/env python3
+"""Refresh engagement stats for Instagram posts via the IG Graph API.
+
+For each posts.platform='instagram' row, look up its media_id (matching by
+permalink against /me/media for the account), then call /{media-id}/insights
+to fetch views, reach, likes, comments, saved, shares. Write into the same
+flat columns the rest of the dashboard uses (posts.upvotes/comments_count/
+views) plus engagement_updated_at, and snapshot to post_views_daily for the
+Trends tab.
+
+Source-of-truth mapping (per `media_posts.target_account`):
+  matt_diak       -> IG_USER_ID + IG_LONG_TOKEN  OR
+                     IG_USER_ID_MATTDIAK + IG_LONG_TOKEN_MATTDIAK
+  matthewheartful -> IG_USER_ID_MATTHEWHEARTFUL + IG_LONG_TOKEN_MATTHEWHEARTFUL
+  omidotme        -> IG_USER_ID_OMIDOTME + IG_LONG_TOKEN_OMIDOTME
+
+Likes -> posts.upvotes (LinkedIn/Twitter convention).
+Views -> posts.views.
+Comments -> posts.comments_count.
+
+Usage:
+    python3 scripts/update_instagram_stats.py [--quiet] [--limit N]
+"""
+
+import argparse
+import json
+import os
+import re
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from http_api import api_get, api_patch, api_post
+
+
+IG_ENV_PATH = Path.home() / "instagram-graph-api" / ".env"
+GRAPH = "https://graph.instagram.com/v22.0"
+SA_CONFIG = Path(__file__).resolve().parent.parent / "config.json"
+
+
+# ── env / credentials ─────────────────────────────────────────────────────────
+
+def load_ig_env():
+    env = {}
+    for line in IG_ENV_PATH.read_text().splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        k, v = line.split("=", 1)
+        env[k.strip()] = v.strip()
+    return env
+
+
+def resolve_account_creds(account_name, ig_env, accounts_cfg):
+    match = next(
+        (a for a in accounts_cfg if a.get("username", "").lower() == account_name.lower()),
+        None,
+    )
+    if match:
+        uid = ig_env.get(match.get("ig_user_id_env", "IG_USER_ID"))
+        tok = ig_env.get(match.get("ig_long_token_env", "IG_LONG_TOKEN"))
+        if uid and tok:
+            return uid, tok
+    # Legacy bare-env fallback (matt_diak historically used IG_USER_ID/IG_LONG_TOKEN).
+    uid = ig_env.get("IG_USER_ID")
+    tok = ig_env.get("IG_LONG_TOKEN")
+    return uid, tok
+
+
+# ── Graph API helpers ─────────────────────────────────────────────────────────
+
+def graph_get(path, token, **params):
+    params["access_token"] = token
+    url = f"{GRAPH}/{path}?{urllib.parse.urlencode(params)}"
+    with urllib.request.urlopen(url, timeout=20) as r:
+        return json.loads(r.read())
+
+
+def shortcode_from_url(url):
+    """Pull the IG shortcode out of a permalink.
+    https://www.instagram.com/reel/DYkkj8RDo9P/ -> DYkkj8RDo9P
+    """
+    m = re.search(r"/(?:reel|p|tv)/([A-Za-z0-9_-]+)", url or "")
+    return m.group(1) if m else None
+
+
+def fetch_media_map(ig_user_id, token, max_pages=10):
+    """Return {shortcode: {id, media_product_type, like_count, comments_count}}
+    for the account's recent media. Pages through /me/media until exhaustion or
+    max_pages safety cap.
+    """
+    out = {}
+    fields = "id,media_type,media_product_type,permalink,like_count,comments_count"
+    url = f"{GRAPH}/{ig_user_id}/media?fields={fields}&limit=100&access_token={token}"
+    pages = 0
+    while url and pages < max_pages:
+        with urllib.request.urlopen(url, timeout=20) as r:
+            data = json.loads(r.read())
+        for item in data.get("data", []):
+            code = shortcode_from_url(item.get("permalink"))
+            if code:
+                out[code] = item
+        url = (data.get("paging") or {}).get("next")
+        pages += 1
+    return out
+
+
+def fetch_insights(media_id, mtype, token):
+    """Return {metric_name: value}. Pick metrics based on media type."""
+    if mtype == "REELS":
+        metrics = "views,reach,likes,comments,saved,shares,total_interactions"
+    elif mtype == "VIDEO":
+        metrics = "views,reach,likes,comments,saved,shares"
+    else:
+        metrics = "reach,likes,comments,saved,shares"
+    try:
+        data = graph_get(f"{media_id}/insights", token, metric=metrics)
+    except urllib.error.HTTPError as e:
+        return {"__error__": f"HTTP {e.code}: {e.read().decode()[:200]}"}
+    result = {}
+    for m in data.get("data", []):
+        name = m.get("name")
+        vals = m.get("values") or []
+        result[name] = (vals[0] or {}).get("value") if vals else None
+    return result
+
+
+# ── main ──────────────────────────────────────────────────────────────────────
+
+def log(msg, quiet=False):
+    if not quiet:
+        print(msg)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--quiet", action="store_true")
+    parser.add_argument("--limit", type=int, default=None)
+    args = parser.parse_args()
+
+    ig_env = load_ig_env()
+    try:
+        cfg = json.loads(SA_CONFIG.read_text())
+    except FileNotFoundError:
+        cfg = {}
+    accounts_cfg = ((cfg.get("instagram") or {}).get("accounts") or [])
+
+    resp = api_get(
+        "/api/v1/posts",
+        query={
+            "platform": "instagram",
+            "status": "active",
+            "has_our_url": "true",
+            "order_by": "id",
+            "order_dir": "asc",
+            "limit": 500,
+        },
+    )
+    rows = (resp.get("data") or {}).get("posts") or []
+    if args.limit:
+        rows = rows[: args.limit]
+
+    log(f"[stats-ig] {len(rows)} active IG rows to check", args.quiet)
+
+    by_account = {}
+    for r in rows:
+        by_account.setdefault(r["our_account"], []).append(r)
+
+    checked = 0
+    updated = 0
+    failed = 0
+    not_found = 0
+    views_refreshed = 0
+
+    for account, account_rows in by_account.items():
+        uid, tok = resolve_account_creds(account, ig_env, accounts_cfg)
+        if not uid or not tok:
+            log(f"[stats-ig] missing creds for account={account}; skipping {len(account_rows)} rows", args.quiet)
+            failed += len(account_rows)
+            continue
+        try:
+            media_map = fetch_media_map(uid, tok)
+        except Exception as e:
+            log(f"[stats-ig] media list failed for {account}: {e}; skipping {len(account_rows)} rows", args.quiet)
+            failed += len(account_rows)
+            continue
+        log(f"[stats-ig] account={account} media-map size={len(media_map)}", args.quiet)
+
+        for r in account_rows:
+            checked += 1
+            code = shortcode_from_url(r["our_url"])
+            if not code:
+                log(f"[stats-ig]   id={r['id']} no shortcode in {r['our_url']}", args.quiet)
+                not_found += 1
+                continue
+            item = media_map.get(code)
+            if not item:
+                log(f"[stats-ig]   id={r['id']} shortcode={code} not in /me/media listing", args.quiet)
+                not_found += 1
+                continue
+
+            media_id = item["id"]
+            mtype = item.get("media_product_type") or item.get("media_type")
+            ins = fetch_insights(media_id, mtype, tok)
+            if "__error__" in ins:
+                log(f"[stats-ig]   id={r['id']} insights error: {ins['__error__']}", args.quiet)
+                failed += 1
+                continue
+
+            likes = ins.get("likes") or item.get("like_count") or 0
+            comments = ins.get("comments") or item.get("comments_count") or 0
+            views = ins.get("views")  # None for photos; that's fine
+
+            old = (r["upvotes"] or 0, r["comments_count"] or 0, r["views"] or 0)
+            new = (likes, comments, views or 0)
+
+            api_patch(
+                f"/api/v1/posts/{r['id']}",
+                {
+                    "upvotes": likes,
+                    "comments_count": comments,
+                    "views": views,
+                    "stamp_engagement_now": True,
+                    "stamp_status_checked_now": True,
+                    "reset_deletion_detect_count": True,
+                },
+            )
+            if views is not None:
+                api_post(
+                    "/api/v1/post-views-daily/snapshot",
+                    {"post_id": r["id"], "views": views},
+                )
+                views_refreshed += 1
+            if new != old:
+                updated += 1
+            log(
+                f"[stats-ig]   id={r['id']} code={code} type={mtype} "
+                f"likes={likes} comments={comments} views={views}",
+                args.quiet,
+            )
+            # Be polite to the Graph API.
+            time.sleep(0.2)
+
+    log(
+        f"[stats-ig] done: checked={checked} updated={updated} "
+        f"not_found={not_found} failed={failed} views_refreshed={views_refreshed}",
+        args.quiet,
+    )
+    # Machine-readable summary for the shell wrapper to consume.
+    print(
+        f"SUMMARY:CHECKED={checked} UPDATED={updated} NOT_FOUND={not_found} "
+        f"FAILED={failed} VIEWS_REFRESHED={views_refreshed}"
+    )
+
+
+if __name__ == "__main__":
+    main()

package/scripts/update_linkedin_stats_from_feed.py

@@ -0,0 +1,328 @@
+#!/usr/bin/env python3
+"""Update LinkedIn engagement stats for OUR engagement-comments stored in `posts`.
+
+This is the posts-table sibling of update_linkedin_comment_stats_from_feed.py.
+Same feed JSON shape (produced by skill/stats-linkedin.sh via
+scrape_linkedin_comment_stats.py); different DB target.
+
+Why this exists separately from the replies-table updater:
+  LinkedIn engagement-comments are stored in the `posts` table (Twitter
+  parity: posts table holds top-level + reply rows alike, identified by
+  the URL extracted from `our_url`). The legacy `replies` table holds an
+  older sliver (~173 rows) from a previous pipeline shape and is updated
+  by update_linkedin_comment_stats_from_feed.py. New rows land in `posts`.
+
+Input JSON shape (one record per OUR comment visible on the activity tab,
+virtualized list, partial coverage per fire is expected; identical to the
+shape the replies-table updater consumes):
+  [
+    {
+      "comment_id":  "7457492815716032512",
+      "parent_kind": "ugcPost"  | "activity" | "share",
+      "parent_id":   "7457485938131161088",
+      "impressions": 156,
+      "reactions":   7,
+      "replies":     1
+    },
+    ...
+  ]
+
+Matching strategy:
+  - Posts written 2026-05-11 onward have `our_url` containing
+    `?commentUrn=urn:li:comment:(...,<comment_id>)` because
+    linkedin_api.py:comment_on_post now embeds it (and reply_to_comment
+    already did). We parse comment_id out of `our_url` and key on it.
+  - Older rows (pre-2026-05-11) where the autoposter set
+    `our_url = thread_url` (parent post URL only, no commentUrn) cannot
+    be matched here and will silently miss. They are not backfillable
+    without per-permalink scraping (the exact pattern that triggered
+    the 2026-04-17 + 2026-05-05 LinkedIn lockouts), so we accept the
+    loss. Going forward every new engagement-comment is captured.
+  - The 97 pre-existing rows that already have `?commentUrn=` in
+    `our_url` (replies-to-comments via reply_to_comment) work
+    immediately.
+
+Behavior:
+  - Match each feed record by comment_id against posts.our_url's
+    `commentUrn=` second-numeric-id field.
+  - If matched: write upvotes (=reactions), comments_count (=replies),
+    views (=impressions), engagement_updated_at = NOW(). Only overwrite
+    a column when the new value is non-null.
+  - Unmatched feed rows are logged but NOT errors (the same feed JSON
+    is consumed by the replies-table updater immediately after this
+    script, so a row unmatched here might match there).
+  - scan_no_change_count IS maintained, matching stats.py's
+    Twitter behavior so dashboard sorting / freshness gates work the
+    same way across platforms.
+
+Output (stdout) one line for stats.sh's extract_field to parse:
+  LinkedInPosts: <T> total, <S> skipped, <C> checked,
+                 <U> updated, <D> deleted, <E> errors
+
+Usage:
+  python3 scripts/update_linkedin_stats_from_feed.py \\
+      --from-json /tmp/li-stats-feed.json \\
+      [--summary  /tmp/li-stats-summary.json] \\
+      [--dry-run] [--quiet]
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+import urllib.parse
+from typing import Optional
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from http_api import api_get, api_post  # noqa: E402
+
+
+# `urn:li:comment:(urn:li:activity:<parent>,<comment_id>)`
+# `urn:li:comment:(urn:li:ugcPost:<parent>,<comment_id>)`
+# `urn:li:comment:(activity:<parent>,<comment_id>)`     ← bare-kind form
+# `urn:li:comment:(ugcPost:<parent>,<comment_id>)`      ← bare-kind form
+#
+# Same lenient regex as update_linkedin_comment_stats_from_feed.py — the
+# inner `urn:li:` prefix on the parent namespace is optional because both
+# forms appear in our data depending on which posting path wrote the row.
+COMMENT_URN_RE = re.compile(
+    r"urn:li:comment:\((?:urn:li:)?(?P<kind>\w+):(?P<parent>\d+),(?P<cid>\d+)\)"
+)
+
+
+def extract_comment_id(our_url: Optional[str]) -> Optional[tuple[str, str, str]]:
+    """Return (parent_kind, parent_id, comment_id) parsed from our_url, or None."""
+    if not our_url:
+        return None
+    decoded = urllib.parse.unquote(our_url)
+    m = COMMENT_URN_RE.search(decoded)
+    if not m:
+        return None
+    return (m.group("kind"), m.group("parent"), m.group("cid"))
+
+
+def load_feed(path: str) -> list[dict]:
+    with open(path) as f:
+        raw = json.load(f)
+    if not isinstance(raw, list):
+        raise ValueError(f"feed file must be a JSON array, got {type(raw).__name__}")
+    out = []
+    for r in raw:
+        if not isinstance(r, dict):
+            continue
+        cid = r.get("comment_id")
+        if not cid:
+            continue
+        out.append({
+            "comment_id":  str(cid),
+            "parent_kind": r.get("parent_kind") or "",
+            "parent_id":   str(r.get("parent_id") or ""),
+            "impressions": r.get("impressions"),
+            "reactions":   r.get("reactions"),
+            "replies":     r.get("replies"),
+        })
+    return out
+
+
+def load_engagement_comments() -> dict:
+    """Return {comment_id: {id, our_url, upvotes, comments_count, views}}.
+
+    Only includes LinkedIn `posts` rows where `our_url` carries a
+    commentUrn (i.e., we can identify OUR comment, not just the parent
+    thread). Status='active' OR 'removed' (removed rows still benefit
+    from a final-stats read in case they come back).
+
+    Migrated 2026-06-01 to GET /api/v1/linkedin-engagement-comments. The
+    server returns every candidate row; the brittle commentUrn regex
+    (extract_comment_id) stays single-sourced here in Python so the API
+    surface never has to replicate it.
+    """
+    resp = api_get("/api/v1/linkedin-engagement-comments")
+    rows = (resp.get("data") or {}).get("rows") or []
+    out = {}
+    for r in rows:
+        parsed = extract_comment_id(r.get("our_url"))
+        if not parsed:
+            continue
+        _, _, cid = parsed
+        out[cid] = {
+            "id":             r["id"],
+            "our_url":        r["our_url"],
+            "upvotes":        int(r.get("upvotes") or 0),
+            "comments_count": int(r.get("comments_count") or 0),
+            "views":          int(r.get("views") or 0),
+        }
+    return out
+
+
+def compute_one(db_row: dict, feed: dict, dry_run: bool, quiet: bool) -> dict:
+    """Compute the update for one feed record against one DB row.
+
+    Returns {post_id, upvotes, comments_count, views, changed}. Only
+    overwrites a column when the feed value is non-null (preserves last
+    known value for fresh comments that don't yet have impressions
+    computed by LinkedIn). The actual write (UPDATE posts +
+    scan_no_change_count maintenance + post_views_daily snapshot) happens
+    server-side when the batch is POSTed.
+    """
+    new_rxn = feed["reactions"]
+    new_imp = feed["impressions"]
+    new_rep = feed["replies"]
+
+    next_upv = db_row["upvotes"]        if new_rxn is None else int(new_rxn)
+    next_cmt = db_row["comments_count"] if new_rep is None else int(new_rep)
+    next_vws = db_row["views"]          if new_imp is None else int(new_imp)
+
+    changed = (
+        next_upv != db_row["upvotes"]
+        or next_cmt != db_row["comments_count"]
+        or next_vws != db_row["views"]
+    )
+
+    if not quiet:
+        tag = "UPDATED" if changed else "same"
+        if dry_run:
+            tag = f"DRY-{tag}"
+        print(
+            f"  [{db_row['id']:>6}] cid={feed['comment_id']:>20s} "
+            f"upv {db_row['upvotes']}->{next_upv}  "
+            f"cmt {db_row['comments_count']}->{next_cmt}  "
+            f"views {db_row['views']}->{next_vws}  [{tag}]",
+            flush=True,
+        )
+
+    return {
+        "post_id":        db_row["id"],
+        "upvotes":        next_upv,
+        "comments_count": next_cmt,
+        "views":          next_vws,
+        "changed":        changed,
+    }
+
+
+def run(from_json: str,
+        summary_path: Optional[str],
+        dry_run: bool,
+        quiet: bool) -> dict:
+    feed = load_feed(from_json)
+    if not feed:
+        return {
+            "ok": True,
+            "total": 0, "skipped": 0, "checked": 0,
+            "updated": 0, "deleted": 0, "errors": 0,
+            "note": "empty_feed",
+        }
+
+    posts_by_cid = load_engagement_comments()
+    if not quiet:
+        print(
+            f"[stats] feed_rows={len(feed)} db_posts_w_commentUrn={len(posts_by_cid)}",
+            flush=True,
+        )
+
+    updates = []
+    unmatched = []
+    errors = 0
+
+    for fr in feed:
+        row = posts_by_cid.get(fr["comment_id"])
+        if row is None:
+            unmatched.append(fr["comment_id"])
+            continue
+        try:
+            updates.append(compute_one(row, fr, dry_run=dry_run, quiet=quiet))
+        except Exception as e:
+            errors += 1
+            if not quiet:
+                print(f"  ERROR id={row['id']} {e}", flush=True)
+            continue
+
+    # Counts come from the locally-computed `changed` flags so the totals
+    # match in both dry-run and live mode. The server applies the writes
+    # (UPDATE posts + scan_no_change_count + post_views_daily snapshot) and
+    # returns its own {updated, unchanged}, which equal these.
+    updated   = sum(1 for u in updates if u["changed"])
+    unchanged = sum(1 for u in updates if not u["changed"])
+
+    if not dry_run and updates:
+        resp = api_post("/api/v1/linkedin-engagement-comments", {"updates": updates})
+        srv = resp.get("data") or {}
+        # Trust the server tallies when present; they should match locals.
+        updated   = int(srv.get("updated", updated))
+        unchanged = int(srv.get("unchanged", unchanged))
+
+    total   = len(feed)
+    checked = updated + unchanged
+    skipped = len(unmatched)
+    deleted = 0
+
+    result = {
+        "ok": True,
+        "total":     total,
+        "skipped":   skipped,
+        "checked":   checked,
+        "updated":   updated,
+        "unchanged": unchanged,
+        "deleted":   deleted,
+        "errors":    errors,
+        "unmatched": unmatched,
+    }
+
+    if summary_path:
+        try:
+            with open(summary_path, "w") as f:
+                json.dump({
+                    "refreshed":   updated,
+                    "removed":     deleted,
+                    "unavailable": 0,
+                    "not_found":   len(unmatched),
+                }, f)
+        except Exception as e:
+            print(
+                f"WARN: failed to write summary {summary_path}: {e}",
+                file=sys.stderr,
+            )
+
+    return result
+
+
+def main() -> None:
+    p = argparse.ArgumentParser(
+        description=(
+            "Apply LinkedIn engagement-comment readings to the posts table."
+        )
+    )
+    p.add_argument("--from-json", required=True,
+                   help="Path to JSON produced by scrape_linkedin_comment_stats.py.")
+    p.add_argument("--summary", default=None,
+                   help="Path to write {refreshed,removed,unavailable,not_found} sidecar.")
+    p.add_argument("--dry-run", action="store_true",
+                   help="Compute updates but do not write to DB.")
+    p.add_argument("--quiet", action="store_true", help="Minimal output.")
+    args = p.parse_args()
+
+    try:
+        result = run(args.from_json, args.summary, args.dry_run, args.quiet)
+    except Exception as e:
+        print(json.dumps({"ok": False, "error": "fatal", "detail": str(e)}),
+              file=sys.stderr)
+        sys.exit(1)
+
+    if not result.get("ok"):
+        print(json.dumps(result, indent=2), file=sys.stderr)
+        sys.exit(1)
+
+    print(
+        f"LinkedInPosts: {result['total']} total, "
+        f"{result['skipped']} skipped, "
+        f"{result['checked']} checked, "
+        f"{result['updated']} updated, "
+        f"{result['deleted']} deleted, "
+        f"{result['errors']} errors"
+    )
+
+
+if __name__ == "__main__":
+    main()

package/scripts/version.py

@@ -0,0 +1,72 @@
+"""Read the social-autoposter package.json version once and cache it.
+
+Every write to posts / replies / dms stamps `autoposter_version` so we can
+attribute engagement back to the release of the autoposter code that
+produced it ("did 1.5.0 outperform 1.4.x on Reddit?").
+
+The value comes from:
+  1. AUTOPOSTER_VERSION env var, if set (lets us pin during testing or
+     override for a one-off backfill).
+  2. package.json `version` field in the repo root.
+
+Returns None when both lookups fail. Callers MUST tolerate None and pass
+it through to the API; the API stores NULL and the column stays empty for
+that row rather than blocking the write.
+
+Why not git SHA: the auto-commit agent at ~/git-dashboard/auto_commit.py
+fires every minute, so the SHA changes constantly without release intent
+and would be noise. The version string is manually bumped per meaningful
+release (bin/cli.js + package.json), which is the right granularity for
+"did this prompt change improve engagement?" analyses.
+"""
+from __future__ import annotations
+
+import json
+import os
+from typing import Optional
+
+_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_PKG_PATH = os.path.join(_REPO_ROOT, "package.json")
+
+_cached: Optional[str] = None
+_cached_loaded = False
+
+
+def read_version() -> Optional[str]:
+    """Return the autoposter version string, or None if unavailable.
+
+    Reads env first, then package.json. Result is cached for the process
+    lifetime since the version never changes mid-run.
+    """
+    global _cached, _cached_loaded
+    if _cached_loaded:
+        return _cached
+
+    env_val = (os.environ.get("AUTOPOSTER_VERSION") or "").strip()
+    if env_val:
+        _cached = env_val
+        _cached_loaded = True
+        return _cached
+
+    try:
+        with open(_PKG_PATH, "r", encoding="utf-8") as f:
+            pkg = json.load(f)
+        v = pkg.get("version")
+        if isinstance(v, str) and v.strip():
+            _cached = v.strip()
+            _cached_loaded = True
+            return _cached
+    except (OSError, json.JSONDecodeError):
+        pass
+
+    _cached_loaded = True
+    _cached = None
+    return None
+
+
+if __name__ == "__main__":
+    # CLI: `python3 scripts/version.py` -> prints the version (or empty
+    # line). Used by shell scripts that want to thread the value into env
+    # before spawning sub-processes.
+    v = read_version()
+    print(v or "")