@sdelsad/commodity-desk-daily 1.0.17 → 1.13.2

package/publish_episode.py

@@ -0,0 +1,367 @@
+#!/usr/bin/env python3
+"""
+Soft Commodity Trading - publisher.
+
+Publishes one episode to npm (which jsDelivr then serves as a public CDN),
+carrying the podcast cover and RSS feed forward and prepending the new episode
+to the feed. Handles version bumping off whatever is currently live, so it is
+safe to run every morning without tracking state anywhere.
+
+Usage:
+    python3 publish_episode.py \
+        --mp3 ep02.mp3 --notes ep02.md \
+        --number 2 --title "Flat Price vs Basis" \
+        --description "Why physical traders don't bet on price." \
+        --duration 612 --pubdate "Tue, 11 Aug 2026 05:00:00 GMT"
+
+Prints PUBLISHED_URL=... and PUBLISHED_VERSION=... on success.
+Exit codes: 0 ok, 1 error.
+"""
+import argparse
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import urllib.request
+
+PKG = "@sdelsad/commodity-desk-daily"
+SITE_URL = "https://storage.googleapis.com/podcast-audio-2647223968/index.html"
+REGISTRY = "https://registry.npmjs.org"
+CDN = "https://cdn.jsdelivr.net/npm"
+
+FEED_SKELETON = """<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
+  <channel>
+    <title>Soft Commodity Trading</title>
+    <link>https://www.npmjs.com/package/{pkg}</link>
+    <description>A daily 10-minute briefing on physical commodity trading.</description>
+    <language>en-us</language>
+    <itunes:author>Sébastien Delsad</itunes:author>
+    <itunes:explicit>false</itunes:explicit>
+    <itunes:category text="Business"/>
+  </channel>
+</rss>
+"""
+
+
+def fetch_json(url):
+    with urllib.request.urlopen(url, timeout=30) as resp:
+        return json.load(resp)
+
+
+def latest_version():
+    try:
+        meta = fetch_json(f"{REGISTRY}/{PKG.replace('/', '%2f')}")
+        return meta["dist-tags"]["latest"]
+    except Exception as exc:
+        print(f"  ! could not read registry ({exc}); starting at 1.0.0")
+        return None
+
+
+def bump(version):
+    if not version:
+        return "1.0.0"
+    major, minor, patch = (version.split(".") + ["0", "0"])[:3]
+    return f"{major}.{minor}.{int(patch) + 1}"
+
+
+def carry_forward(version, workdir):
+    """Extract cover.jpg / feed.xml from the currently published tarball."""
+    if not version:
+        return None
+    name = PKG.split("/")[-1]
+    url = f"{REGISTRY}/{PKG}/-/{name}-{version}.tgz"
+    try:
+        tgz = os.path.join(workdir, "prev.tgz")
+        urllib.request.urlretrieve(url, tgz)
+        with tarfile.open(tgz) as tar:
+            for member in tar.getmembers():
+                base = os.path.basename(member.name)
+                if base in ("cover.jpg", "feed.xml", "covered.md", "glossary.md"):
+                    member.name = base
+                    tar.extract(member, workdir)
+        os.unlink(tgz)
+        return True
+    except Exception as exc:
+        print(f"  ! could not carry forward assets ({exc})")
+        return None
+
+
+def build_item(number, title, description, url, size, duration, pubdate, page_url=""):
+    """One <item>. With page_url, the episode gets a clickable website link.
+
+    Apple Podcasts renders an item's <link> as the episode's website button, and
+    renders anchors inside a CDATA <description>. Both are set, because some
+    apps honour only one of the two.
+    """
+    def esc(text):
+        return (text.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;"))
+
+    link = f"      <link>{esc(page_url)}</link>\n" if page_url else ""
+    if page_url:
+        body = (f"<![CDATA[<p>{esc(description)}</p>"
+                f'<p><a href="{esc(page_url)}">Read this episode, with the charts, '
+                f"the glossary and the quiz &rarr;</a></p>]]>")
+        summary = (f"{description}\n\nRead this episode: {page_url}")
+    else:
+        body = esc(description)
+        summary = description
+
+    return f"""    <item>
+      <title>Ep {number} — {esc(title)}</title>
+{link}      <description>{body}</description>
+      <itunes:summary>{esc(summary)}</itunes:summary>
+      <enclosure url="{url}" length="{size}" type="audio/mpeg"/>
+      <guid isPermaLink="false">{url}</guid>
+      <pubDate>{pubdate}</pubDate>
+      <itunes:duration>{duration}</itunes:duration>
+    </item>
+"""
+
+
+def update_feed(feed_path, item_xml, number):
+    """Prepend the new item; drop any existing item for the same episode."""
+    if os.path.exists(feed_path):
+        feed = open(feed_path, encoding="utf-8").read()
+    else:
+        feed = FEED_SKELETON.format(pkg=PKG)
+    # remove a previous build of this same episode, if any
+    feed = re.sub(
+        rf"    <item>\s*<title>Ep {number} —.*?</item>\n",
+        "",
+        feed,
+        flags=re.DOTALL,
+    )
+    idx = feed.find("<item>")
+    if idx != -1:
+        insert_at = feed.rfind("\n", 0, idx) + 1
+    else:
+        insert_at = feed.rfind("</channel>")
+        if insert_at != -1:
+            insert_at = feed.rfind("\n", 0, insert_at) + 1
+    if insert_at == -1:  # unrecognisable feed: rebuild from skeleton
+        feed = FEED_SKELETON.format(pkg=PKG)
+        insert_at = feed.rfind("\n", 0, feed.rfind("</channel>")) + 1
+    return feed[:insert_at] + item_xml + feed[insert_at:]
+
+
+def set_author(feed: str, author: str) -> str:
+    """Keep the podcast author consistent in the channel metadata."""
+    esc = author.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+    feed = re.sub(r"<itunes:author>.*?</itunes:author>",
+                  f"<itunes:author>{esc}</itunes:author>", feed, count=1, flags=re.DOTALL)
+    feed = re.sub(r"(<itunes:owner>\s*<itunes:name>).*?(</itunes:name>)",
+                  rf"\1{esc}\2", feed, count=1, flags=re.DOTALL)
+    return feed
+
+
+def set_site_link(feed: str, site_url: str) -> str:
+    """Point the channel (and its <image>) at the show's website.
+
+    Apple Podcasts turns the channel <link> into the show's website button, so
+    it must be a real page, not the package registry.
+    """
+    esc = site_url.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
+    # the channel link is the first <link> in the document, before any <item>
+    head, sep, tail = feed.partition("<item>")
+    head = re.sub(r"<link>.*?</link>", f"<link>{esc}</link>", head, flags=re.DOTALL)
+    return head + sep + tail
+
+
+def update_covered(path, number, title, summary, pubdate):
+    """Append this episode to the running log, replacing any earlier entry."""
+    header = "# Soft Commodity Trading — episodes aired\n\nRunning log. Read before writing a new episode: avoid repeating material, and only make callbacks to episodes listed here.\n"
+    lines = []
+    if os.path.exists(path):
+        existing = open(path, encoding="utf-8").read()
+        for line in existing.splitlines():
+            if line.startswith(f"- **Ep {number}**") or line.startswith("#") or not line.strip():
+                continue
+            if line.startswith("Running log."):
+                continue
+            lines.append(line)
+    entry = f"- **Ep {number}** ({pubdate.split(',')[0] if ',' in pubdate else pubdate}) — *{title}*: {summary}"
+    lines.append(entry)
+
+    def sort_key(line):
+        try:
+            return int(line.split("**Ep ")[1].split("**")[0])
+        except (IndexError, ValueError):
+            return 9999
+
+    lines.sort(key=sort_key)
+    return header + "\n" + "\n".join(lines) + "\n"
+
+
+def update_glossary(path, entries, number):
+    """Merge 'term = definition' entries into the cumulative glossary."""
+    header = ("# Soft Commodity Trading — glossary\n\nUnits, conventions and desk expressions, "
+              "accumulated as the show introduces them.\n")
+    known = {}
+    if os.path.exists(path):
+        for line in open(path, encoding="utf-8").read().splitlines():
+            m = re.match(r"- \*\*(.+?)\*\* — (.+?)(?:  _\(ep \d+\)_)?$", line.strip())
+            if m:
+                known[m.group(1).strip().lower()] = (m.group(1).strip(), m.group(2).strip(),
+                                                     line.strip())
+    for raw in entries.split(";"):
+        if "=" not in raw:
+            continue
+        term, _, definition = raw.partition("=")
+        term, definition = term.strip(), definition.strip()
+        if not term or not definition:
+            continue
+        key = term.lower()
+        if key not in known:  # first definition wins; never rewrite history
+            known[key] = (term, definition, f"- **{term}** — {definition}  _(ep {number})_")
+    lines = [v[2] for _, v in sorted(known.items())]
+    return header + "\n" + "\n".join(lines) + "\n"
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--mp3", help="local mp3 file to embed in the npm package (Kokoro path)")
+    ap.add_argument("--mp3-url", help="public URL of an externally hosted mp3 (Gemini/GCS path)")
+    ap.add_argument("--size-bytes", type=int, help="mp3 size in bytes; required with --mp3-url")
+    ap.add_argument("--notes", help="episode .md (transcript + quiz)")
+    ap.add_argument("--number", type=int, required=True)
+    ap.add_argument("--title", required=True)
+    ap.add_argument("--description", default="")
+    ap.add_argument("--page-url", default="",
+                    help="episode page URL; becomes the episode's website link "
+                         "in Apple Podcasts and a clickable link in the notes")
+    ap.add_argument("--site-url", default=SITE_URL,
+                    help="the show's website, used as the feed's channel link")
+    ap.add_argument("--duration", type=int, required=True, help="seconds")
+    ap.add_argument("--pubdate", required=True, help="RFC-822, e.g. 'Tue, 11 Aug 2026 05:00:00 GMT'")
+    ap.add_argument("--covered", default="",
+                    help="one line for covered.md: concepts, vocabulary and the worked example used")
+    ap.add_argument("--feed-out", default="",
+                    help="also write the final feed.xml to this local path (to re-host it elsewhere)")
+    ap.add_argument("--glossary", default="",
+                    help="new terms introduced today: 'term = definition; term2 = definition2'")
+    ap.add_argument("--glossary-out", default="",
+                    help="write the cumulative glossary to this local path (for the email)")
+    ap.add_argument("--script", default="",
+                    help="the segmented spoken script (script.txt) — archived so the audio "
+                         "can be regenerated verbatim later with a different voice")
+    ap.add_argument("--author", default="Sébastien Delsad",
+                    help="podcast author shown in podcast apps")
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    if args.mp3_url:
+        if not args.size_bytes:
+            print("--size-bytes is required with --mp3-url", file=sys.stderr)
+            return 1
+    elif not args.mp3 or not os.path.exists(args.mp3):
+        print("Provide --mp3 <existing file> or --mp3-url + --size-bytes", file=sys.stderr)
+        return 1
+
+    current = latest_version()
+    version = bump(current)
+    nn = f"{args.number:02d}"
+    print(f"[publish] {current or 'none'} -> {version} (episode {nn})")
+
+    workdir = tempfile.mkdtemp(prefix="cdd-")
+    try:
+        carry_forward(current, workdir)
+        if args.notes and os.path.exists(args.notes):
+            shutil.copy(args.notes, os.path.join(workdir, f"ep{nn}.md"))
+        # Archive the exact spoken script: the source of truth for re-generating
+        # this episode's audio verbatim with another voice or engine.
+        if args.script and os.path.exists(args.script):
+            shutil.copy(args.script, os.path.join(workdir, f"ep{nn}.script.txt"))
+
+        if args.mp3_url:
+            url, size = args.mp3_url, args.size_bytes
+        else:
+            shutil.copy(args.mp3, os.path.join(workdir, f"ep{nn}.mp3"))
+            url = f"{CDN}/{PKG}@{version}/ep{nn}.mp3"
+            size = os.path.getsize(args.mp3)
+        feed_path = os.path.join(workdir, "feed.xml")
+        item = build_item(args.number, args.title, args.description, url, size,
+                          args.duration, args.pubdate, args.page_url)
+        # Build the new feed BEFORE opening for write: opening truncates the file.
+        new_feed = set_site_link(
+            set_author(update_feed(feed_path, item, args.number), args.author),
+            args.site_url)
+        with open(feed_path, "w", encoding="utf-8") as fh:
+            fh.write(new_feed)
+        if args.feed_out:
+            with open(args.feed_out, "w", encoding="utf-8") as fh:
+                fh.write(new_feed)
+
+        # Cumulative glossary, carried forward and extended.
+        glossary_path = os.path.join(workdir, "glossary.md")
+        if args.glossary:
+            new_glossary = update_glossary(glossary_path, args.glossary, args.number)
+            with open(glossary_path, "w", encoding="utf-8") as fh:
+                fh.write(new_glossary)
+        if args.glossary_out and os.path.exists(glossary_path):
+            shutil.copy(glossary_path, args.glossary_out)
+
+        # Running log of what has aired, carried forward and appended.
+        covered_path = os.path.join(workdir, "covered.md")
+        summary = args.covered or args.description or args.title
+        new_covered = update_covered(covered_path, args.number, args.title, summary, args.pubdate)
+        with open(covered_path, "w", encoding="utf-8") as fh:
+            fh.write(new_covered)
+
+        json.dump(
+            {
+                "name": PKG,
+                "version": version,
+                "description": f"Soft Commodity Trading - Ep {args.number}: {args.title}",
+                "license": "CC-BY-4.0",
+                "keywords": ["podcast", "commodities", "trading", "education"],
+            },
+            open(os.path.join(workdir, "package.json"), "w"),
+            indent=2,
+        )
+        open(os.path.join(workdir, "README.md"), "w").write(
+            "# Soft Commodity Trading\n\nDaily 10-minute podcast on physical commodity trading.\n\n"
+            f"RSS feed: `{CDN}/{PKG}@latest/feed.xml`\n"
+        )
+
+        # Project-level auth, written next to the package so npm picks it up
+        # from cwd. The token itself never appears here: npm expands NPM_TOKEN
+        # from the environment at publish time. (A user-level ~/.npmrc is not
+        # reliably interpolated when npm runs from a temp directory.)
+        with open(os.path.join(workdir, ".npmrc"), "w") as fh:
+            fh.write("//registry.npmjs.org/:_authToken=${NPM_TOKEN}\n")
+
+        if args.dry_run:
+            print("[publish] dry run; files staged in", workdir)
+            print("\n".join(sorted(os.listdir(workdir))))
+            return 0
+
+        if not os.environ.get("NPM_TOKEN"):
+            print("[publish] NPM_TOKEN is not set in the environment", file=sys.stderr)
+            return 1
+
+        result = subprocess.run(
+            ["npm", "publish", "--access", "public"],
+            cwd=workdir, capture_output=True, text=True,
+        )
+        if result.returncode != 0:
+            print(result.stdout[-1500:] + result.stderr[-1500:], file=sys.stderr)
+            print("[publish] FAILED", file=sys.stderr)
+            return 1
+
+        print(f"[publish] ok")
+        print(f"PUBLISHED_VERSION={version}")
+        print(f"PUBLISHED_URL={url}")
+        print(f"FEED_URL={CDN}/{PKG}@latest/feed.xml")
+        return 0
+    finally:
+        if not args.dry_run:
+            shutil.rmtree(workdir, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/setup.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Soft Commodity Trading - one-shot environment setup.
+# Idempotent: safe to re-run; skips work already done.
+set -euo pipefail
+KDIR="${KOKORO_DIR:-$HOME/kokoro}"
+BASE="https://github.com/thewh1teagle/kokoro-onnx/releases/download/model-files-v1.0"
+echo "[setup] installing python deps..."
+pip install kokoro-onnx soundfile pillow --break-system-packages -q
+mkdir -p "$KDIR"
+fetch() {  # fetch <filename> <min-bytes>
+  local f="$KDIR/$1" min="$2"
+  if [ -f "$f" ] && [ "$(stat -c%s "$f")" -ge "$min" ]; then
+    echo "[setup] $1 already present, skipping"
+  else
+    echo "[setup] downloading $1 ..."
+    curl -sL -o "$f" "$BASE/$1"
+    [ "$(stat -c%s "$f")" -ge "$min" ] || { echo "[setup] FAILED: $1 too small"; exit 1; }
+  fi
+}
+fetch kokoro-v1.0.onnx 300000000
+fetch voices-v1.0.bin  20000000
+command -v ffmpeg >/dev/null || { echo "[setup] WARNING: ffmpeg not found"; }
+echo "[setup] OK -> $KDIR"