bloby-bot 0.50.3 → 0.51.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bloby-bot",
-  "version": "0.50.3",
+  "version": "0.51.0",
   "releaseNotes": [
     "1. Something great..",
     "2. ",

package/supervisor/index.ts

@@ -46,6 +46,11 @@ const PLATFORM_ASSETS = new Set([
   '/manifest.json',
 ]);
 
+// Directory-prefix platform assets — anything under these is served from supervisor/public/.
+// Used for the Morphy animation set: drop new {clip}.png + {clip}.json into public/morphy/
+// and they're automatically served without touching the allowlist.
+const PLATFORM_ASSET_DIRS = ['/morphy/'];
+
 // Ensure dist-bloby exists (postinstall may have failed silently)
 if (!fs.existsSync(DIST_BLOBY)) {
   log.info('Building bloby chat UI (first run)...');
@@ -85,7 +90,7 @@ const SW_JS = `// Service worker — app-shell caching + push notifications
 //   JS/CSS modules                      → stale-while-revalidate
 //   API, WebSocket, Vite internals      → network-only (no cache)
 
-var CACHE = 'bloby-v14';
+var CACHE = 'bloby-v15';
 var HASHED_RE = new RegExp('/assets/.+-[a-zA-Z0-9]{6,}[.](js|css)$');
 
 // Precache the HTML shell on install so the cache is never empty.
@@ -1631,7 +1636,8 @@ mint();
 
     // Platform assets — served from supervisor/public/ so they survive workspace swaps
     const cleanUrl = (req.url || '').split('?')[0];
-    if (PLATFORM_ASSETS.has(cleanUrl)) {
+    const inAssetDir = PLATFORM_ASSET_DIRS.some((d) => cleanUrl.startsWith(d) && !cleanUrl.includes('..'));
+    if (PLATFORM_ASSETS.has(cleanUrl) || inAssetDir) {
       const assetPath = path.join(SUPERVISOR_PUBLIC, cleanUrl);
       try {
         const stat = fs.statSync(assetPath);

package/supervisor/public/morphy/teleporting.json

@@ -0,0 +1,38 @@
+{
+  "name": "teleporting",
+  "spritesheet": "teleporting.png",
+  "frame": {
+    "w": 218,
+    "h": 180
+  },
+  "grid": {
+    "cols": 16,
+    "rows": 8
+  },
+  "totalFrames": 121,
+  "fps": 24,
+  "clips": {
+    "idle": {
+      "from": 19,
+      "to": 19,
+      "mode": "hold"
+    },
+    "enter": {
+      "from": 20,
+      "to": 45,
+      "mode": "forward",
+      "next": "active"
+    },
+    "active": {
+      "from": 46,
+      "to": 68,
+      "mode": "pingpong"
+    },
+    "exit": {
+      "from": 69,
+      "to": 121,
+      "mode": "forward",
+      "next": "idle"
+    }
+  }
+}

package/supervisor/widget.js

@@ -33,24 +33,34 @@
 
   // ══════════════════════════════════════════════════════════════════
   // ── Blob Sprite Sheet Config (splash only) ───────────────────────
+  // Loaded from /morphy/teleporting.json. Clips map to animStates:
+  //   idle (hold)      → animState 'idle'
+  //   enter (forward)  → animState 'melting'
+  //   active (pingpong)→ animState 'traveling'
+  //   exit (forward)   → animState 'reforming'
   // ══════════════════════════════════════════════════════════════════
 
+  var BLOB_CONFIG_URL = '/morphy/teleporting.json';
+  var BLOB_ASSETS_DIR = '/morphy/';
+
+  // Defaults mirror the teleporting.json shipped at the URL above so
+  // animation works even if the fetch is cached at the wrong revision.
   var COLS = 16;
-  var FRAME_W = 125;
-  var FRAME_H = 120;
+  var FRAME_W = 218;
+  var FRAME_H = 180;
   var DISPLAY_H = 58;
   var DISPLAY_W = DISPLAY_H * (FRAME_W / FRAME_H);
 
-  var IDLE_START = 0, IDLE_END = 29;
-  var MELT_START = 30, MELT_END = 52;
-  var TRAVEL_START = 52, TRAVEL_END = 84;
-  var REFORM_START = 84, REFORM_END = 191;
+  var IDLE_START = 19, IDLE_END = 19;
+  var MELT_START = 20, MELT_END = 45;
+  var TRAVEL_START = 46, TRAVEL_END = 68;
+  var REFORM_START = 69, REFORM_END = 121;
 
-  var FPS = 29;
+  var FPS = 24;
   var FRAME_MS = 1000 / FPS;
-  var IDLE_FPS = 22;
+  var IDLE_FPS = 24;
   var IDLE_FRAME_MS = 1000 / IDLE_FPS;
-  var REFORM_FPS = 70;
+  var REFORM_FPS = 24;
   var REFORM_FRAME_MS = 1000 / REFORM_FPS;
 
   var TRAVEL_PX_PER_MS = 0.65;
@@ -147,6 +157,7 @@
   var animState = 'loading';
   var currentFrame = 0;
   var idleDirection = 1;
+  var travelFrameDir = 1;
   var lastFrameTime = 0;
   var travelDuration = 0;
   var travelStartTime = 0;
@@ -181,21 +192,41 @@
   var hpSpeechInstance = null;
   var hpSpeechTranscript = '';
 
-  // ── Load blob sprite sheet ──
+  // ── Load blob sprite sheet (config + image) ──
+  function applyBlobConfig(cfg) {
+    COLS = cfg.grid.cols;
+    FRAME_W = cfg.frame.w;
+    FRAME_H = cfg.frame.h;
+    DISPLAY_W = DISPLAY_H * (FRAME_W / FRAME_H);
+    IDLE_START = cfg.clips.idle.from;   IDLE_END = cfg.clips.idle.to;
+    MELT_START = cfg.clips.enter.from;  MELT_END = cfg.clips.enter.to;
+    TRAVEL_START = cfg.clips.active.from; TRAVEL_END = cfg.clips.active.to;
+    REFORM_START = cfg.clips.exit.from; REFORM_END = cfg.clips.exit.to;
+    FPS = cfg.fps; FRAME_MS = 1000 / FPS;
+    IDLE_FPS = cfg.fps; IDLE_FRAME_MS = 1000 / IDLE_FPS;
+    REFORM_FPS = cfg.fps; REFORM_FRAME_MS = 1000 / REFORM_FPS;
+  }
+
   function loadSprite(onDone, onFail) {
-    var img = new Image();
-    img.onload = function () {
-      spriteSheet = img;
-      center.x = Math.round(W / 2);
-      center.y = Math.round(H / 2);
-      currentFrame = IDLE_START;
-      animState = 'idle';
-      var splash = document.getElementById('splash');
-      if (splash) splash.style.display = 'none';
-      onDone();
-    };
-    img.onerror = function () { if (onFail) onFail(); };
-    img.src = '/spritesheet.webp';
+    fetch(BLOB_CONFIG_URL)
+      .then(function (r) { if (!r.ok) throw new Error('blob config ' + r.status); return r.json(); })
+      .then(function (cfg) {
+        applyBlobConfig(cfg);
+        var img = new Image();
+        img.onload = function () {
+          spriteSheet = img;
+          center.x = Math.round(W / 2);
+          center.y = Math.round(H / 2);
+          currentFrame = IDLE_START;
+          animState = 'idle';
+          var splash = document.getElementById('splash');
+          if (splash) splash.style.display = 'none';
+          onDone();
+        };
+        img.onerror = function () { if (onFail) onFail(); };
+        img.src = BLOB_ASSETS_DIR + cfg.spritesheet;
+      })
+      .catch(function () { if (onFail) onFail(); });
   }
 
   // ── Load headphones sprite sheet ──
@@ -300,10 +331,11 @@
       else if (currentFrame <= IDLE_START) { currentFrame = IDLE_START; idleDirection = 1; }
     } else if (animState === 'melting') {
       currentFrame++;
-      if (currentFrame > MELT_END) { animState = 'traveling'; currentFrame = TRAVEL_START; travelStartTime = now; }
+      if (currentFrame > MELT_END) { animState = 'traveling'; currentFrame = TRAVEL_START; travelStartTime = now; travelFrameDir = 1; }
     } else if (animState === 'traveling') {
-      currentFrame++;
-      if (currentFrame > TRAVEL_END) currentFrame = TRAVEL_START;
+      currentFrame += travelFrameDir;
+      if (currentFrame >= TRAVEL_END) { currentFrame = TRAVEL_END; travelFrameDir = -1; }
+      else if (currentFrame <= TRAVEL_START) { currentFrame = TRAVEL_START; travelFrameDir = 1; }
     } else if (animState === 'reforming') {
       currentFrame++;
       if (currentFrame > REFORM_END) {

package/workspace/client/public/morphy/teleporting.json

@@ -0,0 +1,38 @@
+{
+  "name": "teleporting",
+  "spritesheet": "teleporting.png",
+  "frame": {
+    "w": 218,
+    "h": 180
+  },
+  "grid": {
+    "cols": 16,
+    "rows": 8
+  },
+  "totalFrames": 121,
+  "fps": 24,
+  "clips": {
+    "idle": {
+      "from": 19,
+      "to": 19,
+      "mode": "hold"
+    },
+    "enter": {
+      "from": 20,
+      "to": 45,
+      "mode": "forward",
+      "next": "active"
+    },
+    "active": {
+      "from": 46,
+      "to": 68,
+      "mode": "pingpong"
+    },
+    "exit": {
+      "from": 69,
+      "to": 121,
+      "mode": "forward",
+      "next": "idle"
+    }
+  }
+}

package/workspace/client/public/sw.js

@@ -6,7 +6,7 @@
 //   JS/CSS modules                      → stale-while-revalidate
 //   API, WebSocket, Vite internals      → network-only (no cache)
 
-const CACHE = 'bloby-v5';
+const CACHE = 'bloby-v6';
 
 // Precache the HTML shell on install so the cache is never empty.
 // Without this, the first navigation isn't intercepted (SW wasn't

package/workspace/skills/plaud/SKILL.md

@@ -249,13 +249,27 @@ curl -s -o "workspace/files/audio/plaud/<FILE_ID>.mp3" '<TEMP URL>'
 
 ---
 
-## Transcription — pick a path
+## Transcription — pick a path (with pricing)
 
-Once the audio is on disk, you need text. **Ask the human once** which path they want, then save it as `transcriptionMode` in `.plaud.json` so you don't re-ask every sync.
+Once the audio is on disk, you need text. **Talk to the human about the trade-offs once**, pick a path, save it as `transcriptionMode` in `.plaud.json` so you don't re-ask every sync.
 
-### Path A — Bloby Marketplace `audio-to-text` (easiest, pay-per-minute)
+### The trade-off table (lead with this)
 
-If the bloby is registered with the relay (Quick Tunnel mode → there's a token at `~/.bloby/config.json → relay.token`), just POST the file. No API key to manage, no provider account.
+| Path | Cost | Setup | Notes |
+|---|---|---|---|
+| **Bloby Marketplace** | **$0.22/hr** ($0.0037/min) | None — uses your relay token | **Cheapest hosted option.** Pay-as-you-go from owner's credit balance. No API key to manage. |
+| Groq Whisper (BYO key) | ~$0.04/hr | Human creates a Groq account, gives you the key | Often has a generous free tier. Same model the marketplace uses under the hood. |
+| Mistral Voxtral (BYO key) | varies | Human creates a Mistral account, gives you the key | Newer option. Check [docs.mistral.ai](https://docs.mistral.ai) for current pricing. |
+| OpenAI Whisper (wizard key) | **$1.02/hr** | Already configured if human set whisper_key in the Bloby wizard | **~5× more expensive than the marketplace.** Only worth it if the human really wants to use their existing OpenAI key. |
+| Local (whisper.cpp / faster-whisper) | Free (compute) | Human installs the binary themselves | Fully offline, full privacy. Slower than cloud. |
+
+**Important:** The Bloby wizard today only collects an OpenAI Whisper key. **Don't assume that's the only option.** When you talk to your human about transcription for the first time, **proactively mention** that for $0.22/hr they can use the marketplace, that Groq is even cheaper if they BYO key, and that OpenAI (the wizard default) is actually the most expensive of the cloud options.
+
+### Path A — Bloby Marketplace `audio-to-text` (default recommendation)
+
+If the bloby is registered with the relay (Quick Tunnel mode → there's a token at `~/.bloby/config.json → relay.token`), POST the audio file. Returns the transcript directly. No API key to manage.
+
+The marketplace tries **owner credit balance first** on either route — if your human has any credits at `https://bloby.bot`, neither route needs an on-chain dance and a plain `curl` works:
 
 ```bash
 TOKEN=$(jq -r '.relay.token' ~/.bloby/config.json)
@@ -266,32 +280,119 @@ curl -s -X POST 'https://api.bloby.bot/api/services/audio-to-text/use' \
   -F "language=en"     # optional
 ```
 
-Returns JSON:
+If you get **HTTP 200** — done. Parse the JSON, write the `transcript` to `<FILE_ID>.mp3.txt`.
 
-```json
-{
-  "transcript": "...",
-  "language": "en",
-  "estimatedMinutes": 5,
-  "priceUsd": 0.0185,
-  "paidVia": "balance",
-  "groqDurationSec": 275.4,
-  "model": "whisper-large-v3-turbo"
-}
+If you get **HTTP 402** — balance is empty, you need to settle on-chain. Pick the route matching your wallet's funded network (see "Picking the network" below).
+
+#### Picking the network: Tempo `/use` vs Base `/use-base`
+
+The bloby's wallet (`~/.bloby/config.json → wallet`) can be funded on either Tempo USDC or Base USDC. The human picked when they funded it. **Ask once at first sync** and save it in `.plaud.json`:
+
+```
+Bloby: Your account has no marketplace credits, so I'll need to settle on-chain.
+       Is your wallet funded on Tempo USDC or Base USDC?
+       (If you don't know — open https://bloby.bot, sign in, check your wallet balance.)
 ```
 
-- **Pricing:** $0.0037 per estimated minute, rounded up (~$0.22/hr).
-- **How duration is estimated:** file size ÷ assumed 32kbps bitrate. Plaud-sourced mp3 matches this assumption well. High-bitrate files from other sources would be over-charged proportionally — for those, switch to Path B.
-- **Paid from:** account balance first; falls back to MPP (Tempo USDC) or Base (use `/use-base` instead). Make sure the bloby's account has funds OR its wallet is funded on the matching network.
-- **Limits:** 25MB per file. Mp3 from Plaud comfortably fits — observed 1MB ≈ 4½min.
+Save as `marketplaceNetwork: "tempo" | "base"`. Re-ask only if both routes start failing.
 
-Write the response's `transcript` to `workspace/files/audio/plaud/<FILE_ID>.mp3.txt`.
+#### Tempo path (`/use`) — needs `mppx/client`, NOT curl
 
-### Path B — Bring your own API key (DIY)
+The `mppx` CLI does not support multipart uploads (`-F`). For file-upload services, use the `mppx/client` Node library instead. Write a small helper:
 
-Pick a provider, ask the human for their key, store it as a workspace secret (`workspace/.env` is fine — the backend reloads on .env change). Then call directly from Bash.
+```bash
+# One-time install (in workspace root or skill dir):
+npm install --prefix workspace mppx viem
+```
 
-**Groq Whisper** — cheapest, fastest. Same model the marketplace uses under the hood. Free tier exists.
+```js
+// workspace/skills/plaud/marketplace-tempo.mjs
+import { Mppx, tempo } from 'mppx/client';
+import { privateKeyToAccount } from 'viem/accounts';
+import { readFileSync, writeFileSync } from 'node:fs';
+
+const [, , filePath, language] = process.argv;
+const cfg = JSON.parse(readFileSync(`${process.env.HOME}/.bloby/config.json`, 'utf8'));
+const account = privateKeyToAccount(cfg.wallet.privateKey);
+const mppx = Mppx.create({ methods: [tempo({ account })] });
+
+const form = new FormData();
+form.append('file', new Blob([readFileSync(filePath)]), filePath.split('/').pop());
+if (language) form.append('language', language);
+
+const res = await mppx.fetch('https://api.bloby.bot/api/services/audio-to-text/use', {
+  method: 'POST',
+  headers: { 'X-Bloby-Token': cfg.relay.token },
+  body: form,
+});
+if (!res.ok) { console.error(await res.text()); process.exit(1); }
+const data = await res.json();
+writeFileSync(`${filePath}.txt`, data.transcript);
+console.log(JSON.stringify({ priceUsd: data.priceUsd, paidVia: data.paidVia, transcriptPath: `${filePath}.txt` }, null, 2));
+```
+
+Invoke from Bash:
+```bash
+node workspace/skills/plaud/marketplace-tempo.mjs workspace/files/audio/plaud/<FILE_ID>.mp3 en
+```
+
+#### Base path (`/use-base`) — `x402-fetch` works
+
+Base is easier because `x402-fetch` is a plain `fetch` wrapper that handles FormData natively:
+
+```bash
+npm install --prefix workspace x402-fetch viem
+```
+
+```js
+// workspace/skills/plaud/marketplace-base.mjs
+import { wrapFetchWithPayment } from 'x402-fetch';
+import { privateKeyToAccount } from 'viem/accounts';
+import { readFileSync, writeFileSync } from 'node:fs';
+
+const [, , filePath, language] = process.argv;
+const cfg = JSON.parse(readFileSync(`${process.env.HOME}/.bloby/config.json`, 'utf8'));
+const account = privateKeyToAccount(cfg.wallet.privateKey);
+const fetchWithPay = wrapFetchWithPayment(fetch, account);
+
+const form = new FormData();
+form.append('file', new Blob([readFileSync(filePath)]), filePath.split('/').pop());
+if (language) form.append('language', language);
+
+const res = await fetchWithPay('https://api.bloby.bot/api/services/audio-to-text/use-base', {
+  method: 'POST',
+  headers: { 'X-Bloby-Token': cfg.relay.token },
+  body: form,
+});
+if (!res.ok) { console.error(await res.text()); process.exit(1); }
+const data = await res.json();
+writeFileSync(`${filePath}.txt`, data.transcript);
+console.log(JSON.stringify({ priceUsd: data.priceUsd, paidVia: data.paidVia, transcriptPath: `${filePath}.txt` }, null, 2));
+```
+
+Invoke:
+```bash
+node workspace/skills/plaud/marketplace-base.mjs workspace/files/audio/plaud/<FILE_ID>.mp3 en
+```
+
+#### Suggested flow
+
+1. Try the plain `curl` first — covers the case where the human has credit balance.
+2. If `curl` returns 402, fall through to the helper for `marketplaceNetwork` from `.plaud.json`.
+3. If you don't have `marketplaceNetwork` set yet, ask the human (script above).
+
+Both routes return the same JSON. Pricing:
+- **$0.0037 per estimated minute, rounded up ($0.22/hr).**
+- Duration is estimated from file size ÷ 32 kbps. Plaud mp3 matches; high-bitrate non-Plaud files get over-charged proportionally — use Path B for those.
+- 25MB cap per file (Plaud comfortably fits — observed 1MB ≈ 4½ min).
+
+Set `transcriptionMode: "marketplace"` in `.plaud.json` once it works.
+
+### Path B — Bring your own API key (BYO)
+
+Pick a provider, ask the human for their key, store it (workspace `.env` works — backend auto-reloads on .env change). Then call from Bash.
+
+**Groq Whisper** — currently the cheapest cloud option (~$0.04/hr at our list rate, often free under their free tier). Same model as the marketplace. Recommend this when the human wants to BYO.
 ```bash
 curl -s -X POST 'https://api.groq.com/openai/v1/audio/transcriptions' \
   -H "Authorization: Bearer $GROQ_API_KEY" \
@@ -299,8 +400,9 @@ curl -s -X POST 'https://api.groq.com/openai/v1/audio/transcriptions' \
   -F "model=whisper-large-v3-turbo" \
   -F "response_format=json"
 ```
+Set `transcriptionMode: "groq"`.
 
-**OpenAI Whisper** — the human may already have an OpenAI key from the Bloby wizard. Read it from the settings table directly:
+**OpenAI Whisper** — only do this if the human explicitly prefers it. **$1.02/hr — ~5× more expensive than the marketplace.** The key is the one collected by the Bloby wizard, readable directly from the settings DB:
 ```bash
 WHISPER_KEY=$(sqlite3 ~/.bloby/memory.db "SELECT value FROM settings WHERE key='whisper_key';")
 curl -s -X POST 'https://api.openai.com/v1/audio/transcriptions' \
@@ -308,6 +410,8 @@ curl -s -X POST 'https://api.openai.com/v1/audio/transcriptions' \
   -F "file=@workspace/files/audio/plaud/<FILE_ID>.mp3" \
   -F "model=whisper-1"
 ```
+Before using this path, **say something like**: *"I see you set an OpenAI Whisper key in the wizard. I can use it, but it's about 5× more expensive than the marketplace ($1.02/hr vs $0.22/hr). Want me to use the marketplace instead, or stick with OpenAI?"*
+Set `transcriptionMode: "openai"` if they confirm.
 
 **Mistral Voxtral**:
 ```bash
@@ -316,26 +420,24 @@ curl -s -X POST 'https://api.mistral.ai/v1/audio/transcriptions' \
   -F "file=@workspace/files/audio/plaud/<FILE_ID>.mp3" \
   -F "model=voxtral-mini-latest"
 ```
+Set `transcriptionMode: "mistral"`.
 
-**Local — no API, no cost, fully private:**
-- [whisper.cpp](https://github.com/ggerganov/whisper.cpp) — C++ binary, CPU or Metal/CUDA. Install once, transcribe forever.
+**Local — no API, no cost, fully offline:**
+- [whisper.cpp](https://github.com/ggerganov/whisper.cpp) — C++ binary, CPU or Metal/CUDA.
 - [faster-whisper](https://github.com/SYSTRAN/faster-whisper) — Python, ~4× faster than reference whisper.
 - The human installs one of these themselves. The bloby invokes the CLI from Bash.
 
-After whichever path, extract the `text` field and write it to `workspace/files/audio/plaud/<FILE_ID>.mp3.txt`.
+Set `transcriptionMode: "local"` and add a `localCommand` field to `.plaud.json` with the exact invocation pattern.
+
+After whichever path, extract the `text` field from the response (or stdout for local) and write it to `workspace/files/audio/plaud/<FILE_ID>.mp3.txt`.
+
+### How to talk to the human about this
 
-### Choosing for the human
+First-time setup, before transcribing anything:
 
-If they don't have a preference, recommend **Path A (Marketplace)**:
-- No key setup.
-- Already integrated with the bloby's payment.
-- Pay-as-you-go — no monthly minimum.
-- If their account has any balance from other marketplace use, it just works.
+> *"For transcription I have a few options. Cheapest is the Bloby marketplace at $0.22/hour — no setup, paid from your account credits. If you have a Groq API key, BYO is even cheaper. I see you set an OpenAI Whisper key in the wizard — I can use that too, but at $1.02/hour it's about 5× more expensive than the marketplace, so I'd recommend not using it unless you specifically want to. There's also local transcription if you'd rather install whisper.cpp. What's your preference?"*
 
-Recommend **Path B** if:
-- They're transcribing a lot and want to use a free tier or flat-rate plan.
-- They want 100% local for privacy reasons.
-- They already have a preferred provider.
+After they pick, save it as `transcriptionMode` and don't re-ask.
 
 ---