@caoscompanybr/merlin 3.5.0

package/.merlin-core/skills/general/youtube-transcript/scripts/fetch_transcript.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+"""
+fetch_transcript.py — transcrição timestamped de um vídeo do YouTube (Merlin skill).
+
+Cascata: youtube-transcript-api (Nível 0) → yt-dlp auto-sub (Nível 1). Se NENHUM
+funcionar (vídeo sem legenda), use o gladia_transcribe.mjs (Nível 2).
+
+Metadata via yt-dlp com player_client=android (contorna o anti-bot "This video is
+not available"), com fallbacks (thumbnail via videoId, duração via transcrição).
+
+Uso:
+  python fetch_transcript.py <url|videoId> [--lang pt] [--out arquivo.json] [--proxy URL]
+
+Saída (contrato): { video, sourceUrl, lang, transcript[], transcriptSource }
+Sem --out → imprime o JSON no stdout.
+
+Requisitos: pip install yt-dlp youtube-transcript-api
+Proxy: --proxy http://user:pass@host:port  (ou envs IPROYAL_USER/PASS/HOST/PORT).
+       Só necessário a partir de IPs de datacenter.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+ANDROID = ["--extractor-args", "youtube:player_client=android"]
+
+
+def log(m: str) -> None:
+    print(m, file=sys.stderr, flush=True)
+
+
+def extract_video_id(url: str) -> Optional[str]:
+    for p in (r"(?:v=|/v/|youtu\.be/|embed/|shorts/)([a-zA-Z0-9_-]{11})",):
+        m = re.search(p, url)
+        if m:
+            return m.group(1)
+    return url if re.fullmatch(r"[a-zA-Z0-9_-]{11}", url) else None
+
+
+def build_proxy(cli: Optional[str]) -> Optional[str]:
+    if cli:
+        return cli
+    u, p = os.environ.get("IPROYAL_USER", ""), os.environ.get("IPROYAL_PASS", "")
+    if u and p:
+        h = os.environ.get("IPROYAL_HOST", "geo.iproyal.com")
+        port = os.environ.get("IPROYAL_PORT", "12321")
+        return f"http://{u}:{p}@{h}:{port}"
+    return None
+
+
+def get_metadata(url: str, proxy: Optional[str]) -> Dict[str, Any]:
+    cmd = ["yt-dlp", "--dump-json", "--no-download", "--no-warnings", *ANDROID]
+    if proxy:
+        cmd += ["--proxy", proxy]
+    cmd += [url]
+    r = subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace")
+    if r.returncode != 0 or not r.stdout.strip():
+        log(f"  ⚠ metadata falhou (segue com fallbacks): {r.stderr.strip()[:160]}")
+        return {}
+    try:
+        d = json.loads(r.stdout.strip().splitlines()[0])
+    except Exception:
+        return {}
+    return {
+        "id": d.get("id", ""),
+        "title": d.get("title", ""),
+        "duration": int(d.get("duration") or 0),
+        "channel": d.get("channel") or d.get("uploader") or "",
+        "upload_date": d.get("upload_date", ""),
+        "thumbnail": d.get("thumbnail", ""),
+    }
+
+
+def _norm(s: Any) -> Dict[str, Any]:
+    if isinstance(s, dict):
+        return {"text": s.get("text", ""), "start": float(s.get("start", 0) or 0), "duration": float(s.get("duration", 0) or 0)}
+    return {"text": getattr(s, "text", ""), "start": float(getattr(s, "start", 0) or 0), "duration": float(getattr(s, "duration", 0) or 0)}
+
+
+def _find(tl: Any, lang: str) -> Any:
+    try:
+        return tl.find_transcript([lang])
+    except Exception:
+        return tl.find_transcript(["pt", "en", "es", "pt-BR"])
+
+
+def transcript_via_api(vid: str, lang: str, proxy: Optional[str]) -> List[Dict[str, Any]]:
+    from youtube_transcript_api import YouTubeTranscriptApi as Y  # type: ignore
+
+    if hasattr(Y, "list_transcripts"):  # v0.6.x
+        kw = {"proxies": {"http": proxy, "https": proxy}} if proxy else {}
+        return [_norm(s) for s in _find(Y.list_transcripts(vid, **kw), lang).fetch()]
+    cfg = None  # v1.x
+    if proxy:
+        try:
+            from youtube_transcript_api.proxies import GenericProxyConfig  # type: ignore
+
+            cfg = GenericProxyConfig(http_url=proxy, https_url=proxy)
+        except Exception:
+            cfg = None
+    api = Y(proxy_config=cfg) if cfg else Y()
+    return [_norm(s) for s in _find(api.list(vid), lang).fetch()]
+
+
+def transcript_via_ytdlp(url: str, lang: str, proxy: Optional[str]) -> List[Dict[str, Any]]:
+    with tempfile.TemporaryDirectory() as tmp:
+        cmd = ["yt-dlp", *ANDROID, "--write-auto-sub", "--write-sub", "--sub-lang",
+               f"{lang},en,pt", "--sub-format", "json3", "--skip-download", "--no-warnings",
+               "-o", os.path.join(tmp, "%(id)s")]
+        if proxy:
+            cmd += ["--proxy", proxy]
+        cmd += [url]
+        subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace")
+        subs = list(Path(tmp).glob("*.json3"))
+        if not subs:
+            return []
+        data = json.loads(subs[0].read_text(encoding="utf-8"))
+        out = []
+        for e in data.get("events", []):
+            if not e.get("segs"):
+                continue
+            txt = "".join(g.get("utf8", "") for g in e["segs"])
+            if txt.strip():
+                out.append({"text": txt, "start": (e.get("tStartMs", 0) or 0) / 1000, "duration": (e.get("dDurationMs", 0) or 0) / 1000})
+        return out
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="YouTube → transcrição timestamped (JSON).")
+    ap.add_argument("url")
+    ap.add_argument("--lang", default="pt")
+    ap.add_argument("--out")
+    ap.add_argument("--proxy")
+    a = ap.parse_args()
+
+    vid = extract_video_id(a.url)
+    if not vid:
+        log(f"✖ videoId inválido: {a.url}")
+        return 1
+    url = a.url if a.url.startswith("http") else f"https://youtu.be/{vid}"
+    proxy = build_proxy(a.proxy)
+
+    log(f"═══ {vid} ═══")
+    meta = get_metadata(url, proxy)
+    meta.setdefault("id", vid)
+    if not meta.get("thumbnail"):
+        meta["thumbnail"] = f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg"
+
+    source = "youtube-transcript-api"
+    segs: List[Dict[str, Any]] = []
+    try:
+        segs = transcript_via_api(vid, a.lang, proxy)
+    except Exception as e:
+        log(f"  ⚠ API falhou ({type(e).__name__}: {e}) — tentando yt-dlp auto-sub…")
+    if not segs:
+        segs = transcript_via_ytdlp(url, a.lang, proxy)
+        source = "yt-dlp-autosub"
+    if not segs:
+        log("✖ Sem legenda no YouTube. Use o Nível 2: node gladia_transcribe.mjs " + vid)
+        source = "none"
+
+    if not meta.get("title"):
+        meta["title"] = vid
+    if not meta.get("duration") and segs:
+        last = segs[-1]
+        meta["duration"] = int(last["start"] + last["duration"])
+
+    artifact = {"video": meta, "sourceUrl": url, "lang": a.lang, "transcript": segs, "transcriptSource": source}
+    out_json = json.dumps(artifact, ensure_ascii=False, indent=2)
+    if a.out:
+        Path(a.out).write_text(out_json, encoding="utf-8")
+        log(f"✓ {len(segs)} segmentos · fonte={source} · gravado em {a.out}")
+    else:
+        print(out_json)
+        log(f"✓ {len(segs)} segmentos · fonte={source}")
+    return 0 if segs else 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/.merlin-core/skills/general/youtube-transcript/scripts/gladia_transcribe.mjs

@@ -0,0 +1,230 @@
+/**
+ * gladia_transcribe.mjs — Nível 2 da cascata de transcrição (Merlin skill).
+ *
+ * Para vídeos do YouTube SEM legenda nenhuma: baixa o áudio via yt-dlp
+ * (player_client=android, formato com áudio) e transcreve via Gladia (pt-BR),
+ * convertendo as `utterances` em segmentos {text, start, duration}.
+ *
+ * Uso:
+ *   node gladia_transcribe.mjs <url|videoId> [--out arquivo.json] [--keep-audio]
+ *
+ * Saída (contrato): { video, sourceUrl, lang, transcript[], transcriptSource:"gladia" }
+ * Sem --out → grava ./transcript-<id>.json
+ *
+ * Chave: GLADIA_API_KEY no env, ou em qualquer .secrets-vault/(STAR)/transcription.env.
+ * Requer: yt-dlp acessível via `python -m yt_dlp` (ou ajuste YTDLP abaixo).
+ */
+import { readFile, writeFile, mkdir, unlink, stat } from "node:fs/promises";
+import { existsSync, readdirSync, readFileSync } from "node:fs";
+import { spawn } from "node:child_process";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+
+// ── chave Gladia: env → varredura de .secrets-vault ──
+function loadGladiaKey() {
+  if (process.env.GLADIA_API_KEY) return process.env.GLADIA_API_KEY;
+  // sobe alguns níveis procurando .secrets-vault/<dir>/transcription.env
+  let dir = process.cwd();
+  for (let i = 0; i < 6; i++) {
+    const vault = join(dir, ".secrets-vault");
+    if (existsSync(vault)) {
+      try {
+        for (const d of readdirSync(vault)) {
+          const env = join(vault, d, "transcription.env");
+          if (existsSync(env)) {
+            const m = readFileSync(env, "utf8").match(
+              /^GLADIA_API_KEY\s*=\s*(.+)$/m,
+            );
+            if (m) return m[1].trim().replace(/^["']|["']$/g, "");
+          }
+        }
+      } catch {
+        /* noop */
+      }
+    }
+    dir = resolve(dir, "..");
+  }
+  return null;
+}
+
+const API_KEY = loadGladiaKey();
+if (!API_KEY) {
+  console.error(
+    "✖ GLADIA_API_KEY não encontrado (env nem .secrets-vault/(STAR)/transcription.env)",
+  );
+  process.exit(1);
+}
+
+const args = process.argv.slice(2);
+const positional = args.find((a) => !a.startsWith("--"));
+const KEEP_AUDIO = args.includes("--keep-audio");
+const outIdx = args.indexOf("--out");
+const OUT = outIdx >= 0 ? args[outIdx + 1] : null;
+
+if (!positional) {
+  console.error(
+    "Uso: node gladia_transcribe.mjs <url|videoId> [--out arquivo.json] [--keep-audio]",
+  );
+  process.exit(1);
+}
+
+function extractId(s) {
+  const m = s.match(
+    /(?:v=|\/v\/|youtu\.be\/|embed\/|shorts\/)([a-zA-Z0-9_-]{11})/,
+  );
+  if (m) return m[1];
+  return /^[a-zA-Z0-9_-]{11}$/.test(s) ? s : null;
+}
+const videoId = extractId(positional);
+if (!videoId) {
+  console.error(`✖ videoId inválido: ${positional}`);
+  process.exit(1);
+}
+const url = `https://youtu.be/${videoId}`;
+
+function run(cmd, argv) {
+  return new Promise((res, rej) => {
+    const p = spawn(cmd, argv, { stdio: ["ignore", "inherit", "inherit"] });
+    p.on("error", rej);
+    p.on("close", (c) =>
+      c === 0 ? res() : rej(new Error(`${cmd} exit ${c}`)),
+    );
+  });
+}
+
+async function downloadAudio() {
+  const dir = join(tmpdir(), "merlin-yt");
+  await mkdir(dir, { recursive: true });
+  const out = join(dir, `${videoId}.mp4`);
+  if (existsSync(out)) return out;
+  console.log("→ Baixando áudio (yt-dlp android)…");
+  await run("python", [
+    "-m",
+    "yt_dlp",
+    "--extractor-args",
+    "youtube:player_client=android",
+    "-f",
+    "bestaudio/18",
+    "--no-warnings",
+    "-o",
+    out,
+    url,
+  ]);
+  return out;
+}
+
+async function gladiaUpload(file) {
+  const buf = await readFile(file);
+  const fd = new FormData();
+  fd.append("audio", new Blob([buf]), `${videoId}.mp4`);
+  const r = await fetch("https://api.gladia.io/v2/upload", {
+    method: "POST",
+    headers: { "x-gladia-key": API_KEY },
+    body: fd,
+  });
+  if (!r.ok) throw new Error(`upload ${r.status}: ${await r.text()}`);
+  return (await r.json()).audio_url;
+}
+
+async function gladiaStart(audio_url) {
+  const r = await fetch("https://api.gladia.io/v2/pre-recorded", {
+    method: "POST",
+    headers: { "x-gladia-key": API_KEY, "Content-Type": "application/json" },
+    body: JSON.stringify({
+      audio_url,
+      detect_language: false,
+      language_config: { languages: ["pt"] },
+      punctuation_enhanced: true,
+    }),
+  });
+  if (!r.ok) throw new Error(`start ${r.status}: ${await r.text()}`);
+  return (await r.json()).id;
+}
+
+async function gladiaPoll(id) {
+  let n = 0;
+  while (true) {
+    n++;
+    const r = await fetch(`https://api.gladia.io/v2/pre-recorded/${id}`, {
+      headers: { "x-gladia-key": API_KEY },
+    });
+    if (!r.ok) throw new Error(`poll ${r.status}: ${await r.text()}`);
+    const j = await r.json();
+    if (j.status === "done") return j;
+    if (j.status === "error" || j.status === "failed")
+      throw new Error(`job ${id} falhou`);
+    console.log(`  poll #${n} status=${j.status}`);
+    await new Promise((res) => setTimeout(res, 10000));
+  }
+}
+
+function toSegments(result) {
+  const tx = result.result?.transcription || {};
+  const utt = tx.utterances || [];
+  if (utt.length) {
+    return utt
+      .map((u) => ({
+        text: (u.text || "").trim(),
+        start: Number(u.start) || 0,
+        duration: Math.max(0, (Number(u.end) || 0) - (Number(u.start) || 0)),
+      }))
+      .filter((s) => s.text);
+  }
+  const full = tx.full_transcript || "";
+  return full
+    ? [
+        {
+          text: full,
+          start: 0,
+          duration: result.result?.metadata?.audio_duration || 0,
+        },
+      ]
+    : [];
+}
+
+(async () => {
+  const t0 = Date.now();
+  console.log(`═══ Gladia: ${videoId} ═══`);
+  const audio = await downloadAudio();
+  console.log(`✓ Áudio: ${((await stat(audio)).size / 1048576).toFixed(1)}MB`);
+  console.log("→ Upload Gladia…");
+  const audio_url = await gladiaUpload(audio);
+  const id = await gladiaStart(audio_url);
+  console.log(`  job=${id}`);
+  const result = await gladiaPoll(id);
+  const segments = toSegments(result);
+  console.log(
+    `✓ ${segments.length} segmentos em ${((Date.now() - t0) / 1000).toFixed(0)}s`,
+  );
+  if (!segments.length) {
+    console.error("✖ Gladia retornou vazio.");
+    process.exit(1);
+  }
+
+  const last = segments[segments.length - 1];
+  const artifact = {
+    video: {
+      id: videoId,
+      title: videoId,
+      duration: Math.round(last.start + last.duration),
+      channel: "",
+      thumbnail: `https://i.ytimg.com/vi/${videoId}/hqdefault.jpg`,
+    },
+    sourceUrl: url,
+    lang: "pt",
+    transcript: segments,
+    transcriptSource: "gladia",
+  };
+  const outPath = OUT || `transcript-${videoId}.json`;
+  await writeFile(outPath, JSON.stringify(artifact, null, 2), "utf8");
+  console.log(`✓ Gravado: ${outPath}`);
+
+  if (!KEEP_AUDIO) {
+    try {
+      await unlink(audio);
+    } catch {}
+  }
+})().catch((e) => {
+  console.error("✖ Erro:", e.message);
+  process.exit(1);
+});

package/.merlin-core/tools/commands/activate.js

@@ -0,0 +1,72 @@
+/**
+ * activate -- redeem a Merlin license key on this machine (v3.5)
+ *
+ * Bridges the CLI to the core client activation flow. Parses the license key
+ * and operator email, then drives `activation.activate()` which performs the
+ * full handshake (request -> poll -> verify signed .mlic -> cache).
+ *
+ *   merlin activate <KEY> --email <you@example.com>
+ *
+ * This is the command the gate's denial messages point users at
+ * (gate.js:60). It ships INSIDE the npm package.
+ *
+ * @module tools/commands/activate
+ */
+
+"use strict";
+
+/**
+ * Pure arg parser: first non-flag token is the key; `--email <v>` or
+ * `--email=<v>` is the operator email. Exported for testing.
+ *
+ * @param {string[]} args
+ * @returns {{key: string|undefined, email: string|undefined}}
+ */
+function parseActivateArgs(args) {
+  let key;
+  let email;
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i];
+    if (a === "--email") {
+      email = args[i + 1];
+      i++;
+    } else if (a.startsWith("--email=")) {
+      email = a.slice("--email=".length);
+    } else if (!a.startsWith("-") && key === undefined) {
+      key = a;
+    }
+  }
+  return { key, email };
+}
+
+async function run(_command, args, _projectRoot, deps = {}) {
+  const activation =
+    deps.activation || require("../../core/licensing/activation");
+  const publicKeyHex =
+    deps.publicKeyHex || require("../../core/licensing/license").PUBLIC_KEY_HEX;
+
+  const { key, email } = parseActivateArgs(args);
+  if (!key) {
+    return {
+      success: false,
+      error: "usage: merlin activate <KEY> --email <you@example.com>",
+    };
+  }
+
+  try {
+    const result = await activation.activate({ key, email, publicKeyHex });
+    return { success: true, ...result };
+  } catch (e) {
+    return { success: false, error: e.message };
+  }
+}
+
+function formatHuman(result) {
+  if (!result.success) return `ACTIVATION FAILED: ${result.error}`;
+  const exp = result.expires_at
+    ? new Date(result.expires_at * 1000).toISOString().slice(0, 10)
+    : "?";
+  return `License #${result.serial} activated for ${result.client_id} (expires ${exp}).`;
+}
+
+module.exports = { run, formatHuman, parseActivateArgs };

package/.merlin-core/tools/commands/archive-thoughts.js

@@ -0,0 +1,181 @@
+/**
+ * archive-thoughts -- Move terminal thoughts to cold storage (.archive/)
+ *
+ * Physically moves concluded/done/achieved/expired thoughts older than N days
+ * into thoughts/shared/.archive/{type}/, preserving directory structure.
+ * Rebuilds INDEX after archiving.
+ *
+ * Usage:
+ *   node .merlin-core/tools/merlin-tools.js archive [--days 30] [--dry-run]
+ *
+ * @module tools/commands/archive-thoughts
+ * @version 3.0.0
+ */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const {
+  buildIndex,
+  writeIndex,
+} = require("../../core/alkimia/memory/thoughts-indexer");
+
+const DEFAULT_AGE_DAYS = 30;
+
+function daysSince(dateStr) {
+  if (!dateStr || dateStr === "unknown") return 9999;
+  try {
+    const then = new Date(dateStr);
+    const now = new Date();
+    return Math.max(0, Math.floor((now - then) / 86_400_000));
+  } catch {
+    return 999;
+  }
+}
+
+async function run(_command, args, projectRoot) {
+  const dryRun = args.includes("--dry-run");
+  const daysIdx = args.indexOf("--days");
+  const ageDays =
+    daysIdx !== -1 && args[daysIdx + 1]
+      ? parseInt(args[daysIdx + 1], 10)
+      : DEFAULT_AGE_DAYS;
+
+  const thoughtsPath = path.join(projectRoot, ".merlin-core", "thoughts");
+  const sharedPath = path.join(thoughtsPath, "shared");
+  const archivePath = path.join(sharedPath, ".archive");
+
+  // Build fresh index to get current state
+  const index = buildIndex(thoughtsPath);
+  if (!index || !index.entries) {
+    return { success: false, error: "Could not build index" };
+  }
+
+  // Find candidates: terminal lifecycle + older than N days
+  const candidates = index.entries.filter((entry) => {
+    if (entry.lifecycle !== "terminal") return false;
+    return daysSince(entry.date) > ageDays;
+  });
+
+  if (candidates.length === 0) {
+    return {
+      success: true,
+      archived: 0,
+      message: `No terminal items older than ${ageDays} days found.`,
+      dryRun,
+    };
+  }
+
+  const moved = [];
+  const errors = [];
+
+  for (const entry of candidates) {
+    const sourcePath = path.join(sharedPath, entry.path);
+    const typeDir = path.dirname(entry.path); // e.g. "handoffs"
+    const destDir = path.join(archivePath, typeDir);
+    const destPath = path.join(destDir, path.basename(entry.path));
+
+    if (dryRun) {
+      moved.push({
+        from: entry.path,
+        to: `.archive/${entry.path}`,
+        title: entry.title,
+        date: entry.date,
+        status: entry.status,
+        age: daysSince(entry.date),
+      });
+      continue;
+    }
+
+    try {
+      // Verify source still exists
+      if (!fs.existsSync(sourcePath)) {
+        errors.push({ path: entry.path, error: "Source file not found" });
+        continue;
+      }
+
+      // Ensure archive subdirectory exists
+      if (!fs.existsSync(destDir)) {
+        fs.mkdirSync(destDir, { recursive: true });
+      }
+
+      // Move file (use copy+delete for cross-device compatibility on Windows)
+      fs.copyFileSync(sourcePath, destPath);
+      fs.unlinkSync(sourcePath);
+
+      moved.push({
+        from: entry.path,
+        to: `.archive/${entry.path}`,
+        title: entry.title,
+        date: entry.date,
+        status: entry.status,
+      });
+    } catch (err) {
+      errors.push({ path: entry.path, error: err.message });
+    }
+  }
+
+  // Rebuild index after archiving (only if not dry-run)
+  if (!dryRun && moved.length > 0) {
+    writeIndex(thoughtsPath);
+  }
+
+  // Summarize by type
+  const byType = {};
+  for (const item of moved) {
+    const type = path.dirname(item.from);
+    byType[type] = (byType[type] || 0) + 1;
+  }
+
+  return {
+    success: true,
+    dryRun,
+    archived: moved.length,
+    ageDays,
+    byType,
+    items: moved,
+    errors: errors.length > 0 ? errors : undefined,
+  };
+}
+
+function formatHuman(result) {
+  if (!result.success) return `FAILED: ${result.error || result.message}`;
+
+  const prefix = result.dryRun ? "[DRY RUN] " : "";
+
+  if (result.archived === 0) {
+    return `${prefix}${result.message || "Nothing to archive."}`;
+  }
+
+  const lines = [
+    `${prefix}Archived ${result.archived} items (older than ${result.ageDays} days):`,
+  ];
+
+  for (const [type, count] of Object.entries(result.byType)) {
+    lines.push(`  ${type}: ${count}`);
+  }
+
+  lines.push("");
+  for (const item of result.items) {
+    lines.push(`  ${item.from} → ${item.to}`);
+    lines.push(`    "${item.title}" (${item.date}, status: ${item.status})`);
+  }
+
+  if (result.errors) {
+    lines.push("");
+    lines.push("Errors:");
+    for (const err of result.errors) {
+      lines.push(`  ${err.path}: ${err.error}`);
+    }
+  }
+
+  if (!result.dryRun) {
+    lines.push("");
+    lines.push("INDEX.json rebuilt.");
+  }
+
+  return lines.join("\n");
+}
+
+module.exports = { run, formatHuman };