@danhachuel/thunderbolt 0.4.79 → 0.4.81

package/README.md

@@ -443,3 +443,11 @@ Durante a execução, o backlog mostra a actividade recente do helper e o tempo
 A mensagem do card de vídeo deixou de mostrar apenas os primeiros 240 caracteres. A UI apresenta um resumo de erro com até 700 caracteres e disponibiliza um expander não editável com o diagnóstico completo, incluindo os caminhos do log e do manifesto quando existem. Linhas de sucesso de Pexels, sincronização do projecto, verificação do `uv` e marcadores de caminho são excluídas do resumo terminal.
 
 A verificação `uv sync --frozen` tem agora timeout controlado, detalhe seguro do resolver em caso de falha e confirmação explícita quando termina. O estado do manifesto é marcado como `failed` mesmo quando a falha acontece antes de a CLI do MoneyPrinterTurbo iniciar.
+
+## Optimização nativa de tokens com jusTokenMax
+
+O Thunderbolt integra o [jusTokenMax](https://github.com/Kalmantic/jusTokenMax) como optimizador local e reversível de contexto. Payloads JSON e transcrições muito extensas podem ser compactados antes de serem enviados aos providers LLM. Prompts criativos estruturados e conteúdos pequenos são preservados integralmente para evitar alteração semântica.
+
+A configuração encontra-se em **Configuração API > API Keys > Optimização de tokens — jusTokenMax**. É possível activar ou desactivar o optimizador, controlar os levers por tipo, verificar a versão instalada, consultar métricas e limpar apenas os artefactos derivados. Os originais permanecem em `storage/state/token_optimizer_originals/` e podem ser recuperados pelo hash através do módulo interno.
+
+A instalação do pacote inclui `justokenmax` a partir do repositório oficial. Se o CLI não estiver disponível ou ocorrer um erro, o Thunderbolt usa o conteúdo original e regista um fallback local; a geração não fica bloqueada. API keys, passwords, tokens e blobs base64 são redigidos antes de serem guardados em artefactos derivados.

package/app/main.py

@@ -44,6 +44,10 @@ from app.modules.niche_finder.apify import ApifyError, DEFAULT_ACTOR_ID, abort_a
 from app.modules.niche_finder.core import NicheAnalysisError, run_niche_analysis
 from app.modules.niche_finder.data_loader import DatasetError, download_kaggle_dataset
 from app.modules.niche_finder.summarizer import summarize_items
+from app.modules.token_optimizer.cache_manager import clear_derived_cache
+from app.modules.token_optimizer.compressor import check_installation
+from app.modules.token_optimizer.config import DEFAULTS as TOKEN_OPTIMIZER_DEFAULTS
+from app.modules.token_optimizer.metrics import get_stats as get_token_optimizer_stats
 from app.influencers_ui import render_ai_influencer_characters, render_ai_influencer_content, render_ai_influencers_api_status, render_motion_control, render_ugc_products
 from hermes_ui.blueprints import create_blueprint_from_link, list_branding_files, save_generated_blueprint
 from hermes_ui.thumbnail_blueprints import generate_thumbnail_blueprint, list_thumbnail_blueprint_documents, resolve_thumbnail_blueprint, save_thumbnail_blueprint, save_thumbnail_blueprint_pair, thumbnail_blueprint_catalog, thumbnail_blueprint_for_blueprint, thumbnail_blueprint_for_channel
@@ -7133,6 +7137,24 @@ def render_settings():
             st.subheader("API Keys")
             moneyprinter_path = str(settings.get("moneyprinter_path") or "").strip()
             st.caption(f"Pasta do motor de vídeo: `{moneyprinter_path or 'não configurada'}`")
+            with st.expander("Optimização de tokens — jusTokenMax", expanded=False):
+                st.caption("Compressão local e reversível de contextos volumosos antes das chamadas LLM. Os originais permanecem guardados no storage local.")
+                optimizer_enabled = st.checkbox("Activar optimizador", value=bool(settings.get("token_optimizer_enabled", True)), key="token_optimizer_enabled_ui")
+                optimizer_cols = st.columns(4)
+                optimizer_values = {}
+                for index, (key, label) in enumerate((("json", "JSON/API"), ("log", "Logs"), ("pdf", "PDF"), ("csv", "CSV"), ("diff", "Diffs"), ("code", "Código"))):
+                    with optimizer_cols[index % 4]:
+                        optimizer_values[f"token_optimizer_{key}_enabled"] = st.checkbox(label, value=bool(settings.get(f"token_optimizer_{key}_enabled", True)), key=f"token_optimizer_{key}_ui")
+                if st.button("Guardar optimizador", key="save_token_optimizer", use_container_width=True):
+                    settings.update({"token_optimizer_enabled": optimizer_enabled, **optimizer_values})
+                    write_json("settings.json", settings)
+                    st.success("Configuração do jusTokenMax guardada.")
+                status = check_installation()
+                st.caption(f"Estado: {'instalado' if status.get('installed') else 'não instalado'}{(' · versão ' + str(status.get('version'))) if status.get('version') else ''}")
+                stats = get_token_optimizer_stats()
+                st.caption(f"Contextos: {stats['calls']} · redução acumulada: {stats['reduction_percent']}% · fallbacks: {stats['fallbacks']}")
+                if st.button("Limpar cache derivado", key="clear_token_optimizer_cache"):
+                    st.info(f"{clear_derived_cache()} artefacto(s) derivado(s) removido(s). Os originais da aplicação não foram apagados.")
             with st.form("settings_form"):
                 with st.expander("Niche Finder — Kaggle", expanded=False):
                     st.caption("O dataset permanece no Kaggle. O Thunderbolt usa estas credenciais apenas para publicar/executar a kernel e obter os resultados pequenos da análise.")

package/app/modules/niche_finder/summarizer.py

@@ -4,6 +4,8 @@ from typing import Any
 
 import requests
 
+from app.modules.token_optimizer import compress_text
+
 
 DEFAULT_BASE_URLS = {
     "openai": "https://api.openai.com/v1",
@@ -54,13 +56,16 @@ def summarize_transcript(transcript: str, settings: dict[str, Any], *, timeout:
         "esta estrutura: INTRODUÇÃO: um parágrafo sobre como o vídeo começa. ESTRUTURA: uma lista com "
         "os pontos principais e subpontos da progressão do vídeo."
     )
+    prepared_transcript = transcript
+    if len(transcript) >= 50000:
+        prepared_transcript = compress_text(transcript, "log", settings=settings).content
     payload = {
         "model": model,
         "temperature": 0.2,
         "max_tokens": 1200,
         "messages": [
             {"role": "system", "content": system_prompt},
-            {"role": "user", "content": transcript[:120000]},
+            {"role": "user", "content": prepared_transcript[:120000]},
         ],
     }
     try:

package/app/modules/token_optimizer/__init__.py

@@ -0,0 +1,6 @@
+"""Native jusTokenMax integration for Thunderbolt LLM contexts."""
+
+from .compressor import TokenOptimizationResult, compress_text, compress_payload, check_installation, retrieve_original
+from .metrics import get_stats
+
+__all__ = ["TokenOptimizationResult", "compress_text", "compress_payload", "check_installation", "retrieve_original", "get_stats"]

package/app/modules/token_optimizer/cache_manager.py

@@ -0,0 +1,22 @@
+"""Reversible local cache management for jusTokenMax-derived contexts."""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+from hermes_ui import storage
+
+
+
+def cache_directory() -> Path:
+    return storage.STATE / "token_optimizer_originals"
+
+
+def clear_derived_cache() -> int:
+    directory = cache_directory()
+    if not directory.exists():
+        return 0
+    count = sum(1 for item in directory.iterdir() if item.is_file())
+    shutil.rmtree(directory, ignore_errors=True)
+    return count

package/app/modules/token_optimizer/compressor.py

@@ -0,0 +1,137 @@
+"""Safe adapter around the official jusTokenMax Python package."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import hashlib
+import json
+import re
+from pathlib import Path
+from typing import Any, Mapping
+
+from hermes_ui import storage
+
+
+@dataclass(frozen=True)
+class TokenOptimizationResult:
+    content: str
+    original_hash: str
+    before_chars: int
+    after_chars: int
+    kind: str
+    applied: bool
+    fallback: bool = False
+    note: str = ""
+
+
+def _settings() -> dict[str, Any]:
+    value = storage.read_json("settings.json", {})
+    return value if isinstance(value, dict) else {}
+
+
+def _enabled(settings: Mapping[str, Any], kind: str) -> bool:
+    if not bool(settings.get("token_optimizer_enabled", True)):
+        return False
+    return bool(settings.get(f"token_optimizer_{kind}_enabled", True))
+
+
+def _hash(content: str) -> str:
+    return hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+
+def _record(result: TokenOptimizationResult) -> None:
+    path = storage.STATE / "token_optimizer_metrics.json"
+    current = storage.read_json("token_optimizer_metrics.json", {"calls": 0, "applied": 0, "fallbacks": 0, "before_chars": 0, "after_chars": 0})
+    if not isinstance(current, dict):
+        current = {"calls": 0, "applied": 0, "fallbacks": 0, "before_chars": 0, "after_chars": 0}
+    current["calls"] = int(current.get("calls", 0)) + 1
+    current["applied"] = int(current.get("applied", 0)) + int(result.applied)
+    current["fallbacks"] = int(current.get("fallbacks", 0)) + int(result.fallback)
+    current["before_chars"] = int(current.get("before_chars", 0)) + result.before_chars
+    current["after_chars"] = int(current.get("after_chars", 0)) + result.after_chars
+    storage.write_json(path.name, current)
+
+
+def _cache_original(content: str, digest: str) -> None:
+    directory = storage.STATE / "token_optimizer_originals"
+    directory.mkdir(parents=True, exist_ok=True)
+    target = directory / f"{digest}.txt"
+    if not target.exists():
+        target.write_text(content, encoding="utf-8")
+
+
+def _compress_with_jus_token_max(content: str, kind: str) -> str:
+    from justokenmax.jsoncompress import compress_json
+    from justokenmax.logs import compress_log
+    from justokenmax.redact import mask_secrets
+
+    redacted, _ = mask_secrets(content)
+    redacted = re.sub(
+        r'(["\'](?:api[_-]?key|token|password|secret)["\']\s*:\s*["\'])([^"\']+)(["\'])',
+        r'\1****\3', redacted, flags=re.IGNORECASE,
+    )
+    if kind == "json":
+        compressed, _stats = compress_json(redacted)
+        return compressed
+    if kind == "log":
+        compressed, _stats = compress_log(redacted)
+        return compressed
+    return redacted
+
+
+def compress_text(text: str, content_type: str = "auto", *, settings: Mapping[str, Any] | None = None) -> TokenOptimizationResult:
+    raw = str(text or "")
+    kind = str(content_type or "auto").strip().casefold()
+    if kind == "auto":
+        try:
+            json.loads(raw)
+            kind = "json"
+        except (TypeError, ValueError):
+            kind = "text"
+    if kind not in {"json", "log"} or not _enabled(settings or _settings(), kind):
+        result = TokenOptimizationResult(raw, _hash(raw), len(raw), len(raw), kind, False, note="disabled_or_unsupported")
+        _record(result)
+        return result
+    if len(raw) < 8000:
+        result = TokenOptimizationResult(raw, _hash(raw), len(raw), len(raw), kind, False, note="below_minimum_size")
+        _record(result)
+        return result
+    digest = _hash(raw)
+    try:
+        optimized = _compress_with_jus_token_max(raw, kind)
+        if not optimized or len(optimized) >= len(raw):
+            result = TokenOptimizationResult(raw, digest, len(raw), len(raw), kind, False, note="no_reduction")
+        else:
+            _cache_original(raw, digest)
+            result = TokenOptimizationResult(optimized, digest, len(raw), len(optimized), kind, True)
+    except (ImportError, OSError, RuntimeError, TypeError, ValueError) as exc:
+        result = TokenOptimizationResult(raw, digest, len(raw), len(raw), kind, False, True, f"jusTokenMax indisponível: {type(exc).__name__}")
+    _record(result)
+    return result
+
+
+def compress_payload(payload: Any, *, settings: Mapping[str, Any] | None = None) -> TokenOptimizationResult:
+    raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
+    result = compress_text(raw, "json", settings=settings)
+    if result.applied:
+        return result
+    return result
+
+
+def retrieve_original(original_hash: str) -> str | None:
+    digest = str(original_hash or "").strip().lower()
+    if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest):
+        return None
+    path = storage.STATE / "token_optimizer_originals" / f"{digest}.txt"
+    try:
+        return path.read_text(encoding="utf-8")
+    except OSError:
+        return None
+
+
+def check_installation() -> dict[str, Any]:
+    try:
+        import importlib.metadata
+        return {"installed": True, "version": importlib.metadata.version("justokenmax")}
+    except (ImportError, importlib.metadata.PackageNotFoundError):
+        return {"installed": False, "version": "", "message": "Instale o jusTokenMax para activar a compressão nativa."}

package/app/modules/token_optimizer/config.py

@@ -0,0 +1,36 @@
+"""Persistent token optimizer settings."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from hermes_ui import storage
+
+
+DEFAULTS = {
+    "token_optimizer_enabled": True,
+    "token_optimizer_json_enabled": True,
+    "token_optimizer_log_enabled": True,
+    "token_optimizer_pdf_enabled": True,
+    "token_optimizer_csv_enabled": True,
+    "token_optimizer_diff_enabled": True,
+    "token_optimizer_code_enabled": True,
+    "token_optimizer_redact_enabled": True,
+}
+
+
+def load(settings: dict[str, Any] | None = None) -> dict[str, Any]:
+    result = dict(DEFAULTS)
+    source = settings if isinstance(settings, dict) else storage.read_json("settings.json", {})
+    if isinstance(source, dict):
+        result.update({key: source.get(key, value) for key, value in DEFAULTS.items()})
+    return result
+
+
+def save(values: dict[str, Any]) -> dict[str, Any]:
+    current = storage.read_json("settings.json", {})
+    if not isinstance(current, dict):
+        current = {}
+    current.update({key: bool(values.get(key, default)) for key, default in DEFAULTS.items()})
+    storage.write_json("settings.json", current)
+    return current

package/app/modules/token_optimizer/metrics.py

@@ -0,0 +1,22 @@
+"""Local metrics for token optimization effectiveness."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from hermes_ui import storage
+
+
+def get_stats() -> dict[str, Any]:
+    value = storage.read_json("token_optimizer_metrics.json", {})
+    stats = value if isinstance(value, dict) else {}
+    before = int(stats.get("before_chars", 0) or 0)
+    after = int(stats.get("after_chars", 0) or 0)
+    return {
+        "calls": int(stats.get("calls", 0) or 0),
+        "applied": int(stats.get("applied", 0) or 0),
+        "fallbacks": int(stats.get("fallbacks", 0) or 0),
+        "before_chars": before,
+        "after_chars": after,
+        "reduction_percent": round(max(0, before - after) * 100 / before, 2) if before else 0.0,
+    }

package/hermes_ui/provider_routing.py

@@ -26,6 +26,7 @@ from .llm_providers import (
     provider_definition,
 )
 from .storage import STORAGE, atomic_write, ensure_storage
+from app.modules.token_optimizer import compress_text
 
 
 POOL_LLM = "llm_text"
@@ -450,6 +451,8 @@ def route_json_request(
 
 def route_llm_json(settings: Mapping[str, Any], system_prompt: str, user_prompt: str) -> RoutedResponse:
     """Send an OpenAI-compatible JSON chat call through the LLM pool."""
+    optimized_user_prompt = compress_text(user_prompt, "json", settings=settings) if len(str(user_prompt or "")) >= 50000 else None
+    prepared_user_prompt = optimized_user_prompt.content if optimized_user_prompt is not None else user_prompt
     def request(card: dict[str, Any]) -> Any:
         definition = provider_definition(card.get("provider"))
         base_url = str(card.get("base_url") or definition.default_base_url).strip().rstrip("/")
@@ -462,7 +465,7 @@ def route_llm_json(settings: Mapping[str, Any], system_prompt: str, user_prompt:
             "model": str(card.get("model") or "").strip(),
             "messages": [
                 {"role": "system", "content": system_prompt},
-                {"role": "user", "content": user_prompt},
+                {"role": "user", "content": prepared_user_prompt},
             ],
             "response_format": {"type": "json_object"},
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@danhachuel/thunderbolt",
-  "version": "0.4.79",
+  "version": "0.4.81",
   "description": "Thunderbolt — interface local para operação de canais faceless e motor MoneyPrinterTurbo",
   "license": "MIT",
   "main": "scripts/cli.mjs",

package/requirements.txt

@@ -23,3 +23,4 @@ supabase>=2.4,<3
 bilibili-api-python>=17.4,<18
 playwright>=1.45,<2
 composio>=0.21,<1
+justokenmax @ https://github.com/Kalmantic/jusTokenMax/archive/refs/heads/main.zip#subdirectory=python