arkaos 4.8.0 → 4.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/VERSION +1 -1
- package/arka/skills/research/SKILL.md +6 -6
- package/config/disc-team-validator.sh +7 -7
- package/config/evals/dev.yaml +28 -0
- package/config/evals/finance.yaml +28 -0
- package/config/evals/kb.yaml +26 -0
- package/config/evals/marketing.yaml +28 -0
- package/config/evals/strategy.yaml +26 -0
- package/core/agents/__pycache__/registry_gen.cpython-313.pyc +0 -0
- package/core/agents/registry_gen.py +6 -4
- package/core/evals/__init__.py +19 -0
- package/core/evals/__pycache__/__init__.cpython-313.pyc +0 -0
- package/core/evals/__pycache__/schema.cpython-313.pyc +0 -0
- package/core/evals/__pycache__/verdict_labels.cpython-313.pyc +0 -0
- package/core/evals/schema.py +54 -0
- package/core/evals/verdict_labels.py +72 -0
- package/core/runtime/__pycache__/llm_cost_telemetry.cpython-313.pyc +0 -0
- package/core/runtime/__pycache__/llm_cost_telemetry.cpython-314.pyc +0 -0
- package/core/runtime/llm_cost_telemetry.py +13 -1
- package/core/synapse/__pycache__/engine.cpython-313.pyc +0 -0
- package/core/synapse/__pycache__/engine.cpython-314.pyc +0 -0
- package/core/synapse/__pycache__/layers.cpython-313.pyc +0 -0
- package/core/synapse/__pycache__/layers.cpython-314.pyc +0 -0
- package/core/synapse/engine.py +6 -0
- package/core/synapse/layers.py +17 -0
- package/departments/dev/skills/research/SKILL.md +3 -2
- package/knowledge/commands-registry-v2.json +58 -108
- package/knowledge/commands-registry.json.bak +1 -1
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/scripts/tools/__pycache__/prompt_surface_benchmark.cpython-313.pyc +0 -0
- package/scripts/tools/prompt_surface_benchmark.py +144 -0
- package/knowledge/agents-registry.json +0 -254
package/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
4.
|
|
1
|
+
4.9.0
|
|
@@ -6,13 +6,13 @@ description: >
|
|
|
6
6
|
single cited report, and writes a Knowledge Base note to the Obsidian
|
|
7
7
|
vault. KB-first: Obsidian is searched before any external call.
|
|
8
8
|
TRIGGER: "/arka research", "pesquisa sobre", "investiga",
|
|
9
|
-
"faz research", "research <topic>", "best practices for"
|
|
10
|
-
the art", "o que se sabe sobre" — general, market,
|
|
11
|
-
topics whose deliverable is a synthesised KB note.
|
|
9
|
+
"faz research", "research <topic>", "best practices for" non-code
|
|
10
|
+
topics, "state of the art", "o que se sabe sobre" — general, market,
|
|
11
|
+
tooling, or KB topics whose deliverable is a synthesised KB note.
|
|
12
12
|
SKIP: dev-scoped technical research — library or framework evaluation,
|
|
13
|
-
"que lib uso", "which library/framework"
|
|
14
|
-
/
|
|
15
|
-
|
|
13
|
+
"que lib uso", "which library/framework", best-practice questions
|
|
14
|
+
about implementation choices — dev/research (Lucas) wins; planning a
|
|
15
|
+
task — arka-forge wins.
|
|
16
16
|
allowed-tools: [Agent, Read, Write, mcp__obsidian__search_notes]
|
|
17
17
|
---
|
|
18
18
|
|
|
@@ -1,26 +1,26 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
2
|
# ============================================================================
|
|
3
3
|
# ARKA OS — DISC Team Balance Validator
|
|
4
|
-
# Reads agents-registry.json and validates team DISC distribution
|
|
4
|
+
# Reads agents-registry-v2.json and validates team DISC distribution
|
|
5
5
|
# ============================================================================
|
|
6
6
|
set -e
|
|
7
7
|
|
|
8
8
|
ARKA_OS="${ARKA_OS:-$HOME/.claude/skills/arka}"
|
|
9
9
|
REPO_DIR="${REPO_DIR:-$(cat "$ARKA_OS/.repo-path" 2>/dev/null || echo "")}"
|
|
10
|
-
REGISTRY="${REGISTRY:-$REPO_DIR/knowledge/agents-registry.json}"
|
|
10
|
+
REGISTRY="${REGISTRY:-$REPO_DIR/knowledge/agents-registry-v2.json}"
|
|
11
11
|
|
|
12
12
|
if [ ! -f "$REGISTRY" ]; then
|
|
13
|
-
echo "Error: agents-registry.json not found at $REGISTRY"
|
|
13
|
+
echo "Error: agents-registry-v2.json not found at $REGISTRY"
|
|
14
14
|
exit 1
|
|
15
15
|
fi
|
|
16
16
|
|
|
17
17
|
command -v jq &>/dev/null || { echo "Error: jq is required."; exit 1; }
|
|
18
18
|
|
|
19
19
|
# Read counts from registry
|
|
20
|
-
D_COUNT=$(jq '.
|
|
21
|
-
I_COUNT=$(jq '.
|
|
22
|
-
S_COUNT=$(jq '.
|
|
23
|
-
C_COUNT=$(jq '.
|
|
20
|
+
D_COUNT=$(jq '._meta.disc_distribution.D' "$REGISTRY")
|
|
21
|
+
I_COUNT=$(jq '._meta.disc_distribution.I' "$REGISTRY")
|
|
22
|
+
S_COUNT=$(jq '._meta.disc_distribution.S' "$REGISTRY")
|
|
23
|
+
C_COUNT=$(jq '._meta.disc_distribution.C' "$REGISTRY")
|
|
24
24
|
TOTAL=$(( D_COUNT + I_COUNT + S_COUNT + C_COUNT ))
|
|
25
25
|
|
|
26
26
|
# Calculate percentages
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Dev department reference eval tasks (seed set).
|
|
2
|
+
# Judged against expected_properties + QG verdict; see core/evals/schema.py.
|
|
3
|
+
- id: dev-feature-auth-endpoint
|
|
4
|
+
department: dev
|
|
5
|
+
prompt: >
|
|
6
|
+
Implementa um endpoint de autenticação por token (login + refresh)
|
|
7
|
+
num serviço Laravel existente, com testes.
|
|
8
|
+
expected_properties:
|
|
9
|
+
- FormRequest valida credenciais (nunca validação inline no controller)
|
|
10
|
+
- Lógica em Service/Repository, controller fino
|
|
11
|
+
- Feature tests cobrem sucesso, credenciais inválidas e refresh expirado
|
|
12
|
+
- Nenhum segredo hardcoded; usa config/env
|
|
13
|
+
rubric: >
|
|
14
|
+
Rejeitar se os testes não correram (exit code em registo) ou se o
|
|
15
|
+
fluxo de refresh não invalida o token antigo.
|
|
16
|
+
tags: [backend, laravel, security-adjacent]
|
|
17
|
+
|
|
18
|
+
- id: dev-refactor-n-plus-one
|
|
19
|
+
department: dev
|
|
20
|
+
prompt: >
|
|
21
|
+
Este listing de encomendas faz N+1 queries; corrige sem alterar o
|
|
22
|
+
contrato JSON da API.
|
|
23
|
+
expected_properties:
|
|
24
|
+
- Eager loading ou query única substitui o loop de queries
|
|
25
|
+
- Contrato de resposta byte-idêntico (teste de regressão prova-o)
|
|
26
|
+
- Query count coberto por teste (assertQueryCount ou equivalente)
|
|
27
|
+
rubric: Rejeitar qualquer alteração de schema de resposta.
|
|
28
|
+
tags: [backend, performance]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Finance department reference eval tasks (seed set).
|
|
2
|
+
- id: fin-unit-economics-saas
|
|
3
|
+
department: finance
|
|
4
|
+
prompt: >
|
|
5
|
+
Calcula os unit economics de um SaaS com MRR 8000 EUR, 120 clientes,
|
|
6
|
+
churn mensal 3%, CAC 250 EUR e margem bruta 80%.
|
|
7
|
+
expected_properties:
|
|
8
|
+
- LTV derivado com fórmula explícita (ARPU, churn, margem)
|
|
9
|
+
- Rácio LTV/CAC calculado e interpretado contra o benchmark 3x
|
|
10
|
+
- Payback period em meses
|
|
11
|
+
- Aritmética verificável (números batem certo)
|
|
12
|
+
rubric: >
|
|
13
|
+
Rejeitar qualquer erro aritmético — o arkaos-not-yes-man aplica-se a
|
|
14
|
+
matemática impossível do próprio input.
|
|
15
|
+
tags: [unit-economics, saas]
|
|
16
|
+
|
|
17
|
+
- id: fin-cashflow-13-weeks
|
|
18
|
+
department: finance
|
|
19
|
+
prompt: >
|
|
20
|
+
Constrói um forecast de cash flow a 13 semanas para uma agência com
|
|
21
|
+
3 clientes em retainer e folha salarial fixa.
|
|
22
|
+
expected_properties:
|
|
23
|
+
- Entradas por cliente com datas de recebimento realistas
|
|
24
|
+
- Saídas fixas e variáveis separadas
|
|
25
|
+
- Saldo semanal acumulado com semana crítica identificada
|
|
26
|
+
- Cenário pessimista (cliente atrasa 30 dias) incluído
|
|
27
|
+
rubric: Rejeitar forecast sem cenário de stress.
|
|
28
|
+
tags: [cashflow, forecast]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Knowledge department reference eval tasks (seed set).
|
|
2
|
+
- id: kb-research-note-synthesis
|
|
3
|
+
department: kb
|
|
4
|
+
prompt: >
|
|
5
|
+
Pesquisa "estratégias de retry para APIs rate-limited" e produz uma
|
|
6
|
+
nota KB sintetizada com fontes.
|
|
7
|
+
expected_properties:
|
|
8
|
+
- KB consultada primeiro (citação [[wikilink]] ou gap declarado)
|
|
9
|
+
- Fontes externas citadas com claims atribuídos
|
|
10
|
+
- Nota com frontmatter YAML + wikilinks + estrutura Zettelkasten
|
|
11
|
+
- Síntese própria, não colagem de excertos
|
|
12
|
+
rubric: Rejeitar claims sem fonte ou nota sem ligações ao grafo.
|
|
13
|
+
tags: [research, zettelkasten]
|
|
14
|
+
|
|
15
|
+
- id: kb-moc-reorganize
|
|
16
|
+
department: kb
|
|
17
|
+
prompt: >
|
|
18
|
+
Organiza 15 notas soltas sobre "pricing" num MOC (Map of Content)
|
|
19
|
+
coerente no vault.
|
|
20
|
+
expected_properties:
|
|
21
|
+
- MOC com agrupamento temático justificado
|
|
22
|
+
- Todas as 15 notas ligadas (nenhuma órfã)
|
|
23
|
+
- Hierarquia máxima de 2 níveis
|
|
24
|
+
- Notas duplicadas identificadas para merge
|
|
25
|
+
rubric: Rejeitar agrupamento alfabético ou sem lógica temática.
|
|
26
|
+
tags: [moc, organization]
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Marketing department reference eval tasks (seed set).
|
|
2
|
+
- id: mkt-launch-campaign-plan
|
|
3
|
+
department: marketing
|
|
4
|
+
prompt: >
|
|
5
|
+
Desenha o plano de campanha de lançamento (4 semanas) para um
|
|
6
|
+
micro-SaaS B2B de gestão de inventário, orçamento 2000 EUR.
|
|
7
|
+
expected_properties:
|
|
8
|
+
- Estrutura AARRR explícita com métrica alvo por etapa
|
|
9
|
+
- Canais escolhidos justificados contra o ICP (não lista genérica)
|
|
10
|
+
- Orçamento alocado por canal soma 2000 EUR
|
|
11
|
+
- Calendário semanal com owners
|
|
12
|
+
rubric: >
|
|
13
|
+
Rejeitar clichés de AI copy e canais sem justificação de fit; o
|
|
14
|
+
framework citado tem de ser aplicado, não só nomeado.
|
|
15
|
+
tags: [gtm, aarrr, budget]
|
|
16
|
+
|
|
17
|
+
- id: mkt-seo-content-brief
|
|
18
|
+
department: marketing
|
|
19
|
+
prompt: >
|
|
20
|
+
Cria um brief SEO para um artigo pilar sobre "supplier sync para
|
|
21
|
+
Shopify" com intenção de pesquisa comercial.
|
|
22
|
+
expected_properties:
|
|
23
|
+
- Keyword primária + secundárias com intenção classificada
|
|
24
|
+
- Outline H2/H3 cobre as perguntas do SERP
|
|
25
|
+
- Internal links e CTA definidos
|
|
26
|
+
- Meta title/description dentro dos limites de caracteres
|
|
27
|
+
rubric: Rejeitar keyword stuffing ou outline sem lógica de funil.
|
|
28
|
+
tags: [seo, content]
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Strategy department reference eval tasks (seed set).
|
|
2
|
+
- id: strat-five-forces-entry
|
|
3
|
+
department: strategy
|
|
4
|
+
prompt: >
|
|
5
|
+
Avalia com Porter's Five Forces a entrada de um novo player no
|
|
6
|
+
mercado de ferramentas de orquestração de agentes AI.
|
|
7
|
+
expected_properties:
|
|
8
|
+
- As 5 forças analisadas com evidência específica do mercado, não genérica
|
|
9
|
+
- Intensidade classificada por força com justificação
|
|
10
|
+
- Conclusão acionável (entrar/não entrar/entrar com que moat)
|
|
11
|
+
rubric: >
|
|
12
|
+
Rejeitar análise que apenas define as forças sem as aplicar ao
|
|
13
|
+
mercado concreto.
|
|
14
|
+
tags: [porter, market-entry]
|
|
15
|
+
|
|
16
|
+
- id: strat-moat-audit
|
|
17
|
+
department: strategy
|
|
18
|
+
prompt: >
|
|
19
|
+
Audita os moats de um produto dev-tools open-source com dashboard
|
|
20
|
+
pago, usando 7 Powers.
|
|
21
|
+
expected_properties:
|
|
22
|
+
- Cada power avaliada como presente/ausente/construível
|
|
23
|
+
- Pelo menos um trade-off identificado (ex. open-source vs counter-positioning)
|
|
24
|
+
- Recomendação priorizada de qual power investir primeiro
|
|
25
|
+
rubric: Rejeitar se citar 7 Powers sem mapear ao produto.
|
|
26
|
+
tags: [7-powers, moat]
|
|
Binary file
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
"""Generate agents-registry.json from all agent YAML files.
|
|
1
|
+
"""Generate agents-registry-v2.json from all agent YAML files.
|
|
2
2
|
|
|
3
|
-
Scans departments/*/agents/*.yaml and produces
|
|
4
|
-
registry with all agent metadata including behavioral
|
|
3
|
+
Scans departments/*/agents/*.yaml and produces the single canonical
|
|
4
|
+
machine-readable registry with all agent metadata including behavioral
|
|
5
|
+
DNA. The legacy knowledge/agents-registry.json (v1) was removed —
|
|
6
|
+
tests/python/test_registry_gen.py guards against its resurrection.
|
|
5
7
|
"""
|
|
6
8
|
|
|
7
9
|
import json
|
|
@@ -12,7 +14,7 @@ from core.agents.loader import load_agent
|
|
|
12
14
|
|
|
13
15
|
|
|
14
16
|
def generate_registry(departments_dir: str | Path, output_path: str | Path) -> dict:
|
|
15
|
-
"""Generate agents-registry.json from YAML agent files.
|
|
17
|
+
"""Generate agents-registry-v2.json from YAML agent files.
|
|
16
18
|
|
|
17
19
|
Args:
|
|
18
20
|
departments_dir: Path to departments/ directory.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Eval harness foundation (E2E audit v4.3.6 P2).
|
|
2
|
+
|
|
3
|
+
Reference eval tasks per department (config/evals/*.yaml) plus the
|
|
4
|
+
QG-verdict label log — the free labeled dataset every future eval run
|
|
5
|
+
or local-model distillation consumes.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from core.evals.schema import EvalTask, load_eval_tasks
|
|
9
|
+
from core.evals.verdict_labels import (
|
|
10
|
+
load_verdict_labels,
|
|
11
|
+
record_verdict_label,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"EvalTask",
|
|
16
|
+
"load_eval_tasks",
|
|
17
|
+
"load_verdict_labels",
|
|
18
|
+
"record_verdict_label",
|
|
19
|
+
]
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Eval task schema and loader.
|
|
2
|
+
|
|
3
|
+
Reference tasks live in ``config/evals/<department>.yaml`` — one YAML
|
|
4
|
+
list per department, versioned with the repo so evals evolve with the
|
|
5
|
+
agents they measure. See ADR 2026-07-09-evals-and-distillation.md.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import yaml
|
|
13
|
+
from pydantic import BaseModel, Field
|
|
14
|
+
|
|
15
|
+
DEFAULT_EVALS_DIR = (
|
|
16
|
+
Path(__file__).resolve().parent.parent.parent / "config" / "evals"
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class EvalTask(BaseModel):
|
|
21
|
+
"""One reference task an agent/department run is judged against."""
|
|
22
|
+
|
|
23
|
+
id: str = Field(pattern=r"^[a-z0-9][a-z0-9-]*$")
|
|
24
|
+
department: str
|
|
25
|
+
prompt: str = Field(min_length=10)
|
|
26
|
+
expected_properties: list[str] = Field(
|
|
27
|
+
min_length=1,
|
|
28
|
+
description="Verifiable properties the deliverable must exhibit",
|
|
29
|
+
)
|
|
30
|
+
rubric: str = Field(
|
|
31
|
+
default="",
|
|
32
|
+
description="Free-text judging guidance beyond the properties",
|
|
33
|
+
)
|
|
34
|
+
tags: list[str] = Field(default_factory=list)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def load_eval_tasks(evals_dir: Path | None = None) -> list[EvalTask]:
|
|
38
|
+
"""Load every task from config/evals/*.yaml, validated.
|
|
39
|
+
|
|
40
|
+
Raises on schema violations or duplicate ids — a broken eval set
|
|
41
|
+
must fail loudly, not skew results silently.
|
|
42
|
+
"""
|
|
43
|
+
base = evals_dir or DEFAULT_EVALS_DIR
|
|
44
|
+
tasks: list[EvalTask] = []
|
|
45
|
+
seen: set[str] = set()
|
|
46
|
+
for path in sorted(base.glob("*.yaml")):
|
|
47
|
+
raw = yaml.safe_load(path.read_text(encoding="utf-8")) or []
|
|
48
|
+
for item in raw:
|
|
49
|
+
task = EvalTask.model_validate(item)
|
|
50
|
+
if task.id in seen:
|
|
51
|
+
raise ValueError(f"duplicate eval task id: {task.id}")
|
|
52
|
+
seen.add(task.id)
|
|
53
|
+
tasks.append(task)
|
|
54
|
+
return tasks
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""QG verdicts as free eval labels.
|
|
2
|
+
|
|
3
|
+
Every Quality Gate review already produces a structured ``QGVerdict``
|
|
4
|
+
(APPROVED/REJECTED + blockers + evidence summary). Persisting them turns
|
|
5
|
+
each review into a labeled example — the training/eval signal the E2E
|
|
6
|
+
audit called "labels gratuitos". Append-only JSONL, same conventions as
|
|
7
|
+
the other ``~/.arkaos/telemetry`` writers: never raises, fcntl advisory
|
|
8
|
+
lock via ``llm_cost_telemetry._locked_append`` (QG reviews run as
|
|
9
|
+
parallel subagents and verdict lines far exceed atomic-write sizes, so
|
|
10
|
+
unlocked appends could interleave and corrupt the label corpus),
|
|
11
|
+
``ARKA_QG_LABELS_PATH`` override for tests.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
from datetime import datetime, timezone
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from core.governance.qg_verdict import QGVerdict
|
|
23
|
+
from core.runtime.llm_cost_telemetry import _locked_append
|
|
24
|
+
|
|
25
|
+
DEFAULT_LABELS_PATH = (
|
|
26
|
+
Path.home() / ".arkaos" / "telemetry" / "qg-verdicts.jsonl"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _labels_path() -> Path:
|
|
31
|
+
override = os.environ.get("ARKA_QG_LABELS_PATH", "").strip()
|
|
32
|
+
return Path(override) if override else DEFAULT_LABELS_PATH
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def record_verdict_label(
|
|
36
|
+
verdict: QGVerdict,
|
|
37
|
+
deliverable: str = "",
|
|
38
|
+
department: str = "",
|
|
39
|
+
eval_task_id: str = "",
|
|
40
|
+
session_id: str = "",
|
|
41
|
+
) -> None:
|
|
42
|
+
"""Append one labeled QG example. Never raises (telemetry contract)."""
|
|
43
|
+
try:
|
|
44
|
+
entry: dict[str, Any] = {
|
|
45
|
+
"ts": datetime.now(timezone.utc).isoformat(),
|
|
46
|
+
"deliverable": str(deliverable or ""),
|
|
47
|
+
"department": str(department or ""),
|
|
48
|
+
"eval_task_id": str(eval_task_id or ""),
|
|
49
|
+
"session_id": str(session_id or ""),
|
|
50
|
+
**verdict.model_dump(),
|
|
51
|
+
}
|
|
52
|
+
with _locked_append(_labels_path()) as fh:
|
|
53
|
+
fh.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
|
54
|
+
except Exception: # noqa: BLE001 — telemetry must never raise
|
|
55
|
+
return
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def load_verdict_labels(path: Path | None = None) -> list[dict[str, Any]]:
|
|
59
|
+
"""Read all labeled examples; malformed lines are skipped."""
|
|
60
|
+
target = path or _labels_path()
|
|
61
|
+
if not target.exists():
|
|
62
|
+
return []
|
|
63
|
+
out: list[dict[str, Any]] = []
|
|
64
|
+
for line in target.read_text(encoding="utf-8").splitlines():
|
|
65
|
+
line = line.strip()
|
|
66
|
+
if not line:
|
|
67
|
+
continue
|
|
68
|
+
try:
|
|
69
|
+
out.append(json.loads(line))
|
|
70
|
+
except json.JSONDecodeError:
|
|
71
|
+
continue
|
|
72
|
+
return out
|
|
Binary file
|
|
Binary file
|
|
@@ -232,6 +232,16 @@ def _load_slice(
|
|
|
232
232
|
return kept, corrupt
|
|
233
233
|
|
|
234
234
|
|
|
235
|
+
def _is_fallback_diagnostic(entry: dict[str, Any]) -> bool:
|
|
236
|
+
"""Provider-fallback rows carry the fallback REASON in the model field
|
|
237
|
+
(_log_fallback in llm_provider.py: model='unavailable'/'selected').
|
|
238
|
+
They belong in the by-provider view (degraded chains) but leak bogus
|
|
239
|
+
zero-token rows into any model-keyed breakdown. Totals and by_provider
|
|
240
|
+
keep counting them, so sum(by_model[*].call_count) < call_count when
|
|
241
|
+
fallbacks occurred — intentional, not a reconciliation bug."""
|
|
242
|
+
return str(entry.get("provider") or "").startswith("fallback:")
|
|
243
|
+
|
|
244
|
+
|
|
235
245
|
def _group(entries: list[dict[str, Any]], key: str) -> dict[str, dict[str, Any]]:
|
|
236
246
|
out: dict[str, dict[str, Any]] = {}
|
|
237
247
|
for entry in entries:
|
|
@@ -298,7 +308,9 @@ def summarise(
|
|
|
298
308
|
cache_hit_rate=finalised["cache_hit_rate"],
|
|
299
309
|
call_count=finalised["call_count"],
|
|
300
310
|
by_provider=_group(entries, "provider"),
|
|
301
|
-
by_model=_group(
|
|
311
|
+
by_model=_group(
|
|
312
|
+
[e for e in entries if not _is_fallback_diagnostic(e)], "model"
|
|
313
|
+
),
|
|
302
314
|
by_category=_group(entries, "category"),
|
|
303
315
|
by_session=sessions,
|
|
304
316
|
advisories=_build_advisories(sessions, advisory_threshold_usd),
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/core/synapse/engine.py
CHANGED
|
@@ -118,6 +118,12 @@ class SynapseEngine:
|
|
|
118
118
|
(ctx.user_input or "").encode("utf-8", "replace")
|
|
119
119
|
).hexdigest()[:12]
|
|
120
120
|
cache_key += f":{digest}"
|
|
121
|
+
if getattr(layer, "session_sensitive", False):
|
|
122
|
+
# Session-sensitive layers (KB retrieval) must recompute per
|
|
123
|
+
# session: a cross-session cache hit would skip their
|
|
124
|
+
# per-session side effects (KBSessionCache.store).
|
|
125
|
+
session_id = (ctx.extra or {}).get("session_id", "default")
|
|
126
|
+
cache_key += f":{session_id}"
|
|
121
127
|
|
|
122
128
|
# Check cache
|
|
123
129
|
if layer.cache_ttl > 0:
|
package/core/synapse/layers.py
CHANGED
|
@@ -86,6 +86,19 @@ class Layer(ABC):
|
|
|
86
86
|
"""
|
|
87
87
|
return False
|
|
88
88
|
|
|
89
|
+
@property
|
|
90
|
+
def session_sensitive(self) -> bool:
|
|
91
|
+
"""True when compute() has per-session side effects or output.
|
|
92
|
+
|
|
93
|
+
Session-sensitive layers get ctx.extra['session_id'] added to
|
|
94
|
+
their cache key — without it, a cache hit from one session
|
|
95
|
+
suppresses compute() for a concurrent session in the same cwd,
|
|
96
|
+
skipping that session's side effects (found 2026-07-09: L3.5's
|
|
97
|
+
KBSessionCache.store() never ran for the second session, so its
|
|
98
|
+
KB-injected state and overlap markers belonged to the first).
|
|
99
|
+
"""
|
|
100
|
+
return False
|
|
101
|
+
|
|
89
102
|
@property
|
|
90
103
|
def priority(self) -> int:
|
|
91
104
|
"""Layer priority (lower = computed first)."""
|
|
@@ -546,6 +559,10 @@ class KnowledgeRetrievalLayer(Layer):
|
|
|
546
559
|
def input_sensitive(self) -> bool:
|
|
547
560
|
return True
|
|
548
561
|
|
|
562
|
+
@property
|
|
563
|
+
def session_sensitive(self) -> bool:
|
|
564
|
+
return True
|
|
565
|
+
|
|
549
566
|
@property
|
|
550
567
|
def cache_ttl(self) -> int:
|
|
551
568
|
return 30
|
|
@@ -12,8 +12,9 @@ description: >
|
|
|
12
12
|
adding a new dependency or committing to an architecture-relevant
|
|
13
13
|
library.
|
|
14
14
|
SKIP: general, market, or knowledge-base research whose deliverable
|
|
15
|
-
is an Obsidian KB note
|
|
16
|
-
|
|
15
|
+
is an Obsidian KB note, including "best practices for" a non-code
|
|
16
|
+
topic — arka-research (/arka research, 5-source fan-out) wins;
|
|
17
|
+
requirements definition — arka-dev-spec wins.
|
|
17
18
|
allowed-tools: [Read, Write, Edit, Bash, Grep, Glob, Agent, WebFetch, WebSearch]
|
|
18
19
|
---
|
|
19
20
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"_meta": {
|
|
3
3
|
"version": "2.0.0",
|
|
4
|
-
"generated": "2026-
|
|
5
|
-
"total_commands":
|
|
4
|
+
"generated": "2026-07-09T15:04:23.205671",
|
|
5
|
+
"total_commands": 211,
|
|
6
6
|
"generator": "core/registry/generator.py",
|
|
7
7
|
"departments": {
|
|
8
8
|
"brand": 12,
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"saas": 14,
|
|
22
22
|
"sales": 10,
|
|
23
23
|
"strategy": 10,
|
|
24
|
-
"arka":
|
|
24
|
+
"arka": 19
|
|
25
25
|
}
|
|
26
26
|
},
|
|
27
27
|
"commands": [
|
|
@@ -3587,237 +3587,187 @@
|
|
|
3587
3587
|
"id": "arka-status",
|
|
3588
3588
|
"command": "/arka status",
|
|
3589
3589
|
"department": "arka",
|
|
3590
|
-
"description": "System status (version, departments, agents, active projects)",
|
|
3590
|
+
"description": "System status (version, departments, agents, active projects). Includes **LLM costs (24h)** section: top-line cost + cache hit rate + call count from `core.runtime.llm_cost_telemetry.summarise(period=\"today\")`. Also includes **Enforcement (24h)** section: total calls, block rate, top blocked tools/reasons from `core.governance.enforcement_telemetry.summarise(period=\"today\")` (PR19 v2.41.0). Plus **Reorganization (today)** section: today's proposal path + artifact count from `core.cognition.reorganizer_scheduler.status_summary()` (PR24 v2.46.0). Plus **Model routing** section: gateway live/off + resolved per-slot routes + served counts from `core.runtime.model_routing_check.status_summary()`. Plus **MCP usage (24h)** section: total calls + servers in use + top servers from `core.runtime.mcp_telemetry.summarise(period=\"today\")`.",
|
|
3591
3591
|
"keywords": [],
|
|
3592
3592
|
"tier": 3,
|
|
3593
3593
|
"modifies_code": false,
|
|
3594
3594
|
"requires_branch": false
|
|
3595
3595
|
},
|
|
3596
3596
|
{
|
|
3597
|
-
"id": "arka-
|
|
3598
|
-
"command": "/arka
|
|
3597
|
+
"id": "arka-costs-[period]",
|
|
3598
|
+
"command": "/arka costs [period]",
|
|
3599
3599
|
"department": "arka",
|
|
3600
|
-
"description": "
|
|
3600
|
+
"description": "LLM cost visibility — aggregates telemetry by day/week/month/all, with top expensive sessions. See `arka/skills/costs/SKILL.md`. Shells out to `~/.arkaos/bin/arka-py -m core.runtime.llm_cost_telemetry_cli <period>`.",
|
|
3601
3601
|
"keywords": [],
|
|
3602
3602
|
"tier": 3,
|
|
3603
3603
|
"modifies_code": false,
|
|
3604
3604
|
"requires_branch": false
|
|
3605
3605
|
},
|
|
3606
3606
|
{
|
|
3607
|
-
"id": "arka-
|
|
3608
|
-
"command": "/arka
|
|
3607
|
+
"id": "arka-enforcement-[period]",
|
|
3608
|
+
"command": "/arka enforcement [period]",
|
|
3609
3609
|
"department": "arka",
|
|
3610
|
-
"description": "
|
|
3610
|
+
"description": "Enforcement compliance — aggregates flow-marker enforcement telemetry by day/week/month/all. Shows block rate, top blocked tools, top reasons. Shells out to `~/.arkaos/bin/arka-py -m core.governance.enforcement_telemetry_cli <period>`.",
|
|
3611
3611
|
"keywords": [],
|
|
3612
3612
|
"tier": 3,
|
|
3613
3613
|
"modifies_code": false,
|
|
3614
3614
|
"requires_branch": false
|
|
3615
3615
|
},
|
|
3616
3616
|
{
|
|
3617
|
-
"id": "arka-
|
|
3618
|
-
"command": "/arka
|
|
3617
|
+
"id": "arka-mcps-[period]",
|
|
3618
|
+
"command": "/arka mcps [period]",
|
|
3619
3619
|
"department": "arka",
|
|
3620
|
-
"description": "
|
|
3620
|
+
"description": "MCP usage — aggregates the PostToolUse MCP telemetry (`~/.arkaos/telemetry/mcp-usage.jsonl`) by day/week/month/all. Shows total calls, servers in use, top servers and top tools. Shells out to `~/.arkaos/bin/arka-py -m core.runtime.mcp_telemetry_cli <period>`.",
|
|
3621
3621
|
"keywords": [],
|
|
3622
3622
|
"tier": 3,
|
|
3623
3623
|
"modifies_code": false,
|
|
3624
3624
|
"requires_branch": false
|
|
3625
3625
|
},
|
|
3626
3626
|
{
|
|
3627
|
-
"id": "arka-
|
|
3628
|
-
"command": "/arka
|
|
3627
|
+
"id": "arka-compliance-[period]",
|
|
3628
|
+
"command": "/arka compliance [period]",
|
|
3629
3629
|
"department": "arka",
|
|
3630
|
-
"description": "
|
|
3630
|
+
"description": "Behavior compliance summary (PR29 v2.48.0) — aggregates stop-hook telemetry by day/week/month/all. Shows rates for the four contracts: closing marker, `[arka:meta]` tag, KB citation pass, sycophancy clean. Shells out to `~/.arkaos/bin/arka-py -m core.governance.compliance_telemetry_cli <period>`.",
|
|
3631
3631
|
"keywords": [],
|
|
3632
3632
|
"tier": 3,
|
|
3633
3633
|
"modifies_code": false,
|
|
3634
3634
|
"requires_branch": false
|
|
3635
3635
|
},
|
|
3636
3636
|
{
|
|
3637
|
-
"id": "arka-
|
|
3638
|
-
"command": "/arka
|
|
3637
|
+
"id": "arka-reorganize-[--since-days-N]",
|
|
3638
|
+
"command": "/arka reorganize [--since-days N]",
|
|
3639
3639
|
"department": "arka",
|
|
3640
|
-
"description": "
|
|
3640
|
+
"description": "Dreaming → Agent reorganizer. Aggregates recent KB pattern/anti-pattern/lesson artifacts (default last 7 days) into a markdown proposal at `~/.arkaos/reorganize-proposals/<date>.md`. **Propose-only** — never modifies agent YAMLs. Sanitizes client identifiers from titles and body excerpts; drops `tags:` field entirely to prevent project-name leaks. **Auto-fires on session start when today's proposal is missing** (PR24 v2.46.0 stale-aware trigger, 30s timeout, background). Shells out to `~/.arkaos/bin/arka-py -m core.cognition.reorganizer_cli [--since-days N] [--dry-run]`.",
|
|
3641
3641
|
"keywords": [],
|
|
3642
3642
|
"tier": 3,
|
|
3643
3643
|
"modifies_code": false,
|
|
3644
3644
|
"requires_branch": false
|
|
3645
3645
|
},
|
|
3646
3646
|
{
|
|
3647
|
-
"id": "arka-
|
|
3648
|
-
"command": "/arka
|
|
3649
|
-
"department": "arka",
|
|
3650
|
-
"description": "Activate personal AI advisory board (The Conclave)",
|
|
3651
|
-
"keywords": [],
|
|
3652
|
-
"tier": 3,
|
|
3653
|
-
"modifies_code": false,
|
|
3654
|
-
"requires_branch": false
|
|
3655
|
-
},
|
|
3656
|
-
{
|
|
3657
|
-
"id": "do",
|
|
3658
|
-
"command": "/do <description>",
|
|
3659
|
-
"department": "arka",
|
|
3660
|
-
"description": "Universal routing — natural language to department command",
|
|
3661
|
-
"keywords": [],
|
|
3662
|
-
"tier": 3,
|
|
3663
|
-
"modifies_code": false,
|
|
3664
|
-
"requires_branch": false
|
|
3665
|
-
},
|
|
3666
|
-
{
|
|
3667
|
-
"id": "dev",
|
|
3668
|
-
"command": "/dev",
|
|
3669
|
-
"department": "arka",
|
|
3670
|
-
"description": "Development",
|
|
3671
|
-
"keywords": [],
|
|
3672
|
-
"tier": 3,
|
|
3673
|
-
"modifies_code": false,
|
|
3674
|
-
"requires_branch": false
|
|
3675
|
-
},
|
|
3676
|
-
{
|
|
3677
|
-
"id": "mkt",
|
|
3678
|
-
"command": "/mkt",
|
|
3679
|
-
"department": "arka",
|
|
3680
|
-
"description": "Marketing & Growth",
|
|
3681
|
-
"keywords": [],
|
|
3682
|
-
"tier": 3,
|
|
3683
|
-
"modifies_code": false,
|
|
3684
|
-
"requires_branch": false
|
|
3685
|
-
},
|
|
3686
|
-
{
|
|
3687
|
-
"id": "brand",
|
|
3688
|
-
"command": "/brand",
|
|
3689
|
-
"department": "arka",
|
|
3690
|
-
"description": "Brand & Design",
|
|
3691
|
-
"keywords": [],
|
|
3692
|
-
"tier": 3,
|
|
3693
|
-
"modifies_code": false,
|
|
3694
|
-
"requires_branch": false
|
|
3695
|
-
},
|
|
3696
|
-
{
|
|
3697
|
-
"id": "fin",
|
|
3698
|
-
"command": "/fin",
|
|
3647
|
+
"id": "arka-standup",
|
|
3648
|
+
"command": "/arka standup",
|
|
3699
3649
|
"department": "arka",
|
|
3700
|
-
"description": "
|
|
3650
|
+
"description": "Daily standup (projects, priorities, blockers, updates)",
|
|
3701
3651
|
"keywords": [],
|
|
3702
3652
|
"tier": 3,
|
|
3703
3653
|
"modifies_code": false,
|
|
3704
3654
|
"requires_branch": false
|
|
3705
3655
|
},
|
|
3706
3656
|
{
|
|
3707
|
-
"id": "
|
|
3708
|
-
"command": "/
|
|
3657
|
+
"id": "arka-monitor",
|
|
3658
|
+
"command": "/arka monitor",
|
|
3709
3659
|
"department": "arka",
|
|
3710
|
-
"description": "
|
|
3660
|
+
"description": "System health monitoring",
|
|
3711
3661
|
"keywords": [],
|
|
3712
3662
|
"tier": 3,
|
|
3713
3663
|
"modifies_code": false,
|
|
3714
3664
|
"requires_branch": false
|
|
3715
3665
|
},
|
|
3716
3666
|
{
|
|
3717
|
-
"id": "
|
|
3718
|
-
"command": "/
|
|
3667
|
+
"id": "arka-onboard",
|
|
3668
|
+
"command": "/arka onboard <path>",
|
|
3719
3669
|
"department": "arka",
|
|
3720
|
-
"description": "
|
|
3670
|
+
"description": "Onboard an existing project into ArkaOS",
|
|
3721
3671
|
"keywords": [],
|
|
3722
3672
|
"tier": 3,
|
|
3723
3673
|
"modifies_code": false,
|
|
3724
3674
|
"requires_branch": false
|
|
3725
3675
|
},
|
|
3726
3676
|
{
|
|
3727
|
-
"id": "
|
|
3728
|
-
"command": "/
|
|
3677
|
+
"id": "arka-help",
|
|
3678
|
+
"command": "/arka help",
|
|
3729
3679
|
"department": "arka",
|
|
3730
|
-
"description": "
|
|
3680
|
+
"description": "List all department commands",
|
|
3731
3681
|
"keywords": [],
|
|
3732
3682
|
"tier": 3,
|
|
3733
3683
|
"modifies_code": false,
|
|
3734
3684
|
"requires_branch": false
|
|
3735
3685
|
},
|
|
3736
3686
|
{
|
|
3737
|
-
"id": "
|
|
3738
|
-
"command": "/
|
|
3687
|
+
"id": "arka-setup",
|
|
3688
|
+
"command": "/arka setup",
|
|
3739
3689
|
"department": "arka",
|
|
3740
|
-
"description": "
|
|
3690
|
+
"description": "Interactive profile setup (name, company, role, objectives)",
|
|
3741
3691
|
"keywords": [],
|
|
3742
3692
|
"tier": 3,
|
|
3743
3693
|
"modifies_code": false,
|
|
3744
3694
|
"requires_branch": false
|
|
3745
3695
|
},
|
|
3746
3696
|
{
|
|
3747
|
-
"id": "
|
|
3748
|
-
"command": "/
|
|
3697
|
+
"id": "arka-conclave",
|
|
3698
|
+
"command": "/arka conclave",
|
|
3749
3699
|
"department": "arka",
|
|
3750
|
-
"description": "
|
|
3700
|
+
"description": "Activate personal AI advisory board (The Conclave)",
|
|
3751
3701
|
"keywords": [],
|
|
3752
3702
|
"tier": 3,
|
|
3753
3703
|
"modifies_code": false,
|
|
3754
3704
|
"requires_branch": false
|
|
3755
3705
|
},
|
|
3756
3706
|
{
|
|
3757
|
-
"id": "
|
|
3758
|
-
"command": "/
|
|
3707
|
+
"id": "arka-dashboard",
|
|
3708
|
+
"command": "/arka dashboard",
|
|
3759
3709
|
"department": "arka",
|
|
3760
|
-
"description": "
|
|
3710
|
+
"description": "Open monitoring dashboard (localhost:3333)",
|
|
3761
3711
|
"keywords": [],
|
|
3762
3712
|
"tier": 3,
|
|
3763
3713
|
"modifies_code": false,
|
|
3764
3714
|
"requires_branch": false
|
|
3765
3715
|
},
|
|
3766
3716
|
{
|
|
3767
|
-
"id": "
|
|
3768
|
-
"command": "/
|
|
3717
|
+
"id": "arka-index",
|
|
3718
|
+
"command": "/arka index",
|
|
3769
3719
|
"department": "arka",
|
|
3770
|
-
"description": "
|
|
3720
|
+
"description": "Index Obsidian vault into knowledge base",
|
|
3771
3721
|
"keywords": [],
|
|
3772
3722
|
"tier": 3,
|
|
3773
3723
|
"modifies_code": false,
|
|
3774
3724
|
"requires_branch": false
|
|
3775
3725
|
},
|
|
3776
3726
|
{
|
|
3777
|
-
"id": "
|
|
3778
|
-
"command": "/
|
|
3727
|
+
"id": "arka-search",
|
|
3728
|
+
"command": "/arka search <query>",
|
|
3779
3729
|
"department": "arka",
|
|
3780
|
-
"description": "
|
|
3730
|
+
"description": "Semantic search in knowledge base",
|
|
3781
3731
|
"keywords": [],
|
|
3782
3732
|
"tier": 3,
|
|
3783
3733
|
"modifies_code": false,
|
|
3784
3734
|
"requires_branch": false
|
|
3785
3735
|
},
|
|
3786
3736
|
{
|
|
3787
|
-
"id": "
|
|
3788
|
-
"command": "/
|
|
3737
|
+
"id": "arka-keys",
|
|
3738
|
+
"command": "/arka keys",
|
|
3789
3739
|
"department": "arka",
|
|
3790
|
-
"description": "
|
|
3740
|
+
"description": "Manage API keys (OpenAI, Google, fal.ai)",
|
|
3791
3741
|
"keywords": [],
|
|
3792
3742
|
"tier": 3,
|
|
3793
3743
|
"modifies_code": false,
|
|
3794
3744
|
"requires_branch": false
|
|
3795
3745
|
},
|
|
3796
3746
|
{
|
|
3797
|
-
"id": "
|
|
3798
|
-
"command": "/
|
|
3747
|
+
"id": "arka-personas",
|
|
3748
|
+
"command": "/arka personas",
|
|
3799
3749
|
"department": "arka",
|
|
3800
|
-
"description": "
|
|
3750
|
+
"description": "Manage AI personas (create, clone to agent)",
|
|
3801
3751
|
"keywords": [],
|
|
3802
3752
|
"tier": 3,
|
|
3803
3753
|
"modifies_code": false,
|
|
3804
3754
|
"requires_branch": false
|
|
3805
3755
|
},
|
|
3806
3756
|
{
|
|
3807
|
-
"id": "
|
|
3808
|
-
"command": "/
|
|
3757
|
+
"id": "arka-resume",
|
|
3758
|
+
"command": "/arka resume <PR_URL>",
|
|
3809
3759
|
"department": "arka",
|
|
3810
|
-
"description": "
|
|
3760
|
+
"description": "Re-enter the Claude Code session that produced a PR (GitHub / GitLab / Bitbucket). Wraps the native `/resume` from Claude Code 2.1.122+. Useful with `arka-dev-spec` and `arka-release` archaeology.",
|
|
3811
3761
|
"keywords": [],
|
|
3812
3762
|
"tier": 3,
|
|
3813
3763
|
"modifies_code": false,
|
|
3814
3764
|
"requires_branch": false
|
|
3815
3765
|
},
|
|
3816
3766
|
{
|
|
3817
|
-
"id": "
|
|
3818
|
-
"command": "/
|
|
3767
|
+
"id": "do",
|
|
3768
|
+
"command": "/do <description>",
|
|
3819
3769
|
"department": "arka",
|
|
3820
|
-
"description": "
|
|
3770
|
+
"description": "Universal routing — natural language to department command",
|
|
3821
3771
|
"keywords": [],
|
|
3822
3772
|
"tier": 3,
|
|
3823
3773
|
"modifies_code": false,
|
package/package.json
CHANGED
package/pyproject.toml
CHANGED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""Prompt-surface token benchmark (v4.1 Evidence Reform open item).
|
|
2
|
+
|
|
3
|
+
Measures the UserPromptSubmit hook's injected context (additionalContext)
|
|
4
|
+
across a canonical prompt set, optionally comparing the working tree
|
|
5
|
+
against another git ref (extracted via `git archive`, no worktrees).
|
|
6
|
+
|
|
7
|
+
Token estimate: bytes / 4 — the same coarse heuristic the Synapse layers
|
|
8
|
+
use (`tokens_est`); good enough for a reduction ratio, not billing.
|
|
9
|
+
|
|
10
|
+
Usage:
|
|
11
|
+
arka-py scripts/tools/prompt_surface_benchmark.py [--ref v4.0.2] [--json]
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import argparse
|
|
17
|
+
import json
|
|
18
|
+
import subprocess
|
|
19
|
+
import sys
|
|
20
|
+
import tarfile
|
|
21
|
+
import tempfile
|
|
22
|
+
from io import BytesIO
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
REPO_ROOT = Path(__file__).resolve().parent.parent.parent
|
|
26
|
+
|
|
27
|
+
CANONICAL_PROMPTS: dict[str, str] = {
|
|
28
|
+
"simple": "ok",
|
|
29
|
+
"question": "como funciona o sync de projetos?",
|
|
30
|
+
"code-modifying": "implementa auth no backend com testes",
|
|
31
|
+
"department-routed": "cria uma campanha de marketing para o lançamento",
|
|
32
|
+
"slash-command": "/dev feature user auth",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
HOOK_REL = Path("config") / "hooks" / "user-prompt-submit.sh"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _run_hook(tree: Path, prompt: str) -> int:
|
|
39
|
+
"""Return the additionalContext byte size the hook injects."""
|
|
40
|
+
payload = json.dumps(
|
|
41
|
+
{"prompt": prompt, "cwd": "/tmp", "session_id": "bench-1"}
|
|
42
|
+
)
|
|
43
|
+
proc = subprocess.run(
|
|
44
|
+
["bash", str(tree / HOOK_REL)],
|
|
45
|
+
input=payload,
|
|
46
|
+
capture_output=True,
|
|
47
|
+
text=True,
|
|
48
|
+
timeout=30,
|
|
49
|
+
env={
|
|
50
|
+
"PATH": "/usr/bin:/bin:/usr/local/bin",
|
|
51
|
+
"HOME": str(Path.home()),
|
|
52
|
+
"ARKA_HOOK_FORCE_FALLBACK": "1",
|
|
53
|
+
},
|
|
54
|
+
)
|
|
55
|
+
out = proc.stdout.strip()
|
|
56
|
+
if not out:
|
|
57
|
+
return 0
|
|
58
|
+
try:
|
|
59
|
+
context = json.loads(out).get("additionalContext", "")
|
|
60
|
+
except json.JSONDecodeError:
|
|
61
|
+
context = out
|
|
62
|
+
return len(context.encode("utf-8"))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _extract_ref(ref: str, dest: Path) -> Path:
|
|
66
|
+
archive = subprocess.run(
|
|
67
|
+
["git", "-C", str(REPO_ROOT), "archive", ref],
|
|
68
|
+
capture_output=True,
|
|
69
|
+
check=True,
|
|
70
|
+
timeout=60,
|
|
71
|
+
)
|
|
72
|
+
with tarfile.open(fileobj=BytesIO(archive.stdout)) as tar:
|
|
73
|
+
try:
|
|
74
|
+
tar.extractall(dest, filter="data")
|
|
75
|
+
except TypeError: # Python < 3.12 has no filter param
|
|
76
|
+
tar.extractall(dest) # trusted archive: local git, no network
|
|
77
|
+
return dest
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def measure(tree: Path) -> dict[str, dict[str, int]]:
|
|
81
|
+
results: dict[str, dict[str, int]] = {}
|
|
82
|
+
for name, prompt in CANONICAL_PROMPTS.items():
|
|
83
|
+
size = _run_hook(tree, prompt)
|
|
84
|
+
results[name] = {"bytes": size, "tokens_est": size // 4}
|
|
85
|
+
return results
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def compare(ref: str) -> dict:
|
|
89
|
+
with tempfile.TemporaryDirectory(prefix="arka-bench-") as tmp:
|
|
90
|
+
old_tree = _extract_ref(ref, Path(tmp))
|
|
91
|
+
before = measure(old_tree)
|
|
92
|
+
after = measure(REPO_ROOT)
|
|
93
|
+
total_before = sum(r["bytes"] for r in before.values())
|
|
94
|
+
total_after = sum(r["bytes"] for r in after.values())
|
|
95
|
+
reduction = (
|
|
96
|
+
round(100 * (1 - total_after / total_before), 1)
|
|
97
|
+
if total_before
|
|
98
|
+
else 0.0
|
|
99
|
+
)
|
|
100
|
+
return {
|
|
101
|
+
"ref": ref,
|
|
102
|
+
"before": before,
|
|
103
|
+
"after": after,
|
|
104
|
+
"total_bytes_before": total_before,
|
|
105
|
+
"total_bytes_after": total_after,
|
|
106
|
+
"reduction_pct": reduction,
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def main(argv: list[str] | None = None) -> int:
|
|
111
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
112
|
+
parser.add_argument("--ref", help="git ref to compare against")
|
|
113
|
+
parser.add_argument("--json", action="store_true", dest="as_json")
|
|
114
|
+
args = parser.parse_args(argv)
|
|
115
|
+
|
|
116
|
+
if args.ref:
|
|
117
|
+
report = compare(args.ref)
|
|
118
|
+
else:
|
|
119
|
+
report = {"current": measure(REPO_ROOT)}
|
|
120
|
+
|
|
121
|
+
if args.as_json:
|
|
122
|
+
print(json.dumps(report, indent=2, ensure_ascii=False))
|
|
123
|
+
return 0
|
|
124
|
+
|
|
125
|
+
if "current" in report:
|
|
126
|
+
for name, row in report["current"].items():
|
|
127
|
+
print(f"{name:20s} {row['bytes']:>6d} B ~{row['tokens_est']} tok")
|
|
128
|
+
return 0
|
|
129
|
+
|
|
130
|
+
print(f"Prompt-surface benchmark: {report['ref']} -> HEAD")
|
|
131
|
+
for name in CANONICAL_PROMPTS:
|
|
132
|
+
b = report["before"][name]["bytes"]
|
|
133
|
+
a = report["after"][name]["bytes"]
|
|
134
|
+
print(f"{name:20s} {b:>6d} B -> {a:>6d} B")
|
|
135
|
+
print(
|
|
136
|
+
f"{'TOTAL':20s} {report['total_bytes_before']:>6d} B -> "
|
|
137
|
+
f"{report['total_bytes_after']:>6d} B "
|
|
138
|
+
f"({report['reduction_pct']}% reduction)"
|
|
139
|
+
)
|
|
140
|
+
return 0
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
if __name__ == "__main__":
|
|
144
|
+
raise SystemExit(main())
|
|
@@ -1,254 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": "1.0.0",
|
|
3
|
-
"generated": "2026-03-20",
|
|
4
|
-
"agents": [
|
|
5
|
-
{
|
|
6
|
-
"id": "cto",
|
|
7
|
-
"display_name": "Marco",
|
|
8
|
-
"role": "CTO",
|
|
9
|
-
"department": "dev",
|
|
10
|
-
"tier": 0,
|
|
11
|
-
"disc": { "primary": "D", "secondary": "C", "combination": "D+C", "label": "Driver-Analyst" },
|
|
12
|
-
"file": "departments/dev/agents/cto.md",
|
|
13
|
-
"memory_path": "~/.claude/agent-memory/arka-cto/MEMORY.md",
|
|
14
|
-
"key_authority": ["veto", "approve_architecture", "approve_tech_stack", "block_release"]
|
|
15
|
-
},
|
|
16
|
-
{
|
|
17
|
-
"id": "tech-lead",
|
|
18
|
-
"display_name": "Paulo",
|
|
19
|
-
"role": "Tech Lead",
|
|
20
|
-
"department": "dev",
|
|
21
|
-
"tier": 1,
|
|
22
|
-
"disc": { "primary": "I", "secondary": "S", "combination": "I+S", "label": "Inspirer-Supporter" },
|
|
23
|
-
"file": "departments/dev/agents/tech-lead.md",
|
|
24
|
-
"memory_path": "~/.claude/agent-memory/arka-tech-lead/MEMORY.md",
|
|
25
|
-
"key_authority": ["orchestrate", "assign_tasks"]
|
|
26
|
-
},
|
|
27
|
-
{
|
|
28
|
-
"id": "architect",
|
|
29
|
-
"display_name": "Gabriel",
|
|
30
|
-
"role": "Software Architect",
|
|
31
|
-
"department": "dev",
|
|
32
|
-
"tier": 1,
|
|
33
|
-
"disc": { "primary": "C", "secondary": "D", "combination": "C+D", "label": "Analyst-Driver" },
|
|
34
|
-
"file": "departments/dev/agents/architect.md",
|
|
35
|
-
"memory_path": "~/.claude/agent-memory/arka-architect/MEMORY.md",
|
|
36
|
-
"key_authority": ["design_architecture", "create_adr"]
|
|
37
|
-
},
|
|
38
|
-
{
|
|
39
|
-
"id": "senior-dev",
|
|
40
|
-
"display_name": "Andre",
|
|
41
|
-
"role": "Senior Backend Developer",
|
|
42
|
-
"department": "dev",
|
|
43
|
-
"tier": 2,
|
|
44
|
-
"disc": { "primary": "C", "secondary": "S", "combination": "C+S", "label": "Analyst-Supporter" },
|
|
45
|
-
"file": "departments/dev/agents/senior-dev.md",
|
|
46
|
-
"memory_path": "~/.claude/agent-memory/arka-senior-dev/MEMORY.md",
|
|
47
|
-
"key_authority": ["implement"]
|
|
48
|
-
},
|
|
49
|
-
{
|
|
50
|
-
"id": "frontend-dev",
|
|
51
|
-
"display_name": "Diana",
|
|
52
|
-
"role": "Senior Frontend Developer",
|
|
53
|
-
"department": "dev",
|
|
54
|
-
"tier": 2,
|
|
55
|
-
"disc": { "primary": "I", "secondary": "C", "combination": "I+C", "label": "Inspirer-Analyst" },
|
|
56
|
-
"file": "departments/dev/agents/frontend-dev.md",
|
|
57
|
-
"memory_path": "~/.claude/agent-memory/arka-frontend-dev/MEMORY.md",
|
|
58
|
-
"key_authority": ["implement"]
|
|
59
|
-
},
|
|
60
|
-
{
|
|
61
|
-
"id": "security",
|
|
62
|
-
"display_name": "Bruno",
|
|
63
|
-
"role": "Security Engineer",
|
|
64
|
-
"department": "dev",
|
|
65
|
-
"tier": 2,
|
|
66
|
-
"disc": { "primary": "C", "secondary": "D", "combination": "C+D", "label": "Analyst-Driver" },
|
|
67
|
-
"file": "departments/dev/agents/security.md",
|
|
68
|
-
"memory_path": "~/.claude/agent-memory/arka-security/MEMORY.md",
|
|
69
|
-
"key_authority": ["block_release", "security_audit"]
|
|
70
|
-
},
|
|
71
|
-
{
|
|
72
|
-
"id": "devops",
|
|
73
|
-
"display_name": "Carlos",
|
|
74
|
-
"role": "DevOps Lead",
|
|
75
|
-
"department": "dev",
|
|
76
|
-
"tier": 2,
|
|
77
|
-
"disc": { "primary": "D", "secondary": "C", "combination": "D+C", "label": "Driver-Analyst" },
|
|
78
|
-
"file": "departments/dev/agents/devops.md",
|
|
79
|
-
"memory_path": "~/.claude/agent-memory/arka-devops/MEMORY.md",
|
|
80
|
-
"key_authority": ["push", "deploy", "manage_ci"]
|
|
81
|
-
},
|
|
82
|
-
{
|
|
83
|
-
"id": "qa",
|
|
84
|
-
"display_name": "Rita",
|
|
85
|
-
"role": "QA Lead",
|
|
86
|
-
"department": "dev",
|
|
87
|
-
"tier": 3,
|
|
88
|
-
"disc": { "primary": "C", "secondary": "S", "combination": "C+S", "label": "Analyst-Supporter" },
|
|
89
|
-
"file": "departments/dev/agents/qa.md",
|
|
90
|
-
"memory_path": "~/.claude/agent-memory/arka-qa/MEMORY.md",
|
|
91
|
-
"key_authority": ["block_release", "validate"]
|
|
92
|
-
},
|
|
93
|
-
{
|
|
94
|
-
"id": "analyst",
|
|
95
|
-
"display_name": "Lucas",
|
|
96
|
-
"role": "Technical Analyst",
|
|
97
|
-
"department": "dev",
|
|
98
|
-
"tier": 3,
|
|
99
|
-
"disc": { "primary": "C", "secondary": "I", "combination": "C+I", "label": "Analyst-Inspirer" },
|
|
100
|
-
"file": "departments/dev/agents/analyst.md",
|
|
101
|
-
"memory_path": "~/.claude/agent-memory/arka-analyst/MEMORY.md",
|
|
102
|
-
"key_authority": ["research", "recommend"]
|
|
103
|
-
},
|
|
104
|
-
{
|
|
105
|
-
"id": "cfo",
|
|
106
|
-
"display_name": "Helena",
|
|
107
|
-
"role": "CFO",
|
|
108
|
-
"department": "finance",
|
|
109
|
-
"tier": 0,
|
|
110
|
-
"disc": { "primary": "D", "secondary": "C", "combination": "D+C", "label": "Driver-Analyst" },
|
|
111
|
-
"file": "departments/finance/agents/cfo.md",
|
|
112
|
-
"memory_path": "~/.claude/agent-memory/arka-cfo/MEMORY.md",
|
|
113
|
-
"key_authority": ["veto", "approve_budget", "financial_decisions"]
|
|
114
|
-
},
|
|
115
|
-
{
|
|
116
|
-
"id": "coo",
|
|
117
|
-
"display_name": "Sofia",
|
|
118
|
-
"role": "COO",
|
|
119
|
-
"department": "ops",
|
|
120
|
-
"tier": 0,
|
|
121
|
-
"disc": { "primary": "S", "secondary": "C", "combination": "S+C", "label": "Supporter-Analyst" },
|
|
122
|
-
"file": "departments/ops/agents/ops-lead.yaml",
|
|
123
|
-
"memory_path": "~/.claude/agent-memory/arka-coo/MEMORY.md",
|
|
124
|
-
"key_authority": ["veto", "approve_operations", "cross_department"]
|
|
125
|
-
},
|
|
126
|
-
{
|
|
127
|
-
"id": "content-creator",
|
|
128
|
-
"display_name": "Luna",
|
|
129
|
-
"role": "Content Creator",
|
|
130
|
-
"department": "marketing",
|
|
131
|
-
"tier": 1,
|
|
132
|
-
"disc": { "primary": "I", "secondary": "D", "combination": "I+D", "label": "Inspirer-Driver" },
|
|
133
|
-
"file": "departments/marketing/agents/content-creator.md",
|
|
134
|
-
"memory_path": "~/.claude/agent-memory/arka-content-creator/MEMORY.md",
|
|
135
|
-
"key_authority": ["create_content", "manage_campaigns"]
|
|
136
|
-
},
|
|
137
|
-
{
|
|
138
|
-
"id": "ecommerce-manager",
|
|
139
|
-
"display_name": "Ricardo",
|
|
140
|
-
"role": "E-commerce Manager",
|
|
141
|
-
"department": "ecom",
|
|
142
|
-
"tier": 1,
|
|
143
|
-
"disc": { "primary": "D", "secondary": "I", "combination": "D+I", "label": "Driver-Inspirer" },
|
|
144
|
-
"file": "departments/ecom/agents/ecom-director.yaml",
|
|
145
|
-
"memory_path": "~/.claude/agent-memory/arka-ecommerce-manager/MEMORY.md",
|
|
146
|
-
"key_authority": ["manage_store", "pricing_decisions"]
|
|
147
|
-
},
|
|
148
|
-
{
|
|
149
|
-
"id": "strategist",
|
|
150
|
-
"display_name": "Tomas",
|
|
151
|
-
"role": "Chief Strategist",
|
|
152
|
-
"department": "strategy",
|
|
153
|
-
"tier": 1,
|
|
154
|
-
"disc": { "primary": "I", "secondary": "D", "combination": "I+D", "label": "Inspirer-Driver" },
|
|
155
|
-
"file": "departments/strategy/agents/strategist.md",
|
|
156
|
-
"memory_path": "~/.claude/agent-memory/arka-strategist/MEMORY.md",
|
|
157
|
-
"key_authority": ["strategic_planning", "market_analysis"]
|
|
158
|
-
},
|
|
159
|
-
{
|
|
160
|
-
"id": "knowledge-curator",
|
|
161
|
-
"display_name": "Clara",
|
|
162
|
-
"role": "Knowledge Curator",
|
|
163
|
-
"department": "kb",
|
|
164
|
-
"tier": 1,
|
|
165
|
-
"disc": { "primary": "S", "secondary": "C", "combination": "S+C", "label": "Supporter-Analyst" },
|
|
166
|
-
"file": "departments/kb/agents/knowledge-curator.yaml",
|
|
167
|
-
"memory_path": "~/.claude/agent-memory/arka-knowledge-curator/MEMORY.md",
|
|
168
|
-
"key_authority": ["manage_knowledge", "create_personas"]
|
|
169
|
-
},
|
|
170
|
-
{
|
|
171
|
-
"id": "creative-director",
|
|
172
|
-
"display_name": "Valentina",
|
|
173
|
-
"role": "Creative Director",
|
|
174
|
-
"department": "brand",
|
|
175
|
-
"tier": 1,
|
|
176
|
-
"disc": { "primary": "S", "secondary": "I", "combination": "S+I", "label": "Supporter-Inspirer" },
|
|
177
|
-
"file": "departments/brand/agents/creative-director.md",
|
|
178
|
-
"memory_path": "~/.claude/agent-memory/arka-creative-director/MEMORY.md",
|
|
179
|
-
"key_authority": ["approve_brand", "create_brand", "manage_visuals", "orchestrate"]
|
|
180
|
-
},
|
|
181
|
-
{
|
|
182
|
-
"id": "brand-strategist",
|
|
183
|
-
"display_name": "Mateus",
|
|
184
|
-
"role": "Brand Strategist",
|
|
185
|
-
"department": "brand",
|
|
186
|
-
"tier": 2,
|
|
187
|
-
"disc": { "primary": "C", "secondary": "I", "combination": "C+I", "label": "Analyst-Inspirer" },
|
|
188
|
-
"file": "departments/brand/agents/brand-strategist.md",
|
|
189
|
-
"memory_path": "~/.claude/agent-memory/arka-brand-strategist/MEMORY.md",
|
|
190
|
-
"key_authority": ["brand_research", "brand_positioning", "recommend"]
|
|
191
|
-
},
|
|
192
|
-
{
|
|
193
|
-
"id": "visual-designer",
|
|
194
|
-
"display_name": "Isabel",
|
|
195
|
-
"role": "Visual Designer",
|
|
196
|
-
"department": "brand",
|
|
197
|
-
"tier": 2,
|
|
198
|
-
"disc": { "primary": "I", "secondary": "S", "combination": "I+S", "label": "Inspirer-Supporter" },
|
|
199
|
-
"file": "departments/brand/agents/visual-designer.md",
|
|
200
|
-
"memory_path": "~/.claude/agent-memory/arka-visual-designer/MEMORY.md",
|
|
201
|
-
"key_authority": ["generate_visuals", "create_assets", "implement"]
|
|
202
|
-
},
|
|
203
|
-
{
|
|
204
|
-
"id": "motion-designer",
|
|
205
|
-
"display_name": "Rafael",
|
|
206
|
-
"role": "Motion Designer",
|
|
207
|
-
"department": "brand",
|
|
208
|
-
"tier": 2,
|
|
209
|
-
"disc": { "primary": "D", "secondary": "I", "combination": "D+I", "label": "Driver-Inspirer" },
|
|
210
|
-
"file": "departments/brand/agents/motion-designer.md",
|
|
211
|
-
"memory_path": "~/.claude/agent-memory/arka-motion-designer/MEMORY.md",
|
|
212
|
-
"key_authority": ["generate_video", "create_motion", "implement"]
|
|
213
|
-
},
|
|
214
|
-
{
|
|
215
|
-
"id": "cqo",
|
|
216
|
-
"display_name": "Marta",
|
|
217
|
-
"role": "Chief Quality Officer",
|
|
218
|
-
"department": "quality",
|
|
219
|
-
"tier": 0,
|
|
220
|
-
"disc": { "primary": "C", "secondary": "D", "combination": "C+D", "label": "Analyst-Driver" },
|
|
221
|
-
"file": "departments/quality/agents/cqo.md",
|
|
222
|
-
"memory_path": "~/.claude/agent-memory/arka-cqo/MEMORY.md",
|
|
223
|
-
"key_authority": ["veto", "block_delivery", "block_release", "approve_quality"]
|
|
224
|
-
},
|
|
225
|
-
{
|
|
226
|
-
"id": "copy-director",
|
|
227
|
-
"display_name": "Eduardo",
|
|
228
|
-
"role": "Copy & Language Director",
|
|
229
|
-
"department": "quality",
|
|
230
|
-
"tier": 0,
|
|
231
|
-
"disc": { "primary": "C", "secondary": "S", "combination": "C+S", "label": "Analyst-Supporter" },
|
|
232
|
-
"file": "departments/quality/agents/copy-director.md",
|
|
233
|
-
"memory_path": "~/.claude/agent-memory/arka-copy-director/MEMORY.md",
|
|
234
|
-
"key_authority": ["veto", "block_delivery"]
|
|
235
|
-
},
|
|
236
|
-
{
|
|
237
|
-
"id": "tech-ux-director",
|
|
238
|
-
"display_name": "Francisca",
|
|
239
|
-
"role": "Technical & UX Quality Director",
|
|
240
|
-
"department": "quality",
|
|
241
|
-
"tier": 0,
|
|
242
|
-
"disc": { "primary": "D", "secondary": "C", "combination": "D+C", "label": "Driver-Analyst" },
|
|
243
|
-
"file": "departments/quality/agents/tech-ux-director.md",
|
|
244
|
-
"memory_path": "~/.claude/agent-memory/arka-tech-ux-director/MEMORY.md",
|
|
245
|
-
"key_authority": ["veto", "block_delivery", "block_release"]
|
|
246
|
-
}
|
|
247
|
-
],
|
|
248
|
-
"team_composition": {
|
|
249
|
-
"by_disc_primary": { "D": 6, "I": 5, "S": 3, "C": 8 },
|
|
250
|
-
"by_tier": { "0": 6, "1": 7, "2": 7, "3": 2 },
|
|
251
|
-
"by_department": { "dev": 9, "finance": 1, "operations": 1, "marketing": 1, "ecommerce": 1, "strategy": 1, "knowledge": 1, "brand": 4, "quality": 3 },
|
|
252
|
-
"balance_notes": "22 agents across 9 departments. Quality Gate team (Marta, Eduardo, Francisca) adds 3 Tier 0 supervisors. C profile strong at 36% reflecting quality-driven culture."
|
|
253
|
-
}
|
|
254
|
-
}
|