create-ironclaws 1.0.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/template/CLAUDE.md +105 -51
- package/template/container/Dockerfile.argus +14 -12
- package/template/container/skills/slack-formatting/SKILL.md +46 -47
- package/template/container/tools/README.md +33 -0
- package/template/docs/how-it-works.md +19 -19
- package/template/groups/forge/CLAUDE.md +13 -9
- package/template/scripts/setup-onecli-secrets.sh +25 -87
- package/template/src/container-runner.ts +4 -4
- package/template/src/index.ts +26 -5
- package/template/container/tools/elastic_query.py +0 -161
- package/template/container/tools/gdrive_tool.py +0 -185
- package/template/container/tools/jira_tool.py +0 -433
- package/template/container/tools/slack_history_tool.py +0 -144
- package/template/container/tools/youtube_tool.py +0 -174
- package/template/scripts/register-expense-agent.sh +0 -121
- package/template/setup-agents.sh +0 -142
|
@@ -4,8 +4,11 @@
|
|
|
4
4
|
# Configure OneCLI secrets via the LOCAL HTTP API (handles encryption correctly).
|
|
5
5
|
# Direct DB insertion bypasses encryption and breaks the gateway — never do that.
|
|
6
6
|
#
|
|
7
|
-
# Run
|
|
8
|
-
#
|
|
7
|
+
# Run after deploying IronClaws:
|
|
8
|
+
# bash scripts/setup-onecli-secrets.sh
|
|
9
|
+
#
|
|
10
|
+
# To add a new service: copy one of the existing blocks below, update the
|
|
11
|
+
# variable names, and add a corresponding section in .env.example.
|
|
9
12
|
|
|
10
13
|
set -eo pipefail
|
|
11
14
|
|
|
@@ -78,8 +81,8 @@ link_secret_to_agent() {
|
|
|
78
81
|
local secret_name="$1"
|
|
79
82
|
local agent_name="$2"
|
|
80
83
|
|
|
81
|
-
# Match by identifier (
|
|
82
|
-
# fall back to name for
|
|
84
|
+
# Match by identifier (folder name) first,
|
|
85
|
+
# fall back to name for backward compatibility.
|
|
83
86
|
docker exec onecli-postgres-1 psql -U onecli -d onecli -c "
|
|
84
87
|
INSERT INTO agent_secrets (agent_id, secret_id, created_at, updated_at)
|
|
85
88
|
SELECT a.id, s.id, NOW(), NOW()
|
|
@@ -122,103 +125,38 @@ fi
|
|
|
122
125
|
echo "Reading credentials from $ENV_FILE"
|
|
123
126
|
echo ""
|
|
124
127
|
|
|
125
|
-
# Read
|
|
126
|
-
ELASTIC_API_KEY=$(env_val "ELASTIC_API_KEY")
|
|
127
|
-
ELASTIC_BASE_URL=$(env_val "ELASTIC_BASE_URL")
|
|
128
|
-
ELASTIC_HOST=$(echo "$ELASTIC_BASE_URL" | sed 's|^https://||;s|^http://||' | cut -d/ -f1)
|
|
129
|
-
|
|
130
|
-
JIRA_EMAIL=$(env_val "JIRA_EMAIL")
|
|
131
|
-
JIRA_API_TOKEN=$(env_val "JIRA_API_TOKEN")
|
|
132
|
-
JIRA_BASE_URL=$(env_val "JIRA_BASE_URL")
|
|
133
|
-
JIRA_HOST=$(echo "$JIRA_BASE_URL" | sed 's|^https://||;s|^http://||' | cut -d/ -f1)
|
|
134
|
-
JIRA_BASIC=$(printf '%s:%s' "$JIRA_EMAIL" "$JIRA_API_TOKEN" | base64 | tr -d '\n')
|
|
135
|
-
|
|
136
|
-
CONFLUENCE_USERNAME=$(env_val "CONFLUENCE_USERNAME")
|
|
137
|
-
CONFLUENCE_PASSWORD=$(env_val "CONFLUENCE_PASSWORD")
|
|
138
|
-
CONFLUENCE_BASE_URL=$(env_val "CONFLUENCE_BASE_URL")
|
|
139
|
-
CONFLUENCE_HOST=$(echo "$CONFLUENCE_BASE_URL" | sed 's|^https://||;s|^http://||' | cut -d/ -f1)
|
|
140
|
-
CONFLUENCE_BASIC=$(printf '%s:%s' "$CONFLUENCE_USERNAME" "$CONFLUENCE_PASSWORD" | base64 | tr -d '\n')
|
|
141
|
-
|
|
142
|
-
SLACK_BOT_TOKEN=$(env_val "SLACK_BOT_TOKEN")
|
|
143
|
-
INTERCOM_ACCESS_TOKEN=$(env_val "INTERCOM_ACCESS_TOKEN")
|
|
144
|
-
|
|
145
|
-
XLEDGER_API_TOKEN=$(env_val "XLEDGER_API_TOKEN")
|
|
146
|
-
XLEDGER_API_BASE=$(env_val "XLEDGER_API_BASE")
|
|
147
|
-
XLEDGER_HOST=$(echo "$XLEDGER_API_BASE" | sed 's|^https://||;s|^http://||' | cut -d/ -f1)
|
|
148
|
-
|
|
149
|
-
ARDOQ_API_KEY=$(env_val "ARDOQ_API_KEY")
|
|
150
|
-
ARDOQ_BASE_URL=$(env_val "ARDOQ_BASE_URL")
|
|
151
|
-
ARDOQ_HOST=$(echo "$ARDOQ_BASE_URL" | sed 's|^https://||;s|^http://||' | cut -d/ -f1)
|
|
152
|
-
|
|
128
|
+
# Read credentials from .env
|
|
153
129
|
ANTHROPIC_AUTH_TOKEN=$(env_val "ANTHROPIC_AUTH_TOKEN")
|
|
154
130
|
LLM_GATEWAY=$(env_val "ANTHROPIC_BASE_URL")
|
|
155
131
|
LLM_GATEWAY_HOST=$(echo "$LLM_GATEWAY" | sed 's|^https://||;s|^http://||' | cut -d/ -f1)
|
|
156
132
|
|
|
133
|
+
SLACK_BOT_TOKEN=$(env_val "SLACK_BOT_TOKEN")
|
|
134
|
+
|
|
157
135
|
# ── Register secrets via API (with proper encryption) ─────────────────────────
|
|
158
136
|
|
|
159
137
|
echo "Registering secrets..."
|
|
160
138
|
echo ""
|
|
161
139
|
|
|
162
|
-
|
|
163
|
-
upsert_secret "elastic" "$ELASTIC_HOST" "Authorization" "ApiKey $ELASTIC_API_KEY" || \
|
|
164
|
-
echo " ⚠ Skipping elastic"
|
|
165
|
-
|
|
166
|
-
[ -n "$JIRA_BASIC" ] && [ -n "$JIRA_HOST" ] && \
|
|
167
|
-
upsert_secret "jira" "$JIRA_HOST" "Authorization" "Basic $JIRA_BASIC" || \
|
|
168
|
-
echo " ⚠ Skipping jira"
|
|
169
|
-
|
|
170
|
-
# Confluence: only if host differs from Jira (usually same Atlassian instance)
|
|
171
|
-
if [ -n "$CONFLUENCE_BASIC" ] && [ -n "$CONFLUENCE_HOST" ] && [ "$CONFLUENCE_HOST" != "$JIRA_HOST" ]; then
|
|
172
|
-
upsert_secret "confluence" "$CONFLUENCE_HOST" "Authorization" "Basic $CONFLUENCE_BASIC"
|
|
173
|
-
else
|
|
174
|
-
echo " ✓ Confluence shares host with Jira — using jira secret for both"
|
|
175
|
-
fi
|
|
176
|
-
|
|
177
|
-
[ -n "$SLACK_BOT_TOKEN" ] && \
|
|
178
|
-
upsert_secret "slack" "slack.com" "Authorization" "Bearer $SLACK_BOT_TOKEN" || \
|
|
179
|
-
echo " ⚠ Skipping slack"
|
|
180
|
-
|
|
181
|
-
[ -n "$INTERCOM_ACCESS_TOKEN" ] && \
|
|
182
|
-
upsert_secret "intercom" "api.intercom.io" "Authorization" "Bearer $INTERCOM_ACCESS_TOKEN" || \
|
|
183
|
-
echo " ⚠ Skipping intercom"
|
|
184
|
-
|
|
185
|
-
[ -n "$ARDOQ_API_KEY" ] && [ -n "$ARDOQ_HOST" ] && \
|
|
186
|
-
upsert_secret "ardoq" "$ARDOQ_HOST" "Authorization" "Token token=$ARDOQ_API_KEY" || \
|
|
187
|
-
echo " ⚠ Skipping ardoq"
|
|
188
|
-
|
|
140
|
+
# LiteLLM gateway — required. Routes all Claude API calls through your proxy.
|
|
189
141
|
[ -n "$ANTHROPIC_AUTH_TOKEN" ] && [ -n "$LLM_GATEWAY_HOST" ] && \
|
|
190
142
|
upsert_secret "litellm" "$LLM_GATEWAY_HOST" "Authorization" "Bearer $ANTHROPIC_AUTH_TOKEN" || \
|
|
191
|
-
echo " ⚠ Skipping litellm"
|
|
143
|
+
echo " ⚠ Skipping litellm (ANTHROPIC_AUTH_TOKEN or ANTHROPIC_BASE_URL not set)"
|
|
192
144
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
145
|
+
# Slack — optional. Only needed for agents that call the Slack API directly
|
|
146
|
+
# (e.g. reading channel history). The host handles Slack posting natively.
|
|
147
|
+
[ -n "$SLACK_BOT_TOKEN" ] && \
|
|
148
|
+
upsert_secret "slack" "slack.com" "Authorization" "Bearer $SLACK_BOT_TOKEN"
|
|
196
149
|
|
|
197
|
-
|
|
150
|
+
# ── Add your services here ────────────────────────────────────────────────────
|
|
151
|
+
# For each service your agents call, add a block:
|
|
152
|
+
#
|
|
153
|
+
# MY_TOKEN=$(env_val "MY_SERVICE_TOKEN")
|
|
154
|
+
# MY_HOST="api.myservice.com"
|
|
155
|
+
# [ -n "$MY_TOKEN" ] && upsert_secret "my-service" "$MY_HOST" "Authorization" "Bearer $MY_TOKEN"
|
|
156
|
+
#
|
|
157
|
+
# Then add "my-service" to the agent's onecli_secrets in agents.yaml.
|
|
158
|
+
# The proxy will inject the Authorization header for any call to MY_HOST.
|
|
198
159
|
|
|
199
|
-
# ── Register global-claw agent if missing ────────────────────────────────────
|
|
200
|
-
|
|
201
|
-
echo "Ensuring agents exist..."
|
|
202
|
-
GLOBAL_CLAW_EXISTS=$(docker exec onecli-postgres-1 psql -U onecli -d onecli -t -c \
|
|
203
|
-
"SELECT COUNT(*) FROM agents WHERE name='global-claw';" | tr -d ' \n')
|
|
204
|
-
|
|
205
|
-
if [ "$GLOBAL_CLAW_EXISTS" = "0" ]; then
|
|
206
|
-
RAND_TOKEN=$(openssl rand -hex 20)
|
|
207
|
-
# Try new schema (account_id) first, fall back to old schema (project_id)
|
|
208
|
-
docker exec onecli-postgres-1 psql -U onecli -d onecli -c \
|
|
209
|
-
"INSERT INTO agents (id, name, access_token, identifier, secret_mode, account_id, created_at, updated_at)
|
|
210
|
-
SELECT 'nanoclaw-global-claw', 'global-claw', 'aoc_${RAND_TOKEN}', 'global-claw', 'selective', id, NOW(), NOW()
|
|
211
|
-
FROM accounts LIMIT 1
|
|
212
|
-
ON CONFLICT (id) DO NOTHING;" > /dev/null 2>&1 || \
|
|
213
|
-
docker exec onecli-postgres-1 psql -U onecli -d onecli -c \
|
|
214
|
-
"INSERT INTO agents (id, name, access_token, identifier, secret_mode, project_id, created_at, updated_at)
|
|
215
|
-
VALUES ('nanoclaw-global-claw', 'global-claw', 'aoc_${RAND_TOKEN}', 'global-claw', 'selective',
|
|
216
|
-
(SELECT id FROM projects LIMIT 1), NOW(), NOW())
|
|
217
|
-
ON CONFLICT (id) DO NOTHING;" > /dev/null 2>&1
|
|
218
|
-
echo " ✓ Created global-claw agent"
|
|
219
|
-
else
|
|
220
|
-
echo " ✓ global-claw agent already exists"
|
|
221
|
-
fi
|
|
222
160
|
echo ""
|
|
223
161
|
|
|
224
162
|
# ── Link secrets to agents (driven by agents.yaml — single source of truth) ───
|
|
@@ -148,7 +148,7 @@ function redactContainerArgs(args: string[]): string {
|
|
|
148
148
|
}
|
|
149
149
|
|
|
150
150
|
// Env vars forwarded from .env into every container so Claude's Bash tool
|
|
151
|
-
// can call
|
|
151
|
+
// can call tools (Python scripts, gh CLI, etc.) without
|
|
152
152
|
// secrets being baked into the container image.
|
|
153
153
|
const FORWARDED_ENV_KEYS = [
|
|
154
154
|
'CLAUDE_CODE_API_KEY_HELPER_TTL_MS',
|
|
@@ -172,7 +172,7 @@ const FORWARDED_ENV_KEYS = [
|
|
|
172
172
|
'GITHUB_TOKEN',
|
|
173
173
|
'GITHUB_REPO',
|
|
174
174
|
|
|
175
|
-
// Jira
|
|
175
|
+
// Jira
|
|
176
176
|
'JIRA_BASE_URL',
|
|
177
177
|
'JIRA_EMAIL',
|
|
178
178
|
'JIRA_API_TOKEN',
|
|
@@ -450,8 +450,8 @@ async function buildContainerArgs(
|
|
|
450
450
|
args.push('-e', `TZ=${TIMEZONE}`);
|
|
451
451
|
|
|
452
452
|
// Forward service credentials from .env so Claude's Bash tool can call
|
|
453
|
-
//
|
|
454
|
-
const agentFolder = agentIdentifier || "
|
|
453
|
+
// tools and gh CLI inside the container.
|
|
454
|
+
const agentFolder = agentIdentifier || "forge";
|
|
455
455
|
const forwardedEnv = readEnvFile(getAgentEnvKeys(agentFolder));
|
|
456
456
|
for (const [key, value] of Object.entries(forwardedEnv)) {
|
|
457
457
|
args.push('-e', `${key}=${value}`);
|
package/template/src/index.ts
CHANGED
|
@@ -583,13 +583,13 @@ function ensureContainerSystemRunning(): void {
|
|
|
583
583
|
*
|
|
584
584
|
* On every startup, reads agents.yaml and registers any agent whose channel
|
|
585
585
|
* ID env var is set but isn't in the DB yet. This replaces the need to run
|
|
586
|
-
*
|
|
586
|
+
* Agents auto-register on startup — no manual script needed.
|
|
587
587
|
*/
|
|
588
588
|
/**
|
|
589
589
|
* Ensure every agent from agents.yaml has an entry in the sender allowlist.
|
|
590
590
|
*
|
|
591
591
|
* The allowlist file only needs entries for OVERRIDES — e.g. restricting
|
|
592
|
-
*
|
|
592
|
+
* a main/meta agent to a specific sender. Every other registered agent is
|
|
593
593
|
* automatically allowed with the default open policy so agents.yaml + .env
|
|
594
594
|
* is the single file to maintain.
|
|
595
595
|
*/
|
|
@@ -647,6 +647,7 @@ function autoRegisterAgentsFromYaml(): void {
|
|
|
647
647
|
is_main?: boolean;
|
|
648
648
|
onecli_secrets?: string[];
|
|
649
649
|
onecli_id?: string;
|
|
650
|
+
container_config?: import('./types.js').ContainerConfig;
|
|
650
651
|
}>;
|
|
651
652
|
|
|
652
653
|
try {
|
|
@@ -676,6 +677,7 @@ function autoRegisterAgentsFromYaml(): void {
|
|
|
676
677
|
added_at: new Date().toISOString(),
|
|
677
678
|
requiresTrigger: def.requires_trigger !== false,
|
|
678
679
|
isMain: def.is_main === true,
|
|
680
|
+
...(def.container_config ? { containerConfig: def.container_config } : {}),
|
|
679
681
|
};
|
|
680
682
|
|
|
681
683
|
setRegisteredGroup(jid, group);
|
|
@@ -695,7 +697,7 @@ function autoRegisterAgentsFromYaml(): void {
|
|
|
695
697
|
|
|
696
698
|
// Ensure every registered agent has an entry in the sender allowlist.
|
|
697
699
|
// The allowlist file only needs to exist for OVERRIDES (e.g. restricting
|
|
698
|
-
//
|
|
700
|
+
// a main/meta agent to specific senders). All other agents are auto-added with
|
|
699
701
|
// the default open policy so agents.yaml + .env is the only file to maintain.
|
|
700
702
|
ensureAllowlistEntries(agentDefs);
|
|
701
703
|
|
|
@@ -714,13 +716,32 @@ function ensureOneCLISecrets(): void {
|
|
|
714
716
|
const secretsScript = path.join(process.cwd(), 'scripts', 'setup-onecli-secrets.sh');
|
|
715
717
|
if (!fs.existsSync(secretsScript)) return;
|
|
716
718
|
|
|
717
|
-
// Hash
|
|
719
|
+
// Hash two things:
|
|
720
|
+
// 1. Credential-bearing env vars (re-run when secrets rotate)
|
|
721
|
+
// 2. onecli_secrets from agents.yaml (re-run when an agent's secret list changes)
|
|
718
722
|
const credentialKeys = [
|
|
719
723
|
'ELASTIC_API_KEY', 'JIRA_EMAIL', 'JIRA_API_TOKEN',
|
|
720
724
|
'SLACK_BOT_TOKEN', 'INTERCOM_ACCESS_TOKEN', 'ARDOQ_API_KEY',
|
|
721
725
|
'ANTHROPIC_AUTH_TOKEN', 'CONFLUENCE_USERNAME', 'CONFLUENCE_PASSWORD',
|
|
726
|
+
'XLEDGER_API_TOKEN',
|
|
722
727
|
];
|
|
723
|
-
const
|
|
728
|
+
const envHash = credentialKeys.map(k => `${k}=${process.env[k] || ''}`).join('\n');
|
|
729
|
+
|
|
730
|
+
// Include agents.yaml onecli_secrets so adding/removing a secret triggers re-run
|
|
731
|
+
let agentsSecretsSig = '';
|
|
732
|
+
try {
|
|
733
|
+
const agentsYaml = path.join(process.cwd(), 'agents.yaml');
|
|
734
|
+
if (fs.existsSync(agentsYaml)) {
|
|
735
|
+
const defs = YAML.parse(fs.readFileSync(agentsYaml, 'utf-8')).agents || [];
|
|
736
|
+
agentsSecretsSig = defs
|
|
737
|
+
.map((d: { folder: string; onecli_secrets?: string[] }) =>
|
|
738
|
+
`${d.folder}:${(d.onecli_secrets || []).sort().join(',')}`)
|
|
739
|
+
.sort()
|
|
740
|
+
.join('\n');
|
|
741
|
+
}
|
|
742
|
+
} catch { /* non-fatal — env hash alone is still useful */ }
|
|
743
|
+
|
|
744
|
+
const hashInput = envHash + '\n---agents---\n' + agentsSecretsSig;
|
|
724
745
|
const currentHash = crypto.createHash('sha256').update(hashInput).digest('hex');
|
|
725
746
|
|
|
726
747
|
const hashFile = path.join(STORE_DIR, 'onecli-secrets.hash');
|
|
@@ -1,161 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Standalone Elastic/Kibana log query tool for Argus.
|
|
3
|
-
|
|
4
|
-
Usage:
|
|
5
|
-
python3 elastic_query.py --namespace it-ops
|
|
6
|
-
python3 elastic_query.py --namespace it-ops --container my-service
|
|
7
|
-
python3 elastic_query.py --namespace it-ops --from 2024-01-01T10:00:00Z --to 2024-01-01T11:00:00Z
|
|
8
|
-
python3 elastic_query.py # searches all namespaces with default lookback
|
|
9
|
-
|
|
10
|
-
Environment variables (required):
|
|
11
|
-
ELASTIC_BASE_URL e.g. https://kibana.yourcompany.com
|
|
12
|
-
ELASTIC_API_KEY Kibana API key
|
|
13
|
-
|
|
14
|
-
Optional environment variables:
|
|
15
|
-
ELASTIC_LOOKBACK_MINUTES default 60
|
|
16
|
-
ELASTIC_MAX_LOGS default 100
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
import argparse
|
|
20
|
-
import json
|
|
21
|
-
import os
|
|
22
|
-
import sys
|
|
23
|
-
from datetime import datetime, timedelta, timezone
|
|
24
|
-
|
|
25
|
-
try:
|
|
26
|
-
import requests
|
|
27
|
-
except ImportError:
|
|
28
|
-
print("ERROR: 'requests' is not installed. Run: pip3 install requests", file=sys.stderr)
|
|
29
|
-
sys.exit(1)
|
|
30
|
-
|
|
31
|
-
KIBANA_PROXY_PATH = "/api/console/proxy"
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
def parse_args():
|
|
35
|
-
p = argparse.ArgumentParser(description="Query Elastic error logs via Kibana proxy")
|
|
36
|
-
p.add_argument("--namespace", help="Kubernetes namespace (omit to search all)")
|
|
37
|
-
p.add_argument("--container", help="Kubernetes container name (optional filter)")
|
|
38
|
-
p.add_argument("--from", dest="from_ts", help="Start timestamp (ISO 8601 or 'now-Xm/h/d')")
|
|
39
|
-
p.add_argument("--to", dest="to_ts", help="End timestamp (ISO 8601 or 'now')")
|
|
40
|
-
p.add_argument("--lookback", type=int, help="Lookback minutes (overrides ELASTIC_LOOKBACK_MINUTES)")
|
|
41
|
-
p.add_argument("--max", type=int, help="Max log entries (overrides ELASTIC_MAX_LOGS)")
|
|
42
|
-
return p.parse_args()
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
def resolve_ts(ts: str) -> str:
|
|
46
|
-
"""Resolve 'now-Xm/h/d' and 'now' to ISO timestamps."""
|
|
47
|
-
if not ts.startswith("now"):
|
|
48
|
-
return ts
|
|
49
|
-
import re
|
|
50
|
-
m = re.match(r"now-(\d+)([mhd])$", ts)
|
|
51
|
-
if m:
|
|
52
|
-
amount, unit = int(m.group(1)), m.group(2)
|
|
53
|
-
delta = {"m": timedelta(minutes=amount), "h": timedelta(hours=amount), "d": timedelta(days=amount)}[unit]
|
|
54
|
-
return (datetime.now(timezone.utc) - delta).isoformat()
|
|
55
|
-
return datetime.now(timezone.utc).isoformat()
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def query_logs(namespace, container, from_ts, to_ts, max_logs, base_url, api_key):
|
|
59
|
-
if from_ts and to_ts:
|
|
60
|
-
time_range = {"gte": from_ts, "lte": to_ts}
|
|
61
|
-
else:
|
|
62
|
-
now = datetime.now(timezone.utc)
|
|
63
|
-
lookback = int(os.environ.get("ELASTIC_LOOKBACK_MINUTES", "60"))
|
|
64
|
-
since = now - timedelta(minutes=lookback)
|
|
65
|
-
time_range = {"gte": since.isoformat(), "lte": now.isoformat()}
|
|
66
|
-
|
|
67
|
-
must = [{"range": {"@timestamp": time_range}}]
|
|
68
|
-
if namespace:
|
|
69
|
-
must.append({"match_phrase": {"kubernetes.namespace": namespace}})
|
|
70
|
-
if container:
|
|
71
|
-
must.append({"match_phrase": {"kubernetes.container.name": container}})
|
|
72
|
-
|
|
73
|
-
body = {
|
|
74
|
-
"size": max_logs,
|
|
75
|
-
"sort": [{"@timestamp": {"order": "desc"}}],
|
|
76
|
-
"query": {
|
|
77
|
-
"bool": {
|
|
78
|
-
"must": must,
|
|
79
|
-
"should": [
|
|
80
|
-
{"match_phrase": {"log.level": "error"}},
|
|
81
|
-
{"match_phrase": {"log.level": "ERROR"}},
|
|
82
|
-
{"match": {"message": "error"}},
|
|
83
|
-
{"match": {"message": "exception"}},
|
|
84
|
-
{"match": {"message": "traceback"}},
|
|
85
|
-
{"match": {"message": "Traceback"}},
|
|
86
|
-
],
|
|
87
|
-
"minimum_should_match": 1,
|
|
88
|
-
}
|
|
89
|
-
},
|
|
90
|
-
"_source": [
|
|
91
|
-
"@timestamp",
|
|
92
|
-
"message",
|
|
93
|
-
"log.level",
|
|
94
|
-
"kubernetes.pod.name",
|
|
95
|
-
"kubernetes.container.name",
|
|
96
|
-
],
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
url = f"{base_url.rstrip('/')}{KIBANA_PROXY_PATH}"
|
|
100
|
-
params = {"path": "/filebeat-*/_search", "method": "GET"}
|
|
101
|
-
# Authorization header is optional here — if ELASTIC_API_KEY env var is set,
|
|
102
|
-
# use it directly. If absent, the OneCLI HTTPS proxy injects it automatically.
|
|
103
|
-
headers = {"kbn-xsrf": "true", "Content-Type": "application/json"}
|
|
104
|
-
if api_key:
|
|
105
|
-
headers["Authorization"] = f"ApiKey {api_key}"
|
|
106
|
-
|
|
107
|
-
resp = requests.post(url, params=params, headers=headers, json=body, timeout=30)
|
|
108
|
-
resp.raise_for_status()
|
|
109
|
-
hits = resp.json().get("hits", {}).get("hits", [])
|
|
110
|
-
return [h["_source"] for h in hits]
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
def format_logs(logs):
|
|
114
|
-
if not logs:
|
|
115
|
-
return "No error logs found in the specified time window."
|
|
116
|
-
lines = []
|
|
117
|
-
for log in logs:
|
|
118
|
-
ts = log.get("@timestamp", "?")
|
|
119
|
-
level = log.get("log.level") or (log.get("log") or {}).get("level", "?")
|
|
120
|
-
k8s = log.get("kubernetes") or {}
|
|
121
|
-
pod = (k8s.get("pod") or {}).get("name", "?")
|
|
122
|
-
ctr = (k8s.get("container") or {}).get("name", "?")
|
|
123
|
-
msg = log.get("message", "").strip()
|
|
124
|
-
lines.append(f"[{ts}] [{level}] [{ctr}/{pod}] {msg}")
|
|
125
|
-
return "\n".join(lines)
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
def main():
|
|
129
|
-
args = parse_args()
|
|
130
|
-
|
|
131
|
-
base_url = os.environ.get("ELASTIC_BASE_URL", "").strip()
|
|
132
|
-
if not base_url:
|
|
133
|
-
print("ERROR: ELASTIC_BASE_URL must be set in the environment.", file=sys.stderr)
|
|
134
|
-
sys.exit(1)
|
|
135
|
-
|
|
136
|
-
# api_key is optional — if absent, OneCLI HTTPS proxy injects the Authorization header.
|
|
137
|
-
api_key = os.environ.get("ELASTIC_API_KEY", "").strip()
|
|
138
|
-
|
|
139
|
-
from_ts = resolve_ts(args.from_ts) if args.from_ts else None
|
|
140
|
-
to_ts = resolve_ts(args.to_ts) if args.to_ts else None
|
|
141
|
-
max_logs = args.max or int(os.environ.get("ELASTIC_MAX_LOGS", "100"))
|
|
142
|
-
|
|
143
|
-
try:
|
|
144
|
-
logs = query_logs(
|
|
145
|
-
namespace=args.namespace,
|
|
146
|
-
container=args.container,
|
|
147
|
-
from_ts=from_ts,
|
|
148
|
-
to_ts=to_ts,
|
|
149
|
-
max_logs=max_logs,
|
|
150
|
-
base_url=base_url,
|
|
151
|
-
api_key=api_key,
|
|
152
|
-
)
|
|
153
|
-
except Exception as e:
|
|
154
|
-
print(f"ERROR: Elastic query failed: {e}", file=sys.stderr)
|
|
155
|
-
sys.exit(1)
|
|
156
|
-
|
|
157
|
-
print(format_logs(logs))
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
if __name__ == "__main__":
|
|
161
|
-
main()
|
|
@@ -1,185 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Standalone Google Drive tool for Argus agents.
|
|
3
|
-
Uses OAuth refresh token — no google-auth library needed, just requests.
|
|
4
|
-
|
|
5
|
-
Usage:
|
|
6
|
-
# List files in the standup folder
|
|
7
|
-
python3 gdrive_tool.py list
|
|
8
|
-
|
|
9
|
-
# List only files not yet in processed_transcripts.txt
|
|
10
|
-
python3 gdrive_tool.py list --new-only
|
|
11
|
-
|
|
12
|
-
# Read a Google Doc as plain text
|
|
13
|
-
python3 gdrive_tool.py read --file-id 1abc123...
|
|
14
|
-
|
|
15
|
-
# Mark a file as processed (appends ID to processed_transcripts.txt)
|
|
16
|
-
python3 gdrive_tool.py mark-processed --file-id 1abc123...
|
|
17
|
-
|
|
18
|
-
Environment variables (required):
|
|
19
|
-
GOOGLE_CLIENT_ID
|
|
20
|
-
GOOGLE_CLIENT_SECRET
|
|
21
|
-
GOOGLE_REFRESH_TOKEN
|
|
22
|
-
GOOGLE_DRIVE_STANDUP_FOLDER_ID
|
|
23
|
-
|
|
24
|
-
Optional:
|
|
25
|
-
GDRIVE_PROCESSED_FILE path to processed transcripts list
|
|
26
|
-
default: /workspace/group/processed_transcripts.txt
|
|
27
|
-
"""
|
|
28
|
-
|
|
29
|
-
import argparse
|
|
30
|
-
import json
|
|
31
|
-
import os
|
|
32
|
-
import sys
|
|
33
|
-
|
|
34
|
-
try:
|
|
35
|
-
import requests
|
|
36
|
-
except ImportError:
|
|
37
|
-
print("ERROR: 'requests' is not installed.", file=sys.stderr)
|
|
38
|
-
sys.exit(1)
|
|
39
|
-
|
|
40
|
-
TOKEN_URL = "https://oauth2.googleapis.com/token"
|
|
41
|
-
DRIVE_API = "https://www.googleapis.com/drive/v3"
|
|
42
|
-
|
|
43
|
-
PROCESSED_FILE = os.environ.get(
|
|
44
|
-
"GDRIVE_PROCESSED_FILE", "/workspace/group/processed_transcripts.txt"
|
|
45
|
-
)
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
def get_access_token():
|
|
49
|
-
"""Get Google access token.
|
|
50
|
-
|
|
51
|
-
Uses GOOGLE_ACCESS_TOKEN injected by the NanoClaw host (short-lived, ~1h TTL).
|
|
52
|
-
The OAuth refresh credentials stay on the host — never in the container.
|
|
53
|
-
"""
|
|
54
|
-
token = os.environ.get("GOOGLE_ACCESS_TOKEN", "").strip()
|
|
55
|
-
if not token:
|
|
56
|
-
print(
|
|
57
|
-
"ERROR: GOOGLE_ACCESS_TOKEN must be set. "
|
|
58
|
-
"The NanoClaw host generates this token — check that Google OAuth "
|
|
59
|
-
"credentials (GOOGLE_CLIENT_ID/SECRET/REFRESH_TOKEN) are in the host .env.",
|
|
60
|
-
file=sys.stderr,
|
|
61
|
-
)
|
|
62
|
-
sys.exit(1)
|
|
63
|
-
return token
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
def get_folder_id():
|
|
67
|
-
folder_id = os.environ.get("GOOGLE_DRIVE_STANDUP_FOLDER_ID", "").strip()
|
|
68
|
-
if not folder_id:
|
|
69
|
-
print("ERROR: GOOGLE_DRIVE_STANDUP_FOLDER_ID must be set.", file=sys.stderr)
|
|
70
|
-
sys.exit(1)
|
|
71
|
-
return folder_id
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
def load_processed():
|
|
75
|
-
"""Return set of already-processed file IDs."""
|
|
76
|
-
if not os.path.exists(PROCESSED_FILE):
|
|
77
|
-
return set()
|
|
78
|
-
with open(PROCESSED_FILE) as f:
|
|
79
|
-
return set(line.strip() for line in f if line.strip())
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def mark_processed(file_id):
|
|
83
|
-
"""Append a file ID to the processed list."""
|
|
84
|
-
processed = load_processed()
|
|
85
|
-
if file_id not in processed:
|
|
86
|
-
with open(PROCESSED_FILE, "a") as f:
|
|
87
|
-
f.write(file_id + "\n")
|
|
88
|
-
print(f"Marked {file_id} as processed.")
|
|
89
|
-
else:
|
|
90
|
-
print(f"{file_id} was already marked as processed.")
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
def cmd_list(args):
|
|
94
|
-
token = get_access_token()
|
|
95
|
-
folder_id = get_folder_id()
|
|
96
|
-
|
|
97
|
-
resp = requests.get(
|
|
98
|
-
f"{DRIVE_API}/files",
|
|
99
|
-
headers={"Authorization": f"Bearer {token}"},
|
|
100
|
-
params={
|
|
101
|
-
"q": f"'{folder_id}' in parents and trashed=false",
|
|
102
|
-
"fields": "files(id,name,createdTime,mimeType)",
|
|
103
|
-
"orderBy": "createdTime asc",
|
|
104
|
-
"pageSize": 1000,
|
|
105
|
-
"supportsAllDrives": True,
|
|
106
|
-
"includeItemsFromAllDrives": True,
|
|
107
|
-
},
|
|
108
|
-
timeout=30,
|
|
109
|
-
)
|
|
110
|
-
resp.raise_for_status()
|
|
111
|
-
files = resp.json().get("files", [])
|
|
112
|
-
|
|
113
|
-
if args.new_only:
|
|
114
|
-
processed = load_processed()
|
|
115
|
-
files = [f for f in files if f["id"] not in processed]
|
|
116
|
-
|
|
117
|
-
if not files:
|
|
118
|
-
print("No files found." if not args.new_only else "No new files to process.")
|
|
119
|
-
return
|
|
120
|
-
|
|
121
|
-
print(json.dumps(files, indent=2))
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
def cmd_read(args):
|
|
125
|
-
token = get_access_token()
|
|
126
|
-
|
|
127
|
-
# Try exporting as plain text (works for Google Docs)
|
|
128
|
-
resp = requests.get(
|
|
129
|
-
f"{DRIVE_API}/files/{args.file_id}/export",
|
|
130
|
-
headers={"Authorization": f"Bearer {token}"},
|
|
131
|
-
params={"mimeType": "text/plain"},
|
|
132
|
-
timeout=60,
|
|
133
|
-
)
|
|
134
|
-
|
|
135
|
-
if resp.status_code == 400:
|
|
136
|
-
# Not a Google Doc — try downloading directly
|
|
137
|
-
resp = requests.get(
|
|
138
|
-
f"{DRIVE_API}/files/{args.file_id}?alt=media",
|
|
139
|
-
headers={"Authorization": f"Bearer {token}"},
|
|
140
|
-
timeout=60,
|
|
141
|
-
)
|
|
142
|
-
|
|
143
|
-
resp.raise_for_status()
|
|
144
|
-
print(resp.text)
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
def cmd_mark_processed(args):
|
|
148
|
-
mark_processed(args.file_id)
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
def parse_args():
|
|
152
|
-
p = argparse.ArgumentParser(description="Google Drive tool for Argus agents")
|
|
153
|
-
sub = p.add_subparsers(dest="command")
|
|
154
|
-
|
|
155
|
-
l = sub.add_parser("list", help="List files in the standup folder")
|
|
156
|
-
l.add_argument("--new-only", action="store_true", help="Only show unprocessed files")
|
|
157
|
-
|
|
158
|
-
r = sub.add_parser("read", help="Read a Google Doc as plain text")
|
|
159
|
-
r.add_argument("--file-id", required=True)
|
|
160
|
-
|
|
161
|
-
m = sub.add_parser("mark-processed", help="Mark a file as processed")
|
|
162
|
-
m.add_argument("--file-id", required=True)
|
|
163
|
-
|
|
164
|
-
return p.parse_args()
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
def main():
|
|
168
|
-
args = parse_args()
|
|
169
|
-
if not args.command:
|
|
170
|
-
print("ERROR: specify a command. Use --help for usage.", file=sys.stderr)
|
|
171
|
-
sys.exit(1)
|
|
172
|
-
try:
|
|
173
|
-
{"list": cmd_list, "read": cmd_read, "mark-processed": cmd_mark_processed}[
|
|
174
|
-
args.command
|
|
175
|
-
](args)
|
|
176
|
-
except requests.HTTPError as e:
|
|
177
|
-
print(f"ERROR: Google Drive API error: {e}\n{e.response.text}", file=sys.stderr)
|
|
178
|
-
sys.exit(1)
|
|
179
|
-
except Exception as e:
|
|
180
|
-
print(f"ERROR: {e}", file=sys.stderr)
|
|
181
|
-
sys.exit(1)
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
if __name__ == "__main__":
|
|
185
|
-
main()
|