crucix 2.39.1 → 2.39.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +3 -1
- package/README.md +4 -3
- package/apis/sources/acled.mjs +57 -12
- package/apis/sources/bls.mjs +39 -5
- package/apis/sources/epa.mjs +51 -4
- package/apis/sources/opensky.mjs +22 -1
- package/apis/sources/space.mjs +49 -17
- package/apis/utils/fetch.mjs +6 -2
- package/apis/utils/freshness.mjs +1 -1
- package/apis/utils/quota-cache.mjs +42 -0
- package/apis/utils/rss-pool.mjs +33 -0
- package/crucix.config.mjs +1 -1
- package/dashboard/inject.mjs +33 -12
- package/dashboard/public/live-sources.js +1 -1
- package/lib/llm/budgets.mjs +1 -1
- package/lib/llm/cursor.mjs +12 -2
- package/lib/version.mjs +22 -0
- package/lib/world-rss-runner.mjs +31 -19
- package/package.json +1 -1
- package/scripts/cli.mjs +13 -0
package/.env.example
CHANGED
|
@@ -15,6 +15,8 @@ FRED_API_KEY=
|
|
|
15
15
|
FIRMS_MAP_KEY=
|
|
16
16
|
# Energy Information Administration (free: api.eia.gov/register)
|
|
17
17
|
EIA_API_KEY=
|
|
18
|
+
# BLS Public Data (optional: data.bls.gov/registrationEngine). v1 is ~25/day; Crucix caches 12 hours. v2 is 500/day.
|
|
19
|
+
BLS_API_KEY=
|
|
18
20
|
# Financial Modeling Prep (free: site.financialmodelingprep.com/register, 250 calls/day): equities, indexes, forex in two
|
|
19
21
|
# batch calls per sweep (192/day at the 15-minute sweep). FMP_DAILY_LIMIT caps the calls; without a key Yahoo covers it.
|
|
20
22
|
FMP_API_KEY=
|
|
@@ -94,7 +96,7 @@ ANTHROPIC_BASE_URL=
|
|
|
94
96
|
# Sent as Authorization: Bearer instead of x-api-key; use either this or LLM_API_KEY, not both
|
|
95
97
|
ANTHROPIC_AUTH_TOKEN=
|
|
96
98
|
LLM_IDEAS_MAX_TOKENS=4096
|
|
97
|
-
LLM_IDEAS_TIMEOUT_MS=
|
|
99
|
+
LLM_IDEAS_TIMEOUT_MS=180000
|
|
98
100
|
LLM_ALERT_MAX_TOKENS=800
|
|
99
101
|
LLM_ALERT_TIMEOUT_MS=30000
|
|
100
102
|
# Cited briefings (POST /api/briefing): tokens 128-8192, timeout 1000-360000 ms; a failure falls back to the rule-based briefing
|
package/README.md
CHANGED
|
@@ -95,15 +95,16 @@ It was built for anyone who wants to understand what's actually happening in the
|
|
|
95
95
|
## Quick Start
|
|
96
96
|
|
|
97
97
|
```bash
|
|
98
|
-
# From npm (recommended): first run opens the setup wizard
|
|
98
|
+
# From npm (recommended): Node + runtime source, first run opens the setup wizard
|
|
99
99
|
npx crucix
|
|
100
|
-
|
|
101
|
-
# Or install the command
|
|
102
100
|
npm install -g crucix
|
|
101
|
+
crucix --version
|
|
103
102
|
crucix install # ~/.crucix (.env and runs/)
|
|
104
103
|
crucix # dashboard; wizard if no .env yet
|
|
105
104
|
```
|
|
106
105
|
|
|
106
|
+
That is not the standalone exe. `npm run install:local` (from a git checkout) builds `crucix.exe` and copies it to `%USERPROFILE%\.local\bin` / `~/.local/bin`. Check which one PATH hits with `crucix --version`.
|
|
107
|
+
|
|
107
108
|
Settings the wizard and **Settings → News feeds** save go to `%USERPROFILE%\.crucix` on Windows or `~/.crucix` on Linux/macOS. Set `CRUCIX_HOME` to choose another folder, or `CRUCIX_ENV_FILE` to name the `.env` itself.
|
|
108
109
|
|
|
109
110
|
From a git checkout (development):
|
package/apis/sources/acled.mjs
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { readBoundedText } from '../utils/fetch.mjs';
|
|
2
2
|
// ACLED — Armed Conflict Location & Event Data
|
|
3
3
|
// Auth strategy (tries in order):
|
|
4
|
-
// 1.
|
|
5
|
-
// 2.
|
|
4
|
+
// 1. OAuth Bearer token: POST /oauth/token (password grant + scope=authenticated)
|
|
5
|
+
// 2. Cookie-based session: POST /user/login?_format=json → session cookie
|
|
6
6
|
// Set ACLED_EMAIL and ACLED_PASSWORD in .env (your myACLED login credentials).
|
|
7
7
|
// Data endpoint: GET https://acleddata.com/api/acled/read
|
|
8
8
|
|
|
9
9
|
import { daysAgo } from '../utils/fetch.mjs';
|
|
10
|
+
import { readQuota, writeQuota, clearQuota } from '../utils/quota-cache.mjs';
|
|
10
11
|
import '../utils/env.mjs';
|
|
11
12
|
|
|
12
13
|
const LOGIN_URL = 'https://acleddata.com/user/login?_format=json';
|
|
@@ -15,6 +16,31 @@ const API_BASE = 'https://acleddata.com/api/acled/read';
|
|
|
15
16
|
|
|
16
17
|
// Session cache
|
|
17
18
|
let sessionCache = { cookies: null, token: null, method: null, expires: 0 };
|
|
19
|
+
let lastGood = { at: 0, result: null };
|
|
20
|
+
let hydrated = false;
|
|
21
|
+
const LAST_GOOD_MS = 6 * 60 * 60 * 1000;
|
|
22
|
+
const ACLED_HEADERS = {
|
|
23
|
+
'User-Agent': 'Crucix/2.2',
|
|
24
|
+
Accept: 'application/json',
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
function hydrateQuota() {
|
|
28
|
+
if (hydrated) return;
|
|
29
|
+
hydrated = true;
|
|
30
|
+
const saved = readQuota('acled');
|
|
31
|
+
if (Number(saved?.at) > 0 && saved.result) lastGood = { at: saved.at, result: saved.result };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function resetState({ disk = true } = {}) {
|
|
35
|
+
sessionCache = { cookies: null, token: null, method: null, expires: 0 };
|
|
36
|
+
lastGood = { at: 0, result: null };
|
|
37
|
+
hydrated = false;
|
|
38
|
+
if (disk) clearQuota('acled');
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function isChallenge(status, text) {
|
|
42
|
+
return status === 403 && /just a moment|cf-browser-verification|attention required/i.test(text || '');
|
|
43
|
+
}
|
|
18
44
|
|
|
19
45
|
// Strategy 1: Cookie-based session login (mirrors browser login)
|
|
20
46
|
async function loginCookie(email, password) {
|
|
@@ -22,7 +48,7 @@ async function loginCookie(email, password) {
|
|
|
22
48
|
try {
|
|
23
49
|
const res = await fetch(LOGIN_URL, {
|
|
24
50
|
method: 'POST',
|
|
25
|
-
headers: { 'Content-Type': 'application/json' },
|
|
51
|
+
headers: { ...ACLED_HEADERS, 'Content-Type': 'application/json' },
|
|
26
52
|
body: JSON.stringify({ name: email, pass: password }),
|
|
27
53
|
redirect: 'manual',
|
|
28
54
|
signal,
|
|
@@ -58,21 +84,30 @@ async function loginOAuth(email, password) {
|
|
|
58
84
|
password: password,
|
|
59
85
|
grant_type: 'password',
|
|
60
86
|
client_id: 'acled',
|
|
87
|
+
scope: 'authenticated',
|
|
61
88
|
});
|
|
62
89
|
|
|
63
90
|
const res = await fetch(TOKEN_URL, {
|
|
64
91
|
method: 'POST',
|
|
65
|
-
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
|
92
|
+
headers: { ...ACLED_HEADERS, 'Content-Type': 'application/x-www-form-urlencoded' },
|
|
66
93
|
body: body.toString(),
|
|
67
94
|
signal,
|
|
68
95
|
});
|
|
69
96
|
|
|
97
|
+
const text = await readBoundedText(res, 64 * 1024);
|
|
98
|
+
if (isChallenge(res.status, text)) {
|
|
99
|
+
return { error: `Cloudflare challenge (HTTP ${res.status}); ACLED is blocking scripted login`, blocked: true };
|
|
100
|
+
}
|
|
70
101
|
if (!res.ok) {
|
|
71
|
-
|
|
72
|
-
|
|
102
|
+
let detail = '';
|
|
103
|
+
try {
|
|
104
|
+
const fail = JSON.parse(text);
|
|
105
|
+
detail = fail.error_description || fail.error || fail.message || '';
|
|
106
|
+
} catch { /* HTML or empty */ }
|
|
107
|
+
return { error: `OAuth failed (HTTP ${res.status}${detail ? `: ${String(detail).slice(0, 120)}` : ''})` };
|
|
73
108
|
}
|
|
74
109
|
|
|
75
|
-
const data = JSON.parse(
|
|
110
|
+
const data = JSON.parse(text);
|
|
76
111
|
if (!data.access_token) {
|
|
77
112
|
return { error: 'OAuth response missing access_token' };
|
|
78
113
|
}
|
|
@@ -109,6 +144,9 @@ async function authenticate() {
|
|
|
109
144
|
}
|
|
110
145
|
errors.push(`OAuth: ${oauthResult.error}`);
|
|
111
146
|
if (debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`);
|
|
147
|
+
if (oauthResult.blocked) {
|
|
148
|
+
return { error: `All ACLED auth methods failed.\n${errors.join('\n')}` };
|
|
149
|
+
}
|
|
112
150
|
|
|
113
151
|
// Fall back to cookie-based session
|
|
114
152
|
const cookieResult = await loginCookie(email, password);
|
|
@@ -124,7 +162,7 @@ async function authenticate() {
|
|
|
124
162
|
|
|
125
163
|
// Build headers based on auth method
|
|
126
164
|
function authHeaders(session) {
|
|
127
|
-
const headers = {
|
|
165
|
+
const headers = { ...ACLED_HEADERS, 'Content-Type': 'application/json' };
|
|
128
166
|
if (session.method === 'cookie' && session.cookies) {
|
|
129
167
|
headers['Cookie'] = session.cookies;
|
|
130
168
|
} else if (session.method === 'oauth' && session.token) {
|
|
@@ -228,7 +266,8 @@ function groupBy(events, field) {
|
|
|
228
266
|
}
|
|
229
267
|
|
|
230
268
|
// Briefing — last 7 days of global conflict events
|
|
231
|
-
export async function briefing() {
|
|
269
|
+
export async function briefing({ now = Date.now() } = {}) {
|
|
270
|
+
hydrateQuota();
|
|
232
271
|
if (!process.env.ACLED_EMAIL || !process.env.ACLED_PASSWORD) {
|
|
233
272
|
return {
|
|
234
273
|
source: 'ACLED',
|
|
@@ -248,7 +287,10 @@ export async function briefing() {
|
|
|
248
287
|
});
|
|
249
288
|
|
|
250
289
|
if (data?.error) {
|
|
251
|
-
|
|
290
|
+
if (lastGood.result && now - lastGood.at < LAST_GOOD_MS) {
|
|
291
|
+
return { ...lastGood.result, stale: true, message: data.error };
|
|
292
|
+
}
|
|
293
|
+
return { source: 'ACLED', timestamp: new Date(now).toISOString(), error: data.error };
|
|
252
294
|
}
|
|
253
295
|
|
|
254
296
|
let events = data?.data || [];
|
|
@@ -292,9 +334,9 @@ export async function briefing() {
|
|
|
292
334
|
notes: e.notes?.slice(0, 200),
|
|
293
335
|
}));
|
|
294
336
|
|
|
295
|
-
|
|
337
|
+
const result = {
|
|
296
338
|
source: 'ACLED',
|
|
297
|
-
timestamp: new Date().toISOString(),
|
|
339
|
+
timestamp: new Date(now).toISOString(),
|
|
298
340
|
period: { start, end },
|
|
299
341
|
totalEvents: events.length,
|
|
300
342
|
totalFatalities,
|
|
@@ -303,6 +345,9 @@ export async function briefing() {
|
|
|
303
345
|
topCountries,
|
|
304
346
|
deadliestEvents,
|
|
305
347
|
};
|
|
348
|
+
lastGood = { at: now, result };
|
|
349
|
+
writeQuota('acled', lastGood);
|
|
350
|
+
return result;
|
|
306
351
|
}
|
|
307
352
|
|
|
308
353
|
if (process.argv[1]?.endsWith('acled.mjs')) {
|
package/apis/sources/bls.mjs
CHANGED
|
@@ -1,12 +1,35 @@
|
|
|
1
1
|
import { readBoundedText } from '../utils/fetch.mjs';
|
|
2
2
|
// BLS — Bureau of Labor Statistics
|
|
3
3
|
// CPI, unemployment, nonfarm payrolls, PPI. No auth required (v1 API).
|
|
4
|
-
// v2 with registration key supports more requests; v1 is
|
|
4
|
+
// v2 with registration key supports more requests; v1 is ~25 queries/day. A 12-hour
|
|
5
|
+
// cache keeps a 15-minute sweep under that cap (CPI/labor series update monthly).
|
|
5
6
|
|
|
6
7
|
import { safeFetch, daysAgo } from '../utils/fetch.mjs';
|
|
8
|
+
import { readQuota, writeQuota, clearQuota } from '../utils/quota-cache.mjs';
|
|
7
9
|
|
|
8
10
|
const V1_BASE = 'https://api.bls.gov/publicAPI/v1/timeseries/data/';
|
|
9
11
|
const V2_BASE = 'https://api.bls.gov/publicAPI/v2/timeseries/data/';
|
|
12
|
+
export const CACHE_MS = 12 * 60 * 60 * 1000;
|
|
13
|
+
|
|
14
|
+
let lastGood = { at: 0, result: null };
|
|
15
|
+
let hydrated = false;
|
|
16
|
+
|
|
17
|
+
function hydrateQuota() {
|
|
18
|
+
if (hydrated) return;
|
|
19
|
+
hydrated = true;
|
|
20
|
+
const saved = readQuota('bls');
|
|
21
|
+
if (Number(saved?.at) > 0 && saved.result) lastGood = { at: saved.at, result: saved.result };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function resetState({ disk = true } = {}) {
|
|
25
|
+
lastGood = { at: 0, result: null };
|
|
26
|
+
hydrated = false;
|
|
27
|
+
if (disk) clearQuota('bls');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function cached(message) {
|
|
31
|
+
return { ...lastGood.result, stale: true, message };
|
|
32
|
+
}
|
|
10
33
|
|
|
11
34
|
// Key economic series
|
|
12
35
|
const SERIES = {
|
|
@@ -92,18 +115,26 @@ function momChange(seriesData) {
|
|
|
92
115
|
}
|
|
93
116
|
|
|
94
117
|
// Briefing — pull latest CPI, unemployment, payrolls
|
|
95
|
-
export async function briefing(apiKey) {
|
|
118
|
+
export async function briefing(apiKey, { now = Date.now() } = {}) {
|
|
119
|
+
hydrateQuota();
|
|
120
|
+
if (lastGood.result && now >= lastGood.at && now - lastGood.at < CACHE_MS) {
|
|
121
|
+
return cached(`Cached BLS result; next request after ${new Date(lastGood.at + CACHE_MS).toISOString()}`);
|
|
122
|
+
}
|
|
123
|
+
|
|
96
124
|
const seriesIds = Object.keys(SERIES);
|
|
97
125
|
const resp = await getSeries(seriesIds, { apiKey });
|
|
98
126
|
|
|
99
127
|
if (resp.error) {
|
|
128
|
+
if (lastGood.result) return cached(resp.error);
|
|
100
129
|
return { source: 'BLS', error: resp.error, timestamp: new Date().toISOString() };
|
|
101
130
|
}
|
|
102
131
|
|
|
103
132
|
if (resp.status !== 'REQUEST_SUCCEEDED' || !resp.Results?.series?.length) {
|
|
133
|
+
const error = resp.message?.[0] || 'BLS API returned no data';
|
|
134
|
+
if (lastGood.result) return cached(error);
|
|
104
135
|
return {
|
|
105
136
|
source: 'BLS',
|
|
106
|
-
error
|
|
137
|
+
error,
|
|
107
138
|
rawStatus: resp.status,
|
|
108
139
|
timestamp: new Date().toISOString(),
|
|
109
140
|
};
|
|
@@ -151,12 +182,15 @@ export async function briefing(apiKey) {
|
|
|
151
182
|
}
|
|
152
183
|
}
|
|
153
184
|
|
|
154
|
-
|
|
185
|
+
const result = {
|
|
155
186
|
source: 'BLS',
|
|
156
|
-
timestamp: new Date().toISOString(),
|
|
187
|
+
timestamp: new Date(now).toISOString(),
|
|
157
188
|
indicators,
|
|
158
189
|
signals,
|
|
159
190
|
};
|
|
191
|
+
lastGood = { at: now, result };
|
|
192
|
+
writeQuota('bls', lastGood);
|
|
193
|
+
return result;
|
|
160
194
|
}
|
|
161
195
|
|
|
162
196
|
if (process.argv[1]?.endsWith('bls.mjs')) {
|
package/apis/sources/epa.mjs
CHANGED
|
@@ -4,8 +4,33 @@
|
|
|
4
4
|
// Complements Safecast (citizen science) with official government readings.
|
|
5
5
|
|
|
6
6
|
import { safeFetch } from '../utils/fetch.mjs';
|
|
7
|
+
import { readQuota, writeQuota, clearQuota } from '../utils/quota-cache.mjs';
|
|
7
8
|
|
|
8
9
|
const BASE = 'https://enviro.epa.gov/enviro/efservice';
|
|
10
|
+
const BLOCK_MS = 6 * 60 * 60 * 1000; // EPA's WAF 403 does not clear within a sweep; do not retry for 6 hours
|
|
11
|
+
|
|
12
|
+
let blockedUntil = 0;
|
|
13
|
+
let lastGood = null;
|
|
14
|
+
let hydrated = false;
|
|
15
|
+
|
|
16
|
+
function hydrateQuota() {
|
|
17
|
+
if (hydrated) return;
|
|
18
|
+
hydrated = true;
|
|
19
|
+
const saved = readQuota('epa');
|
|
20
|
+
if (Number(saved?.blockedUntil) > 0) blockedUntil = Number(saved.blockedUntil);
|
|
21
|
+
if (saved?.lastGood) lastGood = saved.lastGood;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function persistQuota() {
|
|
25
|
+
writeQuota('epa', { blockedUntil, lastGood });
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function resetState({ disk = true } = {}) {
|
|
29
|
+
blockedUntil = 0;
|
|
30
|
+
lastGood = null;
|
|
31
|
+
hydrated = false;
|
|
32
|
+
if (disk) clearQuota('epa');
|
|
33
|
+
}
|
|
9
34
|
|
|
10
35
|
// RadNet analytical results endpoint
|
|
11
36
|
const RADNET_ANALYTICAL = `${BASE}/RADNET_ANALYTICAL_RESULTS`;
|
|
@@ -123,14 +148,31 @@ function checkReading(reading) {
|
|
|
123
148
|
return null;
|
|
124
149
|
}
|
|
125
150
|
|
|
151
|
+
function denied(now, error) {
|
|
152
|
+
return lastGood
|
|
153
|
+
? { ...lastGood, stale: true, timestamp: lastGood.timestamp, message: error }
|
|
154
|
+
: { source: 'EPA RadNet', timestamp: new Date(now).toISOString(), error, readings: [], signals: ['Radiation readings unavailable'] };
|
|
155
|
+
}
|
|
156
|
+
|
|
126
157
|
// Briefing — get recent radiation readings from EPA network, flag anomalies
|
|
127
|
-
export async function briefing() {
|
|
158
|
+
export async function briefing({ now = Date.now() } = {}) {
|
|
159
|
+
hydrateQuota();
|
|
160
|
+
if (now < blockedUntil) {
|
|
161
|
+
return denied(now, `EPA RadNet provider access denied (HTTP 403); retry after ${new Date(blockedUntil).toISOString()}`);
|
|
162
|
+
}
|
|
163
|
+
|
|
128
164
|
const readings = [];
|
|
129
165
|
const signals = [];
|
|
130
166
|
|
|
131
167
|
// Fetch recent analytical results (broad pull)
|
|
132
168
|
const recentData = await getAnalyticalResults({ rows: 100 });
|
|
133
|
-
if (recentData.error)
|
|
169
|
+
if (recentData.error) {
|
|
170
|
+
if (recentData.status === 403) {
|
|
171
|
+
blockedUntil = now + BLOCK_MS;
|
|
172
|
+
persistQuota();
|
|
173
|
+
}
|
|
174
|
+
return denied(now, recentData.status === 403 ? 'EPA RadNet provider access denied (HTTP 403)' : recentData.error);
|
|
175
|
+
}
|
|
134
176
|
const recentRecords = Array.isArray(recentData) ? recentData : [];
|
|
135
177
|
|
|
136
178
|
// Compact all readings
|
|
@@ -194,9 +236,9 @@ export async function briefing() {
|
|
|
194
236
|
])
|
|
195
237
|
);
|
|
196
238
|
|
|
197
|
-
|
|
239
|
+
const result = {
|
|
198
240
|
source: 'EPA RadNet',
|
|
199
|
-
timestamp: new Date().toISOString(),
|
|
241
|
+
timestamp: new Date(now).toISOString(),
|
|
200
242
|
totalReadings: readings.length,
|
|
201
243
|
...(analyteResults.some(r => r.error) ? { error: 'Some EPA analyte requests failed' } : {}),
|
|
202
244
|
readings: readings.slice(0, 50), // cap for briefing size
|
|
@@ -210,6 +252,11 @@ export async function briefing() {
|
|
|
210
252
|
thresholds: THRESHOLDS,
|
|
211
253
|
note: 'RadNet data may lag by hours to days. Near-real-time gamma data updates more frequently.',
|
|
212
254
|
};
|
|
255
|
+
if (!result.error) {
|
|
256
|
+
lastGood = result;
|
|
257
|
+
persistQuota();
|
|
258
|
+
}
|
|
259
|
+
return result;
|
|
213
260
|
}
|
|
214
261
|
|
|
215
262
|
// Run standalone
|
package/apis/sources/opensky.mjs
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
// clientId/clientSecret of credentials.json go into OPENSKY_CLIENT_ID/OPENSKY_CLIENT_SECRET.
|
|
8
8
|
|
|
9
9
|
import { safeFetch, readBoundedText } from '../utils/fetch.mjs';
|
|
10
|
+
import { readQuota, writeQuota, clearQuota } from '../utils/quota-cache.mjs';
|
|
10
11
|
import '../utils/env.mjs';
|
|
11
12
|
|
|
12
13
|
const BASE = 'https://opensky-network.org/api';
|
|
@@ -30,12 +31,28 @@ let tokenCache = { token: null, expiresAt: 0 };
|
|
|
30
31
|
let tokenRequest = null; // in-flight token request shared by the parallel hotspots
|
|
31
32
|
let lastRound = { at: 0, result: null }; // last round that got data (and so spent credits)
|
|
32
33
|
let blockedUntil = 0; // set by HTTP 429 from X-Rate-Limit-Retry-After-Seconds
|
|
34
|
+
let hydrated = false;
|
|
33
35
|
|
|
34
|
-
|
|
36
|
+
function persistQuota() {
|
|
37
|
+
writeQuota('opensky', { blockedUntil, lastRound });
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function hydrateQuota() {
|
|
41
|
+
if (hydrated) return;
|
|
42
|
+
hydrated = true;
|
|
43
|
+
const saved = readQuota('opensky');
|
|
44
|
+
if (!saved) return;
|
|
45
|
+
if (Number(saved.blockedUntil) > 0) blockedUntil = Number(saved.blockedUntil);
|
|
46
|
+
if (Number(saved.lastRound?.at) > 0 && saved.lastRound.result) lastRound = saved.lastRound;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function resetState({ disk = true } = {}) {
|
|
35
50
|
tokenCache = { token: null, expiresAt: 0 };
|
|
36
51
|
tokenRequest = null;
|
|
37
52
|
lastRound = { at: 0, result: null };
|
|
38
53
|
blockedUntil = 0;
|
|
54
|
+
hydrated = false;
|
|
55
|
+
if (disk) clearQuota('opensky');
|
|
39
56
|
}
|
|
40
57
|
|
|
41
58
|
function credentials() {
|
|
@@ -177,6 +194,7 @@ function blocked(now) {
|
|
|
177
194
|
|
|
178
195
|
// Briefing — check hotspot regions for flight activity
|
|
179
196
|
export async function briefing({ now = Date.now() } = {}) {
|
|
197
|
+
hydrateQuota();
|
|
180
198
|
if (now < blockedUntil) return blocked(now);
|
|
181
199
|
const deadline = Date.now() + ROUND_BUDGET_MS;
|
|
182
200
|
const creds = credentials();
|
|
@@ -239,7 +257,10 @@ export async function briefing({ now = Date.now() } = {}) {
|
|
|
239
257
|
if (hotspotErrors.length < results.length) lastRound = { at: now, result };
|
|
240
258
|
if (limits.length) {
|
|
241
259
|
blockedUntil = now + Math.min(MAX_BLOCK_MS, Math.max(...limits));
|
|
260
|
+
persistQuota();
|
|
242
261
|
if (hotspotErrors.length === results.length) return blocked(now);
|
|
262
|
+
} else if (hotspotErrors.length < results.length) {
|
|
263
|
+
persistQuota();
|
|
243
264
|
}
|
|
244
265
|
return result;
|
|
245
266
|
}
|
package/apis/sources/space.mjs
CHANGED
|
@@ -2,9 +2,30 @@
|
|
|
2
2
|
// No API key required. Uses CelesTrak for public TLE data and launch info.
|
|
3
3
|
// Tracks: Recent launches, ISS position, satellite decay alerts, space debris.
|
|
4
4
|
|
|
5
|
+
import { Agent } from 'undici';
|
|
5
6
|
import { safeFetch } from '../utils/fetch.mjs';
|
|
7
|
+
import { readQuota, writeQuota, clearQuota } from '../utils/quota-cache.mjs';
|
|
6
8
|
|
|
7
9
|
const CELESTRAK_BASE = 'https://celestrak.org';
|
|
10
|
+
const CACHE_MS = 2 * 60 * 60 * 1000; // GP data updates about every two hours
|
|
11
|
+
const LAST_GOOD_MS = 24 * 60 * 60 * 1000;
|
|
12
|
+
const celestrakAgent = new Agent({ connect: { timeout: 20000, family: 4 }, connections: 3 });
|
|
13
|
+
|
|
14
|
+
let lastGood = { at: 0, result: null };
|
|
15
|
+
let hydrated = false;
|
|
16
|
+
|
|
17
|
+
function hydrateQuota() {
|
|
18
|
+
if (hydrated) return;
|
|
19
|
+
hydrated = true;
|
|
20
|
+
const saved = readQuota('celestrak');
|
|
21
|
+
if (Number(saved?.at) > 0 && saved.result) lastGood = { at: saved.at, result: saved.result };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function resetState({ disk = true } = {}) {
|
|
25
|
+
lastGood = { at: 0, result: null };
|
|
26
|
+
hydrated = false;
|
|
27
|
+
if (disk) clearQuota('celestrak');
|
|
28
|
+
}
|
|
8
29
|
|
|
9
30
|
// Satellite categories for monitoring
|
|
10
31
|
const SAT_CATEGORIES = {
|
|
@@ -20,7 +41,9 @@ const SAT_CATEGORIES = {
|
|
|
20
41
|
async function getTLEs(category) {
|
|
21
42
|
const path = SAT_CATEGORIES[category];
|
|
22
43
|
if (!path) return { error: 'Invalid category' };
|
|
23
|
-
const data = await safeFetch(`${CELESTRAK_BASE}${path}`, {
|
|
44
|
+
const data = await safeFetch(`${CELESTRAK_BASE}${path}`, {
|
|
45
|
+
timeout: 25000, retries: 0, dispatcher: celestrakAgent,
|
|
46
|
+
});
|
|
24
47
|
return data;
|
|
25
48
|
}
|
|
26
49
|
|
|
@@ -95,16 +118,12 @@ async function getMilitaryCount() {
|
|
|
95
118
|
return { count: data.length, byCountry };
|
|
96
119
|
}
|
|
97
120
|
|
|
98
|
-
//
|
|
99
|
-
|
|
100
|
-
const
|
|
101
|
-
getTLEs('starlink'),
|
|
102
|
-
getTLEs('oneweb'),
|
|
103
|
-
]);
|
|
104
|
-
|
|
121
|
+
// Mega-lists (Starlink especially) trip CelesTrak's usage limits. Reuse the last counted totals.
|
|
122
|
+
function getConstellationStats() {
|
|
123
|
+
const saved = readQuota('celestrak-constellations');
|
|
105
124
|
return {
|
|
106
|
-
starlink:
|
|
107
|
-
oneweb:
|
|
125
|
+
starlink: Number(saved?.starlink) || 0,
|
|
126
|
+
oneweb: Number(saved?.oneweb) || 0,
|
|
108
127
|
};
|
|
109
128
|
}
|
|
110
129
|
|
|
@@ -137,21 +156,28 @@ function generateSignals(data) {
|
|
|
137
156
|
}
|
|
138
157
|
|
|
139
158
|
// Briefing export
|
|
140
|
-
export async function briefing() {
|
|
159
|
+
export async function briefing({ now = Date.now() } = {}) {
|
|
160
|
+
hydrateQuota();
|
|
161
|
+
if (lastGood.result && now >= lastGood.at && now - lastGood.at < CACHE_MS) {
|
|
162
|
+
return { ...lastGood.result, stale: true, message: `Cached CelesTrak result; next request after ${new Date(lastGood.at + CACHE_MS).toISOString()}` };
|
|
163
|
+
}
|
|
141
164
|
try {
|
|
142
|
-
const [launches, stations, military
|
|
165
|
+
const [launches, stations, military] = await Promise.all([
|
|
143
166
|
getRecentLaunches(),
|
|
144
167
|
getStationData(),
|
|
145
168
|
getMilitaryCount(),
|
|
146
|
-
getConstellationStats(),
|
|
147
169
|
]);
|
|
170
|
+
const constellations = getConstellationStats();
|
|
148
171
|
|
|
149
172
|
const hasData = !launches.error || !stations.error;
|
|
150
173
|
|
|
151
174
|
if (!hasData) {
|
|
175
|
+
if (lastGood.result && now - lastGood.at < LAST_GOOD_MS) {
|
|
176
|
+
return { ...lastGood.result, stale: true, message: launches.error || stations.error || 'Failed to fetch space data' };
|
|
177
|
+
}
|
|
152
178
|
return {
|
|
153
179
|
source: 'Space/CelesTrak',
|
|
154
|
-
timestamp: new Date().toISOString(),
|
|
180
|
+
timestamp: new Date(now).toISOString(),
|
|
155
181
|
status: 'error',
|
|
156
182
|
error: launches.error || stations.error || 'Failed to fetch space data',
|
|
157
183
|
};
|
|
@@ -160,9 +186,9 @@ export async function briefing() {
|
|
|
160
186
|
const data = { launches, stations, military, constellations };
|
|
161
187
|
const signals = generateSignals(data);
|
|
162
188
|
|
|
163
|
-
|
|
189
|
+
const result = {
|
|
164
190
|
source: 'Space/CelesTrak',
|
|
165
|
-
timestamp: new Date().toISOString(),
|
|
191
|
+
timestamp: new Date(now).toISOString(),
|
|
166
192
|
status: 'active',
|
|
167
193
|
recentLaunches: launches.recentLaunches || [],
|
|
168
194
|
totalNewObjects: launches.totalObjects || 0,
|
|
@@ -174,10 +200,16 @@ export async function briefing() {
|
|
|
174
200
|
constellations: constellations || {},
|
|
175
201
|
signals,
|
|
176
202
|
};
|
|
203
|
+
lastGood = { at: now, result };
|
|
204
|
+
writeQuota('celestrak', lastGood);
|
|
205
|
+
return result;
|
|
177
206
|
} catch (e) {
|
|
207
|
+
if (lastGood.result && now - lastGood.at < LAST_GOOD_MS) {
|
|
208
|
+
return { ...lastGood.result, stale: true, message: e.message };
|
|
209
|
+
}
|
|
178
210
|
return {
|
|
179
211
|
source: 'Space/CelesTrak',
|
|
180
|
-
timestamp: new Date().toISOString(),
|
|
212
|
+
timestamp: new Date(now).toISOString(),
|
|
181
213
|
status: 'error',
|
|
182
214
|
error: e.message,
|
|
183
215
|
};
|
package/apis/utils/fetch.mjs
CHANGED
|
@@ -32,7 +32,8 @@ export async function readBoundedText(response, maxBytes = 10 * 1024 * 1024) {
|
|
|
32
32
|
|
|
33
33
|
export async function safeFetch(url, opts = {}) {
|
|
34
34
|
const { timeout = 15000, retries = 1, headers = {}, maxBytes = 10 * 1024 * 1024,
|
|
35
|
-
format = 'json', method = 'GET', body, retryDelay = 2000, maxRetryDelay = 30000, retryAfterHeader = 'retry-after'
|
|
35
|
+
format = 'json', method = 'GET', body, retryDelay = 2000, maxRetryDelay = 30000, retryAfterHeader = 'retry-after',
|
|
36
|
+
dispatcher } = opts;
|
|
36
37
|
let lastError;
|
|
37
38
|
let lastStatus;
|
|
38
39
|
let lastRetryAfterMs = null;
|
|
@@ -45,6 +46,7 @@ export async function safeFetch(url, opts = {}) {
|
|
|
45
46
|
const response = await fetch(url, {
|
|
46
47
|
signal: controller.signal, method, body,
|
|
47
48
|
headers: { 'User-Agent': 'Crucix/2.2', ...headers },
|
|
49
|
+
...(dispatcher ? { dispatcher } : {}),
|
|
48
50
|
});
|
|
49
51
|
lastStatus = response.status;
|
|
50
52
|
if (!response.ok) {
|
|
@@ -63,7 +65,9 @@ export async function safeFetch(url, opts = {}) {
|
|
|
63
65
|
try { return JSON.parse(text); }
|
|
64
66
|
catch { return { error: 'Invalid JSON response', status: response.status }; }
|
|
65
67
|
} catch (error) {
|
|
66
|
-
|
|
68
|
+
const cause = error?.cause?.code || (typeof error?.cause?.message === 'string' ? error.cause.message : '');
|
|
69
|
+
lastError = controller.signal.aborted ? new Error(`Request timed out after ${timeout}ms`)
|
|
70
|
+
: new Error(cause && !String(error.message).includes(cause) ? `${error.message} (${cause})` : error.message);
|
|
67
71
|
} finally { clearTimeout(timer); }
|
|
68
72
|
if (!retryable || attempt === retries) break;
|
|
69
73
|
await new Promise(resolve => setTimeout(resolve, Math.min(maxRetryDelay, waitMs)));
|
package/apis/utils/freshness.mjs
CHANGED
|
@@ -32,7 +32,7 @@ export const POLICIES = Object.freeze({
|
|
|
32
32
|
'FAA-Airports': { maxAgeMs: 3*HOUR, observationMaxAgeMs: 3*HOUR },
|
|
33
33
|
Regulators: { maxAgeMs: 168*HOUR, observationMaxAgeMs: 336*HOUR },
|
|
34
34
|
'Eurostat-HU': { maxAgeMs: 2160*HOUR, observationMaxAgeMs: 4320*HOUR },
|
|
35
|
-
'Launch-Library': { maxAgeMs:
|
|
35
|
+
'Launch-Library': { maxAgeMs: 72*HOUR, observationMaxAgeMs: 720*HOUR },
|
|
36
36
|
Maritime: { maxAgeMs: 25*60000, observationMaxAgeMs: 25*60000 },
|
|
37
37
|
});
|
|
38
38
|
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// Small JSON files in CRUCIX_HOME/cache so daily quotas survive a process restart.
|
|
2
|
+
// Tests use isolate-profile.mjs, so writes stay in a temp home.
|
|
3
|
+
|
|
4
|
+
import { mkdirSync, readFileSync, writeFileSync, unlinkSync } from 'node:fs';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { crucixHome } from '../../lib/user-paths.mjs';
|
|
7
|
+
|
|
8
|
+
const NAME = /^[a-z0-9-]{1,40}$/;
|
|
9
|
+
const MAX_BYTES = 256 * 1024;
|
|
10
|
+
|
|
11
|
+
export function quotaCachePath(name, options) {
|
|
12
|
+
if (!NAME.test(name)) throw new Error(`invalid quota cache name: ${name}`);
|
|
13
|
+
return join(crucixHome(options), 'cache', `${name}.json`);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function readQuota(name, options) {
|
|
17
|
+
try {
|
|
18
|
+
const raw = readFileSync(quotaCachePath(name, options), 'utf8');
|
|
19
|
+
if (raw.length > MAX_BYTES) return null;
|
|
20
|
+
const data = JSON.parse(raw);
|
|
21
|
+
return data && typeof data === 'object' && !Array.isArray(data) ? data : null;
|
|
22
|
+
} catch {
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function writeQuota(name, value, options) {
|
|
28
|
+
try {
|
|
29
|
+
const dir = join(crucixHome(options), 'cache');
|
|
30
|
+
mkdirSync(dir, { recursive: true });
|
|
31
|
+
const text = JSON.stringify(value);
|
|
32
|
+
if (text.length > MAX_BYTES) return false;
|
|
33
|
+
writeFileSync(quotaCachePath(name, options), text);
|
|
34
|
+
return true;
|
|
35
|
+
} catch {
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function clearQuota(name, options) {
|
|
41
|
+
try { unlinkSync(quotaCachePath(name, options)); } catch { /* missing is fine */ }
|
|
42
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// Shared RSS worker pool. A 15-minute sweep used to start 52 core feeds at once while
|
|
2
|
+
// the world runner already had 35 connections open; on Windows a broken IPv6 path then
|
|
3
|
+
// burned the whole 8 s abort before undici could fall back to IPv4, so every feed "timed out".
|
|
4
|
+
|
|
5
|
+
import { Agent } from 'undici';
|
|
6
|
+
|
|
7
|
+
export const RSS_CONCURRENCY = 8;
|
|
8
|
+
export const RSS_TIMEOUT_MS = 12000;
|
|
9
|
+
|
|
10
|
+
export const rssAgent = new Agent({
|
|
11
|
+
connections: RSS_CONCURRENCY,
|
|
12
|
+
pipelining: 0,
|
|
13
|
+
connect: {
|
|
14
|
+
timeout: 8000,
|
|
15
|
+
family: 4,
|
|
16
|
+
autoSelectFamily: true,
|
|
17
|
+
autoSelectFamilyAttemptTimeout: 250,
|
|
18
|
+
},
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
export async function mapLimited(items, limit, fn) {
|
|
22
|
+
const out = new Array(items.length);
|
|
23
|
+
let next = 0;
|
|
24
|
+
const worker = async () => {
|
|
25
|
+
while (next < items.length) {
|
|
26
|
+
const index = next++;
|
|
27
|
+
try { out[index] = { status: 'fulfilled', value: await fn(items[index], index) }; }
|
|
28
|
+
catch (reason) { out[index] = { status: 'rejected', reason }; }
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
await Promise.all(Array.from({ length: Math.min(Math.max(1, limit), items.length || 1) }, worker));
|
|
32
|
+
return items.length ? out : [];
|
|
33
|
+
}
|
package/crucix.config.mjs
CHANGED
|
@@ -126,7 +126,7 @@ export default {
|
|
|
126
126
|
anthropicAuthToken: process.env.ANTHROPIC_AUTH_TOKEN || null,
|
|
127
127
|
reasoningEffort: process.env.OLLAMA_REASONING_EFFORT || null,
|
|
128
128
|
ideasMaxTokens: envInteger('LLM_IDEAS_MAX_TOKENS', 4096, 128, 16384),
|
|
129
|
-
ideasTimeoutMs: envInteger('LLM_IDEAS_TIMEOUT_MS',
|
|
129
|
+
ideasTimeoutMs: envInteger('LLM_IDEAS_TIMEOUT_MS', 180000, 1000, 360000),
|
|
130
130
|
alertMaxTokens: envInteger('LLM_ALERT_MAX_TOKENS', 800, 128, 4096),
|
|
131
131
|
alertTimeoutMs: envInteger('LLM_ALERT_TIMEOUT_MS', 30000, 1000, 360000),
|
|
132
132
|
briefingMaxTokens: envInteger('LLM_BRIEFING_MAX_TOKENS', 1200, 128, 8192), // cited briefings (getLLMBudget(config, 'briefing'))
|
package/dashboard/inject.mjs
CHANGED
|
@@ -14,6 +14,7 @@ import { openBrowser } from '../lib/open-browser.mjs';
|
|
|
14
14
|
import { inlineJson } from '../lib/html.mjs';
|
|
15
15
|
import { safeFetch } from '../apis/utils/fetch.mjs';
|
|
16
16
|
import { parseFeed } from '../apis/utils/rss.mjs';
|
|
17
|
+
import { mapLimited, rssAgent, RSS_CONCURRENCY, RSS_TIMEOUT_MS } from '../apis/utils/rss-pool.mjs';
|
|
17
18
|
import { vesselName, mmsiOf } from '../apis/utils/ais-collector.mjs';
|
|
18
19
|
import { feedBySource, HUNGARIAN_SOURCES, OFFICIAL_SOURCES } from '../apis/utils/news-feeds.mjs';
|
|
19
20
|
import { activeNewsFeeds } from '../lib/news/feed-store.mjs';
|
|
@@ -345,19 +346,23 @@ export function loadOpenSkyFallback(currentTimestamp, runsDir = config.runsDir |
|
|
|
345
346
|
}
|
|
346
347
|
|
|
347
348
|
// === RSS Fetching ===
|
|
349
|
+
let lastCoreNews = [];
|
|
350
|
+
export function resetNewsCache() { lastCoreNews = []; }
|
|
351
|
+
|
|
348
352
|
async function fetchRSS(url, source, meta = {}) {
|
|
349
353
|
try {
|
|
350
|
-
const response = await safeFetch(url, {
|
|
354
|
+
const response = await safeFetch(url, {
|
|
355
|
+
timeout: RSS_TIMEOUT_MS, retries: 0, format: 'text', maxBytes: 2 * 1024 * 1024, dispatcher: rssAgent,
|
|
356
|
+
});
|
|
351
357
|
if (response.error) throw new Error(response.error);
|
|
352
358
|
const items = [];
|
|
353
359
|
for (const item of parseFeed(response.rawText)) {
|
|
354
360
|
const link = sanitizeExternalUrl(item.link);
|
|
355
361
|
if (item.title !== source) items.push({ title: item.title, date: item.date, source, url: link || undefined, ...(meta.tier ? { tier: meta.tier } : {}), ...(meta.lang && meta.lang !== 'en' ? { lang: meta.lang } : {}), ...(meta.state ? { state: true } : {}), ...(meta.country ? { country: meta.country } : {}) });
|
|
356
362
|
}
|
|
357
|
-
return items.slice(0, MAX_PER_FEED);
|
|
363
|
+
return { source, items: items.slice(0, MAX_PER_FEED), error: null };
|
|
358
364
|
} catch (e) {
|
|
359
|
-
|
|
360
|
-
return [];
|
|
365
|
+
return { source, items: [], error: e.message };
|
|
361
366
|
}
|
|
362
367
|
}
|
|
363
368
|
|
|
@@ -369,16 +374,23 @@ const PER_SOURCE_LIMIT = 5;
|
|
|
369
374
|
|
|
370
375
|
// `customFeeds` takes [url, source] pairs (tests, local overrides); the default is the tiered registry minus the feeds
|
|
371
376
|
// switched off in the dashboard, plus the operator's own feeds (the profile feeds.json, lib/news/feed-store.mjs).
|
|
372
|
-
export async function fetchAllNews(customFeeds) {
|
|
377
|
+
export async function fetchAllNews(customFeeds, { now = Date.now(), concurrency = RSS_CONCURRENCY, budgetMs = 20000 } = {}) {
|
|
373
378
|
const feeds = customFeeds ? customFeeds.map(([url, source]) => ({ ...feedBySource(source), url, source })) : activeNewsFeeds();
|
|
379
|
+
const deadline = now + budgetMs;
|
|
380
|
+
const results = await mapLimited(feeds, concurrency, async feed => {
|
|
381
|
+
if (Date.now() > deadline) return { source: feed.source, items: [], error: 'skipped after RSS budget' };
|
|
382
|
+
return fetchRSS(feed.url, feed.source, feed);
|
|
383
|
+
});
|
|
374
384
|
|
|
375
|
-
const
|
|
376
|
-
|
|
377
|
-
);
|
|
385
|
+
const fetched = results.filter(r => r.status === 'fulfilled').map(r => r.value);
|
|
386
|
+
const failed = fetched.filter(r => r.error);
|
|
387
|
+
const ok = fetched.filter(r => !r.error && r.items.length);
|
|
388
|
+
if (failed.length || fetched.length !== feeds.length) {
|
|
389
|
+
const names = failed.slice(0, 6).map(r => r.source);
|
|
390
|
+
console.log(`RSS: ${ok.length}/${feeds.length} feeds ok, ${failed.length} failed${names.length ? ` (${names.join(', ')}${failed.length > 6 ? '…' : ''})` : ''}; ${concurrency} workers, ${RSS_TIMEOUT_MS / 1000}s each`);
|
|
391
|
+
}
|
|
378
392
|
|
|
379
|
-
const allNews =
|
|
380
|
-
.filter(r => r.status === 'fulfilled')
|
|
381
|
-
.flatMap(r => r.value);
|
|
393
|
+
const allNews = fetched.flatMap(r => r.items);
|
|
382
394
|
|
|
383
395
|
// De-duplicate and geo-tag
|
|
384
396
|
const seen = new Set();
|
|
@@ -433,7 +445,16 @@ export async function fetchAllNews(customFeeds) {
|
|
|
433
445
|
counts.set(item.source, (counts.get(item.source) || 0) + 1);
|
|
434
446
|
}
|
|
435
447
|
selected.sort((a, b) => new Date(b.date || 0) - new Date(a.date || 0));
|
|
436
|
-
|
|
448
|
+
const next = selected.slice(0, NEWS_LIMIT);
|
|
449
|
+
if (next.length) {
|
|
450
|
+
lastCoreNews = next;
|
|
451
|
+
return next;
|
|
452
|
+
}
|
|
453
|
+
if (lastCoreNews.length) {
|
|
454
|
+
console.log(`RSS: no live headlines; keeping ${lastCoreNews.length} from the last successful fetch`);
|
|
455
|
+
return lastCoreNews;
|
|
456
|
+
}
|
|
457
|
+
return next;
|
|
437
458
|
}
|
|
438
459
|
|
|
439
460
|
// === Leverageable Ideas from Signals ===
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
(function(window){
|
|
2
2
|
'use strict';
|
|
3
3
|
const HOUR=3600000;
|
|
4
|
-
const policies={Meteoalarm:{maxAgeMs:3*HOUR,observationMaxAgeMs:48*HOUR},GDACS:{maxAgeMs:6*HOUR,observationMaxAgeMs:72*HOUR},'NOAA-SWPC':{maxAgeMs:HOUR},ECB:{maxAgeMs:120*HOUR},'NASA-EONET':{maxAgeMs:72*HOUR},RIPEstat:{maxAgeMs:12*HOUR},'FIRST-EPSS':{maxAgeMs:48*HOUR},'MET-Norway':{maxAgeMs:8*HOUR},OONI:{maxAgeMs:24*HOUR},'IMF-PortWatch':{maxAgeMs:240*HOUR,observationMaxAgeMs:240*HOUR},EMSC:{maxAgeMs:12*HOUR,observationMaxAgeMs:26*HOUR},'Copernicus-EMS':{maxAgeMs:1080*HOUR,observationMaxAgeMs:720*HOUR},'Aviation-SIGMET':{maxAgeMs:3*HOUR,observationMaxAgeMs:24*HOUR},'ADSB-Military':{maxAgeMs:25*60000,observationMaxAgeMs:25*60000},'OpenSanctions-Index':{maxAgeMs:48*HOUR,observationMaxAgeMs:336*HOUR},'Federal-Register':{maxAgeMs:336*HOUR,observationMaxAgeMs:336*HOUR},'Energy-Charts-HU':{maxAgeMs:6*HOUR,observationMaxAgeMs:6*HOUR},'ENTSOG-HU':{maxAgeMs:72*HOUR,observationMaxAgeMs:72*HOUR},'Prediction-Markets':{maxAgeMs:12*HOUR,observationMaxAgeMs:12*HOUR},ThreatFox:{maxAgeMs:6*HOUR,observationMaxAgeMs:24*HOUR},HIBP:{maxAgeMs:336*HOUR,observationMaxAgeMs:720*HOUR},GPSJam:{maxAgeMs:96*HOUR,observationMaxAgeMs:96*HOUR},'WMO-SWIC':{maxAgeMs:3*HOUR,observationMaxAgeMs:24*HOUR},'UNHCR-Arrivals':{maxAgeMs:504*HOUR,observationMaxAgeMs:1440*HOUR},'SEC-8K':{maxAgeMs:2160*HOUR,observationMaxAgeMs:2160*HOUR},'Open-Meteo-Wind':{maxAgeMs:3*HOUR,observationMaxAgeMs:3*HOUR},'ADSB-Orbits':{maxAgeMs:25*60000,observationMaxAgeMs:25*60000},'NOAA-NHC':{maxAgeMs:8*HOUR,observationMaxAgeMs:12*HOUR},'JMA-Typhoon':{maxAgeMs:8*HOUR,observationMaxAgeMs:12*HOUR},'ECDC-Threats':{maxAgeMs:336*HOUR,observationMaxAgeMs:504*HOUR},'Central-Banks':{maxAgeMs:336*HOUR,observationMaxAgeMs:2880*HOUR},'CFTC-COT':{maxAgeMs:288*HOUR,observationMaxAgeMs:288*HOUR},'FAA-Airports':{maxAgeMs:3*HOUR,observationMaxAgeMs:3*HOUR},Regulators:{maxAgeMs:168*HOUR,observationMaxAgeMs:336*HOUR},'Eurostat-HU':{maxAgeMs:2160*HOUR,observationMaxAgeMs:4320*HOUR},'Launch-Library':{maxAgeMs:
|
|
4
|
+
const policies={Meteoalarm:{maxAgeMs:3*HOUR,observationMaxAgeMs:48*HOUR},GDACS:{maxAgeMs:6*HOUR,observationMaxAgeMs:72*HOUR},'NOAA-SWPC':{maxAgeMs:HOUR},ECB:{maxAgeMs:120*HOUR},'NASA-EONET':{maxAgeMs:72*HOUR},RIPEstat:{maxAgeMs:12*HOUR},'FIRST-EPSS':{maxAgeMs:48*HOUR},'MET-Norway':{maxAgeMs:8*HOUR},OONI:{maxAgeMs:24*HOUR},'IMF-PortWatch':{maxAgeMs:240*HOUR,observationMaxAgeMs:240*HOUR},EMSC:{maxAgeMs:12*HOUR,observationMaxAgeMs:26*HOUR},'Copernicus-EMS':{maxAgeMs:1080*HOUR,observationMaxAgeMs:720*HOUR},'Aviation-SIGMET':{maxAgeMs:3*HOUR,observationMaxAgeMs:24*HOUR},'ADSB-Military':{maxAgeMs:25*60000,observationMaxAgeMs:25*60000},'OpenSanctions-Index':{maxAgeMs:48*HOUR,observationMaxAgeMs:336*HOUR},'Federal-Register':{maxAgeMs:336*HOUR,observationMaxAgeMs:336*HOUR},'Energy-Charts-HU':{maxAgeMs:6*HOUR,observationMaxAgeMs:6*HOUR},'ENTSOG-HU':{maxAgeMs:72*HOUR,observationMaxAgeMs:72*HOUR},'Prediction-Markets':{maxAgeMs:12*HOUR,observationMaxAgeMs:12*HOUR},ThreatFox:{maxAgeMs:6*HOUR,observationMaxAgeMs:24*HOUR},HIBP:{maxAgeMs:336*HOUR,observationMaxAgeMs:720*HOUR},GPSJam:{maxAgeMs:96*HOUR,observationMaxAgeMs:96*HOUR},'WMO-SWIC':{maxAgeMs:3*HOUR,observationMaxAgeMs:24*HOUR},'UNHCR-Arrivals':{maxAgeMs:504*HOUR,observationMaxAgeMs:1440*HOUR},'SEC-8K':{maxAgeMs:2160*HOUR,observationMaxAgeMs:2160*HOUR},'Open-Meteo-Wind':{maxAgeMs:3*HOUR,observationMaxAgeMs:3*HOUR},'ADSB-Orbits':{maxAgeMs:25*60000,observationMaxAgeMs:25*60000},'NOAA-NHC':{maxAgeMs:8*HOUR,observationMaxAgeMs:12*HOUR},'JMA-Typhoon':{maxAgeMs:8*HOUR,observationMaxAgeMs:12*HOUR},'ECDC-Threats':{maxAgeMs:336*HOUR,observationMaxAgeMs:504*HOUR},'Central-Banks':{maxAgeMs:336*HOUR,observationMaxAgeMs:2880*HOUR},'CFTC-COT':{maxAgeMs:288*HOUR,observationMaxAgeMs:288*HOUR},'FAA-Airports':{maxAgeMs:3*HOUR,observationMaxAgeMs:3*HOUR},Regulators:{maxAgeMs:168*HOUR,observationMaxAgeMs:336*HOUR},'Eurostat-HU':{maxAgeMs:2160*HOUR,observationMaxAgeMs:4320*HOUR},'Launch-Library':{maxAgeMs:72*HOUR,observationMaxAgeMs:720*HOUR},Maritime:{maxAgeMs:25*60000,observationMaxAgeMs:25*60000}};
|
|
5
5
|
const esc=value=>String(value??'').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
|
|
6
6
|
function time(value){
|
|
7
7
|
if(typeof value!=='string'||value.length>128)return NaN;
|
package/lib/llm/budgets.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Explicit request budgets keep both remote costs and slow local inference bounded.
|
|
2
2
|
export const LLM_BUDGETS = Object.freeze({
|
|
3
|
-
ideas: Object.freeze({ tokens: 4096, timeout:
|
|
3
|
+
ideas: Object.freeze({ tokens: 4096, timeout: 180000, minTokens: 128, maxTokens: 16384 }),
|
|
4
4
|
alerts: Object.freeze({ tokens: 800, timeout: 30000, minTokens: 128, maxTokens: 4096 }),
|
|
5
5
|
briefing: Object.freeze({ tokens: 1200, timeout: 60000, minTokens: 128, maxTokens: 8192 }),
|
|
6
6
|
ask: Object.freeze({ tokens: 500, timeout: 45000, minTokens: 64, maxTokens: 2048 }),
|
package/lib/llm/cursor.mjs
CHANGED
|
@@ -17,7 +17,7 @@ export class CursorProvider extends LLMProvider {
|
|
|
17
17
|
}
|
|
18
18
|
|
|
19
19
|
async complete(systemPrompt, userMessage, opts = {}) {
|
|
20
|
-
const budget = this.requestOptions(opts, { timeout:
|
|
20
|
+
const budget = this.requestOptions(opts, { timeout: 180000 });
|
|
21
21
|
const isWin = process.platform === 'win32';
|
|
22
22
|
const [cmd, args] = isWin
|
|
23
23
|
? [process.env.COMSPEC || 'cmd.exe', ['/d', '/s', '/c', 'agent', '-p', '--trust', '--mode', 'ask', '--model', this.model]]
|
|
@@ -26,6 +26,7 @@ export class CursorProvider extends LLMProvider {
|
|
|
26
26
|
const input = systemPrompt ? `${systemPrompt}\n\n${userMessage}` : userMessage;
|
|
27
27
|
|
|
28
28
|
return new Promise((resolve, reject) => {
|
|
29
|
+
const started = Date.now();
|
|
29
30
|
let child;
|
|
30
31
|
try {
|
|
31
32
|
child = spawn(cmd, args, { stdio: ['pipe', 'pipe', 'pipe'] });
|
|
@@ -36,18 +37,22 @@ export class CursorProvider extends LLMProvider {
|
|
|
36
37
|
let stdout = '';
|
|
37
38
|
let stderr = '';
|
|
38
39
|
let timedOut = false;
|
|
40
|
+
let firstByteAt = 0;
|
|
39
41
|
|
|
40
42
|
const timer = setTimeout(() => {
|
|
41
43
|
timedOut = true;
|
|
42
44
|
child.kill();
|
|
43
|
-
|
|
45
|
+
const waited = Date.now() - started;
|
|
46
|
+
reject(new Error(`Cursor Agent CLI timed out after ${budget.timeout}ms (waited ${waited}ms, first byte ${firstByteAt ? `${firstByteAt - started}ms` : 'never'})`));
|
|
44
47
|
}, budget.timeout);
|
|
45
48
|
|
|
46
49
|
child.stdout.on('data', (chunk) => {
|
|
50
|
+
if (!firstByteAt) firstByteAt = Date.now();
|
|
47
51
|
stdout += chunk;
|
|
48
52
|
});
|
|
49
53
|
|
|
50
54
|
child.stderr.on('data', (chunk) => {
|
|
55
|
+
if (!firstByteAt) firstByteAt = Date.now();
|
|
51
56
|
stderr += chunk;
|
|
52
57
|
});
|
|
53
58
|
|
|
@@ -59,6 +64,10 @@ export class CursorProvider extends LLMProvider {
|
|
|
59
64
|
child.on('close', (code) => {
|
|
60
65
|
clearTimeout(timer);
|
|
61
66
|
if (timedOut) return;
|
|
67
|
+
const timing = {
|
|
68
|
+
ms: Date.now() - started,
|
|
69
|
+
firstByteMs: firstByteAt ? firstByteAt - started : null,
|
|
70
|
+
};
|
|
62
71
|
|
|
63
72
|
if (code !== 0) {
|
|
64
73
|
const errMsg = stderr.trim() || stdout.trim() || `exited with code ${code}`;
|
|
@@ -69,6 +78,7 @@ export class CursorProvider extends LLMProvider {
|
|
|
69
78
|
text: stdout.trim(),
|
|
70
79
|
usage: { inputTokens: 0, outputTokens: 0 },
|
|
71
80
|
model: this.model,
|
|
81
|
+
timing,
|
|
72
82
|
});
|
|
73
83
|
});
|
|
74
84
|
|
package/lib/version.mjs
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
|
|
5
|
+
const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
|
|
6
|
+
|
|
7
|
+
/** The package.json version, or CRUCIX_VERSION when the SEA binary has no package.json beside it. */
|
|
8
|
+
export function packageVersion() {
|
|
9
|
+
const baked = process.env.CRUCIX_VERSION;
|
|
10
|
+
if (baked) return baked;
|
|
11
|
+
const pkgPath = join(ROOT, 'package.json');
|
|
12
|
+
if (!existsSync(pkgPath)) return '0.0.0';
|
|
13
|
+
try {
|
|
14
|
+
return String(JSON.parse(readFileSync(pkgPath, 'utf8')).version || '0.0.0');
|
|
15
|
+
} catch {
|
|
16
|
+
return '0.0.0';
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function versionLine() {
|
|
21
|
+
return `crucix ${packageVersion()}`;
|
|
22
|
+
}
|
package/lib/world-rss-runner.mjs
CHANGED
|
@@ -3,6 +3,7 @@ import { fileURLToPath } from 'node:url';
|
|
|
3
3
|
import { existsSync, readFileSync } from 'node:fs';
|
|
4
4
|
import { WORLD_FEEDS } from '../apis/utils/news-feeds.mjs';
|
|
5
5
|
import { safeFetch } from '../apis/utils/fetch.mjs';
|
|
6
|
+
import { mapLimited, rssAgent, RSS_CONCURRENCY, RSS_TIMEOUT_MS } from '../apis/utils/rss-pool.mjs';
|
|
6
7
|
import { activeWorldFeeds } from './news/feed-store.mjs';
|
|
7
8
|
import { parseFeed } from '../apis/utils/rss.mjs';
|
|
8
9
|
import { writeJsonAtomic } from './atomic-json.mjs';
|
|
@@ -15,8 +16,8 @@ export function resolveCacheFile() {
|
|
|
15
16
|
return join(resolveRunsDir({ root: ROOT }), 'world-rss-cache.json');
|
|
16
17
|
}
|
|
17
18
|
const CACHE_FILE = resolveCacheFile();
|
|
18
|
-
const TIMEOUT_MS =
|
|
19
|
-
const CONCURRENCY =
|
|
19
|
+
const TIMEOUT_MS = RSS_TIMEOUT_MS;
|
|
20
|
+
const CONCURRENCY = RSS_CONCURRENCY;
|
|
20
21
|
const MAX_PER_FEED = 10;
|
|
21
22
|
const MAX_AGE_MS = 48 * 60 * 60 * 1000; // 48 hours
|
|
22
23
|
|
|
@@ -56,7 +57,7 @@ export function loadWorldRssCache() {
|
|
|
56
57
|
|
|
57
58
|
async function fetchFeed(f) {
|
|
58
59
|
try {
|
|
59
|
-
const res = await safeFetch(f.url, { timeout: TIMEOUT_MS, retries: 0, format: 'text', maxBytes: 1024 * 1024 });
|
|
60
|
+
const res = await safeFetch(f.url, { timeout: TIMEOUT_MS, retries: 0, format: 'text', maxBytes: 1024 * 1024, dispatcher: rssAgent });
|
|
60
61
|
if (res.error) return [];
|
|
61
62
|
const parsed = parseFeed(res.rawText);
|
|
62
63
|
const now = Date.now();
|
|
@@ -89,26 +90,20 @@ export async function runWorldRssSweep({ feeds = activeWorldFeeds() } = {}) {
|
|
|
89
90
|
const start = Date.now();
|
|
90
91
|
console.log(`[WorldRSS] Starting background sweep for ${feeds.length} world feeds...`);
|
|
91
92
|
|
|
92
|
-
const queue = [...feeds];
|
|
93
93
|
const allResults = [];
|
|
94
94
|
let okCount = 0;
|
|
95
95
|
let failCount = 0;
|
|
96
96
|
|
|
97
|
-
const
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
} else {
|
|
106
|
-
failCount++;
|
|
107
|
-
}
|
|
97
|
+
const rows = await mapLimited(feeds, CONCURRENCY, fetchFeed);
|
|
98
|
+
for (const row of rows) {
|
|
99
|
+
const items = row.status === 'fulfilled' ? row.value : [];
|
|
100
|
+
if (items.length > 0) {
|
|
101
|
+
okCount++;
|
|
102
|
+
allResults.push(...items);
|
|
103
|
+
} else {
|
|
104
|
+
failCount++;
|
|
108
105
|
}
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
await Promise.all(workers);
|
|
106
|
+
}
|
|
112
107
|
|
|
113
108
|
// De-duplicate headlines
|
|
114
109
|
const seen = new Set();
|
|
@@ -123,6 +118,23 @@ export async function runWorldRssSweep({ feeds = activeWorldFeeds() } = {}) {
|
|
|
123
118
|
// Sort newest first
|
|
124
119
|
uniqueItems.sort((a, b) => new Date(b.date || 0) - new Date(a.date || 0));
|
|
125
120
|
|
|
121
|
+
if (!uniqueItems.length && cachedItems.length) {
|
|
122
|
+
const durationMs = Date.now() - start;
|
|
123
|
+
lastSweepTime = new Date().toISOString();
|
|
124
|
+
lastStats = {
|
|
125
|
+
totalFeeds: feeds.length,
|
|
126
|
+
catalogFeeds: WORLD_FEEDS.length,
|
|
127
|
+
ok: 0,
|
|
128
|
+
failed: failCount,
|
|
129
|
+
totalItems: cachedItems.length,
|
|
130
|
+
durationMs,
|
|
131
|
+
keptCache: true,
|
|
132
|
+
};
|
|
133
|
+
console.log(`[WorldRSS] Sweep produced no headlines (${failCount}/${feeds.length} failed in ${(durationMs / 1000).toFixed(1)}s); keeping ${cachedItems.length} cached items`);
|
|
134
|
+
isFetching = false;
|
|
135
|
+
return { ok: true, stats: lastStats };
|
|
136
|
+
}
|
|
137
|
+
|
|
126
138
|
cachedItems = uniqueItems;
|
|
127
139
|
rebuildCountryIndex();
|
|
128
140
|
|
|
@@ -201,7 +213,7 @@ export function startWorldRssScheduler(intervalMinutes = 30) {
|
|
|
201
213
|
if (shouldRunNow) {
|
|
202
214
|
setTimeout(() => {
|
|
203
215
|
runWorldRssSweep().catch(err => console.warn('[WorldRSS] Background sweep error:', err.message));
|
|
204
|
-
},
|
|
216
|
+
}, 45000).unref();
|
|
205
217
|
}
|
|
206
218
|
|
|
207
219
|
const timer = setInterval(() => {
|
package/package.json
CHANGED
package/scripts/cli.mjs
CHANGED
|
@@ -21,6 +21,7 @@ import { runSetupWizard } from '../lib/setup-wizard.mjs';
|
|
|
21
21
|
import { getEnvSearchPaths } from '../apis/utils/env.mjs';
|
|
22
22
|
import { installUserProfile } from '../lib/profile-install.mjs';
|
|
23
23
|
import { resolveRunsDir as resolveUserRunsDir } from '../lib/user-paths.mjs';
|
|
24
|
+
import { packageVersion, versionLine } from '../lib/version.mjs';
|
|
24
25
|
|
|
25
26
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
26
27
|
const ROOT = resolve(__dirname, '..');
|
|
@@ -74,6 +75,7 @@ function parseArgs(argv = process.argv.slice(2)) {
|
|
|
74
75
|
live: false,
|
|
75
76
|
json: false,
|
|
76
77
|
help: false,
|
|
78
|
+
version: false,
|
|
77
79
|
noServer: false,
|
|
78
80
|
sweep: false,
|
|
79
81
|
serve: false,
|
|
@@ -117,6 +119,8 @@ function parseArgs(argv = process.argv.slice(2)) {
|
|
|
117
119
|
const arg = args[i];
|
|
118
120
|
if (arg === '-h' || arg === '--help') {
|
|
119
121
|
opts.help = true;
|
|
122
|
+
} else if (arg === '-v' || arg === '-V' || arg === '--version' || arg === 'version') {
|
|
123
|
+
opts.version = true;
|
|
120
124
|
} else if (arg === 'setup' || arg === 'init' || arg === 'config' || arg === '--setup' || arg === '--config' || arg === 'wizard') {
|
|
121
125
|
opts.setup = true;
|
|
122
126
|
} else if (arg === 'install' || arg === '--install-profile') {
|
|
@@ -296,6 +300,7 @@ ${usage}
|
|
|
296
300
|
|
|
297
301
|
Nyelvválasztás & Beállítás:
|
|
298
302
|
-L, --lang <kód> Kimenet nyelve: en, hu, fr (alapértelmezett: en vagy CRUCIX_LANG)
|
|
303
|
+
-v, -V, --version, version A Crucix verziója (${packageVersion()})
|
|
299
304
|
setup, init, --setup Interaktív beállító varázsló (.env, Telegram, Discord, AI, API kulcsok) (--web / --cli)
|
|
300
305
|
install Felhasználói profil (~/.crucix: .env és runs/) létrehozása, majd a varázsló (--no-setup nélküle)
|
|
301
306
|
|
|
@@ -406,6 +411,7 @@ ${usage}
|
|
|
406
411
|
|
|
407
412
|
Language & Setup:
|
|
408
413
|
-L, --lang <code> Output language: en, hu, fr (default: en or CRUCIX_LANG)
|
|
414
|
+
-v, -V, --version, version Print the Crucix version (${packageVersion()})
|
|
409
415
|
setup, init, --setup Interactive setup wizard (.env, Telegram, Discord, AI, API keys) (--web / --cli)
|
|
410
416
|
install Create the user profile (~/.crucix: .env and runs/), then run the wizard (--no-setup skips it)
|
|
411
417
|
|
|
@@ -1489,6 +1495,11 @@ async function main() {
|
|
|
1489
1495
|
const lang = getCliLanguage(opts);
|
|
1490
1496
|
opts.lang = lang;
|
|
1491
1497
|
|
|
1498
|
+
if (opts.version) {
|
|
1499
|
+
console.log(versionLine());
|
|
1500
|
+
return;
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1492
1503
|
if (opts.help) {
|
|
1493
1504
|
printHelp(lang);
|
|
1494
1505
|
return;
|
|
@@ -2148,6 +2159,8 @@ export {
|
|
|
2148
2159
|
isExecutable,
|
|
2149
2160
|
getCliCmd,
|
|
2150
2161
|
printHelp,
|
|
2162
|
+
packageVersion,
|
|
2163
|
+
versionLine,
|
|
2151
2164
|
formatAge,
|
|
2152
2165
|
formatNumber,
|
|
2153
2166
|
formatPct,
|