crucix 2.39.2 → 2.39.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +3 -1
- package/README.md +17 -2
- package/apis/sources/acled.mjs +57 -12
- package/apis/sources/bls.mjs +39 -5
- package/apis/sources/epa.mjs +51 -4
- package/apis/sources/opensky.mjs +22 -1
- package/apis/sources/space.mjs +49 -17
- package/apis/utils/fetch.mjs +6 -2
- package/apis/utils/freshness.mjs +1 -1
- package/apis/utils/quota-cache.mjs +42 -0
- package/apis/utils/rss-pool.mjs +39 -0
- package/crucix.config.mjs +1 -1
- package/dashboard/inject.mjs +33 -12
- package/dashboard/public/live-sources.js +1 -1
- package/lib/http-parser-error.mjs +10 -0
- package/lib/instance-lock.mjs +81 -0
- package/lib/llm/budgets.mjs +1 -1
- package/lib/llm/cursor.mjs +12 -2
- package/lib/release-binary.mjs +32 -0
- package/lib/world-rss-runner.mjs +31 -19
- package/package.json +1 -1
- package/scripts/cli.mjs +6 -0
- package/scripts/install.ps1 +76 -0
- package/scripts/install.sh +80 -0
- package/server.mjs +16 -0
package/.env.example
CHANGED
|
@@ -15,6 +15,8 @@ FRED_API_KEY=
|
|
|
15
15
|
FIRMS_MAP_KEY=
|
|
16
16
|
# Energy Information Administration (free: api.eia.gov/register)
|
|
17
17
|
EIA_API_KEY=
|
|
18
|
+
# BLS Public Data (optional: data.bls.gov/registrationEngine). v1 is ~25/day; Crucix caches 12 hours. v2 is 500/day.
|
|
19
|
+
BLS_API_KEY=
|
|
18
20
|
# Financial Modeling Prep (free: site.financialmodelingprep.com/register, 250 calls/day): equities, indexes, forex in two
|
|
19
21
|
# batch calls per sweep (192/day at the 15-minute sweep). FMP_DAILY_LIMIT caps the calls; without a key Yahoo covers it.
|
|
20
22
|
FMP_API_KEY=
|
|
@@ -94,7 +96,7 @@ ANTHROPIC_BASE_URL=
|
|
|
94
96
|
# Sent as Authorization: Bearer instead of x-api-key; use either this or LLM_API_KEY, not both
|
|
95
97
|
ANTHROPIC_AUTH_TOKEN=
|
|
96
98
|
LLM_IDEAS_MAX_TOKENS=4096
|
|
97
|
-
LLM_IDEAS_TIMEOUT_MS=
|
|
99
|
+
LLM_IDEAS_TIMEOUT_MS=180000
|
|
98
100
|
LLM_ALERT_MAX_TOKENS=800
|
|
99
101
|
LLM_ALERT_TIMEOUT_MS=30000
|
|
100
102
|
# Cited briefings (POST /api/briefing): tokens 128-8192, timeout 1000-360000 ms; a failure falls back to the rule-based briefing
|
package/README.md
CHANGED
|
@@ -94,16 +94,31 @@ It was built for anyone who wants to understand what's actually happening in the
|
|
|
94
94
|
|
|
95
95
|
## Quick Start
|
|
96
96
|
|
|
97
|
+
```powershell
|
|
98
|
+
# Windows: standalone exe from GitHub Releases (no Node)
|
|
99
|
+
irm https://github.com/arlinamid/Crucix/releases/latest/download/install.ps1 | iex
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
# Linux / macOS
|
|
104
|
+
curl -fsSL https://github.com/arlinamid/Crucix/releases/latest/download/install.sh | sh
|
|
105
|
+
```
|
|
106
|
+
|
|
97
107
|
```bash
|
|
98
|
-
# From npm
|
|
108
|
+
# From npm: Node + runtime source, first run opens the setup wizard
|
|
99
109
|
npx crucix
|
|
100
110
|
npm install -g crucix
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Then:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
101
116
|
crucix --version
|
|
102
117
|
crucix install # ~/.crucix (.env and runs/)
|
|
103
118
|
crucix # dashboard; wizard if no .env yet
|
|
104
119
|
```
|
|
105
120
|
|
|
106
|
-
|
|
121
|
+
`npm run install:local` (from a git checkout) builds the exe yourself. Check which one PATH hits with `crucix --version`.
|
|
107
122
|
|
|
108
123
|
Settings the wizard and **Settings → News feeds** save go to `%USERPROFILE%\.crucix` on Windows or `~/.crucix` on Linux/macOS. Set `CRUCIX_HOME` to choose another folder, or `CRUCIX_ENV_FILE` to name the `.env` itself.
|
|
109
124
|
|
package/apis/sources/acled.mjs
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { readBoundedText } from '../utils/fetch.mjs';
|
|
2
2
|
// ACLED — Armed Conflict Location & Event Data
|
|
3
3
|
// Auth strategy (tries in order):
|
|
4
|
-
// 1.
|
|
5
|
-
// 2.
|
|
4
|
+
// 1. OAuth Bearer token: POST /oauth/token (password grant + scope=authenticated)
|
|
5
|
+
// 2. Cookie-based session: POST /user/login?_format=json → session cookie
|
|
6
6
|
// Set ACLED_EMAIL and ACLED_PASSWORD in .env (your myACLED login credentials).
|
|
7
7
|
// Data endpoint: GET https://acleddata.com/api/acled/read
|
|
8
8
|
|
|
9
9
|
import { daysAgo } from '../utils/fetch.mjs';
|
|
10
|
+
import { readQuota, writeQuota, clearQuota } from '../utils/quota-cache.mjs';
|
|
10
11
|
import '../utils/env.mjs';
|
|
11
12
|
|
|
12
13
|
const LOGIN_URL = 'https://acleddata.com/user/login?_format=json';
|
|
@@ -15,6 +16,31 @@ const API_BASE = 'https://acleddata.com/api/acled/read';
|
|
|
15
16
|
|
|
16
17
|
// Session cache
|
|
17
18
|
let sessionCache = { cookies: null, token: null, method: null, expires: 0 };
|
|
19
|
+
let lastGood = { at: 0, result: null };
|
|
20
|
+
let hydrated = false;
|
|
21
|
+
const LAST_GOOD_MS = 6 * 60 * 60 * 1000;
|
|
22
|
+
const ACLED_HEADERS = {
|
|
23
|
+
'User-Agent': 'Crucix/2.2',
|
|
24
|
+
Accept: 'application/json',
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
function hydrateQuota() {
|
|
28
|
+
if (hydrated) return;
|
|
29
|
+
hydrated = true;
|
|
30
|
+
const saved = readQuota('acled');
|
|
31
|
+
if (Number(saved?.at) > 0 && saved.result) lastGood = { at: saved.at, result: saved.result };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function resetState({ disk = true } = {}) {
|
|
35
|
+
sessionCache = { cookies: null, token: null, method: null, expires: 0 };
|
|
36
|
+
lastGood = { at: 0, result: null };
|
|
37
|
+
hydrated = false;
|
|
38
|
+
if (disk) clearQuota('acled');
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function isChallenge(status, text) {
|
|
42
|
+
return status === 403 && /just a moment|cf-browser-verification|attention required/i.test(text || '');
|
|
43
|
+
}
|
|
18
44
|
|
|
19
45
|
// Strategy 1: Cookie-based session login (mirrors browser login)
|
|
20
46
|
async function loginCookie(email, password) {
|
|
@@ -22,7 +48,7 @@ async function loginCookie(email, password) {
|
|
|
22
48
|
try {
|
|
23
49
|
const res = await fetch(LOGIN_URL, {
|
|
24
50
|
method: 'POST',
|
|
25
|
-
headers: { 'Content-Type': 'application/json' },
|
|
51
|
+
headers: { ...ACLED_HEADERS, 'Content-Type': 'application/json' },
|
|
26
52
|
body: JSON.stringify({ name: email, pass: password }),
|
|
27
53
|
redirect: 'manual',
|
|
28
54
|
signal,
|
|
@@ -58,21 +84,30 @@ async function loginOAuth(email, password) {
|
|
|
58
84
|
password: password,
|
|
59
85
|
grant_type: 'password',
|
|
60
86
|
client_id: 'acled',
|
|
87
|
+
scope: 'authenticated',
|
|
61
88
|
});
|
|
62
89
|
|
|
63
90
|
const res = await fetch(TOKEN_URL, {
|
|
64
91
|
method: 'POST',
|
|
65
|
-
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
|
92
|
+
headers: { ...ACLED_HEADERS, 'Content-Type': 'application/x-www-form-urlencoded' },
|
|
66
93
|
body: body.toString(),
|
|
67
94
|
signal,
|
|
68
95
|
});
|
|
69
96
|
|
|
97
|
+
const text = await readBoundedText(res, 64 * 1024);
|
|
98
|
+
if (isChallenge(res.status, text)) {
|
|
99
|
+
return { error: `Cloudflare challenge (HTTP ${res.status}); ACLED is blocking scripted login`, blocked: true };
|
|
100
|
+
}
|
|
70
101
|
if (!res.ok) {
|
|
71
|
-
|
|
72
|
-
|
|
102
|
+
let detail = '';
|
|
103
|
+
try {
|
|
104
|
+
const fail = JSON.parse(text);
|
|
105
|
+
detail = fail.error_description || fail.error || fail.message || '';
|
|
106
|
+
} catch { /* HTML or empty */ }
|
|
107
|
+
return { error: `OAuth failed (HTTP ${res.status}${detail ? `: ${String(detail).slice(0, 120)}` : ''})` };
|
|
73
108
|
}
|
|
74
109
|
|
|
75
|
-
const data = JSON.parse(
|
|
110
|
+
const data = JSON.parse(text);
|
|
76
111
|
if (!data.access_token) {
|
|
77
112
|
return { error: 'OAuth response missing access_token' };
|
|
78
113
|
}
|
|
@@ -109,6 +144,9 @@ async function authenticate() {
|
|
|
109
144
|
}
|
|
110
145
|
errors.push(`OAuth: ${oauthResult.error}`);
|
|
111
146
|
if (debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`);
|
|
147
|
+
if (oauthResult.blocked) {
|
|
148
|
+
return { error: `All ACLED auth methods failed.\n${errors.join('\n')}` };
|
|
149
|
+
}
|
|
112
150
|
|
|
113
151
|
// Fall back to cookie-based session
|
|
114
152
|
const cookieResult = await loginCookie(email, password);
|
|
@@ -124,7 +162,7 @@ async function authenticate() {
|
|
|
124
162
|
|
|
125
163
|
// Build headers based on auth method
|
|
126
164
|
function authHeaders(session) {
|
|
127
|
-
const headers = {
|
|
165
|
+
const headers = { ...ACLED_HEADERS, 'Content-Type': 'application/json' };
|
|
128
166
|
if (session.method === 'cookie' && session.cookies) {
|
|
129
167
|
headers['Cookie'] = session.cookies;
|
|
130
168
|
} else if (session.method === 'oauth' && session.token) {
|
|
@@ -228,7 +266,8 @@ function groupBy(events, field) {
|
|
|
228
266
|
}
|
|
229
267
|
|
|
230
268
|
// Briefing — last 7 days of global conflict events
|
|
231
|
-
export async function briefing() {
|
|
269
|
+
export async function briefing({ now = Date.now() } = {}) {
|
|
270
|
+
hydrateQuota();
|
|
232
271
|
if (!process.env.ACLED_EMAIL || !process.env.ACLED_PASSWORD) {
|
|
233
272
|
return {
|
|
234
273
|
source: 'ACLED',
|
|
@@ -248,7 +287,10 @@ export async function briefing() {
|
|
|
248
287
|
});
|
|
249
288
|
|
|
250
289
|
if (data?.error) {
|
|
251
|
-
|
|
290
|
+
if (lastGood.result && now - lastGood.at < LAST_GOOD_MS) {
|
|
291
|
+
return { ...lastGood.result, stale: true, message: data.error };
|
|
292
|
+
}
|
|
293
|
+
return { source: 'ACLED', timestamp: new Date(now).toISOString(), error: data.error };
|
|
252
294
|
}
|
|
253
295
|
|
|
254
296
|
let events = data?.data || [];
|
|
@@ -292,9 +334,9 @@ export async function briefing() {
|
|
|
292
334
|
notes: e.notes?.slice(0, 200),
|
|
293
335
|
}));
|
|
294
336
|
|
|
295
|
-
|
|
337
|
+
const result = {
|
|
296
338
|
source: 'ACLED',
|
|
297
|
-
timestamp: new Date().toISOString(),
|
|
339
|
+
timestamp: new Date(now).toISOString(),
|
|
298
340
|
period: { start, end },
|
|
299
341
|
totalEvents: events.length,
|
|
300
342
|
totalFatalities,
|
|
@@ -303,6 +345,9 @@ export async function briefing() {
|
|
|
303
345
|
topCountries,
|
|
304
346
|
deadliestEvents,
|
|
305
347
|
};
|
|
348
|
+
lastGood = { at: now, result };
|
|
349
|
+
writeQuota('acled', lastGood);
|
|
350
|
+
return result;
|
|
306
351
|
}
|
|
307
352
|
|
|
308
353
|
if (process.argv[1]?.endsWith('acled.mjs')) {
|
package/apis/sources/bls.mjs
CHANGED
|
@@ -1,12 +1,35 @@
|
|
|
1
1
|
import { readBoundedText } from '../utils/fetch.mjs';
|
|
2
2
|
// BLS — Bureau of Labor Statistics
|
|
3
3
|
// CPI, unemployment, nonfarm payrolls, PPI. No auth required (v1 API).
|
|
4
|
-
// v2 with registration key supports more requests; v1 is
|
|
4
|
+
// v2 with registration key supports more requests; v1 is ~25 queries/day. A 12-hour
|
|
5
|
+
// cache keeps a 15-minute sweep under that cap (CPI/labor series update monthly).
|
|
5
6
|
|
|
6
7
|
import { safeFetch, daysAgo } from '../utils/fetch.mjs';
|
|
8
|
+
import { readQuota, writeQuota, clearQuota } from '../utils/quota-cache.mjs';
|
|
7
9
|
|
|
8
10
|
const V1_BASE = 'https://api.bls.gov/publicAPI/v1/timeseries/data/';
|
|
9
11
|
const V2_BASE = 'https://api.bls.gov/publicAPI/v2/timeseries/data/';
|
|
12
|
+
export const CACHE_MS = 12 * 60 * 60 * 1000;
|
|
13
|
+
|
|
14
|
+
let lastGood = { at: 0, result: null };
|
|
15
|
+
let hydrated = false;
|
|
16
|
+
|
|
17
|
+
function hydrateQuota() {
|
|
18
|
+
if (hydrated) return;
|
|
19
|
+
hydrated = true;
|
|
20
|
+
const saved = readQuota('bls');
|
|
21
|
+
if (Number(saved?.at) > 0 && saved.result) lastGood = { at: saved.at, result: saved.result };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function resetState({ disk = true } = {}) {
|
|
25
|
+
lastGood = { at: 0, result: null };
|
|
26
|
+
hydrated = false;
|
|
27
|
+
if (disk) clearQuota('bls');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function cached(message) {
|
|
31
|
+
return { ...lastGood.result, stale: true, message };
|
|
32
|
+
}
|
|
10
33
|
|
|
11
34
|
// Key economic series
|
|
12
35
|
const SERIES = {
|
|
@@ -92,18 +115,26 @@ function momChange(seriesData) {
|
|
|
92
115
|
}
|
|
93
116
|
|
|
94
117
|
// Briefing — pull latest CPI, unemployment, payrolls
|
|
95
|
-
export async function briefing(apiKey) {
|
|
118
|
+
export async function briefing(apiKey, { now = Date.now() } = {}) {
|
|
119
|
+
hydrateQuota();
|
|
120
|
+
if (lastGood.result && now >= lastGood.at && now - lastGood.at < CACHE_MS) {
|
|
121
|
+
return cached(`Cached BLS result; next request after ${new Date(lastGood.at + CACHE_MS).toISOString()}`);
|
|
122
|
+
}
|
|
123
|
+
|
|
96
124
|
const seriesIds = Object.keys(SERIES);
|
|
97
125
|
const resp = await getSeries(seriesIds, { apiKey });
|
|
98
126
|
|
|
99
127
|
if (resp.error) {
|
|
128
|
+
if (lastGood.result) return cached(resp.error);
|
|
100
129
|
return { source: 'BLS', error: resp.error, timestamp: new Date().toISOString() };
|
|
101
130
|
}
|
|
102
131
|
|
|
103
132
|
if (resp.status !== 'REQUEST_SUCCEEDED' || !resp.Results?.series?.length) {
|
|
133
|
+
const error = resp.message?.[0] || 'BLS API returned no data';
|
|
134
|
+
if (lastGood.result) return cached(error);
|
|
104
135
|
return {
|
|
105
136
|
source: 'BLS',
|
|
106
|
-
error
|
|
137
|
+
error,
|
|
107
138
|
rawStatus: resp.status,
|
|
108
139
|
timestamp: new Date().toISOString(),
|
|
109
140
|
};
|
|
@@ -151,12 +182,15 @@ export async function briefing(apiKey) {
|
|
|
151
182
|
}
|
|
152
183
|
}
|
|
153
184
|
|
|
154
|
-
|
|
185
|
+
const result = {
|
|
155
186
|
source: 'BLS',
|
|
156
|
-
timestamp: new Date().toISOString(),
|
|
187
|
+
timestamp: new Date(now).toISOString(),
|
|
157
188
|
indicators,
|
|
158
189
|
signals,
|
|
159
190
|
};
|
|
191
|
+
lastGood = { at: now, result };
|
|
192
|
+
writeQuota('bls', lastGood);
|
|
193
|
+
return result;
|
|
160
194
|
}
|
|
161
195
|
|
|
162
196
|
if (process.argv[1]?.endsWith('bls.mjs')) {
|
package/apis/sources/epa.mjs
CHANGED
|
@@ -4,8 +4,33 @@
|
|
|
4
4
|
// Complements Safecast (citizen science) with official government readings.
|
|
5
5
|
|
|
6
6
|
import { safeFetch } from '../utils/fetch.mjs';
|
|
7
|
+
import { readQuota, writeQuota, clearQuota } from '../utils/quota-cache.mjs';
|
|
7
8
|
|
|
8
9
|
const BASE = 'https://enviro.epa.gov/enviro/efservice';
|
|
10
|
+
const BLOCK_MS = 6 * 60 * 60 * 1000; // EPA's WAF 403 does not clear within a sweep; do not retry for 6 hours
|
|
11
|
+
|
|
12
|
+
let blockedUntil = 0;
|
|
13
|
+
let lastGood = null;
|
|
14
|
+
let hydrated = false;
|
|
15
|
+
|
|
16
|
+
function hydrateQuota() {
|
|
17
|
+
if (hydrated) return;
|
|
18
|
+
hydrated = true;
|
|
19
|
+
const saved = readQuota('epa');
|
|
20
|
+
if (Number(saved?.blockedUntil) > 0) blockedUntil = Number(saved.blockedUntil);
|
|
21
|
+
if (saved?.lastGood) lastGood = saved.lastGood;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function persistQuota() {
|
|
25
|
+
writeQuota('epa', { blockedUntil, lastGood });
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function resetState({ disk = true } = {}) {
|
|
29
|
+
blockedUntil = 0;
|
|
30
|
+
lastGood = null;
|
|
31
|
+
hydrated = false;
|
|
32
|
+
if (disk) clearQuota('epa');
|
|
33
|
+
}
|
|
9
34
|
|
|
10
35
|
// RadNet analytical results endpoint
|
|
11
36
|
const RADNET_ANALYTICAL = `${BASE}/RADNET_ANALYTICAL_RESULTS`;
|
|
@@ -123,14 +148,31 @@ function checkReading(reading) {
|
|
|
123
148
|
return null;
|
|
124
149
|
}
|
|
125
150
|
|
|
151
|
+
function denied(now, error) {
|
|
152
|
+
return lastGood
|
|
153
|
+
? { ...lastGood, stale: true, timestamp: lastGood.timestamp, message: error }
|
|
154
|
+
: { source: 'EPA RadNet', timestamp: new Date(now).toISOString(), error, readings: [], signals: ['Radiation readings unavailable'] };
|
|
155
|
+
}
|
|
156
|
+
|
|
126
157
|
// Briefing — get recent radiation readings from EPA network, flag anomalies
|
|
127
|
-
export async function briefing() {
|
|
158
|
+
export async function briefing({ now = Date.now() } = {}) {
|
|
159
|
+
hydrateQuota();
|
|
160
|
+
if (now < blockedUntil) {
|
|
161
|
+
return denied(now, `EPA RadNet provider access denied (HTTP 403); retry after ${new Date(blockedUntil).toISOString()}`);
|
|
162
|
+
}
|
|
163
|
+
|
|
128
164
|
const readings = [];
|
|
129
165
|
const signals = [];
|
|
130
166
|
|
|
131
167
|
// Fetch recent analytical results (broad pull)
|
|
132
168
|
const recentData = await getAnalyticalResults({ rows: 100 });
|
|
133
|
-
if (recentData.error)
|
|
169
|
+
if (recentData.error) {
|
|
170
|
+
if (recentData.status === 403) {
|
|
171
|
+
blockedUntil = now + BLOCK_MS;
|
|
172
|
+
persistQuota();
|
|
173
|
+
}
|
|
174
|
+
return denied(now, recentData.status === 403 ? 'EPA RadNet provider access denied (HTTP 403)' : recentData.error);
|
|
175
|
+
}
|
|
134
176
|
const recentRecords = Array.isArray(recentData) ? recentData : [];
|
|
135
177
|
|
|
136
178
|
// Compact all readings
|
|
@@ -194,9 +236,9 @@ export async function briefing() {
|
|
|
194
236
|
])
|
|
195
237
|
);
|
|
196
238
|
|
|
197
|
-
|
|
239
|
+
const result = {
|
|
198
240
|
source: 'EPA RadNet',
|
|
199
|
-
timestamp: new Date().toISOString(),
|
|
241
|
+
timestamp: new Date(now).toISOString(),
|
|
200
242
|
totalReadings: readings.length,
|
|
201
243
|
...(analyteResults.some(r => r.error) ? { error: 'Some EPA analyte requests failed' } : {}),
|
|
202
244
|
readings: readings.slice(0, 50), // cap for briefing size
|
|
@@ -210,6 +252,11 @@ export async function briefing() {
|
|
|
210
252
|
thresholds: THRESHOLDS,
|
|
211
253
|
note: 'RadNet data may lag by hours to days. Near-real-time gamma data updates more frequently.',
|
|
212
254
|
};
|
|
255
|
+
if (!result.error) {
|
|
256
|
+
lastGood = result;
|
|
257
|
+
persistQuota();
|
|
258
|
+
}
|
|
259
|
+
return result;
|
|
213
260
|
}
|
|
214
261
|
|
|
215
262
|
// Run standalone
|
package/apis/sources/opensky.mjs
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
// clientId/clientSecret of credentials.json go into OPENSKY_CLIENT_ID/OPENSKY_CLIENT_SECRET.
|
|
8
8
|
|
|
9
9
|
import { safeFetch, readBoundedText } from '../utils/fetch.mjs';
|
|
10
|
+
import { readQuota, writeQuota, clearQuota } from '../utils/quota-cache.mjs';
|
|
10
11
|
import '../utils/env.mjs';
|
|
11
12
|
|
|
12
13
|
const BASE = 'https://opensky-network.org/api';
|
|
@@ -30,12 +31,28 @@ let tokenCache = { token: null, expiresAt: 0 };
|
|
|
30
31
|
let tokenRequest = null; // in-flight token request shared by the parallel hotspots
|
|
31
32
|
let lastRound = { at: 0, result: null }; // last round that got data (and so spent credits)
|
|
32
33
|
let blockedUntil = 0; // set by HTTP 429 from X-Rate-Limit-Retry-After-Seconds
|
|
34
|
+
let hydrated = false;
|
|
33
35
|
|
|
34
|
-
|
|
36
|
+
function persistQuota() {
|
|
37
|
+
writeQuota('opensky', { blockedUntil, lastRound });
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function hydrateQuota() {
|
|
41
|
+
if (hydrated) return;
|
|
42
|
+
hydrated = true;
|
|
43
|
+
const saved = readQuota('opensky');
|
|
44
|
+
if (!saved) return;
|
|
45
|
+
if (Number(saved.blockedUntil) > 0) blockedUntil = Number(saved.blockedUntil);
|
|
46
|
+
if (Number(saved.lastRound?.at) > 0 && saved.lastRound.result) lastRound = saved.lastRound;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function resetState({ disk = true } = {}) {
|
|
35
50
|
tokenCache = { token: null, expiresAt: 0 };
|
|
36
51
|
tokenRequest = null;
|
|
37
52
|
lastRound = { at: 0, result: null };
|
|
38
53
|
blockedUntil = 0;
|
|
54
|
+
hydrated = false;
|
|
55
|
+
if (disk) clearQuota('opensky');
|
|
39
56
|
}
|
|
40
57
|
|
|
41
58
|
function credentials() {
|
|
@@ -177,6 +194,7 @@ function blocked(now) {
|
|
|
177
194
|
|
|
178
195
|
// Briefing — check hotspot regions for flight activity
|
|
179
196
|
export async function briefing({ now = Date.now() } = {}) {
|
|
197
|
+
hydrateQuota();
|
|
180
198
|
if (now < blockedUntil) return blocked(now);
|
|
181
199
|
const deadline = Date.now() + ROUND_BUDGET_MS;
|
|
182
200
|
const creds = credentials();
|
|
@@ -239,7 +257,10 @@ export async function briefing({ now = Date.now() } = {}) {
|
|
|
239
257
|
if (hotspotErrors.length < results.length) lastRound = { at: now, result };
|
|
240
258
|
if (limits.length) {
|
|
241
259
|
blockedUntil = now + Math.min(MAX_BLOCK_MS, Math.max(...limits));
|
|
260
|
+
persistQuota();
|
|
242
261
|
if (hotspotErrors.length === results.length) return blocked(now);
|
|
262
|
+
} else if (hotspotErrors.length < results.length) {
|
|
263
|
+
persistQuota();
|
|
243
264
|
}
|
|
244
265
|
return result;
|
|
245
266
|
}
|
package/apis/sources/space.mjs
CHANGED
|
@@ -2,9 +2,30 @@
|
|
|
2
2
|
// No API key required. Uses CelesTrak for public TLE data and launch info.
|
|
3
3
|
// Tracks: Recent launches, ISS position, satellite decay alerts, space debris.
|
|
4
4
|
|
|
5
|
+
import { Agent } from 'undici';
|
|
5
6
|
import { safeFetch } from '../utils/fetch.mjs';
|
|
7
|
+
import { readQuota, writeQuota, clearQuota } from '../utils/quota-cache.mjs';
|
|
6
8
|
|
|
7
9
|
const CELESTRAK_BASE = 'https://celestrak.org';
|
|
10
|
+
const CACHE_MS = 2 * 60 * 60 * 1000; // GP data updates about every two hours
|
|
11
|
+
const LAST_GOOD_MS = 24 * 60 * 60 * 1000;
|
|
12
|
+
const celestrakAgent = new Agent({ connect: { timeout: 20000, family: 4 }, connections: 3 });
|
|
13
|
+
|
|
14
|
+
let lastGood = { at: 0, result: null };
|
|
15
|
+
let hydrated = false;
|
|
16
|
+
|
|
17
|
+
function hydrateQuota() {
|
|
18
|
+
if (hydrated) return;
|
|
19
|
+
hydrated = true;
|
|
20
|
+
const saved = readQuota('celestrak');
|
|
21
|
+
if (Number(saved?.at) > 0 && saved.result) lastGood = { at: saved.at, result: saved.result };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function resetState({ disk = true } = {}) {
|
|
25
|
+
lastGood = { at: 0, result: null };
|
|
26
|
+
hydrated = false;
|
|
27
|
+
if (disk) clearQuota('celestrak');
|
|
28
|
+
}
|
|
8
29
|
|
|
9
30
|
// Satellite categories for monitoring
|
|
10
31
|
const SAT_CATEGORIES = {
|
|
@@ -20,7 +41,9 @@ const SAT_CATEGORIES = {
|
|
|
20
41
|
async function getTLEs(category) {
|
|
21
42
|
const path = SAT_CATEGORIES[category];
|
|
22
43
|
if (!path) return { error: 'Invalid category' };
|
|
23
|
-
const data = await safeFetch(`${CELESTRAK_BASE}${path}`, {
|
|
44
|
+
const data = await safeFetch(`${CELESTRAK_BASE}${path}`, {
|
|
45
|
+
timeout: 25000, retries: 0, dispatcher: celestrakAgent,
|
|
46
|
+
});
|
|
24
47
|
return data;
|
|
25
48
|
}
|
|
26
49
|
|
|
@@ -95,16 +118,12 @@ async function getMilitaryCount() {
|
|
|
95
118
|
return { count: data.length, byCountry };
|
|
96
119
|
}
|
|
97
120
|
|
|
98
|
-
//
|
|
99
|
-
|
|
100
|
-
const
|
|
101
|
-
getTLEs('starlink'),
|
|
102
|
-
getTLEs('oneweb'),
|
|
103
|
-
]);
|
|
104
|
-
|
|
121
|
+
// Mega-lists (Starlink especially) trip CelesTrak's usage limits. Reuse the last counted totals.
|
|
122
|
+
function getConstellationStats() {
|
|
123
|
+
const saved = readQuota('celestrak-constellations');
|
|
105
124
|
return {
|
|
106
|
-
starlink:
|
|
107
|
-
oneweb:
|
|
125
|
+
starlink: Number(saved?.starlink) || 0,
|
|
126
|
+
oneweb: Number(saved?.oneweb) || 0,
|
|
108
127
|
};
|
|
109
128
|
}
|
|
110
129
|
|
|
@@ -137,21 +156,28 @@ function generateSignals(data) {
|
|
|
137
156
|
}
|
|
138
157
|
|
|
139
158
|
// Briefing export
|
|
140
|
-
export async function briefing() {
|
|
159
|
+
export async function briefing({ now = Date.now() } = {}) {
|
|
160
|
+
hydrateQuota();
|
|
161
|
+
if (lastGood.result && now >= lastGood.at && now - lastGood.at < CACHE_MS) {
|
|
162
|
+
return { ...lastGood.result, stale: true, message: `Cached CelesTrak result; next request after ${new Date(lastGood.at + CACHE_MS).toISOString()}` };
|
|
163
|
+
}
|
|
141
164
|
try {
|
|
142
|
-
const [launches, stations, military
|
|
165
|
+
const [launches, stations, military] = await Promise.all([
|
|
143
166
|
getRecentLaunches(),
|
|
144
167
|
getStationData(),
|
|
145
168
|
getMilitaryCount(),
|
|
146
|
-
getConstellationStats(),
|
|
147
169
|
]);
|
|
170
|
+
const constellations = getConstellationStats();
|
|
148
171
|
|
|
149
172
|
const hasData = !launches.error || !stations.error;
|
|
150
173
|
|
|
151
174
|
if (!hasData) {
|
|
175
|
+
if (lastGood.result && now - lastGood.at < LAST_GOOD_MS) {
|
|
176
|
+
return { ...lastGood.result, stale: true, message: launches.error || stations.error || 'Failed to fetch space data' };
|
|
177
|
+
}
|
|
152
178
|
return {
|
|
153
179
|
source: 'Space/CelesTrak',
|
|
154
|
-
timestamp: new Date().toISOString(),
|
|
180
|
+
timestamp: new Date(now).toISOString(),
|
|
155
181
|
status: 'error',
|
|
156
182
|
error: launches.error || stations.error || 'Failed to fetch space data',
|
|
157
183
|
};
|
|
@@ -160,9 +186,9 @@ export async function briefing() {
|
|
|
160
186
|
const data = { launches, stations, military, constellations };
|
|
161
187
|
const signals = generateSignals(data);
|
|
162
188
|
|
|
163
|
-
|
|
189
|
+
const result = {
|
|
164
190
|
source: 'Space/CelesTrak',
|
|
165
|
-
timestamp: new Date().toISOString(),
|
|
191
|
+
timestamp: new Date(now).toISOString(),
|
|
166
192
|
status: 'active',
|
|
167
193
|
recentLaunches: launches.recentLaunches || [],
|
|
168
194
|
totalNewObjects: launches.totalObjects || 0,
|
|
@@ -174,10 +200,16 @@ export async function briefing() {
|
|
|
174
200
|
constellations: constellations || {},
|
|
175
201
|
signals,
|
|
176
202
|
};
|
|
203
|
+
lastGood = { at: now, result };
|
|
204
|
+
writeQuota('celestrak', lastGood);
|
|
205
|
+
return result;
|
|
177
206
|
} catch (e) {
|
|
207
|
+
if (lastGood.result && now - lastGood.at < LAST_GOOD_MS) {
|
|
208
|
+
return { ...lastGood.result, stale: true, message: e.message };
|
|
209
|
+
}
|
|
178
210
|
return {
|
|
179
211
|
source: 'Space/CelesTrak',
|
|
180
|
-
timestamp: new Date().toISOString(),
|
|
212
|
+
timestamp: new Date(now).toISOString(),
|
|
181
213
|
status: 'error',
|
|
182
214
|
error: e.message,
|
|
183
215
|
};
|
package/apis/utils/fetch.mjs
CHANGED
|
@@ -32,7 +32,8 @@ export async function readBoundedText(response, maxBytes = 10 * 1024 * 1024) {
|
|
|
32
32
|
|
|
33
33
|
export async function safeFetch(url, opts = {}) {
|
|
34
34
|
const { timeout = 15000, retries = 1, headers = {}, maxBytes = 10 * 1024 * 1024,
|
|
35
|
-
format = 'json', method = 'GET', body, retryDelay = 2000, maxRetryDelay = 30000, retryAfterHeader = 'retry-after'
|
|
35
|
+
format = 'json', method = 'GET', body, retryDelay = 2000, maxRetryDelay = 30000, retryAfterHeader = 'retry-after',
|
|
36
|
+
dispatcher } = opts;
|
|
36
37
|
let lastError;
|
|
37
38
|
let lastStatus;
|
|
38
39
|
let lastRetryAfterMs = null;
|
|
@@ -45,6 +46,7 @@ export async function safeFetch(url, opts = {}) {
|
|
|
45
46
|
const response = await fetch(url, {
|
|
46
47
|
signal: controller.signal, method, body,
|
|
47
48
|
headers: { 'User-Agent': 'Crucix/2.2', ...headers },
|
|
49
|
+
...(dispatcher ? { dispatcher } : {}),
|
|
48
50
|
});
|
|
49
51
|
lastStatus = response.status;
|
|
50
52
|
if (!response.ok) {
|
|
@@ -63,7 +65,9 @@ export async function safeFetch(url, opts = {}) {
|
|
|
63
65
|
try { return JSON.parse(text); }
|
|
64
66
|
catch { return { error: 'Invalid JSON response', status: response.status }; }
|
|
65
67
|
} catch (error) {
|
|
66
|
-
|
|
68
|
+
const cause = error?.cause?.code || (typeof error?.cause?.message === 'string' ? error.cause.message : '');
|
|
69
|
+
lastError = controller.signal.aborted ? new Error(`Request timed out after ${timeout}ms`)
|
|
70
|
+
: new Error(cause && !String(error.message).includes(cause) ? `${error.message} (${cause})` : error.message);
|
|
67
71
|
} finally { clearTimeout(timer); }
|
|
68
72
|
if (!retryable || attempt === retries) break;
|
|
69
73
|
await new Promise(resolve => setTimeout(resolve, Math.min(maxRetryDelay, waitMs)));
|
package/apis/utils/freshness.mjs
CHANGED
|
@@ -32,7 +32,7 @@ export const POLICIES = Object.freeze({
|
|
|
32
32
|
'FAA-Airports': { maxAgeMs: 3*HOUR, observationMaxAgeMs: 3*HOUR },
|
|
33
33
|
Regulators: { maxAgeMs: 168*HOUR, observationMaxAgeMs: 336*HOUR },
|
|
34
34
|
'Eurostat-HU': { maxAgeMs: 2160*HOUR, observationMaxAgeMs: 4320*HOUR },
|
|
35
|
-
'Launch-Library': { maxAgeMs:
|
|
35
|
+
'Launch-Library': { maxAgeMs: 72*HOUR, observationMaxAgeMs: 720*HOUR },
|
|
36
36
|
Maritime: { maxAgeMs: 25*60000, observationMaxAgeMs: 25*60000 },
|
|
37
37
|
});
|
|
38
38
|
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// Small JSON files in CRUCIX_HOME/cache so daily quotas survive a process restart.
|
|
2
|
+
// Tests use isolate-profile.mjs, so writes stay in a temp home.
|
|
3
|
+
|
|
4
|
+
import { mkdirSync, readFileSync, writeFileSync, unlinkSync } from 'node:fs';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { crucixHome } from '../../lib/user-paths.mjs';
|
|
7
|
+
|
|
8
|
+
const NAME = /^[a-z0-9-]{1,40}$/;
|
|
9
|
+
const MAX_BYTES = 256 * 1024;
|
|
10
|
+
|
|
11
|
+
export function quotaCachePath(name, options) {
|
|
12
|
+
if (!NAME.test(name)) throw new Error(`invalid quota cache name: ${name}`);
|
|
13
|
+
return join(crucixHome(options), 'cache', `${name}.json`);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function readQuota(name, options) {
|
|
17
|
+
try {
|
|
18
|
+
const raw = readFileSync(quotaCachePath(name, options), 'utf8');
|
|
19
|
+
if (raw.length > MAX_BYTES) return null;
|
|
20
|
+
const data = JSON.parse(raw);
|
|
21
|
+
return data && typeof data === 'object' && !Array.isArray(data) ? data : null;
|
|
22
|
+
} catch {
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function writeQuota(name, value, options) {
|
|
28
|
+
try {
|
|
29
|
+
const dir = join(crucixHome(options), 'cache');
|
|
30
|
+
mkdirSync(dir, { recursive: true });
|
|
31
|
+
const text = JSON.stringify(value);
|
|
32
|
+
if (text.length > MAX_BYTES) return false;
|
|
33
|
+
writeFileSync(quotaCachePath(name, options), text);
|
|
34
|
+
return true;
|
|
35
|
+
} catch {
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function clearQuota(name, options) {
|
|
41
|
+
try { unlinkSync(quotaCachePath(name, options)); } catch { /* missing is fine */ }
|
|
42
|
+
}
|