bps-mcp-server 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +114 -3
- package/dist/client/allstats-client.d.ts +72 -0
- package/dist/client/allstats-client.d.ts.map +1 -0
- package/dist/client/allstats-client.js +247 -0
- package/dist/client/allstats-client.js.map +1 -0
- package/dist/client/bps-client.d.ts +4 -0
- package/dist/client/bps-client.d.ts.map +1 -1
- package/dist/client/bps-client.js +67 -6
- package/dist/client/bps-client.js.map +1 -1
- package/dist/config/defaults.d.ts +2 -0
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +2 -0
- package/dist/config/defaults.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +11 -2
- package/dist/server.js.map +1 -1
- package/dist/services/data-formatter.d.ts.map +1 -1
- package/dist/services/data-formatter.js +34 -40
- package/dist/services/data-formatter.js.map +1 -1
- package/dist/services/domain-resolver.d.ts +4 -0
- package/dist/services/domain-resolver.d.ts.map +1 -1
- package/dist/services/domain-resolver.js +64 -18
- package/dist/services/domain-resolver.js.map +1 -1
- package/dist/tools/allstats.tools.d.ts +4 -0
- package/dist/tools/allstats.tools.d.ts.map +1 -0
- package/dist/tools/allstats.tools.js +171 -0
- package/dist/tools/allstats.tools.js.map +1 -0
- package/dist/tools/search.tools.d.ts +2 -1
- package/dist/tools/search.tools.d.ts.map +1 -1
- package/dist/tools/search.tools.js +113 -7
- package/dist/tools/search.tools.js.map +1 -1
- package/package.json +10 -3
package/README.md
CHANGED
|
@@ -8,7 +8,9 @@ MCP (Model Context Protocol) server untuk data statistik BPS (Badan Pusat Statis
|
|
|
8
8
|
|
|
9
9
|
## Fitur
|
|
10
10
|
|
|
11
|
-
- **
|
|
11
|
+
- **34 tools** mencakup seluruh endpoint BPS WebAPI v1 + AllStats Search
|
|
12
|
+
- **Integrasi AllStats Search** — pencarian unified + full-text PDF search (tanpa API key)
|
|
13
|
+
- **Smart fallback** — WebAPI search otomatis fallback ke AllStats jika tidak ada hasil
|
|
12
14
|
- **3 MCP Resources** — domain list, kabupaten per provinsi, subjek per domain
|
|
13
15
|
- **5 MCP Prompts** — template analisis data siap pakai
|
|
14
16
|
- **Domain resolver** dengan fuzzy matching (ketik "Jatim" → Jawa Timur)
|
|
@@ -41,6 +43,38 @@ npm run build
|
|
|
41
43
|
BPS_API_KEY=your_key npm start
|
|
42
44
|
```
|
|
43
45
|
|
|
46
|
+
## Akses Remote via Cloudflare Workers
|
|
47
|
+
|
|
48
|
+
Server ini tersedia secara publik di:
|
|
49
|
+
|
|
50
|
+
```
|
|
51
|
+
https://bps-mcp-server.murphi.my.id/mcp
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Tambahkan ke MCP client manapun (Claude Desktop, Cursor, dll.) via remote transport:
|
|
55
|
+
|
|
56
|
+
```json
|
|
57
|
+
{
|
|
58
|
+
"mcpServers": {
|
|
59
|
+
"bps-statistics": {
|
|
60
|
+
"type": "http",
|
|
61
|
+
"url": "https://bps-mcp-server.murphi.my.id/mcp",
|
|
62
|
+
"headers": {
|
|
63
|
+
"X-BPS-API-Key": "your_api_key_here"
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Self-hosted
|
|
71
|
+
|
|
72
|
+
Ingin deploy sendiri? Deploy sebagai serverless worker di akun Cloudflare kamu:
|
|
73
|
+
|
|
74
|
+
[](https://deploy.workers.cloudflare.com/?url=https://github.com/setiapam/bps-mcp-server)
|
|
75
|
+
|
|
76
|
+
Lihat panduan lengkap di [docs/DEPLOY-WORKERS.md](docs/DEPLOY-WORKERS.md).
|
|
77
|
+
|
|
44
78
|
## Konfigurasi MCP Client
|
|
45
79
|
|
|
46
80
|
### Claude Desktop
|
|
@@ -101,7 +135,9 @@ File `~/.cursor/mcp.json` atau `.vscode/mcp.json`:
|
|
|
101
135
|
}
|
|
102
136
|
```
|
|
103
137
|
|
|
104
|
-
## Tools (
|
|
138
|
+
## Tools (34)
|
|
139
|
+
|
|
140
|
+
### WebAPI Tools (32)
|
|
105
141
|
|
|
106
142
|
| Tool | Deskripsi |
|
|
107
143
|
|------|-----------|
|
|
@@ -135,9 +171,82 @@ File `~/.cursor/mcp.json` atau `.vscode/mcp.json`:
|
|
|
135
171
|
| `list_csa_tables` | Tabel CSA per subjek |
|
|
136
172
|
| `get_csa_table` | Detail tabel CSA (HTML) |
|
|
137
173
|
| `list_glossary` | Glosarium istilah statistik |
|
|
138
|
-
| `search` | Pencarian lintas tipe |
|
|
174
|
+
| `search` | Pencarian lintas tipe (WebAPI + AllStats fallback) |
|
|
139
175
|
| `cache_clear` | Bersihkan cache |
|
|
140
176
|
|
|
177
|
+
### AllStats Search Tools (2)
|
|
178
|
+
|
|
179
|
+
| Tool | Deskripsi |
|
|
180
|
+
|------|-----------|
|
|
181
|
+
| `allstats_search` | Pencarian unified semua konten BPS (publikasi, tabel, BRS, infografis, data mikro, glosarium, klasifikasi) |
|
|
182
|
+
| `allstats_deep_search` | Full-text search di dalam isi PDF publikasi BPS — **fitur unik, tidak tersedia di WebAPI** |
|
|
183
|
+
|
|
184
|
+
## Integrasi AllStats Search
|
|
185
|
+
|
|
186
|
+
Server ini mengintegrasikan dua sumber data yang saling melengkapi:
|
|
187
|
+
|
|
188
|
+
```
|
|
189
|
+
┌─────────────────────────────────────────────────────────┐
|
|
190
|
+
│ BPS MCP Server │
|
|
191
|
+
│ │
|
|
192
|
+
│ ┌──────────────────┐ ┌────────────────────────┐ │
|
|
193
|
+
│ │ WebAPI BPS │ │ AllStats Search │ │
|
|
194
|
+
│ │ (Primary) │ │ (Supplementary) │ │
|
|
195
|
+
│ │ │ │ │ │
|
|
196
|
+
│ │ + Structured │ │ + Full-text PDF search │ │
|
|
197
|
+
│ │ data (JSON) │ │ + Unified search │ │
|
|
198
|
+
│ │ + Dynamic tables │ │ semua tipe konten │ │
|
|
199
|
+
│ │ + Ekspor/Impor │ │ + Tanpa API key │ │
|
|
200
|
+
│ │ + Sensus data │ │ + Filter wilayah 550+ │ │
|
|
201
|
+
│ │ - No PDF search │ │ - HTML scraping │ │
|
|
202
|
+
│ └──────────────────┘ └────────────────────────┘ │
|
|
203
|
+
│ │
|
|
204
|
+
│ Strategi interaksi: │
|
|
205
|
+
│ 1. search → WebAPI dulu, fallback AllStats jika kosong │
|
|
206
|
+
│ 2. allstats_search → langsung ke AllStats Search │
|
|
207
|
+
│ 3. allstats_deep_search → cari teks di dalam PDF │
|
|
208
|
+
└─────────────────────────────────────────────────────────┘
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
### Cara Kerja
|
|
212
|
+
|
|
213
|
+
**`search` (smart fallback)**
|
|
214
|
+
- Prioritas: WebAPI BPS (structured JSON)
|
|
215
|
+
- Jika WebAPI mengembalikan hasil kosong atau error, otomatis fallback ke AllStats Search
|
|
216
|
+
- User mendapat notifikasi sumber data yang digunakan
|
|
217
|
+
|
|
218
|
+
**`allstats_search` (unified discovery)**
|
|
219
|
+
- Langsung query ke `searchengine.web.bps.go.id`
|
|
220
|
+
- Mendukung filter: tipe konten, wilayah, rentang tahun, urutan
|
|
221
|
+
- Tidak memerlukan API key BPS
|
|
222
|
+
- Cocok untuk discovery atau pencarian broad
|
|
223
|
+
|
|
224
|
+
**`allstats_deep_search` (PDF full-text)**
|
|
225
|
+
- Cari teks di dalam isi PDF publikasi BPS
|
|
226
|
+
- Memerlukan `publication_id` (24 karakter hex) dari hasil `allstats_search`
|
|
227
|
+
- Mengembalikan halaman PDF yang cocok beserta cuplikan teks
|
|
228
|
+
- Fitur unik yang tidak tersedia di WebAPI
|
|
229
|
+
|
|
230
|
+
### Workflow Contoh
|
|
231
|
+
|
|
232
|
+
```
|
|
233
|
+
1. Discovery → Deep Search:
|
|
234
|
+
allstats_search("akses internet", content="publication")
|
|
235
|
+
→ dapat publication_id
|
|
236
|
+
→ allstats_deep_search("akses internet", publication_id="131385d0253c6aae7c7a59fa")
|
|
237
|
+
→ halaman PDF yang membahas "akses internet"
|
|
238
|
+
|
|
239
|
+
2. Smart fallback:
|
|
240
|
+
search(keyword="kemiskinan Papua")
|
|
241
|
+
→ WebAPI kosong → otomatis cari via AllStats
|
|
242
|
+
→ hasil dari AllStats ditampilkan dengan catatan fallback
|
|
243
|
+
|
|
244
|
+
3. Parallel enrichment (oleh AI):
|
|
245
|
+
- get_dynamic_data → data angka terstruktur
|
|
246
|
+
- allstats_search("inflasi", content="pressrelease") → BRS terbaru
|
|
247
|
+
→ AI menggabungkan data angka + konteks dari BRS
|
|
248
|
+
```
|
|
249
|
+
|
|
141
250
|
## Resources (3)
|
|
142
251
|
|
|
143
252
|
| URI | Deskripsi |
|
|
@@ -163,6 +272,8 @@ File `~/.cursor/mcp.json` atau `.vscode/mcp.json`:
|
|
|
163
272
|
"Bandingkan angka kemiskinan Jawa Timur vs Jawa Barat 2020-2023"
|
|
164
273
|
"Cari BRS terbaru tentang inflasi"
|
|
165
274
|
"Data ekspor kopi Indonesia tahun 2024"
|
|
275
|
+
"Cari publikasi tentang statistik telekomunikasi"
|
|
276
|
+
"Cari teks tentang akses internet di dalam publikasi BPS"
|
|
166
277
|
```
|
|
167
278
|
|
|
168
279
|
## Environment Variables
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { ICacheProvider } from "../services/cache.js";
|
|
2
|
+
export type AllStatsContentType = "all" | "publication" | "table" | "pressrelease" | "infographic" | "microdata" | "news" | "glosarium" | "kbli2020" | "kbli2017" | "kbli2015" | "kbli2009";
|
|
3
|
+
export type AllStatsSortOrder = "terbaru" | "relevansi";
|
|
4
|
+
export interface AllStatsSearchParams {
|
|
5
|
+
query: string;
|
|
6
|
+
content?: AllStatsContentType;
|
|
7
|
+
domain?: string;
|
|
8
|
+
page?: number;
|
|
9
|
+
titleOnly?: boolean;
|
|
10
|
+
yearFrom?: string;
|
|
11
|
+
yearTo?: string;
|
|
12
|
+
sort?: AllStatsSortOrder;
|
|
13
|
+
}
|
|
14
|
+
export interface AllStatsSearchResult {
|
|
15
|
+
url: string;
|
|
16
|
+
title: string;
|
|
17
|
+
description: string;
|
|
18
|
+
contentType: string;
|
|
19
|
+
domain: string;
|
|
20
|
+
deepSearchId?: string;
|
|
21
|
+
}
|
|
22
|
+
export interface AllStatsSearchResponse {
|
|
23
|
+
query: string;
|
|
24
|
+
totalResults: number;
|
|
25
|
+
currentPage: number;
|
|
26
|
+
totalPages: number;
|
|
27
|
+
results: AllStatsSearchResult[];
|
|
28
|
+
}
|
|
29
|
+
export interface AllStatsDeepSearchParams {
|
|
30
|
+
query: string;
|
|
31
|
+
publicationId: string;
|
|
32
|
+
domain?: string;
|
|
33
|
+
page?: number;
|
|
34
|
+
}
|
|
35
|
+
export interface AllStatsDeepSearchMatch {
|
|
36
|
+
pageNumber: number;
|
|
37
|
+
excerpt: string;
|
|
38
|
+
highlights: string[];
|
|
39
|
+
pdfViewerUrl: string;
|
|
40
|
+
}
|
|
41
|
+
export interface AllStatsDeepSearchPublication {
|
|
42
|
+
title: string;
|
|
43
|
+
publisher: string;
|
|
44
|
+
coverUrl: string;
|
|
45
|
+
publicationUrl: string;
|
|
46
|
+
pdfDownloadBase: string;
|
|
47
|
+
}
|
|
48
|
+
export interface AllStatsDeepSearchResponse {
|
|
49
|
+
query: string;
|
|
50
|
+
publication: AllStatsDeepSearchPublication;
|
|
51
|
+
totalMatches: number;
|
|
52
|
+
currentPage: number;
|
|
53
|
+
totalPages: number;
|
|
54
|
+
matches: AllStatsDeepSearchMatch[];
|
|
55
|
+
}
|
|
56
|
+
export declare class AllStatsClient {
|
|
57
|
+
private readonly cache;
|
|
58
|
+
constructor(cache: ICacheProvider | null);
|
|
59
|
+
private fetchHtml;
|
|
60
|
+
private fetchWithRetry;
|
|
61
|
+
private doFetch;
|
|
62
|
+
search(params: AllStatsSearchParams): Promise<AllStatsSearchResponse>;
|
|
63
|
+
private parseSearchResults;
|
|
64
|
+
deepSearch(params: AllStatsDeepSearchParams): Promise<AllStatsDeepSearchResponse>;
|
|
65
|
+
private parseDeepSearchResults;
|
|
66
|
+
}
|
|
67
|
+
export declare class AllStatsError extends Error {
|
|
68
|
+
readonly statusCode?: number | undefined;
|
|
69
|
+
readonly endpoint?: string | undefined;
|
|
70
|
+
constructor(message: string, statusCode?: number | undefined, endpoint?: string | undefined);
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=allstats-client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"allstats-client.d.ts","sourceRoot":"","sources":["../../src/client/allstats-client.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAa3D,MAAM,MAAM,mBAAmB,GAC3B,KAAK,GACL,aAAa,GACb,OAAO,GACP,cAAc,GACd,aAAa,GACb,WAAW,GACX,MAAM,GACN,WAAW,GACX,UAAU,GACV,UAAU,GACV,UAAU,GACV,UAAU,CAAC;AAEf,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,WAAW,CAAC;AAExD,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,iBAAiB,CAAC;CAC1B;AAED,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,oBAAoB,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,wBAAwB;IACvC,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,uBAAuB;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,6BAA6B;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,0BAA0B;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,6BAA6B,CAAC;IAC3C,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,uBAAuB,EAAE,CAAC;CACpC;AAiBD,qBAAa,cAAc;IACb,OAAO,CAAC,QAAQ,CAAC,KAAK;gBAAL,KAAK,EAAE,cAAc,GAAG,IAAI;YAI3C,SAAS;YA2BT,cAAc;YA4Bd,OAAO;IAiCf,MAAM,CAAC,MAAM,EAAE,oBAAoB,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAe3E,OAAO,CAAC,kBAAkB;IAoEpB,UAAU,CACd,MAAM,EAAE,wBAAwB,GAC/B,OAAO,CAAC,0BAA0B,CAAC;IAYtC,OAAO,CAAC,sBAAsB;CAwE/B;AAID,qBAAa,aAAc,SAAQ,KAAK;aAGpB,UAAU,CAAC,EAAE,MAAM;aACnB,QAAQ,CAAC,EAAE,MAAM;gBAFjC,OAAO,EAAE,MAAM,EACC,UAAU,CAAC,EAAE,MAAM,YAAA,EACnB,QAAQ,CAAC,EAAE,MAAM,YAAA;CAKpC"}
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
import * as cheerio from "cheerio";
|
|
2
|
+
import { logger } from "../utils/logger.js";
|
|
3
|
+
// ========== Constants ==========
|
|
4
|
+
const ALLSTATS_BASE = "https://searchengine.web.bps.go.id";
|
|
5
|
+
const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36";
|
|
6
|
+
const FETCH_TIMEOUT_MS = 30_000;
|
|
7
|
+
const MAX_RETRIES = 3;
|
|
8
|
+
const RETRY_BASE_DELAY_MS = 1_000;
|
|
9
|
+
// ========== OCR Cleanup ==========
|
|
10
|
+
const OCR_PATTERNS = [
|
|
11
|
+
/^Ps\s*\/\/\s*W\s*Ww\s*\.Bp\s*S\.G\s*O\.I\s*D\s*/i,
|
|
12
|
+
/^Ht\s*Tp\s*\/\/\s*.*?\.Bp\s*S\s*\.\s*Go\s*\.\s*Id\s*/i,
|
|
13
|
+
];
|
|
14
|
+
function cleanExcerpt(text) {
|
|
15
|
+
let cleaned = text;
|
|
16
|
+
for (const p of OCR_PATTERNS)
|
|
17
|
+
cleaned = cleaned.replace(p, "");
|
|
18
|
+
return cleaned.replace(/\s+/g, " ").trim();
|
|
19
|
+
}
|
|
20
|
+
// ========== Client ==========
|
|
21
|
+
export class AllStatsClient {
|
|
22
|
+
cache;
|
|
23
|
+
constructor(cache) {
|
|
24
|
+
this.cache = cache;
|
|
25
|
+
}
|
|
26
|
+
// ---------- HTTP ----------
|
|
27
|
+
async fetchHtml(path, params) {
|
|
28
|
+
const url = new URL(path, ALLSTATS_BASE);
|
|
29
|
+
for (const [k, v] of Object.entries(params)) {
|
|
30
|
+
url.searchParams.set(k, v);
|
|
31
|
+
}
|
|
32
|
+
const cacheKey = `allstats:${url.toString()}`;
|
|
33
|
+
// Check cache
|
|
34
|
+
if (this.cache) {
|
|
35
|
+
const cached = await this.cache.get(cacheKey);
|
|
36
|
+
if (cached) {
|
|
37
|
+
logger.debug(`AllStats cache hit: ${cacheKey}`);
|
|
38
|
+
return cached;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const html = await this.fetchWithRetry(url.toString());
|
|
42
|
+
// Cache for 30 minutes
|
|
43
|
+
if (this.cache) {
|
|
44
|
+
await this.cache.set(cacheKey, html, 30 * 60);
|
|
45
|
+
}
|
|
46
|
+
return html;
|
|
47
|
+
}
|
|
48
|
+
async fetchWithRetry(url) {
|
|
49
|
+
let lastError;
|
|
50
|
+
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
|
51
|
+
try {
|
|
52
|
+
return await this.doFetch(url);
|
|
53
|
+
}
|
|
54
|
+
catch (error) {
|
|
55
|
+
lastError = error;
|
|
56
|
+
// Retry on 403 (rate limit) and 5xx
|
|
57
|
+
const isRetryable = error instanceof AllStatsError &&
|
|
58
|
+
error.statusCode !== undefined &&
|
|
59
|
+
(error.statusCode === 403 || error.statusCode >= 500);
|
|
60
|
+
if (!isRetryable || attempt === MAX_RETRIES - 1) {
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
const delay = RETRY_BASE_DELAY_MS * Math.pow(2, attempt);
|
|
64
|
+
logger.warn(`AllStats retry (${attempt + 1}/${MAX_RETRIES}) after ${delay}ms: ${url}`);
|
|
65
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
throw lastError;
|
|
69
|
+
}
|
|
70
|
+
async doFetch(url) {
|
|
71
|
+
logger.debug(`AllStats fetch: ${url}`);
|
|
72
|
+
const controller = new AbortController();
|
|
73
|
+
const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
|
|
74
|
+
let res;
|
|
75
|
+
try {
|
|
76
|
+
res = await fetch(url, {
|
|
77
|
+
headers: {
|
|
78
|
+
"User-Agent": USER_AGENT,
|
|
79
|
+
Accept: "text/html,application/xhtml+xml",
|
|
80
|
+
},
|
|
81
|
+
signal: controller.signal,
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
if (error instanceof DOMException && error.name === "AbortError") {
|
|
86
|
+
throw new AllStatsError(`Request timeout after ${FETCH_TIMEOUT_MS}ms`, 408, url);
|
|
87
|
+
}
|
|
88
|
+
throw error;
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
clearTimeout(timeout);
|
|
92
|
+
}
|
|
93
|
+
if (!res.ok) {
|
|
94
|
+
throw new AllStatsError(`AllStats HTTP ${res.status} ${res.statusText}`, res.status, url);
|
|
95
|
+
}
|
|
96
|
+
return res.text();
|
|
97
|
+
}
|
|
98
|
+
// ---------- Search ----------
|
|
99
|
+
async search(params) {
|
|
100
|
+
const html = await this.fetchHtml("/search", {
|
|
101
|
+
q: params.query,
|
|
102
|
+
content: params.content || "all",
|
|
103
|
+
page: String(params.page || 1),
|
|
104
|
+
title: params.titleOnly ? "1" : "0",
|
|
105
|
+
mfd: params.domain || "0000",
|
|
106
|
+
from: params.yearFrom || "all",
|
|
107
|
+
to: params.yearTo || "all",
|
|
108
|
+
sort: params.sort || "terbaru",
|
|
109
|
+
});
|
|
110
|
+
return this.parseSearchResults(html, params.query, params.page || 1);
|
|
111
|
+
}
|
|
112
|
+
parseSearchResults(html, query, currentPage) {
|
|
113
|
+
const $ = cheerio.load(html);
|
|
114
|
+
// Total results: "Menampilkan 4.079 hasil pencarian dalam 0.367 detik"
|
|
115
|
+
const totalMatch = $("body")
|
|
116
|
+
.text()
|
|
117
|
+
.match(/Menampilkan\s+([\d.]+)\s+hasil pencarian/);
|
|
118
|
+
const totalResults = totalMatch
|
|
119
|
+
? parseInt(totalMatch[1].replace(/\./g, ""))
|
|
120
|
+
: 0;
|
|
121
|
+
// Pagination
|
|
122
|
+
const pages = [];
|
|
123
|
+
$("a[onclick*='changePage']").each((_, el) => {
|
|
124
|
+
const m = $(el).attr("onclick")?.match(/changePage\((\d+)\)/);
|
|
125
|
+
if (m)
|
|
126
|
+
pages.push(parseInt(m[1]));
|
|
127
|
+
});
|
|
128
|
+
const totalPages = pages.length > 0 ? Math.max(...pages) : 1;
|
|
129
|
+
// Results
|
|
130
|
+
const results = [];
|
|
131
|
+
$("div.card-result").each((_, el) => {
|
|
132
|
+
const $c = $(el);
|
|
133
|
+
const url = $c.find("a.stretched-link").attr("href") || "";
|
|
134
|
+
const title = $c.find("h5.fw-medium").text().trim();
|
|
135
|
+
const description = $c
|
|
136
|
+
.find("p.text-body-secondary.text-truncate")
|
|
137
|
+
.text()
|
|
138
|
+
.trim();
|
|
139
|
+
const badges = $c.find("div.badge");
|
|
140
|
+
const contentType = badges.first().text().trim();
|
|
141
|
+
const domain = $c.find("div.badge.text-bg-light").text().trim() ||
|
|
142
|
+
badges.last().text().trim();
|
|
143
|
+
let deepSearchId;
|
|
144
|
+
const deepHref = $c.find('a[href*="deep"]').attr("href");
|
|
145
|
+
if (deepHref) {
|
|
146
|
+
const m = deepHref.match(/id=([a-f0-9]{24})/);
|
|
147
|
+
if (m)
|
|
148
|
+
deepSearchId = m[1];
|
|
149
|
+
}
|
|
150
|
+
results.push({
|
|
151
|
+
url,
|
|
152
|
+
title,
|
|
153
|
+
description,
|
|
154
|
+
contentType,
|
|
155
|
+
domain,
|
|
156
|
+
deepSearchId,
|
|
157
|
+
});
|
|
158
|
+
});
|
|
159
|
+
return {
|
|
160
|
+
query,
|
|
161
|
+
totalResults,
|
|
162
|
+
currentPage,
|
|
163
|
+
totalPages,
|
|
164
|
+
results,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
// ---------- Deep Search ----------
|
|
168
|
+
async deepSearch(params) {
|
|
169
|
+
const html = await this.fetchHtml("/deep", {
|
|
170
|
+
q: params.query,
|
|
171
|
+
id: params.publicationId,
|
|
172
|
+
content: "publication",
|
|
173
|
+
mfd: params.domain || "0000",
|
|
174
|
+
page: String(params.page || 1),
|
|
175
|
+
});
|
|
176
|
+
return this.parseDeepSearchResults(html, params.query, params.page || 1);
|
|
177
|
+
}
|
|
178
|
+
parseDeepSearchResults(html, query, currentPage) {
|
|
179
|
+
const $ = cheerio.load(html);
|
|
180
|
+
// Publication metadata
|
|
181
|
+
const publication = {
|
|
182
|
+
title: $("h5.card-title.fw-semibold").first().text().trim(),
|
|
183
|
+
publisher: $("p.card-text.text-body-secondary").first().text().trim(),
|
|
184
|
+
coverUrl: $("img.foreground").attr("src") || "",
|
|
185
|
+
publicationUrl: $('a.btn-info[href*="bps.go.id/publication"]').attr("href") || "",
|
|
186
|
+
pdfDownloadBase: "",
|
|
187
|
+
};
|
|
188
|
+
// PDF download base from inline script
|
|
189
|
+
$("script").each((_, el) => {
|
|
190
|
+
const content = $(el).html() || "";
|
|
191
|
+
const m = content.match(/https:\/\/web-api\.bps\.go\.id\/download\.php\?f=[^#"]*/);
|
|
192
|
+
if (m)
|
|
193
|
+
publication.pdfDownloadBase = m[0];
|
|
194
|
+
});
|
|
195
|
+
// Total matches: "Menampilkan 11 halaman dengan kata kunci "akses internet""
|
|
196
|
+
const totalMatch = $("body")
|
|
197
|
+
.text()
|
|
198
|
+
.match(/Menampilkan\s+(\d+)\s+halaman dengan kata kunci/);
|
|
199
|
+
const totalMatches = totalMatch ? parseInt(totalMatch[1]) : 0;
|
|
200
|
+
// Pagination
|
|
201
|
+
const pages = [];
|
|
202
|
+
$("a[onclick*='changePage']").each((_, el) => {
|
|
203
|
+
const m = $(el).attr("onclick")?.match(/changePage\((\d+)\)/);
|
|
204
|
+
if (m)
|
|
205
|
+
pages.push(parseInt(m[1]));
|
|
206
|
+
});
|
|
207
|
+
const totalPages = pages.length > 0 ? Math.max(...pages) : 1;
|
|
208
|
+
// Matches
|
|
209
|
+
const matches = [];
|
|
210
|
+
$("div.card-result").each((_, el) => {
|
|
211
|
+
const $c = $(el);
|
|
212
|
+
const pageNumber = parseInt($c.find("a.linkhalaman").attr("data-page") || "0");
|
|
213
|
+
const excerptEl = $c.find('p[id^="deskripsi-"]');
|
|
214
|
+
const excerpt = cleanExcerpt(excerptEl.text());
|
|
215
|
+
const highlights = [];
|
|
216
|
+
excerptEl.find("mark").each((_, mark) => {
|
|
217
|
+
const kw = $(mark).text().trim().toLowerCase();
|
|
218
|
+
if (!highlights.includes(kw))
|
|
219
|
+
highlights.push(kw);
|
|
220
|
+
});
|
|
221
|
+
const pdfViewerUrl = publication.pdfDownloadBase
|
|
222
|
+
? `${publication.pdfDownloadBase}#page=${pageNumber}`
|
|
223
|
+
: "";
|
|
224
|
+
matches.push({ pageNumber, excerpt, highlights, pdfViewerUrl });
|
|
225
|
+
});
|
|
226
|
+
return {
|
|
227
|
+
query,
|
|
228
|
+
publication,
|
|
229
|
+
totalMatches,
|
|
230
|
+
currentPage,
|
|
231
|
+
totalPages,
|
|
232
|
+
matches,
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
// ========== Error ==========
|
|
237
|
+
export class AllStatsError extends Error {
|
|
238
|
+
statusCode;
|
|
239
|
+
endpoint;
|
|
240
|
+
constructor(message, statusCode, endpoint) {
|
|
241
|
+
super(message);
|
|
242
|
+
this.statusCode = statusCode;
|
|
243
|
+
this.endpoint = endpoint;
|
|
244
|
+
this.name = "AllStatsError";
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
//# sourceMappingURL=allstats-client.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"allstats-client.js","sourceRoot":"","sources":["../../src/client/allstats-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,kCAAkC;AAElC,MAAM,aAAa,GAAG,oCAAoC,CAAC;AAC3D,MAAM,UAAU,GAAG,uGAAuG,CAAC;AAC3H,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAChC,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,mBAAmB,GAAG,KAAK,CAAC;AA+ElC,oCAAoC;AAEpC,MAAM,YAAY,GAAG;IACnB,kDAAkD;IAClD,uDAAuD;CACxD,CAAC;AAEF,SAAS,YAAY,CAAC,IAAY;IAChC,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,KAAK,MAAM,CAAC,IAAI,YAAY;QAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/D,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC7C,CAAC;AAED,+BAA+B;AAE/B,MAAM,OAAO,cAAc;IACI;IAA7B,YAA6B,KAA4B;QAA5B,UAAK,GAAL,KAAK,CAAuB;IAAG,CAAC;IAE7D,6BAA6B;IAErB,KAAK,CAAC,SAAS,CAAC,IAAY,EAAE,MAA8B;QAClE,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QACzC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5C,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,CAAC;QAED,MAAM,QAAQ,GAAG,YAAY,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC;QAE9C,cAAc;QACd,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC9C,IAAI,MAAM,EAAE,CAAC;gBACX,MAAM,CAAC,KAAK,CAAC,uBAAuB,QAAQ,EAAE,CAAC,CAAC;gBAChD,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QAEvD,uBAAuB;QACvB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAChD,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,GAAW;QACtC,IAAI,SAAkB,CAAC;QAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,KAAK,CAAC;gBAElB,oCAAoC;gBACpC,MAAM,WAAW,GACf,KAAK,YAAY,aAAa;oBAC9B,KAAK,CAAC,UAAU,KAAK,SAAS;oBAC9B,CAAC,KAAK,CAAC,UAAU,KAAK,GAAG,IAAI,KAAK,CAAC,UAAU,IAAI,GAAG,CAAC,CAAC;gBAExD,IAAI,CAAC,WAAW,IAAI,OAAO,KAAK,WAAW,GAAG,CAAC,EAAE,CAAC;oBAChD,MAAM,KAAK,CAAC;gBACd,CAAC;gBAED,MAAM,KAAK,GAAG,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACzD,MAAM,CAAC,IAAI,CAAC,mBAAmB,OAAO,GAAG,CAAC,IAAI,WAAW,WAAW,KAAK,OAAO,GAAG,EAAE,CAAC,CAAC;gBACvF,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QAED,MAAM,SAAS,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,GAAW;QAC/B,MAAM,CAAC,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC;QAEvC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,gBAAgB,CAAC,CAAC;QAEvE,IAAI,GAAa,CAAC;QAClB,IAAI,CAAC;YACH,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBACrB,OAAO,EAAE;oBACP,YAAY,EAAE,UAAU;oBACxB,MAAM,EAAE,iCAAiC;iBAC1C;gBACD,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBACjE,MAAM,IAAI,aAAa,CAAC,yBAAyB,gBAAgB,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YACnF,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,OAAO,CAAC,CAAC;QACxB,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,aAAa,CAAC,iBAAiB,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC5F,CAAC;QAED,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAED,+BAA+B;IAE/B,KAAK,CAAC,MAAM,CAAC,MAA4B;QACvC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE;YAC3C,CAAC,EAAE,MAAM,CAAC,KAAK;YACf,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,KAAK;YAChC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;YAC9B,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;YACnC,GAAG,EAAE,MAAM,CAAC,MAAM,IAAI,MAAM;YAC5B,IAAI,EAAE,MAAM,CAAC,QAAQ,IAAI,KAAK;YAC9B,EAAE,EAAE,MAAM,CAAC,MAAM,IAAI,KAAK;YAC1B,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,SAAS;SAC/B,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;IACvE,CAAC;IAEO,kBAAkB,CACxB,IAAY,EACZ,KAAa,EACb,WAAmB;QAEnB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,uEAAuE;QACvE,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;aACzB,IAAI,EAAE;aACN,KAAK,CAAC,0CAA0C,CAAC,CAAC;QACrD,MAAM,YAAY,GAAG,UAAU;YAC7B,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC5C,CAAC,CAAC,CAAC,CAAC;QAEN,aAAa;QACb,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,0BAA0B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC3C,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAC9D,IAAI,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE7D,UAAU;QACV,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,CAAC,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAClC,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YACjB,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACpD,MAAM,WAAW,GAAG,EAAE;iBACnB,IAAI,CAAC,qCAAqC,CAAC;iBAC3C,IAAI,EAAE;iBACN,IAAI,EAAE,CAAC;YAEV,MAAM,MAAM,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACpC,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACjD,MAAM,MAAM,GACV,EAAE,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;gBAChD,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAE9B,IAAI,YAAgC,CAAC;YACrC,MAAM,QAAQ,GAAG,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;gBAC9C,IAAI,CAAC;oBAAE,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,CAAC;YAED,OAAO,CAAC,IAAI,CAAC;gBACX,GAAG;gBACH,KAAK;gBACL,WAAW;gBACX,WAAW;gBACX,MAAM;gBACN,YAAY;aACb,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,KAAK;YACL,YAAY;YACZ,WAAW;YACX,UAAU;YACV,OAAO;SACR,CAAC;IACJ,CAAC;IAED,oCAAoC;IAEpC,KAAK,CAAC,UAAU,CACd,MAAgC;QAEhC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE;YACzC,CAAC,EAAE,MAAM,CAAC,KAAK;YACf,EAAE,EAAE,MAAM,CAAC,aAAa;YACxB,OAAO,EAAE,aAAa;YACtB,GAAG,EAAE,MAAM,CAAC,MAAM,IAAI,MAAM;YAC5B,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;SAC/B,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,sBAAsB,CAAC,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;IAC3E,CAAC;IAEO,sBAAsB,CAC5B,IAAY,EACZ,KAAa,EACb,WAAmB;QAEnB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,uBAAuB;QACvB,MAAM,WAAW,GAAkC;YACjD,KAAK,EAAE,CAAC,CAAC,2BAA2B,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;YAC3D,SAAS,EAAE,CAAC,CAAC,iCAAiC,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;YACrE,QAAQ,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;YAC/C,cAAc,EACZ,CAAC,CAAC,2CAA2C,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;YACnE,eAAe,EAAE,EAAE;SACpB,CAAC;QAEF,uCAAuC;QACvC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACzB,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CACrB,yDAAyD,CAC1D,CAAC;YACF,IAAI,CAAC;gBAAE,WAAW,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,6EAA6E;QAC7E,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;aACzB,IAAI,EAAE;aACN,KAAK,CAAC,iDAAiD,CAAC,CAAC;QAC5D,MAAM,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9D,aAAa;QACb,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,CAAC,CAAC,0BAA0B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAC3C,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,qBAAqB,CAAC,CAAC;YAC9D,IAAI,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE7D,UAAU;QACV,MAAM,OAAO,GAA8B,EAAE,CAAC;QAC9C,CAAC,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YAClC,MAAM,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YACjB,MAAM,UAAU,GAAG,QAAQ,CACzB,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,CAClD,CAAC;YACF,MAAM,SAAS,GAAG,EAAE,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;YACjD,MAAM,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;YAE/C,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;gBACtC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;gBAC/C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAAE,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACpD,CAAC,CAAC,CAAC;YAEH,MAAM,YAAY,GAAG,WAAW,CAAC,eAAe;gBAC9C,CAAC,CAAC,GAAG,WAAW,CAAC,eAAe,SAAS,UAAU,EAAE;gBACrD,CAAC,CAAC,EAAE,CAAC;YAEP,OAAO,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,OAAO;YACL,KAAK;YACL,WAAW;YACX,YAAY;YACZ,WAAW;YACX,UAAU;YACV,OAAO;SACR,CAAC;IACJ,CAAC;CACF;AAED,8BAA8B;AAE9B,MAAM,OAAO,aAAc,SAAQ,KAAK;IAGpB;IACA;IAHlB,YACE,OAAe,EACC,UAAmB,EACnB,QAAiB;QAEjC,KAAK,CAAC,OAAO,CAAC,CAAC;QAHC,eAAU,GAAV,UAAU,CAAS;QACnB,aAAQ,GAAR,QAAQ,CAAS;QAGjC,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC;IAC9B,CAAC;CACF"}
|
|
@@ -8,8 +8,12 @@ export declare class BpsClient {
|
|
|
8
8
|
private readonly baseUrl;
|
|
9
9
|
private readonly defaultLang;
|
|
10
10
|
private readonly defaultDomain;
|
|
11
|
+
/** In-flight request deduplication map: cacheKey → pending Promise */
|
|
12
|
+
private readonly inflight;
|
|
11
13
|
constructor(auth: IAuthProvider, cache: ICacheProvider | null, config: Config);
|
|
12
14
|
private fetchJson;
|
|
15
|
+
private fetchWithRetry;
|
|
16
|
+
private doFetch;
|
|
13
17
|
private authParams;
|
|
14
18
|
/**
|
|
15
19
|
* Perform a list request: /api/list/model/{model}/...
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bps-client.d.ts","sourceRoot":"","sources":["../../src/client/bps-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AASjD,OAAO,KAAK,EACV,SAAS,EACT,UAAU,EACV,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,kBAAkB,EAClB,SAAS,EACT,gBAAgB,EAChB,OAAO,EACP,sBAAsB,EACtB,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,cAAc,EACd,qBAAqB,EACrB,cAAc,EACd,oBAAoB,EACpB,OAAO,EACP,aAAa,EACb,eAAe,EACf,qBAAqB,EACrB,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,QAAQ,EACT,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"bps-client.d.ts","sourceRoot":"","sources":["../../src/client/bps-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AASjD,OAAO,KAAK,EACV,SAAS,EACT,UAAU,EACV,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,kBAAkB,EAClB,SAAS,EACT,gBAAgB,EAChB,OAAO,EACP,sBAAsB,EACtB,cAAc,EACd,oBAAoB,EACpB,eAAe,EACf,cAAc,EACd,qBAAqB,EACrB,cAAc,EACd,oBAAoB,EACpB,OAAO,EACP,aAAa,EACb,eAAe,EACf,qBAAqB,EACrB,aAAa,EACb,WAAW,EACX,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,QAAQ,EACT,MAAM,YAAY,CAAC;AAapB,qBAAa,SAAS;IASlB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,KAAK;IATxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IAEvC,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAuC;gBAG7C,IAAI,EAAE,aAAa,EACnB,KAAK,EAAE,cAAc,GAAG,IAAI,EAC7C,MAAM,EAAE,MAAM;YAOF,SAAS;YA2BT,cAAc;YA4Bd,OAAO;YAsDP,UAAU;IAIxB;;OAEG;YACW,WAAW;IAkBzB;;OAEG;YACW,WAAW;IAkBzB,OAAO,CAAC,gBAAgB;IAUlB,WAAW,CACf,IAAI,GAAE,KAAK,GAAG,MAAM,GAAG,KAAK,GAAG,WAAmB,EAClD,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC;QAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;IAiB5C,YAAY,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,UAAU,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;IAShG,qBAAqB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAYrE,aAAa,CACjB,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,MAAM,EACb,IAAI,CAAC,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC;QAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;IAS9C,qBAAqB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAStF,oBAAoB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IASpF,WAAW,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;IASlE,kBAAkB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAShF,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAW9C,cAAc,CAClB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,EAAE,CAAC,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,MAAM,EACf,KAAK,CAAC,EAAE,MAAM,GACb,OAAO,CAAC,sBAAsB,CAAC;IAU5B,gBAAgB,CACpB,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,MAAM,EACb,KAAK,CAAC,EAAE,MAAM,EACd,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,IAAI,EAAE,cAAc,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;IASjD,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAWzE,iBAAiB,CACrB,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,MAAM,EACb,KAAK,CAAC,EAAE,MAAM,EACd,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,IAAI,EAAE,eAAe,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;IASlD,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAWrE,gBAAgB,CACpB,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,MAAM,EACb,KAAK,CAAC,EAAE,MAAM,EACd,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,IAAI,EAAE,cAAc,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;IASjD,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;IAWnE,uBAAuB,CAC3B,MAAM,CAAC,EAAE,MAAM,EACf,KAAK,CAAC,EAAE,MAAM,EACd,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,IAAI,EAAE,qBAAqB,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;IAWxD,YAAY,CAChB,MAAM,EAAE,CAAC,GAAG,CAAC,EACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,OAAO,CAAC;IAmBb,gBAAgB,CACpB,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,IAAI,EAAE,cAAc,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;IASjD,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAWzE,QAAQ,CACZ,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;IAS1C,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAW3D,YAAY,CAChB,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,IAAI,EAAE,eAAe,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;IAWlD,wBAAwB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAC;IAS3E,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;IAStG,aAAa,CACjB,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;IAS9C,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAWnE,gBAAgB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAQ7C,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAU7D,MAAM,CACV,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,KAAK,CAAC,EAAE,MAAM,EACd,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC;QAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAAC,IAAI,CAAC,EAAE,QAAQ,CAAA;KAAE,CAAC;CASjD"}
|
|
@@ -1,12 +1,20 @@
|
|
|
1
1
|
import { buildListUrl, buildViewUrl, buildDomainUrl, buildTradeUrl, buildInteropUrl, MODELS, } from "./endpoints.js";
|
|
2
2
|
import { BpsApiError, BpsAuthError, BpsNotFoundError } from "../utils/error.js";
|
|
3
3
|
import { logger } from "../utils/logger.js";
|
|
4
|
+
/** Default fetch timeout in milliseconds */
|
|
5
|
+
const FETCH_TIMEOUT_MS = 30_000;
|
|
6
|
+
/** Maximum retry attempts for transient errors */
|
|
7
|
+
const MAX_RETRIES = 3;
|
|
8
|
+
/** Base delay for exponential backoff in milliseconds */
|
|
9
|
+
const RETRY_BASE_DELAY_MS = 500;
|
|
4
10
|
export class BpsClient {
|
|
5
11
|
auth;
|
|
6
12
|
cache;
|
|
7
13
|
baseUrl;
|
|
8
14
|
defaultLang;
|
|
9
15
|
defaultDomain;
|
|
16
|
+
/** In-flight request deduplication map: cacheKey → pending Promise */
|
|
17
|
+
inflight = new Map();
|
|
10
18
|
constructor(auth, cache, config) {
|
|
11
19
|
this.auth = auth;
|
|
12
20
|
this.cache = cache;
|
|
@@ -23,14 +31,67 @@ export class BpsClient {
|
|
|
23
31
|
return JSON.parse(cached);
|
|
24
32
|
}
|
|
25
33
|
}
|
|
34
|
+
// Deduplicate concurrent requests for the same cache key
|
|
35
|
+
const existing = this.inflight.get(cacheKey);
|
|
36
|
+
if (existing) {
|
|
37
|
+
logger.debug(`Dedup hit: ${cacheKey}`);
|
|
38
|
+
return existing;
|
|
39
|
+
}
|
|
40
|
+
const promise = this.fetchWithRetry(url, cacheKey, ttl);
|
|
41
|
+
this.inflight.set(cacheKey, promise);
|
|
42
|
+
try {
|
|
43
|
+
return await promise;
|
|
44
|
+
}
|
|
45
|
+
finally {
|
|
46
|
+
this.inflight.delete(cacheKey);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
async fetchWithRetry(url, cacheKey, ttl) {
|
|
50
|
+
let lastError;
|
|
51
|
+
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
|
52
|
+
try {
|
|
53
|
+
return await this.doFetch(url, cacheKey, ttl);
|
|
54
|
+
}
|
|
55
|
+
catch (error) {
|
|
56
|
+
lastError = error;
|
|
57
|
+
// Only retry on transient server errors (5xx)
|
|
58
|
+
const isRetryable = error instanceof BpsApiError &&
|
|
59
|
+
error.statusCode !== undefined &&
|
|
60
|
+
error.statusCode >= 500;
|
|
61
|
+
if (!isRetryable || attempt === MAX_RETRIES - 1) {
|
|
62
|
+
throw error;
|
|
63
|
+
}
|
|
64
|
+
const delay = RETRY_BASE_DELAY_MS * Math.pow(2, attempt);
|
|
65
|
+
logger.warn(`Retrying (${attempt + 1}/${MAX_RETRIES}) after ${delay}ms: ${url}`);
|
|
66
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
throw lastError;
|
|
70
|
+
}
|
|
71
|
+
async doFetch(url, cacheKey, ttl) {
|
|
26
72
|
logger.debug(`Fetching: ${url}`);
|
|
27
73
|
const authHeaders = await this.auth.getHeaders();
|
|
28
|
-
const
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
74
|
+
const controller = new AbortController();
|
|
75
|
+
const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
|
|
76
|
+
let res;
|
|
77
|
+
try {
|
|
78
|
+
res = await fetch(url, {
|
|
79
|
+
headers: {
|
|
80
|
+
Accept: "application/json",
|
|
81
|
+
...authHeaders,
|
|
82
|
+
},
|
|
83
|
+
signal: controller.signal,
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
catch (error) {
|
|
87
|
+
if (error instanceof DOMException && error.name === "AbortError") {
|
|
88
|
+
throw new BpsApiError(`Request timeout after ${FETCH_TIMEOUT_MS}ms`, 408, url);
|
|
89
|
+
}
|
|
90
|
+
throw error;
|
|
91
|
+
}
|
|
92
|
+
finally {
|
|
93
|
+
clearTimeout(timeout);
|
|
94
|
+
}
|
|
34
95
|
if (res.status === 401 || res.status === 403) {
|
|
35
96
|
throw new BpsAuthError();
|
|
36
97
|
}
|