@tandem-language-exchange/content-store 1.1.2 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +163 -20
- package/dist/chunk-EQ3DSPTJ.js +86 -0
- package/dist/chunk-EQ3DSPTJ.js.map +1 -0
- package/dist/{chunk-JBFJU4JA.js → chunk-LOCC2BXB.js} +118 -13
- package/dist/chunk-LOCC2BXB.js.map +1 -0
- package/dist/chunk-OTZLCMZ6.js +396 -0
- package/dist/chunk-OTZLCMZ6.js.map +1 -0
- package/dist/{chunk-YOREZCXB.js → chunk-PQJ2MGH7.js} +180 -29
- package/dist/chunk-PQJ2MGH7.js.map +1 -0
- package/dist/client/cli.js +31 -8
- package/dist/client/cli.js.map +1 -1
- package/dist/client/{fetch-bundles.js → fetch-content-bundles.js} +8 -7
- package/dist/client/fetch-content-bundles.js.map +1 -0
- package/dist/client/fetch-merged-translation-bundles.js +40 -0
- package/dist/client/fetch-merged-translation-bundles.js.map +1 -0
- package/dist/client/fetch-translation-bundles.js +43 -0
- package/dist/client/fetch-translation-bundles.js.map +1 -0
- package/dist/index-kfqHGgMO.d.ts +118 -0
- package/dist/index.d.ts +1 -1
- package/dist/node.browser.js +20 -6
- package/dist/node.browser.js.map +1 -1
- package/dist/node.d.ts +18 -5
- package/dist/node.js +527 -41
- package/dist/node.js.map +1 -1
- package/package.json +25 -11
- package/dist/chunk-JBFJU4JA.js.map +0 -1
- package/dist/chunk-UWGOF36L.js +0 -85
- package/dist/chunk-UWGOF36L.js.map +0 -1
- package/dist/chunk-YOREZCXB.js.map +0 -1
- package/dist/client/fetch-bundles.js.map +0 -1
- package/dist/index-DJXkO17k.d.ts +0 -83
package/README.md
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
# Content Store
|
|
2
2
|
|
|
3
|
-
SDK for fetching CMS
|
|
3
|
+
SDK for fetching **CMS** and **translation** bundles from Amazon S3 and, for CMS data, querying them locally from the filesystem. CMS bundles come from syncs of **Contentful** or **Sanity**; translation bundles come from **Lingohub** projects synced to S3 (see the [Server & CLI README](src/server/README.md) for the upload pipeline).
|
|
4
4
|
|
|
5
|
-
For the
|
|
5
|
+
For the Express API, server CLI, scheduling, and deployment, see the [Server & CLI README](src/server/README.md).
|
|
6
6
|
|
|
7
7
|
### Package entry points
|
|
8
8
|
|
|
9
9
|
- **`@tandem-language-exchange/content-store`** (default) — **types only** at runtime. Safe to import from shared code that Next.js, Vite, or Turbopack may bundle for the browser.
|
|
10
|
-
- **`@tandem-language-exchange/content-store/node`** — `ContentStoreSDK`, `
|
|
10
|
+
- **`@tandem-language-exchange/content-store/node`** — `ContentStoreSDK`, `fetchCmsBundles`, `fetchTranslationBundles`, `fetchMergedTranslationBundles`, `queryCmsBundle`, `ContentStore`, `getDefaultS3RetryConfig`, and `trimDepth`. Real implementations use the filesystem and S3 and run only under the Node (`node`) export condition (Route Handlers, `getServerSideProps`, CLI, etc.).
|
|
11
11
|
|
|
12
12
|
For **browser** bundles (including anything imported from `_app.tsx`, client components, or shared modules that reach the client graph), bundlers should resolve the `browser` / `edge-light` conditions to a **stub** that does not import `fs`. That stub throws if you call server-only APIs; **`trimDepth` is fully implemented** and safe on the client.
|
|
13
13
|
|
|
@@ -16,9 +16,11 @@ For the server, CLI, and deployment documentation, see the [Server & CLI README]
|
|
|
16
16
|
## Installation
|
|
17
17
|
|
|
18
18
|
```bash
|
|
19
|
-
npm install content-store
|
|
19
|
+
npm install @tandem-language-exchange/content-store
|
|
20
20
|
```
|
|
21
21
|
|
|
22
|
+
Use the same scoped name in `package.json` dependencies and in `npx` / import paths.
|
|
23
|
+
|
|
22
24
|
## Initialisation
|
|
23
25
|
|
|
24
26
|
```typescript
|
|
@@ -70,12 +72,12 @@ const sdk = new ContentStoreSDK({
|
|
|
70
72
|
|
|
71
73
|
---
|
|
72
74
|
|
|
73
|
-
## `
|
|
75
|
+
## `fetchCmsBundles(options)`
|
|
74
76
|
|
|
75
77
|
Downloads the latest content bundles from S3 and saves them as JSON files to `outputDir`.
|
|
76
78
|
|
|
77
79
|
```typescript
|
|
78
|
-
const files = await sdk.
|
|
80
|
+
const files = await sdk.fetchCmsBundles({
|
|
79
81
|
cms: 'contentful',
|
|
80
82
|
contentTypes: ['gridLayout', 'iconWithText', 'page'],
|
|
81
83
|
});
|
|
@@ -87,6 +89,7 @@ const files = await sdk.fetchBundles({
|
|
|
87
89
|
| --- | --- | --- |
|
|
88
90
|
| `cms` | `'contentful' \| 'sanity'` | Which CMS the bundles were synced from |
|
|
89
91
|
| `contentTypes` | `string[]` | Content types to download |
|
|
92
|
+
| `retry` | `S3RetryConfig` | Optional. Overrides [S3 download retries](#s3-download-retries). |
|
|
90
93
|
|
|
91
94
|
**Returns:** `Record<string, string>` — a map of content type to absolute file path.
|
|
92
95
|
|
|
@@ -100,12 +103,82 @@ const files = await sdk.fetchBundles({
|
|
|
100
103
|
|
|
101
104
|
Files are written to `outputDir` with the naming pattern `{cms}-{contentType}.json`.
|
|
102
105
|
|
|
103
|
-
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## `fetchTranslationBundles(options)`
|
|
109
|
+
|
|
110
|
+
Downloads translation objects from S3. The **sync** stores each Lingohub file **verbatim** (raw UTF-8); this call **parses** each file (JSON / `.strings` / Android XML per `src/shared/lingohub.ts`) and writes **normalized JSON** under `outputDir` (see file naming below).
|
|
111
|
+
|
|
112
|
+
```typescript
|
|
113
|
+
const files = await sdk.fetchTranslationBundles({
|
|
114
|
+
projects: ['tandem', 'tandem-(website)'],
|
|
115
|
+
locales: ['en', 'de'], // omit or leave empty to use the package default locale list
|
|
116
|
+
});
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
**Parameters:**
|
|
120
|
+
|
|
121
|
+
| Field | Type | Description |
|
|
122
|
+
| --- | --- | --- |
|
|
123
|
+
| `projects` | `string[]` | Lingohub project ids to fetch (must match keys configured in the package, e.g. `tandem`, `tandem-(android)`). |
|
|
124
|
+
| `locales` | `string[]` | Optional. Locale codes to fetch (e.g. `pt-br`, `zh-hans`). If omitted or empty, a built-in default list is used. |
|
|
125
|
+
| `retry` | `S3RetryConfig` | Optional. Overrides [S3 download retries](#s3-download-retries). |
|
|
126
|
+
|
|
127
|
+
**Returns:** `TranslationBundleInfo` — nested map **`project → S3 object key → absolute file path`** on disk.
|
|
128
|
+
|
|
129
|
+
```typescript
|
|
130
|
+
{
|
|
131
|
+
'tandem-(website)': {
|
|
132
|
+
'lingohub-tandem-(website).en.json': '/abs/path/to/content-cache/lingohub-tandem-(website).en.json',
|
|
133
|
+
'lingohub-tandem-(website).AI.en.json': '/abs/path/to/...'
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
**S3 object keys** follow `lingohub-{project}.{fileName}` where `{fileName}` is the Lingohub resource template with `[locale]` replaced by the **mapped** locale when a resource defines `localeMapping` (same rules as the server sync). On disk, non-`.json` keys gain a trailing `.json` (e.g. `…en.strings` → `…en.strings.json`) containing the parsed structure as JSON.
|
|
139
|
+
|
|
140
|
+
Downloads are run **in parallel** (per project); [S3 download retries](#s3-download-retries) apply by default.
|
|
141
|
+
|
|
142
|
+
---
|
|
143
|
+
|
|
144
|
+
## `fetchMergedTranslationBundles(options)`
|
|
145
|
+
|
|
146
|
+
Same **`projects`**, **`locales`**, and **`retry`** as `fetchTranslationBundles`. Downloads and parses every matching resource, flattens each file to string key/value pairs, then **merges** all pairs per **catalog locale** into a single file **`{locale}.json`** in `outputDir` (e.g. `en.json`). Duplicate keys across resources or projects: **last wins** (order: projects → resources → locales loop).
|
|
147
|
+
|
|
148
|
+
**Returns:** `Record<string, string>` — locale code → absolute path of the merged file.
|
|
149
|
+
|
|
150
|
+
```typescript
|
|
151
|
+
const mergedPaths = await sdk.fetchMergedTranslationBundles({
|
|
152
|
+
projects: ['tandem-(new-website)', 'tandem-(website)'],
|
|
153
|
+
locales: ['en'],
|
|
154
|
+
});
|
|
155
|
+
// mergedPaths.en → path to one big en.json
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
---
|
|
159
|
+
|
|
160
|
+
## S3 download retries
|
|
161
|
+
|
|
162
|
+
`fetchCmsBundles`, `fetchTranslationBundles`, and `fetchMergedTranslationBundles` use retry + exponential backoff on **transient** S3/network failures (for example HTTP 503 “Slow Down”, throttling, timeouts). They do **not** retry clear client errors such as **404** (missing key).
|
|
163
|
+
|
|
164
|
+
Default limits are read from the environment (highest precedence first):
|
|
165
|
+
|
|
166
|
+
| Variable | Fallback | Purpose |
|
|
167
|
+
| --- | --- | --- |
|
|
168
|
+
| `S3_RETRY_MAX_RETRIES` | `RETRY_MAX_RETRIES` (default `5`) | Maximum retry attempts after the first try |
|
|
169
|
+
| `S3_RETRY_BASE_DELAY_MS` | `RETRY_BASE_DELAY_MS` (default `1000`) | Base delay for exponential backoff |
|
|
170
|
+
| `S3_RETRY_MAX_DELAY_MS` | `RETRY_MAX_DELAY_MS` (default `60000`) | Cap on backoff delay |
|
|
171
|
+
|
|
172
|
+
Override per call with **`retry: { maxRetries, baseDelayMs, maxDelayMs }`**, or import **`getDefaultS3RetryConfig()`** to merge with your own defaults.
|
|
173
|
+
|
|
174
|
+
---
|
|
175
|
+
|
|
176
|
+
## `queryCmsBundle(cms, contentType, options?)`
|
|
104
177
|
|
|
105
178
|
Reads a previously fetched bundle from the local filesystem and returns a filtered, shaped result set.
|
|
106
179
|
|
|
107
180
|
```typescript
|
|
108
|
-
const results = await sdk.
|
|
181
|
+
const results = await sdk.queryCmsBundle('contentful', 'gridLayout', {
|
|
109
182
|
fields: { columns: '2' },
|
|
110
183
|
select: ['title', 'bodyBefore'],
|
|
111
184
|
limit: 10,
|
|
@@ -231,10 +304,17 @@ Query options are applied in this order:
|
|
|
231
304
|
|
|
232
305
|
## Standalone functions
|
|
233
306
|
|
|
234
|
-
The
|
|
307
|
+
The same operations are available as standalone imports (no `ContentStoreSDK` wrapper):
|
|
235
308
|
|
|
236
309
|
```typescript
|
|
237
|
-
import {
|
|
310
|
+
import {
|
|
311
|
+
fetchCmsBundles,
|
|
312
|
+
fetchTranslationBundles,
|
|
313
|
+
fetchMergedTranslationBundles,
|
|
314
|
+
queryCmsBundle,
|
|
315
|
+
getDefaultS3RetryConfig,
|
|
316
|
+
ContentStore,
|
|
317
|
+
} from '@tandem-language-exchange/content-store/node';
|
|
238
318
|
|
|
239
319
|
const store = new ContentStore({
|
|
240
320
|
bucket: 'beta-content-store',
|
|
@@ -243,12 +323,23 @@ const store = new ContentStore({
|
|
|
243
323
|
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
|
|
244
324
|
});
|
|
245
325
|
|
|
246
|
-
await
|
|
326
|
+
await fetchCmsBundles(store, './content-cache', {
|
|
247
327
|
cms: 'contentful',
|
|
248
328
|
contentTypes: ['gridLayout'],
|
|
249
329
|
});
|
|
250
330
|
|
|
251
|
-
|
|
331
|
+
await fetchTranslationBundles(store, './content-cache', {
|
|
332
|
+
projects: ['tandem-(website)'],
|
|
333
|
+
locales: ['en', 'fr'],
|
|
334
|
+
retry: getDefaultS3RetryConfig(),
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
await fetchMergedTranslationBundles(store, './content-cache/merged', {
|
|
338
|
+
projects: ['tandem-(new-website)', 'tandem-(website)'],
|
|
339
|
+
locales: ['en'],
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
const results = await queryCmsBundle('./content-cache', 'contentful', 'gridLayout', {
|
|
252
343
|
fields: { columns: '2' },
|
|
253
344
|
limit: 5,
|
|
254
345
|
});
|
|
@@ -269,14 +360,19 @@ const sdk = new ContentStoreSDK({
|
|
|
269
360
|
outputDir: './.content-cache',
|
|
270
361
|
});
|
|
271
362
|
|
|
272
|
-
// 1. Pull latest bundles from S3 to disk
|
|
273
|
-
await sdk.
|
|
363
|
+
// 1. Pull latest CMS bundles from S3 to disk
|
|
364
|
+
await sdk.fetchCmsBundles({
|
|
274
365
|
cms: 'contentful',
|
|
275
366
|
contentTypes: ['page', 'gridLayout'],
|
|
276
367
|
});
|
|
277
368
|
|
|
278
|
-
//
|
|
279
|
-
|
|
369
|
+
// Optional: pull translation bundles written by the Lingohub → S3 sync
|
|
370
|
+
await sdk.fetchTranslationBundles({
|
|
371
|
+
projects: ['tandem-(website)'],
|
|
372
|
+
});
|
|
373
|
+
|
|
374
|
+
// 2. Query CMS bundle locally — no further network calls
|
|
375
|
+
const grids = await sdk.queryCmsBundle('contentful', 'gridLayout', {
|
|
280
376
|
fields: { columns: '2' },
|
|
281
377
|
select: ['title', 'refs'],
|
|
282
378
|
include: 2,
|
|
@@ -287,7 +383,7 @@ console.log(grids);
|
|
|
287
383
|
```
|
|
288
384
|
## CLI
|
|
289
385
|
|
|
290
|
-
The package ships
|
|
386
|
+
The package ships **`fetch-content-bundles`**, **`fetch-translation-bundles`**, and **`fetch-merged-translation-bundles`** as **`bin`** commands. Additional commands live in **`dist/client/cli.js`** (`fetch-cms`, `query-cms`, …); run with **`node node_modules/@tandem-language-exchange/content-store/dist/client/cli.js <command>`** or npm scripts.
|
|
291
387
|
|
|
292
388
|
### `fetch-content-bundles` — download bundles from S3
|
|
293
389
|
|
|
@@ -313,12 +409,49 @@ Files are written as `{cms}-{contentType}.json` inside the output directory.
|
|
|
313
409
|
}
|
|
314
410
|
```
|
|
315
411
|
|
|
316
|
-
|
|
412
|
+
All **`bin`** tools read S3 settings from [S3 config via environment variables](#s3-config-via-environment-variables).
|
|
317
413
|
|
|
318
|
-
|
|
414
|
+
### `fetch-translation-bundles`
|
|
319
415
|
|
|
320
416
|
```bash
|
|
321
|
-
npx
|
|
417
|
+
npx fetch-translation-bundles --projects 'tandem-(new-website),tandem-(website)' --output ./content-cache
|
|
418
|
+
```
|
|
419
|
+
|
|
420
|
+
| Flag | Required | Default | Description |
|
|
421
|
+
| --- | --- | --- | --- |
|
|
422
|
+
| `--projects <list>` | Yes | | Comma-separated Lingohub project ids (must match `src/shared/lingohub.ts`) |
|
|
423
|
+
| `--locales <list>` | No | | Comma-separated locales; omit to use the built-in default list |
|
|
424
|
+
| `--output <dir>` | No | `./content-cache` | Output directory |
|
|
425
|
+
|
|
426
|
+
**zsh / bash:** Project ids often contain **`(`** and **`)`**. You must **quote** the argument when using `=` form, or use a space so the value is a separate token:
|
|
427
|
+
|
|
428
|
+
```bash
|
|
429
|
+
# Good — quoted
|
|
430
|
+
fetch-translation-bundles --projects='tandem-(new-website)' --output=src/data/cache
|
|
431
|
+
|
|
432
|
+
# Good — space form (value quoted or unambiguous)
|
|
433
|
+
fetch-translation-bundles --projects 'tandem-(new-website)' --output src/data/cache
|
|
434
|
+
|
|
435
|
+
# Bad in zsh — unquoted parentheses are shell syntax
|
|
436
|
+
fetch-translation-bundles --projects=tandem-(new-website) --output=src/data/cache
|
|
437
|
+
```
|
|
438
|
+
|
|
439
|
+
### `fetch-merged-translation-bundles`
|
|
440
|
+
|
|
441
|
+
Writes merged **`{locale}.json`** files (string key/value map; duplicate keys: last wins).
|
|
442
|
+
|
|
443
|
+
```bash
|
|
444
|
+
npx fetch-merged-translation-bundles --projects 'tandem-(new-website),tandem-(website)' --locales en --output ./content-cache/merged
|
|
445
|
+
```
|
|
446
|
+
|
|
447
|
+
Alternatively call **`fetchTranslationBundles`** / **`fetchMergedTranslationBundles`** from a Node script, or use the server’s **`POST /getTranslationBundles`** API (see [Server & CLI README](src/server/README.md)).
|
|
448
|
+
|
|
449
|
+
### `query-cms` — query a local bundle
|
|
450
|
+
|
|
451
|
+
Reads a previously fetched bundle from disk and prints JSON to stdout. This command is **not** a separate `bin`; run the built client CLI (after `npm install` of this package):
|
|
452
|
+
|
|
453
|
+
```bash
|
|
454
|
+
node node_modules/@tandem-language-exchange/content-store/dist/client/cli.js query-cms \
|
|
322
455
|
--cms contentful --type gridLayout \
|
|
323
456
|
--fields '{"columns":"2"}' \
|
|
324
457
|
--select title,bodyBefore \
|
|
@@ -326,6 +459,16 @@ npx @tandem-language-exchange/content-store query \
|
|
|
326
459
|
--include 2
|
|
327
460
|
```
|
|
328
461
|
|
|
462
|
+
**Typical `package.json` shortcut:**
|
|
463
|
+
|
|
464
|
+
```json
|
|
465
|
+
"scripts": {
|
|
466
|
+
"query:cms": "node ./node_modules/@tandem-language-exchange/content-store/dist/client/cli.js query-cms"
|
|
467
|
+
}
|
|
468
|
+
```
|
|
469
|
+
|
|
470
|
+
Then: `npm run query:cms -- --cms contentful --type gridLayout …`
|
|
471
|
+
|
|
329
472
|
| Flag | Required | Default | Description |
|
|
330
473
|
| --- | --- | --- | --- |
|
|
331
474
|
| `--cms <provider>` | Yes | | `contentful` or `sanity` |
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/shared/s3-retry.ts
|
|
4
|
+
function computeDelay(attempt, baseDelayMs, maxDelayMs) {
|
|
5
|
+
const exponential = baseDelayMs * Math.pow(2, attempt);
|
|
6
|
+
const jitter = Math.random() * baseDelayMs;
|
|
7
|
+
return Math.min(exponential + jitter, maxDelayMs);
|
|
8
|
+
}
|
|
9
|
+
function getDefaultS3RetryConfig() {
|
|
10
|
+
return {
|
|
11
|
+
maxRetries: parseInt(
|
|
12
|
+
process.env.S3_RETRY_MAX_RETRIES ?? process.env.RETRY_MAX_RETRIES ?? "5",
|
|
13
|
+
10
|
|
14
|
+
),
|
|
15
|
+
baseDelayMs: parseInt(
|
|
16
|
+
process.env.S3_RETRY_BASE_DELAY_MS ?? process.env.RETRY_BASE_DELAY_MS ?? "1000",
|
|
17
|
+
10
|
|
18
|
+
),
|
|
19
|
+
maxDelayMs: parseInt(
|
|
20
|
+
process.env.S3_RETRY_MAX_DELAY_MS ?? process.env.RETRY_MAX_DELAY_MS ?? "60000",
|
|
21
|
+
10
|
|
22
|
+
)
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
function isRetryableS3DownloadError(err) {
|
|
26
|
+
if (err === null || err === void 0) return false;
|
|
27
|
+
if (typeof err === "object") {
|
|
28
|
+
const e = err;
|
|
29
|
+
const status = e.$metadata?.httpStatusCode;
|
|
30
|
+
if (status === 404 || status === 403 || status === 401 || status === 400) {
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
if (status !== void 0 && (status === 408 || status === 429 || status === 500 || status === 502 || status === 503 || status === 504)) {
|
|
34
|
+
return true;
|
|
35
|
+
}
|
|
36
|
+
const name = e.name ?? "";
|
|
37
|
+
const code = e.Code ?? "";
|
|
38
|
+
if (/SlowDown|Throttl|Timeout|TooManyRequests|ServiceUnavailable|InternalError/i.test(
|
|
39
|
+
name
|
|
40
|
+
) || /SlowDown|Throttl/i.test(code)) {
|
|
41
|
+
return true;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (err instanceof Error) {
|
|
45
|
+
const m = err.message;
|
|
46
|
+
if (/ECONNRESET|ETIMEDOUT|EPIPE|ECONNREFUSED|socket hang up|getaddrinfo/i.test(
|
|
47
|
+
m
|
|
48
|
+
)) {
|
|
49
|
+
return true;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
async function withS3Retry(fn, { maxRetries, baseDelayMs, maxDelayMs }) {
|
|
55
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
56
|
+
try {
|
|
57
|
+
return await fn();
|
|
58
|
+
} catch (err) {
|
|
59
|
+
if (!isRetryableS3DownloadError(err)) {
|
|
60
|
+
throw err;
|
|
61
|
+
}
|
|
62
|
+
if (attempt === maxRetries) {
|
|
63
|
+
throw err;
|
|
64
|
+
}
|
|
65
|
+
const delay = computeDelay(attempt, baseDelayMs, maxDelayMs);
|
|
66
|
+
console.warn(
|
|
67
|
+
` S3 request failed (attempt ${attempt + 1}/${maxRetries + 1}): ${err instanceof Error ? err.message : String(err)}. Retrying in ${Math.round(delay)}ms\u2026`
|
|
68
|
+
);
|
|
69
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
throw new Error("withS3Retry: unreachable");
|
|
73
|
+
}
|
|
74
|
+
async function downloadWithRetry(store, key, cfg) {
|
|
75
|
+
return withS3Retry(() => store.download(key), cfg);
|
|
76
|
+
}
|
|
77
|
+
async function downloadRawWithRetry(store, key, cfg) {
|
|
78
|
+
return withS3Retry(() => store.downloadRaw(key), cfg);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export {
|
|
82
|
+
getDefaultS3RetryConfig,
|
|
83
|
+
downloadWithRetry,
|
|
84
|
+
downloadRawWithRetry
|
|
85
|
+
};
|
|
86
|
+
//# sourceMappingURL=chunk-EQ3DSPTJ.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/shared/s3-retry.ts"],"sourcesContent":["import type { ContentStore } from './s3';\n\n/** Retry/backoff for S3 reads (aligned with CMS `RetryConfig` shape). */\nexport interface S3RetryConfig {\n maxRetries: number;\n baseDelayMs: number;\n maxDelayMs: number;\n}\n\nfunction computeDelay(\n attempt: number,\n baseDelayMs: number,\n maxDelayMs: number,\n): number {\n const exponential = baseDelayMs * Math.pow(2, attempt);\n const jitter = Math.random() * baseDelayMs;\n return Math.min(exponential + jitter, maxDelayMs);\n}\n\n/**\n * Default S3 download retry policy from env.\n * `S3_RETRY_*` overrides `RETRY_*` when set.\n */\nexport function getDefaultS3RetryConfig(): S3RetryConfig {\n return {\n maxRetries: parseInt(\n process.env.S3_RETRY_MAX_RETRIES ??\n process.env.RETRY_MAX_RETRIES ??\n '5',\n 10,\n ),\n baseDelayMs: parseInt(\n process.env.S3_RETRY_BASE_DELAY_MS ??\n process.env.RETRY_BASE_DELAY_MS ??\n '1000',\n 10,\n ),\n maxDelayMs: parseInt(\n process.env.S3_RETRY_MAX_DELAY_MS ??\n process.env.RETRY_MAX_DELAY_MS ??\n '60000',\n 10,\n ),\n };\n}\n\n/**\n * True when a failed S3 GET may succeed after a short wait (503 Slow Down,\n * transient network, throttling). Never true for definitive client errors (404, etc.).\n */\nexport function isRetryableS3DownloadError(err: unknown): boolean {\n if (err === null || err === undefined) return false;\n\n if (typeof err === 'object') {\n const e = err as {\n name?: string;\n Code?: string;\n $metadata?: { httpStatusCode?: number };\n };\n const status = e.$metadata?.httpStatusCode;\n if (status === 404 || status === 403 || status === 401 || status === 400) {\n return false;\n }\n if (\n status !== undefined &&\n (status === 408 ||\n status === 429 ||\n status === 500 ||\n status === 502 ||\n status === 503 ||\n status === 504)\n ) {\n return true;\n }\n\n const name = e.name ?? '';\n const code = e.Code ?? '';\n if (\n /SlowDown|Throttl|Timeout|TooManyRequests|ServiceUnavailable|InternalError/i.test(\n name,\n ) ||\n /SlowDown|Throttl/i.test(code)\n ) {\n return true;\n }\n }\n\n if (err instanceof Error) {\n const m = err.message;\n if (\n /ECONNRESET|ETIMEDOUT|EPIPE|ECONNREFUSED|socket hang up|getaddrinfo/i.test(\n m,\n )\n ) {\n return true;\n }\n }\n\n return false;\n}\n\n/**\n * Runs `fn` with retries when `isRetryableS3DownloadError` applies; otherwise throws immediately.\n */\nexport async function withS3Retry<T>(\n fn: () => Promise<T>,\n { maxRetries, baseDelayMs, maxDelayMs }: S3RetryConfig,\n): Promise<T> {\n for (let attempt = 0; attempt <= maxRetries; attempt++) {\n try {\n return await fn();\n } catch (err) {\n if (!isRetryableS3DownloadError(err)) {\n throw err;\n }\n if (attempt === maxRetries) {\n throw err;\n }\n\n const delay = computeDelay(attempt, baseDelayMs, maxDelayMs);\n console.warn(\n ` S3 request failed (attempt ${attempt + 1}/${maxRetries + 1}): ${\n err instanceof Error ? err.message : String(err)\n }. Retrying in ${Math.round(delay)}ms…`,\n );\n await new Promise((resolve) => setTimeout(resolve, delay));\n }\n }\n\n throw new Error('withS3Retry: unreachable');\n}\n\nexport async function downloadWithRetry(\n store: ContentStore,\n key: string,\n cfg: S3RetryConfig,\n): Promise<unknown> {\n return withS3Retry(() => store.download(key), cfg);\n}\n\nexport async function downloadRawWithRetry(\n store: ContentStore,\n key: string,\n cfg: S3RetryConfig,\n): Promise<string> {\n return withS3Retry(() => store.downloadRaw(key), cfg);\n}\n"],"mappings":";;;AASA,SAAS,aACP,SACA,aACA,YACQ;AACR,QAAM,cAAc,cAAc,KAAK,IAAI,GAAG,OAAO;AACrD,QAAM,SAAS,KAAK,OAAO,IAAI;AAC/B,SAAO,KAAK,IAAI,cAAc,QAAQ,UAAU;AAClD;AAMO,SAAS,0BAAyC;AACvD,SAAO;AAAA,IACL,YAAY;AAAA,MACV,QAAQ,IAAI,wBACV,QAAQ,IAAI,qBACZ;AAAA,MACF;AAAA,IACF;AAAA,IACA,aAAa;AAAA,MACX,QAAQ,IAAI,0BACV,QAAQ,IAAI,uBACZ;AAAA,MACF;AAAA,IACF;AAAA,IACA,YAAY;AAAA,MACV,QAAQ,IAAI,yBACV,QAAQ,IAAI,sBACZ;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAMO,SAAS,2BAA2B,KAAuB;AAChE,MAAI,QAAQ,QAAQ,QAAQ,OAAW,QAAO;AAE9C,MAAI,OAAO,QAAQ,UAAU;AAC3B,UAAM,IAAI;AAKV,UAAM,SAAS,EAAE,WAAW;AAC5B,QAAI,WAAW,OAAO,WAAW,OAAO,WAAW,OAAO,WAAW,KAAK;AACxE,aAAO;AAAA,IACT;AACA,QACE,WAAW,WACV,WAAW,OACV,WAAW,OACX,WAAW,OACX,WAAW,OACX,WAAW,OACX,WAAW,MACb;AACA,aAAO;AAAA,IACT;AAEA,UAAM,OAAO,EAAE,QAAQ;AACvB,UAAM,OAAO,EAAE,QAAQ;AACvB,QACE,6EAA6E;AAAA,MAC3E;AAAA,IACF,KACA,oBAAoB,KAAK,IAAI,GAC7B;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAEA,MAAI,eAAe,OAAO;AACxB,UAAM,IAAI,IAAI;AACd,QACE,sEAAsE;AAAA,MACpE;AAAA,IACF,GACA;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;AAKA,eAAsB,YACpB,IACA,EAAE,YAAY,aAAa,WAAW,GAC1B;AACZ,WAAS,UAAU,GAAG,WAAW,YAAY,WAAW;AACtD,QAAI;AACF,aAAO,MAAM,GAAG;AAAA,IAClB,SAAS,KAAK;AACZ,UAAI,CAAC,2BAA2B,GAAG,GAAG;AACpC,cAAM;AAAA,MACR;AACA,UAAI,YAAY,YAAY;AAC1B,cAAM;AAAA,MACR;AAEA,YAAM,QAAQ,aAAa,SAAS,aAAa,UAAU;AAC3D,cAAQ;AAAA,QACN,gCAAgC,UAAU,CAAC,IAAI,aAAa,CAAC,MAC3D,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CACjD,iBAAiB,KAAK,MAAM,KAAK,CAAC;AAAA,MACpC;AACA,YAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,KAAK,CAAC;AAAA,IAC3D;AAAA,EACF;AAEA,QAAM,IAAI,MAAM,0BAA0B;AAC5C;AAEA,eAAsB,kBACpB,OACA,KACA,KACkB;AAClB,SAAO,YAAY,MAAM,MAAM,SAAS,GAAG,GAAG,GAAG;AACnD;AAEA,eAAsB,qBACpB,OACA,KACA,KACiB;AACjB,SAAO,YAAY,MAAM,MAAM,YAAY,GAAG,GAAG,GAAG;AACtD;","names":[]}
|
|
@@ -1,7 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
downloadRawWithRetry,
|
|
4
|
+
downloadWithRetry,
|
|
5
|
+
getDefaultS3RetryConfig
|
|
6
|
+
} from "./chunk-EQ3DSPTJ.js";
|
|
7
|
+
import {
|
|
8
|
+
allProjects,
|
|
9
|
+
buildCmsObjectKey,
|
|
10
|
+
buildTranslationObjectKey,
|
|
11
|
+
config,
|
|
12
|
+
defaultLocales,
|
|
13
|
+
parseTranslationResourceRaw,
|
|
14
|
+
toFlatStringMap,
|
|
15
|
+
translationJsonOutputPath
|
|
16
|
+
} from "./chunk-OTZLCMZ6.js";
|
|
5
17
|
|
|
6
18
|
// src/client/config.ts
|
|
7
19
|
import dotenv from "dotenv";
|
|
@@ -101,25 +113,116 @@ function setNestedAt(target, dottedPath, value) {
|
|
|
101
113
|
}
|
|
102
114
|
setNestedAt(nested, rest, value);
|
|
103
115
|
}
|
|
104
|
-
async function
|
|
116
|
+
async function fetchCmsBundles(store, outputDir, options) {
|
|
105
117
|
const { cms, contentTypes } = options;
|
|
118
|
+
const retry = options.retry ?? getDefaultS3RetryConfig();
|
|
106
119
|
await fs.mkdir(outputDir, { recursive: true });
|
|
107
120
|
const result = {};
|
|
108
121
|
await Promise.all(
|
|
109
122
|
contentTypes.map(async (contentType) => {
|
|
110
|
-
const key =
|
|
111
|
-
const data = await store
|
|
112
|
-
const filePath = path.resolve(
|
|
113
|
-
outputDir,
|
|
114
|
-
`${cms}-${contentType}.json`
|
|
115
|
-
);
|
|
123
|
+
const key = buildCmsObjectKey(cms, contentType);
|
|
124
|
+
const data = await downloadWithRetry(store, key, retry);
|
|
125
|
+
const filePath = path.resolve(outputDir, key);
|
|
116
126
|
await fs.writeFile(filePath, JSON.stringify(data, null, 2), "utf-8");
|
|
117
127
|
result[contentType] = filePath;
|
|
118
128
|
})
|
|
119
129
|
);
|
|
120
130
|
return result;
|
|
121
131
|
}
|
|
122
|
-
async function
|
|
132
|
+
async function fetchTranslationBundles(store, outputDir, options) {
|
|
133
|
+
const { projects, locales } = options;
|
|
134
|
+
const retry = options.retry ?? getDefaultS3RetryConfig();
|
|
135
|
+
await fs.mkdir(outputDir, { recursive: true });
|
|
136
|
+
const result = {};
|
|
137
|
+
const localesToSync = locales && locales.length ? locales : defaultLocales;
|
|
138
|
+
await Promise.all(
|
|
139
|
+
projects.map(async (project) => {
|
|
140
|
+
const resources = allProjects[project];
|
|
141
|
+
if (!resources?.length) {
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
const resourceTasks = [];
|
|
145
|
+
for (const resource of resources) {
|
|
146
|
+
for (const loc of localesToSync) {
|
|
147
|
+
const locale = resource.localeMapping && resource.localeMapping[loc] ? resource.localeMapping[loc] : loc;
|
|
148
|
+
const objectKey = buildTranslationObjectKey(
|
|
149
|
+
project,
|
|
150
|
+
resource.fileName,
|
|
151
|
+
locale
|
|
152
|
+
);
|
|
153
|
+
resourceTasks.push({ objectKey, resource });
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
await Promise.all(
|
|
157
|
+
resourceTasks.map(async ({ objectKey, resource }) => {
|
|
158
|
+
const raw = await downloadRawWithRetry(store, objectKey, retry);
|
|
159
|
+
const parsed = parseTranslationResourceRaw(raw, resource);
|
|
160
|
+
const filePath = translationJsonOutputPath(outputDir, objectKey);
|
|
161
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
162
|
+
await fs.writeFile(
|
|
163
|
+
filePath,
|
|
164
|
+
JSON.stringify(parsed, null, 2),
|
|
165
|
+
"utf-8"
|
|
166
|
+
);
|
|
167
|
+
if (!result[project]) {
|
|
168
|
+
result[project] = {};
|
|
169
|
+
}
|
|
170
|
+
result[project][objectKey] = filePath;
|
|
171
|
+
})
|
|
172
|
+
);
|
|
173
|
+
})
|
|
174
|
+
);
|
|
175
|
+
return result;
|
|
176
|
+
}
|
|
177
|
+
async function fetchMergedTranslationBundles(store, outputDir, options) {
|
|
178
|
+
const { projects, locales } = options;
|
|
179
|
+
const retry = options.retry ?? getDefaultS3RetryConfig();
|
|
180
|
+
await fs.mkdir(outputDir, { recursive: true });
|
|
181
|
+
const localesToSync = locales && locales.length ? locales : defaultLocales;
|
|
182
|
+
const tasks = [];
|
|
183
|
+
for (const project of projects) {
|
|
184
|
+
const resources = allProjects[project];
|
|
185
|
+
if (!resources?.length) continue;
|
|
186
|
+
for (const resource of resources) {
|
|
187
|
+
for (const loc of localesToSync) {
|
|
188
|
+
const mappedLocale = resource.localeMapping && resource.localeMapping[loc] ? resource.localeMapping[loc] : loc;
|
|
189
|
+
const objectKey = buildTranslationObjectKey(
|
|
190
|
+
project,
|
|
191
|
+
resource.fileName,
|
|
192
|
+
mappedLocale
|
|
193
|
+
);
|
|
194
|
+
tasks.push({ catalogLocale: loc, objectKey, resource });
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
const results = await Promise.all(
|
|
199
|
+
tasks.map(async ({ catalogLocale, objectKey, resource }) => {
|
|
200
|
+
const raw = await downloadRawWithRetry(store, objectKey, retry);
|
|
201
|
+
const parsed = parseTranslationResourceRaw(raw, resource);
|
|
202
|
+
const stringMap = toFlatStringMap(parsed);
|
|
203
|
+
return { catalogLocale, stringMap };
|
|
204
|
+
})
|
|
205
|
+
);
|
|
206
|
+
const merged = {};
|
|
207
|
+
for (const loc of localesToSync) {
|
|
208
|
+
merged[loc] = {};
|
|
209
|
+
}
|
|
210
|
+
for (const { catalogLocale, stringMap } of results) {
|
|
211
|
+
Object.assign(merged[catalogLocale], stringMap);
|
|
212
|
+
}
|
|
213
|
+
const out = {};
|
|
214
|
+
for (const loc of localesToSync) {
|
|
215
|
+
const filePath = path.resolve(outputDir, `${loc}.json`);
|
|
216
|
+
await fs.writeFile(
|
|
217
|
+
filePath,
|
|
218
|
+
JSON.stringify(merged[loc], null, 2),
|
|
219
|
+
"utf-8"
|
|
220
|
+
);
|
|
221
|
+
out[loc] = filePath;
|
|
222
|
+
}
|
|
223
|
+
return out;
|
|
224
|
+
}
|
|
225
|
+
async function queryCmsBundle(outputDir, cms, contentType, options = {}) {
|
|
123
226
|
const filePath = path.resolve(
|
|
124
227
|
outputDir,
|
|
125
228
|
`${cms}-${contentType}.json`
|
|
@@ -158,7 +261,9 @@ async function queryBundle(outputDir, cms, contentType, options = {}) {
|
|
|
158
261
|
|
|
159
262
|
export {
|
|
160
263
|
config2 as config,
|
|
161
|
-
|
|
162
|
-
|
|
264
|
+
fetchCmsBundles,
|
|
265
|
+
fetchTranslationBundles,
|
|
266
|
+
fetchMergedTranslationBundles,
|
|
267
|
+
queryCmsBundle
|
|
163
268
|
};
|
|
164
|
-
//# sourceMappingURL=chunk-
|
|
269
|
+
//# sourceMappingURL=chunk-LOCC2BXB.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/client/config.ts","../src/shared/bundles.ts","../src/shared/trimDepth.ts"],"sourcesContent":["import dotenv from 'dotenv';\nimport type { S3Config, CMSProvider } from '../shared/types';\nimport {SharedConfig, config as sharedConfig} from '../shared/config';\n\ndotenv.config({ path: '.env.local' });\ndotenv.config();\n\nexport type { CMSProvider, S3Config };\n\nexport const config:SharedConfig = {\n ...sharedConfig\n};\n","import fs from 'node:fs/promises';\nimport path from 'node:path';\nimport type { CMSProvider } from './types';\nimport { buildCmsObjectKey, buildTranslationObjectKey, ContentStore } from './s3';\nimport {\n downloadRawWithRetry,\n downloadWithRetry,\n getDefaultS3RetryConfig,\n type S3RetryConfig,\n} from './s3-retry';\nimport { trimDepth } from './trimDepth';\nimport {\n allProjects,\n defaultLocales,\n type LingohubResource,\n} from './lingohub';\nimport {\n parseTranslationResourceRaw,\n toFlatStringMap,\n translationJsonOutputPath,\n} from './translationResource';\n\nexport { trimDepth } from './trimDepth';\nexport type { S3RetryConfig } from './s3-retry';\nexport { getDefaultS3RetryConfig } from './s3-retry';\n\nexport interface CmsBundleInfo {\n [key: string]: string;\n}\n\nexport interface TranslationBundleInfo {\n [key: string] : Record<string, string>;\n}\n\nexport interface BundleItem {\n [key: string]: any; // eslint-disable-line @typescript-eslint/no-explicit-any\n}\n\nexport interface FetchCmsBundlesOptions {\n cms: CMSProvider;\n contentTypes: string[];\n /** S3 GET retry; defaults via {@link getDefaultS3RetryConfig}. */\n retry?: S3RetryConfig;\n}\n\nexport interface FetchTranslationBundlesOptions {\n projects: string[];\n locales?: string[] | undefined;\n /** S3 GET retry; defaults via {@link getDefaultS3RetryConfig}. */\n retry?: S3RetryConfig;\n}\n\n/** One merged JSON file per catalog locale (`en.json`, …). Same options as {@link FetchTranslationBundlesOptions}. */\nexport type FetchMergedTranslationBundlesOptions = FetchTranslationBundlesOptions;\n\nexport interface QueryOptions {\n /**\n * Filter items by matching property values (exact equality, or array for IN).\n * Keys may use dot notation for nested paths, e.g. `{ 'meta._id': 'abc' }` matches\n * `item.meta._id`.\n *\n * Contentful-style operators on the path (before `[`):\n * - `{ 'field[exists]': true }` — field is present and non-empty (not null, undefined,\n * `''`, `[]`, or `{}`).\n * - `{ 'field[exists]': false }` — field is missing or empty (same emptiness rules).\n * Nested paths work, e.g. `{ 'blocks.hero[exists]': false }`.\n */\n fields?: Record<string, unknown>;\n /**\n * Properties to include in each result object. Omit to return all properties.\n * Keys may use dot notation; nested segments become nested objects in the result,\n * e.g. `['slug', 'meta._updatedAt']` → `{ slug, meta: { _updatedAt } }`.\n */\n select?: string[];\n /** Maximum number of items to return. */\n limit?: number;\n /**\n * How many levels deep to return.\n * - 1 = the item's own scalar properties only; nested objects/refs are nulled.\n * - 2 = the item including its direct references; refs inside those are nulled.\n * - 3 = three levels deep, and so on.\n * - Omit to return the full depth.\n */\n include?: number;\n}\n\n/** Read a value at a dotted path; returns whether every segment existed. */\nfunction getAtPath(\n obj: unknown,\n dottedPath: string,\n): { found: true; value: unknown } | { found: false } {\n const parts = dottedPath.split('.').filter((p) => p.length > 0);\n if (parts.length === 0) return { found: false };\n\n let cur: unknown = obj;\n for (const p of parts) {\n if (cur === null || typeof cur !== 'object' || Array.isArray(cur)) {\n return { found: false };\n }\n const rec = cur as Record<string, unknown>;\n if (!(p in rec)) return { found: false };\n cur = rec[p];\n }\n return { found: true, value: cur };\n}\n\nconst FIELD_OP_KEY = /^(.+)\\[([^\\]]+)]$/;\n\nfunction parseFieldKey(key: string): { path: string; operator?: string } {\n const m = FIELD_OP_KEY.exec(key);\n if (!m) return { path: key };\n return { path: m[1]!, operator: m[2]! };\n}\n\n/** True when a bundle value counts as “no content” (aligned with typical CMS “empty”). */\nfunction isEmptyValue(value: unknown): boolean {\n if (value === null || value === undefined) return true;\n if (value === '') return true;\n if (Array.isArray(value) && value.length === 0) {\n return true;\n }\n return typeof value === 'object' &&\n !Array.isArray(value) &&\n Object.keys(value).length === 0;\n\n}\n\nfunction matchesFieldFilter(\n item: BundleItem,\n key: string,\n expected: unknown,\n): boolean {\n const { path, operator } = parseFieldKey(key);\n const at = getAtPath(item, path);\n\n if (operator === 'exists') {\n if (expected !== true && expected !== false) return false;\n const empty = !at.found || isEmptyValue(at.value);\n return !expected ? empty : !empty;\n }\n\n const actual = at.found ? at.value : undefined;\n if (Array.isArray(expected)) return expected.includes(actual);\n return actual === expected;\n}\n\n/** Set `value` on `target` at a dotted path, creating plain objects as needed. */\nfunction setNestedAt(\n target: Record<string, unknown>,\n dottedPath: string,\n value: unknown,\n): void {\n const parts = dottedPath.split('.').filter((p) => p.length > 0);\n if (parts.length === 0) return;\n\n if (parts.length === 1) {\n target[parts[0]!] = value;\n return;\n }\n\n const head = parts[0]!;\n const rest = parts.slice(1).join('.');\n let nested = target[head];\n if (\n nested === null ||\n typeof nested !== 'object' ||\n Array.isArray(nested)\n ) {\n nested = {};\n target[head] = nested;\n }\n setNestedAt(nested as Record<string, unknown>, rest, value);\n}\n\n/**\n * Downloads the latest CMS bundles from S3 and writes them as JSON files to `outputDir`.\n *\n * @returns A map of contentType to absolute file path.\n */\nexport async function fetchCmsBundles(\n store: ContentStore,\n outputDir: string,\n options: FetchCmsBundlesOptions,\n): Promise<Record<string, string>> {\n const { cms, contentTypes } = options;\n const retry = options.retry ?? getDefaultS3RetryConfig();\n await fs.mkdir(outputDir, { recursive: true });\n\n const result: CmsBundleInfo = {};\n\n await Promise.all(\n contentTypes.map(async (contentType) => {\n const key = buildCmsObjectKey(cms, contentType);\n const data = await downloadWithRetry(store, key, retry);\n const filePath = path.resolve(outputDir, key);\n await fs.writeFile(filePath, JSON.stringify(data, null, 2), 'utf-8');\n result[contentType] = filePath;\n }),\n );\n\n return result;\n}\n\n/**\n * Downloads the latest translation bundles from S3 and writes them as JSON files to `outputDir`.\n *\n * @returns For each project, a map of S3 object key to absolute file path on disk.\n */\nexport async function fetchTranslationBundles(\n store: ContentStore,\n outputDir: string,\n options: FetchTranslationBundlesOptions,\n): Promise<TranslationBundleInfo> {\n const { projects, locales } = options;\n const retry = options.retry ?? getDefaultS3RetryConfig();\n await fs.mkdir(outputDir, { recursive: true });\n\n const result: TranslationBundleInfo = {};\n const localesToSync = locales && locales.length ? locales : defaultLocales;\n\n await Promise.all(\n projects.map(async (project) => {\n const resources = allProjects[project];\n if (!resources?.length) {\n return;\n }\n\n const resourceTasks: { objectKey: string; resource: LingohubResource }[] =\n [];\n for (const resource of resources) {\n for (const loc of localesToSync) {\n const locale =\n resource.localeMapping && resource.localeMapping[loc]\n ? resource.localeMapping[loc]\n : loc;\n const objectKey = buildTranslationObjectKey(\n project,\n resource.fileName,\n locale,\n );\n resourceTasks.push({ objectKey, resource });\n }\n }\n\n await Promise.all(\n resourceTasks.map(async ({ objectKey, resource }) => {\n const raw = await downloadRawWithRetry(store, objectKey, retry);\n const parsed = parseTranslationResourceRaw(raw, resource);\n const filePath = translationJsonOutputPath(outputDir, objectKey);\n await fs.mkdir(path.dirname(filePath), { recursive: true });\n await fs.writeFile(\n filePath,\n JSON.stringify(parsed, null, 2),\n 'utf-8',\n );\n if (!result[project]) {\n result[project] = {};\n }\n result[project][objectKey] = filePath;\n }),\n );\n }),\n );\n\n return result;\n}\n\n/**\n * Downloads translation bundles from S3, parses each raw Lingohub file, and merges all\n * key/value pairs per **catalog** locale into a single JSON file (e.g. `en.json`).\n * Duplicate keys across resources or projects: **last write wins** (iteration order:\n * projects → resources → locales).\n *\n * @returns Map of locale code to absolute path of the merged `{locale}.json` file.\n */\nexport async function fetchMergedTranslationBundles(\n store: ContentStore,\n outputDir: string,\n options: FetchMergedTranslationBundlesOptions,\n): Promise<Record<string, string>> {\n const { projects, locales } = options;\n const retry = options.retry ?? getDefaultS3RetryConfig();\n await fs.mkdir(outputDir, { recursive: true });\n\n const localesToSync = locales && locales.length ? locales : defaultLocales;\n\n type Task = {\n catalogLocale: string;\n objectKey: string;\n resource: LingohubResource;\n };\n const tasks: Task[] = [];\n\n for (const project of projects) {\n const resources = allProjects[project];\n if (!resources?.length) continue;\n for (const resource of resources) {\n for (const loc of localesToSync) {\n const mappedLocale =\n resource.localeMapping && resource.localeMapping[loc]\n ? resource.localeMapping[loc]\n : loc;\n const objectKey = buildTranslationObjectKey(\n project,\n resource.fileName,\n mappedLocale,\n );\n tasks.push({ catalogLocale: loc, objectKey, resource });\n }\n }\n }\n\n const results = await Promise.all(\n tasks.map(async ({ catalogLocale, objectKey, resource }) => {\n const raw = await downloadRawWithRetry(store, objectKey, retry);\n const parsed = parseTranslationResourceRaw(raw, resource);\n const stringMap = toFlatStringMap(parsed);\n return { catalogLocale, stringMap };\n }),\n );\n\n const merged: Record<string, Record<string, string>> = {};\n for (const loc of localesToSync) {\n merged[loc] = {};\n }\n for (const { catalogLocale, stringMap } of results) {\n Object.assign(merged[catalogLocale]!, stringMap);\n }\n\n const out: Record<string, string> = {};\n for (const loc of localesToSync) {\n const filePath = path.resolve(outputDir, `${loc}.json`);\n await fs.writeFile(\n filePath,\n JSON.stringify(merged[loc], null, 2),\n 'utf-8',\n );\n out[loc] = filePath;\n }\n\n return out;\n}\n\n/**\n * Queries a previously fetched bundle from the local filesystem.\n */\nexport async function queryCmsBundle(\n outputDir: string,\n cms: CMSProvider,\n contentType: string,\n options: QueryOptions = {},\n): Promise<unknown[]> {\n const filePath = path.resolve(\n outputDir,\n `${cms}-${contentType}.json`,\n );\n const raw = await fs.readFile(filePath, 'utf-8');\n let items:BundleItem[] = JSON.parse(raw) as Record<string, unknown>[];\n\n if (options.fields) {\n const filters = options.fields;\n items = items.filter((item) =>\n Object.entries(filters).every(([key, expected]) =>\n matchesFieldFilter(item, key, expected),\n ),\n );\n }\n\n if (options.limit !== undefined && options.limit > 0) {\n items = items.slice(0, options.limit);\n }\n\n if (options.include !== undefined) {\n items = items.map(\n (item) => trimDepth(item, options.include!) as Record<string, unknown>,\n );\n }\n\n if (options.select?.length) {\n const keys = options.select;\n items = items.map((item) => {\n const picked: Record<string, unknown> = {};\n for (const k of keys) {\n const at = getAtPath(item, k);\n if (at.found) setNestedAt(picked, k, at.value);\n }\n return picked;\n });\n }\n\n return items;\n}\n","/**\n * Trims nested object depth.\n *\n * `remaining` represents how many levels the current object is allowed.\n * - Scalar properties are always kept.\n * - Nested objects / arrays-of-objects consume one level. When `remaining`\n * drops to 1 they are replaced with `null` (no budget left for refs).\n */\nexport function trimDepth(value: unknown, remaining: number): unknown {\n if (value === null || value === undefined || typeof value !== 'object') {\n return value;\n }\n\n if (Array.isArray(value)) {\n return value.map((item) => trimDepth(item, remaining));\n }\n\n const obj = value as Record<string, unknown>;\n const result: Record<string, unknown> = {};\n\n for (const [k, v] of Object.entries(obj)) {\n if (v === null || v === undefined || typeof v !== 'object') {\n result[k] = v;\n } else if (remaining <= 1) {\n result[k] = null;\n } else if (Array.isArray(v)) {\n result[k] = v.map((item) => {\n if (item !== null && typeof item === 'object' && !Array.isArray(item)) {\n return trimDepth(item, remaining - 1);\n }\n return item;\n });\n } else {\n result[k] = trimDepth(v, remaining - 1);\n }\n }\n\n return result;\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA,OAAO,YAAY;AAInB,OAAO,OAAO,EAAE,MAAM,aAAa,CAAC;AACpC,OAAO,OAAO;AAIP,IAAMA,UAAsB;AAAA,EAC/B,GAAG;AACP;;;ACXA,OAAO,QAAQ;AACf,OAAO,UAAU;;;ACOV,SAAS,UAAU,OAAgB,WAA4B;AACpE,MAAI,UAAU,QAAQ,UAAU,UAAa,OAAO,UAAU,UAAU;AACtE,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,WAAO,MAAM,IAAI,CAAC,SAAS,UAAU,MAAM,SAAS,CAAC;AAAA,EACvD;AAEA,QAAM,MAAM;AACZ,QAAM,SAAkC,CAAC;AAEzC,aAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,GAAG,GAAG;AACxC,QAAI,MAAM,QAAQ,MAAM,UAAa,OAAO,MAAM,UAAU;AAC1D,aAAO,CAAC,IAAI;AAAA,IACd,WAAW,aAAa,GAAG;AACzB,aAAO,CAAC,IAAI;AAAA,IACd,WAAW,MAAM,QAAQ,CAAC,GAAG;AAC3B,aAAO,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS;AAC1B,YAAI,SAAS,QAAQ,OAAO,SAAS,YAAY,CAAC,MAAM,QAAQ,IAAI,GAAG;AACrE,iBAAO,UAAU,MAAM,YAAY,CAAC;AAAA,QACtC;AACA,eAAO;AAAA,MACT,CAAC;AAAA,IACH,OAAO;AACL,aAAO,CAAC,IAAI,UAAU,GAAG,YAAY,CAAC;AAAA,IACxC;AAAA,EACF;AAEA,SAAO;AACT;;;ADiDA,SAAS,UACP,KACA,YACoD;AACpD,QAAM,QAAQ,WAAW,MAAM,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AAC9D,MAAI,MAAM,WAAW,EAAG,QAAO,EAAE,OAAO,MAAM;AAE9C,MAAI,MAAe;AACnB,aAAW,KAAK,OAAO;AACrB,QAAI,QAAQ,QAAQ,OAAO,QAAQ,YAAY,MAAM,QAAQ,GAAG,GAAG;AACjE,aAAO,EAAE,OAAO,MAAM;AAAA,IACxB;AACA,UAAM,MAAM;AACZ,QAAI,EAAE,KAAK,KAAM,QAAO,EAAE,OAAO,MAAM;AACvC,UAAM,IAAI,CAAC;AAAA,EACb;AACA,SAAO,EAAE,OAAO,MAAM,OAAO,IAAI;AACnC;AAEA,IAAM,eAAe;AAErB,SAAS,cAAc,KAAkD;AACvE,QAAM,IAAI,aAAa,KAAK,GAAG;AAC/B,MAAI,CAAC,EAAG,QAAO,EAAE,MAAM,IAAI;AAC3B,SAAO,EAAE,MAAM,EAAE,CAAC,GAAI,UAAU,EAAE,CAAC,EAAG;AACxC;AAGA,SAAS,aAAa,OAAyB;AAC7C,MAAI,UAAU,QAAQ,UAAU,OAAW,QAAO;AAClD,MAAI,UAAU,GAAI,QAAO;AACzB,MAAI,MAAM,QAAQ,KAAK,KAAK,MAAM,WAAW,GAAG;AAC9C,WAAO;AAAA,EACT;AACA,SAAO,OAAO,UAAU,YACpB,CAAC,MAAM,QAAQ,KAAK,KACpB,OAAO,KAAK,KAAK,EAAE,WAAW;AAEpC;AAEA,SAAS,mBACP,MACA,KACA,UACS;AACT,QAAM,EAAE,MAAAC,OAAM,SAAS,IAAI,cAAc,GAAG;AAC5C,QAAM,KAAK,UAAU,MAAMA,KAAI;AAE/B,MAAI,aAAa,UAAU;AACzB,QAAI,aAAa,QAAQ,aAAa,MAAO,QAAO;AACpD,UAAM,QAAQ,CAAC,GAAG,SAAS,aAAa,GAAG,KAAK;AAChD,WAAO,CAAC,WAAW,QAAQ,CAAC;AAAA,EAC9B;AAEA,QAAM,SAAS,GAAG,QAAQ,GAAG,QAAQ;AACrC,MAAI,MAAM,QAAQ,QAAQ,EAAG,QAAO,SAAS,SAAS,MAAM;AAC5D,SAAO,WAAW;AACpB;AAGA,SAAS,YACP,QACA,YACA,OACM;AACN,QAAM,QAAQ,WAAW,MAAM,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AAC9D,MAAI,MAAM,WAAW,EAAG;AAExB,MAAI,MAAM,WAAW,GAAG;AACtB,WAAO,MAAM,CAAC,CAAE,IAAI;AACpB;AAAA,EACF;AAEA,QAAM,OAAO,MAAM,CAAC;AACpB,QAAM,OAAO,MAAM,MAAM,CAAC,EAAE,KAAK,GAAG;AACpC,MAAI,SAAS,OAAO,IAAI;AACxB,MACE,WAAW,QACX,OAAO,WAAW,YAClB,MAAM,QAAQ,MAAM,GACpB;AACA,aAAS,CAAC;AACV,WAAO,IAAI,IAAI;AAAA,EACjB;AACA,cAAY,QAAmC,MAAM,KAAK;AAC5D;AAOA,eAAsB,gBACpB,OACA,WACA,SACiC;AACjC,QAAM,EAAE,KAAK,aAAa,IAAI;AAC9B,QAAM,QAAQ,QAAQ,SAAS,wBAAwB;AACvD,QAAM,GAAG,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAE7C,QAAM,SAAwB,CAAC;AAE/B,QAAM,QAAQ;AAAA,IACZ,aAAa,IAAI,OAAO,gBAAgB;AACtC,YAAM,MAAM,kBAAkB,KAAK,WAAW;AAC9C,YAAM,OAAO,MAAM,kBAAkB,OAAO,KAAK,KAAK;AACtD,YAAM,WAAW,KAAK,QAAQ,WAAW,GAAG;AAC5C,YAAM,GAAG,UAAU,UAAU,KAAK,UAAU,MAAM,MAAM,CAAC,GAAG,OAAO;AACnE,aAAO,WAAW,IAAI;AAAA,IACxB,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAOA,eAAsB,wBACpB,OACA,WACA,SACgC;AAChC,QAAM,EAAE,UAAU,QAAQ,IAAI;AAC9B,QAAM,QAAQ,QAAQ,SAAS,wBAAwB;AACvD,QAAM,GAAG,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAE7C,QAAM,SAAgC,CAAC;AACvC,QAAM,gBAAgB,WAAW,QAAQ,SAAS,UAAU;AAE5D,QAAM,QAAQ;AAAA,IACZ,SAAS,IAAI,OAAO,YAAY;AAC9B,YAAM,YAAY,YAAY,OAAO;AACrC,UAAI,CAAC,WAAW,QAAQ;AACtB;AAAA,MACF;AAEA,YAAM,gBACJ,CAAC;AACH,iBAAW,YAAY,WAAW;AAChC,mBAAW,OAAO,eAAe;AAC/B,gBAAM,SACJ,SAAS,iBAAiB,SAAS,cAAc,GAAG,IAChD,SAAS,cAAc,GAAG,IAC1B;AACN,gBAAM,YAAY;AAAA,YAChB;AAAA,YACA,SAAS;AAAA,YACT;AAAA,UACF;AACA,wBAAc,KAAK,EAAE,WAAW,SAAS,CAAC;AAAA,QAC5C;AAAA,MACF;AAEA,YAAM,QAAQ;AAAA,QACZ,cAAc,IAAI,OAAO,EAAE,WAAW,SAAS,MAAM;AACnD,gBAAM,MAAM,MAAM,qBAAqB,OAAO,WAAW,KAAK;AAC9D,gBAAM,SAAS,4BAA4B,KAAK,QAAQ;AACxD,gBAAM,WAAW,0BAA0B,WAAW,SAAS;AAC/D,gBAAM,GAAG,MAAM,KAAK,QAAQ,QAAQ,GAAG,EAAE,WAAW,KAAK,CAAC;AAC1D,gBAAM,GAAG;AAAA,YACP;AAAA,YACA,KAAK,UAAU,QAAQ,MAAM,CAAC;AAAA,YAC9B;AAAA,UACF;AACA,cAAI,CAAC,OAAO,OAAO,GAAG;AACpB,mBAAO,OAAO,IAAI,CAAC;AAAA,UACrB;AACA,iBAAO,OAAO,EAAE,SAAS,IAAI;AAAA,QAC/B,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO;AACT;AAUA,eAAsB,8BACpB,OACA,WACA,SACiC;AACjC,QAAM,EAAE,UAAU,QAAQ,IAAI;AAC9B,QAAM,QAAQ,QAAQ,SAAS,wBAAwB;AACvD,QAAM,GAAG,MAAM,WAAW,EAAE,WAAW,KAAK,CAAC;AAE7C,QAAM,gBAAgB,WAAW,QAAQ,SAAS,UAAU;AAO5D,QAAM,QAAgB,CAAC;AAEvB,aAAW,WAAW,UAAU;AAC9B,UAAM,YAAY,YAAY,OAAO;AACrC,QAAI,CAAC,WAAW,OAAQ;AACxB,eAAW,YAAY,WAAW;AAChC,iBAAW,OAAO,eAAe;AAC/B,cAAM,eACJ,SAAS,iBAAiB,SAAS,cAAc,GAAG,IAChD,SAAS,cAAc,GAAG,IAC1B;AACN,cAAM,YAAY;AAAA,UAChB;AAAA,UACA,SAAS;AAAA,UACT;AAAA,QACF;AACA,cAAM,KAAK,EAAE,eAAe,KAAK,WAAW,SAAS,CAAC;AAAA,MACxD;AAAA,IACF;AAAA,EACF;AAEA,QAAM,UAAU,MAAM,QAAQ;AAAA,IAC5B,MAAM,IAAI,OAAO,EAAE,eAAe,WAAW,SAAS,MAAM;AAC1D,YAAM,MAAM,MAAM,qBAAqB,OAAO,WAAW,KAAK;AAC9D,YAAM,SAAS,4BAA4B,KAAK,QAAQ;AACxD,YAAM,YAAY,gBAAgB,MAAM;AACxC,aAAO,EAAE,eAAe,UAAU;AAAA,IACpC,CAAC;AAAA,EACH;AAEA,QAAM,SAAiD,CAAC;AACxD,aAAW,OAAO,eAAe;AAC/B,WAAO,GAAG,IAAI,CAAC;AAAA,EACjB;AACA,aAAW,EAAE,eAAe,UAAU,KAAK,SAAS;AAClD,WAAO,OAAO,OAAO,aAAa,GAAI,SAAS;AAAA,EACjD;AAEA,QAAM,MAA8B,CAAC;AACrC,aAAW,OAAO,eAAe;AAC/B,UAAM,WAAW,KAAK,QAAQ,WAAW,GAAG,GAAG,OAAO;AACtD,UAAM,GAAG;AAAA,MACP;AAAA,MACA,KAAK,UAAU,OAAO,GAAG,GAAG,MAAM,CAAC;AAAA,MACnC;AAAA,IACF;AACA,QAAI,GAAG,IAAI;AAAA,EACb;AAEA,SAAO;AACT;AAKA,eAAsB,eACpB,WACA,KACA,aACA,UAAwB,CAAC,GACL;AACpB,QAAM,WAAW,KAAK;AAAA,IACpB;AAAA,IACA,GAAG,GAAG,IAAI,WAAW;AAAA,EACvB;AACA,QAAM,MAAM,MAAM,GAAG,SAAS,UAAU,OAAO;AAC/C,MAAI,QAAqB,KAAK,MAAM,GAAG;AAEvC,MAAI,QAAQ,QAAQ;AAClB,UAAM,UAAU,QAAQ;AACxB,YAAQ,MAAM;AAAA,MAAO,CAAC,SACpB,OAAO,QAAQ,OAAO,EAAE;AAAA,QAAM,CAAC,CAAC,KAAK,QAAQ,MAC3C,mBAAmB,MAAM,KAAK,QAAQ;AAAA,MACxC;AAAA,IACF;AAAA,EACF;AAEA,MAAI,QAAQ,UAAU,UAAa,QAAQ,QAAQ,GAAG;AACpD,YAAQ,MAAM,MAAM,GAAG,QAAQ,KAAK;AAAA,EACtC;AAEA,MAAI,QAAQ,YAAY,QAAW;AACjC,YAAQ,MAAM;AAAA,MACZ,CAAC,SAAS,UAAU,MAAM,QAAQ,OAAQ;AAAA,IAC5C;AAAA,EACF;AAEA,MAAI,QAAQ,QAAQ,QAAQ;AAC1B,UAAM,OAAO,QAAQ;AACrB,YAAQ,MAAM,IAAI,CAAC,SAAS;AAC1B,YAAM,SAAkC,CAAC;AACzC,iBAAW,KAAK,MAAM;AACpB,cAAM,KAAK,UAAU,MAAM,CAAC;AAC5B,YAAI,GAAG,MAAO,aAAY,QAAQ,GAAG,GAAG,KAAK;AAAA,MAC/C;AACA,aAAO;AAAA,IACT,CAAC;AAAA,EACH;AAEA,SAAO;AACT;","names":["config","path"]}
|