@tandem-language-exchange/content-store 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +111 -21
- package/dist/{chunk-EGDGHYGI.js → chunk-UCBZUEUP.js} +221 -30
- package/dist/chunk-UCBZUEUP.js.map +1 -0
- package/dist/{chunk-JFI26IB3.js → chunk-VRWRAFDK.js} +67 -18
- package/dist/chunk-VRWRAFDK.js.map +1 -0
- package/dist/chunk-XP3USUQC.js +82 -0
- package/dist/chunk-XP3USUQC.js.map +1 -0
- package/dist/chunk-YZSLCPN6.js +272 -0
- package/dist/chunk-YZSLCPN6.js.map +1 -0
- package/dist/client/cli.js +31 -8
- package/dist/client/cli.js.map +1 -1
- package/dist/client/{fetch-bundles.js → fetch-content-bundles.js} +8 -7
- package/dist/client/fetch-content-bundles.js.map +1 -0
- package/dist/client/fetch-translation-bundles.js +36 -0
- package/dist/client/fetch-translation-bundles.js.map +1 -0
- package/dist/{index-DxoMnE4K.d.ts → index-Db97SUTy.d.ts} +33 -22
- package/dist/index.d.ts +1 -1
- package/dist/node.browser.js +71 -0
- package/dist/node.browser.js.map +1 -0
- package/dist/node.d.ts +22 -5
- package/dist/node.js +353 -44
- package/dist/node.js.map +1 -1
- package/package.json +28 -11
- package/dist/chunk-EGDGHYGI.js.map +0 -1
- package/dist/chunk-JFI26IB3.js.map +0 -1
- package/dist/chunk-UWGOF36L.js +0 -85
- package/dist/chunk-UWGOF36L.js.map +0 -1
- package/dist/client/fetch-bundles.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,20 +1,26 @@
|
|
|
1
1
|
# Content Store
|
|
2
2
|
|
|
3
|
-
SDK for fetching CMS
|
|
3
|
+
SDK for fetching **CMS** and **translation** bundles from Amazon S3 and, for CMS data, querying them locally from the filesystem. CMS bundles come from syncs of **Contentful** or **Sanity**; translation bundles come from **Lingohub** projects synced to S3 (see the [Server & CLI README](src/server/README.md) for the upload pipeline).
|
|
4
4
|
|
|
5
|
-
For the
|
|
5
|
+
For the Express API, server CLI, scheduling, and deployment, see the [Server & CLI README](src/server/README.md).
|
|
6
6
|
|
|
7
7
|
### Package entry points
|
|
8
8
|
|
|
9
|
-
- **`@tandem-language-exchange/content-store`** (default) — **types only** at runtime. Safe to import from shared code that Next.js, Vite, or Turbopack may bundle for the browser.
|
|
10
|
-
- **`@tandem-language-exchange/content-store/node`** — `ContentStoreSDK`, `
|
|
9
|
+
- **`@tandem-language-exchange/content-store`** (default) — **types only** at runtime. Safe to import from shared code that Next.js, Vite, or Turbopack may bundle for the browser.
|
|
10
|
+
- **`@tandem-language-exchange/content-store/node`** — `ContentStoreSDK`, `fetchCmsBundles`, `fetchTranslationBundles`, `queryCmsBundle`, `ContentStore`, `getDefaultS3RetryConfig`, and `trimDepth`. Real implementations use the filesystem and S3 and run only under the Node (`node`) export condition (Route Handlers, `getServerSideProps`, CLI, etc.).
|
|
11
|
+
|
|
12
|
+
For **browser** bundles (including anything imported from `_app.tsx`, client components, or shared modules that reach the client graph), bundlers should resolve the `browser` / `edge-light` conditions to a **stub** that does not import `fs`. That stub throws if you call server-only APIs; **`trimDepth` is fully implemented** and safe on the client.
|
|
13
|
+
|
|
14
|
+
Prefer **not** importing `/node` from files that `_app` or layouts load: keep SDK usage in server-only modules and pass data in as props. If the stub throws at runtime, move the import to server-only code.
|
|
11
15
|
|
|
12
16
|
## Installation
|
|
13
17
|
|
|
14
18
|
```bash
|
|
15
|
-
npm install content-store
|
|
19
|
+
npm install @tandem-language-exchange/content-store
|
|
16
20
|
```
|
|
17
21
|
|
|
22
|
+
Use the same scoped name in `package.json` dependencies and in `npx` / import paths.
|
|
23
|
+
|
|
18
24
|
## Initialisation
|
|
19
25
|
|
|
20
26
|
```typescript
|
|
@@ -66,12 +72,12 @@ const sdk = new ContentStoreSDK({
|
|
|
66
72
|
|
|
67
73
|
---
|
|
68
74
|
|
|
69
|
-
## `
|
|
75
|
+
## `fetchCmsBundles(options)`
|
|
70
76
|
|
|
71
77
|
Downloads the latest content bundles from S3 and saves them as JSON files to `outputDir`.
|
|
72
78
|
|
|
73
79
|
```typescript
|
|
74
|
-
const files = await sdk.
|
|
80
|
+
const files = await sdk.fetchCmsBundles({
|
|
75
81
|
cms: 'contentful',
|
|
76
82
|
contentTypes: ['gridLayout', 'iconWithText', 'page'],
|
|
77
83
|
});
|
|
@@ -83,6 +89,7 @@ const files = await sdk.fetchBundles({
|
|
|
83
89
|
| --- | --- | --- |
|
|
84
90
|
| `cms` | `'contentful' \| 'sanity'` | Which CMS the bundles were synced from |
|
|
85
91
|
| `contentTypes` | `string[]` | Content types to download |
|
|
92
|
+
| `retry` | `S3RetryConfig` | Optional. Overrides [S3 download retries](#s3-download-retries). |
|
|
86
93
|
|
|
87
94
|
**Returns:** `Record<string, string>` — a map of content type to absolute file path.
|
|
88
95
|
|
|
@@ -96,12 +103,66 @@ const files = await sdk.fetchBundles({
|
|
|
96
103
|
|
|
97
104
|
Files are written to `outputDir` with the naming pattern `{cms}-{contentType}.json`.
|
|
98
105
|
|
|
99
|
-
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## `fetchTranslationBundles(options)`
|
|
109
|
+
|
|
110
|
+
Downloads translation bundles that were previously uploaded to S3 (after a Lingohub sync). Files are written under `outputDir` using the same **S3 object key** as the path (see below).
|
|
111
|
+
|
|
112
|
+
```typescript
|
|
113
|
+
const files = await sdk.fetchTranslationBundles({
|
|
114
|
+
projects: ['tandem', 'tandem-(website)'],
|
|
115
|
+
locales: ['en', 'de'], // omit or leave empty to use the package default locale list
|
|
116
|
+
});
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
**Parameters:**
|
|
120
|
+
|
|
121
|
+
| Field | Type | Description |
|
|
122
|
+
| --- | --- | --- |
|
|
123
|
+
| `projects` | `string[]` | Lingohub project ids to fetch (must match keys configured in the package, e.g. `tandem`, `tandem-(android)`). |
|
|
124
|
+
| `locales` | `string[]` | Optional. Locale codes to fetch (e.g. `pt-br`, `zh-hans`). If omitted or empty, a built-in default list is used. |
|
|
125
|
+
| `retry` | `S3RetryConfig` | Optional. Overrides [S3 download retries](#s3-download-retries). |
|
|
126
|
+
|
|
127
|
+
**Returns:** `TranslationBundleInfo` — nested map **`project → S3 object key → absolute file path`** on disk.
|
|
128
|
+
|
|
129
|
+
```typescript
|
|
130
|
+
{
|
|
131
|
+
'tandem-(website)': {
|
|
132
|
+
'lingohub-tandem-(website).en.json': '/abs/path/to/content-cache/lingohub-tandem-(website).en.json',
|
|
133
|
+
'lingohub-tandem-(website).AI.en.json': '/abs/path/to/...'
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
**S3 object keys** follow `lingohub-{project}.{fileName}` where `{fileName}` is the Lingohub resource template with `[locale]` replaced by the **mapped** locale when a resource defines `localeMapping` (same rules as the server sync). Consumers should use the returned keys or list objects by project as shown above.
|
|
139
|
+
|
|
140
|
+
Downloads are run **in parallel** (per project); [S3 download retries](#s3-download-retries) apply by default.
|
|
141
|
+
|
|
142
|
+
---
|
|
143
|
+
|
|
144
|
+
## S3 download retries
|
|
145
|
+
|
|
146
|
+
`fetchCmsBundles` and `fetchTranslationBundles` use retry + exponential backoff on **transient** S3/network failures (for example HTTP 503 “Slow Down”, throttling, timeouts). They do **not** retry clear client errors such as **404** (missing key).
|
|
147
|
+
|
|
148
|
+
Default limits are read from the environment (highest precedence first):
|
|
149
|
+
|
|
150
|
+
| Variable | Fallback | Purpose |
|
|
151
|
+
| --- | --- | --- |
|
|
152
|
+
| `S3_RETRY_MAX_RETRIES` | `RETRY_MAX_RETRIES` (default `5`) | Maximum retry attempts after the first try |
|
|
153
|
+
| `S3_RETRY_BASE_DELAY_MS` | `RETRY_BASE_DELAY_MS` (default `1000`) | Base delay for exponential backoff |
|
|
154
|
+
| `S3_RETRY_MAX_DELAY_MS` | `RETRY_MAX_DELAY_MS` (default `60000`) | Cap on backoff delay |
|
|
155
|
+
|
|
156
|
+
Override per call with **`retry: { maxRetries, baseDelayMs, maxDelayMs }`**, or import **`getDefaultS3RetryConfig()`** to merge with your own defaults.
|
|
157
|
+
|
|
158
|
+
---
|
|
159
|
+
|
|
160
|
+
## `queryCmsBundle(cms, contentType, options?)`
|
|
100
161
|
|
|
101
162
|
Reads a previously fetched bundle from the local filesystem and returns a filtered, shaped result set.
|
|
102
163
|
|
|
103
164
|
```typescript
|
|
104
|
-
const results = await sdk.
|
|
165
|
+
const results = await sdk.queryCmsBundle('contentful', 'gridLayout', {
|
|
105
166
|
fields: { columns: '2' },
|
|
106
167
|
select: ['title', 'bodyBefore'],
|
|
107
168
|
limit: 10,
|
|
@@ -227,10 +288,16 @@ Query options are applied in this order:
|
|
|
227
288
|
|
|
228
289
|
## Standalone functions
|
|
229
290
|
|
|
230
|
-
The
|
|
291
|
+
The same operations are available as standalone imports (no `ContentStoreSDK` wrapper):
|
|
231
292
|
|
|
232
293
|
```typescript
|
|
233
|
-
import {
|
|
294
|
+
import {
|
|
295
|
+
fetchCmsBundles,
|
|
296
|
+
fetchTranslationBundles,
|
|
297
|
+
queryCmsBundle,
|
|
298
|
+
getDefaultS3RetryConfig,
|
|
299
|
+
ContentStore,
|
|
300
|
+
} from '@tandem-language-exchange/content-store/node';
|
|
234
301
|
|
|
235
302
|
const store = new ContentStore({
|
|
236
303
|
bucket: 'beta-content-store',
|
|
@@ -239,12 +306,18 @@ const store = new ContentStore({
|
|
|
239
306
|
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
|
|
240
307
|
});
|
|
241
308
|
|
|
242
|
-
await
|
|
309
|
+
await fetchCmsBundles(store, './content-cache', {
|
|
243
310
|
cms: 'contentful',
|
|
244
311
|
contentTypes: ['gridLayout'],
|
|
245
312
|
});
|
|
246
313
|
|
|
247
|
-
|
|
314
|
+
await fetchTranslationBundles(store, './content-cache', {
|
|
315
|
+
projects: ['tandem-(website)'],
|
|
316
|
+
locales: ['en', 'fr'],
|
|
317
|
+
retry: getDefaultS3RetryConfig(),
|
|
318
|
+
});
|
|
319
|
+
|
|
320
|
+
const results = await queryCmsBundle('./content-cache', 'contentful', 'gridLayout', {
|
|
248
321
|
fields: { columns: '2' },
|
|
249
322
|
limit: 5,
|
|
250
323
|
});
|
|
@@ -265,14 +338,19 @@ const sdk = new ContentStoreSDK({
|
|
|
265
338
|
outputDir: './.content-cache',
|
|
266
339
|
});
|
|
267
340
|
|
|
268
|
-
// 1. Pull latest bundles from S3 to disk
|
|
269
|
-
await sdk.
|
|
341
|
+
// 1. Pull latest CMS bundles from S3 to disk
|
|
342
|
+
await sdk.fetchCmsBundles({
|
|
270
343
|
cms: 'contentful',
|
|
271
344
|
contentTypes: ['page', 'gridLayout'],
|
|
272
345
|
});
|
|
273
346
|
|
|
274
|
-
//
|
|
275
|
-
|
|
347
|
+
// Optional: pull translation bundles written by the Lingohub → S3 sync
|
|
348
|
+
await sdk.fetchTranslationBundles({
|
|
349
|
+
projects: ['tandem-(website)'],
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
// 2. Query CMS bundle locally — no further network calls
|
|
353
|
+
const grids = await sdk.queryCmsBundle('contentful', 'gridLayout', {
|
|
276
354
|
fields: { columns: '2' },
|
|
277
355
|
select: ['title', 'refs'],
|
|
278
356
|
include: 2,
|
|
@@ -283,7 +361,7 @@ console.log(grids);
|
|
|
283
361
|
```
|
|
284
362
|
## CLI
|
|
285
363
|
|
|
286
|
-
The package
|
|
364
|
+
The published package exposes one **global binary** — **`fetch-content-bundles`** (CMS downloads). Other commands live in **`dist/client/cli.js`** as subcommands; call them with **`node node_modules/@tandem-language-exchange/content-store/dist/client/cli.js <command>`** after install, or wrap them in your app’s **`package.json` `scripts`** (see examples below).
|
|
287
365
|
|
|
288
366
|
### `fetch-content-bundles` — download bundles from S3
|
|
289
367
|
|
|
@@ -309,12 +387,14 @@ Files are written as `{cms}-{contentType}.json` inside the output directory.
|
|
|
309
387
|
}
|
|
310
388
|
```
|
|
311
389
|
|
|
312
|
-
|
|
390
|
+
The published **`bin`** only registers **`fetch-content-bundles`** (CMS). To **download translation bundles** from a script without embedding credentials in the shell, use a small Node script that calls **`fetchTranslationBundles`** (or `ContentStoreSDK`) with the same env vars as in [S3 config via environment variables](#s3-config-via-environment-variables), or call the server’s **`POST /getTranslationBundles`** API if you run the content-store service (see [Server & CLI README](src/server/README.md)).
|
|
313
391
|
|
|
314
|
-
|
|
392
|
+
### `query-cms` — query a local bundle
|
|
393
|
+
|
|
394
|
+
Reads a previously fetched bundle from disk and prints JSON to stdout. This command is **not** a separate `bin`; run the built client CLI (after `npm install` of this package):
|
|
315
395
|
|
|
316
396
|
```bash
|
|
317
|
-
|
|
397
|
+
node node_modules/@tandem-language-exchange/content-store/dist/client/cli.js query-cms \
|
|
318
398
|
--cms contentful --type gridLayout \
|
|
319
399
|
--fields '{"columns":"2"}' \
|
|
320
400
|
--select title,bodyBefore \
|
|
@@ -322,6 +402,16 @@ npx @tandem-language-exchange/content-store query \
|
|
|
322
402
|
--include 2
|
|
323
403
|
```
|
|
324
404
|
|
|
405
|
+
**Typical `package.json` shortcut:**
|
|
406
|
+
|
|
407
|
+
```json
|
|
408
|
+
"scripts": {
|
|
409
|
+
"query:cms": "node ./node_modules/@tandem-language-exchange/content-store/dist/client/cli.js query-cms"
|
|
410
|
+
}
|
|
411
|
+
```
|
|
412
|
+
|
|
413
|
+
Then: `npm run query:cms -- --cms contentful --type gridLayout …`
|
|
414
|
+
|
|
325
415
|
| Flag | Required | Default | Description |
|
|
326
416
|
| --- | --- | --- | --- |
|
|
327
417
|
| `--cms <provider>` | Yes | | `contentful` or `sanity` |
|
|
@@ -1,13 +1,49 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
ContentStore,
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
allProjects,
|
|
5
|
+
buildCmsObjectKey,
|
|
6
|
+
buildTranslationObjectKey,
|
|
7
|
+
config,
|
|
8
|
+
defaultLocales
|
|
9
|
+
} from "./chunk-YZSLCPN6.js";
|
|
6
10
|
|
|
7
11
|
// src/server/config.ts
|
|
8
12
|
import dotenv from "dotenv";
|
|
9
13
|
dotenv.config({ path: ".env.local" });
|
|
10
14
|
dotenv.config();
|
|
15
|
+
function parseScheduledCmsJobConfig() {
|
|
16
|
+
const intervalMinutes = parseInt(process.env.SCHEDULE_INTERVAL_MINUTES ?? "0", 60);
|
|
17
|
+
const enabled = Number.isFinite(intervalMinutes) && intervalMinutes > 0;
|
|
18
|
+
const runOnStart = (process.env.SCHEDULE_RUN_ON_START ?? "true").toLowerCase() !== "false";
|
|
19
|
+
const task = (process.env.SCHEDULE_TASK ?? "sync").trim();
|
|
20
|
+
const syncCms = (process.env.SCHEDULE_SYNC_CMS ?? "").trim();
|
|
21
|
+
const rawTypes = process.env.SCHEDULE_SYNC_TYPES?.trim();
|
|
22
|
+
const syncTypes = rawTypes ? rawTypes.split(",").map((s) => s.trim()).filter(Boolean) : void 0;
|
|
23
|
+
return {
|
|
24
|
+
enabled,
|
|
25
|
+
intervalMinutes,
|
|
26
|
+
runOnStart,
|
|
27
|
+
task,
|
|
28
|
+
syncCms: syncCms || void 0,
|
|
29
|
+
syncTypes
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
function parseScheduledTranslationJobConfig() {
|
|
33
|
+
const intervalMinutes = parseInt(process.env.SCHEDULE_INTERVAL_MINUTES ?? "0", 60);
|
|
34
|
+
const enabled = Number.isFinite(intervalMinutes) && intervalMinutes > 0;
|
|
35
|
+
const runOnStart = (process.env.SCHEDULE_RUN_ON_START ?? "true").toLowerCase() !== "false";
|
|
36
|
+
const task = (process.env.SCHEDULE_TASK ?? "sync").trim();
|
|
37
|
+
const rawProjects = process.env.SCHEDULE_SYNC_TRANSLATION_PROJECTS?.trim();
|
|
38
|
+
const syncProjects = rawProjects ? rawProjects.split(",").map((s) => s.trim()).filter(Boolean) : void 0;
|
|
39
|
+
return {
|
|
40
|
+
enabled,
|
|
41
|
+
intervalMinutes,
|
|
42
|
+
runOnStart,
|
|
43
|
+
task,
|
|
44
|
+
syncProjects
|
|
45
|
+
};
|
|
46
|
+
}
|
|
11
47
|
var config2 = {
|
|
12
48
|
...config,
|
|
13
49
|
contentful: {
|
|
@@ -17,11 +53,14 @@ var config2 = {
|
|
|
17
53
|
batchSize: 1e3,
|
|
18
54
|
maxDepth: 4,
|
|
19
55
|
contentTypes: [
|
|
20
|
-
"
|
|
21
|
-
"
|
|
22
|
-
"
|
|
56
|
+
"asset",
|
|
57
|
+
"page",
|
|
58
|
+
"longtailPage",
|
|
59
|
+
"customJson",
|
|
60
|
+
"banner",
|
|
61
|
+
"cookieBanner",
|
|
62
|
+
"downloadPage"
|
|
23
63
|
// Add Contentful content type IDs here to limit sync scope.
|
|
24
|
-
// Leave empty to sync all content types in the space.
|
|
25
64
|
]
|
|
26
65
|
},
|
|
27
66
|
sanity: {
|
|
@@ -30,6 +69,10 @@ var config2 = {
|
|
|
30
69
|
token: process.env.SANITY_API_TOKEN ?? "",
|
|
31
70
|
apiVersion: "2024-01-01"
|
|
32
71
|
},
|
|
72
|
+
lingohub: {
|
|
73
|
+
authToken: process.env.LINGOHUB_AUTH_TOKEN ?? "",
|
|
74
|
+
workspace: process.env.LINGOHUB_WORKSPACE ?? ""
|
|
75
|
+
},
|
|
33
76
|
retry: {
|
|
34
77
|
maxRetries: parseInt(process.env.RETRY_MAX_RETRIES ?? "5", 10),
|
|
35
78
|
baseDelayMs: parseInt(process.env.RETRY_BASE_DELAY_MS ?? "1000", 10),
|
|
@@ -38,6 +81,18 @@ var config2 = {
|
|
|
38
81
|
api: {
|
|
39
82
|
port: parseInt(process.env.PORT ?? "3010"),
|
|
40
83
|
apiToken: process.env.CONTENT_STORE_API_TOKEN ?? ""
|
|
84
|
+
},
|
|
85
|
+
scheduledCmsJob: parseScheduledCmsJobConfig(),
|
|
86
|
+
scheduledTranslationJob: parseScheduledTranslationJobConfig(),
|
|
87
|
+
cleanup: {
|
|
88
|
+
enabled: (process.env.CLEANUP_ENABLED ?? "true").toLowerCase() !== "false",
|
|
89
|
+
retentionDays: parseInt(process.env.CLEANUP_RETENTION_DAYS ?? "30", 10)
|
|
90
|
+
},
|
|
91
|
+
slack: {
|
|
92
|
+
enabled: (process.env.SLACK_BOT_TOKEN ?? "").length > 0,
|
|
93
|
+
botToken: process.env.SLACK_BOT_TOKEN ?? "",
|
|
94
|
+
signingSecret: process.env.SLACK_SIGNING_SECRET ?? "",
|
|
95
|
+
appToken: process.env.SLACK_APP_TOKEN ?? ""
|
|
41
96
|
}
|
|
42
97
|
};
|
|
43
98
|
|
|
@@ -110,7 +165,7 @@ function stripEnvelope(value, maxDepth, depth = 0, path = /* @__PURE__ */ new We
|
|
|
110
165
|
result2[k] = stripEnvelope(v, maxDepth, depth + 1, path);
|
|
111
166
|
}
|
|
112
167
|
const sys = obj.sys;
|
|
113
|
-
if (sys
|
|
168
|
+
if (sys) {
|
|
114
169
|
const existingMeta = result2.meta;
|
|
115
170
|
const metaBase = typeof existingMeta === "object" && existingMeta !== null && !Array.isArray(existingMeta) ? existingMeta : {};
|
|
116
171
|
const _contentType = sys.contentType ? sys.contentType.sys.id : "Asset";
|
|
@@ -134,15 +189,15 @@ var ContentfulAdapter = class {
|
|
|
134
189
|
maxDepth;
|
|
135
190
|
allowedTypes;
|
|
136
191
|
retryConfig;
|
|
137
|
-
constructor(
|
|
192
|
+
constructor(cfg2, retryConfig) {
|
|
138
193
|
this.client = createClient({
|
|
139
|
-
space:
|
|
140
|
-
accessToken:
|
|
141
|
-
host:
|
|
194
|
+
space: cfg2.spaceId,
|
|
195
|
+
accessToken: cfg2.accessToken,
|
|
196
|
+
host: cfg2.host
|
|
142
197
|
});
|
|
143
|
-
this.batchSize =
|
|
144
|
-
this.maxDepth =
|
|
145
|
-
this.allowedTypes =
|
|
198
|
+
this.batchSize = cfg2.batchSize;
|
|
199
|
+
this.maxDepth = cfg2.maxDepth;
|
|
200
|
+
this.allowedTypes = cfg2.contentTypes;
|
|
146
201
|
this.retryConfig = retryConfig;
|
|
147
202
|
}
|
|
148
203
|
async getContentTypes() {
|
|
@@ -160,8 +215,13 @@ var ContentfulAdapter = class {
|
|
|
160
215
|
* Fetches every entry for a content type using batched pagination.
|
|
161
216
|
* Contentful caps `getEntries` at 1 000 items per call, so we page through
|
|
162
217
|
* with `skip` until all items are collected.
|
|
218
|
+
*
|
|
219
|
+
* The reserved content type `"asset"` fetches from `getAssets()` instead.
|
|
163
220
|
*/
|
|
164
221
|
async fetchAll(contentType, includeLevels = 4) {
|
|
222
|
+
if (contentType === "asset") {
|
|
223
|
+
return this.fetchAllAssets();
|
|
224
|
+
}
|
|
165
225
|
const allItems = [];
|
|
166
226
|
let skip = 0;
|
|
167
227
|
let total = 0;
|
|
@@ -188,6 +248,30 @@ var ContentfulAdapter = class {
|
|
|
188
248
|
total
|
|
189
249
|
};
|
|
190
250
|
}
|
|
251
|
+
async fetchAllAssets() {
|
|
252
|
+
const allItems = [];
|
|
253
|
+
let skip = 0;
|
|
254
|
+
let total = 0;
|
|
255
|
+
do {
|
|
256
|
+
const response = await withRetry(
|
|
257
|
+
() => this.client.getAssets({ limit: this.batchSize, skip }),
|
|
258
|
+
this.retryConfig
|
|
259
|
+
);
|
|
260
|
+
total = response.total;
|
|
261
|
+
allItems.push(...response.items);
|
|
262
|
+
skip += response.items.length;
|
|
263
|
+
if (total > this.batchSize) {
|
|
264
|
+
console.log(
|
|
265
|
+
` [contentful] asset: fetched ${allItems.length}/${total}`
|
|
266
|
+
);
|
|
267
|
+
}
|
|
268
|
+
} while (skip < total);
|
|
269
|
+
return {
|
|
270
|
+
contentType: "asset",
|
|
271
|
+
items: allItems.map((item) => stripEnvelope(item, this.maxDepth)),
|
|
272
|
+
total
|
|
273
|
+
};
|
|
274
|
+
}
|
|
191
275
|
};
|
|
192
276
|
|
|
193
277
|
// src/server/adapters/sanity.ts
|
|
@@ -196,19 +280,19 @@ var SanityAdapter = class {
|
|
|
196
280
|
name = "sanity";
|
|
197
281
|
client;
|
|
198
282
|
retryConfig;
|
|
199
|
-
constructor(
|
|
283
|
+
constructor(cfg2, retryConfig) {
|
|
200
284
|
this.client = createClient2({
|
|
201
|
-
projectId:
|
|
202
|
-
dataset:
|
|
203
|
-
token:
|
|
204
|
-
apiVersion:
|
|
285
|
+
projectId: cfg2.projectId,
|
|
286
|
+
dataset: cfg2.dataset,
|
|
287
|
+
token: cfg2.token,
|
|
288
|
+
apiVersion: cfg2.apiVersion,
|
|
205
289
|
useCdn: false
|
|
206
290
|
});
|
|
207
291
|
this.retryConfig = retryConfig;
|
|
208
292
|
}
|
|
209
293
|
async getContentTypes() {
|
|
210
294
|
const types = await withRetry(
|
|
211
|
-
() => this.client.fetch(
|
|
295
|
+
() => this.client.fetch("array::unique(*[]._type)"),
|
|
212
296
|
this.retryConfig
|
|
213
297
|
);
|
|
214
298
|
return types.filter(
|
|
@@ -217,7 +301,7 @@ var SanityAdapter = class {
|
|
|
217
301
|
}
|
|
218
302
|
async fetchAll(contentType) {
|
|
219
303
|
const items = await withRetry(
|
|
220
|
-
() => this.client.fetch(
|
|
304
|
+
() => this.client.fetch("*[_type == $type]", { type: contentType }),
|
|
221
305
|
this.retryConfig
|
|
222
306
|
);
|
|
223
307
|
console.log(` [sanity] ${contentType}: fetched ${items.length} items`);
|
|
@@ -237,8 +321,72 @@ function createAdapter(cms) {
|
|
|
237
321
|
}
|
|
238
322
|
}
|
|
239
323
|
|
|
324
|
+
// src/shared/utils.ts
|
|
325
|
+
import convert from "xml-js";
|
|
326
|
+
import set from "lodash.set";
|
|
327
|
+
import merge from "lodash.merge";
|
|
328
|
+
var convertXMLToJS = (xml) => {
|
|
329
|
+
const converted = convert.xml2js(xml, {
|
|
330
|
+
ignoreComment: true,
|
|
331
|
+
ignoreDeclaration: true,
|
|
332
|
+
ignoreInstruction: true,
|
|
333
|
+
compact: true
|
|
334
|
+
});
|
|
335
|
+
let mapped = {};
|
|
336
|
+
converted.resources.string.forEach((item) => {
|
|
337
|
+
mapped = {
|
|
338
|
+
...mapped,
|
|
339
|
+
[item._attributes.name]: item._text
|
|
340
|
+
};
|
|
341
|
+
});
|
|
342
|
+
return mapped;
|
|
343
|
+
};
|
|
344
|
+
var parseIOSStrings = (strings) => {
|
|
345
|
+
const parsedObj = {};
|
|
346
|
+
strings.split("\n").filter((line) => line.startsWith('"') && line.endsWith(";")).map((line) => line.trim().slice(0, -1)).forEach((line) => {
|
|
347
|
+
let [key, value] = line.split(" = ");
|
|
348
|
+
if (!key || !value) return;
|
|
349
|
+
key = key.slice(1, -1);
|
|
350
|
+
value = value.slice(1, -1);
|
|
351
|
+
parsedObj[key] = value;
|
|
352
|
+
});
|
|
353
|
+
return parsedObj;
|
|
354
|
+
};
|
|
355
|
+
|
|
356
|
+
// src/server/adapters/lingohub.ts
|
|
357
|
+
var cfg = config2.lingohub;
|
|
358
|
+
var apiUrl = "https://api.lingohub.com/v1/" + cfg.workspace + "/projects/";
|
|
359
|
+
var getResourceForLocale = async (project, resource, locale) => {
|
|
360
|
+
const urlForResourceLocalised = `${apiUrl}${project}/resources/${resource.fileName}?auth_token=${cfg.authToken}`.replace("[locale]", locale);
|
|
361
|
+
const result = await fetch(
|
|
362
|
+
urlForResourceLocalised,
|
|
363
|
+
{
|
|
364
|
+
method: "GET"
|
|
365
|
+
}
|
|
366
|
+
).then(async (res) => {
|
|
367
|
+
if (!res.ok) {
|
|
368
|
+
throw new Error(`Failed to fetch resource: ${res.status} - ${res.statusText}`);
|
|
369
|
+
}
|
|
370
|
+
if (resource.type === "json") {
|
|
371
|
+
return res.json();
|
|
372
|
+
}
|
|
373
|
+
if (resource.type === "xml") {
|
|
374
|
+
const xml = await res.text();
|
|
375
|
+
return convertXMLToJS(xml);
|
|
376
|
+
}
|
|
377
|
+
if (resource.type === "strings") {
|
|
378
|
+
const strings = await res.text();
|
|
379
|
+
return parseIOSStrings(strings);
|
|
380
|
+
}
|
|
381
|
+
throw new Error("Invalid resource type");
|
|
382
|
+
}).catch((err) => {
|
|
383
|
+
throw err;
|
|
384
|
+
});
|
|
385
|
+
return result;
|
|
386
|
+
};
|
|
387
|
+
|
|
240
388
|
// src/server/sync/engine.ts
|
|
241
|
-
async function
|
|
389
|
+
async function syncCmsContent(cms, contentTypes, includeLevels) {
|
|
242
390
|
const adapter = createAdapter(cms);
|
|
243
391
|
const store = new ContentStore(config2.s3);
|
|
244
392
|
const timestamp = Math.floor(Date.now() / 1e3);
|
|
@@ -252,17 +400,15 @@ Starting sync from ${cms} at ${new Date(timestamp * 1e3).toISOString()}`);
|
|
|
252
400
|
for (const contentType of typesToSync) {
|
|
253
401
|
try {
|
|
254
402
|
const result = await adapter.fetchAll(contentType, includeLevels);
|
|
255
|
-
const
|
|
256
|
-
await store.upload(
|
|
257
|
-
const latestKey = await store.copyToLatest(versionedKey, cms, contentType);
|
|
403
|
+
const objectKey = buildCmsObjectKey(cms, contentType);
|
|
404
|
+
await store.upload(objectKey, result.items);
|
|
258
405
|
entries.push({
|
|
259
406
|
contentType,
|
|
260
407
|
itemCount: result.total,
|
|
261
|
-
|
|
262
|
-
latestKey
|
|
408
|
+
objectKey
|
|
263
409
|
});
|
|
264
410
|
console.log(
|
|
265
|
-
` + ${contentType}: ${result.total} items -> ${
|
|
411
|
+
` + ${contentType}: ${result.total} items -> ${objectKey}`
|
|
266
412
|
);
|
|
267
413
|
} catch (err) {
|
|
268
414
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -277,9 +423,54 @@ Sync complete: ${entries.length} succeeded, ${errors.length} failed
|
|
|
277
423
|
);
|
|
278
424
|
return { cms, timestamp, entries, errors };
|
|
279
425
|
}
|
|
426
|
+
async function syncTranslations(projects, locales) {
|
|
427
|
+
const store = new ContentStore(config2.s3);
|
|
428
|
+
const entries = [];
|
|
429
|
+
const errors = [];
|
|
430
|
+
const timestamp = Math.floor(Date.now() / 1e3);
|
|
431
|
+
if (!locales) {
|
|
432
|
+
locales = defaultLocales;
|
|
433
|
+
}
|
|
434
|
+
if (!projects) {
|
|
435
|
+
projects = Object.keys(allProjects);
|
|
436
|
+
}
|
|
437
|
+
for (const project of projects) {
|
|
438
|
+
const resources = allProjects[project];
|
|
439
|
+
if (!resources) {
|
|
440
|
+
console.error(`No resources found for ${project}`);
|
|
441
|
+
continue;
|
|
442
|
+
}
|
|
443
|
+
for (const resource of resources) {
|
|
444
|
+
for (const loc of locales) {
|
|
445
|
+
const locale = resource.localeMapping && resource.localeMapping[loc] ? resource.localeMapping[loc] : loc;
|
|
446
|
+
try {
|
|
447
|
+
const result = await getResourceForLocale(project, resource, locale);
|
|
448
|
+
const objectKey = buildTranslationObjectKey(project, resource.fileName, locale);
|
|
449
|
+
await store.upload(objectKey, result);
|
|
450
|
+
const itemCount = Object.keys(result).length;
|
|
451
|
+
entries.push({
|
|
452
|
+
project,
|
|
453
|
+
locale,
|
|
454
|
+
itemCount,
|
|
455
|
+
objectKey
|
|
456
|
+
});
|
|
457
|
+
console.log(
|
|
458
|
+
` + ${project} - ${locale}: ${itemCount} items -> ${objectKey}`
|
|
459
|
+
);
|
|
460
|
+
} catch (err) {
|
|
461
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
462
|
+
errors.push({ project, locale, error: message });
|
|
463
|
+
console.error(` x ${project} - ${locale}: ${message}`);
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
return { timestamp, entries, errors };
|
|
469
|
+
}
|
|
280
470
|
|
|
281
471
|
export {
|
|
282
472
|
config2 as config,
|
|
283
|
-
|
|
473
|
+
syncCmsContent,
|
|
474
|
+
syncTranslations
|
|
284
475
|
};
|
|
285
|
-
//# sourceMappingURL=chunk-
|
|
476
|
+
//# sourceMappingURL=chunk-UCBZUEUP.js.map
|