@tandem-language-exchange/content-store 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/cli.d.ts +1 -0
- package/dist/client/cli.js +72 -0
- package/dist/client/cli.js.map +1 -0
- package/dist/client/config.d.ts +6 -0
- package/dist/client/config.js +8 -0
- package/dist/client/config.js.map +1 -0
- package/dist/index.d.ts +5 -5
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/sdk/client.d.ts +2 -2
- package/dist/sdk/client.js +2 -2
- package/dist/sdk/client.js.map +1 -1
- package/dist/sdk/index.d.ts +5 -5
- package/dist/sdk/index.js +3 -3
- package/dist/sdk/index.js.map +1 -1
- package/dist/shared/bundles.d.ts +2 -2
- package/dist/shared/config.d.ts +6 -0
- package/dist/shared/config.js +12 -0
- package/dist/shared/config.js.map +1 -0
- package/dist/shared/s3.d.ts +1 -1
- package/dist/shared/sync/engine.d.ts +17 -0
- package/dist/shared/sync/engine.js +38 -0
- package/dist/shared/sync/engine.js.map +1 -0
- package/dist/shared/sync/retry.d.ts +7 -0
- package/dist/shared/sync/retry.js +57 -0
- package/dist/shared/sync/retry.js.map +1 -0
- package/package.json +3 -2
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { Command } from 'commander';
|
|
2
|
+
import { config } from './config';
|
|
3
|
+
import { ContentStore } from '../shared/s3';
|
|
4
|
+
import { fetchBundles, queryBundle } from '../shared/bundles';
|
|
5
|
+
const program = new Command();
|
|
6
|
+
program
|
|
7
|
+
.name('content-store')
|
|
8
|
+
.description('Sync CMS content to S3')
|
|
9
|
+
.version('1.0.0');
|
|
10
|
+
program
|
|
11
|
+
.command('fetch')
|
|
12
|
+
.description('Download latest bundles from S3 to the local filesystem')
|
|
13
|
+
.requiredOption('--cms <provider>', 'CMS provider: contentful | sanity')
|
|
14
|
+
.requiredOption('--types <types>', 'Comma-separated content types to fetch')
|
|
15
|
+
.option('--output <directory>', 'Output directory', './content-cache')
|
|
16
|
+
.action(async (opts) => {
|
|
17
|
+
const cms = opts.cms;
|
|
18
|
+
if (!['contentful', 'sanity'].includes(cms)) {
|
|
19
|
+
console.error(`Invalid CMS provider: ${cms}`);
|
|
20
|
+
process.exit(1);
|
|
21
|
+
}
|
|
22
|
+
const contentTypes = opts.types.split(',').map((s) => s.trim());
|
|
23
|
+
const store = new ContentStore(config.s3);
|
|
24
|
+
try {
|
|
25
|
+
const files = await fetchBundles(store, opts.output, {
|
|
26
|
+
cms,
|
|
27
|
+
contentTypes,
|
|
28
|
+
});
|
|
29
|
+
console.log('Fetched bundles:');
|
|
30
|
+
for (const [type, filePath] of Object.entries(files)) {
|
|
31
|
+
console.log(` ${type} -> ${filePath}`);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
catch (err) {
|
|
35
|
+
console.error('Fetch failed:', err);
|
|
36
|
+
process.exit(1);
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
program
|
|
40
|
+
.command('query')
|
|
41
|
+
.description('Query a previously fetched bundle from the local filesystem')
|
|
42
|
+
.requiredOption('--cms <provider>', 'CMS provider: contentful | sanity')
|
|
43
|
+
.requiredOption('--type <type>', 'Content type to query')
|
|
44
|
+
.option('--output <directory>', 'Directory where bundles are stored', './content-cache')
|
|
45
|
+
.option('--fields <json>', 'Filter by fields (JSON object, e.g. \'{"columns":"2"}\')')
|
|
46
|
+
.option('--select <props>', 'Comma-separated properties to include in results')
|
|
47
|
+
.option('--limit <n>', 'Maximum number of results', parseInt)
|
|
48
|
+
.option('--include <n>', 'Depth of nested references to include', parseInt)
|
|
49
|
+
.action(async (opts) => {
|
|
50
|
+
const cms = opts.cms;
|
|
51
|
+
if (!['contentful', 'sanity'].includes(cms)) {
|
|
52
|
+
console.error(`Invalid CMS provider: ${cms}`);
|
|
53
|
+
process.exit(1);
|
|
54
|
+
}
|
|
55
|
+
try {
|
|
56
|
+
const results = await queryBundle(opts.output, cms, opts.type, {
|
|
57
|
+
fields: opts.fields ? JSON.parse(opts.fields) : undefined,
|
|
58
|
+
select: opts.select
|
|
59
|
+
? opts.select.split(',').map((s) => s.trim())
|
|
60
|
+
: undefined,
|
|
61
|
+
limit: opts.limit,
|
|
62
|
+
include: opts.include,
|
|
63
|
+
});
|
|
64
|
+
console.log(JSON.stringify(results, null, 2));
|
|
65
|
+
}
|
|
66
|
+
catch (err) {
|
|
67
|
+
console.error('Query failed:', err);
|
|
68
|
+
process.exit(1);
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
program.parse();
|
|
72
|
+
//# sourceMappingURL=cli.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/client/cli.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAoB,MAAM,UAAU,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE9D,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,eAAe,CAAC;KACrB,WAAW,CAAC,wBAAwB,CAAC;KACrC,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACF,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,yDAAyD,CAAC;KACtE,cAAc,CAAC,kBAAkB,EAAE,mCAAmC,CAAC;KACvE,cAAc,CAAC,iBAAiB,EAAE,wCAAwC,CAAC;KAC3E,MAAM,CAAC,sBAAsB,EAAE,kBAAkB,EAAE,iBAAiB,CAAC;KACrE,MAAM,CAAC,KAAK,EAAE,IAAoD,EAAE,EAAE;IACvE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAkB,CAAC;IAEpC,IAAI,CAAC,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,CAAC,KAAK,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChE,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAE1C,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE;YACnD,GAAG;YACH,YAAY;SACb,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QAChC,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,OAAO,QAAQ,EAAE,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,6DAA6D,CAAC;KAC1E,cAAc,CAAC,kBAAkB,EAAE,mCAAmC,CAAC;KACvE,cAAc,CAAC,eAAe,EAAE,uBAAuB,CAAC;KACxD,MAAM,CAAC,sBAAsB,EAAE,oCAAoC,EAAE,iBAAiB,CAAC;KACvF,MAAM,CAAC,iBAAiB,EAAE,0DAA0D,CAAC;KACrF,MAAM,CAAC,kBAAkB,EAAE,kDAAkD,CAAC;KAC9E,MAAM,CAAC,aAAa,EAAE,2BAA2B,EAAE,QAAQ,CAAC;KAC5D,MAAM,CAAC,eAAe,EAAE,uCAAuC,EAAE,QAAQ,CAAC;KAC1E,MAAM,CACL,KAAK,EAAE,IAQN,EAAE,EAAE;IACH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAkB,CAAC;IAEpC,IAAI,CAAC,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC5C,OAAO,CAAC,KAAK,CAAC,yBAAyB,GAAG,EAAE,CAAC,CAAC;QAC9C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE;YAC7D,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;YACzD,MAAM,EAAE,IAAI,CAAC,MAAM;gBACjB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC7C,CAAC,CAAC,SAAS;YACb,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC;QACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CACF,CAAC;AAEJ,OAAO,CAAC,KAAK,EAAE,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/client/config.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAe,MAAM,IAAI,YAAY,EAAC,MAAM,kBAAkB,CAAC;AAEtE,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;AACtC,MAAM,CAAC,MAAM,EAAE,CAAC;AAOhB,MAAM,CAAC,MAAM,MAAM,GAAgC;IAC/C,GAAG,YAAY;CAClB,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export { ContentStoreSDK } from './sdk/index
|
|
2
|
-
export type { SDKConfig, FetchBundlesOptions, QueryOptions } from './sdk/index
|
|
3
|
-
export { fetchBundles, queryBundle, trimDepth } from './shared/bundles
|
|
4
|
-
export { ContentStore } from './shared/s3
|
|
5
|
-
export type { CMSProvider, S3Config } from './shared/types
|
|
1
|
+
export { ContentStoreSDK } from './sdk/index';
|
|
2
|
+
export type { SDKConfig, FetchBundlesOptions, QueryOptions } from './sdk/index';
|
|
3
|
+
export { fetchBundles, queryBundle, trimDepth } from './shared/bundles';
|
|
4
|
+
export { ContentStore } from './shared/s3';
|
|
5
|
+
export type { CMSProvider, S3Config } from './shared/types';
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { ContentStoreSDK } from './sdk/index
|
|
2
|
-
export { fetchBundles, queryBundle, trimDepth } from './shared/bundles
|
|
3
|
-
export { ContentStore } from './shared/s3
|
|
1
|
+
export { ContentStoreSDK } from './sdk/index';
|
|
2
|
+
export { fetchBundles, queryBundle, trimDepth } from './shared/bundles';
|
|
3
|
+
export { ContentStore } from './shared/s3';
|
|
4
4
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC"}
|
package/dist/sdk/client.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import type { S3Config, CMSProvider } from '../shared/types
|
|
2
|
-
import { type FetchBundlesOptions, type QueryOptions } from '../shared/bundles
|
|
1
|
+
import type { S3Config, CMSProvider } from '../shared/types';
|
|
2
|
+
import { type FetchBundlesOptions, type QueryOptions } from '../shared/bundles';
|
|
3
3
|
export type { FetchBundlesOptions, QueryOptions };
|
|
4
4
|
export interface SDKConfig {
|
|
5
5
|
s3: S3Config;
|
package/dist/sdk/client.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { ContentStore } from '../shared/s3
|
|
2
|
-
import { fetchBundles, queryBundle, } from '../shared/bundles
|
|
1
|
+
import { ContentStore } from '../shared/s3';
|
|
2
|
+
import { fetchBundles, queryBundle, } from '../shared/bundles';
|
|
3
3
|
export class ContentStoreSDK {
|
|
4
4
|
store;
|
|
5
5
|
outputDir;
|
package/dist/sdk/client.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/sdk/client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"client.js","sourceRoot":"","sources":["../../src/sdk/client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EACL,YAAY,EACZ,WAAW,GAGZ,MAAM,mBAAmB,CAAC;AAU3B,MAAM,OAAO,eAAe;IAClB,KAAK,CAAe;IACpB,SAAS,CAAS;IAE1B,YAAY,MAAiB;QAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;IACpC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,YAAY,CAChB,OAA4B;QAE5B,OAAO,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC3D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CACf,GAAgB,EAChB,WAAmB,EACnB,UAAwB,EAAE;QAE1B,OAAO,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;IAChE,CAAC;CACF"}
|
package/dist/sdk/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export { ContentStoreSDK } from './client
|
|
2
|
-
export type { SDKConfig, FetchBundlesOptions, QueryOptions } from './client
|
|
3
|
-
export { fetchBundles, queryBundle, trimDepth } from '../shared/bundles
|
|
4
|
-
export { ContentStore } from '../shared/s3
|
|
5
|
-
export type { S3Config, CMSProvider } from '../shared/types
|
|
1
|
+
export { ContentStoreSDK } from './client';
|
|
2
|
+
export type { SDKConfig, FetchBundlesOptions, QueryOptions } from './client';
|
|
3
|
+
export { fetchBundles, queryBundle, trimDepth } from '../shared/bundles';
|
|
4
|
+
export { ContentStore } from '../shared/s3';
|
|
5
|
+
export type { S3Config, CMSProvider } from '../shared/types';
|
package/dist/sdk/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { ContentStoreSDK } from './client
|
|
2
|
-
export { fetchBundles, queryBundle, trimDepth } from '../shared/bundles
|
|
3
|
-
export { ContentStore } from '../shared/s3
|
|
1
|
+
export { ContentStoreSDK } from './client';
|
|
2
|
+
export { fetchBundles, queryBundle, trimDepth } from '../shared/bundles';
|
|
3
|
+
export { ContentStore } from '../shared/s3';
|
|
4
4
|
//# sourceMappingURL=index.js.map
|
package/dist/sdk/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/sdk/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/sdk/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE3C,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AACzE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC"}
|
package/dist/shared/bundles.d.ts
CHANGED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import dotenv from 'dotenv';
|
|
2
|
+
dotenv.config({ path: '.env.local' });
|
|
3
|
+
dotenv.config();
|
|
4
|
+
export const config = {
|
|
5
|
+
s3: {
|
|
6
|
+
bucket: process.env.CONTENT_STORE_S3_BUCKET ?? '',
|
|
7
|
+
region: process.env.CONTENT_STORE_S3_REGION ?? 'eu-central-1',
|
|
8
|
+
accessKeyId: process.env.AWS_ACCESS_KEY ?? '',
|
|
9
|
+
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ?? '',
|
|
10
|
+
}
|
|
11
|
+
};
|
|
12
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/shared/config.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,QAAQ,CAAC;AAG5B,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;AACtC,MAAM,CAAC,MAAM,EAAE,CAAC;AAQhB,MAAM,CAAC,MAAM,MAAM,GAAiB;IAChC,EAAE,EAAE;QACA,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,EAAE;QACjD,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,cAAc;QAC7D,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE;QAC7C,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,EAAE;KAC3D;CACJ,CAAC"}
|
package/dist/shared/s3.d.ts
CHANGED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { CMSProvider } from '../config.js';
|
|
2
|
+
export interface SyncResultEntry {
|
|
3
|
+
contentType: string;
|
|
4
|
+
itemCount: number;
|
|
5
|
+
versionedKey: string;
|
|
6
|
+
latestKey: string;
|
|
7
|
+
}
|
|
8
|
+
export interface SyncResult {
|
|
9
|
+
cms: CMSProvider;
|
|
10
|
+
timestamp: number;
|
|
11
|
+
entries: SyncResultEntry[];
|
|
12
|
+
errors: Array<{
|
|
13
|
+
contentType: string;
|
|
14
|
+
error: string;
|
|
15
|
+
}>;
|
|
16
|
+
}
|
|
17
|
+
export declare function runSync(cms: CMSProvider, contentTypes?: string[]): Promise<SyncResult>;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { config } from '../config.js';
|
|
2
|
+
import { createAdapter } from '../adapters/index.js';
|
|
3
|
+
import { ContentStore } from '../../shared/s3.js';
|
|
4
|
+
export async function runSync(cms, contentTypes) {
|
|
5
|
+
const adapter = createAdapter(cms);
|
|
6
|
+
const store = new ContentStore(config.s3);
|
|
7
|
+
const timestamp = Math.floor(Date.now() / 1000);
|
|
8
|
+
console.log(`\nStarting sync from ${cms} at ${new Date(timestamp * 1000).toISOString()}`);
|
|
9
|
+
const typesToSync = contentTypes && contentTypes.length > 0
|
|
10
|
+
? contentTypes
|
|
11
|
+
: await adapter.getContentTypes();
|
|
12
|
+
console.log(`Content types to sync: ${typesToSync.join(', ')}\n`);
|
|
13
|
+
const entries = [];
|
|
14
|
+
const errors = [];
|
|
15
|
+
for (const contentType of typesToSync) {
|
|
16
|
+
try {
|
|
17
|
+
const result = await adapter.fetchAll(contentType);
|
|
18
|
+
const versionedKey = store.buildVersionedKey(cms, contentType, timestamp);
|
|
19
|
+
await store.upload(versionedKey, result.items);
|
|
20
|
+
const latestKey = await store.copyToLatest(versionedKey, cms, contentType);
|
|
21
|
+
entries.push({
|
|
22
|
+
contentType,
|
|
23
|
+
itemCount: result.total,
|
|
24
|
+
versionedKey,
|
|
25
|
+
latestKey,
|
|
26
|
+
});
|
|
27
|
+
console.log(` + ${contentType}: ${result.total} items -> ${versionedKey}`);
|
|
28
|
+
}
|
|
29
|
+
catch (err) {
|
|
30
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
31
|
+
errors.push({ contentType, error: message });
|
|
32
|
+
console.error(` x ${contentType}: ${message}`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
console.log(`\nSync complete: ${entries.length} succeeded, ${errors.length} failed\n`);
|
|
36
|
+
return { cms, timestamp, entries, errors };
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=engine.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"engine.js","sourceRoot":"","sources":["../../../src/shared/sync/engine.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAgBlD,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,GAAgB,EAChB,YAAuB;IAEvB,MAAM,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;IAEhD,OAAO,CAAC,GAAG,CAAC,wBAAwB,GAAG,OAAO,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IAE1F,MAAM,WAAW,GACf,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC;QACrC,CAAC,CAAC,YAAY;QACd,CAAC,CAAC,MAAM,OAAO,CAAC,eAAe,EAAE,CAAC;IAEtC,OAAO,CAAC,GAAG,CAAC,0BAA0B,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAElE,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,MAAM,MAAM,GAAkD,EAAE,CAAC;IAEjE,KAAK,MAAM,WAAW,IAAI,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAEnD,MAAM,YAAY,GAAG,KAAK,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;YAC1E,MAAM,KAAK,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;YAE/C,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;YAE3E,OAAO,CAAC,IAAI,CAAC;gBACX,WAAW;gBACX,SAAS,EAAE,MAAM,CAAC,KAAK;gBACvB,YAAY;gBACZ,SAAS;aACV,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CACT,OAAO,WAAW,KAAK,MAAM,CAAC,KAAK,aAAa,YAAY,EAAE,CAC/D,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;YAC7C,OAAO,CAAC,KAAK,CAAC,OAAO,WAAW,KAAK,OAAO,EAAE,CAAC,CAAC;QAClD,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CACT,oBAAoB,OAAO,CAAC,MAAM,eAAe,MAAM,CAAC,MAAM,WAAW,CAC1E,CAAC;IAEF,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC7C,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { RetryConfig } from '../config.js';
|
|
2
|
+
/**
|
|
3
|
+
* Executes `fn` with automatic retry + exponential backoff.
|
|
4
|
+
* Rate-limit (429) responses are handled specially: if the API provides a
|
|
5
|
+
* Retry-After / reset header, that value is respected instead of computed backoff.
|
|
6
|
+
*/
|
|
7
|
+
export declare function withRetry<T>(fn: () => Promise<T>, { maxRetries, baseDelayMs, maxDelayMs }: RetryConfig): Promise<T>;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inspects an error to determine if it represents an API rate-limit (HTTP 429).
|
|
3
|
+
* Returns the suggested wait time in ms when available, otherwise `0` to signal
|
|
4
|
+
* that the caller should fall back to computed backoff. Returns `null` when the
|
|
5
|
+
* error is *not* a rate-limit error.
|
|
6
|
+
*/
|
|
7
|
+
function rateLimitDelayMs(err) {
|
|
8
|
+
const e = err;
|
|
9
|
+
if (e?.status === 429 || e?.statusCode === 429) {
|
|
10
|
+
const reset = e?.headers?.['x-contentful-ratelimit-reset'];
|
|
11
|
+
return reset ? parseFloat(reset) * 1000 : 0;
|
|
12
|
+
}
|
|
13
|
+
const resp = e?.response;
|
|
14
|
+
if (resp?.status === 429) {
|
|
15
|
+
const headers = resp?.headers;
|
|
16
|
+
const reset = headers?.['x-contentful-ratelimit-reset'];
|
|
17
|
+
return reset ? parseFloat(reset) * 1000 : 0;
|
|
18
|
+
}
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
function computeDelay(attempt, baseDelayMs, maxDelayMs) {
|
|
22
|
+
const exponential = baseDelayMs * Math.pow(2, attempt);
|
|
23
|
+
const jitter = Math.random() * baseDelayMs;
|
|
24
|
+
return Math.min(exponential + jitter, maxDelayMs);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Executes `fn` with automatic retry + exponential backoff.
|
|
28
|
+
* Rate-limit (429) responses are handled specially: if the API provides a
|
|
29
|
+
* Retry-After / reset header, that value is respected instead of computed backoff.
|
|
30
|
+
*/
|
|
31
|
+
export async function withRetry(fn, { maxRetries, baseDelayMs, maxDelayMs }) {
|
|
32
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
33
|
+
try {
|
|
34
|
+
return await fn();
|
|
35
|
+
}
|
|
36
|
+
catch (err) {
|
|
37
|
+
if (attempt === maxRetries)
|
|
38
|
+
throw err;
|
|
39
|
+
const rlDelay = rateLimitDelayMs(err);
|
|
40
|
+
let delay;
|
|
41
|
+
if (rlDelay !== null) {
|
|
42
|
+
delay =
|
|
43
|
+
rlDelay > 0 ? rlDelay : computeDelay(attempt, baseDelayMs, maxDelayMs);
|
|
44
|
+
console.warn(` Rate limited (attempt ${attempt + 1}/${maxRetries}). ` +
|
|
45
|
+
`Waiting ${Math.round(delay)}ms…`);
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
delay = computeDelay(attempt, baseDelayMs, maxDelayMs);
|
|
49
|
+
console.warn(` Request failed (attempt ${attempt + 1}/${maxRetries}): ` +
|
|
50
|
+
`${err.message}. Retrying in ${Math.round(delay)}ms…`);
|
|
51
|
+
}
|
|
52
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
throw new Error('withRetry: unreachable');
|
|
56
|
+
}
|
|
57
|
+
//# sourceMappingURL=retry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retry.js","sourceRoot":"","sources":["../../../src/shared/sync/retry.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,SAAS,gBAAgB,CAAC,GAAY;IACpC,MAAM,CAAC,GAAG,GAA8B,CAAC;IAEzC,IAAI,CAAC,EAAE,MAAM,KAAK,GAAG,IAAI,CAAC,EAAE,UAAU,KAAK,GAAG,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAI,CAAC,EAAE,OAA8C,EAAE,CAChE,8BAA8B,CAC/B,CAAC;QACF,OAAO,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,EAAE,QAA+C,CAAC;IAChE,IAAI,IAAI,EAAE,MAAM,KAAK,GAAG,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,EAAE,OAA6C,CAAC;QACpE,MAAM,KAAK,GAAG,OAAO,EAAE,CAAC,8BAA8B,CAAC,CAAC;QACxD,OAAO,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,YAAY,CACnB,OAAe,EACf,WAAmB,EACnB,UAAkB;IAElB,MAAM,WAAW,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACvD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC;IAC3C,OAAO,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,MAAM,EAAE,UAAU,CAAC,CAAC;AACpD,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,EAAoB,EACpB,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAe;IAEpD,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,OAAO,KAAK,UAAU;gBAAE,MAAM,GAAG,CAAC;YAEtC,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,KAAa,CAAC;YAElB,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACrB,KAAK;oBACH,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;gBACzE,OAAO,CAAC,IAAI,CACV,2BAA2B,OAAO,GAAG,CAAC,IAAI,UAAU,KAAK;oBACvD,WAAW,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CACpC,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,YAAY,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;gBACvD,OAAO,CAAC,IAAI,CACV,6BAA6B,OAAO,GAAG,CAAC,IAAI,UAAU,KAAK;oBACzD,GAAI,GAAa,CAAC,OAAO,iBAAiB,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CACnE,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;AAC5C,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tandem-language-exchange/content-store",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.4",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -13,10 +13,11 @@
|
|
|
13
13
|
"files": [
|
|
14
14
|
"dist/index.*",
|
|
15
15
|
"dist/sdk/",
|
|
16
|
+
"dist/client/",
|
|
16
17
|
"dist/shared/"
|
|
17
18
|
],
|
|
18
19
|
"bin": {
|
|
19
|
-
"fetch-content-bundles": "dist/
|
|
20
|
+
"fetch-content-bundles": "dist/client/cli.js fetch"
|
|
20
21
|
},
|
|
21
22
|
"engines": {
|
|
22
23
|
"npm": "^11.3.0",
|