@learnpack/learnpack 5.0.351 → 5.0.352

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,6 +11,7 @@ export type PackageManifestLesson = {
11
11
  position: number;
12
12
  type: LessonType;
13
13
  title: Record<string, string>;
14
+ description: Record<string, string | null>;
14
15
  video: Record<string, string | null>;
15
16
  };
16
17
  export type PackageManifest = {
@@ -48,6 +49,9 @@ export declare function resolvePreviewUrl(slug: string, preview?: string): strin
48
49
  export declare function extractReadmeH1(body: string): string | null;
49
50
  export declare function normalizeReadmeTitle(raw: string): string | null;
50
51
  export declare function buildPackageManifestFromSources(input: PackageManifestBuildInput): PackageManifest;
52
+ export declare function parseReadmeContent(content: string): ReadmeFrontmatter;
53
+ export declare function comparePackageManifests(existing: PackageManifest | null, next: PackageManifest): "created" | "updated" | "skipped";
54
+ export declare function serializePackageManifest(manifest: PackageManifest): string;
51
55
  export declare function generateAndPersistPackageManifest(bucket: Bucket, courseSlug: string, options: {
52
56
  publishedAt: string | null;
53
57
  localDir?: string;
@@ -6,7 +6,11 @@ exports.resolvePreviewUrl = resolvePreviewUrl;
6
6
  exports.extractReadmeH1 = extractReadmeH1;
7
7
  exports.normalizeReadmeTitle = normalizeReadmeTitle;
8
8
  exports.buildPackageManifestFromSources = buildPackageManifestFromSources;
9
+ exports.parseReadmeContent = parseReadmeContent;
10
+ exports.comparePackageManifests = comparePackageManifests;
11
+ exports.serializePackageManifest = serializePackageManifest;
9
12
  exports.generateAndPersistPackageManifest = generateAndPersistPackageManifest;
13
+ const tslib_1 = require("tslib");
10
14
  const fs = require("fs");
11
15
  const mkdirp = require("mkdirp");
12
16
  const path = require("path");
@@ -90,6 +94,14 @@ function resolveLessonTitle(exercise, sidebar, syllabusLesson, courseLang, readm
90
94
  }
91
95
  return titles;
92
96
  }
97
+ function resolveLessonDescriptions(exercise) {
98
+ const description = {};
99
+ for (const lang of Object.keys(exercise.translations)) {
100
+ // TODO: add description content
101
+ description[lang] = null;
102
+ }
103
+ return description;
104
+ }
93
105
  function resolveLessonVideos(exercise, readmeFrontmatters) {
94
106
  const video = {};
95
107
  const frontmatters = readmeFrontmatters[exercise.slug] || {};
@@ -122,6 +134,7 @@ function buildPackageManifestFromSources(input) {
122
134
  position: exercise.position,
123
135
  type,
124
136
  title: resolveLessonTitle(exercise, sidebar, syllabusLesson, courseLang, (_b = readmeFrontmatters[exercise.slug]) !== null && _b !== void 0 ? _b : {}),
137
+ description: resolveLessonDescriptions(exercise),
125
138
  video: resolveLessonVideos(exercise, readmeFrontmatters),
126
139
  };
127
140
  });
@@ -150,17 +163,34 @@ async function downloadJsonFile(bucket, filePath) {
150
163
  return null;
151
164
  }
152
165
  }
166
+ function parseReadmeContent(content) {
167
+ const parsed = frontMatter(content);
168
+ const attributes = parsed.attributes || {};
169
+ const h1 = extractReadmeH1(parsed.body || "");
170
+ const title = h1 ? normalizeReadmeTitle(h1) : null;
171
+ return Object.assign(Object.assign(Object.assign({}, (attributes.tutorial ? { tutorial: attributes.tutorial } : {})), (attributes.intro ? { intro: attributes.intro } : {})), (title ? { title } : {}));
172
+ }
173
+ function comparePackageManifests(existing, next) {
174
+ if (existing === null) {
175
+ return "created";
176
+ }
177
+ const { generatedAt: _existingGeneratedAt } = existing, existingRest = tslib_1.__rest(existing, ["generatedAt"]);
178
+ const { generatedAt: _nextGeneratedAt } = next, nextRest = tslib_1.__rest(next, ["generatedAt"]);
179
+ if (JSON.stringify(existingRest) === JSON.stringify(nextRest)) {
180
+ return "skipped";
181
+ }
182
+ return "updated";
183
+ }
184
+ function serializePackageManifest(manifest) {
185
+ return JSON.stringify(manifest, null, 2);
186
+ }
153
187
  async function readReadmeFrontmatter(bucket, exerciseSlug, lang, readmePath) {
154
188
  try {
155
189
  const [buf] = await bucket.file(readmePath).download();
156
- const parsed = frontMatter(buf.toString());
157
- const attributes = parsed.attributes || {};
158
- const h1 = extractReadmeH1(parsed.body || "");
159
- const title = h1 ? normalizeReadmeTitle(h1) : null;
160
190
  return {
161
191
  exerciseSlug,
162
192
  lang,
163
- frontmatter: Object.assign({ tutorial: attributes.tutorial, intro: attributes.intro }, (title ? { title } : {})),
193
+ frontmatter: parseReadmeContent(buf.toString()),
164
194
  };
165
195
  }
166
196
  catch (error) {
@@ -185,7 +215,7 @@ async function collectReadmeFrontmatters(bucket, courseSlug, exercises) {
185
215
  return result;
186
216
  }
187
217
  function writeManifestJson(manifest, localDir) {
188
- const json = JSON.stringify(manifest, null, 2);
218
+ const json = serializePackageManifest(manifest);
189
219
  if (localDir) {
190
220
  const learnDir = path.join(localDir, ".learn");
191
221
  mkdirp.sync(learnDir);
@@ -0,0 +1,82 @@
1
+ import { Exercise } from "../configBuilder";
2
+ import { Syllabus } from "../../models/creator";
3
+ import { PackageManifest, ReadmeFrontmatter } from "../packageManifest";
4
+ export type PackageResultStatus = "created" | "updated" | "skipped" | "failed";
5
+ export type PackageResult = {
6
+ slug: string;
7
+ status: PackageResultStatus;
8
+ durationMs: number;
9
+ error?: string;
10
+ step?: string;
11
+ sources?: {
12
+ learnJson: boolean;
13
+ configJson: boolean;
14
+ sidebar: boolean;
15
+ syllabus: boolean;
16
+ readmes: number;
17
+ };
18
+ };
19
+ export type BackfillOptions = {
20
+ dryRun: boolean;
21
+ force: boolean;
22
+ skipExisting: boolean;
23
+ noInvalidate: boolean;
24
+ concurrency: number;
25
+ publishedAt: string | null;
26
+ resumeFrom?: string;
27
+ reportOut: string;
28
+ slug?: string;
29
+ bucket: string;
30
+ region: string;
31
+ cloudFrontDistributionId?: string;
32
+ };
33
+ export type BatchReport = {
34
+ total: number;
35
+ created: number;
36
+ updated: number;
37
+ skipped: number;
38
+ failed: number;
39
+ startedAt: string;
40
+ finishedAt: string;
41
+ options: BackfillOptions;
42
+ results: PackageResult[];
43
+ invalidation?: {
44
+ attempted: boolean;
45
+ skippedReason?: string;
46
+ id?: string;
47
+ paths?: string[];
48
+ error?: string;
49
+ };
50
+ };
51
+ export type AwsClient = {
52
+ send(command: unknown): Promise<unknown>;
53
+ };
54
+ type PackageSources = {
55
+ learnJson: Record<string, unknown>;
56
+ exercises: Exercise[];
57
+ sidebar: Record<string, Record<string, string>>;
58
+ syllabus: Syllabus | null;
59
+ readmeFrontmatters: Record<string, Record<string, ReadmeFrontmatter>>;
60
+ meta: PackageResult["sources"];
61
+ };
62
+ export declare function streamToString(body: unknown): Promise<string>;
63
+ export declare function withRetry<T>(fn: () => Promise<T>, attempts?: number, attempt?: number): Promise<T>;
64
+ export declare function mapWithConcurrency<T, R>(items: T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;
65
+ export declare function listPublishedSlugs(s3: AwsClient, bucket: string): Promise<string[]>;
66
+ export declare function fetchJsonObject<T>(s3: AwsClient, bucket: string, key: string): Promise<T | null>;
67
+ export declare function fetchPackageSources(s3: AwsClient, bucket: string, slug: string): Promise<PackageSources>;
68
+ export declare function buildManifestFromS3Sources(slug: string, sources: PackageSources, options: {
69
+ publishedAt: string | null;
70
+ generatedAt?: string;
71
+ }): PackageManifest;
72
+ export declare function processPackage(s3: AwsClient, bucket: string, slug: string, options: Pick<BackfillOptions, "dryRun" | "force" | "skipExisting" | "publishedAt">): Promise<PackageResult>;
73
+ export declare function invalidatePackageManifestPaths(cf: AwsClient, distributionId: string, paths: string[]): Promise<{
74
+ id: string;
75
+ }>;
76
+ export declare function runBatch(options: BackfillOptions, clients?: {
77
+ s3?: AwsClient;
78
+ cf?: AwsClient;
79
+ onProgress?: (index: number, total: number, result: PackageResult) => void;
80
+ }): Promise<BatchReport>;
81
+ export declare function getBatchExitCode(report: BatchReport): number;
82
+ export {};
@@ -0,0 +1,485 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.streamToString = streamToString;
4
+ exports.withRetry = withRetry;
5
+ exports.mapWithConcurrency = mapWithConcurrency;
6
+ exports.listPublishedSlugs = listPublishedSlugs;
7
+ exports.fetchJsonObject = fetchJsonObject;
8
+ exports.fetchPackageSources = fetchPackageSources;
9
+ exports.buildManifestFromS3Sources = buildManifestFromS3Sources;
10
+ exports.processPackage = processPackage;
11
+ exports.invalidatePackageManifestPaths = invalidatePackageManifestPaths;
12
+ exports.runBatch = runBatch;
13
+ exports.getBatchExitCode = getBatchExitCode;
14
+ const tslib_1 = require("tslib");
15
+ const client_cloudfront_1 = require("@aws-sdk/client-cloudfront");
16
+ const client_s3_1 = require("@aws-sdk/client-s3");
17
+ const packageManifest_1 = require("../packageManifest");
18
+ const README_FILENAME_PATTERN = /^readme(?:\.[a-z]{2})?\.md$/i;
19
+ const INDEX_HTML_PATTERN = /^[^/]+\/index\.html$/;
20
+ const TRANSIENT_ERROR_CODES = new Set([
21
+ "SlowDown",
22
+ "ServiceUnavailable",
23
+ "RequestTimeout",
24
+ "Throttling",
25
+ "ThrottlingException",
26
+ "TooManyRequestsException",
27
+ ]);
28
+ const CLOUDFRONT_INVALIDATION_CHUNK_SIZE = 3000;
29
+ async function streamToString(body) {
30
+ var _a, e_1, _b, _c;
31
+ if (!body) {
32
+ return "";
33
+ }
34
+ if (typeof body === "string") {
35
+ return body;
36
+ }
37
+ if (Buffer.isBuffer(body)) {
38
+ return body.toString("utf-8");
39
+ }
40
+ if (body instanceof Uint8Array) {
41
+ return Buffer.from(body).toString("utf-8");
42
+ }
43
+ const chunks = [];
44
+ try {
45
+ for (var _d = true, _e = tslib_1.__asyncValues(body), _f; _f = await _e.next(), _a = _f.done, !_a; _d = true) {
46
+ _c = _f.value;
47
+ _d = false;
48
+ const chunk = _c;
49
+ chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
50
+ }
51
+ }
52
+ catch (e_1_1) { e_1 = { error: e_1_1 }; }
53
+ finally {
54
+ try {
55
+ if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
56
+ }
57
+ finally { if (e_1) throw e_1.error; }
58
+ }
59
+ return Buffer.concat(chunks).toString("utf-8");
60
+ }
61
+ function isTransientError(error) {
62
+ var _a;
63
+ const err = error;
64
+ const code = err.name || err.Code || "";
65
+ if (TRANSIENT_ERROR_CODES.has(code)) {
66
+ return true;
67
+ }
68
+ const status = (_a = err.$metadata) === null || _a === void 0 ? void 0 : _a.httpStatusCode;
69
+ return status === 429 || status === 503;
70
+ }
71
+ function sleep(ms) {
72
+ return new Promise(resolve => {
73
+ setTimeout(resolve, ms);
74
+ });
75
+ }
76
+ async function withRetry(fn, attempts = 3, attempt = 1) {
77
+ try {
78
+ return await fn();
79
+ }
80
+ catch (error) {
81
+ if (!isTransientError(error) || attempt === attempts) {
82
+ throw error;
83
+ }
84
+ const multiplier = 2 ** (attempt - 1);
85
+ const delayMs = 250 * multiplier;
86
+ await sleep(delayMs);
87
+ return withRetry(fn, attempts, attempt + 1);
88
+ }
89
+ }
90
+ async function mapWithConcurrency(items, limit, fn) {
91
+ const results = Array.from({ length: items.length });
92
+ let nextIndex = 0;
93
+ async function worker() {
94
+ while (nextIndex < items.length) {
95
+ const currentIndex = nextIndex;
96
+ nextIndex += 1;
97
+ // eslint-disable-next-line no-await-in-loop -- each worker processes items sequentially
98
+ results[currentIndex] = await fn(items[currentIndex]);
99
+ }
100
+ }
101
+ const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
102
+ await Promise.all(workers);
103
+ return results;
104
+ }
105
+ function manifestKey(slug) {
106
+ return `${slug}/${packageManifest_1.PACKAGE_MANIFEST_REL_PATH}`;
107
+ }
108
+ function cloudFrontManifestPath(slug) {
109
+ return `/${slug}/${packageManifest_1.PACKAGE_MANIFEST_REL_PATH}`;
110
+ }
111
+ function isNotFoundError(error) {
112
+ const err = error;
113
+ const code = err.name || err.Code || "";
114
+ return code === "NoSuchKey" || code === "NotFound" || code === "NoSuchBucket";
115
+ }
116
+ async function listPublishedSlugs(s3, bucket) {
117
+ const slugs = new Set();
118
+ let continuationToken;
119
+ do {
120
+ // eslint-disable-next-line no-await-in-loop -- S3 pagination requires sequential requests
121
+ const response = (await withRetry(() => s3.send(new client_s3_1.ListObjectsV2Command({
122
+ Bucket: bucket,
123
+ ContinuationToken: continuationToken,
124
+ }))));
125
+ for (const item of response.Contents || []) {
126
+ const key = item.Key;
127
+ if (key && INDEX_HTML_PATTERN.test(key)) {
128
+ slugs.add(key.split("/")[0]);
129
+ }
130
+ }
131
+ continuationToken = response.NextContinuationToken;
132
+ } while (continuationToken);
133
+ return [...slugs].sort();
134
+ }
135
+ async function fetchJsonObject(s3, bucket, key) {
136
+ try {
137
+ const response = (await withRetry(() => s3.send(new client_s3_1.GetObjectCommand({ Bucket: bucket, Key: key }))));
138
+ const content = await streamToString(response.Body);
139
+ return JSON.parse(content);
140
+ }
141
+ catch (error) {
142
+ if (isNotFoundError(error)) {
143
+ return null;
144
+ }
145
+ throw error;
146
+ }
147
+ }
148
+ async function fetchTextObject(s3, bucket, key) {
149
+ try {
150
+ const response = (await withRetry(() => s3.send(new client_s3_1.GetObjectCommand({ Bucket: bucket, Key: key }))));
151
+ return await streamToString(response.Body);
152
+ }
153
+ catch (error) {
154
+ if (isNotFoundError(error)) {
155
+ return null;
156
+ }
157
+ throw error;
158
+ }
159
+ }
160
+ async function listObjectKeys(s3, bucket, prefix) {
161
+ const keys = [];
162
+ let continuationToken;
163
+ do {
164
+ // eslint-disable-next-line no-await-in-loop -- S3 pagination requires sequential requests
165
+ const response = (await withRetry(() => s3.send(new client_s3_1.ListObjectsV2Command({
166
+ Bucket: bucket,
167
+ Prefix: prefix,
168
+ ContinuationToken: continuationToken,
169
+ }))));
170
+ for (const item of response.Contents || []) {
171
+ if (item.Key) {
172
+ keys.push(item.Key);
173
+ }
174
+ }
175
+ continuationToken = response.NextContinuationToken;
176
+ } while (continuationToken);
177
+ return keys;
178
+ }
179
+ function resolveReadmeLang(filename, exercise) {
180
+ const lower = filename.toLowerCase();
181
+ for (const [lang, readmeFile] of Object.entries(exercise.translations)) {
182
+ if (readmeFile.toLowerCase() === lower) {
183
+ return lang;
184
+ }
185
+ }
186
+ return null;
187
+ }
188
+ function normalizeLearnAndExercises(slug, learnJson, configJson) {
189
+ const exercises = configJson === null || configJson === void 0 ? void 0 : configJson.exercises;
190
+ if (!Array.isArray(exercises) || exercises.length === 0) {
191
+ throw new Error(`Missing or empty exercises array in ${slug}/config.json`);
192
+ }
193
+ const resolvedLearn = learnJson ||
194
+ (configJson === null || configJson === void 0 ? void 0 : configJson.config) ||
195
+ null;
196
+ if (!resolvedLearn) {
197
+ throw new Error(`Missing learn metadata: no ${slug}/learn.json and no config.config`);
198
+ }
199
+ return { learnJson: resolvedLearn, exercises };
200
+ }
201
+ async function fetchPackageSources(s3, bucket, slug) {
202
+ const [learnJsonRaw, configJsonRoot, configJsonLearn, sidebarLearn, sidebarRoot, syllabus,] = await Promise.all([
203
+ fetchJsonObject(s3, bucket, `${slug}/learn.json`),
204
+ fetchJsonObject(s3, bucket, `${slug}/config.json`),
205
+ fetchJsonObject(s3, bucket, `${slug}/.learn/config.json`),
206
+ fetchJsonObject(s3, bucket, `${slug}/.learn/sidebar.json`),
207
+ fetchJsonObject(s3, bucket, `${slug}/sidebar.json`),
208
+ fetchJsonObject(s3, bucket, `${slug}/.learn/initialSyllabus.json`),
209
+ ]);
210
+ const configJson = configJsonRoot || configJsonLearn;
211
+ if (!configJson) {
212
+ throw new Error(`Missing required file: ${slug}/config.json or ${slug}/.learn/config.json`);
213
+ }
214
+ const { learnJson, exercises } = normalizeLearnAndExercises(slug, learnJsonRaw, configJson);
215
+ const exerciseSlugs = new Set(exercises.map(exercise => exercise.slug));
216
+ const readmeKeys = (await listObjectKeys(s3, bucket, `${slug}/exercises/`))
217
+ .filter(key => {
218
+ const filename = key.split("/").pop() || "";
219
+ return README_FILENAME_PATTERN.test(filename);
220
+ })
221
+ .filter(key => {
222
+ const parts = key.split("/");
223
+ const exerciseSlug = parts[2];
224
+ return Boolean(exerciseSlug && exerciseSlugs.has(exerciseSlug));
225
+ });
226
+ const readmeFrontmatters = {};
227
+ await Promise.all(readmeKeys.map(async (key) => {
228
+ const parts = key.split("/");
229
+ const exerciseSlug = parts[2];
230
+ const filename = parts[parts.length - 1];
231
+ const exercise = exercises.find(item => item.slug === exerciseSlug);
232
+ if (!exercise) {
233
+ return;
234
+ }
235
+ const lang = resolveReadmeLang(filename, exercise);
236
+ if (!lang) {
237
+ return;
238
+ }
239
+ const content = await fetchTextObject(s3, bucket, key);
240
+ if (!content) {
241
+ return;
242
+ }
243
+ if (!readmeFrontmatters[exerciseSlug]) {
244
+ readmeFrontmatters[exerciseSlug] = {};
245
+ }
246
+ readmeFrontmatters[exerciseSlug][lang] = (0, packageManifest_1.parseReadmeContent)(content);
247
+ }));
248
+ const readmeCount = Object.values(readmeFrontmatters).reduce((total, byLang) => total + Object.keys(byLang).length, 0);
249
+ return {
250
+ learnJson,
251
+ exercises,
252
+ sidebar: sidebarLearn || sidebarRoot || {},
253
+ syllabus,
254
+ readmeFrontmatters,
255
+ meta: {
256
+ learnJson: Boolean(learnJsonRaw || configJson.config),
257
+ configJson: true,
258
+ sidebar: Boolean(sidebarLearn || sidebarRoot),
259
+ syllabus: Boolean(syllabus),
260
+ readmes: readmeCount,
261
+ },
262
+ };
263
+ }
264
+ function buildManifestFromS3Sources(slug, sources, options) {
265
+ return (0, packageManifest_1.buildPackageManifestFromSources)({
266
+ learnJson: sources.learnJson,
267
+ syllabus: sources.syllabus,
268
+ sidebar: sources.sidebar,
269
+ exercises: sources.exercises,
270
+ readmeFrontmatters: sources.readmeFrontmatters,
271
+ options: {
272
+ slug,
273
+ generatedAt: options.generatedAt || new Date().toISOString(),
274
+ publishedAt: options.publishedAt,
275
+ },
276
+ });
277
+ }
278
+ async function processPackage(s3, bucket, slug, options) {
279
+ const started = Date.now();
280
+ try {
281
+ const existing = await fetchJsonObject(s3, bucket, manifestKey(slug));
282
+ if (options.skipExisting && existing) {
283
+ return {
284
+ slug,
285
+ status: "skipped",
286
+ durationMs: Date.now() - started,
287
+ sources: {
288
+ learnJson: false,
289
+ configJson: false,
290
+ sidebar: false,
291
+ syllabus: false,
292
+ readmes: 0,
293
+ },
294
+ };
295
+ }
296
+ const sources = await fetchPackageSources(s3, bucket, slug);
297
+ const next = buildManifestFromS3Sources(slug, sources, {
298
+ publishedAt: options.publishedAt,
299
+ });
300
+ let status = (0, packageManifest_1.comparePackageManifests)(existing, next);
301
+ if (options.force && status === "skipped") {
302
+ status = "updated";
303
+ }
304
+ if (!options.dryRun && (status === "created" || status === "updated")) {
305
+ await withRetry(() => s3.send(new client_s3_1.PutObjectCommand({
306
+ Bucket: bucket,
307
+ Key: manifestKey(slug),
308
+ Body: (0, packageManifest_1.serializePackageManifest)(next),
309
+ ContentType: "application/json",
310
+ })));
311
+ }
312
+ return {
313
+ slug,
314
+ status,
315
+ durationMs: Date.now() - started,
316
+ sources: sources.meta,
317
+ };
318
+ }
319
+ catch (error) {
320
+ const err = error;
321
+ return {
322
+ slug,
323
+ status: "failed",
324
+ durationMs: Date.now() - started,
325
+ error: err.message,
326
+ step: err.step || inferFailureStep(err.message),
327
+ };
328
+ }
329
+ }
330
+ function inferFailureStep(message) {
331
+ if (message.includes("config.json") || message.includes("learn")) {
332
+ return "fetch_sources";
333
+ }
334
+ if (message.includes("PutObject")) {
335
+ return "put_object";
336
+ }
337
+ return "build";
338
+ }
339
+ async function invalidatePackageManifestPaths(cf, distributionId, paths) {
340
+ var _a;
341
+ if (paths.length === 0) {
342
+ throw new Error("No paths to invalidate");
343
+ }
344
+ const callerReference = `package-manifest-backfill-${Date.now()}`;
345
+ let lastId = "";
346
+ for (let index = 0; index < paths.length; index += CLOUDFRONT_INVALIDATION_CHUNK_SIZE) {
347
+ const chunk = paths.slice(index, index + CLOUDFRONT_INVALIDATION_CHUNK_SIZE);
348
+ // eslint-disable-next-line no-await-in-loop -- sequential invalidation chunks to avoid rate limits
349
+ const response = (await withRetry(() => cf.send(new client_cloudfront_1.CreateInvalidationCommand({
350
+ DistributionId: distributionId,
351
+ InvalidationBatch: {
352
+ CallerReference: `${callerReference}-${index}`,
353
+ Paths: {
354
+ Quantity: chunk.length,
355
+ Items: chunk,
356
+ },
357
+ },
358
+ }))));
359
+ lastId = ((_a = response.Invalidation) === null || _a === void 0 ? void 0 : _a.Id) || lastId;
360
+ }
361
+ return { id: lastId };
362
+ }
363
+ function loadResumeSlugs(reportPath) {
364
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
365
+ const fs = require("fs");
366
+ const raw = fs.readFileSync(reportPath, "utf-8");
367
+ const report = JSON.parse(raw);
368
+ const skipStatuses = new Set([
369
+ "created",
370
+ "updated",
371
+ "skipped",
372
+ ]);
373
+ return new Set(report.results
374
+ .filter(result => skipStatuses.has(result.status))
375
+ .map(result => result.slug));
376
+ }
377
+ function summarizeResults(results) {
378
+ return {
379
+ created: results.filter(result => result.status === "created").length,
380
+ updated: results.filter(result => result.status === "updated").length,
381
+ skipped: results.filter(result => result.status === "skipped").length,
382
+ failed: results.filter(result => result.status === "failed").length,
383
+ };
384
+ }
385
+ async function runBatch(options, clients) {
386
+ const startedAt = new Date().toISOString();
387
+ const s3 = (clients === null || clients === void 0 ? void 0 : clients.s3) ||
388
+ new client_s3_1.S3Client({
389
+ region: options.region,
390
+ credentials: {
391
+ accessKeyId: process.env.AWS_ACCESS_KEY_ID || "",
392
+ secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "",
393
+ },
394
+ });
395
+ const cf = (clients === null || clients === void 0 ? void 0 : clients.cf) ||
396
+ new client_cloudfront_1.CloudFrontClient({
397
+ region: "us-east-1",
398
+ credentials: {
399
+ accessKeyId: process.env.AWS_ACCESS_KEY_ID || "",
400
+ secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "",
401
+ },
402
+ });
403
+ let slugs = options.slug ?
404
+ [options.slug] :
405
+ await listPublishedSlugs(s3, options.bucket);
406
+ if (options.resumeFrom) {
407
+ const resumeSlugs = loadResumeSlugs(options.resumeFrom);
408
+ slugs = slugs.filter(slug => !resumeSlugs.has(slug));
409
+ }
410
+ let completed = 0;
411
+ const results = await mapWithConcurrency(slugs, options.concurrency, async (slug) => {
412
+ var _a;
413
+ const result = await processPackage(s3, options.bucket, slug, options);
414
+ completed += 1;
415
+ (_a = clients === null || clients === void 0 ? void 0 : clients.onProgress) === null || _a === void 0 ? void 0 : _a.call(clients, completed, slugs.length, result);
416
+ return result;
417
+ });
418
+ const summary = summarizeResults(results);
419
+ const invalidationPaths = results
420
+ .filter(result => result.status === "created" || result.status === "updated")
421
+ .map(result => cloudFrontManifestPath(result.slug));
422
+ let invalidation;
423
+ if (options.dryRun) {
424
+ invalidation = {
425
+ attempted: false,
426
+ skippedReason: "dry-run",
427
+ };
428
+ }
429
+ else if (options.noInvalidate) {
430
+ invalidation = {
431
+ attempted: false,
432
+ skippedReason: "no-invalidate",
433
+ };
434
+ }
435
+ else if (invalidationPaths.length === 0) {
436
+ invalidation = {
437
+ attempted: false,
438
+ skippedReason: "no-paths",
439
+ };
440
+ }
441
+ else if (!options.cloudFrontDistributionId) {
442
+ invalidation = {
443
+ attempted: true,
444
+ paths: invalidationPaths,
445
+ error: "Missing CLOUDFRONT_DISTRIBUTION_ID",
446
+ };
447
+ }
448
+ else {
449
+ try {
450
+ const result = await invalidatePackageManifestPaths(cf, options.cloudFrontDistributionId, invalidationPaths);
451
+ invalidation = {
452
+ attempted: true,
453
+ id: result.id,
454
+ paths: invalidationPaths,
455
+ };
456
+ }
457
+ catch (error) {
458
+ invalidation = {
459
+ attempted: true,
460
+ paths: invalidationPaths,
461
+ error: error.message,
462
+ };
463
+ }
464
+ }
465
+ const report = Object.assign(Object.assign({ total: slugs.length }, summary), { startedAt, finishedAt: new Date().toISOString(), options,
466
+ results,
467
+ invalidation });
468
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
469
+ const fs = require("fs");
470
+ // eslint-disable-next-line @typescript-eslint/no-var-requires
471
+ const path = require("path");
472
+ fs.mkdirSync(path.dirname(options.reportOut), { recursive: true });
473
+ fs.writeFileSync(options.reportOut, JSON.stringify(report, null, 2), "utf-8");
474
+ return report;
475
+ }
476
+ function getBatchExitCode(report) {
477
+ var _a;
478
+ if (report.failed > 0) {
479
+ return 1;
480
+ }
481
+ if (((_a = report.invalidation) === null || _a === void 0 ? void 0 : _a.attempted) && report.invalidation.error) {
482
+ return 1;
483
+ }
484
+ return 0;
485
+ }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@learnpack/learnpack",
3
3
  "description": "Seamlessly build, sell and/or take interactive & auto-graded tutorials, start learning now or build a new tutorial to your audience.",
4
- "version": "5.0.351",
4
+ "version": "5.0.352",
5
5
  "author": "Alejandro Sanchez @alesanchezr",
6
6
  "contributors": [
7
7
  {
@@ -21,6 +21,8 @@
21
21
  "url": "https://github.com/learnpack/learnpack-cli/issues"
22
22
  },
23
23
  "dependencies": {
24
+ "@aws-sdk/client-cloudfront": "^3.1092.0",
25
+ "@aws-sdk/client-s3": "^3.1092.0",
24
26
  "@google-cloud/storage": "^7.16.0",
25
27
  "@oclif/command": "^1.6.1",
26
28
  "@oclif/config": "^1.15.1",
@@ -161,6 +163,7 @@
161
163
  ]
162
164
  },
163
165
  "scripts": {
166
+ "backfill:package-manifest": "ts-node --project ./tsconfig.json ./scripts/backfill-package-manifest-s3.ts",
164
167
  "copy-assets": "npx cpy src/creatorDist/**/* lib/creatorDist --parents --verbose && npx cpy src/utils/templates/**/* lib/utils/templates --parents --verbose && npx cpy src/lua/**/* lib/lua --parents --verbose",
165
168
  "tsc": "tsc -b",
166
169
  "postpack": "rm -f oclif.manifest.json && eslint . --ext .ts --config .eslintrc",