@learnpack/learnpack 5.0.351 → 5.0.352
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/utils/packageManifest.d.ts +4 -0
- package/lib/utils/packageManifest.js +36 -6
- package/lib/utils/s3/packageManifestBackfill.d.ts +82 -0
- package/lib/utils/s3/packageManifestBackfill.js +485 -0
- package/package.json +4 -1
- package/src/ui/_app/app.js +2 -2
- package/src/ui/app.tar.gz +0 -0
- package/src/utils/export/README.md +0 -2
- package/src/utils/packageManifest.ts +51 -10
- package/src/utils/s3/packageManifestBackfill.ts +776 -0
|
@@ -11,6 +11,7 @@ export type PackageManifestLesson = {
|
|
|
11
11
|
position: number;
|
|
12
12
|
type: LessonType;
|
|
13
13
|
title: Record<string, string>;
|
|
14
|
+
description: Record<string, string | null>;
|
|
14
15
|
video: Record<string, string | null>;
|
|
15
16
|
};
|
|
16
17
|
export type PackageManifest = {
|
|
@@ -48,6 +49,9 @@ export declare function resolvePreviewUrl(slug: string, preview?: string): strin
|
|
|
48
49
|
export declare function extractReadmeH1(body: string): string | null;
|
|
49
50
|
export declare function normalizeReadmeTitle(raw: string): string | null;
|
|
50
51
|
export declare function buildPackageManifestFromSources(input: PackageManifestBuildInput): PackageManifest;
|
|
52
|
+
export declare function parseReadmeContent(content: string): ReadmeFrontmatter;
|
|
53
|
+
export declare function comparePackageManifests(existing: PackageManifest | null, next: PackageManifest): "created" | "updated" | "skipped";
|
|
54
|
+
export declare function serializePackageManifest(manifest: PackageManifest): string;
|
|
51
55
|
export declare function generateAndPersistPackageManifest(bucket: Bucket, courseSlug: string, options: {
|
|
52
56
|
publishedAt: string | null;
|
|
53
57
|
localDir?: string;
|
|
@@ -6,7 +6,11 @@ exports.resolvePreviewUrl = resolvePreviewUrl;
|
|
|
6
6
|
exports.extractReadmeH1 = extractReadmeH1;
|
|
7
7
|
exports.normalizeReadmeTitle = normalizeReadmeTitle;
|
|
8
8
|
exports.buildPackageManifestFromSources = buildPackageManifestFromSources;
|
|
9
|
+
exports.parseReadmeContent = parseReadmeContent;
|
|
10
|
+
exports.comparePackageManifests = comparePackageManifests;
|
|
11
|
+
exports.serializePackageManifest = serializePackageManifest;
|
|
9
12
|
exports.generateAndPersistPackageManifest = generateAndPersistPackageManifest;
|
|
13
|
+
const tslib_1 = require("tslib");
|
|
10
14
|
const fs = require("fs");
|
|
11
15
|
const mkdirp = require("mkdirp");
|
|
12
16
|
const path = require("path");
|
|
@@ -90,6 +94,14 @@ function resolveLessonTitle(exercise, sidebar, syllabusLesson, courseLang, readm
|
|
|
90
94
|
}
|
|
91
95
|
return titles;
|
|
92
96
|
}
|
|
97
|
+
function resolveLessonDescriptions(exercise) {
|
|
98
|
+
const description = {};
|
|
99
|
+
for (const lang of Object.keys(exercise.translations)) {
|
|
100
|
+
// TODO: add description content
|
|
101
|
+
description[lang] = null;
|
|
102
|
+
}
|
|
103
|
+
return description;
|
|
104
|
+
}
|
|
93
105
|
function resolveLessonVideos(exercise, readmeFrontmatters) {
|
|
94
106
|
const video = {};
|
|
95
107
|
const frontmatters = readmeFrontmatters[exercise.slug] || {};
|
|
@@ -122,6 +134,7 @@ function buildPackageManifestFromSources(input) {
|
|
|
122
134
|
position: exercise.position,
|
|
123
135
|
type,
|
|
124
136
|
title: resolveLessonTitle(exercise, sidebar, syllabusLesson, courseLang, (_b = readmeFrontmatters[exercise.slug]) !== null && _b !== void 0 ? _b : {}),
|
|
137
|
+
description: resolveLessonDescriptions(exercise),
|
|
125
138
|
video: resolveLessonVideos(exercise, readmeFrontmatters),
|
|
126
139
|
};
|
|
127
140
|
});
|
|
@@ -150,17 +163,34 @@ async function downloadJsonFile(bucket, filePath) {
|
|
|
150
163
|
return null;
|
|
151
164
|
}
|
|
152
165
|
}
|
|
166
|
+
function parseReadmeContent(content) {
|
|
167
|
+
const parsed = frontMatter(content);
|
|
168
|
+
const attributes = parsed.attributes || {};
|
|
169
|
+
const h1 = extractReadmeH1(parsed.body || "");
|
|
170
|
+
const title = h1 ? normalizeReadmeTitle(h1) : null;
|
|
171
|
+
return Object.assign(Object.assign(Object.assign({}, (attributes.tutorial ? { tutorial: attributes.tutorial } : {})), (attributes.intro ? { intro: attributes.intro } : {})), (title ? { title } : {}));
|
|
172
|
+
}
|
|
173
|
+
function comparePackageManifests(existing, next) {
|
|
174
|
+
if (existing === null) {
|
|
175
|
+
return "created";
|
|
176
|
+
}
|
|
177
|
+
const { generatedAt: _existingGeneratedAt } = existing, existingRest = tslib_1.__rest(existing, ["generatedAt"]);
|
|
178
|
+
const { generatedAt: _nextGeneratedAt } = next, nextRest = tslib_1.__rest(next, ["generatedAt"]);
|
|
179
|
+
if (JSON.stringify(existingRest) === JSON.stringify(nextRest)) {
|
|
180
|
+
return "skipped";
|
|
181
|
+
}
|
|
182
|
+
return "updated";
|
|
183
|
+
}
|
|
184
|
+
function serializePackageManifest(manifest) {
|
|
185
|
+
return JSON.stringify(manifest, null, 2);
|
|
186
|
+
}
|
|
153
187
|
async function readReadmeFrontmatter(bucket, exerciseSlug, lang, readmePath) {
|
|
154
188
|
try {
|
|
155
189
|
const [buf] = await bucket.file(readmePath).download();
|
|
156
|
-
const parsed = frontMatter(buf.toString());
|
|
157
|
-
const attributes = parsed.attributes || {};
|
|
158
|
-
const h1 = extractReadmeH1(parsed.body || "");
|
|
159
|
-
const title = h1 ? normalizeReadmeTitle(h1) : null;
|
|
160
190
|
return {
|
|
161
191
|
exerciseSlug,
|
|
162
192
|
lang,
|
|
163
|
-
frontmatter:
|
|
193
|
+
frontmatter: parseReadmeContent(buf.toString()),
|
|
164
194
|
};
|
|
165
195
|
}
|
|
166
196
|
catch (error) {
|
|
@@ -185,7 +215,7 @@ async function collectReadmeFrontmatters(bucket, courseSlug, exercises) {
|
|
|
185
215
|
return result;
|
|
186
216
|
}
|
|
187
217
|
function writeManifestJson(manifest, localDir) {
|
|
188
|
-
const json =
|
|
218
|
+
const json = serializePackageManifest(manifest);
|
|
189
219
|
if (localDir) {
|
|
190
220
|
const learnDir = path.join(localDir, ".learn");
|
|
191
221
|
mkdirp.sync(learnDir);
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { Exercise } from "../configBuilder";
|
|
2
|
+
import { Syllabus } from "../../models/creator";
|
|
3
|
+
import { PackageManifest, ReadmeFrontmatter } from "../packageManifest";
|
|
4
|
+
export type PackageResultStatus = "created" | "updated" | "skipped" | "failed";
|
|
5
|
+
export type PackageResult = {
|
|
6
|
+
slug: string;
|
|
7
|
+
status: PackageResultStatus;
|
|
8
|
+
durationMs: number;
|
|
9
|
+
error?: string;
|
|
10
|
+
step?: string;
|
|
11
|
+
sources?: {
|
|
12
|
+
learnJson: boolean;
|
|
13
|
+
configJson: boolean;
|
|
14
|
+
sidebar: boolean;
|
|
15
|
+
syllabus: boolean;
|
|
16
|
+
readmes: number;
|
|
17
|
+
};
|
|
18
|
+
};
|
|
19
|
+
export type BackfillOptions = {
|
|
20
|
+
dryRun: boolean;
|
|
21
|
+
force: boolean;
|
|
22
|
+
skipExisting: boolean;
|
|
23
|
+
noInvalidate: boolean;
|
|
24
|
+
concurrency: number;
|
|
25
|
+
publishedAt: string | null;
|
|
26
|
+
resumeFrom?: string;
|
|
27
|
+
reportOut: string;
|
|
28
|
+
slug?: string;
|
|
29
|
+
bucket: string;
|
|
30
|
+
region: string;
|
|
31
|
+
cloudFrontDistributionId?: string;
|
|
32
|
+
};
|
|
33
|
+
export type BatchReport = {
|
|
34
|
+
total: number;
|
|
35
|
+
created: number;
|
|
36
|
+
updated: number;
|
|
37
|
+
skipped: number;
|
|
38
|
+
failed: number;
|
|
39
|
+
startedAt: string;
|
|
40
|
+
finishedAt: string;
|
|
41
|
+
options: BackfillOptions;
|
|
42
|
+
results: PackageResult[];
|
|
43
|
+
invalidation?: {
|
|
44
|
+
attempted: boolean;
|
|
45
|
+
skippedReason?: string;
|
|
46
|
+
id?: string;
|
|
47
|
+
paths?: string[];
|
|
48
|
+
error?: string;
|
|
49
|
+
};
|
|
50
|
+
};
|
|
51
|
+
export type AwsClient = {
|
|
52
|
+
send(command: unknown): Promise<unknown>;
|
|
53
|
+
};
|
|
54
|
+
type PackageSources = {
|
|
55
|
+
learnJson: Record<string, unknown>;
|
|
56
|
+
exercises: Exercise[];
|
|
57
|
+
sidebar: Record<string, Record<string, string>>;
|
|
58
|
+
syllabus: Syllabus | null;
|
|
59
|
+
readmeFrontmatters: Record<string, Record<string, ReadmeFrontmatter>>;
|
|
60
|
+
meta: PackageResult["sources"];
|
|
61
|
+
};
|
|
62
|
+
export declare function streamToString(body: unknown): Promise<string>;
|
|
63
|
+
export declare function withRetry<T>(fn: () => Promise<T>, attempts?: number, attempt?: number): Promise<T>;
|
|
64
|
+
export declare function mapWithConcurrency<T, R>(items: T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;
|
|
65
|
+
export declare function listPublishedSlugs(s3: AwsClient, bucket: string): Promise<string[]>;
|
|
66
|
+
export declare function fetchJsonObject<T>(s3: AwsClient, bucket: string, key: string): Promise<T | null>;
|
|
67
|
+
export declare function fetchPackageSources(s3: AwsClient, bucket: string, slug: string): Promise<PackageSources>;
|
|
68
|
+
export declare function buildManifestFromS3Sources(slug: string, sources: PackageSources, options: {
|
|
69
|
+
publishedAt: string | null;
|
|
70
|
+
generatedAt?: string;
|
|
71
|
+
}): PackageManifest;
|
|
72
|
+
export declare function processPackage(s3: AwsClient, bucket: string, slug: string, options: Pick<BackfillOptions, "dryRun" | "force" | "skipExisting" | "publishedAt">): Promise<PackageResult>;
|
|
73
|
+
export declare function invalidatePackageManifestPaths(cf: AwsClient, distributionId: string, paths: string[]): Promise<{
|
|
74
|
+
id: string;
|
|
75
|
+
}>;
|
|
76
|
+
export declare function runBatch(options: BackfillOptions, clients?: {
|
|
77
|
+
s3?: AwsClient;
|
|
78
|
+
cf?: AwsClient;
|
|
79
|
+
onProgress?: (index: number, total: number, result: PackageResult) => void;
|
|
80
|
+
}): Promise<BatchReport>;
|
|
81
|
+
export declare function getBatchExitCode(report: BatchReport): number;
|
|
82
|
+
export {};
|
|
@@ -0,0 +1,485 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.streamToString = streamToString;
|
|
4
|
+
exports.withRetry = withRetry;
|
|
5
|
+
exports.mapWithConcurrency = mapWithConcurrency;
|
|
6
|
+
exports.listPublishedSlugs = listPublishedSlugs;
|
|
7
|
+
exports.fetchJsonObject = fetchJsonObject;
|
|
8
|
+
exports.fetchPackageSources = fetchPackageSources;
|
|
9
|
+
exports.buildManifestFromS3Sources = buildManifestFromS3Sources;
|
|
10
|
+
exports.processPackage = processPackage;
|
|
11
|
+
exports.invalidatePackageManifestPaths = invalidatePackageManifestPaths;
|
|
12
|
+
exports.runBatch = runBatch;
|
|
13
|
+
exports.getBatchExitCode = getBatchExitCode;
|
|
14
|
+
const tslib_1 = require("tslib");
|
|
15
|
+
const client_cloudfront_1 = require("@aws-sdk/client-cloudfront");
|
|
16
|
+
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
17
|
+
const packageManifest_1 = require("../packageManifest");
|
|
18
|
+
const README_FILENAME_PATTERN = /^readme(?:\.[a-z]{2})?\.md$/i;
|
|
19
|
+
const INDEX_HTML_PATTERN = /^[^/]+\/index\.html$/;
|
|
20
|
+
const TRANSIENT_ERROR_CODES = new Set([
|
|
21
|
+
"SlowDown",
|
|
22
|
+
"ServiceUnavailable",
|
|
23
|
+
"RequestTimeout",
|
|
24
|
+
"Throttling",
|
|
25
|
+
"ThrottlingException",
|
|
26
|
+
"TooManyRequestsException",
|
|
27
|
+
]);
|
|
28
|
+
const CLOUDFRONT_INVALIDATION_CHUNK_SIZE = 3000;
|
|
29
|
+
async function streamToString(body) {
|
|
30
|
+
var _a, e_1, _b, _c;
|
|
31
|
+
if (!body) {
|
|
32
|
+
return "";
|
|
33
|
+
}
|
|
34
|
+
if (typeof body === "string") {
|
|
35
|
+
return body;
|
|
36
|
+
}
|
|
37
|
+
if (Buffer.isBuffer(body)) {
|
|
38
|
+
return body.toString("utf-8");
|
|
39
|
+
}
|
|
40
|
+
if (body instanceof Uint8Array) {
|
|
41
|
+
return Buffer.from(body).toString("utf-8");
|
|
42
|
+
}
|
|
43
|
+
const chunks = [];
|
|
44
|
+
try {
|
|
45
|
+
for (var _d = true, _e = tslib_1.__asyncValues(body), _f; _f = await _e.next(), _a = _f.done, !_a; _d = true) {
|
|
46
|
+
_c = _f.value;
|
|
47
|
+
_d = false;
|
|
48
|
+
const chunk = _c;
|
|
49
|
+
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
53
|
+
finally {
|
|
54
|
+
try {
|
|
55
|
+
if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
|
|
56
|
+
}
|
|
57
|
+
finally { if (e_1) throw e_1.error; }
|
|
58
|
+
}
|
|
59
|
+
return Buffer.concat(chunks).toString("utf-8");
|
|
60
|
+
}
|
|
61
|
+
function isTransientError(error) {
|
|
62
|
+
var _a;
|
|
63
|
+
const err = error;
|
|
64
|
+
const code = err.name || err.Code || "";
|
|
65
|
+
if (TRANSIENT_ERROR_CODES.has(code)) {
|
|
66
|
+
return true;
|
|
67
|
+
}
|
|
68
|
+
const status = (_a = err.$metadata) === null || _a === void 0 ? void 0 : _a.httpStatusCode;
|
|
69
|
+
return status === 429 || status === 503;
|
|
70
|
+
}
|
|
71
|
+
function sleep(ms) {
|
|
72
|
+
return new Promise(resolve => {
|
|
73
|
+
setTimeout(resolve, ms);
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
async function withRetry(fn, attempts = 3, attempt = 1) {
|
|
77
|
+
try {
|
|
78
|
+
return await fn();
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
if (!isTransientError(error) || attempt === attempts) {
|
|
82
|
+
throw error;
|
|
83
|
+
}
|
|
84
|
+
const multiplier = 2 ** (attempt - 1);
|
|
85
|
+
const delayMs = 250 * multiplier;
|
|
86
|
+
await sleep(delayMs);
|
|
87
|
+
return withRetry(fn, attempts, attempt + 1);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
async function mapWithConcurrency(items, limit, fn) {
|
|
91
|
+
const results = Array.from({ length: items.length });
|
|
92
|
+
let nextIndex = 0;
|
|
93
|
+
async function worker() {
|
|
94
|
+
while (nextIndex < items.length) {
|
|
95
|
+
const currentIndex = nextIndex;
|
|
96
|
+
nextIndex += 1;
|
|
97
|
+
// eslint-disable-next-line no-await-in-loop -- each worker processes items sequentially
|
|
98
|
+
results[currentIndex] = await fn(items[currentIndex]);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
|
|
102
|
+
await Promise.all(workers);
|
|
103
|
+
return results;
|
|
104
|
+
}
|
|
105
|
+
function manifestKey(slug) {
|
|
106
|
+
return `${slug}/${packageManifest_1.PACKAGE_MANIFEST_REL_PATH}`;
|
|
107
|
+
}
|
|
108
|
+
function cloudFrontManifestPath(slug) {
|
|
109
|
+
return `/${slug}/${packageManifest_1.PACKAGE_MANIFEST_REL_PATH}`;
|
|
110
|
+
}
|
|
111
|
+
function isNotFoundError(error) {
|
|
112
|
+
const err = error;
|
|
113
|
+
const code = err.name || err.Code || "";
|
|
114
|
+
return code === "NoSuchKey" || code === "NotFound" || code === "NoSuchBucket";
|
|
115
|
+
}
|
|
116
|
+
async function listPublishedSlugs(s3, bucket) {
|
|
117
|
+
const slugs = new Set();
|
|
118
|
+
let continuationToken;
|
|
119
|
+
do {
|
|
120
|
+
// eslint-disable-next-line no-await-in-loop -- S3 pagination requires sequential requests
|
|
121
|
+
const response = (await withRetry(() => s3.send(new client_s3_1.ListObjectsV2Command({
|
|
122
|
+
Bucket: bucket,
|
|
123
|
+
ContinuationToken: continuationToken,
|
|
124
|
+
}))));
|
|
125
|
+
for (const item of response.Contents || []) {
|
|
126
|
+
const key = item.Key;
|
|
127
|
+
if (key && INDEX_HTML_PATTERN.test(key)) {
|
|
128
|
+
slugs.add(key.split("/")[0]);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
continuationToken = response.NextContinuationToken;
|
|
132
|
+
} while (continuationToken);
|
|
133
|
+
return [...slugs].sort();
|
|
134
|
+
}
|
|
135
|
+
async function fetchJsonObject(s3, bucket, key) {
|
|
136
|
+
try {
|
|
137
|
+
const response = (await withRetry(() => s3.send(new client_s3_1.GetObjectCommand({ Bucket: bucket, Key: key }))));
|
|
138
|
+
const content = await streamToString(response.Body);
|
|
139
|
+
return JSON.parse(content);
|
|
140
|
+
}
|
|
141
|
+
catch (error) {
|
|
142
|
+
if (isNotFoundError(error)) {
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
throw error;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
async function fetchTextObject(s3, bucket, key) {
|
|
149
|
+
try {
|
|
150
|
+
const response = (await withRetry(() => s3.send(new client_s3_1.GetObjectCommand({ Bucket: bucket, Key: key }))));
|
|
151
|
+
return await streamToString(response.Body);
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
if (isNotFoundError(error)) {
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
throw error;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
async function listObjectKeys(s3, bucket, prefix) {
|
|
161
|
+
const keys = [];
|
|
162
|
+
let continuationToken;
|
|
163
|
+
do {
|
|
164
|
+
// eslint-disable-next-line no-await-in-loop -- S3 pagination requires sequential requests
|
|
165
|
+
const response = (await withRetry(() => s3.send(new client_s3_1.ListObjectsV2Command({
|
|
166
|
+
Bucket: bucket,
|
|
167
|
+
Prefix: prefix,
|
|
168
|
+
ContinuationToken: continuationToken,
|
|
169
|
+
}))));
|
|
170
|
+
for (const item of response.Contents || []) {
|
|
171
|
+
if (item.Key) {
|
|
172
|
+
keys.push(item.Key);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
continuationToken = response.NextContinuationToken;
|
|
176
|
+
} while (continuationToken);
|
|
177
|
+
return keys;
|
|
178
|
+
}
|
|
179
|
+
function resolveReadmeLang(filename, exercise) {
|
|
180
|
+
const lower = filename.toLowerCase();
|
|
181
|
+
for (const [lang, readmeFile] of Object.entries(exercise.translations)) {
|
|
182
|
+
if (readmeFile.toLowerCase() === lower) {
|
|
183
|
+
return lang;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return null;
|
|
187
|
+
}
|
|
188
|
+
function normalizeLearnAndExercises(slug, learnJson, configJson) {
|
|
189
|
+
const exercises = configJson === null || configJson === void 0 ? void 0 : configJson.exercises;
|
|
190
|
+
if (!Array.isArray(exercises) || exercises.length === 0) {
|
|
191
|
+
throw new Error(`Missing or empty exercises array in ${slug}/config.json`);
|
|
192
|
+
}
|
|
193
|
+
const resolvedLearn = learnJson ||
|
|
194
|
+
(configJson === null || configJson === void 0 ? void 0 : configJson.config) ||
|
|
195
|
+
null;
|
|
196
|
+
if (!resolvedLearn) {
|
|
197
|
+
throw new Error(`Missing learn metadata: no ${slug}/learn.json and no config.config`);
|
|
198
|
+
}
|
|
199
|
+
return { learnJson: resolvedLearn, exercises };
|
|
200
|
+
}
|
|
201
|
+
async function fetchPackageSources(s3, bucket, slug) {
|
|
202
|
+
const [learnJsonRaw, configJsonRoot, configJsonLearn, sidebarLearn, sidebarRoot, syllabus,] = await Promise.all([
|
|
203
|
+
fetchJsonObject(s3, bucket, `${slug}/learn.json`),
|
|
204
|
+
fetchJsonObject(s3, bucket, `${slug}/config.json`),
|
|
205
|
+
fetchJsonObject(s3, bucket, `${slug}/.learn/config.json`),
|
|
206
|
+
fetchJsonObject(s3, bucket, `${slug}/.learn/sidebar.json`),
|
|
207
|
+
fetchJsonObject(s3, bucket, `${slug}/sidebar.json`),
|
|
208
|
+
fetchJsonObject(s3, bucket, `${slug}/.learn/initialSyllabus.json`),
|
|
209
|
+
]);
|
|
210
|
+
const configJson = configJsonRoot || configJsonLearn;
|
|
211
|
+
if (!configJson) {
|
|
212
|
+
throw new Error(`Missing required file: ${slug}/config.json or ${slug}/.learn/config.json`);
|
|
213
|
+
}
|
|
214
|
+
const { learnJson, exercises } = normalizeLearnAndExercises(slug, learnJsonRaw, configJson);
|
|
215
|
+
const exerciseSlugs = new Set(exercises.map(exercise => exercise.slug));
|
|
216
|
+
const readmeKeys = (await listObjectKeys(s3, bucket, `${slug}/exercises/`))
|
|
217
|
+
.filter(key => {
|
|
218
|
+
const filename = key.split("/").pop() || "";
|
|
219
|
+
return README_FILENAME_PATTERN.test(filename);
|
|
220
|
+
})
|
|
221
|
+
.filter(key => {
|
|
222
|
+
const parts = key.split("/");
|
|
223
|
+
const exerciseSlug = parts[2];
|
|
224
|
+
return Boolean(exerciseSlug && exerciseSlugs.has(exerciseSlug));
|
|
225
|
+
});
|
|
226
|
+
const readmeFrontmatters = {};
|
|
227
|
+
await Promise.all(readmeKeys.map(async (key) => {
|
|
228
|
+
const parts = key.split("/");
|
|
229
|
+
const exerciseSlug = parts[2];
|
|
230
|
+
const filename = parts[parts.length - 1];
|
|
231
|
+
const exercise = exercises.find(item => item.slug === exerciseSlug);
|
|
232
|
+
if (!exercise) {
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
const lang = resolveReadmeLang(filename, exercise);
|
|
236
|
+
if (!lang) {
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
const content = await fetchTextObject(s3, bucket, key);
|
|
240
|
+
if (!content) {
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
if (!readmeFrontmatters[exerciseSlug]) {
|
|
244
|
+
readmeFrontmatters[exerciseSlug] = {};
|
|
245
|
+
}
|
|
246
|
+
readmeFrontmatters[exerciseSlug][lang] = (0, packageManifest_1.parseReadmeContent)(content);
|
|
247
|
+
}));
|
|
248
|
+
const readmeCount = Object.values(readmeFrontmatters).reduce((total, byLang) => total + Object.keys(byLang).length, 0);
|
|
249
|
+
return {
|
|
250
|
+
learnJson,
|
|
251
|
+
exercises,
|
|
252
|
+
sidebar: sidebarLearn || sidebarRoot || {},
|
|
253
|
+
syllabus,
|
|
254
|
+
readmeFrontmatters,
|
|
255
|
+
meta: {
|
|
256
|
+
learnJson: Boolean(learnJsonRaw || configJson.config),
|
|
257
|
+
configJson: true,
|
|
258
|
+
sidebar: Boolean(sidebarLearn || sidebarRoot),
|
|
259
|
+
syllabus: Boolean(syllabus),
|
|
260
|
+
readmes: readmeCount,
|
|
261
|
+
},
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
function buildManifestFromS3Sources(slug, sources, options) {
|
|
265
|
+
return (0, packageManifest_1.buildPackageManifestFromSources)({
|
|
266
|
+
learnJson: sources.learnJson,
|
|
267
|
+
syllabus: sources.syllabus,
|
|
268
|
+
sidebar: sources.sidebar,
|
|
269
|
+
exercises: sources.exercises,
|
|
270
|
+
readmeFrontmatters: sources.readmeFrontmatters,
|
|
271
|
+
options: {
|
|
272
|
+
slug,
|
|
273
|
+
generatedAt: options.generatedAt || new Date().toISOString(),
|
|
274
|
+
publishedAt: options.publishedAt,
|
|
275
|
+
},
|
|
276
|
+
});
|
|
277
|
+
}
|
|
278
|
+
async function processPackage(s3, bucket, slug, options) {
|
|
279
|
+
const started = Date.now();
|
|
280
|
+
try {
|
|
281
|
+
const existing = await fetchJsonObject(s3, bucket, manifestKey(slug));
|
|
282
|
+
if (options.skipExisting && existing) {
|
|
283
|
+
return {
|
|
284
|
+
slug,
|
|
285
|
+
status: "skipped",
|
|
286
|
+
durationMs: Date.now() - started,
|
|
287
|
+
sources: {
|
|
288
|
+
learnJson: false,
|
|
289
|
+
configJson: false,
|
|
290
|
+
sidebar: false,
|
|
291
|
+
syllabus: false,
|
|
292
|
+
readmes: 0,
|
|
293
|
+
},
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
const sources = await fetchPackageSources(s3, bucket, slug);
|
|
297
|
+
const next = buildManifestFromS3Sources(slug, sources, {
|
|
298
|
+
publishedAt: options.publishedAt,
|
|
299
|
+
});
|
|
300
|
+
let status = (0, packageManifest_1.comparePackageManifests)(existing, next);
|
|
301
|
+
if (options.force && status === "skipped") {
|
|
302
|
+
status = "updated";
|
|
303
|
+
}
|
|
304
|
+
if (!options.dryRun && (status === "created" || status === "updated")) {
|
|
305
|
+
await withRetry(() => s3.send(new client_s3_1.PutObjectCommand({
|
|
306
|
+
Bucket: bucket,
|
|
307
|
+
Key: manifestKey(slug),
|
|
308
|
+
Body: (0, packageManifest_1.serializePackageManifest)(next),
|
|
309
|
+
ContentType: "application/json",
|
|
310
|
+
})));
|
|
311
|
+
}
|
|
312
|
+
return {
|
|
313
|
+
slug,
|
|
314
|
+
status,
|
|
315
|
+
durationMs: Date.now() - started,
|
|
316
|
+
sources: sources.meta,
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
catch (error) {
|
|
320
|
+
const err = error;
|
|
321
|
+
return {
|
|
322
|
+
slug,
|
|
323
|
+
status: "failed",
|
|
324
|
+
durationMs: Date.now() - started,
|
|
325
|
+
error: err.message,
|
|
326
|
+
step: err.step || inferFailureStep(err.message),
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
function inferFailureStep(message) {
|
|
331
|
+
if (message.includes("config.json") || message.includes("learn")) {
|
|
332
|
+
return "fetch_sources";
|
|
333
|
+
}
|
|
334
|
+
if (message.includes("PutObject")) {
|
|
335
|
+
return "put_object";
|
|
336
|
+
}
|
|
337
|
+
return "build";
|
|
338
|
+
}
|
|
339
|
+
async function invalidatePackageManifestPaths(cf, distributionId, paths) {
|
|
340
|
+
var _a;
|
|
341
|
+
if (paths.length === 0) {
|
|
342
|
+
throw new Error("No paths to invalidate");
|
|
343
|
+
}
|
|
344
|
+
const callerReference = `package-manifest-backfill-${Date.now()}`;
|
|
345
|
+
let lastId = "";
|
|
346
|
+
for (let index = 0; index < paths.length; index += CLOUDFRONT_INVALIDATION_CHUNK_SIZE) {
|
|
347
|
+
const chunk = paths.slice(index, index + CLOUDFRONT_INVALIDATION_CHUNK_SIZE);
|
|
348
|
+
// eslint-disable-next-line no-await-in-loop -- sequential invalidation chunks to avoid rate limits
|
|
349
|
+
const response = (await withRetry(() => cf.send(new client_cloudfront_1.CreateInvalidationCommand({
|
|
350
|
+
DistributionId: distributionId,
|
|
351
|
+
InvalidationBatch: {
|
|
352
|
+
CallerReference: `${callerReference}-${index}`,
|
|
353
|
+
Paths: {
|
|
354
|
+
Quantity: chunk.length,
|
|
355
|
+
Items: chunk,
|
|
356
|
+
},
|
|
357
|
+
},
|
|
358
|
+
}))));
|
|
359
|
+
lastId = ((_a = response.Invalidation) === null || _a === void 0 ? void 0 : _a.Id) || lastId;
|
|
360
|
+
}
|
|
361
|
+
return { id: lastId };
|
|
362
|
+
}
|
|
363
|
+
function loadResumeSlugs(reportPath) {
|
|
364
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
365
|
+
const fs = require("fs");
|
|
366
|
+
const raw = fs.readFileSync(reportPath, "utf-8");
|
|
367
|
+
const report = JSON.parse(raw);
|
|
368
|
+
const skipStatuses = new Set([
|
|
369
|
+
"created",
|
|
370
|
+
"updated",
|
|
371
|
+
"skipped",
|
|
372
|
+
]);
|
|
373
|
+
return new Set(report.results
|
|
374
|
+
.filter(result => skipStatuses.has(result.status))
|
|
375
|
+
.map(result => result.slug));
|
|
376
|
+
}
|
|
377
|
+
function summarizeResults(results) {
|
|
378
|
+
return {
|
|
379
|
+
created: results.filter(result => result.status === "created").length,
|
|
380
|
+
updated: results.filter(result => result.status === "updated").length,
|
|
381
|
+
skipped: results.filter(result => result.status === "skipped").length,
|
|
382
|
+
failed: results.filter(result => result.status === "failed").length,
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
async function runBatch(options, clients) {
|
|
386
|
+
const startedAt = new Date().toISOString();
|
|
387
|
+
const s3 = (clients === null || clients === void 0 ? void 0 : clients.s3) ||
|
|
388
|
+
new client_s3_1.S3Client({
|
|
389
|
+
region: options.region,
|
|
390
|
+
credentials: {
|
|
391
|
+
accessKeyId: process.env.AWS_ACCESS_KEY_ID || "",
|
|
392
|
+
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "",
|
|
393
|
+
},
|
|
394
|
+
});
|
|
395
|
+
const cf = (clients === null || clients === void 0 ? void 0 : clients.cf) ||
|
|
396
|
+
new client_cloudfront_1.CloudFrontClient({
|
|
397
|
+
region: "us-east-1",
|
|
398
|
+
credentials: {
|
|
399
|
+
accessKeyId: process.env.AWS_ACCESS_KEY_ID || "",
|
|
400
|
+
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY || "",
|
|
401
|
+
},
|
|
402
|
+
});
|
|
403
|
+
let slugs = options.slug ?
|
|
404
|
+
[options.slug] :
|
|
405
|
+
await listPublishedSlugs(s3, options.bucket);
|
|
406
|
+
if (options.resumeFrom) {
|
|
407
|
+
const resumeSlugs = loadResumeSlugs(options.resumeFrom);
|
|
408
|
+
slugs = slugs.filter(slug => !resumeSlugs.has(slug));
|
|
409
|
+
}
|
|
410
|
+
let completed = 0;
|
|
411
|
+
const results = await mapWithConcurrency(slugs, options.concurrency, async (slug) => {
|
|
412
|
+
var _a;
|
|
413
|
+
const result = await processPackage(s3, options.bucket, slug, options);
|
|
414
|
+
completed += 1;
|
|
415
|
+
(_a = clients === null || clients === void 0 ? void 0 : clients.onProgress) === null || _a === void 0 ? void 0 : _a.call(clients, completed, slugs.length, result);
|
|
416
|
+
return result;
|
|
417
|
+
});
|
|
418
|
+
const summary = summarizeResults(results);
|
|
419
|
+
const invalidationPaths = results
|
|
420
|
+
.filter(result => result.status === "created" || result.status === "updated")
|
|
421
|
+
.map(result => cloudFrontManifestPath(result.slug));
|
|
422
|
+
let invalidation;
|
|
423
|
+
if (options.dryRun) {
|
|
424
|
+
invalidation = {
|
|
425
|
+
attempted: false,
|
|
426
|
+
skippedReason: "dry-run",
|
|
427
|
+
};
|
|
428
|
+
}
|
|
429
|
+
else if (options.noInvalidate) {
|
|
430
|
+
invalidation = {
|
|
431
|
+
attempted: false,
|
|
432
|
+
skippedReason: "no-invalidate",
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
else if (invalidationPaths.length === 0) {
|
|
436
|
+
invalidation = {
|
|
437
|
+
attempted: false,
|
|
438
|
+
skippedReason: "no-paths",
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
else if (!options.cloudFrontDistributionId) {
|
|
442
|
+
invalidation = {
|
|
443
|
+
attempted: true,
|
|
444
|
+
paths: invalidationPaths,
|
|
445
|
+
error: "Missing CLOUDFRONT_DISTRIBUTION_ID",
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
else {
|
|
449
|
+
try {
|
|
450
|
+
const result = await invalidatePackageManifestPaths(cf, options.cloudFrontDistributionId, invalidationPaths);
|
|
451
|
+
invalidation = {
|
|
452
|
+
attempted: true,
|
|
453
|
+
id: result.id,
|
|
454
|
+
paths: invalidationPaths,
|
|
455
|
+
};
|
|
456
|
+
}
|
|
457
|
+
catch (error) {
|
|
458
|
+
invalidation = {
|
|
459
|
+
attempted: true,
|
|
460
|
+
paths: invalidationPaths,
|
|
461
|
+
error: error.message,
|
|
462
|
+
};
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
const report = Object.assign(Object.assign({ total: slugs.length }, summary), { startedAt, finishedAt: new Date().toISOString(), options,
|
|
466
|
+
results,
|
|
467
|
+
invalidation });
|
|
468
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
469
|
+
const fs = require("fs");
|
|
470
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
471
|
+
const path = require("path");
|
|
472
|
+
fs.mkdirSync(path.dirname(options.reportOut), { recursive: true });
|
|
473
|
+
fs.writeFileSync(options.reportOut, JSON.stringify(report, null, 2), "utf-8");
|
|
474
|
+
return report;
|
|
475
|
+
}
|
|
476
|
+
function getBatchExitCode(report) {
|
|
477
|
+
var _a;
|
|
478
|
+
if (report.failed > 0) {
|
|
479
|
+
return 1;
|
|
480
|
+
}
|
|
481
|
+
if (((_a = report.invalidation) === null || _a === void 0 ? void 0 : _a.attempted) && report.invalidation.error) {
|
|
482
|
+
return 1;
|
|
483
|
+
}
|
|
484
|
+
return 0;
|
|
485
|
+
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@learnpack/learnpack",
|
|
3
3
|
"description": "Seamlessly build, sell and/or take interactive & auto-graded tutorials, start learning now or build a new tutorial to your audience.",
|
|
4
|
-
"version": "5.0.
|
|
4
|
+
"version": "5.0.352",
|
|
5
5
|
"author": "Alejandro Sanchez @alesanchezr",
|
|
6
6
|
"contributors": [
|
|
7
7
|
{
|
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
"url": "https://github.com/learnpack/learnpack-cli/issues"
|
|
22
22
|
},
|
|
23
23
|
"dependencies": {
|
|
24
|
+
"@aws-sdk/client-cloudfront": "^3.1092.0",
|
|
25
|
+
"@aws-sdk/client-s3": "^3.1092.0",
|
|
24
26
|
"@google-cloud/storage": "^7.16.0",
|
|
25
27
|
"@oclif/command": "^1.6.1",
|
|
26
28
|
"@oclif/config": "^1.15.1",
|
|
@@ -161,6 +163,7 @@
|
|
|
161
163
|
]
|
|
162
164
|
},
|
|
163
165
|
"scripts": {
|
|
166
|
+
"backfill:package-manifest": "ts-node --project ./tsconfig.json ./scripts/backfill-package-manifest-s3.ts",
|
|
164
167
|
"copy-assets": "npx cpy src/creatorDist/**/* lib/creatorDist --parents --verbose && npx cpy src/utils/templates/**/* lib/utils/templates --parents --verbose && npx cpy src/lua/**/* lib/lua --parents --verbose",
|
|
165
168
|
"tsc": "tsc -b",
|
|
166
169
|
"postpack": "rm -f oclif.manifest.json && eslint . --ext .ts --config .eslintrc",
|