@learnpack/learnpack 5.0.353 → 5.0.354
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/scripts/descriptionsS3Backfill.js +142 -22
- package/lib/utils/awsCredentials.d.ts +20 -0
- package/lib/utils/awsCredentials.js +43 -0
- package/lib/utils/descriptions/backfillEvents.d.ts +60 -0
- package/lib/utils/descriptions/backfillEvents.js +107 -0
- package/lib/utils/descriptions/generateCourseDescriptions.d.ts +7 -0
- package/lib/utils/descriptions/generateCourseDescriptions.js +3 -0
- package/lib/utils/descriptions/publishStage.js +14 -3
- package/lib/utils/descriptions/resumePublication.js +17 -2
- package/lib/utils/descriptions/s3Storage.js +13 -6
- package/lib/utils/packageManifest.d.ts +8 -0
- package/lib/utils/packageManifest.js +8 -0
- package/lib/utils/repair/legacyPackageRepair.d.ts +131 -0
- package/lib/utils/repair/legacyPackageRepair.js +492 -0
- package/lib/utils/repair/repairStorage.d.ts +68 -0
- package/lib/utils/repair/repairStorage.js +89 -0
- package/lib/utils/s3/packageManifestBackfill.js +3 -8
- package/lib/utils/s3/packageSourcesAudit.d.ts +75 -0
- package/lib/utils/s3/packageSourcesAudit.js +184 -0
- package/package.json +3 -1
- package/src/scripts/README.md +244 -0
- package/src/scripts/descriptionsS3Backfill.ts +188 -20
- package/src/utils/awsCredentials.ts +57 -0
- package/src/utils/descriptions/backfillEvents.ts +152 -0
- package/src/utils/descriptions/generateCourseDescriptions.ts +10 -0
- package/src/utils/descriptions/publishStage.ts +394 -382
- package/src/utils/descriptions/resumePublication.ts +217 -200
- package/src/utils/descriptions/s3Storage.ts +214 -206
- package/src/utils/packageManifest.ts +8 -1
- package/src/utils/repair/legacyPackageRepair.ts +731 -0
- package/src/utils/repair/repairStorage.ts +168 -0
- package/src/utils/s3/packageManifestBackfill.ts +771 -776
- package/src/utils/s3/packageSourcesAudit.ts +311 -0
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { Syllabus } from "../../models/creator";
|
|
2
|
+
import { Exercise } from "../configBuilder";
|
|
3
|
+
import { LessonType } from "../packageManifest";
|
|
4
|
+
import { RepairStorage, RepairTarget } from "./repairStorage";
|
|
5
|
+
/**
|
|
6
|
+
* Reconstruct the manifest inputs missing from legacy packages.
|
|
7
|
+
*
|
|
8
|
+
* Some packages predate the creator — a few were uploaded straight from a GitHub
|
|
9
|
+
* repository — and never got a `.learn/initialSyllabus.json`, a
|
|
10
|
+
* `.learn/sidebar.json`, or a usable `config.json`. The manifest tolerates the
|
|
11
|
+
* first two and comes out degraded: every lesson typed `READ`, and descriptions
|
|
12
|
+
* pinned to `null` forever, because `descriptionsS3Backfill` builds its work
|
|
13
|
+
* list from the syllabus keys and never even sees a course without one.
|
|
14
|
+
*
|
|
15
|
+
* This module writes those files back, derived from what the package *does*
|
|
16
|
+
* have: its exercise folders and their READMEs. It never invents content — a
|
|
17
|
+
* title comes from the README's own `# H1`, never from an LLM.
|
|
18
|
+
*
|
|
19
|
+
* It runs once per bucket (see `repairStorage.ts` for why the draft is repaired
|
|
20
|
+
* from the draft rather than copied from the snapshot). Ordering afterwards:
|
|
21
|
+
*
|
|
22
|
+
* 1. this script, on both targets (structure)
|
|
23
|
+
* 2. `descriptionsS3Backfill --reproject-manifest --mirror-draft`
|
|
24
|
+
* 3. `backfill:package-manifest`, unless step 2 projected it
|
|
25
|
+
*
|
|
26
|
+
* Safety: with one documented exception (a `config.json` whose `exercises` array
|
|
27
|
+
* is empty, which is merged rather than replaced) it only ever writes keys that
|
|
28
|
+
* do not exist. It never touches a file the real pipeline produced.
|
|
29
|
+
*/
|
|
30
|
+
export type RepairFile = "config" | "syllabus" | "sidebar";
|
|
31
|
+
export declare const ALL_REPAIR_FILES: RepairFile[];
|
|
32
|
+
export type FileOutcome = "created" | "merged" | "present" | "not-requested" | "impossible";
|
|
33
|
+
export type FileResult = {
|
|
34
|
+
outcome: FileOutcome;
|
|
35
|
+
key?: string;
|
|
36
|
+
reason?: string;
|
|
37
|
+
};
|
|
38
|
+
export type RepairStats = {
|
|
39
|
+
exercises: number;
|
|
40
|
+
languages: string[];
|
|
41
|
+
primaryLanguage: string;
|
|
42
|
+
lessonTypes: Record<LessonType, number>;
|
|
43
|
+
titlesFromReadme: number;
|
|
44
|
+
titlesFromFolderName: number;
|
|
45
|
+
};
|
|
46
|
+
export type RepairResult = {
|
|
47
|
+
slug: string;
|
|
48
|
+
target: RepairTarget;
|
|
49
|
+
/**
|
|
50
|
+
* `absent` is not a failure: half the legacy packages published in S3 have no
|
|
51
|
+
* draft in GCS at all, so a run over both buckets is expected to find nothing
|
|
52
|
+
* to repair on one side. Only a package that *is* in the bucket but yields no
|
|
53
|
+
* exercises counts as `failed`.
|
|
54
|
+
*/
|
|
55
|
+
status: "repaired" | "unchanged" | "absent" | "failed";
|
|
56
|
+
durationMs: number;
|
|
57
|
+
files: Record<RepairFile, FileResult>;
|
|
58
|
+
stats?: RepairStats;
|
|
59
|
+
error?: string;
|
|
60
|
+
};
|
|
61
|
+
export type RepairOptions = {
|
|
62
|
+
dryRun: boolean;
|
|
63
|
+
files: RepairFile[];
|
|
64
|
+
};
|
|
65
|
+
/** README body + parsed title, by exercise slug and language. */
|
|
66
|
+
export type ReadmeIndex = Record<string, Record<string, {
|
|
67
|
+
title: string | null;
|
|
68
|
+
body: string;
|
|
69
|
+
}>>;
|
|
70
|
+
/**
|
|
71
|
+
* Exercises from the bucket listing alone, mirroring `configBuilder.buildConfig`
|
|
72
|
+
* key for key: same README language rule, same natural ordering, same
|
|
73
|
+
* `graded: false` default.
|
|
74
|
+
*/
|
|
75
|
+
export declare function buildExercisesFromKeys(exercisesPrefix: string, keys: string[]): Exercise[];
|
|
76
|
+
/**
|
|
77
|
+
* Language the course is really written in.
|
|
78
|
+
*
|
|
79
|
+
* Same rule as `workList.inferCourseBaseLanguage`: a declared language is
|
|
80
|
+
* trusted only when some exercise actually has a README in it, otherwise the
|
|
81
|
+
* most frequent translation wins. Defaulting to "en" would label a Spanish
|
|
82
|
+
* course English and anchor every later description to a translation.
|
|
83
|
+
*
|
|
84
|
+
* A bilingual course ties on count — most of the legacy ones are — so the tie is
|
|
85
|
+
* broken on the file names rather than arbitrarily: `getReadmeExtension` gives
|
|
86
|
+
* the *original* README no language suffix and suffixes every translation, so
|
|
87
|
+
* the language mapped to a bare `README.md` is the one the course was written
|
|
88
|
+
* in. This matters beyond the title: the descriptions backfill generates from
|
|
89
|
+
* `courseInfo.language` and would otherwise work from a translation.
|
|
90
|
+
*/
|
|
91
|
+
export declare function inferPrimaryLanguage(exercises: Exercise[], declared?: string): string;
|
|
92
|
+
/**
|
|
93
|
+
* Lesson type, inferred — the one field of a synthesized syllabus that is a
|
|
94
|
+
* guess rather than a reading. Callers should treat it as such.
|
|
95
|
+
*/
|
|
96
|
+
export declare function inferLessonType(input: {
|
|
97
|
+
graded: boolean;
|
|
98
|
+
files: Array<{
|
|
99
|
+
name: string;
|
|
100
|
+
}>;
|
|
101
|
+
readmeBody: string;
|
|
102
|
+
}): LessonType;
|
|
103
|
+
export type BuildSyllabusInput = {
|
|
104
|
+
slug: string;
|
|
105
|
+
learnJson: Record<string, unknown> | null;
|
|
106
|
+
exercises: Exercise[];
|
|
107
|
+
readmes: ReadmeIndex;
|
|
108
|
+
primaryLanguage: string;
|
|
109
|
+
};
|
|
110
|
+
export type BuildSyllabusResult = {
|
|
111
|
+
syllabus: Syllabus;
|
|
112
|
+
stats: Omit<RepairStats, "exercises" | "languages" | "primaryLanguage">;
|
|
113
|
+
};
|
|
114
|
+
/**
|
|
115
|
+
* A syllabus equivalent to what the creator would have written, derived from the
|
|
116
|
+
* exercise folders and their READMEs.
|
|
117
|
+
*
|
|
118
|
+
* `uid` is set to the exercise folder slug on purpose. `findSyllabusLesson`
|
|
119
|
+
* matches on `slugify(id + "-" + title)` *or* `uid`, and only the second is
|
|
120
|
+
* guaranteed to hold once a title comes from an H1 the folder name never had —
|
|
121
|
+
* it is also what lets the S3 and GCS repairs, run independently, still line up
|
|
122
|
+
* for the descriptions mirror.
|
|
123
|
+
*/
|
|
124
|
+
export declare function buildSyllabusFromSources(input: BuildSyllabusInput): BuildSyllabusResult;
|
|
125
|
+
/** `sidebar[exerciseSlug][lang] = title`, from the same READMEs. */
|
|
126
|
+
export declare function buildSidebarFromReadmes(exercises: Exercise[], readmes: ReadmeIndex): Record<string, Record<string, string>>;
|
|
127
|
+
/**
|
|
128
|
+
* Repair one package in one bucket. Returns what it did (or would do, under
|
|
129
|
+
* `dryRun`) without throwing: a batch reports failures per slug.
|
|
130
|
+
*/
|
|
131
|
+
export declare function repairPackage(storage: RepairStorage, slug: string, options: RepairOptions): Promise<RepairResult>;
|
|
@@ -0,0 +1,492 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ALL_REPAIR_FILES = void 0;
|
|
4
|
+
exports.buildExercisesFromKeys = buildExercisesFromKeys;
|
|
5
|
+
exports.inferPrimaryLanguage = inferPrimaryLanguage;
|
|
6
|
+
exports.inferLessonType = inferLessonType;
|
|
7
|
+
exports.buildSyllabusFromSources = buildSyllabusFromSources;
|
|
8
|
+
exports.buildSidebarFromReadmes = buildSidebarFromReadmes;
|
|
9
|
+
exports.repairPackage = repairPackage;
|
|
10
|
+
const exerciseFileOrder_1 = require("../exerciseFileOrder");
|
|
11
|
+
const packageManifest_1 = require("../packageManifest");
|
|
12
|
+
exports.ALL_REPAIR_FILES = ["config", "syllabus", "sidebar"];
|
|
13
|
+
const README_FILENAME_PATTERN = /^readme(?:\.([a-z]{2}))?\.md$/i;
|
|
14
|
+
/**
|
|
15
|
+
* A markdown task list is how LearnPack renders a quiz, so two or more options
|
|
16
|
+
* in one README is the only signal a legacy package gives us about its type.
|
|
17
|
+
*/
|
|
18
|
+
const QUIZ_OPTION_PATTERN = /^\s*[*-]\s+\[[\sXx]]\s+\S/gm;
|
|
19
|
+
const CODE_FILE_EXTENSIONS = new Set([
|
|
20
|
+
"c",
|
|
21
|
+
"cpp",
|
|
22
|
+
"cs",
|
|
23
|
+
"css",
|
|
24
|
+
"go",
|
|
25
|
+
"html",
|
|
26
|
+
"java",
|
|
27
|
+
"js",
|
|
28
|
+
"json",
|
|
29
|
+
"jsx",
|
|
30
|
+
"php",
|
|
31
|
+
"py",
|
|
32
|
+
"rb",
|
|
33
|
+
"rs",
|
|
34
|
+
"scss",
|
|
35
|
+
"sh",
|
|
36
|
+
"sql",
|
|
37
|
+
"ts",
|
|
38
|
+
"tsx",
|
|
39
|
+
"vue",
|
|
40
|
+
]);
|
|
41
|
+
function humanizeFolderName(folderSlug) {
|
|
42
|
+
const match = folderSlug.match(/^(\d+(?:\.\d+)?)(?:-(.*))?$/);
|
|
43
|
+
const titlePart = match && match[2] ? match[2] : folderSlug;
|
|
44
|
+
return titlePart.replace(/-/g, " ").trim() || folderSlug;
|
|
45
|
+
}
|
|
46
|
+
function naturalCompare(a, b) {
|
|
47
|
+
const regex = /(\d+|\D+)/g;
|
|
48
|
+
const ax = a.match(regex) || [];
|
|
49
|
+
const bx = b.match(regex) || [];
|
|
50
|
+
for (let i = 0; i < Math.max(ax.length, bx.length); i++) {
|
|
51
|
+
const an = Number.parseInt(ax[i], 10);
|
|
52
|
+
const bn = Number.parseInt(bx[i], 10);
|
|
53
|
+
if (!Number.isNaN(an) && !Number.isNaN(bn)) {
|
|
54
|
+
if (an !== bn) {
|
|
55
|
+
return an - bn;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
else if (ax[i] !== bx[i]) {
|
|
59
|
+
return (ax[i] || "").localeCompare(bx[i] || "");
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return 0;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Exercises from the bucket listing alone, mirroring `configBuilder.buildConfig`
|
|
66
|
+
* key for key: same README language rule, same natural ordering, same
|
|
67
|
+
* `graded: false` default.
|
|
68
|
+
*/
|
|
69
|
+
function buildExercisesFromKeys(exercisesPrefix, keys) {
|
|
70
|
+
const map = new Map();
|
|
71
|
+
for (const key of keys) {
|
|
72
|
+
if (!key.startsWith(exercisesPrefix)) {
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
const parts = key.slice(exercisesPrefix.length).split("/");
|
|
76
|
+
if (parts.length < 2) {
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
const exerciseSlug = parts[0];
|
|
80
|
+
const filename = parts[parts.length - 1];
|
|
81
|
+
if (!exerciseSlug || !filename) {
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
let exercise = map.get(exerciseSlug);
|
|
85
|
+
if (!exercise) {
|
|
86
|
+
exercise = {
|
|
87
|
+
title: exerciseSlug,
|
|
88
|
+
slug: exerciseSlug,
|
|
89
|
+
graded: false,
|
|
90
|
+
files: [],
|
|
91
|
+
translations: {},
|
|
92
|
+
};
|
|
93
|
+
map.set(exerciseSlug, exercise);
|
|
94
|
+
}
|
|
95
|
+
const readmeMatch = filename.match(README_FILENAME_PATTERN);
|
|
96
|
+
// Nested files keep their folder-relative name, as the IDE expects.
|
|
97
|
+
if (readmeMatch && parts.length === 2) {
|
|
98
|
+
exercise.translations[readmeMatch[1] || "en"] = filename;
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
exercise.files.push({
|
|
102
|
+
name: parts.slice(1).join("/"),
|
|
103
|
+
slug: parts.slice(1).join("/"),
|
|
104
|
+
hidden: false,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return [...map.values()]
|
|
109
|
+
.sort((a, b) => naturalCompare(a.slug, b.slug))
|
|
110
|
+
.map((exercise, index) => (Object.assign(Object.assign({}, exercise), { files: (0, exerciseFileOrder_1.sortExerciseFiles)(exercise.files), position: index })));
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Language the course is really written in.
|
|
114
|
+
*
|
|
115
|
+
* Same rule as `workList.inferCourseBaseLanguage`: a declared language is
|
|
116
|
+
* trusted only when some exercise actually has a README in it, otherwise the
|
|
117
|
+
* most frequent translation wins. Defaulting to "en" would label a Spanish
|
|
118
|
+
* course English and anchor every later description to a translation.
|
|
119
|
+
*
|
|
120
|
+
* A bilingual course ties on count — most of the legacy ones are — so the tie is
|
|
121
|
+
* broken on the file names rather than arbitrarily: `getReadmeExtension` gives
|
|
122
|
+
* the *original* README no language suffix and suffixes every translation, so
|
|
123
|
+
* the language mapped to a bare `README.md` is the one the course was written
|
|
124
|
+
* in. This matters beyond the title: the descriptions backfill generates from
|
|
125
|
+
* `courseInfo.language` and would otherwise work from a translation.
|
|
126
|
+
*/
|
|
127
|
+
function inferPrimaryLanguage(exercises, declared) {
|
|
128
|
+
const counts = new Map();
|
|
129
|
+
const unsuffixed = new Set();
|
|
130
|
+
for (const exercise of exercises) {
|
|
131
|
+
for (const [lang, filename] of Object.entries(exercise.translations || {})) {
|
|
132
|
+
counts.set(lang, (counts.get(lang) || 0) + 1);
|
|
133
|
+
if (/^readme\.md$/i.test(filename)) {
|
|
134
|
+
unsuffixed.add(lang);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
if (counts.size === 0) {
|
|
139
|
+
return declared || "en";
|
|
140
|
+
}
|
|
141
|
+
const normalized = declared === null || declared === void 0 ? void 0 : declared.trim().toLowerCase();
|
|
142
|
+
if (normalized) {
|
|
143
|
+
for (const lang of counts.keys()) {
|
|
144
|
+
if (lang.toLowerCase() === normalized) {
|
|
145
|
+
return lang;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
const ranked = [...counts.entries()].sort((a, b) => {
|
|
150
|
+
if (a[1] !== b[1]) {
|
|
151
|
+
return b[1] - a[1];
|
|
152
|
+
}
|
|
153
|
+
const aBase = unsuffixed.has(a[0]) ? 0 : 1;
|
|
154
|
+
const bBase = unsuffixed.has(b[0]) ? 0 : 1;
|
|
155
|
+
return aBase === bBase ? a[0].localeCompare(b[0]) : aBase - bBase;
|
|
156
|
+
});
|
|
157
|
+
return ranked[0][0];
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Lesson type, inferred — the one field of a synthesized syllabus that is a
|
|
161
|
+
* guess rather than a reading. Callers should treat it as such.
|
|
162
|
+
*/
|
|
163
|
+
function inferLessonType(input) {
|
|
164
|
+
const quizOptions = input.readmeBody.match(QUIZ_OPTION_PATTERN) || [];
|
|
165
|
+
if (quizOptions.length >= 2) {
|
|
166
|
+
return "QUIZ";
|
|
167
|
+
}
|
|
168
|
+
if (input.graded) {
|
|
169
|
+
return "CODE";
|
|
170
|
+
}
|
|
171
|
+
const hasCodeFile = input.files.some(file => {
|
|
172
|
+
var _a;
|
|
173
|
+
const extension = ((_a = file.name.split(".").pop()) === null || _a === void 0 ? void 0 : _a.toLowerCase()) || "";
|
|
174
|
+
return CODE_FILE_EXTENSIONS.has(extension);
|
|
175
|
+
});
|
|
176
|
+
return hasCodeFile ? "CODE" : "READ";
|
|
177
|
+
}
|
|
178
|
+
function pickLocalized(value, primaryLanguage) {
|
|
179
|
+
if (typeof value === "string") {
|
|
180
|
+
return value;
|
|
181
|
+
}
|
|
182
|
+
if (!value || typeof value !== "object") {
|
|
183
|
+
return undefined;
|
|
184
|
+
}
|
|
185
|
+
const map = value;
|
|
186
|
+
const candidate = map[primaryLanguage] || Object.values(map)[0];
|
|
187
|
+
return typeof candidate === "string" ? candidate : undefined;
|
|
188
|
+
}
|
|
189
|
+
function buildCourseInfo(input) {
|
|
190
|
+
const learnJson = input.learnJson || {};
|
|
191
|
+
const difficulty = learnJson.difficulty;
|
|
192
|
+
const duration = learnJson.duration;
|
|
193
|
+
return {
|
|
194
|
+
title: pickLocalized(learnJson.title, input.primaryLanguage) || input.slug,
|
|
195
|
+
description: pickLocalized(learnJson.description, input.primaryLanguage) || "",
|
|
196
|
+
duration: typeof duration === "number" ? duration : 0,
|
|
197
|
+
difficulty: (typeof difficulty === "string" ?
|
|
198
|
+
difficulty :
|
|
199
|
+
"beginner"),
|
|
200
|
+
language: input.primaryLanguage,
|
|
201
|
+
technologies: Array.isArray(learnJson.technologies) ?
|
|
202
|
+
learnJson.technologies :
|
|
203
|
+
[],
|
|
204
|
+
slug: input.slug,
|
|
205
|
+
// Fields the creator wizard owns. A repaired package was never authored
|
|
206
|
+
// there, so they carry the "nothing pending" value rather than a guess.
|
|
207
|
+
hasContentIndex: false,
|
|
208
|
+
contentIndex: "",
|
|
209
|
+
isCompleted: true,
|
|
210
|
+
variables: [],
|
|
211
|
+
currentStep: "",
|
|
212
|
+
purpose: "",
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* A syllabus equivalent to what the creator would have written, derived from the
|
|
217
|
+
* exercise folders and their READMEs.
|
|
218
|
+
*
|
|
219
|
+
* `uid` is set to the exercise folder slug on purpose. `findSyllabusLesson`
|
|
220
|
+
* matches on `slugify(id + "-" + title)` *or* `uid`, and only the second is
|
|
221
|
+
* guaranteed to hold once a title comes from an H1 the folder name never had —
|
|
222
|
+
* it is also what lets the S3 and GCS repairs, run independently, still line up
|
|
223
|
+
* for the descriptions mirror.
|
|
224
|
+
*/
|
|
225
|
+
function buildSyllabusFromSources(input) {
|
|
226
|
+
const now = Date.now();
|
|
227
|
+
const lessonTypes = { READ: 0, CODE: 0, QUIZ: 0 };
|
|
228
|
+
let titlesFromReadme = 0;
|
|
229
|
+
let titlesFromFolderName = 0;
|
|
230
|
+
const lessons = [...input.exercises]
|
|
231
|
+
.sort((a, b) => a.position - b.position)
|
|
232
|
+
.map(exercise => {
|
|
233
|
+
const readmesByLang = input.readmes[exercise.slug] || {};
|
|
234
|
+
const primaryReadme = readmesByLang[input.primaryLanguage] ||
|
|
235
|
+
Object.values(readmesByLang)[0] || { title: null, body: "" };
|
|
236
|
+
const title = primaryReadme.title || humanizeFolderName(exercise.slug);
|
|
237
|
+
if (primaryReadme.title) {
|
|
238
|
+
titlesFromReadme += 1;
|
|
239
|
+
}
|
|
240
|
+
else {
|
|
241
|
+
titlesFromFolderName += 1;
|
|
242
|
+
}
|
|
243
|
+
const type = inferLessonType({
|
|
244
|
+
graded: exercise.graded,
|
|
245
|
+
files: exercise.files,
|
|
246
|
+
readmeBody: primaryReadme.body,
|
|
247
|
+
});
|
|
248
|
+
lessonTypes[type] += 1;
|
|
249
|
+
const translations = {};
|
|
250
|
+
for (const lang of Object.keys(exercise.translations || {})) {
|
|
251
|
+
// Same shape `syllabusSync` repairs a missing slot with. Deliberately
|
|
252
|
+
// left without a fingerprint or prompt version so that
|
|
253
|
+
// `workList.needsGeneration` returns true and the descriptions backfill
|
|
254
|
+
// picks the course up on its next run.
|
|
255
|
+
translations[lang] = {
|
|
256
|
+
completionId: 0,
|
|
257
|
+
startedAt: now,
|
|
258
|
+
completedAt: now,
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
return {
|
|
262
|
+
id: (0, packageManifest_1.deriveLessonId)(exercise.slug),
|
|
263
|
+
uid: exercise.slug,
|
|
264
|
+
title,
|
|
265
|
+
type,
|
|
266
|
+
// `Lesson.description` is the generation topic, not the step
|
|
267
|
+
// description shown to students — that one lives per language in
|
|
268
|
+
// `translations[lang].description`, which the backfill fills in.
|
|
269
|
+
description: title,
|
|
270
|
+
duration: 2,
|
|
271
|
+
generated: true,
|
|
272
|
+
status: "DONE",
|
|
273
|
+
translations,
|
|
274
|
+
};
|
|
275
|
+
});
|
|
276
|
+
return {
|
|
277
|
+
syllabus: {
|
|
278
|
+
lessons,
|
|
279
|
+
courseInfo: buildCourseInfo(input),
|
|
280
|
+
},
|
|
281
|
+
stats: { lessonTypes, titlesFromReadme, titlesFromFolderName },
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
/** `sidebar[exerciseSlug][lang] = title`, from the same READMEs. */
|
|
285
|
+
function buildSidebarFromReadmes(exercises, readmes) {
|
|
286
|
+
var _a;
|
|
287
|
+
const sidebar = {};
|
|
288
|
+
for (const exercise of exercises) {
|
|
289
|
+
const readmesByLang = readmes[exercise.slug] || {};
|
|
290
|
+
const entry = {};
|
|
291
|
+
for (const lang of Object.keys(exercise.translations || {})) {
|
|
292
|
+
entry[lang] =
|
|
293
|
+
((_a = readmesByLang[lang]) === null || _a === void 0 ? void 0 : _a.title) || humanizeFolderName(exercise.slug);
|
|
294
|
+
}
|
|
295
|
+
if (Object.keys(entry).length > 0) {
|
|
296
|
+
sidebar[exercise.slug] = entry;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
return sidebar;
|
|
300
|
+
}
|
|
301
|
+
async function readReadmes(storage, slug, exercises) {
|
|
302
|
+
const index = {};
|
|
303
|
+
const prefix = storage.layout.exercisesPrefix(slug);
|
|
304
|
+
await Promise.all(exercises.flatMap(exercise => Object.entries(exercise.translations || {}).map(async ([lang, filename]) => {
|
|
305
|
+
const content = await storage.readText(`${prefix}${exercise.slug}/${filename}`);
|
|
306
|
+
if (content === null) {
|
|
307
|
+
return;
|
|
308
|
+
}
|
|
309
|
+
const { title } = (0, packageManifest_1.parseReadmeContent)(content);
|
|
310
|
+
if (!index[exercise.slug]) {
|
|
311
|
+
index[exercise.slug] = {};
|
|
312
|
+
}
|
|
313
|
+
index[exercise.slug][lang] = { title: title || null, body: content };
|
|
314
|
+
})));
|
|
315
|
+
return index;
|
|
316
|
+
}
|
|
317
|
+
/** First key that resolves, with the parsed value. */
|
|
318
|
+
async function readFirst(storage, keys) {
|
|
319
|
+
for (const key of keys) {
|
|
320
|
+
// eslint-disable-next-line no-await-in-loop -- later keys are fallbacks
|
|
321
|
+
const value = await storage.readJson(key);
|
|
322
|
+
if (value) {
|
|
323
|
+
return { value, key };
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
return null;
|
|
327
|
+
}
|
|
328
|
+
function initialFiles(files) {
|
|
329
|
+
const requested = new Set(files);
|
|
330
|
+
const base = "not-requested";
|
|
331
|
+
return {
|
|
332
|
+
config: { outcome: requested.has("config") ? "present" : base },
|
|
333
|
+
syllabus: { outcome: requested.has("syllabus") ? "present" : base },
|
|
334
|
+
sidebar: { outcome: requested.has("sidebar") ? "present" : base },
|
|
335
|
+
};
|
|
336
|
+
}
|
|
337
|
+
/**
|
|
338
|
+
* Repair one package in one bucket. Returns what it did (or would do, under
|
|
339
|
+
* `dryRun`) without throwing: a batch reports failures per slug.
|
|
340
|
+
*/
|
|
341
|
+
async function repairPackage(storage, slug, options) {
|
|
342
|
+
var _a;
|
|
343
|
+
const started = Date.now();
|
|
344
|
+
const { layout } = storage;
|
|
345
|
+
const wants = new Set(options.files);
|
|
346
|
+
const files = initialFiles(options.files);
|
|
347
|
+
const result = (status, extra = {}) => (Object.assign({ slug, target: storage.target, status, durationMs: Date.now() - started, files }, extra));
|
|
348
|
+
try {
|
|
349
|
+
const [learnJson, existingConfig, existingSyllabus, existingSidebar] = await Promise.all([
|
|
350
|
+
storage.readJson(layout.learnJsonKey(slug)),
|
|
351
|
+
readFirst(storage, layout.configKeys(slug)),
|
|
352
|
+
storage.readJson(layout.syllabusKey(slug)),
|
|
353
|
+
readFirst(storage, layout.sidebarKeys(slug)),
|
|
354
|
+
]);
|
|
355
|
+
const configuredExercises = Array.isArray(existingConfig === null || existingConfig === void 0 ? void 0 : existingConfig.value.exercises) ?
|
|
356
|
+
existingConfig.value.exercises :
|
|
357
|
+
[];
|
|
358
|
+
const needsConfig = wants.has("config") && configuredExercises.length === 0;
|
|
359
|
+
const needsSyllabus = wants.has("syllabus") && !existingSyllabus;
|
|
360
|
+
const needsSidebar = wants.has("sidebar") && !existingSidebar;
|
|
361
|
+
if (wants.has("config") && !needsConfig) {
|
|
362
|
+
files.config = { outcome: "present", key: existingConfig.key };
|
|
363
|
+
}
|
|
364
|
+
if (wants.has("syllabus") && !needsSyllabus) {
|
|
365
|
+
files.syllabus = { outcome: "present", key: layout.syllabusKey(slug) };
|
|
366
|
+
}
|
|
367
|
+
if (wants.has("sidebar") && !needsSidebar) {
|
|
368
|
+
files.sidebar = { outcome: "present", key: existingSidebar.key };
|
|
369
|
+
}
|
|
370
|
+
// Nothing to build: stop before the listing and the READMEs, which is what
|
|
371
|
+
// makes a catalogue-wide pass over healthy packages cheap.
|
|
372
|
+
if (!needsConfig && !needsSyllabus && !needsSidebar) {
|
|
373
|
+
return result("unchanged");
|
|
374
|
+
}
|
|
375
|
+
const useConfigExercises = layout.exercisesFromConfig && configuredExercises.length > 0;
|
|
376
|
+
const derivedExercises = useConfigExercises ?
|
|
377
|
+
[] :
|
|
378
|
+
buildExercisesFromKeys(layout.exercisesPrefix(slug), await storage.listKeys(layout.exercisesPrefix(slug)));
|
|
379
|
+
// Even where the listing is the authority on the exercise list, `graded` is
|
|
380
|
+
// worth keeping: the listing cannot express it (`buildConfig` hardcodes
|
|
381
|
+
// false) and it is a signal for typing a lesson CODE.
|
|
382
|
+
const gradedBySlug = new Map(configuredExercises.map(exercise => [exercise.slug, exercise.graded]));
|
|
383
|
+
const exercises = useConfigExercises ?
|
|
384
|
+
configuredExercises :
|
|
385
|
+
derivedExercises.map(exercise => {
|
|
386
|
+
var _a;
|
|
387
|
+
return (Object.assign(Object.assign({}, exercise), { graded: (_a = gradedBySlug.get(exercise.slug)) !== null && _a !== void 0 ? _a : exercise.graded }));
|
|
388
|
+
});
|
|
389
|
+
if (exercises.length === 0) {
|
|
390
|
+
const anythingAtAll = Boolean(learnJson || existingConfig || existingSyllabus || existingSidebar);
|
|
391
|
+
if (anythingAtAll) {
|
|
392
|
+
return result("failed", {
|
|
393
|
+
error: `No exercises found for "${slug}" in ${storage.target}: neither its config.json nor its exercises/ listing yields any`,
|
|
394
|
+
});
|
|
395
|
+
}
|
|
396
|
+
for (const file of options.files) {
|
|
397
|
+
files[file] = {
|
|
398
|
+
outcome: "impossible",
|
|
399
|
+
reason: `the package is not in ${storage.target}`,
|
|
400
|
+
};
|
|
401
|
+
}
|
|
402
|
+
return result("absent");
|
|
403
|
+
}
|
|
404
|
+
const readmes = await readReadmes(storage, slug, exercises);
|
|
405
|
+
const primaryLanguage = inferPrimaryLanguage(exercises, (learnJson === null || learnJson === void 0 ? void 0 : learnJson.language) ||
|
|
406
|
+
((_a = existingSyllabus === null || existingSyllabus === void 0 ? void 0 : existingSyllabus.courseInfo) === null || _a === void 0 ? void 0 : _a.language));
|
|
407
|
+
let wrote = false;
|
|
408
|
+
// --- config.json ----------------------------------------------------
|
|
409
|
+
if (needsConfig) {
|
|
410
|
+
// `config` is not decoration: the publish route destructures
|
|
411
|
+
// `{ config, exercises }` and reads `config.title` right after, so a
|
|
412
|
+
// config carrying only exercises would break the next publication.
|
|
413
|
+
const configField = (existingConfig === null || existingConfig === void 0 ? void 0 : existingConfig.value.config) || learnJson;
|
|
414
|
+
if (configField) {
|
|
415
|
+
const key = (existingConfig === null || existingConfig === void 0 ? void 0 : existingConfig.key) || layout.configKeys(slug)[0];
|
|
416
|
+
const merged = Object.assign(Object.assign({}, existingConfig === null || existingConfig === void 0 ? void 0 : existingConfig.value), { config: configField, exercises: derivedExercises });
|
|
417
|
+
files.config = {
|
|
418
|
+
outcome: existingConfig ? "merged" : "created",
|
|
419
|
+
key,
|
|
420
|
+
reason: existingConfig ?
|
|
421
|
+
"existing config.json had no exercises; other fields preserved" :
|
|
422
|
+
"no config.json in the package",
|
|
423
|
+
};
|
|
424
|
+
if (!options.dryRun) {
|
|
425
|
+
await storage.writeJson(key, merged);
|
|
426
|
+
}
|
|
427
|
+
wrote = true;
|
|
428
|
+
}
|
|
429
|
+
else {
|
|
430
|
+
files.config = {
|
|
431
|
+
outcome: "impossible",
|
|
432
|
+
reason: "no learn.json and no config.config to build it from",
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
// --- initialSyllabus.json -------------------------------------------
|
|
437
|
+
let syllabusStats;
|
|
438
|
+
if (needsSyllabus) {
|
|
439
|
+
const built = buildSyllabusFromSources({
|
|
440
|
+
slug,
|
|
441
|
+
learnJson,
|
|
442
|
+
exercises,
|
|
443
|
+
readmes,
|
|
444
|
+
primaryLanguage,
|
|
445
|
+
});
|
|
446
|
+
syllabusStats = built.stats;
|
|
447
|
+
files.syllabus = { outcome: "created", key: layout.syllabusKey(slug) };
|
|
448
|
+
if (!options.dryRun) {
|
|
449
|
+
await storage.writeJson(layout.syllabusKey(slug), built.syllabus);
|
|
450
|
+
}
|
|
451
|
+
wrote = true;
|
|
452
|
+
}
|
|
453
|
+
// --- sidebar.json ----------------------------------------------------
|
|
454
|
+
if (needsSidebar) {
|
|
455
|
+
const sidebar = buildSidebarFromReadmes(exercises, readmes);
|
|
456
|
+
if (Object.keys(sidebar).length === 0) {
|
|
457
|
+
files.sidebar = {
|
|
458
|
+
outcome: "impossible",
|
|
459
|
+
reason: "no README titles to build it from",
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
else {
|
|
463
|
+
const key = layout.sidebarKeys(slug)[0];
|
|
464
|
+
files.sidebar = { outcome: "created", key };
|
|
465
|
+
if (!options.dryRun) {
|
|
466
|
+
await storage.writeJson(key, sidebar);
|
|
467
|
+
}
|
|
468
|
+
wrote = true;
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
const languages = [
|
|
472
|
+
...new Set(exercises.flatMap(exercise => Object.keys(exercise.translations || {}))),
|
|
473
|
+
].sort();
|
|
474
|
+
return result(wrote ? "repaired" : "unchanged", {
|
|
475
|
+
stats: {
|
|
476
|
+
exercises: exercises.length,
|
|
477
|
+
languages,
|
|
478
|
+
primaryLanguage,
|
|
479
|
+
lessonTypes: (syllabusStats === null || syllabusStats === void 0 ? void 0 : syllabusStats.lessonTypes) || {
|
|
480
|
+
READ: 0,
|
|
481
|
+
CODE: 0,
|
|
482
|
+
QUIZ: 0,
|
|
483
|
+
},
|
|
484
|
+
titlesFromReadme: (syllabusStats === null || syllabusStats === void 0 ? void 0 : syllabusStats.titlesFromReadme) || 0,
|
|
485
|
+
titlesFromFolderName: (syllabusStats === null || syllabusStats === void 0 ? void 0 : syllabusStats.titlesFromFolderName) || 0,
|
|
486
|
+
},
|
|
487
|
+
});
|
|
488
|
+
}
|
|
489
|
+
catch (error) {
|
|
490
|
+
return result("failed", { error: error.message });
|
|
491
|
+
}
|
|
492
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { Bucket } from "@google-cloud/storage";
|
|
2
|
+
import { AwsClient } from "../s3/packageManifestBackfill";
|
|
3
|
+
/**
|
|
4
|
+
* The two buckets a package lives in, behind one interface.
|
|
5
|
+
*
|
|
6
|
+
* A legacy package is repaired **twice, independently** — once against the
|
|
7
|
+
* published snapshot in S3 and once against the draft in GCS — never by copying
|
|
8
|
+
* one into the other. The two hold different content (the draft keeps moving
|
|
9
|
+
* after a publication) and the synthesized structure is a projection of that
|
|
10
|
+
* content: a syllabus derived from the snapshot could name steps the draft does
|
|
11
|
+
* not have. This is the same rule `mirrorDescriptions` enforces with its content
|
|
12
|
+
* hash; deriving from each bucket needs no guard at all.
|
|
13
|
+
*
|
|
14
|
+
* What crosses between them is only the descriptions, through
|
|
15
|
+
* `descriptionsS3Backfill --mirror-draft`, which is hash-guarded per step.
|
|
16
|
+
*/
|
|
17
|
+
export type RepairTarget = "s3" | "gcs";
|
|
18
|
+
export declare const ALL_REPAIR_TARGETS: RepairTarget[];
|
|
19
|
+
/**
|
|
20
|
+
* Where each input lives. The two buckets do not agree on this, and the
|
|
21
|
+
* disagreement is load-bearing:
|
|
22
|
+
*
|
|
23
|
+
* - S3 reads `{slug}/config.json` first (`fetchPackageSources`), which is where
|
|
24
|
+
* the publish writes it into the zip.
|
|
25
|
+
* - GCS reads `courses/{slug}/.learn/config.json` — the publish route downloads
|
|
26
|
+
* that one before anything else, so a config written anywhere else in the
|
|
27
|
+
* draft would not make the package publishable.
|
|
28
|
+
*/
|
|
29
|
+
export type PackageLayout = {
|
|
30
|
+
learnJsonKey(slug: string): string;
|
|
31
|
+
/** Config locations in resolution order; the first is the write target. */
|
|
32
|
+
configKeys(slug: string): string[];
|
|
33
|
+
syllabusKey(slug: string): string;
|
|
34
|
+
/** Sidebar locations in resolution order; the first is the write target. */
|
|
35
|
+
sidebarKeys(slug: string): string[];
|
|
36
|
+
exercisesPrefix(slug: string): string;
|
|
37
|
+
/**
|
|
38
|
+
* Whether `config.json` is the authority on the exercise list, or the folder
|
|
39
|
+
* listing is.
|
|
40
|
+
*
|
|
41
|
+
* This decides the **language keys** of the synthesized syllabus, and the two
|
|
42
|
+
* buckets genuinely disagree:
|
|
43
|
+
*
|
|
44
|
+
* - S3 reads the published `config.json` (`fetchPackageSources`), where these
|
|
45
|
+
* legacy packages carry `us` for the unsuffixed README.
|
|
46
|
+
* - GCS has no such reader. Both the publish-time manifest and the
|
|
47
|
+
* descriptions pipeline go through `configBuilder.buildConfig`, which
|
|
48
|
+
* derives exercises from the listing and maps an unsuffixed `README.md` to
|
|
49
|
+
* `en` — regardless of what the draft's `.learn/config.json` says.
|
|
50
|
+
*
|
|
51
|
+
* A syllabus keyed the other way is not read at all: the manifest would look
|
|
52
|
+
* up `translations["en"]`, find nothing, and publish `description: null` for
|
|
53
|
+
* every step. So each side is built the way its own reader will read it.
|
|
54
|
+
*/
|
|
55
|
+
exercisesFromConfig: boolean;
|
|
56
|
+
};
|
|
57
|
+
export type RepairStorage = {
|
|
58
|
+
target: RepairTarget;
|
|
59
|
+
layout: PackageLayout;
|
|
60
|
+
readJson<T>(key: string): Promise<T | null>;
|
|
61
|
+
readText(key: string): Promise<string | null>;
|
|
62
|
+
listKeys(prefix: string): Promise<string[]>;
|
|
63
|
+
writeJson(key: string, value: unknown): Promise<void>;
|
|
64
|
+
};
|
|
65
|
+
export declare const s3Layout: PackageLayout;
|
|
66
|
+
export declare const gcsLayout: PackageLayout;
|
|
67
|
+
export declare function createS3RepairStorage(s3: AwsClient, bucket: string): RepairStorage;
|
|
68
|
+
export declare function createGcsRepairStorage(bucket: Bucket): RepairStorage;
|