@learnpack/learnpack 5.0.352 → 5.0.354

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/lib/commands/publish.js +6 -0
  2. package/lib/commands/serve.js +85 -259
  3. package/lib/models/creator.d.ts +6 -0
  4. package/lib/scripts/descriptionsGcsBackfill.d.ts +1 -0
  5. package/lib/scripts/descriptionsGcsBackfill.js +141 -0
  6. package/lib/scripts/descriptionsS3Backfill.d.ts +1 -0
  7. package/lib/scripts/descriptionsS3Backfill.js +277 -0
  8. package/lib/scripts/descriptionsSweep.d.ts +1 -0
  9. package/lib/scripts/descriptionsSweep.js +142 -0
  10. package/lib/utils/api.d.ts +7 -0
  11. package/lib/utils/api.js +8 -1
  12. package/lib/utils/awsCredentials.d.ts +20 -0
  13. package/lib/utils/awsCredentials.js +43 -0
  14. package/lib/utils/creatorUtilities.js +2 -1
  15. package/lib/utils/descriptionHash.d.ts +66 -0
  16. package/lib/utils/descriptionHash.js +173 -0
  17. package/lib/utils/descriptions/backfillEvents.d.ts +60 -0
  18. package/lib/utils/descriptions/backfillEvents.js +107 -0
  19. package/lib/utils/descriptions/gcsStorage.d.ts +16 -0
  20. package/lib/utils/descriptions/gcsStorage.js +60 -0
  21. package/lib/utils/descriptions/generateCourseDescriptions.d.ts +66 -0
  22. package/lib/utils/descriptions/generateCourseDescriptions.js +176 -0
  23. package/lib/utils/descriptions/mirrorDescriptions.d.ts +49 -0
  24. package/lib/utils/descriptions/mirrorDescriptions.js +109 -0
  25. package/lib/utils/descriptions/publishStage.d.ts +69 -0
  26. package/lib/utils/descriptions/publishStage.js +245 -0
  27. package/lib/utils/descriptions/resumePublication.d.ts +36 -0
  28. package/lib/utils/descriptions/resumePublication.js +128 -0
  29. package/lib/utils/descriptions/s3Storage.d.ts +30 -0
  30. package/lib/utils/descriptions/s3Storage.js +141 -0
  31. package/lib/utils/descriptions/workList.d.ts +75 -0
  32. package/lib/utils/descriptions/workList.js +177 -0
  33. package/lib/utils/gcsBucketName.d.ts +10 -0
  34. package/lib/utils/gcsBucketName.js +19 -0
  35. package/lib/utils/packageManifest.d.ts +22 -0
  36. package/lib/utils/packageManifest.js +57 -5
  37. package/lib/utils/publishEvents.d.ts +66 -0
  38. package/lib/utils/publishEvents.js +111 -0
  39. package/lib/utils/publishJournal.d.ts +119 -0
  40. package/lib/utils/publishJournal.js +275 -0
  41. package/lib/utils/repair/legacyPackageRepair.d.ts +131 -0
  42. package/lib/utils/repair/legacyPackageRepair.js +492 -0
  43. package/lib/utils/repair/repairStorage.d.ts +68 -0
  44. package/lib/utils/repair/repairStorage.js +89 -0
  45. package/lib/utils/rigoActions.d.ts +44 -0
  46. package/lib/utils/rigoActions.js +75 -1
  47. package/lib/utils/s3/packageManifestBackfill.d.ts +2 -0
  48. package/lib/utils/s3/packageManifestBackfill.js +5 -8
  49. package/lib/utils/s3/packageSourcesAudit.d.ts +75 -0
  50. package/lib/utils/s3/packageSourcesAudit.js +184 -0
  51. package/lib/utils/syllabusSync.d.ts +71 -0
  52. package/lib/utils/syllabusSync.js +273 -0
  53. package/package.json +3 -1
  54. package/src/commands/publish.ts +7 -0
  55. package/src/commands/serve.ts +144 -335
  56. package/src/models/creator.ts +9 -0
  57. package/src/scripts/README.md +244 -0
  58. package/src/scripts/descriptionsGcsBackfill.ts +193 -0
  59. package/src/scripts/descriptionsS3Backfill.ts +376 -0
  60. package/src/scripts/descriptionsSweep.ts +185 -0
  61. package/src/ui/_app/app.css +1 -1
  62. package/src/ui/_app/app.js +142 -140
  63. package/src/ui/app.tar.gz +0 -0
  64. package/src/utils/api.ts +9 -0
  65. package/src/utils/awsCredentials.ts +57 -0
  66. package/src/utils/creatorUtilities.ts +2 -1
  67. package/src/utils/descriptionHash.ts +196 -0
  68. package/src/utils/descriptions/backfillEvents.ts +152 -0
  69. package/src/utils/descriptions/gcsStorage.ts +67 -0
  70. package/src/utils/descriptions/generateCourseDescriptions.ts +311 -0
  71. package/src/utils/descriptions/mirrorDescriptions.ts +191 -0
  72. package/src/utils/descriptions/publishStage.ts +394 -0
  73. package/src/utils/descriptions/resumePublication.ts +217 -0
  74. package/src/utils/descriptions/s3Storage.ts +214 -0
  75. package/src/utils/descriptions/workList.ts +283 -0
  76. package/src/utils/gcsBucketName.ts +19 -0
  77. package/src/utils/packageManifest.ts +62 -5
  78. package/src/utils/publishEvents.ts +181 -0
  79. package/src/utils/publishJournal.ts +383 -0
  80. package/src/utils/repair/legacyPackageRepair.ts +731 -0
  81. package/src/utils/repair/repairStorage.ts +168 -0
  82. package/src/utils/rigoActions.ts +130 -0
  83. package/src/utils/s3/packageManifestBackfill.ts +771 -776
  84. package/src/utils/s3/packageSourcesAudit.ts +311 -0
  85. package/src/utils/syllabusSync.ts +390 -0
@@ -0,0 +1,492 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.ALL_REPAIR_FILES = void 0;
4
+ exports.buildExercisesFromKeys = buildExercisesFromKeys;
5
+ exports.inferPrimaryLanguage = inferPrimaryLanguage;
6
+ exports.inferLessonType = inferLessonType;
7
+ exports.buildSyllabusFromSources = buildSyllabusFromSources;
8
+ exports.buildSidebarFromReadmes = buildSidebarFromReadmes;
9
+ exports.repairPackage = repairPackage;
10
+ const exerciseFileOrder_1 = require("../exerciseFileOrder");
11
+ const packageManifest_1 = require("../packageManifest");
12
+ exports.ALL_REPAIR_FILES = ["config", "syllabus", "sidebar"];
13
+ const README_FILENAME_PATTERN = /^readme(?:\.([a-z]{2}))?\.md$/i;
14
+ /**
15
+ * A markdown task list is how LearnPack renders a quiz, so two or more options
16
+ * in one README is the only signal a legacy package gives us about its type.
17
+ */
18
+ const QUIZ_OPTION_PATTERN = /^\s*[*-]\s+\[[\sXx]]\s+\S/gm;
19
+ const CODE_FILE_EXTENSIONS = new Set([
20
+ "c",
21
+ "cpp",
22
+ "cs",
23
+ "css",
24
+ "go",
25
+ "html",
26
+ "java",
27
+ "js",
28
+ "json",
29
+ "jsx",
30
+ "php",
31
+ "py",
32
+ "rb",
33
+ "rs",
34
+ "scss",
35
+ "sh",
36
+ "sql",
37
+ "ts",
38
+ "tsx",
39
+ "vue",
40
+ ]);
41
+ function humanizeFolderName(folderSlug) {
42
+ const match = folderSlug.match(/^(\d+(?:\.\d+)?)(?:-(.*))?$/);
43
+ const titlePart = match && match[2] ? match[2] : folderSlug;
44
+ return titlePart.replace(/-/g, " ").trim() || folderSlug;
45
+ }
46
+ function naturalCompare(a, b) {
47
+ const regex = /(\d+|\D+)/g;
48
+ const ax = a.match(regex) || [];
49
+ const bx = b.match(regex) || [];
50
+ for (let i = 0; i < Math.max(ax.length, bx.length); i++) {
51
+ const an = Number.parseInt(ax[i], 10);
52
+ const bn = Number.parseInt(bx[i], 10);
53
+ if (!Number.isNaN(an) && !Number.isNaN(bn)) {
54
+ if (an !== bn) {
55
+ return an - bn;
56
+ }
57
+ }
58
+ else if (ax[i] !== bx[i]) {
59
+ return (ax[i] || "").localeCompare(bx[i] || "");
60
+ }
61
+ }
62
+ return 0;
63
+ }
64
+ /**
65
+ * Exercises from the bucket listing alone, mirroring `configBuilder.buildConfig`
66
+ * key for key: same README language rule, same natural ordering, same
67
+ * `graded: false` default.
68
+ */
69
+ function buildExercisesFromKeys(exercisesPrefix, keys) {
70
+ const map = new Map();
71
+ for (const key of keys) {
72
+ if (!key.startsWith(exercisesPrefix)) {
73
+ continue;
74
+ }
75
+ const parts = key.slice(exercisesPrefix.length).split("/");
76
+ if (parts.length < 2) {
77
+ continue;
78
+ }
79
+ const exerciseSlug = parts[0];
80
+ const filename = parts[parts.length - 1];
81
+ if (!exerciseSlug || !filename) {
82
+ continue;
83
+ }
84
+ let exercise = map.get(exerciseSlug);
85
+ if (!exercise) {
86
+ exercise = {
87
+ title: exerciseSlug,
88
+ slug: exerciseSlug,
89
+ graded: false,
90
+ files: [],
91
+ translations: {},
92
+ };
93
+ map.set(exerciseSlug, exercise);
94
+ }
95
+ const readmeMatch = filename.match(README_FILENAME_PATTERN);
96
+ // Nested files keep their folder-relative name, as the IDE expects.
97
+ if (readmeMatch && parts.length === 2) {
98
+ exercise.translations[readmeMatch[1] || "en"] = filename;
99
+ }
100
+ else {
101
+ exercise.files.push({
102
+ name: parts.slice(1).join("/"),
103
+ slug: parts.slice(1).join("/"),
104
+ hidden: false,
105
+ });
106
+ }
107
+ }
108
+ return [...map.values()]
109
+ .sort((a, b) => naturalCompare(a.slug, b.slug))
110
+ .map((exercise, index) => (Object.assign(Object.assign({}, exercise), { files: (0, exerciseFileOrder_1.sortExerciseFiles)(exercise.files), position: index })));
111
+ }
112
+ /**
113
+ * Language the course is really written in.
114
+ *
115
+ * Same rule as `workList.inferCourseBaseLanguage`: a declared language is
116
+ * trusted only when some exercise actually has a README in it, otherwise the
117
+ * most frequent translation wins. Defaulting to "en" would label a Spanish
118
+ * course English and anchor every later description to a translation.
119
+ *
120
+ * A bilingual course ties on count — most of the legacy ones are — so the tie is
121
+ * broken on the file names rather than arbitrarily: `getReadmeExtension` gives
122
+ * the *original* README no language suffix and suffixes every translation, so
123
+ * the language mapped to a bare `README.md` is the one the course was written
124
+ * in. This matters beyond the title: the descriptions backfill generates from
125
+ * `courseInfo.language` and would otherwise work from a translation.
126
+ */
127
+ function inferPrimaryLanguage(exercises, declared) {
128
+ const counts = new Map();
129
+ const unsuffixed = new Set();
130
+ for (const exercise of exercises) {
131
+ for (const [lang, filename] of Object.entries(exercise.translations || {})) {
132
+ counts.set(lang, (counts.get(lang) || 0) + 1);
133
+ if (/^readme\.md$/i.test(filename)) {
134
+ unsuffixed.add(lang);
135
+ }
136
+ }
137
+ }
138
+ if (counts.size === 0) {
139
+ return declared || "en";
140
+ }
141
+ const normalized = declared === null || declared === void 0 ? void 0 : declared.trim().toLowerCase();
142
+ if (normalized) {
143
+ for (const lang of counts.keys()) {
144
+ if (lang.toLowerCase() === normalized) {
145
+ return lang;
146
+ }
147
+ }
148
+ }
149
+ const ranked = [...counts.entries()].sort((a, b) => {
150
+ if (a[1] !== b[1]) {
151
+ return b[1] - a[1];
152
+ }
153
+ const aBase = unsuffixed.has(a[0]) ? 0 : 1;
154
+ const bBase = unsuffixed.has(b[0]) ? 0 : 1;
155
+ return aBase === bBase ? a[0].localeCompare(b[0]) : aBase - bBase;
156
+ });
157
+ return ranked[0][0];
158
+ }
159
+ /**
160
+ * Lesson type, inferred — the one field of a synthesized syllabus that is a
161
+ * guess rather than a reading. Callers should treat it as such.
162
+ */
163
+ function inferLessonType(input) {
164
+ const quizOptions = input.readmeBody.match(QUIZ_OPTION_PATTERN) || [];
165
+ if (quizOptions.length >= 2) {
166
+ return "QUIZ";
167
+ }
168
+ if (input.graded) {
169
+ return "CODE";
170
+ }
171
+ const hasCodeFile = input.files.some(file => {
172
+ var _a;
173
+ const extension = ((_a = file.name.split(".").pop()) === null || _a === void 0 ? void 0 : _a.toLowerCase()) || "";
174
+ return CODE_FILE_EXTENSIONS.has(extension);
175
+ });
176
+ return hasCodeFile ? "CODE" : "READ";
177
+ }
178
+ function pickLocalized(value, primaryLanguage) {
179
+ if (typeof value === "string") {
180
+ return value;
181
+ }
182
+ if (!value || typeof value !== "object") {
183
+ return undefined;
184
+ }
185
+ const map = value;
186
+ const candidate = map[primaryLanguage] || Object.values(map)[0];
187
+ return typeof candidate === "string" ? candidate : undefined;
188
+ }
189
+ function buildCourseInfo(input) {
190
+ const learnJson = input.learnJson || {};
191
+ const difficulty = learnJson.difficulty;
192
+ const duration = learnJson.duration;
193
+ return {
194
+ title: pickLocalized(learnJson.title, input.primaryLanguage) || input.slug,
195
+ description: pickLocalized(learnJson.description, input.primaryLanguage) || "",
196
+ duration: typeof duration === "number" ? duration : 0,
197
+ difficulty: (typeof difficulty === "string" ?
198
+ difficulty :
199
+ "beginner"),
200
+ language: input.primaryLanguage,
201
+ technologies: Array.isArray(learnJson.technologies) ?
202
+ learnJson.technologies :
203
+ [],
204
+ slug: input.slug,
205
+ // Fields the creator wizard owns. A repaired package was never authored
206
+ // there, so they carry the "nothing pending" value rather than a guess.
207
+ hasContentIndex: false,
208
+ contentIndex: "",
209
+ isCompleted: true,
210
+ variables: [],
211
+ currentStep: "",
212
+ purpose: "",
213
+ };
214
+ }
215
+ /**
216
+ * A syllabus equivalent to what the creator would have written, derived from the
217
+ * exercise folders and their READMEs.
218
+ *
219
+ * `uid` is set to the exercise folder slug on purpose. `findSyllabusLesson`
220
+ * matches on `slugify(id + "-" + title)` *or* `uid`, and only the second is
221
+ * guaranteed to hold once a title comes from an H1 the folder name never had —
222
+ * it is also what lets the S3 and GCS repairs, run independently, still line up
223
+ * for the descriptions mirror.
224
+ */
225
+ function buildSyllabusFromSources(input) {
226
+ const now = Date.now();
227
+ const lessonTypes = { READ: 0, CODE: 0, QUIZ: 0 };
228
+ let titlesFromReadme = 0;
229
+ let titlesFromFolderName = 0;
230
+ const lessons = [...input.exercises]
231
+ .sort((a, b) => a.position - b.position)
232
+ .map(exercise => {
233
+ const readmesByLang = input.readmes[exercise.slug] || {};
234
+ const primaryReadme = readmesByLang[input.primaryLanguage] ||
235
+ Object.values(readmesByLang)[0] || { title: null, body: "" };
236
+ const title = primaryReadme.title || humanizeFolderName(exercise.slug);
237
+ if (primaryReadme.title) {
238
+ titlesFromReadme += 1;
239
+ }
240
+ else {
241
+ titlesFromFolderName += 1;
242
+ }
243
+ const type = inferLessonType({
244
+ graded: exercise.graded,
245
+ files: exercise.files,
246
+ readmeBody: primaryReadme.body,
247
+ });
248
+ lessonTypes[type] += 1;
249
+ const translations = {};
250
+ for (const lang of Object.keys(exercise.translations || {})) {
251
+ // Same shape `syllabusSync` repairs a missing slot with. Deliberately
252
+ // left without a fingerprint or prompt version so that
253
+ // `workList.needsGeneration` returns true and the descriptions backfill
254
+ // picks the course up on its next run.
255
+ translations[lang] = {
256
+ completionId: 0,
257
+ startedAt: now,
258
+ completedAt: now,
259
+ };
260
+ }
261
+ return {
262
+ id: (0, packageManifest_1.deriveLessonId)(exercise.slug),
263
+ uid: exercise.slug,
264
+ title,
265
+ type,
266
+ // `Lesson.description` is the generation topic, not the step
267
+ // description shown to students — that one lives per language in
268
+ // `translations[lang].description`, which the backfill fills in.
269
+ description: title,
270
+ duration: 2,
271
+ generated: true,
272
+ status: "DONE",
273
+ translations,
274
+ };
275
+ });
276
+ return {
277
+ syllabus: {
278
+ lessons,
279
+ courseInfo: buildCourseInfo(input),
280
+ },
281
+ stats: { lessonTypes, titlesFromReadme, titlesFromFolderName },
282
+ };
283
+ }
284
+ /** `sidebar[exerciseSlug][lang] = title`, from the same READMEs. */
285
+ function buildSidebarFromReadmes(exercises, readmes) {
286
+ var _a;
287
+ const sidebar = {};
288
+ for (const exercise of exercises) {
289
+ const readmesByLang = readmes[exercise.slug] || {};
290
+ const entry = {};
291
+ for (const lang of Object.keys(exercise.translations || {})) {
292
+ entry[lang] =
293
+ ((_a = readmesByLang[lang]) === null || _a === void 0 ? void 0 : _a.title) || humanizeFolderName(exercise.slug);
294
+ }
295
+ if (Object.keys(entry).length > 0) {
296
+ sidebar[exercise.slug] = entry;
297
+ }
298
+ }
299
+ return sidebar;
300
+ }
301
+ async function readReadmes(storage, slug, exercises) {
302
+ const index = {};
303
+ const prefix = storage.layout.exercisesPrefix(slug);
304
+ await Promise.all(exercises.flatMap(exercise => Object.entries(exercise.translations || {}).map(async ([lang, filename]) => {
305
+ const content = await storage.readText(`${prefix}${exercise.slug}/${filename}`);
306
+ if (content === null) {
307
+ return;
308
+ }
309
+ const { title } = (0, packageManifest_1.parseReadmeContent)(content);
310
+ if (!index[exercise.slug]) {
311
+ index[exercise.slug] = {};
312
+ }
313
+ index[exercise.slug][lang] = { title: title || null, body: content };
314
+ })));
315
+ return index;
316
+ }
317
+ /** First key that resolves, with the parsed value. */
318
+ async function readFirst(storage, keys) {
319
+ for (const key of keys) {
320
+ // eslint-disable-next-line no-await-in-loop -- later keys are fallbacks
321
+ const value = await storage.readJson(key);
322
+ if (value) {
323
+ return { value, key };
324
+ }
325
+ }
326
+ return null;
327
+ }
328
+ function initialFiles(files) {
329
+ const requested = new Set(files);
330
+ const base = "not-requested";
331
+ return {
332
+ config: { outcome: requested.has("config") ? "present" : base },
333
+ syllabus: { outcome: requested.has("syllabus") ? "present" : base },
334
+ sidebar: { outcome: requested.has("sidebar") ? "present" : base },
335
+ };
336
+ }
337
+ /**
338
+ * Repair one package in one bucket. Returns what it did (or would do, under
339
+ * `dryRun`) without throwing: a batch reports failures per slug.
340
+ */
341
+ async function repairPackage(storage, slug, options) {
342
+ var _a;
343
+ const started = Date.now();
344
+ const { layout } = storage;
345
+ const wants = new Set(options.files);
346
+ const files = initialFiles(options.files);
347
+ const result = (status, extra = {}) => (Object.assign({ slug, target: storage.target, status, durationMs: Date.now() - started, files }, extra));
348
+ try {
349
+ const [learnJson, existingConfig, existingSyllabus, existingSidebar] = await Promise.all([
350
+ storage.readJson(layout.learnJsonKey(slug)),
351
+ readFirst(storage, layout.configKeys(slug)),
352
+ storage.readJson(layout.syllabusKey(slug)),
353
+ readFirst(storage, layout.sidebarKeys(slug)),
354
+ ]);
355
+ const configuredExercises = Array.isArray(existingConfig === null || existingConfig === void 0 ? void 0 : existingConfig.value.exercises) ?
356
+ existingConfig.value.exercises :
357
+ [];
358
+ const needsConfig = wants.has("config") && configuredExercises.length === 0;
359
+ const needsSyllabus = wants.has("syllabus") && !existingSyllabus;
360
+ const needsSidebar = wants.has("sidebar") && !existingSidebar;
361
+ if (wants.has("config") && !needsConfig) {
362
+ files.config = { outcome: "present", key: existingConfig.key };
363
+ }
364
+ if (wants.has("syllabus") && !needsSyllabus) {
365
+ files.syllabus = { outcome: "present", key: layout.syllabusKey(slug) };
366
+ }
367
+ if (wants.has("sidebar") && !needsSidebar) {
368
+ files.sidebar = { outcome: "present", key: existingSidebar.key };
369
+ }
370
+ // Nothing to build: stop before the listing and the READMEs, which is what
371
+ // makes a catalogue-wide pass over healthy packages cheap.
372
+ if (!needsConfig && !needsSyllabus && !needsSidebar) {
373
+ return result("unchanged");
374
+ }
375
+ const useConfigExercises = layout.exercisesFromConfig && configuredExercises.length > 0;
376
+ const derivedExercises = useConfigExercises ?
377
+ [] :
378
+ buildExercisesFromKeys(layout.exercisesPrefix(slug), await storage.listKeys(layout.exercisesPrefix(slug)));
379
+ // Even where the listing is the authority on the exercise list, `graded` is
380
+ // worth keeping: the listing cannot express it (`buildConfig` hardcodes
381
+ // false) and it is a signal for typing a lesson CODE.
382
+ const gradedBySlug = new Map(configuredExercises.map(exercise => [exercise.slug, exercise.graded]));
383
+ const exercises = useConfigExercises ?
384
+ configuredExercises :
385
+ derivedExercises.map(exercise => {
386
+ var _a;
387
+ return (Object.assign(Object.assign({}, exercise), { graded: (_a = gradedBySlug.get(exercise.slug)) !== null && _a !== void 0 ? _a : exercise.graded }));
388
+ });
389
+ if (exercises.length === 0) {
390
+ const anythingAtAll = Boolean(learnJson || existingConfig || existingSyllabus || existingSidebar);
391
+ if (anythingAtAll) {
392
+ return result("failed", {
393
+ error: `No exercises found for "${slug}" in ${storage.target}: neither its config.json nor its exercises/ listing yields any`,
394
+ });
395
+ }
396
+ for (const file of options.files) {
397
+ files[file] = {
398
+ outcome: "impossible",
399
+ reason: `the package is not in ${storage.target}`,
400
+ };
401
+ }
402
+ return result("absent");
403
+ }
404
+ const readmes = await readReadmes(storage, slug, exercises);
405
+ const primaryLanguage = inferPrimaryLanguage(exercises, (learnJson === null || learnJson === void 0 ? void 0 : learnJson.language) ||
406
+ ((_a = existingSyllabus === null || existingSyllabus === void 0 ? void 0 : existingSyllabus.courseInfo) === null || _a === void 0 ? void 0 : _a.language));
407
+ let wrote = false;
408
+ // --- config.json ----------------------------------------------------
409
+ if (needsConfig) {
410
+ // `config` is not decoration: the publish route destructures
411
+ // `{ config, exercises }` and reads `config.title` right after, so a
412
+ // config carrying only exercises would break the next publication.
413
+ const configField = (existingConfig === null || existingConfig === void 0 ? void 0 : existingConfig.value.config) || learnJson;
414
+ if (configField) {
415
+ const key = (existingConfig === null || existingConfig === void 0 ? void 0 : existingConfig.key) || layout.configKeys(slug)[0];
416
+ const merged = Object.assign(Object.assign({}, existingConfig === null || existingConfig === void 0 ? void 0 : existingConfig.value), { config: configField, exercises: derivedExercises });
417
+ files.config = {
418
+ outcome: existingConfig ? "merged" : "created",
419
+ key,
420
+ reason: existingConfig ?
421
+ "existing config.json had no exercises; other fields preserved" :
422
+ "no config.json in the package",
423
+ };
424
+ if (!options.dryRun) {
425
+ await storage.writeJson(key, merged);
426
+ }
427
+ wrote = true;
428
+ }
429
+ else {
430
+ files.config = {
431
+ outcome: "impossible",
432
+ reason: "no learn.json and no config.config to build it from",
433
+ };
434
+ }
435
+ }
436
+ // --- initialSyllabus.json -------------------------------------------
437
+ let syllabusStats;
438
+ if (needsSyllabus) {
439
+ const built = buildSyllabusFromSources({
440
+ slug,
441
+ learnJson,
442
+ exercises,
443
+ readmes,
444
+ primaryLanguage,
445
+ });
446
+ syllabusStats = built.stats;
447
+ files.syllabus = { outcome: "created", key: layout.syllabusKey(slug) };
448
+ if (!options.dryRun) {
449
+ await storage.writeJson(layout.syllabusKey(slug), built.syllabus);
450
+ }
451
+ wrote = true;
452
+ }
453
+ // --- sidebar.json ----------------------------------------------------
454
+ if (needsSidebar) {
455
+ const sidebar = buildSidebarFromReadmes(exercises, readmes);
456
+ if (Object.keys(sidebar).length === 0) {
457
+ files.sidebar = {
458
+ outcome: "impossible",
459
+ reason: "no README titles to build it from",
460
+ };
461
+ }
462
+ else {
463
+ const key = layout.sidebarKeys(slug)[0];
464
+ files.sidebar = { outcome: "created", key };
465
+ if (!options.dryRun) {
466
+ await storage.writeJson(key, sidebar);
467
+ }
468
+ wrote = true;
469
+ }
470
+ }
471
+ const languages = [
472
+ ...new Set(exercises.flatMap(exercise => Object.keys(exercise.translations || {}))),
473
+ ].sort();
474
+ return result(wrote ? "repaired" : "unchanged", {
475
+ stats: {
476
+ exercises: exercises.length,
477
+ languages,
478
+ primaryLanguage,
479
+ lessonTypes: (syllabusStats === null || syllabusStats === void 0 ? void 0 : syllabusStats.lessonTypes) || {
480
+ READ: 0,
481
+ CODE: 0,
482
+ QUIZ: 0,
483
+ },
484
+ titlesFromReadme: (syllabusStats === null || syllabusStats === void 0 ? void 0 : syllabusStats.titlesFromReadme) || 0,
485
+ titlesFromFolderName: (syllabusStats === null || syllabusStats === void 0 ? void 0 : syllabusStats.titlesFromFolderName) || 0,
486
+ },
487
+ });
488
+ }
489
+ catch (error) {
490
+ return result("failed", { error: error.message });
491
+ }
492
+ }
@@ -0,0 +1,68 @@
1
+ import { Bucket } from "@google-cloud/storage";
2
+ import { AwsClient } from "../s3/packageManifestBackfill";
3
+ /**
4
+ * The two buckets a package lives in, behind one interface.
5
+ *
6
+ * A legacy package is repaired **twice, independently** — once against the
7
+ * published snapshot in S3 and once against the draft in GCS — never by copying
8
+ * one into the other. The two hold different content (the draft keeps moving
9
+ * after a publication) and the synthesized structure is a projection of that
10
+ * content: a syllabus derived from the snapshot could name steps the draft does
11
+ * not have. This is the same rule `mirrorDescriptions` enforces with its content
12
+ * hash; deriving from each bucket needs no guard at all.
13
+ *
14
+ * What crosses between them is only the descriptions, through
15
+ * `descriptionsS3Backfill --mirror-draft`, which is hash-guarded per step.
16
+ */
17
+ export type RepairTarget = "s3" | "gcs";
18
+ export declare const ALL_REPAIR_TARGETS: RepairTarget[];
19
+ /**
20
+ * Where each input lives. The two buckets do not agree on this, and the
21
+ * disagreement is load-bearing:
22
+ *
23
+ * - S3 reads `{slug}/config.json` first (`fetchPackageSources`), which is where
24
+ * the publish writes it into the zip.
25
+ * - GCS reads `courses/{slug}/.learn/config.json` — the publish route downloads
26
+ * that one before anything else, so a config written anywhere else in the
27
+ * draft would not make the package publishable.
28
+ */
29
+ export type PackageLayout = {
30
+ learnJsonKey(slug: string): string;
31
+ /** Config locations in resolution order; the first is the write target. */
32
+ configKeys(slug: string): string[];
33
+ syllabusKey(slug: string): string;
34
+ /** Sidebar locations in resolution order; the first is the write target. */
35
+ sidebarKeys(slug: string): string[];
36
+ exercisesPrefix(slug: string): string;
37
+ /**
38
+ * Whether `config.json` is the authority on the exercise list, or the folder
39
+ * listing is.
40
+ *
41
+ * This decides the **language keys** of the synthesized syllabus, and the two
42
+ * buckets genuinely disagree:
43
+ *
44
+ * - S3 reads the published `config.json` (`fetchPackageSources`), where these
45
+ * legacy packages carry `us` for the unsuffixed README.
46
+ * - GCS has no such reader. Both the publish-time manifest and the
47
+ * descriptions pipeline go through `configBuilder.buildConfig`, which
48
+ * derives exercises from the listing and maps an unsuffixed `README.md` to
49
+ * `en` — regardless of what the draft's `.learn/config.json` says.
50
+ *
51
+ * A syllabus keyed the other way is not read at all: the manifest would look
52
+ * up `translations["en"]`, find nothing, and publish `description: null` for
53
+ * every step. So each side is built the way its own reader will read it.
54
+ */
55
+ exercisesFromConfig: boolean;
56
+ };
57
+ export type RepairStorage = {
58
+ target: RepairTarget;
59
+ layout: PackageLayout;
60
+ readJson<T>(key: string): Promise<T | null>;
61
+ readText(key: string): Promise<string | null>;
62
+ listKeys(prefix: string): Promise<string[]>;
63
+ writeJson(key: string, value: unknown): Promise<void>;
64
+ };
65
+ export declare const s3Layout: PackageLayout;
66
+ export declare const gcsLayout: PackageLayout;
67
+ export declare function createS3RepairStorage(s3: AwsClient, bucket: string): RepairStorage;
68
+ export declare function createGcsRepairStorage(bucket: Bucket): RepairStorage;
@@ -0,0 +1,89 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.gcsLayout = exports.s3Layout = exports.ALL_REPAIR_TARGETS = void 0;
4
+ exports.createS3RepairStorage = createS3RepairStorage;
5
+ exports.createGcsRepairStorage = createGcsRepairStorage;
6
+ const client_s3_1 = require("@aws-sdk/client-s3");
7
+ const packageManifestBackfill_1 = require("../s3/packageManifestBackfill");
8
+ exports.ALL_REPAIR_TARGETS = ["s3", "gcs"];
9
+ exports.s3Layout = {
10
+ learnJsonKey: slug => `${slug}/learn.json`,
11
+ configKeys: slug => [`${slug}/config.json`, `${slug}/.learn/config.json`],
12
+ syllabusKey: slug => `${slug}/.learn/initialSyllabus.json`,
13
+ // 25 of the published packages keep their sidebar at the root. It has to be
14
+ // resolved before writing: `fetchPackageSources` prefers the `.learn/` copy,
15
+ // so a synthesized one would shadow the published titles.
16
+ sidebarKeys: slug => [
17
+ `${slug}/.learn/sidebar.json`,
18
+ `${slug}/sidebar.json`,
19
+ ],
20
+ exercisesPrefix: slug => `${slug}/exercises/`,
21
+ exercisesFromConfig: true,
22
+ };
23
+ exports.gcsLayout = {
24
+ learnJsonKey: slug => `courses/${slug}/learn.json`,
25
+ configKeys: slug => [
26
+ `courses/${slug}/.learn/config.json`,
27
+ `courses/${slug}/config.json`,
28
+ ],
29
+ syllabusKey: slug => `courses/${slug}/.learn/initialSyllabus.json`,
30
+ sidebarKeys: slug => [`courses/${slug}/.learn/sidebar.json`],
31
+ exercisesPrefix: slug => `courses/${slug}/exercises/`,
32
+ exercisesFromConfig: false,
33
+ };
34
+ function createS3RepairStorage(s3, bucket) {
35
+ return {
36
+ target: "s3",
37
+ layout: exports.s3Layout,
38
+ readJson: key => (0, packageManifestBackfill_1.fetchJsonObject)(s3, bucket, key),
39
+ readText: key => (0, packageManifestBackfill_1.fetchTextObject)(s3, bucket, key),
40
+ listKeys: prefix => (0, packageManifestBackfill_1.listObjectKeys)(s3, bucket, prefix),
41
+ async writeJson(key, value) {
42
+ await (0, packageManifestBackfill_1.withRetry)(() => s3.send(new client_s3_1.PutObjectCommand({
43
+ Bucket: bucket,
44
+ Key: key,
45
+ Body: JSON.stringify(value, null, 2),
46
+ ContentType: "application/json",
47
+ })));
48
+ },
49
+ };
50
+ }
51
+ function createGcsRepairStorage(bucket) {
52
+ const download = async (key) => {
53
+ try {
54
+ const [buf] = await bucket.file(key).download();
55
+ return buf.toString();
56
+ }
57
+ catch (_a) {
58
+ return null;
59
+ }
60
+ };
61
+ return {
62
+ target: "gcs",
63
+ layout: exports.gcsLayout,
64
+ async readJson(key) {
65
+ const content = await download(key);
66
+ if (content === null) {
67
+ return null;
68
+ }
69
+ try {
70
+ return JSON.parse(content);
71
+ }
72
+ catch (_a) {
73
+ return null;
74
+ }
75
+ },
76
+ readText: download,
77
+ async listKeys(prefix) {
78
+ const [files] = await bucket.getFiles({ prefix });
79
+ return files.map(file => file.name);
80
+ },
81
+ async writeJson(key, value) {
82
+ await bucket
83
+ .file(key)
84
+ .save(Buffer.from(JSON.stringify(value, null, 2), "utf8"), {
85
+ contentType: "application/json",
86
+ });
87
+ },
88
+ };
89
+ }