@learnpack/learnpack 5.0.352 → 5.0.353
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/commands/publish.js +6 -0
- package/lib/commands/serve.js +85 -259
- package/lib/models/creator.d.ts +6 -0
- package/lib/scripts/descriptionsGcsBackfill.d.ts +1 -0
- package/lib/scripts/descriptionsGcsBackfill.js +141 -0
- package/lib/scripts/descriptionsS3Backfill.d.ts +1 -0
- package/lib/scripts/descriptionsS3Backfill.js +157 -0
- package/lib/scripts/descriptionsSweep.d.ts +1 -0
- package/lib/scripts/descriptionsSweep.js +142 -0
- package/lib/utils/api.d.ts +7 -0
- package/lib/utils/api.js +8 -1
- package/lib/utils/creatorUtilities.js +2 -1
- package/lib/utils/descriptionHash.d.ts +66 -0
- package/lib/utils/descriptionHash.js +173 -0
- package/lib/utils/descriptions/gcsStorage.d.ts +16 -0
- package/lib/utils/descriptions/gcsStorage.js +60 -0
- package/lib/utils/descriptions/generateCourseDescriptions.d.ts +59 -0
- package/lib/utils/descriptions/generateCourseDescriptions.js +173 -0
- package/lib/utils/descriptions/mirrorDescriptions.d.ts +49 -0
- package/lib/utils/descriptions/mirrorDescriptions.js +109 -0
- package/lib/utils/descriptions/publishStage.d.ts +69 -0
- package/lib/utils/descriptions/publishStage.js +234 -0
- package/lib/utils/descriptions/resumePublication.d.ts +36 -0
- package/lib/utils/descriptions/resumePublication.js +113 -0
- package/lib/utils/descriptions/s3Storage.d.ts +30 -0
- package/lib/utils/descriptions/s3Storage.js +134 -0
- package/lib/utils/descriptions/workList.d.ts +75 -0
- package/lib/utils/descriptions/workList.js +177 -0
- package/lib/utils/gcsBucketName.d.ts +10 -0
- package/lib/utils/gcsBucketName.js +19 -0
- package/lib/utils/packageManifest.d.ts +14 -0
- package/lib/utils/packageManifest.js +49 -5
- package/lib/utils/publishEvents.d.ts +66 -0
- package/lib/utils/publishEvents.js +111 -0
- package/lib/utils/publishJournal.d.ts +119 -0
- package/lib/utils/publishJournal.js +275 -0
- package/lib/utils/rigoActions.d.ts +44 -0
- package/lib/utils/rigoActions.js +75 -1
- package/lib/utils/s3/packageManifestBackfill.d.ts +2 -0
- package/lib/utils/s3/packageManifestBackfill.js +2 -0
- package/lib/utils/syllabusSync.d.ts +71 -0
- package/lib/utils/syllabusSync.js +273 -0
- package/package.json +1 -1
- package/src/commands/publish.ts +7 -0
- package/src/commands/serve.ts +144 -335
- package/src/models/creator.ts +9 -0
- package/src/scripts/descriptionsGcsBackfill.ts +193 -0
- package/src/scripts/descriptionsS3Backfill.ts +208 -0
- package/src/scripts/descriptionsSweep.ts +185 -0
- package/src/ui/_app/app.css +1 -1
- package/src/ui/_app/app.js +142 -140
- package/src/ui/app.tar.gz +0 -0
- package/src/utils/api.ts +9 -0
- package/src/utils/creatorUtilities.ts +2 -1
- package/src/utils/descriptionHash.ts +196 -0
- package/src/utils/descriptions/gcsStorage.ts +67 -0
- package/src/utils/descriptions/generateCourseDescriptions.ts +301 -0
- package/src/utils/descriptions/mirrorDescriptions.ts +191 -0
- package/src/utils/descriptions/publishStage.ts +382 -0
- package/src/utils/descriptions/resumePublication.ts +200 -0
- package/src/utils/descriptions/s3Storage.ts +206 -0
- package/src/utils/descriptions/workList.ts +283 -0
- package/src/utils/gcsBucketName.ts +19 -0
- package/src/utils/packageManifest.ts +54 -4
- package/src/utils/publishEvents.ts +181 -0
- package/src/utils/publishJournal.ts +383 -0
- package/src/utils/rigoActions.ts +130 -0
- package/src/utils/s3/packageManifestBackfill.ts +2 -2
- package/src/utils/syllabusSync.ts +390 -0
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.findSyllabusLesson = findSyllabusLesson;
|
|
4
|
+
exports.inferCourseBaseLanguage = inferCourseBaseLanguage;
|
|
5
|
+
exports.needsGeneration = needsGeneration;
|
|
6
|
+
exports.buildDescriptionWorkList = buildDescriptionWorkList;
|
|
7
|
+
exports.countWorkItems = countWorkItems;
|
|
8
|
+
exports.isCourseSettled = isCourseSettled;
|
|
9
|
+
const creatorUtilities_1 = require("../creatorUtilities");
|
|
10
|
+
const descriptionHash_1 = require("../descriptionHash");
|
|
11
|
+
const packageManifest_1 = require("../packageManifest");
|
|
12
|
+
function findSyllabusLesson(syllabus, exerciseSlug) {
|
|
13
|
+
return syllabus.lessons.find(lesson => lesson.uid === exerciseSlug ||
|
|
14
|
+
(0, creatorUtilities_1.slugify)(lesson.id + "-" + lesson.title) === exerciseSlug);
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Language a course is actually written in.
|
|
18
|
+
*
|
|
19
|
+
* `courseInfo.language` is trusted only when some exercise really has a README
|
|
20
|
+
* in it; otherwise the language is inferred from the content itself (the most
|
|
21
|
+
* frequent translation across exercises). Defaulting to "en" when the field is
|
|
22
|
+
* missing would label Spanish courses as English and generate their
|
|
23
|
+
* descriptions from a translation instead of the original.
|
|
24
|
+
*/
|
|
25
|
+
function inferCourseBaseLanguage(syllabus, exercises) {
|
|
26
|
+
var _a, _b;
|
|
27
|
+
const counts = new Map();
|
|
28
|
+
for (const exercise of exercises) {
|
|
29
|
+
for (const lang of Object.keys(exercise.translations || {})) {
|
|
30
|
+
counts.set(lang, (counts.get(lang) || 0) + 1);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
if (counts.size === 0) {
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
const declared = (_b = (_a = syllabus.courseInfo) === null || _a === void 0 ? void 0 : _a.language) === null || _b === void 0 ? void 0 : _b.toLowerCase();
|
|
37
|
+
if (declared) {
|
|
38
|
+
for (const lang of counts.keys()) {
|
|
39
|
+
if (lang.toLowerCase() === declared) {
|
|
40
|
+
return lang;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
let best = null;
|
|
45
|
+
let bestCount = 0;
|
|
46
|
+
for (const [lang, count] of counts) {
|
|
47
|
+
if (count > bestCount) {
|
|
48
|
+
best = lang;
|
|
49
|
+
bestCount = count;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return best;
|
|
53
|
+
}
|
|
54
|
+
/** Language to generate this step from: the course's, when it has it. */
|
|
55
|
+
function resolveStepBaseLanguage(exercise, courseBaseLanguage) {
|
|
56
|
+
const langs = Object.keys(exercise.translations || {});
|
|
57
|
+
if (langs.length === 0) {
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
if (courseBaseLanguage && langs.includes(courseBaseLanguage)) {
|
|
61
|
+
return courseBaseLanguage;
|
|
62
|
+
}
|
|
63
|
+
return langs[0];
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Whether a (step, language) needs generating. Human-edited descriptions are
|
|
67
|
+
* never touched; an explicit "not enough content" answer (a stored null with a
|
|
68
|
+
* fingerprint) is respected until the content itself changes.
|
|
69
|
+
*/
|
|
70
|
+
function needsGeneration(slot, fingerprint, options) {
|
|
71
|
+
var _a;
|
|
72
|
+
if ((slot === null || slot === void 0 ? void 0 : slot.descriptionSource) === "human") {
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
if (!slot) {
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
if (options.force) {
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
if (slot.descriptionStatus === "error") {
|
|
82
|
+
return true;
|
|
83
|
+
}
|
|
84
|
+
if (((_a = slot.descriptionPromptVersion) !== null && _a !== void 0 ? _a : 0) < options.promptVersion) {
|
|
85
|
+
return true;
|
|
86
|
+
}
|
|
87
|
+
return (0, descriptionHash_1.isDescriptionStale)({ sha256: slot.sourceContentHash, simhash: slot.sourceSimHash }, fingerprint, options.simhashThreshold);
|
|
88
|
+
}
|
|
89
|
+
/** The (step, language) pairs whose description is missing or out of date. */
|
|
90
|
+
function buildDescriptionWorkList(input) {
|
|
91
|
+
var _a, _b, _c, _d;
|
|
92
|
+
const promptVersion = (_a = input.promptVersion) !== null && _a !== void 0 ? _a : packageManifest_1.DESCRIPTION_PROMPT_VERSION;
|
|
93
|
+
const simhashThreshold = (_b = input.simhashThreshold) !== null && _b !== void 0 ? _b : (0, descriptionHash_1.getSimhashThreshold)();
|
|
94
|
+
const force = (_c = input.force) !== null && _c !== void 0 ? _c : false;
|
|
95
|
+
const courseBaseLanguage = inferCourseBaseLanguage(input.syllabus, input.exercises);
|
|
96
|
+
const work = [];
|
|
97
|
+
for (const exercise of input.exercises) {
|
|
98
|
+
const lesson = findSyllabusLesson(input.syllabus, exercise.slug);
|
|
99
|
+
if (!lesson) {
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
const baseLanguage = resolveStepBaseLanguage(exercise, courseBaseLanguage);
|
|
103
|
+
if (!baseLanguage) {
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
const exerciseReadmes = input.readmes[exercise.slug] || {};
|
|
107
|
+
const baseContent = exerciseReadmes[baseLanguage];
|
|
108
|
+
if (typeof baseContent !== "string") {
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
const languages = [];
|
|
112
|
+
for (const lang of Object.keys(exercise.translations || {})) {
|
|
113
|
+
const content = exerciseReadmes[lang];
|
|
114
|
+
if (typeof content !== "string") {
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
// Each language's staleness is anchored to its OWN content, even though
|
|
118
|
+
// the description is generated from the base language: that is what makes
|
|
119
|
+
// "the Spanish README changed" regenerate only Spanish.
|
|
120
|
+
const fingerprint = (0, descriptionHash_1.fingerprintReadme)(content);
|
|
121
|
+
if (!needsGeneration((_d = lesson.translations) === null || _d === void 0 ? void 0 : _d[lang], fingerprint, {
|
|
122
|
+
promptVersion,
|
|
123
|
+
simhashThreshold,
|
|
124
|
+
force,
|
|
125
|
+
})) {
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
languages.push({ lang, fingerprint });
|
|
129
|
+
}
|
|
130
|
+
if (languages.length === 0) {
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
work.push({
|
|
134
|
+
exerciseSlug: exercise.slug,
|
|
135
|
+
lesson,
|
|
136
|
+
baseLanguage,
|
|
137
|
+
baseContent,
|
|
138
|
+
languages,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
return work;
|
|
142
|
+
}
|
|
143
|
+
/** Total (step, language) pairs in a work list. */
|
|
144
|
+
function countWorkItems(work) {
|
|
145
|
+
return work.reduce((total, step) => total + step.languages.length, 0);
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Cheap "has this course ever been fully processed at this prompt version?"
|
|
149
|
+
* check, from the syllabus alone.
|
|
150
|
+
*
|
|
151
|
+
* Deliberately weaker than the work list: it cannot see content changes because
|
|
152
|
+
* it never reads a README. That is the point — it lets a batched run skip
|
|
153
|
+
* finished courses without paying thousands of object reads. Use the work list
|
|
154
|
+
* whenever correctness matters; use this only as a pre-filter.
|
|
155
|
+
*/
|
|
156
|
+
function isCourseSettled(syllabus, exercises, promptVersion = packageManifest_1.DESCRIPTION_PROMPT_VERSION) {
|
|
157
|
+
var _a, _b;
|
|
158
|
+
for (const exercise of exercises) {
|
|
159
|
+
const lesson = findSyllabusLesson(syllabus, exercise.slug);
|
|
160
|
+
if (!lesson) {
|
|
161
|
+
return false;
|
|
162
|
+
}
|
|
163
|
+
for (const lang of Object.keys(exercise.translations || {})) {
|
|
164
|
+
const slot = (_a = lesson.translations) === null || _a === void 0 ? void 0 : _a[lang];
|
|
165
|
+
if ((slot === null || slot === void 0 ? void 0 : slot.descriptionSource) === "human") {
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
if (!slot ||
|
|
169
|
+
slot.descriptionStatus === "error" ||
|
|
170
|
+
!slot.sourceContentHash ||
|
|
171
|
+
((_b = slot.descriptionPromptVersion) !== null && _b !== void 0 ? _b : 0) < promptVersion) {
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return true;
|
|
177
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The draft bucket name, required rather than defaulted.
|
|
3
|
+
*
|
|
4
|
+
* The code used to fall back to "learnpack-packages", a name the service
|
|
5
|
+
* account cannot reach: with the variable missing, the process would start
|
|
6
|
+
* cleanly and then fail with a 403 on every single object operation. Failing at
|
|
7
|
+
* boot with the reason turns a puzzling runtime outage into an obvious
|
|
8
|
+
* misconfiguration.
|
|
9
|
+
*/
|
|
10
|
+
export declare function requireGcsBucketName(): string;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.requireGcsBucketName = requireGcsBucketName;
|
|
4
|
+
/**
|
|
5
|
+
* The draft bucket name, required rather than defaulted.
|
|
6
|
+
*
|
|
7
|
+
* The code used to fall back to "learnpack-packages", a name the service
|
|
8
|
+
* account cannot reach: with the variable missing, the process would start
|
|
9
|
+
* cleanly and then fail with a 403 on every single object operation. Failing at
|
|
10
|
+
* boot with the reason turns a puzzling runtime outage into an obvious
|
|
11
|
+
* misconfiguration.
|
|
12
|
+
*/
|
|
13
|
+
function requireGcsBucketName() {
|
|
14
|
+
const name = (process.env.GCP_BUCKET_NAME || "").trim();
|
|
15
|
+
if (!name) {
|
|
16
|
+
throw new Error("GCP_BUCKET_NAME (env) is required: it names the bucket holding course drafts");
|
|
17
|
+
}
|
|
18
|
+
return name;
|
|
19
|
+
}
|
|
@@ -4,6 +4,8 @@ import { Exercise } from "./configBuilder";
|
|
|
4
4
|
export declare const PACKAGE_MANIFEST_FILENAME = "package-manifest.json";
|
|
5
5
|
export declare const PACKAGE_MANIFEST_REL_PATH = ".learn/package-manifest.json";
|
|
6
6
|
export declare const SCHEMA_VERSION = 1;
|
|
7
|
+
export declare const DESCRIPTION_PROMPT_VERSION = 1;
|
|
8
|
+
export declare const DESCRIPTION_TARGET_WORD_COUNT = 25;
|
|
7
9
|
export type LessonType = "READ" | "CODE" | "QUIZ";
|
|
8
10
|
export type PackageManifestLesson = {
|
|
9
11
|
id: string;
|
|
@@ -52,6 +54,18 @@ export declare function buildPackageManifestFromSources(input: PackageManifestBu
|
|
|
52
54
|
export declare function parseReadmeContent(content: string): ReadmeFrontmatter;
|
|
53
55
|
export declare function comparePackageManifests(existing: PackageManifest | null, next: PackageManifest): "created" | "updated" | "skipped";
|
|
54
56
|
export declare function serializePackageManifest(manifest: PackageManifest): string;
|
|
57
|
+
/** The manifest as it travels inside a publication event. */
|
|
58
|
+
export type PackageManifestEventView = Omit<PackageManifest, "generatedAt">;
|
|
59
|
+
/**
|
|
60
|
+
* Project a manifest into the shape sent to breathecode.
|
|
61
|
+
*
|
|
62
|
+
* Today this is nearly the identity — only `generatedAt`, an internal detail of
|
|
63
|
+
* the projection, is dropped. It exists as an explicit pick so that fields
|
|
64
|
+
* added to the manifest later (hashes, generation flags) do not silently become
|
|
65
|
+
* part of an external contract: joining the event payload has to be a decision
|
|
66
|
+
* visible in a diff.
|
|
67
|
+
*/
|
|
68
|
+
export declare function serializePackageManifestForEvent(manifest: PackageManifest): PackageManifestEventView;
|
|
55
69
|
export declare function generateAndPersistPackageManifest(bucket: Bucket, courseSlug: string, options: {
|
|
56
70
|
publishedAt: string | null;
|
|
57
71
|
localDir?: string;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.SCHEMA_VERSION = exports.PACKAGE_MANIFEST_REL_PATH = exports.PACKAGE_MANIFEST_FILENAME = void 0;
|
|
3
|
+
exports.DESCRIPTION_TARGET_WORD_COUNT = exports.DESCRIPTION_PROMPT_VERSION = exports.SCHEMA_VERSION = exports.PACKAGE_MANIFEST_REL_PATH = exports.PACKAGE_MANIFEST_FILENAME = void 0;
|
|
4
4
|
exports.packageManifestGcsPath = packageManifestGcsPath;
|
|
5
5
|
exports.resolvePreviewUrl = resolvePreviewUrl;
|
|
6
6
|
exports.extractReadmeH1 = extractReadmeH1;
|
|
@@ -9,6 +9,7 @@ exports.buildPackageManifestFromSources = buildPackageManifestFromSources;
|
|
|
9
9
|
exports.parseReadmeContent = parseReadmeContent;
|
|
10
10
|
exports.comparePackageManifests = comparePackageManifests;
|
|
11
11
|
exports.serializePackageManifest = serializePackageManifest;
|
|
12
|
+
exports.serializePackageManifestForEvent = serializePackageManifestForEvent;
|
|
12
13
|
exports.generateAndPersistPackageManifest = generateAndPersistPackageManifest;
|
|
13
14
|
const tslib_1 = require("tslib");
|
|
14
15
|
const fs = require("fs");
|
|
@@ -22,6 +23,13 @@ const frontMatter = require("front-matter");
|
|
|
22
23
|
exports.PACKAGE_MANIFEST_FILENAME = "package-manifest.json";
|
|
23
24
|
exports.PACKAGE_MANIFEST_REL_PATH = ".learn/package-manifest.json";
|
|
24
25
|
exports.SCHEMA_VERSION = 1;
|
|
26
|
+
// Version of the Rigobot prompt/model used to generate step descriptions. It
|
|
27
|
+
// travels inside the completion inputs, so bumping it both busts Rigobot's
|
|
28
|
+
// cache and marks every description stored at a lower version for regeneration.
|
|
29
|
+
// Keep in sync with the prompt body provisioned in Rigobot.
|
|
30
|
+
exports.DESCRIPTION_PROMPT_VERSION = 1;
|
|
31
|
+
// Target length, in words, of a generated step description.
|
|
32
|
+
exports.DESCRIPTION_TARGET_WORD_COUNT = 25;
|
|
25
33
|
const VALID_LESSON_TYPES = new Set(["READ", "CODE", "QUIZ"]);
|
|
26
34
|
function packageManifestGcsPath(courseSlug) {
|
|
27
35
|
return `courses/${courseSlug}/${exports.PACKAGE_MANIFEST_REL_PATH}`;
|
|
@@ -94,11 +102,16 @@ function resolveLessonTitle(exercise, sidebar, syllabusLesson, courseLang, readm
|
|
|
94
102
|
}
|
|
95
103
|
return titles;
|
|
96
104
|
}
|
|
97
|
-
function resolveLessonDescriptions(exercise) {
|
|
105
|
+
function resolveLessonDescriptions(exercise, syllabusLesson) {
|
|
106
|
+
var _a, _b, _c;
|
|
98
107
|
const description = {};
|
|
108
|
+
// Descriptions are a projection: the source of truth lives in the syllabus
|
|
109
|
+
// lesson (translations[lang].description), populated by the sweep/backfill.
|
|
110
|
+
// When absent we emit null (the frontend hides the area) rather than a
|
|
111
|
+
// low-quality placeholder.
|
|
99
112
|
for (const lang of Object.keys(exercise.translations)) {
|
|
100
|
-
|
|
101
|
-
|
|
113
|
+
description[lang] =
|
|
114
|
+
(_c = (_b = (_a = syllabusLesson === null || syllabusLesson === void 0 ? void 0 : syllabusLesson.translations) === null || _a === void 0 ? void 0 : _a[lang]) === null || _b === void 0 ? void 0 : _b.description) !== null && _c !== void 0 ? _c : null;
|
|
102
115
|
}
|
|
103
116
|
return description;
|
|
104
117
|
}
|
|
@@ -134,7 +147,7 @@ function buildPackageManifestFromSources(input) {
|
|
|
134
147
|
position: exercise.position,
|
|
135
148
|
type,
|
|
136
149
|
title: resolveLessonTitle(exercise, sidebar, syllabusLesson, courseLang, (_b = readmeFrontmatters[exercise.slug]) !== null && _b !== void 0 ? _b : {}),
|
|
137
|
-
description: resolveLessonDescriptions(exercise),
|
|
150
|
+
description: resolveLessonDescriptions(exercise, syllabusLesson),
|
|
138
151
|
video: resolveLessonVideos(exercise, readmeFrontmatters),
|
|
139
152
|
};
|
|
140
153
|
});
|
|
@@ -184,6 +197,37 @@ function comparePackageManifests(existing, next) {
|
|
|
184
197
|
function serializePackageManifest(manifest) {
|
|
185
198
|
return JSON.stringify(manifest, null, 2);
|
|
186
199
|
}
|
|
200
|
+
/**
|
|
201
|
+
* Project a manifest into the shape sent to breathecode.
|
|
202
|
+
*
|
|
203
|
+
* Today this is nearly the identity — only `generatedAt`, an internal detail of
|
|
204
|
+
* the projection, is dropped. It exists as an explicit pick so that fields
|
|
205
|
+
* added to the manifest later (hashes, generation flags) do not silently become
|
|
206
|
+
* part of an external contract: joining the event payload has to be a decision
|
|
207
|
+
* visible in a diff.
|
|
208
|
+
*/
|
|
209
|
+
function serializePackageManifestForEvent(manifest) {
|
|
210
|
+
return {
|
|
211
|
+
schemaVersion: manifest.schemaVersion,
|
|
212
|
+
publishedAt: manifest.publishedAt,
|
|
213
|
+
slug: manifest.slug,
|
|
214
|
+
title: manifest.title,
|
|
215
|
+
description: manifest.description,
|
|
216
|
+
preview: manifest.preview,
|
|
217
|
+
technologies: manifest.technologies,
|
|
218
|
+
difficulty: manifest.difficulty,
|
|
219
|
+
duration: manifest.duration,
|
|
220
|
+
lessons: manifest.lessons.map(lesson => ({
|
|
221
|
+
id: lesson.id,
|
|
222
|
+
slug: lesson.slug,
|
|
223
|
+
position: lesson.position,
|
|
224
|
+
type: lesson.type,
|
|
225
|
+
title: lesson.title,
|
|
226
|
+
description: lesson.description,
|
|
227
|
+
video: lesson.video,
|
|
228
|
+
})),
|
|
229
|
+
};
|
|
230
|
+
}
|
|
187
231
|
async function readReadmeFrontmatter(bucket, exerciseSlug, lang, readmePath) {
|
|
188
232
|
try {
|
|
189
233
|
const [buf] = await bucket.file(readmePath).download();
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { PackageManifest, PackageManifestEventView } from "./packageManifest";
|
|
2
|
+
/**
|
|
3
|
+
* Publication events sent to breathecode.
|
|
4
|
+
*
|
|
5
|
+
* They are emitted from here, and not from Rigobot, because learnpack-cli is
|
|
6
|
+
* the only party that witnesses the facts: Rigobot's part ends when the zip is
|
|
7
|
+
* deployed, before the breathecode asset sync, and it never sees the end of the
|
|
8
|
+
* asynchronous description stage.
|
|
9
|
+
*
|
|
10
|
+
* Transport is the existing telemetry endpoint. The discriminator is a
|
|
11
|
+
* top-level `event` key, which is what breathecode's webhook log already looks
|
|
12
|
+
* for; a body without it keeps being processed as plain telemetry.
|
|
13
|
+
*
|
|
14
|
+
* `package_slug` and `package_id` are duplicated at the top level on purpose:
|
|
15
|
+
* `LearnPack.add_webhook_to_log` fills the indexed columns from there, so
|
|
16
|
+
* without them the events would land as rows nobody can filter in the webhook
|
|
17
|
+
* viewer.
|
|
18
|
+
*/
|
|
19
|
+
export declare const PACKAGE_PUBLISHED_EVENT = "package_published";
|
|
20
|
+
export declare const PACKAGE_MANIFEST_UPDATED_EVENT = "package_manifest_updated";
|
|
21
|
+
/** Whether a second event is coming for this publication. */
|
|
22
|
+
export type DescriptionsOutcome = "queued" | "up_to_date";
|
|
23
|
+
export type PublishEventEnvelope = {
|
|
24
|
+
event: string;
|
|
25
|
+
package_slug: string;
|
|
26
|
+
package_id?: number;
|
|
27
|
+
payload: Record<string, unknown>;
|
|
28
|
+
};
|
|
29
|
+
export type PackagePublishedPayload = {
|
|
30
|
+
publish_id: string;
|
|
31
|
+
package: Record<string, unknown> | null;
|
|
32
|
+
manifest: PackageManifestEventView | null;
|
|
33
|
+
descriptions: DescriptionsOutcome;
|
|
34
|
+
};
|
|
35
|
+
export type PackageManifestUpdatedPayload = {
|
|
36
|
+
publish_id: string;
|
|
37
|
+
status: "success" | "failed";
|
|
38
|
+
package: Record<string, unknown> | null;
|
|
39
|
+
manifest: PackageManifestEventView | null;
|
|
40
|
+
stats?: {
|
|
41
|
+
generated: number;
|
|
42
|
+
failed: number;
|
|
43
|
+
missing: number;
|
|
44
|
+
};
|
|
45
|
+
};
|
|
46
|
+
export type PublishEventContext = {
|
|
47
|
+
publishId: string;
|
|
48
|
+
courseSlug: string;
|
|
49
|
+
/** `GET /v1/learnpack/package/{slug}/` from Rigobot. */
|
|
50
|
+
packageInfo: Record<string, unknown> | null;
|
|
51
|
+
manifest: PackageManifest | null;
|
|
52
|
+
};
|
|
53
|
+
export declare function buildPackagePublishedEvent(context: PublishEventContext, descriptions: DescriptionsOutcome): PublishEventEnvelope;
|
|
54
|
+
export declare function buildPackageManifestUpdatedEvent(context: PublishEventContext, status: "success" | "failed", stats?: PackageManifestUpdatedPayload["stats"]): PublishEventEnvelope;
|
|
55
|
+
/**
|
|
56
|
+
* Fire and forget: a publication must never fail because breathecode is down,
|
|
57
|
+
* so every error is swallowed and logged. Returns whether it was delivered, for
|
|
58
|
+
* the journal, not for control flow.
|
|
59
|
+
*/
|
|
60
|
+
export declare function sendPublishEvent(envelope: PublishEventEnvelope, breathecodeToken: string, timeoutMs?: number): Promise<boolean>;
|
|
61
|
+
/**
|
|
62
|
+
* Package information for the event payload, straight from Rigobot's own
|
|
63
|
+
* serializer so the shape stays owned by the system that owns the model.
|
|
64
|
+
* Returns null on failure: an event with less context still beats no event.
|
|
65
|
+
*/
|
|
66
|
+
export declare function fetchPackageInfo(courseSlug: string, rigobotToken: string): Promise<Record<string, unknown> | null>;
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PACKAGE_MANIFEST_UPDATED_EVENT = exports.PACKAGE_PUBLISHED_EVENT = void 0;
|
|
4
|
+
exports.buildPackagePublishedEvent = buildPackagePublishedEvent;
|
|
5
|
+
exports.buildPackageManifestUpdatedEvent = buildPackageManifestUpdatedEvent;
|
|
6
|
+
exports.sendPublishEvent = sendPublishEvent;
|
|
7
|
+
exports.fetchPackageInfo = fetchPackageInfo;
|
|
8
|
+
const axios_1 = require("axios");
|
|
9
|
+
const api_1 = require("./api");
|
|
10
|
+
const packageManifest_1 = require("./packageManifest");
|
|
11
|
+
/**
|
|
12
|
+
* Publication events sent to breathecode.
|
|
13
|
+
*
|
|
14
|
+
* They are emitted from here, and not from Rigobot, because learnpack-cli is
|
|
15
|
+
* the only party that witnesses the facts: Rigobot's part ends when the zip is
|
|
16
|
+
* deployed, before the breathecode asset sync, and it never sees the end of the
|
|
17
|
+
* asynchronous description stage.
|
|
18
|
+
*
|
|
19
|
+
* Transport is the existing telemetry endpoint. The discriminator is a
|
|
20
|
+
* top-level `event` key, which is what breathecode's webhook log already looks
|
|
21
|
+
* for; a body without it keeps being processed as plain telemetry.
|
|
22
|
+
*
|
|
23
|
+
* `package_slug` and `package_id` are duplicated at the top level on purpose:
|
|
24
|
+
* `LearnPack.add_webhook_to_log` fills the indexed columns from there, so
|
|
25
|
+
* without them the events would land as rows nobody can filter in the webhook
|
|
26
|
+
* viewer.
|
|
27
|
+
*/
|
|
28
|
+
exports.PACKAGE_PUBLISHED_EVENT = "package_published";
|
|
29
|
+
exports.PACKAGE_MANIFEST_UPDATED_EVENT = "package_manifest_updated";
|
|
30
|
+
function packageIdOf(packageInfo) {
|
|
31
|
+
const id = packageInfo === null || packageInfo === void 0 ? void 0 : packageInfo.id;
|
|
32
|
+
return typeof id === "number" ? id : undefined;
|
|
33
|
+
}
|
|
34
|
+
function buildPackagePublishedEvent(context, descriptions) {
|
|
35
|
+
const payload = {
|
|
36
|
+
publish_id: context.publishId,
|
|
37
|
+
package: context.packageInfo,
|
|
38
|
+
manifest: context.manifest ?
|
|
39
|
+
(0, packageManifest_1.serializePackageManifestForEvent)(context.manifest) :
|
|
40
|
+
null,
|
|
41
|
+
descriptions,
|
|
42
|
+
};
|
|
43
|
+
return {
|
|
44
|
+
event: exports.PACKAGE_PUBLISHED_EVENT,
|
|
45
|
+
package_slug: context.courseSlug,
|
|
46
|
+
package_id: packageIdOf(context.packageInfo),
|
|
47
|
+
payload: payload,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
function buildPackageManifestUpdatedEvent(context, status, stats) {
|
|
51
|
+
const payload = {
|
|
52
|
+
publish_id: context.publishId,
|
|
53
|
+
status,
|
|
54
|
+
package: context.packageInfo,
|
|
55
|
+
manifest: context.manifest ?
|
|
56
|
+
(0, packageManifest_1.serializePackageManifestForEvent)(context.manifest) :
|
|
57
|
+
null,
|
|
58
|
+
};
|
|
59
|
+
if (stats) {
|
|
60
|
+
payload.stats = stats;
|
|
61
|
+
}
|
|
62
|
+
return {
|
|
63
|
+
event: exports.PACKAGE_MANIFEST_UPDATED_EVENT,
|
|
64
|
+
package_slug: context.courseSlug,
|
|
65
|
+
package_id: packageIdOf(context.packageInfo),
|
|
66
|
+
payload: payload,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Fire and forget: a publication must never fail because breathecode is down,
|
|
71
|
+
* so every error is swallowed and logged. Returns whether it was delivered, for
|
|
72
|
+
* the journal, not for control flow.
|
|
73
|
+
*/
|
|
74
|
+
async function sendPublishEvent(envelope, breathecodeToken, timeoutMs = 10000) {
|
|
75
|
+
if (!breathecodeToken) {
|
|
76
|
+
console.warn(`[publish-events] Skipping "${envelope.event}": no breathecode token`);
|
|
77
|
+
return false;
|
|
78
|
+
}
|
|
79
|
+
try {
|
|
80
|
+
await axios_1.default.post(api_1.BREATHECODE_EVENTS_URL, envelope, {
|
|
81
|
+
timeout: timeoutMs,
|
|
82
|
+
headers: {
|
|
83
|
+
"Content-Type": "application/json",
|
|
84
|
+
Authorization: "Token " + breathecodeToken.trim(),
|
|
85
|
+
},
|
|
86
|
+
});
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
console.error(`[publish-events] Could not deliver "${envelope.event}" for "${envelope.package_slug}":`, error.message);
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Package information for the event payload, straight from Rigobot's own
|
|
96
|
+
* serializer so the shape stays owned by the system that owns the model.
|
|
97
|
+
* Returns null on failure: an event with less context still beats no event.
|
|
98
|
+
*/
|
|
99
|
+
async function fetchPackageInfo(courseSlug, rigobotToken) {
|
|
100
|
+
try {
|
|
101
|
+
const response = await axios_1.default.get(`${api_1.RIGOBOT_HOST}/v1/learnpack/package/${courseSlug}/`, {
|
|
102
|
+
timeout: 10000,
|
|
103
|
+
headers: { Authorization: "Token " + rigobotToken.trim() },
|
|
104
|
+
});
|
|
105
|
+
return response.data;
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
console.error(`[publish-events] Could not read package "${courseSlug}" from Rigobot:`, error.message);
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { Bucket } from "@google-cloud/storage";
|
|
2
|
+
/**
|
|
3
|
+
* Publish journal — per-stage state of a publication.
|
|
4
|
+
*
|
|
5
|
+
* Publishing is a distributed transaction across three systems (S3 via Rigobot,
|
|
6
|
+
* breathecode assets, and our own buckets) plus an asynchronous description
|
|
7
|
+
* stage. Some of its outcomes are NOT derivable from content afterwards: in
|
|
8
|
+
* particular "was the manifest-updated event already emitted?" leaves no trace
|
|
9
|
+
* in S3 or GCS. The journal records what happened per stage so the sweep can
|
|
10
|
+
* resume an interrupted publication and honour the event contract (a publish
|
|
11
|
+
* that announced `descriptions: "queued"` must always be followed by exactly
|
|
12
|
+
* one `package_manifest_updated`).
|
|
13
|
+
*
|
|
14
|
+
* Design notes:
|
|
15
|
+
*
|
|
16
|
+
* - **Best effort, never a lock.** Every write swallows its errors: if the
|
|
17
|
+
* journal cannot be written the publication still proceeds. Losing the
|
|
18
|
+
* journal degrades us to the previous behaviour (no recovery, no diagnosis),
|
|
19
|
+
* which is acceptable; blocking a publication because a bookkeeping write
|
|
20
|
+
* failed is not.
|
|
21
|
+
* - **Flat key space.** Journals live in `publish-journal/{slug}__{id}.json`,
|
|
22
|
+
* a sibling prefix of `courses/`, so the sweep can list every unfinished
|
|
23
|
+
* publication with one prefix query and the concurrency guard can list a
|
|
24
|
+
* single course with another, without maintaining an index.
|
|
25
|
+
* - **Explicit cleanup.** A journal is deleted once every stage succeeded, and
|
|
26
|
+
* kept when any stage failed (visibility + retry). GCS lifecycle rules would
|
|
27
|
+
* be the natural alternative, but the service account only holds
|
|
28
|
+
* object-level permissions (`storage.buckets.*` is denied), so the cleanup
|
|
29
|
+
* has to happen in code.
|
|
30
|
+
*/
|
|
31
|
+
export declare const JOURNAL_PREFIX = "publish-journal/";
|
|
32
|
+
/**
|
|
33
|
+
* Stages of a publication, in execution order.
|
|
34
|
+
*
|
|
35
|
+
* Only stages the orchestrator can observe on its own: writing the asset id
|
|
36
|
+
* back into Rigobot happens inside the asset sync call, so it is covered by
|
|
37
|
+
* `assetsSync` rather than being a stage that could never be marked.
|
|
38
|
+
*/
|
|
39
|
+
export declare const STAGE_NAMES: readonly ["deploy", "assetsSync", "publishedEvent", "descriptions", "gcsMirror", "manifestEvent"];
|
|
40
|
+
export type StageName = typeof STAGE_NAMES[number];
|
|
41
|
+
export type StageStatus = "pending" | "running" | "done" | "failed";
|
|
42
|
+
export type Stage = {
|
|
43
|
+
status: StageStatus;
|
|
44
|
+
at?: string;
|
|
45
|
+
error?: string;
|
|
46
|
+
meta?: unknown;
|
|
47
|
+
};
|
|
48
|
+
export type PublishJournal = {
|
|
49
|
+
publishId: string;
|
|
50
|
+
courseSlug: string;
|
|
51
|
+
startedAt: string;
|
|
52
|
+
updatedAt: string;
|
|
53
|
+
/** Times the sweep has picked this journal up, to bound retries. */
|
|
54
|
+
attempts: number;
|
|
55
|
+
/** Set once retries are exhausted: kept for a human, ignored by the sweep. */
|
|
56
|
+
abandoned?: boolean;
|
|
57
|
+
stages: Record<StageName, Stage>;
|
|
58
|
+
};
|
|
59
|
+
export type JournalRef = {
|
|
60
|
+
courseSlug: string;
|
|
61
|
+
publishId: string;
|
|
62
|
+
key: string;
|
|
63
|
+
};
|
|
64
|
+
/**
|
|
65
|
+
* Minimal storage surface the journal needs, so callers can pass the real
|
|
66
|
+
* bucket adapter or an in-memory double. Object-level operations only.
|
|
67
|
+
*/
|
|
68
|
+
export interface JournalStorage {
|
|
69
|
+
save(key: string, contents: string): Promise<void>;
|
|
70
|
+
read(key: string): Promise<string | null>;
|
|
71
|
+
remove(key: string): Promise<void>;
|
|
72
|
+
list(prefix: string): Promise<string[]>;
|
|
73
|
+
}
|
|
74
|
+
export declare function journalKey(courseSlug: string, publishId: string): string;
|
|
75
|
+
export declare function coursePrefix(courseSlug: string): string;
|
|
76
|
+
/** Inverse of `journalKey`; returns null for keys that don't match the shape. */
|
|
77
|
+
export declare function parseJournalKey(key: string): JournalRef | null;
|
|
78
|
+
/**
|
|
79
|
+
* Start a journal for a new publication. The generated `publishId` correlates
|
|
80
|
+
* the two publish events and is returned even if persisting failed.
|
|
81
|
+
*/
|
|
82
|
+
export declare function createJournal(storage: JournalStorage, courseSlug: string, publishId?: string): Promise<PublishJournal>;
|
|
83
|
+
/** Update one stage and persist. Mutates and returns the same journal object. */
|
|
84
|
+
export declare function markStage(storage: JournalStorage, journal: PublishJournal, stage: StageName, status: StageStatus, extra?: {
|
|
85
|
+
error?: string;
|
|
86
|
+
meta?: unknown;
|
|
87
|
+
}): Promise<PublishJournal>;
|
|
88
|
+
/** Read a journal. Returns null when missing or unreadable. */
|
|
89
|
+
export declare function readJournal(storage: JournalStorage, courseSlug: string, publishId: string): Promise<PublishJournal | null>;
|
|
90
|
+
export declare function readJournalByKey(storage: JournalStorage, key: string): Promise<PublishJournal | null>;
|
|
91
|
+
/**
|
|
92
|
+
* Every journal still stored, i.e. every publication that did not finish
|
|
93
|
+
* cleanly. Returns an empty list on error (the sweep falls back to its deep
|
|
94
|
+
* scan rather than crashing).
|
|
95
|
+
*/
|
|
96
|
+
export declare function listJournalRefs(storage: JournalStorage): Promise<JournalRef[]>;
|
|
97
|
+
/** Journals of a single course; backs the concurrent-publish guard. */
|
|
98
|
+
export declare function listJournalRefsForCourse(storage: JournalStorage, courseSlug: string): Promise<JournalRef[]>;
|
|
99
|
+
export declare function isStageTerminal(stage: Stage): boolean;
|
|
100
|
+
export declare function hasFailedStage(journal: PublishJournal): boolean;
|
|
101
|
+
/** True when no stage is left to run (all done, or done/failed mix). */
|
|
102
|
+
export declare function isJournalSettled(journal: PublishJournal): boolean;
|
|
103
|
+
/** Stages the sweep still has work to do on. */
|
|
104
|
+
export declare function pendingStages(journal: PublishJournal): StageName[];
|
|
105
|
+
/** A journal untouched for longer than `maxAgeMs` is presumed interrupted. */
|
|
106
|
+
export declare function isJournalStale(journal: PublishJournal, maxAgeMs: number, now?: number): boolean;
|
|
107
|
+
export declare function incrementAttempts(storage: JournalStorage, journal: PublishJournal): Promise<PublishJournal>;
|
|
108
|
+
/**
|
|
109
|
+
* Give up on a journal after too many attempts. It is kept, not deleted: a
|
|
110
|
+
* publication that never completed is exactly what someone needs to see.
|
|
111
|
+
*/
|
|
112
|
+
export declare function abandonJournal(storage: JournalStorage, journal: PublishJournal): Promise<PublishJournal>;
|
|
113
|
+
/**
|
|
114
|
+
* Close a journal: delete it when every stage succeeded, keep it otherwise so
|
|
115
|
+
* the sweep can retry and a human can inspect it. Never throws.
|
|
116
|
+
*/
|
|
117
|
+
export declare function finalizeJournal(storage: JournalStorage, journal: PublishJournal): Promise<"deleted" | "kept">;
|
|
118
|
+
/** GCS-backed adapter. Object-level operations only, by design. */
|
|
119
|
+
export declare function createGcsJournalStorage(bucket: Bucket): JournalStorage;
|