@learnpack/learnpack 5.0.352 → 5.0.354
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/commands/publish.js +6 -0
- package/lib/commands/serve.js +85 -259
- package/lib/models/creator.d.ts +6 -0
- package/lib/scripts/descriptionsGcsBackfill.d.ts +1 -0
- package/lib/scripts/descriptionsGcsBackfill.js +141 -0
- package/lib/scripts/descriptionsS3Backfill.d.ts +1 -0
- package/lib/scripts/descriptionsS3Backfill.js +277 -0
- package/lib/scripts/descriptionsSweep.d.ts +1 -0
- package/lib/scripts/descriptionsSweep.js +142 -0
- package/lib/utils/api.d.ts +7 -0
- package/lib/utils/api.js +8 -1
- package/lib/utils/awsCredentials.d.ts +20 -0
- package/lib/utils/awsCredentials.js +43 -0
- package/lib/utils/creatorUtilities.js +2 -1
- package/lib/utils/descriptionHash.d.ts +66 -0
- package/lib/utils/descriptionHash.js +173 -0
- package/lib/utils/descriptions/backfillEvents.d.ts +60 -0
- package/lib/utils/descriptions/backfillEvents.js +107 -0
- package/lib/utils/descriptions/gcsStorage.d.ts +16 -0
- package/lib/utils/descriptions/gcsStorage.js +60 -0
- package/lib/utils/descriptions/generateCourseDescriptions.d.ts +66 -0
- package/lib/utils/descriptions/generateCourseDescriptions.js +176 -0
- package/lib/utils/descriptions/mirrorDescriptions.d.ts +49 -0
- package/lib/utils/descriptions/mirrorDescriptions.js +109 -0
- package/lib/utils/descriptions/publishStage.d.ts +69 -0
- package/lib/utils/descriptions/publishStage.js +245 -0
- package/lib/utils/descriptions/resumePublication.d.ts +36 -0
- package/lib/utils/descriptions/resumePublication.js +128 -0
- package/lib/utils/descriptions/s3Storage.d.ts +30 -0
- package/lib/utils/descriptions/s3Storage.js +141 -0
- package/lib/utils/descriptions/workList.d.ts +75 -0
- package/lib/utils/descriptions/workList.js +177 -0
- package/lib/utils/gcsBucketName.d.ts +10 -0
- package/lib/utils/gcsBucketName.js +19 -0
- package/lib/utils/packageManifest.d.ts +22 -0
- package/lib/utils/packageManifest.js +57 -5
- package/lib/utils/publishEvents.d.ts +66 -0
- package/lib/utils/publishEvents.js +111 -0
- package/lib/utils/publishJournal.d.ts +119 -0
- package/lib/utils/publishJournal.js +275 -0
- package/lib/utils/repair/legacyPackageRepair.d.ts +131 -0
- package/lib/utils/repair/legacyPackageRepair.js +492 -0
- package/lib/utils/repair/repairStorage.d.ts +68 -0
- package/lib/utils/repair/repairStorage.js +89 -0
- package/lib/utils/rigoActions.d.ts +44 -0
- package/lib/utils/rigoActions.js +75 -1
- package/lib/utils/s3/packageManifestBackfill.d.ts +2 -0
- package/lib/utils/s3/packageManifestBackfill.js +5 -8
- package/lib/utils/s3/packageSourcesAudit.d.ts +75 -0
- package/lib/utils/s3/packageSourcesAudit.js +184 -0
- package/lib/utils/syllabusSync.d.ts +71 -0
- package/lib/utils/syllabusSync.js +273 -0
- package/package.json +3 -1
- package/src/commands/publish.ts +7 -0
- package/src/commands/serve.ts +144 -335
- package/src/models/creator.ts +9 -0
- package/src/scripts/README.md +244 -0
- package/src/scripts/descriptionsGcsBackfill.ts +193 -0
- package/src/scripts/descriptionsS3Backfill.ts +376 -0
- package/src/scripts/descriptionsSweep.ts +185 -0
- package/src/ui/_app/app.css +1 -1
- package/src/ui/_app/app.js +142 -140
- package/src/ui/app.tar.gz +0 -0
- package/src/utils/api.ts +9 -0
- package/src/utils/awsCredentials.ts +57 -0
- package/src/utils/creatorUtilities.ts +2 -1
- package/src/utils/descriptionHash.ts +196 -0
- package/src/utils/descriptions/backfillEvents.ts +152 -0
- package/src/utils/descriptions/gcsStorage.ts +67 -0
- package/src/utils/descriptions/generateCourseDescriptions.ts +311 -0
- package/src/utils/descriptions/mirrorDescriptions.ts +191 -0
- package/src/utils/descriptions/publishStage.ts +394 -0
- package/src/utils/descriptions/resumePublication.ts +217 -0
- package/src/utils/descriptions/s3Storage.ts +214 -0
- package/src/utils/descriptions/workList.ts +283 -0
- package/src/utils/gcsBucketName.ts +19 -0
- package/src/utils/packageManifest.ts +62 -5
- package/src/utils/publishEvents.ts +181 -0
- package/src/utils/publishJournal.ts +383 -0
- package/src/utils/repair/legacyPackageRepair.ts +731 -0
- package/src/utils/repair/repairStorage.ts +168 -0
- package/src/utils/rigoActions.ts +130 -0
- package/src/utils/s3/packageManifestBackfill.ts +771 -776
- package/src/utils/s3/packageSourcesAudit.ts +311 -0
- package/src/utils/syllabusSync.ts +390 -0
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.manifestKey = exports.syllabusKey = void 0;
|
|
4
|
+
exports.fetchExercises = fetchExercises;
|
|
5
|
+
exports.createS3SyllabusSyncStorage = createS3SyllabusSyncStorage;
|
|
6
|
+
exports.createS3DescriptionsStorageFromEnv = createS3DescriptionsStorageFromEnv;
|
|
7
|
+
exports.createS3DescriptionsStorage = createS3DescriptionsStorage;
|
|
8
|
+
const client_s3_1 = require("@aws-sdk/client-s3");
|
|
9
|
+
const client_cloudfront_1 = require("@aws-sdk/client-cloudfront");
|
|
10
|
+
const packageManifest_1 = require("../packageManifest");
|
|
11
|
+
const packageManifestBackfill_1 = require("../s3/packageManifestBackfill");
|
|
12
|
+
const awsCredentials_1 = require("../awsCredentials");
|
|
13
|
+
/**
|
|
14
|
+
* Published-bucket (S3) adapter for the description flows.
|
|
15
|
+
*
|
|
16
|
+
* The published package is the snapshot descriptions are generated from: it is
|
|
17
|
+
* immutable between publications, unlike the draft in GCS, which keeps moving
|
|
18
|
+
* while the teacher edits.
|
|
19
|
+
*/
|
|
20
|
+
const syllabusKey = (slug) => `${slug}/.learn/initialSyllabus.json`;
|
|
21
|
+
exports.syllabusKey = syllabusKey;
|
|
22
|
+
const manifestKey = (slug) => `${slug}/${packageManifest_1.PACKAGE_MANIFEST_REL_PATH}`;
|
|
23
|
+
exports.manifestKey = manifestKey;
|
|
24
|
+
const configKeys = (slug) => [
|
|
25
|
+
`${slug}/.learn/config.json`,
|
|
26
|
+
`${slug}/config.json`,
|
|
27
|
+
];
|
|
28
|
+
async function fetchExercises(s3, bucket, slug) {
|
|
29
|
+
for (const key of configKeys(slug)) {
|
|
30
|
+
// eslint-disable-next-line no-await-in-loop -- the second key is a fallback
|
|
31
|
+
const config = await (0, packageManifestBackfill_1.fetchJsonObject)(s3, bucket, key);
|
|
32
|
+
if ((config === null || config === void 0 ? void 0 : config.exercises) && Array.isArray(config.exercises)) {
|
|
33
|
+
return config.exercises;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return [];
|
|
37
|
+
}
|
|
38
|
+
function createS3SyllabusSyncStorage(s3, bucket) {
|
|
39
|
+
return {
|
|
40
|
+
async listExerciseFolderFileCounts(courseSlug) {
|
|
41
|
+
const prefix = `${courseSlug}/exercises/`;
|
|
42
|
+
const keys = await (0, packageManifestBackfill_1.listObjectKeys)(s3, bucket, prefix);
|
|
43
|
+
const counts = new Map();
|
|
44
|
+
for (const key of keys) {
|
|
45
|
+
const segment = key.slice(prefix.length).split("/")[0];
|
|
46
|
+
if (segment) {
|
|
47
|
+
counts.set(segment, (counts.get(segment) || 0) + 1);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return counts;
|
|
51
|
+
},
|
|
52
|
+
async translationLangsBySlug(courseSlug) {
|
|
53
|
+
const exercises = await fetchExercises(s3, bucket, courseSlug);
|
|
54
|
+
const map = new Map();
|
|
55
|
+
for (const exercise of exercises) {
|
|
56
|
+
const langs = Object.keys(exercise.translations || {})
|
|
57
|
+
.map(lang => lang.toLowerCase())
|
|
58
|
+
.filter(Boolean);
|
|
59
|
+
if (langs.length > 0) {
|
|
60
|
+
map.set(exercise.slug, [...new Set(langs)]);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return map;
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Storage for the published bucket, wired from the environment. Used by the
|
|
69
|
+
* post-publish stage and the sweep, which have no CLI flags to read.
|
|
70
|
+
*/
|
|
71
|
+
function createS3DescriptionsStorageFromEnv() {
|
|
72
|
+
// Validated here so a missing variable fails with its own name, instead of
|
|
73
|
+
// surfacing minutes later as "Could not load credentials from any providers"
|
|
74
|
+
// from inside a background job.
|
|
75
|
+
const credentials = (0, awsCredentials_1.requireAwsCredentials)();
|
|
76
|
+
const region = (0, awsCredentials_1.awsRegion)();
|
|
77
|
+
const s3 = new client_s3_1.S3Client({ region, credentials });
|
|
78
|
+
const distributionId = (process.env.CLOUDFRONT_DISTRIBUTION_ID || "").trim();
|
|
79
|
+
const cloudFront = distributionId ?
|
|
80
|
+
{
|
|
81
|
+
client: new client_cloudfront_1.CloudFrontClient({
|
|
82
|
+
region,
|
|
83
|
+
credentials,
|
|
84
|
+
}),
|
|
85
|
+
distributionId,
|
|
86
|
+
} :
|
|
87
|
+
undefined;
|
|
88
|
+
if (!cloudFront) {
|
|
89
|
+
console.warn("[descriptions] CLOUDFRONT_DISTRIBUTION_ID is not set: the manifest will be re-projected but not invalidated");
|
|
90
|
+
}
|
|
91
|
+
return createS3DescriptionsStorage(s3, (0, awsCredentials_1.requireS3PackagesBucket)(), {
|
|
92
|
+
cloudFront,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
function createS3DescriptionsStorage(s3, bucket, options = {}) {
|
|
96
|
+
const storage = {
|
|
97
|
+
async readSyllabus(courseSlug) {
|
|
98
|
+
return (0, packageManifestBackfill_1.fetchJsonObject)(s3, bucket, (0, exports.syllabusKey)(courseSlug));
|
|
99
|
+
},
|
|
100
|
+
async writeSyllabus(courseSlug, syllabus) {
|
|
101
|
+
await (0, packageManifestBackfill_1.withRetry)(() => s3.send(new client_s3_1.PutObjectCommand({
|
|
102
|
+
Bucket: bucket,
|
|
103
|
+
Key: (0, exports.syllabusKey)(courseSlug),
|
|
104
|
+
Body: JSON.stringify(syllabus, null, 2),
|
|
105
|
+
ContentType: "application/json",
|
|
106
|
+
})));
|
|
107
|
+
},
|
|
108
|
+
async listExercises(courseSlug) {
|
|
109
|
+
return fetchExercises(s3, bucket, courseSlug);
|
|
110
|
+
},
|
|
111
|
+
async readReadme(courseSlug, exerciseSlug, fileName) {
|
|
112
|
+
return (0, packageManifestBackfill_1.fetchTextObject)(s3, bucket, `${courseSlug}/exercises/${exerciseSlug}/${fileName}`);
|
|
113
|
+
},
|
|
114
|
+
async readManifest(courseSlug) {
|
|
115
|
+
return (0, packageManifestBackfill_1.fetchJsonObject)(s3, bucket, (0, exports.manifestKey)(courseSlug));
|
|
116
|
+
},
|
|
117
|
+
syllabusSyncStorage: createS3SyllabusSyncStorage(s3, bucket),
|
|
118
|
+
};
|
|
119
|
+
if (options.reprojectManifest !== false) {
|
|
120
|
+
storage.reprojectManifest = async (courseSlug) => {
|
|
121
|
+
var _a;
|
|
122
|
+
// Preserve the publication timestamp: this is a re-projection of an
|
|
123
|
+
// already published package, not a new publication.
|
|
124
|
+
const existing = await (0, packageManifestBackfill_1.fetchJsonObject)(s3, bucket, (0, exports.manifestKey)(courseSlug));
|
|
125
|
+
const result = await (0, packageManifestBackfill_1.processPackage)(s3, bucket, courseSlug, {
|
|
126
|
+
dryRun: false,
|
|
127
|
+
force: false,
|
|
128
|
+
skipExisting: false,
|
|
129
|
+
publishedAt: (_a = existing === null || existing === void 0 ? void 0 : existing.publishedAt) !== null && _a !== void 0 ? _a : null,
|
|
130
|
+
});
|
|
131
|
+
if (result.status === "failed") {
|
|
132
|
+
throw new Error(result.error || "manifest projection failed");
|
|
133
|
+
}
|
|
134
|
+
if (result.status === "skipped" || !options.cloudFront) {
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
await (0, packageManifestBackfill_1.invalidatePackageManifestPaths)(options.cloudFront.client, options.cloudFront.distributionId, [`/${(0, exports.manifestKey)(courseSlug)}`]);
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
return storage;
|
|
141
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { Lesson, Syllabus } from "../../models/creator";
|
|
2
|
+
import { ContentFingerprint } from "../descriptionHash";
|
|
3
|
+
/**
|
|
4
|
+
* Deciding WHAT needs a description, separated from generating it.
|
|
5
|
+
*
|
|
6
|
+
* The publish route needs the answer before doing any work (to tell breathecode
|
|
7
|
+
* whether a second event is coming), and the generation service needs the very
|
|
8
|
+
* same list right after. Keeping the decision here, pure and content-only,
|
|
9
|
+
* means both answer identically — the alternative, two implementations of
|
|
10
|
+
* "is this stale?", is exactly how the backfill and the sweep drifted apart.
|
|
11
|
+
*/
|
|
12
|
+
export type CourseExercise = {
|
|
13
|
+
slug: string;
|
|
14
|
+
/** language code -> README file name */
|
|
15
|
+
translations: Record<string, string>;
|
|
16
|
+
};
|
|
17
|
+
/** README contents by exercise slug and language. */
|
|
18
|
+
export type CourseReadmes = Record<string, Record<string, string>>;
|
|
19
|
+
export type LanguageWork = {
|
|
20
|
+
lang: string;
|
|
21
|
+
fingerprint: ContentFingerprint;
|
|
22
|
+
};
|
|
23
|
+
export type StepWork = {
|
|
24
|
+
exerciseSlug: string;
|
|
25
|
+
/** Reference into the syllabus; the generation service writes through it. */
|
|
26
|
+
lesson: Lesson;
|
|
27
|
+
/** Language the description is written from and translated out of. */
|
|
28
|
+
baseLanguage: string;
|
|
29
|
+
baseContent: string;
|
|
30
|
+
languages: LanguageWork[];
|
|
31
|
+
};
|
|
32
|
+
export type BuildWorkListInput = {
|
|
33
|
+
syllabus: Syllabus;
|
|
34
|
+
exercises: CourseExercise[];
|
|
35
|
+
readmes: CourseReadmes;
|
|
36
|
+
promptVersion?: number;
|
|
37
|
+
simhashThreshold?: number;
|
|
38
|
+
/** Regenerate everything except human-edited descriptions. */
|
|
39
|
+
force?: boolean;
|
|
40
|
+
};
|
|
41
|
+
export declare function findSyllabusLesson(syllabus: Syllabus, exerciseSlug: string): Lesson | undefined;
|
|
42
|
+
/**
|
|
43
|
+
* Language a course is actually written in.
|
|
44
|
+
*
|
|
45
|
+
* `courseInfo.language` is trusted only when some exercise really has a README
|
|
46
|
+
* in it; otherwise the language is inferred from the content itself (the most
|
|
47
|
+
* frequent translation across exercises). Defaulting to "en" when the field is
|
|
48
|
+
* missing would label Spanish courses as English and generate their
|
|
49
|
+
* descriptions from a translation instead of the original.
|
|
50
|
+
*/
|
|
51
|
+
export declare function inferCourseBaseLanguage(syllabus: Syllabus, exercises: CourseExercise[]): string | null;
|
|
52
|
+
/**
|
|
53
|
+
* Whether a (step, language) needs generating. Human-edited descriptions are
|
|
54
|
+
* never touched; an explicit "not enough content" answer (a stored null with a
|
|
55
|
+
* fingerprint) is respected until the content itself changes.
|
|
56
|
+
*/
|
|
57
|
+
export declare function needsGeneration(slot: NonNullable<Lesson["translations"]>[string] | undefined, fingerprint: ContentFingerprint, options: {
|
|
58
|
+
promptVersion: number;
|
|
59
|
+
simhashThreshold: number;
|
|
60
|
+
force: boolean;
|
|
61
|
+
}): boolean;
|
|
62
|
+
/** The (step, language) pairs whose description is missing or out of date. */
|
|
63
|
+
export declare function buildDescriptionWorkList(input: BuildWorkListInput): StepWork[];
|
|
64
|
+
/** Total (step, language) pairs in a work list. */
|
|
65
|
+
export declare function countWorkItems(work: StepWork[]): number;
|
|
66
|
+
/**
|
|
67
|
+
* Cheap "has this course ever been fully processed at this prompt version?"
|
|
68
|
+
* check, from the syllabus alone.
|
|
69
|
+
*
|
|
70
|
+
* Deliberately weaker than the work list: it cannot see content changes because
|
|
71
|
+
* it never reads a README. That is the point — it lets a batched run skip
|
|
72
|
+
* finished courses without paying thousands of object reads. Use the work list
|
|
73
|
+
* whenever correctness matters; use this only as a pre-filter.
|
|
74
|
+
*/
|
|
75
|
+
export declare function isCourseSettled(syllabus: Syllabus, exercises: CourseExercise[], promptVersion?: number): boolean;
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.findSyllabusLesson = findSyllabusLesson;
|
|
4
|
+
exports.inferCourseBaseLanguage = inferCourseBaseLanguage;
|
|
5
|
+
exports.needsGeneration = needsGeneration;
|
|
6
|
+
exports.buildDescriptionWorkList = buildDescriptionWorkList;
|
|
7
|
+
exports.countWorkItems = countWorkItems;
|
|
8
|
+
exports.isCourseSettled = isCourseSettled;
|
|
9
|
+
const creatorUtilities_1 = require("../creatorUtilities");
|
|
10
|
+
const descriptionHash_1 = require("../descriptionHash");
|
|
11
|
+
const packageManifest_1 = require("../packageManifest");
|
|
12
|
+
function findSyllabusLesson(syllabus, exerciseSlug) {
|
|
13
|
+
return syllabus.lessons.find(lesson => lesson.uid === exerciseSlug ||
|
|
14
|
+
(0, creatorUtilities_1.slugify)(lesson.id + "-" + lesson.title) === exerciseSlug);
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Language a course is actually written in.
|
|
18
|
+
*
|
|
19
|
+
* `courseInfo.language` is trusted only when some exercise really has a README
|
|
20
|
+
* in it; otherwise the language is inferred from the content itself (the most
|
|
21
|
+
* frequent translation across exercises). Defaulting to "en" when the field is
|
|
22
|
+
* missing would label Spanish courses as English and generate their
|
|
23
|
+
* descriptions from a translation instead of the original.
|
|
24
|
+
*/
|
|
25
|
+
function inferCourseBaseLanguage(syllabus, exercises) {
|
|
26
|
+
var _a, _b;
|
|
27
|
+
const counts = new Map();
|
|
28
|
+
for (const exercise of exercises) {
|
|
29
|
+
for (const lang of Object.keys(exercise.translations || {})) {
|
|
30
|
+
counts.set(lang, (counts.get(lang) || 0) + 1);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
if (counts.size === 0) {
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
const declared = (_b = (_a = syllabus.courseInfo) === null || _a === void 0 ? void 0 : _a.language) === null || _b === void 0 ? void 0 : _b.toLowerCase();
|
|
37
|
+
if (declared) {
|
|
38
|
+
for (const lang of counts.keys()) {
|
|
39
|
+
if (lang.toLowerCase() === declared) {
|
|
40
|
+
return lang;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
let best = null;
|
|
45
|
+
let bestCount = 0;
|
|
46
|
+
for (const [lang, count] of counts) {
|
|
47
|
+
if (count > bestCount) {
|
|
48
|
+
best = lang;
|
|
49
|
+
bestCount = count;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return best;
|
|
53
|
+
}
|
|
54
|
+
/** Language to generate this step from: the course's, when it has it. */
|
|
55
|
+
function resolveStepBaseLanguage(exercise, courseBaseLanguage) {
|
|
56
|
+
const langs = Object.keys(exercise.translations || {});
|
|
57
|
+
if (langs.length === 0) {
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
if (courseBaseLanguage && langs.includes(courseBaseLanguage)) {
|
|
61
|
+
return courseBaseLanguage;
|
|
62
|
+
}
|
|
63
|
+
return langs[0];
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Whether a (step, language) needs generating. Human-edited descriptions are
|
|
67
|
+
* never touched; an explicit "not enough content" answer (a stored null with a
|
|
68
|
+
* fingerprint) is respected until the content itself changes.
|
|
69
|
+
*/
|
|
70
|
+
function needsGeneration(slot, fingerprint, options) {
|
|
71
|
+
var _a;
|
|
72
|
+
if ((slot === null || slot === void 0 ? void 0 : slot.descriptionSource) === "human") {
|
|
73
|
+
return false;
|
|
74
|
+
}
|
|
75
|
+
if (!slot) {
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
if (options.force) {
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
if (slot.descriptionStatus === "error") {
|
|
82
|
+
return true;
|
|
83
|
+
}
|
|
84
|
+
if (((_a = slot.descriptionPromptVersion) !== null && _a !== void 0 ? _a : 0) < options.promptVersion) {
|
|
85
|
+
return true;
|
|
86
|
+
}
|
|
87
|
+
return (0, descriptionHash_1.isDescriptionStale)({ sha256: slot.sourceContentHash, simhash: slot.sourceSimHash }, fingerprint, options.simhashThreshold);
|
|
88
|
+
}
|
|
89
|
+
/** The (step, language) pairs whose description is missing or out of date. */
|
|
90
|
+
function buildDescriptionWorkList(input) {
|
|
91
|
+
var _a, _b, _c, _d;
|
|
92
|
+
const promptVersion = (_a = input.promptVersion) !== null && _a !== void 0 ? _a : packageManifest_1.DESCRIPTION_PROMPT_VERSION;
|
|
93
|
+
const simhashThreshold = (_b = input.simhashThreshold) !== null && _b !== void 0 ? _b : (0, descriptionHash_1.getSimhashThreshold)();
|
|
94
|
+
const force = (_c = input.force) !== null && _c !== void 0 ? _c : false;
|
|
95
|
+
const courseBaseLanguage = inferCourseBaseLanguage(input.syllabus, input.exercises);
|
|
96
|
+
const work = [];
|
|
97
|
+
for (const exercise of input.exercises) {
|
|
98
|
+
const lesson = findSyllabusLesson(input.syllabus, exercise.slug);
|
|
99
|
+
if (!lesson) {
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
const baseLanguage = resolveStepBaseLanguage(exercise, courseBaseLanguage);
|
|
103
|
+
if (!baseLanguage) {
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
const exerciseReadmes = input.readmes[exercise.slug] || {};
|
|
107
|
+
const baseContent = exerciseReadmes[baseLanguage];
|
|
108
|
+
if (typeof baseContent !== "string") {
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
const languages = [];
|
|
112
|
+
for (const lang of Object.keys(exercise.translations || {})) {
|
|
113
|
+
const content = exerciseReadmes[lang];
|
|
114
|
+
if (typeof content !== "string") {
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
// Each language's staleness is anchored to its OWN content, even though
|
|
118
|
+
// the description is generated from the base language: that is what makes
|
|
119
|
+
// "the Spanish README changed" regenerate only Spanish.
|
|
120
|
+
const fingerprint = (0, descriptionHash_1.fingerprintReadme)(content);
|
|
121
|
+
if (!needsGeneration((_d = lesson.translations) === null || _d === void 0 ? void 0 : _d[lang], fingerprint, {
|
|
122
|
+
promptVersion,
|
|
123
|
+
simhashThreshold,
|
|
124
|
+
force,
|
|
125
|
+
})) {
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
languages.push({ lang, fingerprint });
|
|
129
|
+
}
|
|
130
|
+
if (languages.length === 0) {
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
work.push({
|
|
134
|
+
exerciseSlug: exercise.slug,
|
|
135
|
+
lesson,
|
|
136
|
+
baseLanguage,
|
|
137
|
+
baseContent,
|
|
138
|
+
languages,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
return work;
|
|
142
|
+
}
|
|
143
|
+
/** Total (step, language) pairs in a work list. */
|
|
144
|
+
function countWorkItems(work) {
|
|
145
|
+
return work.reduce((total, step) => total + step.languages.length, 0);
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Cheap "has this course ever been fully processed at this prompt version?"
|
|
149
|
+
* check, from the syllabus alone.
|
|
150
|
+
*
|
|
151
|
+
* Deliberately weaker than the work list: it cannot see content changes because
|
|
152
|
+
* it never reads a README. That is the point — it lets a batched run skip
|
|
153
|
+
* finished courses without paying thousands of object reads. Use the work list
|
|
154
|
+
* whenever correctness matters; use this only as a pre-filter.
|
|
155
|
+
*/
|
|
156
|
+
function isCourseSettled(syllabus, exercises, promptVersion = packageManifest_1.DESCRIPTION_PROMPT_VERSION) {
|
|
157
|
+
var _a, _b;
|
|
158
|
+
for (const exercise of exercises) {
|
|
159
|
+
const lesson = findSyllabusLesson(syllabus, exercise.slug);
|
|
160
|
+
if (!lesson) {
|
|
161
|
+
return false;
|
|
162
|
+
}
|
|
163
|
+
for (const lang of Object.keys(exercise.translations || {})) {
|
|
164
|
+
const slot = (_a = lesson.translations) === null || _a === void 0 ? void 0 : _a[lang];
|
|
165
|
+
if ((slot === null || slot === void 0 ? void 0 : slot.descriptionSource) === "human") {
|
|
166
|
+
continue;
|
|
167
|
+
}
|
|
168
|
+
if (!slot ||
|
|
169
|
+
slot.descriptionStatus === "error" ||
|
|
170
|
+
!slot.sourceContentHash ||
|
|
171
|
+
((_b = slot.descriptionPromptVersion) !== null && _b !== void 0 ? _b : 0) < promptVersion) {
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return true;
|
|
177
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The draft bucket name, required rather than defaulted.
|
|
3
|
+
*
|
|
4
|
+
* The code used to fall back to "learnpack-packages", a name the service
|
|
5
|
+
* account cannot reach: with the variable missing, the process would start
|
|
6
|
+
* cleanly and then fail with a 403 on every single object operation. Failing at
|
|
7
|
+
* boot with the reason turns a puzzling runtime outage into an obvious
|
|
8
|
+
* misconfiguration.
|
|
9
|
+
*/
|
|
10
|
+
export declare function requireGcsBucketName(): string;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.requireGcsBucketName = requireGcsBucketName;
|
|
4
|
+
/**
|
|
5
|
+
* The draft bucket name, required rather than defaulted.
|
|
6
|
+
*
|
|
7
|
+
* The code used to fall back to "learnpack-packages", a name the service
|
|
8
|
+
* account cannot reach: with the variable missing, the process would start
|
|
9
|
+
* cleanly and then fail with a 403 on every single object operation. Failing at
|
|
10
|
+
* boot with the reason turns a puzzling runtime outage into an obvious
|
|
11
|
+
* misconfiguration.
|
|
12
|
+
*/
|
|
13
|
+
function requireGcsBucketName() {
|
|
14
|
+
const name = (process.env.GCP_BUCKET_NAME || "").trim();
|
|
15
|
+
if (!name) {
|
|
16
|
+
throw new Error("GCP_BUCKET_NAME (env) is required: it names the bucket holding course drafts");
|
|
17
|
+
}
|
|
18
|
+
return name;
|
|
19
|
+
}
|
|
@@ -4,6 +4,8 @@ import { Exercise } from "./configBuilder";
|
|
|
4
4
|
export declare const PACKAGE_MANIFEST_FILENAME = "package-manifest.json";
|
|
5
5
|
export declare const PACKAGE_MANIFEST_REL_PATH = ".learn/package-manifest.json";
|
|
6
6
|
export declare const SCHEMA_VERSION = 1;
|
|
7
|
+
export declare const DESCRIPTION_PROMPT_VERSION = 1;
|
|
8
|
+
export declare const DESCRIPTION_TARGET_WORD_COUNT = 25;
|
|
7
9
|
export type LessonType = "READ" | "CODE" | "QUIZ";
|
|
8
10
|
export type PackageManifestLesson = {
|
|
9
11
|
id: string;
|
|
@@ -46,12 +48,32 @@ export type PackageManifestBuildInput = {
|
|
|
46
48
|
};
|
|
47
49
|
export declare function packageManifestGcsPath(courseSlug: string): string;
|
|
48
50
|
export declare function resolvePreviewUrl(slug: string, preview?: string): string;
|
|
51
|
+
/**
|
|
52
|
+
* Lesson id when no syllabus entry matches: the numeric prefix of the folder.
|
|
53
|
+
*
|
|
54
|
+
* Exported because `legacyPackageRepair` has to synthesize syllabus ids with
|
|
55
|
+
* exactly this rule — a repaired package whose ids differed from the ones its
|
|
56
|
+
* manifest already carries would look like a content change to every consumer.
|
|
57
|
+
*/
|
|
58
|
+
export declare function deriveLessonId(exerciseSlug: string): string;
|
|
49
59
|
export declare function extractReadmeH1(body: string): string | null;
|
|
50
60
|
export declare function normalizeReadmeTitle(raw: string): string | null;
|
|
51
61
|
export declare function buildPackageManifestFromSources(input: PackageManifestBuildInput): PackageManifest;
|
|
52
62
|
export declare function parseReadmeContent(content: string): ReadmeFrontmatter;
|
|
53
63
|
export declare function comparePackageManifests(existing: PackageManifest | null, next: PackageManifest): "created" | "updated" | "skipped";
|
|
54
64
|
export declare function serializePackageManifest(manifest: PackageManifest): string;
|
|
65
|
+
/** The manifest as it travels inside a publication event. */
|
|
66
|
+
export type PackageManifestEventView = Omit<PackageManifest, "generatedAt">;
|
|
67
|
+
/**
|
|
68
|
+
* Project a manifest into the shape sent to breathecode.
|
|
69
|
+
*
|
|
70
|
+
* Today this is nearly the identity — only `generatedAt`, an internal detail of
|
|
71
|
+
* the projection, is dropped. It exists as an explicit pick so that fields
|
|
72
|
+
* added to the manifest later (hashes, generation flags) do not silently become
|
|
73
|
+
* part of an external contract: joining the event payload has to be a decision
|
|
74
|
+
* visible in a diff.
|
|
75
|
+
*/
|
|
76
|
+
export declare function serializePackageManifestForEvent(manifest: PackageManifest): PackageManifestEventView;
|
|
55
77
|
export declare function generateAndPersistPackageManifest(bucket: Bucket, courseSlug: string, options: {
|
|
56
78
|
publishedAt: string | null;
|
|
57
79
|
localDir?: string;
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.SCHEMA_VERSION = exports.PACKAGE_MANIFEST_REL_PATH = exports.PACKAGE_MANIFEST_FILENAME = void 0;
|
|
3
|
+
exports.DESCRIPTION_TARGET_WORD_COUNT = exports.DESCRIPTION_PROMPT_VERSION = exports.SCHEMA_VERSION = exports.PACKAGE_MANIFEST_REL_PATH = exports.PACKAGE_MANIFEST_FILENAME = void 0;
|
|
4
4
|
exports.packageManifestGcsPath = packageManifestGcsPath;
|
|
5
5
|
exports.resolvePreviewUrl = resolvePreviewUrl;
|
|
6
|
+
exports.deriveLessonId = deriveLessonId;
|
|
6
7
|
exports.extractReadmeH1 = extractReadmeH1;
|
|
7
8
|
exports.normalizeReadmeTitle = normalizeReadmeTitle;
|
|
8
9
|
exports.buildPackageManifestFromSources = buildPackageManifestFromSources;
|
|
9
10
|
exports.parseReadmeContent = parseReadmeContent;
|
|
10
11
|
exports.comparePackageManifests = comparePackageManifests;
|
|
11
12
|
exports.serializePackageManifest = serializePackageManifest;
|
|
13
|
+
exports.serializePackageManifestForEvent = serializePackageManifestForEvent;
|
|
12
14
|
exports.generateAndPersistPackageManifest = generateAndPersistPackageManifest;
|
|
13
15
|
const tslib_1 = require("tslib");
|
|
14
16
|
const fs = require("fs");
|
|
@@ -22,6 +24,13 @@ const frontMatter = require("front-matter");
|
|
|
22
24
|
exports.PACKAGE_MANIFEST_FILENAME = "package-manifest.json";
|
|
23
25
|
exports.PACKAGE_MANIFEST_REL_PATH = ".learn/package-manifest.json";
|
|
24
26
|
exports.SCHEMA_VERSION = 1;
|
|
27
|
+
// Version of the Rigobot prompt/model used to generate step descriptions. It
|
|
28
|
+
// travels inside the completion inputs, so bumping it both busts Rigobot's
|
|
29
|
+
// cache and marks every description stored at a lower version for regeneration.
|
|
30
|
+
// Keep in sync with the prompt body provisioned in Rigobot.
|
|
31
|
+
exports.DESCRIPTION_PROMPT_VERSION = 1;
|
|
32
|
+
// Target length, in words, of a generated step description.
|
|
33
|
+
exports.DESCRIPTION_TARGET_WORD_COUNT = 25;
|
|
25
34
|
const VALID_LESSON_TYPES = new Set(["READ", "CODE", "QUIZ"]);
|
|
26
35
|
function packageManifestGcsPath(courseSlug) {
|
|
27
36
|
return `courses/${courseSlug}/${exports.PACKAGE_MANIFEST_REL_PATH}`;
|
|
@@ -41,6 +50,13 @@ function findSyllabusLesson(exerciseSlug, lessons) {
|
|
|
41
50
|
return candidates.includes(exerciseSlug);
|
|
42
51
|
});
|
|
43
52
|
}
|
|
53
|
+
/**
|
|
54
|
+
* Lesson id when no syllabus entry matches: the numeric prefix of the folder.
|
|
55
|
+
*
|
|
56
|
+
* Exported because `legacyPackageRepair` has to synthesize syllabus ids with
|
|
57
|
+
* exactly this rule — a repaired package whose ids differed from the ones its
|
|
58
|
+
* manifest already carries would look like a content change to every consumer.
|
|
59
|
+
*/
|
|
44
60
|
function deriveLessonId(exerciseSlug) {
|
|
45
61
|
const match = exerciseSlug.match(/^(\d+(?:\.\d+)?)/);
|
|
46
62
|
return match ? match[1] : exerciseSlug;
|
|
@@ -94,11 +110,16 @@ function resolveLessonTitle(exercise, sidebar, syllabusLesson, courseLang, readm
|
|
|
94
110
|
}
|
|
95
111
|
return titles;
|
|
96
112
|
}
|
|
97
|
-
function resolveLessonDescriptions(exercise) {
|
|
113
|
+
function resolveLessonDescriptions(exercise, syllabusLesson) {
|
|
114
|
+
var _a, _b, _c;
|
|
98
115
|
const description = {};
|
|
116
|
+
// Descriptions are a projection: the source of truth lives in the syllabus
|
|
117
|
+
// lesson (translations[lang].description), populated by the sweep/backfill.
|
|
118
|
+
// When absent we emit null (the frontend hides the area) rather than a
|
|
119
|
+
// low-quality placeholder.
|
|
99
120
|
for (const lang of Object.keys(exercise.translations)) {
|
|
100
|
-
|
|
101
|
-
|
|
121
|
+
description[lang] =
|
|
122
|
+
(_c = (_b = (_a = syllabusLesson === null || syllabusLesson === void 0 ? void 0 : syllabusLesson.translations) === null || _a === void 0 ? void 0 : _a[lang]) === null || _b === void 0 ? void 0 : _b.description) !== null && _c !== void 0 ? _c : null;
|
|
102
123
|
}
|
|
103
124
|
return description;
|
|
104
125
|
}
|
|
@@ -134,7 +155,7 @@ function buildPackageManifestFromSources(input) {
|
|
|
134
155
|
position: exercise.position,
|
|
135
156
|
type,
|
|
136
157
|
title: resolveLessonTitle(exercise, sidebar, syllabusLesson, courseLang, (_b = readmeFrontmatters[exercise.slug]) !== null && _b !== void 0 ? _b : {}),
|
|
137
|
-
description: resolveLessonDescriptions(exercise),
|
|
158
|
+
description: resolveLessonDescriptions(exercise, syllabusLesson),
|
|
138
159
|
video: resolveLessonVideos(exercise, readmeFrontmatters),
|
|
139
160
|
};
|
|
140
161
|
});
|
|
@@ -184,6 +205,37 @@ function comparePackageManifests(existing, next) {
|
|
|
184
205
|
function serializePackageManifest(manifest) {
|
|
185
206
|
return JSON.stringify(manifest, null, 2);
|
|
186
207
|
}
|
|
208
|
+
/**
|
|
209
|
+
* Project a manifest into the shape sent to breathecode.
|
|
210
|
+
*
|
|
211
|
+
* Today this is nearly the identity — only `generatedAt`, an internal detail of
|
|
212
|
+
* the projection, is dropped. It exists as an explicit pick so that fields
|
|
213
|
+
* added to the manifest later (hashes, generation flags) do not silently become
|
|
214
|
+
* part of an external contract: joining the event payload has to be a decision
|
|
215
|
+
* visible in a diff.
|
|
216
|
+
*/
|
|
217
|
+
function serializePackageManifestForEvent(manifest) {
|
|
218
|
+
return {
|
|
219
|
+
schemaVersion: manifest.schemaVersion,
|
|
220
|
+
publishedAt: manifest.publishedAt,
|
|
221
|
+
slug: manifest.slug,
|
|
222
|
+
title: manifest.title,
|
|
223
|
+
description: manifest.description,
|
|
224
|
+
preview: manifest.preview,
|
|
225
|
+
technologies: manifest.technologies,
|
|
226
|
+
difficulty: manifest.difficulty,
|
|
227
|
+
duration: manifest.duration,
|
|
228
|
+
lessons: manifest.lessons.map(lesson => ({
|
|
229
|
+
id: lesson.id,
|
|
230
|
+
slug: lesson.slug,
|
|
231
|
+
position: lesson.position,
|
|
232
|
+
type: lesson.type,
|
|
233
|
+
title: lesson.title,
|
|
234
|
+
description: lesson.description,
|
|
235
|
+
video: lesson.video,
|
|
236
|
+
})),
|
|
237
|
+
};
|
|
238
|
+
}
|
|
187
239
|
async function readReadmeFrontmatter(bucket, exerciseSlug, lang, readmePath) {
|
|
188
240
|
try {
|
|
189
241
|
const [buf] = await bucket.file(readmePath).download();
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { PackageManifest, PackageManifestEventView } from "./packageManifest";
|
|
2
|
+
/**
|
|
3
|
+
* Publication events sent to breathecode.
|
|
4
|
+
*
|
|
5
|
+
* They are emitted from here, and not from Rigobot, because learnpack-cli is
|
|
6
|
+
* the only party that witnesses the facts: Rigobot's part ends when the zip is
|
|
7
|
+
* deployed, before the breathecode asset sync, and it never sees the end of the
|
|
8
|
+
* asynchronous description stage.
|
|
9
|
+
*
|
|
10
|
+
* Transport is the existing telemetry endpoint. The discriminator is a
|
|
11
|
+
* top-level `event` key, which is what breathecode's webhook log already looks
|
|
12
|
+
* for; a body without it keeps being processed as plain telemetry.
|
|
13
|
+
*
|
|
14
|
+
* `package_slug` and `package_id` are duplicated at the top level on purpose:
|
|
15
|
+
* `LearnPack.add_webhook_to_log` fills the indexed columns from there, so
|
|
16
|
+
* without them the events would land as rows nobody can filter in the webhook
|
|
17
|
+
* viewer.
|
|
18
|
+
*/
|
|
19
|
+
export declare const PACKAGE_PUBLISHED_EVENT = "package_published";
|
|
20
|
+
export declare const PACKAGE_MANIFEST_UPDATED_EVENT = "package_manifest_updated";
|
|
21
|
+
/** Whether a second event is coming for this publication. */
|
|
22
|
+
export type DescriptionsOutcome = "queued" | "up_to_date";
|
|
23
|
+
export type PublishEventEnvelope = {
|
|
24
|
+
event: string;
|
|
25
|
+
package_slug: string;
|
|
26
|
+
package_id?: number;
|
|
27
|
+
payload: Record<string, unknown>;
|
|
28
|
+
};
|
|
29
|
+
export type PackagePublishedPayload = {
|
|
30
|
+
publish_id: string;
|
|
31
|
+
package: Record<string, unknown> | null;
|
|
32
|
+
manifest: PackageManifestEventView | null;
|
|
33
|
+
descriptions: DescriptionsOutcome;
|
|
34
|
+
};
|
|
35
|
+
export type PackageManifestUpdatedPayload = {
|
|
36
|
+
publish_id: string;
|
|
37
|
+
status: "success" | "failed";
|
|
38
|
+
package: Record<string, unknown> | null;
|
|
39
|
+
manifest: PackageManifestEventView | null;
|
|
40
|
+
stats?: {
|
|
41
|
+
generated: number;
|
|
42
|
+
failed: number;
|
|
43
|
+
missing: number;
|
|
44
|
+
};
|
|
45
|
+
};
|
|
46
|
+
export type PublishEventContext = {
|
|
47
|
+
publishId: string;
|
|
48
|
+
courseSlug: string;
|
|
49
|
+
/** `GET /v1/learnpack/package/{slug}/` from Rigobot. */
|
|
50
|
+
packageInfo: Record<string, unknown> | null;
|
|
51
|
+
manifest: PackageManifest | null;
|
|
52
|
+
};
|
|
53
|
+
export declare function buildPackagePublishedEvent(context: PublishEventContext, descriptions: DescriptionsOutcome): PublishEventEnvelope;
|
|
54
|
+
export declare function buildPackageManifestUpdatedEvent(context: PublishEventContext, status: "success" | "failed", stats?: PackageManifestUpdatedPayload["stats"]): PublishEventEnvelope;
|
|
55
|
+
/**
|
|
56
|
+
* Fire and forget: a publication must never fail because breathecode is down,
|
|
57
|
+
* so every error is swallowed and logged. Returns whether it was delivered, for
|
|
58
|
+
* the journal, not for control flow.
|
|
59
|
+
*/
|
|
60
|
+
export declare function sendPublishEvent(envelope: PublishEventEnvelope, breathecodeToken: string, timeoutMs?: number): Promise<boolean>;
|
|
61
|
+
/**
|
|
62
|
+
* Package information for the event payload, straight from Rigobot's own
|
|
63
|
+
* serializer so the shape stays owned by the system that owns the model.
|
|
64
|
+
* Returns null on failure: an event with less context still beats no event.
|
|
65
|
+
*/
|
|
66
|
+
export declare function fetchPackageInfo(courseSlug: string, rigobotToken: string): Promise<Record<string, unknown> | null>;
|