@learnpack/learnpack 5.0.353 → 5.0.354

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/lib/scripts/descriptionsS3Backfill.js +142 -22
  2. package/lib/utils/awsCredentials.d.ts +20 -0
  3. package/lib/utils/awsCredentials.js +43 -0
  4. package/lib/utils/descriptions/backfillEvents.d.ts +60 -0
  5. package/lib/utils/descriptions/backfillEvents.js +107 -0
  6. package/lib/utils/descriptions/generateCourseDescriptions.d.ts +7 -0
  7. package/lib/utils/descriptions/generateCourseDescriptions.js +3 -0
  8. package/lib/utils/descriptions/publishStage.js +14 -3
  9. package/lib/utils/descriptions/resumePublication.js +17 -2
  10. package/lib/utils/descriptions/s3Storage.js +13 -6
  11. package/lib/utils/packageManifest.d.ts +8 -0
  12. package/lib/utils/packageManifest.js +8 -0
  13. package/lib/utils/repair/legacyPackageRepair.d.ts +131 -0
  14. package/lib/utils/repair/legacyPackageRepair.js +492 -0
  15. package/lib/utils/repair/repairStorage.d.ts +68 -0
  16. package/lib/utils/repair/repairStorage.js +89 -0
  17. package/lib/utils/s3/packageManifestBackfill.js +3 -8
  18. package/lib/utils/s3/packageSourcesAudit.d.ts +75 -0
  19. package/lib/utils/s3/packageSourcesAudit.js +184 -0
  20. package/package.json +3 -1
  21. package/src/scripts/README.md +244 -0
  22. package/src/scripts/descriptionsS3Backfill.ts +188 -20
  23. package/src/utils/awsCredentials.ts +57 -0
  24. package/src/utils/descriptions/backfillEvents.ts +152 -0
  25. package/src/utils/descriptions/generateCourseDescriptions.ts +10 -0
  26. package/src/utils/descriptions/publishStage.ts +394 -382
  27. package/src/utils/descriptions/resumePublication.ts +217 -200
  28. package/src/utils/descriptions/s3Storage.ts +214 -206
  29. package/src/utils/packageManifest.ts +8 -1
  30. package/src/utils/repair/legacyPackageRepair.ts +731 -0
  31. package/src/utils/repair/repairStorage.ts +168 -0
  32. package/src/utils/s3/packageManifestBackfill.ts +771 -776
  33. package/src/utils/s3/packageSourcesAudit.ts +311 -0
@@ -1,382 +1,394 @@
1
- import * as fs from "fs"
2
- import * as path from "path"
3
- import { Bucket } from "@google-cloud/storage"
4
- import { Syllabus } from "../../models/creator"
5
- import {
6
- finalizeJournal,
7
- JournalStorage,
8
- markStage,
9
- PublishJournal,
10
- } from "../publishJournal"
11
- import {
12
- CourseDescriptionsStorage,
13
- generateCourseDescriptions,
14
- GenerateCourseDescriptionsOptions,
15
- GenerateCourseDescriptionsResult,
16
- } from "./generateCourseDescriptions"
17
- import { createGcsDescriptionsStorage } from "./gcsStorage"
18
- import { mirrorDescriptionsToDraft, MirrorResult } from "./mirrorDescriptions"
19
- import { createS3DescriptionsStorageFromEnv } from "./s3Storage"
20
- import {
21
- buildPackageManifestUpdatedEvent,
22
- PublishEventContext,
23
- sendPublishEvent,
24
- } from "../publishEvents"
25
- import {
26
- buildDescriptionWorkList,
27
- countWorkItems,
28
- CourseExercise,
29
- CourseReadmes,
30
- } from "./workList"
31
-
32
- /**
33
- * Stage B of a publication: generate the descriptions the new content needs.
34
- *
35
- * It runs in the background, after the HTTP response, because a single
36
- * completion takes ~15s and a course may need several — far past any request
37
- * limit. The publication itself is already finished and durable by then; this
38
- * stage only enriches the published package.
39
- */
40
-
41
- /**
42
- * Whether the freshly built package needs any description, decided from the
43
- * build directory that is still on disk after zipping.
44
- *
45
- * Reading from disk (instead of re-reading GCS or S3) is free and, at this
46
- * exact moment, all three hold the same content: the answer is what the publish
47
- * response promises to breathecode, so it must be cheap enough to compute
48
- * before responding.
49
- */
50
- /** Reads a path relative to the build root; null when it does not exist. */
51
- export type BuildFileReader = (relativePath: string) => string | null;
52
-
53
- export function createBuildFileReader(buildRoot: string): BuildFileReader {
54
- return relativePath => {
55
- try {
56
- return fs.readFileSync(
57
- path.join(buildRoot, ...relativePath.split("/")),
58
- "utf8"
59
- )
60
- } catch {
61
- return null
62
- }
63
- }
64
- }
65
-
66
- export function countPendingDescriptions(
67
- buildRoot: string,
68
- exercises: CourseExercise[],
69
- readFile: BuildFileReader = createBuildFileReader(buildRoot)
70
- ): number {
71
- const rawSyllabus = readFile(".learn/initialSyllabus.json")
72
- if (!rawSyllabus) {
73
- return 0
74
- }
75
-
76
- let syllabus: Syllabus
77
- try {
78
- syllabus = JSON.parse(rawSyllabus) as Syllabus
79
- } catch {
80
- return 0
81
- }
82
-
83
- const readmes: CourseReadmes = {}
84
- for (const exercise of exercises) {
85
- for (const [lang, fileName] of Object.entries(
86
- exercise.translations || {}
87
- )) {
88
- const content = readFile(`exercises/${exercise.slug}/${fileName}`)
89
- if (content === null) {
90
- // Missing translation file: nothing to fingerprint, nothing to ask for.
91
- continue
92
- }
93
-
94
- if (!readmes[exercise.slug]) {
95
- readmes[exercise.slug] = {}
96
- }
97
-
98
- readmes[exercise.slug][lang] = content
99
- }
100
- }
101
-
102
- return countWorkItems(
103
- buildDescriptionWorkList({ syllabus, exercises, readmes })
104
- )
105
- }
106
-
107
- export type PublishDescriptionsStageResult = {
108
- generation: GenerateCourseDescriptionsResult | null;
109
- mirror: MirrorResult | null;
110
- /** False when generation failed outright; drives the event status. */
111
- succeeded: boolean;
112
- };
113
-
114
- /**
115
- * Generate against the published package, then mirror what still matches back
116
- * into the draft. Stages are recorded as they go so an interrupted run can be
117
- * resumed by the sweep.
118
- */
119
- export async function runPublishDescriptionsStage(params: {
120
- courseSlug: string;
121
- rigobotToken: string;
122
- gcsBucket?: Bucket;
123
- journal: PublishJournal;
124
- journalStorage: JournalStorage;
125
- /** Injection points for tests; production resolves them from env/bucket. */
126
- publishedStorage?: CourseDescriptionsStorage;
127
- draftStorage?: CourseDescriptionsStorage;
128
- generate?: GenerateCourseDescriptionsOptions["generate"];
129
- }): Promise<PublishDescriptionsStageResult> {
130
- const { courseSlug, journal, journalStorage } = params
131
- const publishedStorage =
132
- params.publishedStorage ?? createS3DescriptionsStorageFromEnv()
133
-
134
- await markStage(journalStorage, journal, "descriptions", "running")
135
-
136
- let generation: GenerateCourseDescriptionsResult | null = null
137
- try {
138
- generation = await generateCourseDescriptions(
139
- publishedStorage,
140
- courseSlug,
141
- {
142
- token: params.rigobotToken,
143
- reconcile: true,
144
- generate: params.generate,
145
- }
146
- )
147
- } catch (error) {
148
- await markStage(journalStorage, journal, "descriptions", "failed", {
149
- error: (error as Error).message,
150
- })
151
- return { generation: null, mirror: null, succeeded: false }
152
- }
153
-
154
- const succeeded = generation.status !== "failed"
155
- await markStage(
156
- journalStorage,
157
- journal,
158
- "descriptions",
159
- succeeded ? "done" : "failed",
160
- {
161
- error: succeeded ? undefined : generation.errors.join("; "),
162
- meta: {
163
- steps: generation.steps,
164
- generated: generation.generated,
165
- failed: generation.failed,
166
- missing: generation.missing,
167
- durationsSeconds: generation.durationsSeconds,
168
- },
169
- }
170
- )
171
-
172
- if (!succeeded) {
173
- return { generation, mirror: null, succeeded: false }
174
- }
175
-
176
- // Mirror back so the draft keeps the fingerprints: without it the next
177
- // publication would regenerate the whole course.
178
- await markStage(journalStorage, journal, "gcsMirror", "running")
179
- try {
180
- const draftStorage =
181
- params.draftStorage ??
182
- createGcsDescriptionsStorage(params.gcsBucket as Bucket)
183
- const publishedSyllabus = await publishedStorage.readSyllabus(courseSlug)
184
- const mirror = await mirrorDescriptionsToDraft(
185
- draftStorage,
186
- courseSlug,
187
- publishedSyllabus
188
- )
189
- await markStage(journalStorage, journal, "gcsMirror", "done", {
190
- meta: mirror,
191
- })
192
- return { generation, mirror, succeeded: true }
193
- } catch (error) {
194
- // The published package is already correct; only the draft lags behind.
195
- await markStage(journalStorage, journal, "gcsMirror", "failed", {
196
- error: (error as Error).message,
197
- })
198
- return { generation, mirror: null, succeeded: true }
199
- }
200
- }
201
-
202
- /**
203
- * Entry point for the detached run after publishing. Never throws: it is not
204
- * awaited by anyone, so an escaping rejection would only surface as an
205
- * unhandled promise.
206
- */
207
- export async function runDescriptionsStageAfterPublish(params: {
208
- courseSlug: string;
209
- gcsBucket?: Bucket;
210
- journal: PublishJournal;
211
- journalStorage: JournalStorage;
212
- hasWork: boolean;
213
- /** Present when the caller announced `descriptions: "queued"`. */
214
- eventContext?: PublishEventContext;
215
- breathecodeToken?: string;
216
- publishedStorage?: CourseDescriptionsStorage;
217
- draftStorage?: CourseDescriptionsStorage;
218
- generate?: GenerateCourseDescriptionsOptions["generate"];
219
- /** Injection point for tests. */
220
- emit?: typeof sendPublishEvent;
221
- }): Promise<PublishDescriptionsStageResult> {
222
- const { courseSlug, journal, journalStorage } = params
223
- let outcome: PublishDescriptionsStageResult = {
224
- generation: null,
225
- mirror: null,
226
- succeeded: true,
227
- }
228
-
229
- // Resolved on demand: the common "nothing to describe" path must not build
230
- // AWS clients it will never use.
231
- let resolved = params.publishedStorage
232
- const publishedStorage = () => {
233
- if (!resolved) {
234
- resolved = createS3DescriptionsStorageFromEnv()
235
- }
236
-
237
- return resolved
238
- }
239
-
240
- try {
241
- if (!params.hasWork) {
242
- // Nothing changed enough to describe; both stages are settled, and no
243
- // second event was promised, so none is sent.
244
- await markStage(journalStorage, journal, "descriptions", "done", {
245
- meta: { reason: "up_to_date" },
246
- })
247
- await markStage(journalStorage, journal, "gcsMirror", "done", {
248
- meta: { reason: "up_to_date" },
249
- })
250
- await markStage(journalStorage, journal, "manifestEvent", "done", {
251
- meta: { reason: "not_promised" },
252
- })
253
- return outcome
254
- }
255
-
256
- // Descriptions are platform enrichment, not something the publisher is
257
- // billed for, so they always run under the service account.
258
- const token = process.env.RIGOBOT_SYSTEM_TOKEN
259
- if (!token) {
260
- await markStage(journalStorage, journal, "descriptions", "failed", {
261
- error: "RIGOBOT_SYSTEM_TOKEN is not configured",
262
- })
263
- console.error(
264
- `[descriptions] Skipping stage for "${courseSlug}": RIGOBOT_SYSTEM_TOKEN is not configured`
265
- )
266
- outcome = { generation: null, mirror: null, succeeded: false }
267
- return outcome
268
- }
269
-
270
- outcome = await runPublishDescriptionsStage({
271
- courseSlug,
272
- rigobotToken: token,
273
- gcsBucket: params.gcsBucket,
274
- journal,
275
- journalStorage,
276
- publishedStorage: publishedStorage(),
277
- draftStorage: params.draftStorage,
278
- generate: params.generate,
279
- })
280
-
281
- if (outcome.succeeded) {
282
- console.log(
283
- `[descriptions] "${courseSlug}": ${
284
- outcome.generation?.generated ?? 0
285
- } description(s), ${outcome.mirror?.copied ?? 0} mirrored`
286
- )
287
- } else {
288
- console.error(
289
- `[descriptions] "${courseSlug}": stage failed${
290
- outcome.generation?.errors.length ?
291
- ` (${outcome.generation.errors.join("; ")})` :
292
- ""
293
- }`
294
- )
295
- }
296
-
297
- return outcome
298
- } catch (error) {
299
- console.error(
300
- `[descriptions] Stage failed for "${courseSlug}":`,
301
- (error as Error).message
302
- )
303
- outcome = { generation: null, mirror: null, succeeded: false }
304
- return outcome
305
- } finally {
306
- // The contract: a publication that announced `descriptions: "queued"` is
307
- // always followed by exactly one manifest event, success or failure.
308
- // Emitting from `finally` is what makes "exactly one, on every exit path"
309
- // true by construction rather than by remembering to call it.
310
- if (params.hasWork) {
311
- await emitManifestUpdated({
312
- courseSlug,
313
- journal,
314
- journalStorage,
315
- outcome,
316
- eventContext: params.eventContext,
317
- breathecodeToken: params.breathecodeToken,
318
- publishedStorage: publishedStorage(),
319
- emit: params.emit ?? sendPublishEvent,
320
- })
321
- }
322
-
323
- await finalizeJournal(journalStorage, journal)
324
- }
325
- }
326
-
327
- async function emitManifestUpdated(params: {
328
- courseSlug: string;
329
- journal: PublishJournal;
330
- journalStorage: JournalStorage;
331
- outcome: PublishDescriptionsStageResult;
332
- eventContext?: PublishEventContext;
333
- breathecodeToken?: string;
334
- publishedStorage: CourseDescriptionsStorage;
335
- emit: typeof sendPublishEvent;
336
- }): Promise<void> {
337
- const { journal, journalStorage } = params
338
-
339
- if (!params.eventContext || !params.breathecodeToken) {
340
- // Nothing was announced to breathecode, so nothing is owed.
341
- await markStage(journalStorage, journal, "manifestEvent", "done", {
342
- meta: { reason: "no_event_context" },
343
- })
344
- return
345
- }
346
-
347
- try {
348
- // Report the manifest as it is NOW: the descriptions are the whole point of
349
- // this event, so the one captured at publish time would be stale.
350
- const updatedManifest =
351
- (await params.publishedStorage.readManifest?.(params.courseSlug)) ??
352
- params.eventContext.manifest
353
-
354
- const generation = params.outcome.generation
355
- const delivered = await params.emit(
356
- buildPackageManifestUpdatedEvent(
357
- { ...params.eventContext, manifest: updatedManifest },
358
- params.outcome.succeeded ? "success" : "failed",
359
- generation ?
360
- {
361
- generated: generation.generated,
362
- failed: generation.failed,
363
- missing: generation.missing,
364
- } :
365
- undefined
366
- ),
367
- params.breathecodeToken
368
- )
369
-
370
- await markStage(
371
- journalStorage,
372
- journal,
373
- "manifestEvent",
374
- delivered ? "done" : "failed",
375
- { error: delivered ? undefined : "delivery failed" }
376
- )
377
- } catch (error) {
378
- await markStage(journalStorage, journal, "manifestEvent", "failed", {
379
- error: (error as Error).message,
380
- })
381
- }
382
- }
1
+ import * as fs from "fs"
2
+ import * as path from "path"
3
+ import { Bucket } from "@google-cloud/storage"
4
+ import { Syllabus } from "../../models/creator"
5
+ import {
6
+ finalizeJournal,
7
+ JournalStorage,
8
+ markStage,
9
+ PublishJournal,
10
+ } from "../publishJournal"
11
+ import {
12
+ CourseDescriptionsStorage,
13
+ generateCourseDescriptions,
14
+ GenerateCourseDescriptionsOptions,
15
+ GenerateCourseDescriptionsResult,
16
+ } from "./generateCourseDescriptions"
17
+ import { createGcsDescriptionsStorage } from "./gcsStorage"
18
+ import { mirrorDescriptionsToDraft, MirrorResult } from "./mirrorDescriptions"
19
+ import { createS3DescriptionsStorageFromEnv } from "./s3Storage"
20
+ import {
21
+ buildPackageManifestUpdatedEvent,
22
+ PublishEventContext,
23
+ sendPublishEvent,
24
+ } from "../publishEvents"
25
+ import {
26
+ buildDescriptionWorkList,
27
+ countWorkItems,
28
+ CourseExercise,
29
+ CourseReadmes,
30
+ } from "./workList"
31
+
32
+ /**
33
+ * Stage B of a publication: generate the descriptions the new content needs.
34
+ *
35
+ * It runs in the background, after the HTTP response, because a single
36
+ * completion takes ~15s and a course may need several — far past any request
37
+ * limit. The publication itself is already finished and durable by then; this
38
+ * stage only enriches the published package.
39
+ */
40
+
41
+ /**
42
+ * Whether the freshly built package needs any description, decided from the
43
+ * build directory that is still on disk after zipping.
44
+ *
45
+ * Reading from disk (instead of re-reading GCS or S3) is free and, at this
46
+ * exact moment, all three hold the same content: the answer is what the publish
47
+ * response promises to breathecode, so it must be cheap enough to compute
48
+ * before responding.
49
+ */
50
+ /** Reads a path relative to the build root; null when it does not exist. */
51
+ export type BuildFileReader = (relativePath: string) => string | null;
52
+
53
+ export function createBuildFileReader(buildRoot: string): BuildFileReader {
54
+ return relativePath => {
55
+ try {
56
+ return fs.readFileSync(
57
+ path.join(buildRoot, ...relativePath.split("/")),
58
+ "utf8"
59
+ )
60
+ } catch {
61
+ return null
62
+ }
63
+ }
64
+ }
65
+
66
+ export function countPendingDescriptions(
67
+ buildRoot: string,
68
+ exercises: CourseExercise[],
69
+ readFile: BuildFileReader = createBuildFileReader(buildRoot)
70
+ ): number {
71
+ const rawSyllabus = readFile(".learn/initialSyllabus.json")
72
+ if (!rawSyllabus) {
73
+ return 0
74
+ }
75
+
76
+ let syllabus: Syllabus
77
+ try {
78
+ syllabus = JSON.parse(rawSyllabus) as Syllabus
79
+ } catch {
80
+ return 0
81
+ }
82
+
83
+ const readmes: CourseReadmes = {}
84
+ for (const exercise of exercises) {
85
+ for (const [lang, fileName] of Object.entries(
86
+ exercise.translations || {}
87
+ )) {
88
+ const content = readFile(`exercises/${exercise.slug}/${fileName}`)
89
+ if (content === null) {
90
+ // Missing translation file: nothing to fingerprint, nothing to ask for.
91
+ continue
92
+ }
93
+
94
+ if (!readmes[exercise.slug]) {
95
+ readmes[exercise.slug] = {}
96
+ }
97
+
98
+ readmes[exercise.slug][lang] = content
99
+ }
100
+ }
101
+
102
+ return countWorkItems(
103
+ buildDescriptionWorkList({ syllabus, exercises, readmes })
104
+ )
105
+ }
106
+
107
+ export type PublishDescriptionsStageResult = {
108
+ generation: GenerateCourseDescriptionsResult | null;
109
+ mirror: MirrorResult | null;
110
+ /** False when generation failed outright; drives the event status. */
111
+ succeeded: boolean;
112
+ };
113
+
114
+ /**
115
+ * Generate against the published package, then mirror what still matches back
116
+ * into the draft. Stages are recorded as they go so an interrupted run can be
117
+ * resumed by the sweep.
118
+ */
119
+ export async function runPublishDescriptionsStage(params: {
120
+ courseSlug: string;
121
+ rigobotToken: string;
122
+ gcsBucket?: Bucket;
123
+ journal: PublishJournal;
124
+ journalStorage: JournalStorage;
125
+ /** Injection points for tests; production resolves them from env/bucket. */
126
+ publishedStorage?: CourseDescriptionsStorage;
127
+ draftStorage?: CourseDescriptionsStorage;
128
+ generate?: GenerateCourseDescriptionsOptions["generate"];
129
+ }): Promise<PublishDescriptionsStageResult> {
130
+ const { courseSlug, journal, journalStorage } = params
131
+ const publishedStorage =
132
+ params.publishedStorage ?? createS3DescriptionsStorageFromEnv()
133
+
134
+ await markStage(journalStorage, journal, "descriptions", "running")
135
+
136
+ let generation: GenerateCourseDescriptionsResult | null = null
137
+ try {
138
+ generation = await generateCourseDescriptions(
139
+ publishedStorage,
140
+ courseSlug,
141
+ {
142
+ token: params.rigobotToken,
143
+ reconcile: true,
144
+ generate: params.generate,
145
+ }
146
+ )
147
+ } catch (error) {
148
+ await markStage(journalStorage, journal, "descriptions", "failed", {
149
+ error: (error as Error).message,
150
+ })
151
+ return { generation: null, mirror: null, succeeded: false }
152
+ }
153
+
154
+ const succeeded = generation.status !== "failed"
155
+ await markStage(
156
+ journalStorage,
157
+ journal,
158
+ "descriptions",
159
+ succeeded ? "done" : "failed",
160
+ {
161
+ error: succeeded ? undefined : generation.errors.join("; "),
162
+ meta: {
163
+ steps: generation.steps,
164
+ generated: generation.generated,
165
+ failed: generation.failed,
166
+ missing: generation.missing,
167
+ durationsSeconds: generation.durationsSeconds,
168
+ },
169
+ }
170
+ )
171
+
172
+ if (!succeeded) {
173
+ return { generation, mirror: null, succeeded: false }
174
+ }
175
+
176
+ // Mirror back so the draft keeps the fingerprints: without it the next
177
+ // publication would regenerate the whole course.
178
+ await markStage(journalStorage, journal, "gcsMirror", "running")
179
+ try {
180
+ const draftStorage =
181
+ params.draftStorage ??
182
+ createGcsDescriptionsStorage(params.gcsBucket as Bucket)
183
+ const publishedSyllabus = await publishedStorage.readSyllabus(courseSlug)
184
+ const mirror = await mirrorDescriptionsToDraft(
185
+ draftStorage,
186
+ courseSlug,
187
+ publishedSyllabus
188
+ )
189
+ await markStage(journalStorage, journal, "gcsMirror", "done", {
190
+ meta: mirror,
191
+ })
192
+ return { generation, mirror, succeeded: true }
193
+ } catch (error) {
194
+ // The published package is already correct; only the draft lags behind.
195
+ await markStage(journalStorage, journal, "gcsMirror", "failed", {
196
+ error: (error as Error).message,
197
+ })
198
+ return { generation, mirror: null, succeeded: true }
199
+ }
200
+ }
201
+
202
+ /**
203
+ * Entry point for the detached run after publishing. Never throws: it is not
204
+ * awaited by anyone, so an escaping rejection would only surface as an
205
+ * unhandled promise.
206
+ */
207
+ export async function runDescriptionsStageAfterPublish(params: {
208
+ courseSlug: string;
209
+ gcsBucket?: Bucket;
210
+ journal: PublishJournal;
211
+ journalStorage: JournalStorage;
212
+ hasWork: boolean;
213
+ /** Present when the caller announced `descriptions: "queued"`. */
214
+ eventContext?: PublishEventContext;
215
+ breathecodeToken?: string;
216
+ publishedStorage?: CourseDescriptionsStorage;
217
+ draftStorage?: CourseDescriptionsStorage;
218
+ generate?: GenerateCourseDescriptionsOptions["generate"];
219
+ /** Injection point for tests. */
220
+ emit?: typeof sendPublishEvent;
221
+ }): Promise<PublishDescriptionsStageResult> {
222
+ const { courseSlug, journal, journalStorage } = params
223
+ let outcome: PublishDescriptionsStageResult = {
224
+ generation: null,
225
+ mirror: null,
226
+ succeeded: true,
227
+ }
228
+
229
+ // Resolved on demand: the common "nothing to describe" path must not build
230
+ // AWS clients it will never use.
231
+ let resolved = params.publishedStorage
232
+ const publishedStorage = () => {
233
+ if (!resolved) {
234
+ resolved = createS3DescriptionsStorageFromEnv()
235
+ }
236
+
237
+ return resolved
238
+ }
239
+
240
+ try {
241
+ if (!params.hasWork) {
242
+ // Nothing changed enough to describe; both stages are settled, and no
243
+ // second event was promised, so none is sent.
244
+ await markStage(journalStorage, journal, "descriptions", "done", {
245
+ meta: { reason: "up_to_date" },
246
+ })
247
+ await markStage(journalStorage, journal, "gcsMirror", "done", {
248
+ meta: { reason: "up_to_date" },
249
+ })
250
+ await markStage(journalStorage, journal, "manifestEvent", "done", {
251
+ meta: { reason: "not_promised" },
252
+ })
253
+ return outcome
254
+ }
255
+
256
+ // Descriptions are platform enrichment, not something the publisher is
257
+ // billed for, so they always run under the service account.
258
+ const token = process.env.RIGOBOT_SYSTEM_TOKEN
259
+ if (!token) {
260
+ await markStage(journalStorage, journal, "descriptions", "failed", {
261
+ error: "RIGOBOT_SYSTEM_TOKEN is not configured",
262
+ })
263
+ console.error(
264
+ `[descriptions] Skipping stage for "${courseSlug}": RIGOBOT_SYSTEM_TOKEN is not configured`
265
+ )
266
+ outcome = { generation: null, mirror: null, succeeded: false }
267
+ return outcome
268
+ }
269
+
270
+ outcome = await runPublishDescriptionsStage({
271
+ courseSlug,
272
+ rigobotToken: token,
273
+ gcsBucket: params.gcsBucket,
274
+ journal,
275
+ journalStorage,
276
+ publishedStorage: publishedStorage(),
277
+ draftStorage: params.draftStorage,
278
+ generate: params.generate,
279
+ })
280
+
281
+ if (outcome.succeeded) {
282
+ console.log(
283
+ `[descriptions] "${courseSlug}": ${
284
+ outcome.generation?.generated ?? 0
285
+ } description(s), ${outcome.mirror?.copied ?? 0} mirrored`
286
+ )
287
+ } else {
288
+ console.error(
289
+ `[descriptions] "${courseSlug}": stage failed${
290
+ outcome.generation?.errors.length ?
291
+ ` (${outcome.generation.errors.join("; ")})` :
292
+ ""
293
+ }`
294
+ )
295
+ }
296
+
297
+ return outcome
298
+ } catch (error) {
299
+ console.error(
300
+ `[descriptions] Stage failed for "${courseSlug}":`,
301
+ (error as Error).message
302
+ )
303
+ outcome = { generation: null, mirror: null, succeeded: false }
304
+ return outcome
305
+ } finally {
306
+ // The contract: a publication that announced `descriptions: "queued"` is
307
+ // always followed by exactly one manifest event, success or failure.
308
+ // Emitting from `finally` is what makes "exactly one, on every exit path"
309
+ // true by construction rather than by remembering to call it.
310
+ if (params.hasWork) {
311
+ await emitManifestUpdated({
312
+ courseSlug,
313
+ journal,
314
+ journalStorage,
315
+ outcome,
316
+ eventContext: params.eventContext,
317
+ breathecodeToken: params.breathecodeToken,
318
+ publishedStorage: publishedStorage(),
319
+ emit: params.emit ?? sendPublishEvent,
320
+ })
321
+ }
322
+
323
+ await finalizeJournal(journalStorage, journal)
324
+ }
325
+ }
326
+
327
+ async function emitManifestUpdated(params: {
328
+ courseSlug: string;
329
+ journal: PublishJournal;
330
+ journalStorage: JournalStorage;
331
+ outcome: PublishDescriptionsStageResult;
332
+ eventContext?: PublishEventContext;
333
+ breathecodeToken?: string;
334
+ publishedStorage: CourseDescriptionsStorage;
335
+ emit: typeof sendPublishEvent;
336
+ }): Promise<void> {
337
+ const { journal, journalStorage } = params
338
+
339
+ if (!params.eventContext || !params.breathecodeToken) {
340
+ // Nothing was announced to breathecode, so nothing is owed.
341
+ await markStage(journalStorage, journal, "manifestEvent", "done", {
342
+ meta: { reason: "no_event_context" },
343
+ })
344
+ return
345
+ }
346
+
347
+ // Enriching the payload must never prevent the delivery. The manifest is
348
+ // optional in the contract (`manifest: null` is valid); the event is not:
349
+ // breathecode was promised one and is waiting for it. Reading the manifest
350
+ // from the published bucket can fail for reasons that have nothing to do with
351
+ // the event — a bad AWS credential, say — and an event with less context still
352
+ // beats an event that never arrives.
353
+ let updatedManifest = params.eventContext.manifest
354
+ try {
355
+ updatedManifest =
356
+ (await params.publishedStorage.readManifest?.(params.courseSlug)) ??
357
+ params.eventContext.manifest
358
+ } catch (error) {
359
+ console.error(
360
+ `[descriptions] Could not read the updated manifest for "${params.courseSlug}", reporting without it:`,
361
+ (error as Error).message
362
+ )
363
+ }
364
+
365
+ try {
366
+ const generation = params.outcome.generation
367
+ const delivered = await params.emit(
368
+ buildPackageManifestUpdatedEvent(
369
+ { ...params.eventContext, manifest: updatedManifest },
370
+ params.outcome.succeeded ? "success" : "failed",
371
+ generation ?
372
+ {
373
+ generated: generation.generated,
374
+ failed: generation.failed,
375
+ missing: generation.missing,
376
+ } :
377
+ undefined
378
+ ),
379
+ params.breathecodeToken
380
+ )
381
+
382
+ await markStage(
383
+ journalStorage,
384
+ journal,
385
+ "manifestEvent",
386
+ delivered ? "done" : "failed",
387
+ { error: delivered ? undefined : "delivery failed" }
388
+ )
389
+ } catch (error) {
390
+ await markStage(journalStorage, journal, "manifestEvent", "failed", {
391
+ error: (error as Error).message,
392
+ })
393
+ }
394
+ }