@learnpack/learnpack 5.0.353 → 5.0.355
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/commands/serve.js +1 -1
- package/lib/scripts/descriptionsS3Backfill.js +142 -22
- package/lib/utils/awsCredentials.d.ts +20 -0
- package/lib/utils/awsCredentials.js +43 -0
- package/lib/utils/descriptions/backfillEvents.d.ts +60 -0
- package/lib/utils/descriptions/backfillEvents.js +107 -0
- package/lib/utils/descriptions/generateCourseDescriptions.d.ts +7 -0
- package/lib/utils/descriptions/generateCourseDescriptions.js +3 -0
- package/lib/utils/descriptions/publishStage.js +14 -3
- package/lib/utils/descriptions/resumePublication.js +17 -2
- package/lib/utils/descriptions/s3Storage.js +13 -6
- package/lib/utils/packageManifest.d.ts +8 -0
- package/lib/utils/packageManifest.js +8 -0
- package/lib/utils/repair/legacyPackageRepair.d.ts +131 -0
- package/lib/utils/repair/legacyPackageRepair.js +492 -0
- package/lib/utils/repair/repairStorage.d.ts +68 -0
- package/lib/utils/repair/repairStorage.js +89 -0
- package/lib/utils/s3/packageManifestBackfill.js +3 -8
- package/lib/utils/s3/packageSourcesAudit.d.ts +75 -0
- package/lib/utils/s3/packageSourcesAudit.js +184 -0
- package/package.json +3 -1
- package/src/commands/serve.ts +1 -1
- package/src/scripts/README.md +244 -0
- package/src/scripts/descriptionsS3Backfill.ts +188 -20
- package/src/utils/awsCredentials.ts +57 -0
- package/src/utils/descriptions/backfillEvents.ts +152 -0
- package/src/utils/descriptions/generateCourseDescriptions.ts +10 -0
- package/src/utils/descriptions/publishStage.ts +394 -382
- package/src/utils/descriptions/resumePublication.ts +217 -200
- package/src/utils/descriptions/s3Storage.ts +214 -206
- package/src/utils/packageManifest.ts +8 -1
- package/src/utils/repair/legacyPackageRepair.ts +731 -0
- package/src/utils/repair/repairStorage.ts +168 -0
- package/src/utils/s3/packageManifestBackfill.ts +771 -776
- package/src/utils/s3/packageSourcesAudit.ts +311 -0
|
@@ -1,382 +1,394 @@
|
|
|
1
|
-
import * as fs from "fs"
|
|
2
|
-
import * as path from "path"
|
|
3
|
-
import { Bucket } from "@google-cloud/storage"
|
|
4
|
-
import { Syllabus } from "../../models/creator"
|
|
5
|
-
import {
|
|
6
|
-
finalizeJournal,
|
|
7
|
-
JournalStorage,
|
|
8
|
-
markStage,
|
|
9
|
-
PublishJournal,
|
|
10
|
-
} from "../publishJournal"
|
|
11
|
-
import {
|
|
12
|
-
CourseDescriptionsStorage,
|
|
13
|
-
generateCourseDescriptions,
|
|
14
|
-
GenerateCourseDescriptionsOptions,
|
|
15
|
-
GenerateCourseDescriptionsResult,
|
|
16
|
-
} from "./generateCourseDescriptions"
|
|
17
|
-
import { createGcsDescriptionsStorage } from "./gcsStorage"
|
|
18
|
-
import { mirrorDescriptionsToDraft, MirrorResult } from "./mirrorDescriptions"
|
|
19
|
-
import { createS3DescriptionsStorageFromEnv } from "./s3Storage"
|
|
20
|
-
import {
|
|
21
|
-
buildPackageManifestUpdatedEvent,
|
|
22
|
-
PublishEventContext,
|
|
23
|
-
sendPublishEvent,
|
|
24
|
-
} from "../publishEvents"
|
|
25
|
-
import {
|
|
26
|
-
buildDescriptionWorkList,
|
|
27
|
-
countWorkItems,
|
|
28
|
-
CourseExercise,
|
|
29
|
-
CourseReadmes,
|
|
30
|
-
} from "./workList"
|
|
31
|
-
|
|
32
|
-
/**
|
|
33
|
-
* Stage B of a publication: generate the descriptions the new content needs.
|
|
34
|
-
*
|
|
35
|
-
* It runs in the background, after the HTTP response, because a single
|
|
36
|
-
* completion takes ~15s and a course may need several — far past any request
|
|
37
|
-
* limit. The publication itself is already finished and durable by then; this
|
|
38
|
-
* stage only enriches the published package.
|
|
39
|
-
*/
|
|
40
|
-
|
|
41
|
-
/**
|
|
42
|
-
* Whether the freshly built package needs any description, decided from the
|
|
43
|
-
* build directory that is still on disk after zipping.
|
|
44
|
-
*
|
|
45
|
-
* Reading from disk (instead of re-reading GCS or S3) is free and, at this
|
|
46
|
-
* exact moment, all three hold the same content: the answer is what the publish
|
|
47
|
-
* response promises to breathecode, so it must be cheap enough to compute
|
|
48
|
-
* before responding.
|
|
49
|
-
*/
|
|
50
|
-
/** Reads a path relative to the build root; null when it does not exist. */
|
|
51
|
-
export type BuildFileReader = (relativePath: string) => string | null;
|
|
52
|
-
|
|
53
|
-
export function createBuildFileReader(buildRoot: string): BuildFileReader {
|
|
54
|
-
return relativePath => {
|
|
55
|
-
try {
|
|
56
|
-
return fs.readFileSync(
|
|
57
|
-
path.join(buildRoot, ...relativePath.split("/")),
|
|
58
|
-
"utf8"
|
|
59
|
-
)
|
|
60
|
-
} catch {
|
|
61
|
-
return null
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
export function countPendingDescriptions(
|
|
67
|
-
buildRoot: string,
|
|
68
|
-
exercises: CourseExercise[],
|
|
69
|
-
readFile: BuildFileReader = createBuildFileReader(buildRoot)
|
|
70
|
-
): number {
|
|
71
|
-
const rawSyllabus = readFile(".learn/initialSyllabus.json")
|
|
72
|
-
if (!rawSyllabus) {
|
|
73
|
-
return 0
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
let syllabus: Syllabus
|
|
77
|
-
try {
|
|
78
|
-
syllabus = JSON.parse(rawSyllabus) as Syllabus
|
|
79
|
-
} catch {
|
|
80
|
-
return 0
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
const readmes: CourseReadmes = {}
|
|
84
|
-
for (const exercise of exercises) {
|
|
85
|
-
for (const [lang, fileName] of Object.entries(
|
|
86
|
-
exercise.translations || {}
|
|
87
|
-
)) {
|
|
88
|
-
const content = readFile(`exercises/${exercise.slug}/${fileName}`)
|
|
89
|
-
if (content === null) {
|
|
90
|
-
// Missing translation file: nothing to fingerprint, nothing to ask for.
|
|
91
|
-
continue
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
if (!readmes[exercise.slug]) {
|
|
95
|
-
readmes[exercise.slug] = {}
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
readmes[exercise.slug][lang] = content
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
return countWorkItems(
|
|
103
|
-
buildDescriptionWorkList({ syllabus, exercises, readmes })
|
|
104
|
-
)
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
export type PublishDescriptionsStageResult = {
|
|
108
|
-
generation: GenerateCourseDescriptionsResult | null;
|
|
109
|
-
mirror: MirrorResult | null;
|
|
110
|
-
/** False when generation failed outright; drives the event status. */
|
|
111
|
-
succeeded: boolean;
|
|
112
|
-
};
|
|
113
|
-
|
|
114
|
-
/**
|
|
115
|
-
* Generate against the published package, then mirror what still matches back
|
|
116
|
-
* into the draft. Stages are recorded as they go so an interrupted run can be
|
|
117
|
-
* resumed by the sweep.
|
|
118
|
-
*/
|
|
119
|
-
export async function runPublishDescriptionsStage(params: {
|
|
120
|
-
courseSlug: string;
|
|
121
|
-
rigobotToken: string;
|
|
122
|
-
gcsBucket?: Bucket;
|
|
123
|
-
journal: PublishJournal;
|
|
124
|
-
journalStorage: JournalStorage;
|
|
125
|
-
/** Injection points for tests; production resolves them from env/bucket. */
|
|
126
|
-
publishedStorage?: CourseDescriptionsStorage;
|
|
127
|
-
draftStorage?: CourseDescriptionsStorage;
|
|
128
|
-
generate?: GenerateCourseDescriptionsOptions["generate"];
|
|
129
|
-
}): Promise<PublishDescriptionsStageResult> {
|
|
130
|
-
const { courseSlug, journal, journalStorage } = params
|
|
131
|
-
const publishedStorage =
|
|
132
|
-
params.publishedStorage ?? createS3DescriptionsStorageFromEnv()
|
|
133
|
-
|
|
134
|
-
await markStage(journalStorage, journal, "descriptions", "running")
|
|
135
|
-
|
|
136
|
-
let generation: GenerateCourseDescriptionsResult | null = null
|
|
137
|
-
try {
|
|
138
|
-
generation = await generateCourseDescriptions(
|
|
139
|
-
publishedStorage,
|
|
140
|
-
courseSlug,
|
|
141
|
-
{
|
|
142
|
-
token: params.rigobotToken,
|
|
143
|
-
reconcile: true,
|
|
144
|
-
generate: params.generate,
|
|
145
|
-
}
|
|
146
|
-
)
|
|
147
|
-
} catch (error) {
|
|
148
|
-
await markStage(journalStorage, journal, "descriptions", "failed", {
|
|
149
|
-
error: (error as Error).message,
|
|
150
|
-
})
|
|
151
|
-
return { generation: null, mirror: null, succeeded: false }
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
const succeeded = generation.status !== "failed"
|
|
155
|
-
await markStage(
|
|
156
|
-
journalStorage,
|
|
157
|
-
journal,
|
|
158
|
-
"descriptions",
|
|
159
|
-
succeeded ? "done" : "failed",
|
|
160
|
-
{
|
|
161
|
-
error: succeeded ? undefined : generation.errors.join("; "),
|
|
162
|
-
meta: {
|
|
163
|
-
steps: generation.steps,
|
|
164
|
-
generated: generation.generated,
|
|
165
|
-
failed: generation.failed,
|
|
166
|
-
missing: generation.missing,
|
|
167
|
-
durationsSeconds: generation.durationsSeconds,
|
|
168
|
-
},
|
|
169
|
-
}
|
|
170
|
-
)
|
|
171
|
-
|
|
172
|
-
if (!succeeded) {
|
|
173
|
-
return { generation, mirror: null, succeeded: false }
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
// Mirror back so the draft keeps the fingerprints: without it the next
|
|
177
|
-
// publication would regenerate the whole course.
|
|
178
|
-
await markStage(journalStorage, journal, "gcsMirror", "running")
|
|
179
|
-
try {
|
|
180
|
-
const draftStorage =
|
|
181
|
-
params.draftStorage ??
|
|
182
|
-
createGcsDescriptionsStorage(params.gcsBucket as Bucket)
|
|
183
|
-
const publishedSyllabus = await publishedStorage.readSyllabus(courseSlug)
|
|
184
|
-
const mirror = await mirrorDescriptionsToDraft(
|
|
185
|
-
draftStorage,
|
|
186
|
-
courseSlug,
|
|
187
|
-
publishedSyllabus
|
|
188
|
-
)
|
|
189
|
-
await markStage(journalStorage, journal, "gcsMirror", "done", {
|
|
190
|
-
meta: mirror,
|
|
191
|
-
})
|
|
192
|
-
return { generation, mirror, succeeded: true }
|
|
193
|
-
} catch (error) {
|
|
194
|
-
// The published package is already correct; only the draft lags behind.
|
|
195
|
-
await markStage(journalStorage, journal, "gcsMirror", "failed", {
|
|
196
|
-
error: (error as Error).message,
|
|
197
|
-
})
|
|
198
|
-
return { generation, mirror: null, succeeded: true }
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
/**
|
|
203
|
-
* Entry point for the detached run after publishing. Never throws: it is not
|
|
204
|
-
* awaited by anyone, so an escaping rejection would only surface as an
|
|
205
|
-
* unhandled promise.
|
|
206
|
-
*/
|
|
207
|
-
export async function runDescriptionsStageAfterPublish(params: {
|
|
208
|
-
courseSlug: string;
|
|
209
|
-
gcsBucket?: Bucket;
|
|
210
|
-
journal: PublishJournal;
|
|
211
|
-
journalStorage: JournalStorage;
|
|
212
|
-
hasWork: boolean;
|
|
213
|
-
/** Present when the caller announced `descriptions: "queued"`. */
|
|
214
|
-
eventContext?: PublishEventContext;
|
|
215
|
-
breathecodeToken?: string;
|
|
216
|
-
publishedStorage?: CourseDescriptionsStorage;
|
|
217
|
-
draftStorage?: CourseDescriptionsStorage;
|
|
218
|
-
generate?: GenerateCourseDescriptionsOptions["generate"];
|
|
219
|
-
/** Injection point for tests. */
|
|
220
|
-
emit?: typeof sendPublishEvent;
|
|
221
|
-
}): Promise<PublishDescriptionsStageResult> {
|
|
222
|
-
const { courseSlug, journal, journalStorage } = params
|
|
223
|
-
let outcome: PublishDescriptionsStageResult = {
|
|
224
|
-
generation: null,
|
|
225
|
-
mirror: null,
|
|
226
|
-
succeeded: true,
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
// Resolved on demand: the common "nothing to describe" path must not build
|
|
230
|
-
// AWS clients it will never use.
|
|
231
|
-
let resolved = params.publishedStorage
|
|
232
|
-
const publishedStorage = () => {
|
|
233
|
-
if (!resolved) {
|
|
234
|
-
resolved = createS3DescriptionsStorageFromEnv()
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
return resolved
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
try {
|
|
241
|
-
if (!params.hasWork) {
|
|
242
|
-
// Nothing changed enough to describe; both stages are settled, and no
|
|
243
|
-
// second event was promised, so none is sent.
|
|
244
|
-
await markStage(journalStorage, journal, "descriptions", "done", {
|
|
245
|
-
meta: { reason: "up_to_date" },
|
|
246
|
-
})
|
|
247
|
-
await markStage(journalStorage, journal, "gcsMirror", "done", {
|
|
248
|
-
meta: { reason: "up_to_date" },
|
|
249
|
-
})
|
|
250
|
-
await markStage(journalStorage, journal, "manifestEvent", "done", {
|
|
251
|
-
meta: { reason: "not_promised" },
|
|
252
|
-
})
|
|
253
|
-
return outcome
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
// Descriptions are platform enrichment, not something the publisher is
|
|
257
|
-
// billed for, so they always run under the service account.
|
|
258
|
-
const token = process.env.RIGOBOT_SYSTEM_TOKEN
|
|
259
|
-
if (!token) {
|
|
260
|
-
await markStage(journalStorage, journal, "descriptions", "failed", {
|
|
261
|
-
error: "RIGOBOT_SYSTEM_TOKEN is not configured",
|
|
262
|
-
})
|
|
263
|
-
console.error(
|
|
264
|
-
`[descriptions] Skipping stage for "${courseSlug}": RIGOBOT_SYSTEM_TOKEN is not configured`
|
|
265
|
-
)
|
|
266
|
-
outcome = { generation: null, mirror: null, succeeded: false }
|
|
267
|
-
return outcome
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
outcome = await runPublishDescriptionsStage({
|
|
271
|
-
courseSlug,
|
|
272
|
-
rigobotToken: token,
|
|
273
|
-
gcsBucket: params.gcsBucket,
|
|
274
|
-
journal,
|
|
275
|
-
journalStorage,
|
|
276
|
-
publishedStorage: publishedStorage(),
|
|
277
|
-
draftStorage: params.draftStorage,
|
|
278
|
-
generate: params.generate,
|
|
279
|
-
})
|
|
280
|
-
|
|
281
|
-
if (outcome.succeeded) {
|
|
282
|
-
console.log(
|
|
283
|
-
`[descriptions] "${courseSlug}": ${
|
|
284
|
-
outcome.generation?.generated ?? 0
|
|
285
|
-
} description(s), ${outcome.mirror?.copied ?? 0} mirrored`
|
|
286
|
-
)
|
|
287
|
-
} else {
|
|
288
|
-
console.error(
|
|
289
|
-
`[descriptions] "${courseSlug}": stage failed${
|
|
290
|
-
outcome.generation?.errors.length ?
|
|
291
|
-
` (${outcome.generation.errors.join("; ")})` :
|
|
292
|
-
""
|
|
293
|
-
}`
|
|
294
|
-
)
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
return outcome
|
|
298
|
-
} catch (error) {
|
|
299
|
-
console.error(
|
|
300
|
-
`[descriptions] Stage failed for "${courseSlug}":`,
|
|
301
|
-
(error as Error).message
|
|
302
|
-
)
|
|
303
|
-
outcome = { generation: null, mirror: null, succeeded: false }
|
|
304
|
-
return outcome
|
|
305
|
-
} finally {
|
|
306
|
-
// The contract: a publication that announced `descriptions: "queued"` is
|
|
307
|
-
// always followed by exactly one manifest event, success or failure.
|
|
308
|
-
// Emitting from `finally` is what makes "exactly one, on every exit path"
|
|
309
|
-
// true by construction rather than by remembering to call it.
|
|
310
|
-
if (params.hasWork) {
|
|
311
|
-
await emitManifestUpdated({
|
|
312
|
-
courseSlug,
|
|
313
|
-
journal,
|
|
314
|
-
journalStorage,
|
|
315
|
-
outcome,
|
|
316
|
-
eventContext: params.eventContext,
|
|
317
|
-
breathecodeToken: params.breathecodeToken,
|
|
318
|
-
publishedStorage: publishedStorage(),
|
|
319
|
-
emit: params.emit ?? sendPublishEvent,
|
|
320
|
-
})
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
await finalizeJournal(journalStorage, journal)
|
|
324
|
-
}
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
async function emitManifestUpdated(params: {
|
|
328
|
-
courseSlug: string;
|
|
329
|
-
journal: PublishJournal;
|
|
330
|
-
journalStorage: JournalStorage;
|
|
331
|
-
outcome: PublishDescriptionsStageResult;
|
|
332
|
-
eventContext?: PublishEventContext;
|
|
333
|
-
breathecodeToken?: string;
|
|
334
|
-
publishedStorage: CourseDescriptionsStorage;
|
|
335
|
-
emit: typeof sendPublishEvent;
|
|
336
|
-
}): Promise<void> {
|
|
337
|
-
const { journal, journalStorage } = params
|
|
338
|
-
|
|
339
|
-
if (!params.eventContext || !params.breathecodeToken) {
|
|
340
|
-
// Nothing was announced to breathecode, so nothing is owed.
|
|
341
|
-
await markStage(journalStorage, journal, "manifestEvent", "done", {
|
|
342
|
-
meta: { reason: "no_event_context" },
|
|
343
|
-
})
|
|
344
|
-
return
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
1
|
+
import * as fs from "fs"
|
|
2
|
+
import * as path from "path"
|
|
3
|
+
import { Bucket } from "@google-cloud/storage"
|
|
4
|
+
import { Syllabus } from "../../models/creator"
|
|
5
|
+
import {
|
|
6
|
+
finalizeJournal,
|
|
7
|
+
JournalStorage,
|
|
8
|
+
markStage,
|
|
9
|
+
PublishJournal,
|
|
10
|
+
} from "../publishJournal"
|
|
11
|
+
import {
|
|
12
|
+
CourseDescriptionsStorage,
|
|
13
|
+
generateCourseDescriptions,
|
|
14
|
+
GenerateCourseDescriptionsOptions,
|
|
15
|
+
GenerateCourseDescriptionsResult,
|
|
16
|
+
} from "./generateCourseDescriptions"
|
|
17
|
+
import { createGcsDescriptionsStorage } from "./gcsStorage"
|
|
18
|
+
import { mirrorDescriptionsToDraft, MirrorResult } from "./mirrorDescriptions"
|
|
19
|
+
import { createS3DescriptionsStorageFromEnv } from "./s3Storage"
|
|
20
|
+
import {
|
|
21
|
+
buildPackageManifestUpdatedEvent,
|
|
22
|
+
PublishEventContext,
|
|
23
|
+
sendPublishEvent,
|
|
24
|
+
} from "../publishEvents"
|
|
25
|
+
import {
|
|
26
|
+
buildDescriptionWorkList,
|
|
27
|
+
countWorkItems,
|
|
28
|
+
CourseExercise,
|
|
29
|
+
CourseReadmes,
|
|
30
|
+
} from "./workList"
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Stage B of a publication: generate the descriptions the new content needs.
|
|
34
|
+
*
|
|
35
|
+
* It runs in the background, after the HTTP response, because a single
|
|
36
|
+
* completion takes ~15s and a course may need several — far past any request
|
|
37
|
+
* limit. The publication itself is already finished and durable by then; this
|
|
38
|
+
* stage only enriches the published package.
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Whether the freshly built package needs any description, decided from the
|
|
43
|
+
* build directory that is still on disk after zipping.
|
|
44
|
+
*
|
|
45
|
+
* Reading from disk (instead of re-reading GCS or S3) is free and, at this
|
|
46
|
+
* exact moment, all three hold the same content: the answer is what the publish
|
|
47
|
+
* response promises to breathecode, so it must be cheap enough to compute
|
|
48
|
+
* before responding.
|
|
49
|
+
*/
|
|
50
|
+
/** Reads a path relative to the build root; null when it does not exist. */
|
|
51
|
+
export type BuildFileReader = (relativePath: string) => string | null;
|
|
52
|
+
|
|
53
|
+
export function createBuildFileReader(buildRoot: string): BuildFileReader {
|
|
54
|
+
return relativePath => {
|
|
55
|
+
try {
|
|
56
|
+
return fs.readFileSync(
|
|
57
|
+
path.join(buildRoot, ...relativePath.split("/")),
|
|
58
|
+
"utf8"
|
|
59
|
+
)
|
|
60
|
+
} catch {
|
|
61
|
+
return null
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function countPendingDescriptions(
|
|
67
|
+
buildRoot: string,
|
|
68
|
+
exercises: CourseExercise[],
|
|
69
|
+
readFile: BuildFileReader = createBuildFileReader(buildRoot)
|
|
70
|
+
): number {
|
|
71
|
+
const rawSyllabus = readFile(".learn/initialSyllabus.json")
|
|
72
|
+
if (!rawSyllabus) {
|
|
73
|
+
return 0
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
let syllabus: Syllabus
|
|
77
|
+
try {
|
|
78
|
+
syllabus = JSON.parse(rawSyllabus) as Syllabus
|
|
79
|
+
} catch {
|
|
80
|
+
return 0
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const readmes: CourseReadmes = {}
|
|
84
|
+
for (const exercise of exercises) {
|
|
85
|
+
for (const [lang, fileName] of Object.entries(
|
|
86
|
+
exercise.translations || {}
|
|
87
|
+
)) {
|
|
88
|
+
const content = readFile(`exercises/${exercise.slug}/${fileName}`)
|
|
89
|
+
if (content === null) {
|
|
90
|
+
// Missing translation file: nothing to fingerprint, nothing to ask for.
|
|
91
|
+
continue
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
if (!readmes[exercise.slug]) {
|
|
95
|
+
readmes[exercise.slug] = {}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
readmes[exercise.slug][lang] = content
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
return countWorkItems(
|
|
103
|
+
buildDescriptionWorkList({ syllabus, exercises, readmes })
|
|
104
|
+
)
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export type PublishDescriptionsStageResult = {
|
|
108
|
+
generation: GenerateCourseDescriptionsResult | null;
|
|
109
|
+
mirror: MirrorResult | null;
|
|
110
|
+
/** False when generation failed outright; drives the event status. */
|
|
111
|
+
succeeded: boolean;
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Generate against the published package, then mirror what still matches back
|
|
116
|
+
* into the draft. Stages are recorded as they go so an interrupted run can be
|
|
117
|
+
* resumed by the sweep.
|
|
118
|
+
*/
|
|
119
|
+
export async function runPublishDescriptionsStage(params: {
|
|
120
|
+
courseSlug: string;
|
|
121
|
+
rigobotToken: string;
|
|
122
|
+
gcsBucket?: Bucket;
|
|
123
|
+
journal: PublishJournal;
|
|
124
|
+
journalStorage: JournalStorage;
|
|
125
|
+
/** Injection points for tests; production resolves them from env/bucket. */
|
|
126
|
+
publishedStorage?: CourseDescriptionsStorage;
|
|
127
|
+
draftStorage?: CourseDescriptionsStorage;
|
|
128
|
+
generate?: GenerateCourseDescriptionsOptions["generate"];
|
|
129
|
+
}): Promise<PublishDescriptionsStageResult> {
|
|
130
|
+
const { courseSlug, journal, journalStorage } = params
|
|
131
|
+
const publishedStorage =
|
|
132
|
+
params.publishedStorage ?? createS3DescriptionsStorageFromEnv()
|
|
133
|
+
|
|
134
|
+
await markStage(journalStorage, journal, "descriptions", "running")
|
|
135
|
+
|
|
136
|
+
let generation: GenerateCourseDescriptionsResult | null = null
|
|
137
|
+
try {
|
|
138
|
+
generation = await generateCourseDescriptions(
|
|
139
|
+
publishedStorage,
|
|
140
|
+
courseSlug,
|
|
141
|
+
{
|
|
142
|
+
token: params.rigobotToken,
|
|
143
|
+
reconcile: true,
|
|
144
|
+
generate: params.generate,
|
|
145
|
+
}
|
|
146
|
+
)
|
|
147
|
+
} catch (error) {
|
|
148
|
+
await markStage(journalStorage, journal, "descriptions", "failed", {
|
|
149
|
+
error: (error as Error).message,
|
|
150
|
+
})
|
|
151
|
+
return { generation: null, mirror: null, succeeded: false }
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const succeeded = generation.status !== "failed"
|
|
155
|
+
await markStage(
|
|
156
|
+
journalStorage,
|
|
157
|
+
journal,
|
|
158
|
+
"descriptions",
|
|
159
|
+
succeeded ? "done" : "failed",
|
|
160
|
+
{
|
|
161
|
+
error: succeeded ? undefined : generation.errors.join("; "),
|
|
162
|
+
meta: {
|
|
163
|
+
steps: generation.steps,
|
|
164
|
+
generated: generation.generated,
|
|
165
|
+
failed: generation.failed,
|
|
166
|
+
missing: generation.missing,
|
|
167
|
+
durationsSeconds: generation.durationsSeconds,
|
|
168
|
+
},
|
|
169
|
+
}
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
if (!succeeded) {
|
|
173
|
+
return { generation, mirror: null, succeeded: false }
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// Mirror back so the draft keeps the fingerprints: without it the next
|
|
177
|
+
// publication would regenerate the whole course.
|
|
178
|
+
await markStage(journalStorage, journal, "gcsMirror", "running")
|
|
179
|
+
try {
|
|
180
|
+
const draftStorage =
|
|
181
|
+
params.draftStorage ??
|
|
182
|
+
createGcsDescriptionsStorage(params.gcsBucket as Bucket)
|
|
183
|
+
const publishedSyllabus = await publishedStorage.readSyllabus(courseSlug)
|
|
184
|
+
const mirror = await mirrorDescriptionsToDraft(
|
|
185
|
+
draftStorage,
|
|
186
|
+
courseSlug,
|
|
187
|
+
publishedSyllabus
|
|
188
|
+
)
|
|
189
|
+
await markStage(journalStorage, journal, "gcsMirror", "done", {
|
|
190
|
+
meta: mirror,
|
|
191
|
+
})
|
|
192
|
+
return { generation, mirror, succeeded: true }
|
|
193
|
+
} catch (error) {
|
|
194
|
+
// The published package is already correct; only the draft lags behind.
|
|
195
|
+
await markStage(journalStorage, journal, "gcsMirror", "failed", {
|
|
196
|
+
error: (error as Error).message,
|
|
197
|
+
})
|
|
198
|
+
return { generation, mirror: null, succeeded: true }
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Entry point for the detached run after publishing. Never throws: it is not
|
|
204
|
+
* awaited by anyone, so an escaping rejection would only surface as an
|
|
205
|
+
* unhandled promise.
|
|
206
|
+
*/
|
|
207
|
+
export async function runDescriptionsStageAfterPublish(params: {
|
|
208
|
+
courseSlug: string;
|
|
209
|
+
gcsBucket?: Bucket;
|
|
210
|
+
journal: PublishJournal;
|
|
211
|
+
journalStorage: JournalStorage;
|
|
212
|
+
hasWork: boolean;
|
|
213
|
+
/** Present when the caller announced `descriptions: "queued"`. */
|
|
214
|
+
eventContext?: PublishEventContext;
|
|
215
|
+
breathecodeToken?: string;
|
|
216
|
+
publishedStorage?: CourseDescriptionsStorage;
|
|
217
|
+
draftStorage?: CourseDescriptionsStorage;
|
|
218
|
+
generate?: GenerateCourseDescriptionsOptions["generate"];
|
|
219
|
+
/** Injection point for tests. */
|
|
220
|
+
emit?: typeof sendPublishEvent;
|
|
221
|
+
}): Promise<PublishDescriptionsStageResult> {
|
|
222
|
+
const { courseSlug, journal, journalStorage } = params
|
|
223
|
+
let outcome: PublishDescriptionsStageResult = {
|
|
224
|
+
generation: null,
|
|
225
|
+
mirror: null,
|
|
226
|
+
succeeded: true,
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
// Resolved on demand: the common "nothing to describe" path must not build
|
|
230
|
+
// AWS clients it will never use.
|
|
231
|
+
let resolved = params.publishedStorage
|
|
232
|
+
const publishedStorage = () => {
|
|
233
|
+
if (!resolved) {
|
|
234
|
+
resolved = createS3DescriptionsStorageFromEnv()
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
return resolved
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
try {
|
|
241
|
+
if (!params.hasWork) {
|
|
242
|
+
// Nothing changed enough to describe; both stages are settled, and no
|
|
243
|
+
// second event was promised, so none is sent.
|
|
244
|
+
await markStage(journalStorage, journal, "descriptions", "done", {
|
|
245
|
+
meta: { reason: "up_to_date" },
|
|
246
|
+
})
|
|
247
|
+
await markStage(journalStorage, journal, "gcsMirror", "done", {
|
|
248
|
+
meta: { reason: "up_to_date" },
|
|
249
|
+
})
|
|
250
|
+
await markStage(journalStorage, journal, "manifestEvent", "done", {
|
|
251
|
+
meta: { reason: "not_promised" },
|
|
252
|
+
})
|
|
253
|
+
return outcome
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// Descriptions are platform enrichment, not something the publisher is
|
|
257
|
+
// billed for, so they always run under the service account.
|
|
258
|
+
const token = process.env.RIGOBOT_SYSTEM_TOKEN
|
|
259
|
+
if (!token) {
|
|
260
|
+
await markStage(journalStorage, journal, "descriptions", "failed", {
|
|
261
|
+
error: "RIGOBOT_SYSTEM_TOKEN is not configured",
|
|
262
|
+
})
|
|
263
|
+
console.error(
|
|
264
|
+
`[descriptions] Skipping stage for "${courseSlug}": RIGOBOT_SYSTEM_TOKEN is not configured`
|
|
265
|
+
)
|
|
266
|
+
outcome = { generation: null, mirror: null, succeeded: false }
|
|
267
|
+
return outcome
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
outcome = await runPublishDescriptionsStage({
|
|
271
|
+
courseSlug,
|
|
272
|
+
rigobotToken: token,
|
|
273
|
+
gcsBucket: params.gcsBucket,
|
|
274
|
+
journal,
|
|
275
|
+
journalStorage,
|
|
276
|
+
publishedStorage: publishedStorage(),
|
|
277
|
+
draftStorage: params.draftStorage,
|
|
278
|
+
generate: params.generate,
|
|
279
|
+
})
|
|
280
|
+
|
|
281
|
+
if (outcome.succeeded) {
|
|
282
|
+
console.log(
|
|
283
|
+
`[descriptions] "${courseSlug}": ${
|
|
284
|
+
outcome.generation?.generated ?? 0
|
|
285
|
+
} description(s), ${outcome.mirror?.copied ?? 0} mirrored`
|
|
286
|
+
)
|
|
287
|
+
} else {
|
|
288
|
+
console.error(
|
|
289
|
+
`[descriptions] "${courseSlug}": stage failed${
|
|
290
|
+
outcome.generation?.errors.length ?
|
|
291
|
+
` (${outcome.generation.errors.join("; ")})` :
|
|
292
|
+
""
|
|
293
|
+
}`
|
|
294
|
+
)
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
return outcome
|
|
298
|
+
} catch (error) {
|
|
299
|
+
console.error(
|
|
300
|
+
`[descriptions] Stage failed for "${courseSlug}":`,
|
|
301
|
+
(error as Error).message
|
|
302
|
+
)
|
|
303
|
+
outcome = { generation: null, mirror: null, succeeded: false }
|
|
304
|
+
return outcome
|
|
305
|
+
} finally {
|
|
306
|
+
// The contract: a publication that announced `descriptions: "queued"` is
|
|
307
|
+
// always followed by exactly one manifest event, success or failure.
|
|
308
|
+
// Emitting from `finally` is what makes "exactly one, on every exit path"
|
|
309
|
+
// true by construction rather than by remembering to call it.
|
|
310
|
+
if (params.hasWork) {
|
|
311
|
+
await emitManifestUpdated({
|
|
312
|
+
courseSlug,
|
|
313
|
+
journal,
|
|
314
|
+
journalStorage,
|
|
315
|
+
outcome,
|
|
316
|
+
eventContext: params.eventContext,
|
|
317
|
+
breathecodeToken: params.breathecodeToken,
|
|
318
|
+
publishedStorage: publishedStorage(),
|
|
319
|
+
emit: params.emit ?? sendPublishEvent,
|
|
320
|
+
})
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
await finalizeJournal(journalStorage, journal)
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
async function emitManifestUpdated(params: {
|
|
328
|
+
courseSlug: string;
|
|
329
|
+
journal: PublishJournal;
|
|
330
|
+
journalStorage: JournalStorage;
|
|
331
|
+
outcome: PublishDescriptionsStageResult;
|
|
332
|
+
eventContext?: PublishEventContext;
|
|
333
|
+
breathecodeToken?: string;
|
|
334
|
+
publishedStorage: CourseDescriptionsStorage;
|
|
335
|
+
emit: typeof sendPublishEvent;
|
|
336
|
+
}): Promise<void> {
|
|
337
|
+
const { journal, journalStorage } = params
|
|
338
|
+
|
|
339
|
+
if (!params.eventContext || !params.breathecodeToken) {
|
|
340
|
+
// Nothing was announced to breathecode, so nothing is owed.
|
|
341
|
+
await markStage(journalStorage, journal, "manifestEvent", "done", {
|
|
342
|
+
meta: { reason: "no_event_context" },
|
|
343
|
+
})
|
|
344
|
+
return
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// Enriching the payload must never prevent the delivery. The manifest is
|
|
348
|
+
// optional in the contract (`manifest: null` is valid); the event is not:
|
|
349
|
+
// breathecode was promised one and is waiting for it. Reading the manifest
|
|
350
|
+
// from the published bucket can fail for reasons that have nothing to do with
|
|
351
|
+
// the event — a bad AWS credential, say — and an event with less context still
|
|
352
|
+
// beats an event that never arrives.
|
|
353
|
+
let updatedManifest = params.eventContext.manifest
|
|
354
|
+
try {
|
|
355
|
+
updatedManifest =
|
|
356
|
+
(await params.publishedStorage.readManifest?.(params.courseSlug)) ??
|
|
357
|
+
params.eventContext.manifest
|
|
358
|
+
} catch (error) {
|
|
359
|
+
console.error(
|
|
360
|
+
`[descriptions] Could not read the updated manifest for "${params.courseSlug}", reporting without it:`,
|
|
361
|
+
(error as Error).message
|
|
362
|
+
)
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
try {
|
|
366
|
+
const generation = params.outcome.generation
|
|
367
|
+
const delivered = await params.emit(
|
|
368
|
+
buildPackageManifestUpdatedEvent(
|
|
369
|
+
{ ...params.eventContext, manifest: updatedManifest },
|
|
370
|
+
params.outcome.succeeded ? "success" : "failed",
|
|
371
|
+
generation ?
|
|
372
|
+
{
|
|
373
|
+
generated: generation.generated,
|
|
374
|
+
failed: generation.failed,
|
|
375
|
+
missing: generation.missing,
|
|
376
|
+
} :
|
|
377
|
+
undefined
|
|
378
|
+
),
|
|
379
|
+
params.breathecodeToken
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
await markStage(
|
|
383
|
+
journalStorage,
|
|
384
|
+
journal,
|
|
385
|
+
"manifestEvent",
|
|
386
|
+
delivered ? "done" : "failed",
|
|
387
|
+
{ error: delivered ? undefined : "delivery failed" }
|
|
388
|
+
)
|
|
389
|
+
} catch (error) {
|
|
390
|
+
await markStage(journalStorage, journal, "manifestEvent", "failed", {
|
|
391
|
+
error: (error as Error).message,
|
|
392
|
+
})
|
|
393
|
+
}
|
|
394
|
+
}
|