@learnpack/learnpack 5.0.352 → 5.0.354
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/commands/publish.js +6 -0
- package/lib/commands/serve.js +85 -259
- package/lib/models/creator.d.ts +6 -0
- package/lib/scripts/descriptionsGcsBackfill.d.ts +1 -0
- package/lib/scripts/descriptionsGcsBackfill.js +141 -0
- package/lib/scripts/descriptionsS3Backfill.d.ts +1 -0
- package/lib/scripts/descriptionsS3Backfill.js +277 -0
- package/lib/scripts/descriptionsSweep.d.ts +1 -0
- package/lib/scripts/descriptionsSweep.js +142 -0
- package/lib/utils/api.d.ts +7 -0
- package/lib/utils/api.js +8 -1
- package/lib/utils/awsCredentials.d.ts +20 -0
- package/lib/utils/awsCredentials.js +43 -0
- package/lib/utils/creatorUtilities.js +2 -1
- package/lib/utils/descriptionHash.d.ts +66 -0
- package/lib/utils/descriptionHash.js +173 -0
- package/lib/utils/descriptions/backfillEvents.d.ts +60 -0
- package/lib/utils/descriptions/backfillEvents.js +107 -0
- package/lib/utils/descriptions/gcsStorage.d.ts +16 -0
- package/lib/utils/descriptions/gcsStorage.js +60 -0
- package/lib/utils/descriptions/generateCourseDescriptions.d.ts +66 -0
- package/lib/utils/descriptions/generateCourseDescriptions.js +176 -0
- package/lib/utils/descriptions/mirrorDescriptions.d.ts +49 -0
- package/lib/utils/descriptions/mirrorDescriptions.js +109 -0
- package/lib/utils/descriptions/publishStage.d.ts +69 -0
- package/lib/utils/descriptions/publishStage.js +245 -0
- package/lib/utils/descriptions/resumePublication.d.ts +36 -0
- package/lib/utils/descriptions/resumePublication.js +128 -0
- package/lib/utils/descriptions/s3Storage.d.ts +30 -0
- package/lib/utils/descriptions/s3Storage.js +141 -0
- package/lib/utils/descriptions/workList.d.ts +75 -0
- package/lib/utils/descriptions/workList.js +177 -0
- package/lib/utils/gcsBucketName.d.ts +10 -0
- package/lib/utils/gcsBucketName.js +19 -0
- package/lib/utils/packageManifest.d.ts +22 -0
- package/lib/utils/packageManifest.js +57 -5
- package/lib/utils/publishEvents.d.ts +66 -0
- package/lib/utils/publishEvents.js +111 -0
- package/lib/utils/publishJournal.d.ts +119 -0
- package/lib/utils/publishJournal.js +275 -0
- package/lib/utils/repair/legacyPackageRepair.d.ts +131 -0
- package/lib/utils/repair/legacyPackageRepair.js +492 -0
- package/lib/utils/repair/repairStorage.d.ts +68 -0
- package/lib/utils/repair/repairStorage.js +89 -0
- package/lib/utils/rigoActions.d.ts +44 -0
- package/lib/utils/rigoActions.js +75 -1
- package/lib/utils/s3/packageManifestBackfill.d.ts +2 -0
- package/lib/utils/s3/packageManifestBackfill.js +5 -8
- package/lib/utils/s3/packageSourcesAudit.d.ts +75 -0
- package/lib/utils/s3/packageSourcesAudit.js +184 -0
- package/lib/utils/syllabusSync.d.ts +71 -0
- package/lib/utils/syllabusSync.js +273 -0
- package/package.json +3 -1
- package/src/commands/publish.ts +7 -0
- package/src/commands/serve.ts +144 -335
- package/src/models/creator.ts +9 -0
- package/src/scripts/README.md +244 -0
- package/src/scripts/descriptionsGcsBackfill.ts +193 -0
- package/src/scripts/descriptionsS3Backfill.ts +376 -0
- package/src/scripts/descriptionsSweep.ts +185 -0
- package/src/ui/_app/app.css +1 -1
- package/src/ui/_app/app.js +142 -140
- package/src/ui/app.tar.gz +0 -0
- package/src/utils/api.ts +9 -0
- package/src/utils/awsCredentials.ts +57 -0
- package/src/utils/creatorUtilities.ts +2 -1
- package/src/utils/descriptionHash.ts +196 -0
- package/src/utils/descriptions/backfillEvents.ts +152 -0
- package/src/utils/descriptions/gcsStorage.ts +67 -0
- package/src/utils/descriptions/generateCourseDescriptions.ts +311 -0
- package/src/utils/descriptions/mirrorDescriptions.ts +191 -0
- package/src/utils/descriptions/publishStage.ts +394 -0
- package/src/utils/descriptions/resumePublication.ts +217 -0
- package/src/utils/descriptions/s3Storage.ts +214 -0
- package/src/utils/descriptions/workList.ts +283 -0
- package/src/utils/gcsBucketName.ts +19 -0
- package/src/utils/packageManifest.ts +62 -5
- package/src/utils/publishEvents.ts +181 -0
- package/src/utils/publishJournal.ts +383 -0
- package/src/utils/repair/legacyPackageRepair.ts +731 -0
- package/src/utils/repair/repairStorage.ts +168 -0
- package/src/utils/rigoActions.ts +130 -0
- package/src/utils/s3/packageManifestBackfill.ts +771 -776
- package/src/utils/s3/packageSourcesAudit.ts +311 -0
- package/src/utils/syllabusSync.ts +390 -0
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PACKAGE_MANIFEST_UPDATED_EVENT = exports.PACKAGE_PUBLISHED_EVENT = void 0;
|
|
4
|
+
exports.buildPackagePublishedEvent = buildPackagePublishedEvent;
|
|
5
|
+
exports.buildPackageManifestUpdatedEvent = buildPackageManifestUpdatedEvent;
|
|
6
|
+
exports.sendPublishEvent = sendPublishEvent;
|
|
7
|
+
exports.fetchPackageInfo = fetchPackageInfo;
|
|
8
|
+
const axios_1 = require("axios");
|
|
9
|
+
const api_1 = require("./api");
|
|
10
|
+
const packageManifest_1 = require("./packageManifest");
|
|
11
|
+
/**
|
|
12
|
+
* Publication events sent to breathecode.
|
|
13
|
+
*
|
|
14
|
+
* They are emitted from here, and not from Rigobot, because learnpack-cli is
|
|
15
|
+
* the only party that witnesses the facts: Rigobot's part ends when the zip is
|
|
16
|
+
* deployed, before the breathecode asset sync, and it never sees the end of the
|
|
17
|
+
* asynchronous description stage.
|
|
18
|
+
*
|
|
19
|
+
* Transport is the existing telemetry endpoint. The discriminator is a
|
|
20
|
+
* top-level `event` key, which is what breathecode's webhook log already looks
|
|
21
|
+
* for; a body without it keeps being processed as plain telemetry.
|
|
22
|
+
*
|
|
23
|
+
* `package_slug` and `package_id` are duplicated at the top level on purpose:
|
|
24
|
+
* `LearnPack.add_webhook_to_log` fills the indexed columns from there, so
|
|
25
|
+
* without them the events would land as rows nobody can filter in the webhook
|
|
26
|
+
* viewer.
|
|
27
|
+
*/
|
|
28
|
+
exports.PACKAGE_PUBLISHED_EVENT = "package_published";
|
|
29
|
+
exports.PACKAGE_MANIFEST_UPDATED_EVENT = "package_manifest_updated";
|
|
30
|
+
function packageIdOf(packageInfo) {
|
|
31
|
+
const id = packageInfo === null || packageInfo === void 0 ? void 0 : packageInfo.id;
|
|
32
|
+
return typeof id === "number" ? id : undefined;
|
|
33
|
+
}
|
|
34
|
+
function buildPackagePublishedEvent(context, descriptions) {
|
|
35
|
+
const payload = {
|
|
36
|
+
publish_id: context.publishId,
|
|
37
|
+
package: context.packageInfo,
|
|
38
|
+
manifest: context.manifest ?
|
|
39
|
+
(0, packageManifest_1.serializePackageManifestForEvent)(context.manifest) :
|
|
40
|
+
null,
|
|
41
|
+
descriptions,
|
|
42
|
+
};
|
|
43
|
+
return {
|
|
44
|
+
event: exports.PACKAGE_PUBLISHED_EVENT,
|
|
45
|
+
package_slug: context.courseSlug,
|
|
46
|
+
package_id: packageIdOf(context.packageInfo),
|
|
47
|
+
payload: payload,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
function buildPackageManifestUpdatedEvent(context, status, stats) {
|
|
51
|
+
const payload = {
|
|
52
|
+
publish_id: context.publishId,
|
|
53
|
+
status,
|
|
54
|
+
package: context.packageInfo,
|
|
55
|
+
manifest: context.manifest ?
|
|
56
|
+
(0, packageManifest_1.serializePackageManifestForEvent)(context.manifest) :
|
|
57
|
+
null,
|
|
58
|
+
};
|
|
59
|
+
if (stats) {
|
|
60
|
+
payload.stats = stats;
|
|
61
|
+
}
|
|
62
|
+
return {
|
|
63
|
+
event: exports.PACKAGE_MANIFEST_UPDATED_EVENT,
|
|
64
|
+
package_slug: context.courseSlug,
|
|
65
|
+
package_id: packageIdOf(context.packageInfo),
|
|
66
|
+
payload: payload,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Fire and forget: a publication must never fail because breathecode is down,
|
|
71
|
+
* so every error is swallowed and logged. Returns whether it was delivered, for
|
|
72
|
+
* the journal, not for control flow.
|
|
73
|
+
*/
|
|
74
|
+
async function sendPublishEvent(envelope, breathecodeToken, timeoutMs = 10000) {
|
|
75
|
+
if (!breathecodeToken) {
|
|
76
|
+
console.warn(`[publish-events] Skipping "${envelope.event}": no breathecode token`);
|
|
77
|
+
return false;
|
|
78
|
+
}
|
|
79
|
+
try {
|
|
80
|
+
await axios_1.default.post(api_1.BREATHECODE_EVENTS_URL, envelope, {
|
|
81
|
+
timeout: timeoutMs,
|
|
82
|
+
headers: {
|
|
83
|
+
"Content-Type": "application/json",
|
|
84
|
+
Authorization: "Token " + breathecodeToken.trim(),
|
|
85
|
+
},
|
|
86
|
+
});
|
|
87
|
+
return true;
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
console.error(`[publish-events] Could not deliver "${envelope.event}" for "${envelope.package_slug}":`, error.message);
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Package information for the event payload, straight from Rigobot's own
|
|
96
|
+
* serializer so the shape stays owned by the system that owns the model.
|
|
97
|
+
* Returns null on failure: an event with less context still beats no event.
|
|
98
|
+
*/
|
|
99
|
+
async function fetchPackageInfo(courseSlug, rigobotToken) {
|
|
100
|
+
try {
|
|
101
|
+
const response = await axios_1.default.get(`${api_1.RIGOBOT_HOST}/v1/learnpack/package/${courseSlug}/`, {
|
|
102
|
+
timeout: 10000,
|
|
103
|
+
headers: { Authorization: "Token " + rigobotToken.trim() },
|
|
104
|
+
});
|
|
105
|
+
return response.data;
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
console.error(`[publish-events] Could not read package "${courseSlug}" from Rigobot:`, error.message);
|
|
109
|
+
return null;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { Bucket } from "@google-cloud/storage";
|
|
2
|
+
/**
|
|
3
|
+
* Publish journal — per-stage state of a publication.
|
|
4
|
+
*
|
|
5
|
+
* Publishing is a distributed transaction across three systems (S3 via Rigobot,
|
|
6
|
+
* breathecode assets, and our own buckets) plus an asynchronous description
|
|
7
|
+
* stage. Some of its outcomes are NOT derivable from content afterwards: in
|
|
8
|
+
* particular "was the manifest-updated event already emitted?" leaves no trace
|
|
9
|
+
* in S3 or GCS. The journal records what happened per stage so the sweep can
|
|
10
|
+
* resume an interrupted publication and honour the event contract (a publish
|
|
11
|
+
* that announced `descriptions: "queued"` must always be followed by exactly
|
|
12
|
+
* one `package_manifest_updated`).
|
|
13
|
+
*
|
|
14
|
+
* Design notes:
|
|
15
|
+
*
|
|
16
|
+
* - **Best effort, never a lock.** Every write swallows its errors: if the
|
|
17
|
+
* journal cannot be written the publication still proceeds. Losing the
|
|
18
|
+
* journal degrades us to the previous behaviour (no recovery, no diagnosis),
|
|
19
|
+
* which is acceptable; blocking a publication because a bookkeeping write
|
|
20
|
+
* failed is not.
|
|
21
|
+
* - **Flat key space.** Journals live in `publish-journal/{slug}__{id}.json`,
|
|
22
|
+
* a sibling prefix of `courses/`, so the sweep can list every unfinished
|
|
23
|
+
* publication with one prefix query and the concurrency guard can list a
|
|
24
|
+
* single course with another, without maintaining an index.
|
|
25
|
+
* - **Explicit cleanup.** A journal is deleted once every stage succeeded, and
|
|
26
|
+
* kept when any stage failed (visibility + retry). GCS lifecycle rules would
|
|
27
|
+
* be the natural alternative, but the service account only holds
|
|
28
|
+
* object-level permissions (`storage.buckets.*` is denied), so the cleanup
|
|
29
|
+
* has to happen in code.
|
|
30
|
+
*/
|
|
31
|
+
export declare const JOURNAL_PREFIX = "publish-journal/";
|
|
32
|
+
/**
|
|
33
|
+
* Stages of a publication, in execution order.
|
|
34
|
+
*
|
|
35
|
+
* Only stages the orchestrator can observe on its own: writing the asset id
|
|
36
|
+
* back into Rigobot happens inside the asset sync call, so it is covered by
|
|
37
|
+
* `assetsSync` rather than being a stage that could never be marked.
|
|
38
|
+
*/
|
|
39
|
+
export declare const STAGE_NAMES: readonly ["deploy", "assetsSync", "publishedEvent", "descriptions", "gcsMirror", "manifestEvent"];
|
|
40
|
+
export type StageName = typeof STAGE_NAMES[number];
|
|
41
|
+
export type StageStatus = "pending" | "running" | "done" | "failed";
|
|
42
|
+
export type Stage = {
|
|
43
|
+
status: StageStatus;
|
|
44
|
+
at?: string;
|
|
45
|
+
error?: string;
|
|
46
|
+
meta?: unknown;
|
|
47
|
+
};
|
|
48
|
+
export type PublishJournal = {
|
|
49
|
+
publishId: string;
|
|
50
|
+
courseSlug: string;
|
|
51
|
+
startedAt: string;
|
|
52
|
+
updatedAt: string;
|
|
53
|
+
/** Times the sweep has picked this journal up, to bound retries. */
|
|
54
|
+
attempts: number;
|
|
55
|
+
/** Set once retries are exhausted: kept for a human, ignored by the sweep. */
|
|
56
|
+
abandoned?: boolean;
|
|
57
|
+
stages: Record<StageName, Stage>;
|
|
58
|
+
};
|
|
59
|
+
export type JournalRef = {
|
|
60
|
+
courseSlug: string;
|
|
61
|
+
publishId: string;
|
|
62
|
+
key: string;
|
|
63
|
+
};
|
|
64
|
+
/**
|
|
65
|
+
* Minimal storage surface the journal needs, so callers can pass the real
|
|
66
|
+
* bucket adapter or an in-memory double. Object-level operations only.
|
|
67
|
+
*/
|
|
68
|
+
export interface JournalStorage {
|
|
69
|
+
save(key: string, contents: string): Promise<void>;
|
|
70
|
+
read(key: string): Promise<string | null>;
|
|
71
|
+
remove(key: string): Promise<void>;
|
|
72
|
+
list(prefix: string): Promise<string[]>;
|
|
73
|
+
}
|
|
74
|
+
export declare function journalKey(courseSlug: string, publishId: string): string;
|
|
75
|
+
export declare function coursePrefix(courseSlug: string): string;
|
|
76
|
+
/** Inverse of `journalKey`; returns null for keys that don't match the shape. */
|
|
77
|
+
export declare function parseJournalKey(key: string): JournalRef | null;
|
|
78
|
+
/**
|
|
79
|
+
* Start a journal for a new publication. The generated `publishId` correlates
|
|
80
|
+
* the two publish events and is returned even if persisting failed.
|
|
81
|
+
*/
|
|
82
|
+
export declare function createJournal(storage: JournalStorage, courseSlug: string, publishId?: string): Promise<PublishJournal>;
|
|
83
|
+
/** Update one stage and persist. Mutates and returns the same journal object. */
|
|
84
|
+
export declare function markStage(storage: JournalStorage, journal: PublishJournal, stage: StageName, status: StageStatus, extra?: {
|
|
85
|
+
error?: string;
|
|
86
|
+
meta?: unknown;
|
|
87
|
+
}): Promise<PublishJournal>;
|
|
88
|
+
/** Read a journal. Returns null when missing or unreadable. */
|
|
89
|
+
export declare function readJournal(storage: JournalStorage, courseSlug: string, publishId: string): Promise<PublishJournal | null>;
|
|
90
|
+
export declare function readJournalByKey(storage: JournalStorage, key: string): Promise<PublishJournal | null>;
|
|
91
|
+
/**
|
|
92
|
+
* Every journal still stored, i.e. every publication that did not finish
|
|
93
|
+
* cleanly. Returns an empty list on error (the sweep falls back to its deep
|
|
94
|
+
* scan rather than crashing).
|
|
95
|
+
*/
|
|
96
|
+
export declare function listJournalRefs(storage: JournalStorage): Promise<JournalRef[]>;
|
|
97
|
+
/** Journals of a single course; backs the concurrent-publish guard. */
|
|
98
|
+
export declare function listJournalRefsForCourse(storage: JournalStorage, courseSlug: string): Promise<JournalRef[]>;
|
|
99
|
+
export declare function isStageTerminal(stage: Stage): boolean;
|
|
100
|
+
export declare function hasFailedStage(journal: PublishJournal): boolean;
|
|
101
|
+
/** True when no stage is left to run (all done, or done/failed mix). */
|
|
102
|
+
export declare function isJournalSettled(journal: PublishJournal): boolean;
|
|
103
|
+
/** Stages the sweep still has work to do on. */
|
|
104
|
+
export declare function pendingStages(journal: PublishJournal): StageName[];
|
|
105
|
+
/** A journal untouched for longer than `maxAgeMs` is presumed interrupted. */
|
|
106
|
+
export declare function isJournalStale(journal: PublishJournal, maxAgeMs: number, now?: number): boolean;
|
|
107
|
+
export declare function incrementAttempts(storage: JournalStorage, journal: PublishJournal): Promise<PublishJournal>;
|
|
108
|
+
/**
|
|
109
|
+
* Give up on a journal after too many attempts. It is kept, not deleted: a
|
|
110
|
+
* publication that never completed is exactly what someone needs to see.
|
|
111
|
+
*/
|
|
112
|
+
export declare function abandonJournal(storage: JournalStorage, journal: PublishJournal): Promise<PublishJournal>;
|
|
113
|
+
/**
|
|
114
|
+
* Close a journal: delete it when every stage succeeded, keep it otherwise so
|
|
115
|
+
* the sweep can retry and a human can inspect it. Never throws.
|
|
116
|
+
*/
|
|
117
|
+
export declare function finalizeJournal(storage: JournalStorage, journal: PublishJournal): Promise<"deleted" | "kept">;
|
|
118
|
+
/** GCS-backed adapter. Object-level operations only, by design. */
|
|
119
|
+
export declare function createGcsJournalStorage(bucket: Bucket): JournalStorage;
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.STAGE_NAMES = exports.JOURNAL_PREFIX = void 0;
|
|
4
|
+
exports.journalKey = journalKey;
|
|
5
|
+
exports.coursePrefix = coursePrefix;
|
|
6
|
+
exports.parseJournalKey = parseJournalKey;
|
|
7
|
+
exports.createJournal = createJournal;
|
|
8
|
+
exports.markStage = markStage;
|
|
9
|
+
exports.readJournal = readJournal;
|
|
10
|
+
exports.readJournalByKey = readJournalByKey;
|
|
11
|
+
exports.listJournalRefs = listJournalRefs;
|
|
12
|
+
exports.listJournalRefsForCourse = listJournalRefsForCourse;
|
|
13
|
+
exports.isStageTerminal = isStageTerminal;
|
|
14
|
+
exports.hasFailedStage = hasFailedStage;
|
|
15
|
+
exports.isJournalSettled = isJournalSettled;
|
|
16
|
+
exports.pendingStages = pendingStages;
|
|
17
|
+
exports.isJournalStale = isJournalStale;
|
|
18
|
+
exports.incrementAttempts = incrementAttempts;
|
|
19
|
+
exports.abandonJournal = abandonJournal;
|
|
20
|
+
exports.finalizeJournal = finalizeJournal;
|
|
21
|
+
exports.createGcsJournalStorage = createGcsJournalStorage;
|
|
22
|
+
const uuid_1 = require("uuid");
|
|
23
|
+
/**
|
|
24
|
+
* Publish journal — per-stage state of a publication.
|
|
25
|
+
*
|
|
26
|
+
* Publishing is a distributed transaction across three systems (S3 via Rigobot,
|
|
27
|
+
* breathecode assets, and our own buckets) plus an asynchronous description
|
|
28
|
+
* stage. Some of its outcomes are NOT derivable from content afterwards: in
|
|
29
|
+
* particular "was the manifest-updated event already emitted?" leaves no trace
|
|
30
|
+
* in S3 or GCS. The journal records what happened per stage so the sweep can
|
|
31
|
+
* resume an interrupted publication and honour the event contract (a publish
|
|
32
|
+
* that announced `descriptions: "queued"` must always be followed by exactly
|
|
33
|
+
* one `package_manifest_updated`).
|
|
34
|
+
*
|
|
35
|
+
* Design notes:
|
|
36
|
+
*
|
|
37
|
+
* - **Best effort, never a lock.** Every write swallows its errors: if the
|
|
38
|
+
* journal cannot be written the publication still proceeds. Losing the
|
|
39
|
+
* journal degrades us to the previous behaviour (no recovery, no diagnosis),
|
|
40
|
+
* which is acceptable; blocking a publication because a bookkeeping write
|
|
41
|
+
* failed is not.
|
|
42
|
+
* - **Flat key space.** Journals live in `publish-journal/{slug}__{id}.json`,
|
|
43
|
+
* a sibling prefix of `courses/`, so the sweep can list every unfinished
|
|
44
|
+
* publication with one prefix query and the concurrency guard can list a
|
|
45
|
+
* single course with another, without maintaining an index.
|
|
46
|
+
* - **Explicit cleanup.** A journal is deleted once every stage succeeded, and
|
|
47
|
+
* kept when any stage failed (visibility + retry). GCS lifecycle rules would
|
|
48
|
+
* be the natural alternative, but the service account only holds
|
|
49
|
+
* object-level permissions (`storage.buckets.*` is denied), so the cleanup
|
|
50
|
+
* has to happen in code.
|
|
51
|
+
*/
|
|
52
|
+
exports.JOURNAL_PREFIX = "publish-journal/";
|
|
53
|
+
/**
|
|
54
|
+
* Stages of a publication, in execution order.
|
|
55
|
+
*
|
|
56
|
+
* Only stages the orchestrator can observe on its own: writing the asset id
|
|
57
|
+
* back into Rigobot happens inside the asset sync call, so it is covered by
|
|
58
|
+
* `assetsSync` rather than being a stage that could never be marked.
|
|
59
|
+
*/
|
|
60
|
+
exports.STAGE_NAMES = [
|
|
61
|
+
"deploy",
|
|
62
|
+
"assetsSync",
|
|
63
|
+
"publishedEvent",
|
|
64
|
+
"descriptions",
|
|
65
|
+
"gcsMirror",
|
|
66
|
+
"manifestEvent",
|
|
67
|
+
];
|
|
68
|
+
function journalKey(courseSlug, publishId) {
|
|
69
|
+
return `${exports.JOURNAL_PREFIX}${courseSlug}__${publishId}.json`;
|
|
70
|
+
}
|
|
71
|
+
function coursePrefix(courseSlug) {
|
|
72
|
+
return `${exports.JOURNAL_PREFIX}${courseSlug}__`;
|
|
73
|
+
}
|
|
74
|
+
/** Inverse of `journalKey`; returns null for keys that don't match the shape. */
|
|
75
|
+
function parseJournalKey(key) {
|
|
76
|
+
if (!key.startsWith(exports.JOURNAL_PREFIX) || !key.endsWith(".json")) {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
const name = key.slice(exports.JOURNAL_PREFIX.length, -".json".length);
|
|
80
|
+
const separator = name.indexOf("__");
|
|
81
|
+
if (separator <= 0) {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
const courseSlug = name.slice(0, separator);
|
|
85
|
+
const publishId = name.slice(separator + 2);
|
|
86
|
+
if (!courseSlug || !publishId) {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
return { courseSlug, publishId, key };
|
|
90
|
+
}
|
|
91
|
+
function emptyStages() {
|
|
92
|
+
const stages = {};
|
|
93
|
+
for (const name of exports.STAGE_NAMES) {
|
|
94
|
+
stages[name] = { status: "pending" };
|
|
95
|
+
}
|
|
96
|
+
return stages;
|
|
97
|
+
}
|
|
98
|
+
/** Persist the journal. Never throws: bookkeeping must not break a publish. */
|
|
99
|
+
async function persist(storage, journal) {
|
|
100
|
+
try {
|
|
101
|
+
await storage.save(journalKey(journal.courseSlug, journal.publishId), JSON.stringify(journal, null, 2));
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
console.warn(`[publish-journal] Could not persist journal for "${journal.courseSlug}":`, error.message);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Start a journal for a new publication. The generated `publishId` correlates
|
|
109
|
+
* the two publish events and is returned even if persisting failed.
|
|
110
|
+
*/
|
|
111
|
+
async function createJournal(storage, courseSlug, publishId = (0, uuid_1.v4)()) {
|
|
112
|
+
const now = new Date().toISOString();
|
|
113
|
+
const journal = {
|
|
114
|
+
publishId,
|
|
115
|
+
courseSlug,
|
|
116
|
+
startedAt: now,
|
|
117
|
+
updatedAt: now,
|
|
118
|
+
attempts: 0,
|
|
119
|
+
stages: emptyStages(),
|
|
120
|
+
};
|
|
121
|
+
await persist(storage, journal);
|
|
122
|
+
return journal;
|
|
123
|
+
}
|
|
124
|
+
/** Update one stage and persist. Mutates and returns the same journal object. */
|
|
125
|
+
async function markStage(storage, journal, stage, status, extra = {}) {
|
|
126
|
+
const next = { status, at: new Date().toISOString() };
|
|
127
|
+
if (extra.error !== undefined) {
|
|
128
|
+
next.error = extra.error;
|
|
129
|
+
}
|
|
130
|
+
if (extra.meta !== undefined) {
|
|
131
|
+
next.meta = extra.meta;
|
|
132
|
+
}
|
|
133
|
+
journal.stages[stage] = next;
|
|
134
|
+
journal.updatedAt = next.at;
|
|
135
|
+
await persist(storage, journal);
|
|
136
|
+
return journal;
|
|
137
|
+
}
|
|
138
|
+
/** Read a journal. Returns null when missing or unreadable. */
|
|
139
|
+
async function readJournal(storage, courseSlug, publishId) {
|
|
140
|
+
return readJournalByKey(storage, journalKey(courseSlug, publishId));
|
|
141
|
+
}
|
|
142
|
+
async function readJournalByKey(storage, key) {
|
|
143
|
+
try {
|
|
144
|
+
const raw = await storage.read(key);
|
|
145
|
+
if (!raw) {
|
|
146
|
+
return null;
|
|
147
|
+
}
|
|
148
|
+
const parsed = JSON.parse(raw);
|
|
149
|
+
if (!parsed || !parsed.publishId || !parsed.stages) {
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
// Tolerate journals written before a stage was added to STAGE_NAMES.
|
|
153
|
+
for (const name of exports.STAGE_NAMES) {
|
|
154
|
+
if (!parsed.stages[name]) {
|
|
155
|
+
parsed.stages[name] = { status: "pending" };
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return parsed;
|
|
159
|
+
}
|
|
160
|
+
catch (error) {
|
|
161
|
+
console.warn(`[publish-journal] Could not read journal "${key}":`, error.message);
|
|
162
|
+
return null;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Every journal still stored, i.e. every publication that did not finish
|
|
167
|
+
* cleanly. Returns an empty list on error (the sweep falls back to its deep
|
|
168
|
+
* scan rather than crashing).
|
|
169
|
+
*/
|
|
170
|
+
async function listJournalRefs(storage) {
|
|
171
|
+
return listRefsByPrefix(storage, exports.JOURNAL_PREFIX);
|
|
172
|
+
}
|
|
173
|
+
/** Journals of a single course; backs the concurrent-publish guard. */
|
|
174
|
+
async function listJournalRefsForCourse(storage, courseSlug) {
|
|
175
|
+
return listRefsByPrefix(storage, coursePrefix(courseSlug));
|
|
176
|
+
}
|
|
177
|
+
async function listRefsByPrefix(storage, prefix) {
|
|
178
|
+
try {
|
|
179
|
+
const keys = await storage.list(prefix);
|
|
180
|
+
const refs = [];
|
|
181
|
+
for (const key of keys) {
|
|
182
|
+
const ref = parseJournalKey(key);
|
|
183
|
+
if (ref) {
|
|
184
|
+
refs.push(ref);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return refs;
|
|
188
|
+
}
|
|
189
|
+
catch (error) {
|
|
190
|
+
console.warn(`[publish-journal] Could not list journals "${prefix}":`, error.message);
|
|
191
|
+
return [];
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
function isStageTerminal(stage) {
|
|
195
|
+
return stage.status === "done" || stage.status === "failed";
|
|
196
|
+
}
|
|
197
|
+
function hasFailedStage(journal) {
|
|
198
|
+
return exports.STAGE_NAMES.some(name => journal.stages[name].status === "failed");
|
|
199
|
+
}
|
|
200
|
+
/** True when no stage is left to run (all done, or done/failed mix). */
|
|
201
|
+
function isJournalSettled(journal) {
|
|
202
|
+
return exports.STAGE_NAMES.every(name => isStageTerminal(journal.stages[name]));
|
|
203
|
+
}
|
|
204
|
+
/** Stages the sweep still has work to do on. */
|
|
205
|
+
function pendingStages(journal) {
|
|
206
|
+
return exports.STAGE_NAMES.filter(name => !isStageTerminal(journal.stages[name]));
|
|
207
|
+
}
|
|
208
|
+
/** A journal untouched for longer than `maxAgeMs` is presumed interrupted. */
|
|
209
|
+
function isJournalStale(journal, maxAgeMs, now = Date.now()) {
|
|
210
|
+
const updatedAt = Date.parse(journal.updatedAt);
|
|
211
|
+
if (Number.isNaN(updatedAt)) {
|
|
212
|
+
return true;
|
|
213
|
+
}
|
|
214
|
+
return now - updatedAt > maxAgeMs;
|
|
215
|
+
}
|
|
216
|
+
async function incrementAttempts(storage, journal) {
|
|
217
|
+
journal.attempts += 1;
|
|
218
|
+
journal.updatedAt = new Date().toISOString();
|
|
219
|
+
await persist(storage, journal);
|
|
220
|
+
return journal;
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Give up on a journal after too many attempts. It is kept, not deleted: a
|
|
224
|
+
* publication that never completed is exactly what someone needs to see.
|
|
225
|
+
*/
|
|
226
|
+
async function abandonJournal(storage, journal) {
|
|
227
|
+
journal.abandoned = true;
|
|
228
|
+
journal.updatedAt = new Date().toISOString();
|
|
229
|
+
await persist(storage, journal);
|
|
230
|
+
return journal;
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Close a journal: delete it when every stage succeeded, keep it otherwise so
|
|
234
|
+
* the sweep can retry and a human can inspect it. Never throws.
|
|
235
|
+
*/
|
|
236
|
+
async function finalizeJournal(storage, journal) {
|
|
237
|
+
if (!isJournalSettled(journal) || hasFailedStage(journal)) {
|
|
238
|
+
await persist(storage, journal);
|
|
239
|
+
return "kept";
|
|
240
|
+
}
|
|
241
|
+
try {
|
|
242
|
+
await storage.remove(journalKey(journal.courseSlug, journal.publishId));
|
|
243
|
+
return "deleted";
|
|
244
|
+
}
|
|
245
|
+
catch (error) {
|
|
246
|
+
console.warn(`[publish-journal] Could not delete journal for "${journal.courseSlug}":`, error.message);
|
|
247
|
+
return "kept";
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
/** GCS-backed adapter. Object-level operations only, by design. */
|
|
251
|
+
function createGcsJournalStorage(bucket) {
|
|
252
|
+
return {
|
|
253
|
+
async save(key, contents) {
|
|
254
|
+
await bucket.file(key).save(Buffer.from(contents, "utf8"), {
|
|
255
|
+
contentType: "application/json",
|
|
256
|
+
});
|
|
257
|
+
},
|
|
258
|
+
async read(key) {
|
|
259
|
+
try {
|
|
260
|
+
const [buf] = await bucket.file(key).download();
|
|
261
|
+
return buf.toString();
|
|
262
|
+
}
|
|
263
|
+
catch (_a) {
|
|
264
|
+
return null;
|
|
265
|
+
}
|
|
266
|
+
},
|
|
267
|
+
async remove(key) {
|
|
268
|
+
await bucket.file(key).delete();
|
|
269
|
+
},
|
|
270
|
+
async list(prefix) {
|
|
271
|
+
const [files] = await bucket.getFiles({ prefix });
|
|
272
|
+
return files.map(file => file.name);
|
|
273
|
+
},
|
|
274
|
+
};
|
|
275
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { Syllabus } from "../../models/creator";
|
|
2
|
+
import { Exercise } from "../configBuilder";
|
|
3
|
+
import { LessonType } from "../packageManifest";
|
|
4
|
+
import { RepairStorage, RepairTarget } from "./repairStorage";
|
|
5
|
+
/**
|
|
6
|
+
* Reconstruct the manifest inputs missing from legacy packages.
|
|
7
|
+
*
|
|
8
|
+
* Some packages predate the creator — a few were uploaded straight from a GitHub
|
|
9
|
+
* repository — and never got a `.learn/initialSyllabus.json`, a
|
|
10
|
+
* `.learn/sidebar.json`, or a usable `config.json`. The manifest tolerates the
|
|
11
|
+
* first two and comes out degraded: every lesson typed `READ`, and descriptions
|
|
12
|
+
* pinned to `null` forever, because `descriptionsS3Backfill` builds its work
|
|
13
|
+
* list from the syllabus keys and never even sees a course without one.
|
|
14
|
+
*
|
|
15
|
+
* This module writes those files back, derived from what the package *does*
|
|
16
|
+
* have: its exercise folders and their READMEs. It never invents content — a
|
|
17
|
+
* title comes from the README's own `# H1`, never from an LLM.
|
|
18
|
+
*
|
|
19
|
+
* It runs once per bucket (see `repairStorage.ts` for why the draft is repaired
|
|
20
|
+
* from the draft rather than copied from the snapshot). Ordering afterwards:
|
|
21
|
+
*
|
|
22
|
+
* 1. this script, on both targets (structure)
|
|
23
|
+
* 2. `descriptionsS3Backfill --reproject-manifest --mirror-draft`
|
|
24
|
+
* 3. `backfill:package-manifest`, unless step 2 projected it
|
|
25
|
+
*
|
|
26
|
+
* Safety: with one documented exception (a `config.json` whose `exercises` array
|
|
27
|
+
* is empty, which is merged rather than replaced) it only ever writes keys that
|
|
28
|
+
* do not exist. It never touches a file the real pipeline produced.
|
|
29
|
+
*/
|
|
30
|
+
export type RepairFile = "config" | "syllabus" | "sidebar";
|
|
31
|
+
export declare const ALL_REPAIR_FILES: RepairFile[];
|
|
32
|
+
export type FileOutcome = "created" | "merged" | "present" | "not-requested" | "impossible";
|
|
33
|
+
export type FileResult = {
|
|
34
|
+
outcome: FileOutcome;
|
|
35
|
+
key?: string;
|
|
36
|
+
reason?: string;
|
|
37
|
+
};
|
|
38
|
+
export type RepairStats = {
|
|
39
|
+
exercises: number;
|
|
40
|
+
languages: string[];
|
|
41
|
+
primaryLanguage: string;
|
|
42
|
+
lessonTypes: Record<LessonType, number>;
|
|
43
|
+
titlesFromReadme: number;
|
|
44
|
+
titlesFromFolderName: number;
|
|
45
|
+
};
|
|
46
|
+
export type RepairResult = {
|
|
47
|
+
slug: string;
|
|
48
|
+
target: RepairTarget;
|
|
49
|
+
/**
|
|
50
|
+
* `absent` is not a failure: half the legacy packages published in S3 have no
|
|
51
|
+
* draft in GCS at all, so a run over both buckets is expected to find nothing
|
|
52
|
+
* to repair on one side. Only a package that *is* in the bucket but yields no
|
|
53
|
+
* exercises counts as `failed`.
|
|
54
|
+
*/
|
|
55
|
+
status: "repaired" | "unchanged" | "absent" | "failed";
|
|
56
|
+
durationMs: number;
|
|
57
|
+
files: Record<RepairFile, FileResult>;
|
|
58
|
+
stats?: RepairStats;
|
|
59
|
+
error?: string;
|
|
60
|
+
};
|
|
61
|
+
export type RepairOptions = {
|
|
62
|
+
dryRun: boolean;
|
|
63
|
+
files: RepairFile[];
|
|
64
|
+
};
|
|
65
|
+
/** README body + parsed title, by exercise slug and language. */
|
|
66
|
+
export type ReadmeIndex = Record<string, Record<string, {
|
|
67
|
+
title: string | null;
|
|
68
|
+
body: string;
|
|
69
|
+
}>>;
|
|
70
|
+
/**
|
|
71
|
+
* Exercises from the bucket listing alone, mirroring `configBuilder.buildConfig`
|
|
72
|
+
* key for key: same README language rule, same natural ordering, same
|
|
73
|
+
* `graded: false` default.
|
|
74
|
+
*/
|
|
75
|
+
export declare function buildExercisesFromKeys(exercisesPrefix: string, keys: string[]): Exercise[];
|
|
76
|
+
/**
|
|
77
|
+
* Language the course is really written in.
|
|
78
|
+
*
|
|
79
|
+
* Same rule as `workList.inferCourseBaseLanguage`: a declared language is
|
|
80
|
+
* trusted only when some exercise actually has a README in it, otherwise the
|
|
81
|
+
* most frequent translation wins. Defaulting to "en" would label a Spanish
|
|
82
|
+
* course English and anchor every later description to a translation.
|
|
83
|
+
*
|
|
84
|
+
* A bilingual course ties on count — most of the legacy ones are — so the tie is
|
|
85
|
+
* broken on the file names rather than arbitrarily: `getReadmeExtension` gives
|
|
86
|
+
* the *original* README no language suffix and suffixes every translation, so
|
|
87
|
+
* the language mapped to a bare `README.md` is the one the course was written
|
|
88
|
+
* in. This matters beyond the title: the descriptions backfill generates from
|
|
89
|
+
* `courseInfo.language` and would otherwise work from a translation.
|
|
90
|
+
*/
|
|
91
|
+
export declare function inferPrimaryLanguage(exercises: Exercise[], declared?: string): string;
|
|
92
|
+
/**
|
|
93
|
+
* Lesson type, inferred — the one field of a synthesized syllabus that is a
|
|
94
|
+
* guess rather than a reading. Callers should treat it as such.
|
|
95
|
+
*/
|
|
96
|
+
export declare function inferLessonType(input: {
|
|
97
|
+
graded: boolean;
|
|
98
|
+
files: Array<{
|
|
99
|
+
name: string;
|
|
100
|
+
}>;
|
|
101
|
+
readmeBody: string;
|
|
102
|
+
}): LessonType;
|
|
103
|
+
export type BuildSyllabusInput = {
|
|
104
|
+
slug: string;
|
|
105
|
+
learnJson: Record<string, unknown> | null;
|
|
106
|
+
exercises: Exercise[];
|
|
107
|
+
readmes: ReadmeIndex;
|
|
108
|
+
primaryLanguage: string;
|
|
109
|
+
};
|
|
110
|
+
export type BuildSyllabusResult = {
|
|
111
|
+
syllabus: Syllabus;
|
|
112
|
+
stats: Omit<RepairStats, "exercises" | "languages" | "primaryLanguage">;
|
|
113
|
+
};
|
|
114
|
+
/**
|
|
115
|
+
* A syllabus equivalent to what the creator would have written, derived from the
|
|
116
|
+
* exercise folders and their READMEs.
|
|
117
|
+
*
|
|
118
|
+
* `uid` is set to the exercise folder slug on purpose. `findSyllabusLesson`
|
|
119
|
+
* matches on `slugify(id + "-" + title)` *or* `uid`, and only the second is
|
|
120
|
+
* guaranteed to hold once a title comes from an H1 the folder name never had —
|
|
121
|
+
* it is also what lets the S3 and GCS repairs, run independently, still line up
|
|
122
|
+
* for the descriptions mirror.
|
|
123
|
+
*/
|
|
124
|
+
export declare function buildSyllabusFromSources(input: BuildSyllabusInput): BuildSyllabusResult;
|
|
125
|
+
/** `sidebar[exerciseSlug][lang] = title`, from the same READMEs. */
|
|
126
|
+
export declare function buildSidebarFromReadmes(exercises: Exercise[], readmes: ReadmeIndex): Record<string, Record<string, string>>;
|
|
127
|
+
/**
|
|
128
|
+
* Repair one package in one bucket. Returns what it did (or would do, under
|
|
129
|
+
* `dryRun`) without throwing: a batch reports failures per slug.
|
|
130
|
+
*/
|
|
131
|
+
export declare function repairPackage(storage: RepairStorage, slug: string, options: RepairOptions): Promise<RepairResult>;
|