@mcptoolshop/style-dataset-lab 3.3.0 → 3.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/CHANGELOG.md +60 -0
  2. package/README.es.md +117 -55
  3. package/README.fr.md +125 -65
  4. package/README.hi.md +117 -68
  5. package/README.it.md +119 -58
  6. package/README.ja.md +124 -74
  7. package/README.md +73 -7
  8. package/README.pt-BR.md +128 -63
  9. package/README.zh.md +116 -65
  10. package/bin/sdlab.js +100 -4
  11. package/lib/adapters/ai-toolkit.js +224 -35
  12. package/lib/adapters/comfyui-runner.js +79 -26
  13. package/lib/adapters/comfyui-workflows.js +94 -11
  14. package/lib/adapters/diffusers-lora.js +37 -5
  15. package/lib/adapters/generic-image-caption.js +43 -5
  16. package/lib/args.js +16 -1
  17. package/lib/asset-ingest.js +575 -0
  18. package/lib/asset-masks.js +323 -0
  19. package/lib/asset-source.js +1176 -0
  20. package/lib/batch-modes.js +29 -8
  21. package/lib/batch-runs.js +29 -1
  22. package/lib/batch-sheet-render.js +13 -3
  23. package/lib/brief-compiler.js +19 -0
  24. package/lib/candidate-sheet.js +327 -0
  25. package/lib/canon-author.js +350 -0
  26. package/lib/canon-build/build.js +146 -93
  27. package/lib/canon-review-gate.js +205 -0
  28. package/lib/captions.js +57 -0
  29. package/lib/card.js +74 -2
  30. package/lib/colab-launcher-guards.js +285 -0
  31. package/lib/comfyui.js +116 -18
  32. package/lib/config.js +116 -12
  33. package/lib/critique-engine.js +36 -2
  34. package/lib/eligibility.js +121 -9
  35. package/lib/eval-pack.js +14 -14
  36. package/lib/eval-runs.js +33 -0
  37. package/lib/export.js +122 -34
  38. package/lib/freeze-gate.js +16 -3
  39. package/lib/freeze-stamp.js +28 -3
  40. package/lib/generated-provenance.js +194 -6
  41. package/lib/implementation-packs.js +95 -17
  42. package/lib/ingest.js +396 -0
  43. package/lib/judgment-provenance.js +220 -0
  44. package/lib/lane1-render.js +668 -0
  45. package/lib/measure.js +422 -0
  46. package/lib/memory-store.js +663 -0
  47. package/lib/model-licences.js +305 -0
  48. package/lib/npy-meta.js +130 -0
  49. package/lib/paths.js +196 -7
  50. package/lib/png-meta.js +512 -0
  51. package/lib/records.js +54 -8
  52. package/lib/reingest-selected.js +113 -17
  53. package/lib/reingest.js +27 -3
  54. package/lib/run-manifest.js +159 -25
  55. package/lib/run-summary.js +12 -6
  56. package/lib/runtime-runs.js +45 -4
  57. package/lib/snapshot.js +114 -15
  58. package/lib/split.js +364 -48
  59. package/lib/train-runs.js +214 -0
  60. package/lib/training-manifests.js +114 -21
  61. package/lib/training-packages.js +147 -15
  62. package/lib/verifier.js +3 -0
  63. package/lib/verify-chain.js +367 -0
  64. package/lib/workflow-profiles.js +24 -3
  65. package/package.json +4 -3
  66. package/runtime/comfyui/txt2img-qwen.json +38 -0
  67. package/runtime/comfyui/txt2img-standard.json +21 -0
  68. package/scripts/_resume.js +62 -0
  69. package/scripts/_shared-args.js +44 -0
  70. package/scripts/asset-ingest.js +160 -0
  71. package/scripts/brief-compile.js +2 -1
  72. package/scripts/brief-show.js +2 -1
  73. package/scripts/canon-bind.js +2 -2
  74. package/scripts/canon-build.js +27 -20
  75. package/scripts/canon-draft.js +82 -0
  76. package/scripts/canon-drift.js +16 -7
  77. package/scripts/canon-freeze-status.js +14 -6
  78. package/scripts/canon-freeze.js +31 -11
  79. package/scripts/canon-unfreeze.js +17 -9
  80. package/scripts/card.js +2 -2
  81. package/scripts/colab/REAL_SERVICE_ASSUMPTIONS.md +14 -0
  82. package/scripts/colab/ai-toolkit-override-py315.txt +45 -0
  83. package/scripts/colab/colab_window.sh +587 -0
  84. package/scripts/colab/lane1_render.py +970 -0
  85. package/scripts/colab/launch_guards.py +564 -0
  86. package/scripts/colab/launch_lane1_stub.sh +309 -0
  87. package/scripts/colab/launch_lane3_stub.sh +366 -0
  88. package/scripts/colab/remote_runner.py +59 -0
  89. package/scripts/compare.js +2 -2
  90. package/scripts/critique.js +16 -1
  91. package/scripts/curate.js +2 -2
  92. package/scripts/doctor.js +182 -55
  93. package/scripts/eligibility.js +42 -2
  94. package/scripts/eval-pack.js +2 -2
  95. package/scripts/eval-run.js +2 -2
  96. package/scripts/export.js +2 -2
  97. package/scripts/generate-controlnet.js +45 -25
  98. package/scripts/generate-identity.js +53 -23
  99. package/scripts/generate-ipadapter.js +44 -23
  100. package/scripts/generate.js +86 -17
  101. package/scripts/implementation-pack.js +2 -2
  102. package/scripts/ingest.js +145 -0
  103. package/scripts/init.js +73 -13
  104. package/scripts/measure.js +170 -0
  105. package/scripts/measure_image.py +458 -0
  106. package/scripts/migrate-records.js +142 -0
  107. package/scripts/painterly-test.js +4 -3
  108. package/scripts/painterly.js +7 -3
  109. package/scripts/qwen_generate.py +138 -20
  110. package/scripts/reingest.js +2 -2
  111. package/scripts/sheet.js +142 -0
  112. package/scripts/snapshot.js +2 -2
  113. package/scripts/split.js +2 -2
  114. package/scripts/training-manifest.js +2 -2
  115. package/scripts/training-package.js +2 -2
  116. package/scripts/training-profile.js +2 -2
  117. package/scripts/verify.js +67 -0
  118. package/scripts/workflow-list.js +2 -1
  119. package/scripts/workflow-show.js +2 -1
@@ -0,0 +1,350 @@
1
+ /**
2
+ * Lane 2 canon and prompt authoring adapter (DESIGN-v3 §3.2, §9.2 item 7, §11, G10).
3
+ *
4
+ * Calls a local loopback OpenAI-compatible HTTP endpoint (e.g., llama-server or
5
+ * Ollama serving Muse-Glimmer-30B-A3B-GGUF under Apache-2.0) to draft visual
6
+ * canon entries and per-class prompts + negatives.
7
+ *
8
+ * Every model-authored output starts with `status: "draft"` and `judgment: null`,
9
+ * carrying the model name, digest, SPDX licence (`Apache-2.0`), credit note,
10
+ * `messages_sha256`, and `draft_sha256`.
11
+ */
12
+
13
+ import { createHash } from 'node:crypto';
14
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
15
+ import { dirname } from 'node:path';
16
+ import { canonicalize } from './freeze-stamp.js';
17
+ import { inputError as rawInputError } from './errors.js';
18
+
19
+ function inputError(code, msg) {
20
+ return rawInputError(code, `${code}: ${msg}`);
21
+ }
22
+
23
+ export const MIN_LLAMA_SERVER_BUILD = 10353;
24
+
25
+ export const GLIMMER_CREDIT_NOTE =
26
+ 'Canon and prompt drafting assisted by Muse-Glimmer-30B-A3B (GlimmerLabs, Apache-2.0); vision inspection via Qwen3-VL mmproj (Apache-2.0).';
27
+
28
+ export const GLIMMER_DEFAULT_MODEL = Object.freeze({
29
+ name: 'Muse-Glimmer-30B-A3B-Q4_K_M.gguf',
30
+ repo: 'GlimmerLabs/Muse-Glimmer-30B-A3B-GGUF',
31
+ digest: 'sha256:' + '3'.repeat(64),
32
+ licence_id: 'Apache-2.0',
33
+ licence_url: 'https://huggingface.co/GlimmerLabs/Muse-Glimmer-30B-A3B-GGUF/blob/main/LICENSE',
34
+ date_read: '2026-11-05',
35
+ credit_note: GLIMMER_CREDIT_NOTE,
36
+ });
37
+
38
+ export const LOOPBACK_HOSTS = new Set(['127.0.0.1', 'localhost', '::1', '[::1]']);
39
+
40
+ function sha256String(str) {
41
+ return 'sha256:' + createHash('sha256').update(str, 'utf-8').digest('hex');
42
+ }
43
+
44
+ function sha256Buffer(buf) {
45
+ return 'sha256:' + createHash('sha256').update(buf).digest('hex');
46
+ }
47
+
48
+ function normalizeSha256(val) {
49
+ if (typeof val !== 'string') return null;
50
+ const trimmed = val.trim().toLowerCase();
51
+ const hex = trimmed.startsWith('sha256:') ? trimmed.slice(7) : trimmed;
52
+ if (!/^[0-9a-f]{64}$/.test(hex)) return null;
53
+ return `sha256:${hex}`;
54
+ }
55
+
56
+ /**
57
+ * Enforce that the Lane 2 LLM endpoint is strictly loopback-bound (§3.2, §6, §11.3).
58
+ *
59
+ * @param {string} endpointUrl
60
+ * @returns {URL}
61
+ */
62
+ export function assertLoopbackEndpoint(endpointUrl) {
63
+ let parsed;
64
+ try {
65
+ parsed = new URL(endpointUrl);
66
+ } catch {
67
+ throw inputError(
68
+ 'LANE2_NON_LOOPBACK_ENDPOINT',
69
+ `Invalid Lane 2 endpoint URL "${endpointUrl}". Must be http://127.0.0.1:<port>/...`
70
+ );
71
+ }
72
+ const host = parsed.hostname.toLowerCase();
73
+ if (!LOOPBACK_HOSTS.has(host)) {
74
+ throw inputError(
75
+ 'LANE2_NON_LOOPBACK_ENDPOINT',
76
+ `Lane 2 endpoint "${endpointUrl}" binds to non-loopback host "${host}". Must bind to 127.0.0.1.`
77
+ );
78
+ }
79
+ return parsed;
80
+ }
81
+
82
+ /**
83
+ * Pre-check the llama.cpp `llama-server` build (`>= b10353` for Qwen3.5 /
84
+ * Muse-Glimmer-30B-A3B) and optional `mmproj` vision projector (§9.2 item 7, §11.3).
85
+ *
86
+ * @param {Object} opts
87
+ * @param {string|number} opts.llamaServerBuild — e.g. 'b10353' or 10400
88
+ * @param {boolean} [opts.vision=false] — whether vision inspection / plate reading is active
89
+ * @param {string} [opts.mmprojPath]
90
+ * @param {string} [opts.mmprojSha256]
91
+ * @param {boolean} [opts.verifyFiles=false]
92
+ * @returns {{ ok: true, buildNumber: number, vision: boolean, mmprojSha256: string|null }}
93
+ */
94
+ export function verifyGlimmerRuntime({
95
+ llamaServerBuild,
96
+ vision = false,
97
+ mmprojPath = null,
98
+ mmprojSha256 = null,
99
+ verifyFiles = false,
100
+ } = {}) {
101
+ const rawBuild = String(llamaServerBuild ?? '').trim();
102
+ const match = /(\d{4,6})/.exec(rawBuild);
103
+ const buildNumber = match ? Number(match[1]) : NaN;
104
+ if (!Number.isFinite(buildNumber) || buildNumber < MIN_LLAMA_SERVER_BUILD) {
105
+ throw inputError(
106
+ 'LANE2_LLAMA_SERVER_TOO_OLD',
107
+ `Muse-Glimmer-30B-A3B (Qwen3.5 architecture) requires llama-server >= b${MIN_LLAMA_SERVER_BUILD}; got "${rawBuild}".`
108
+ );
109
+ }
110
+
111
+ let normalizedMmprojSha = null;
112
+ if (vision) {
113
+ normalizedMmprojSha = normalizeSha256(mmprojSha256);
114
+ if (!mmprojPath || !normalizedMmprojSha) {
115
+ throw inputError(
116
+ 'LANE2_MMPROJ_MISSING',
117
+ 'Vision mode on Muse-Glimmer-30B-A3B requires --mmproj (mmproj-F16.gguf or mmproj-BF16.gguf) with a pinned 64-hex sha256.'
118
+ );
119
+ }
120
+ if (verifyFiles) {
121
+ if (!existsSync(mmprojPath)) {
122
+ throw inputError('LANE2_MMPROJ_MISSING', `mmproj file not found at "${mmprojPath}".`);
123
+ }
124
+ const actualSha = sha256Buffer(readFileSync(mmprojPath));
125
+ if (actualSha !== normalizedMmprojSha) {
126
+ throw inputError(
127
+ 'LANE2_MMPROJ_SHA_MISMATCH',
128
+ `mmproj sha256 mismatch: got ${actualSha}, expected ${normalizedMmprojSha}.`
129
+ );
130
+ }
131
+ }
132
+ }
133
+
134
+ return {
135
+ ok: true,
136
+ buildNumber,
137
+ vision: Boolean(vision),
138
+ mmprojSha256: normalizedMmprojSha,
139
+ };
140
+ }
141
+
142
+ /**
143
+ * Compute the canonical content hash of a canon/prompt draft so any edit after
144
+ * Director ratification is detected by the freeze gate.
145
+ *
146
+ * @param {Object} draft
147
+ * @returns {string}
148
+ */
149
+ const RESERVED_MODEL_KEYS = Object.freeze([
150
+ 'ratification',
151
+ 'judgment',
152
+ 'status',
153
+ 'drafted_by_model',
154
+ 'author_model',
155
+ ]);
156
+
157
+ function stripReservedModelKeys(obj) {
158
+ if (!obj || typeof obj !== 'object' || Array.isArray(obj)) return {};
159
+ const cleaned = { ...obj };
160
+ for (const key of RESERVED_MODEL_KEYS) {
161
+ delete cleaned[key];
162
+ }
163
+ return cleaned;
164
+ }
165
+
166
+ export function computeDraftContentSha256(draft) {
167
+ const payload = {
168
+ class_id: draft.class_id || draft.entry_id || null,
169
+ prompt: draft.prompt || '',
170
+ negative_prompt: draft.negative_prompt || '',
171
+ canon_entry_draft: draft.canon_entry_draft || null,
172
+ messages_sha256: draft.messages_sha256 || null,
173
+ };
174
+ return sha256String(canonicalize(payload));
175
+ }
176
+
177
+ /**
178
+ * Build the request messages for drafting a visual canon class and prompt.
179
+ *
180
+ * @param {Object} opts
181
+ * @param {string} opts.classId
182
+ * @param {string} opts.brief
183
+ * @param {Object} [opts.canonContext]
184
+ * @returns {{ messages: Array<Object>, messagesSha256: string }}
185
+ */
186
+ export function buildCanonDraftMessages({ classId, brief, canonContext = {} }) {
187
+ const messages = [
188
+ {
189
+ role: 'system',
190
+ content:
191
+ 'You are the sdlab Lane 2 canon and prompt drafter (Muse-Glimmer-30B-A3B, Apache-2.0). ' +
192
+ 'Return strict JSON with keys: class_id, canon_entry_draft, prompt, negative_prompt.',
193
+ },
194
+ {
195
+ role: 'user',
196
+ content: JSON.stringify({
197
+ class_id: classId,
198
+ brief,
199
+ canon_context: canonContext,
200
+ }),
201
+ },
202
+ ];
203
+ const messagesSha256 = sha256String(canonicalize(messages));
204
+ return { messages, messagesSha256 };
205
+ }
206
+
207
+ /**
208
+ * Draft a visual canon entry and class prompt via a loopback OpenAI-compatible
209
+ * endpoint (`http://127.0.0.1:<port>/v1/chat/completions`) (G10, §3.2, §11).
210
+ *
211
+ * Always emits `status: "draft"` and `judgment: null`.
212
+ *
213
+ * @param {Object} opts
214
+ * @param {string} opts.endpoint
215
+ * @param {string} opts.classId
216
+ * @param {string} opts.brief
217
+ * @param {Object} [opts.canonContext]
218
+ * @param {Object} [opts.modelPin]
219
+ * @param {string} [opts.createdAt]
220
+ * @returns {Promise<Object>}
221
+ */
222
+ export async function draftCanonClassViaEndpoint({
223
+ endpoint,
224
+ classId,
225
+ brief,
226
+ canonContext = {},
227
+ modelPin = GLIMMER_DEFAULT_MODEL,
228
+ createdAt = new Date().toISOString(),
229
+ }) {
230
+ assertLoopbackEndpoint(endpoint);
231
+ if (!classId || typeof classId !== 'string') {
232
+ throw inputError('LANE2_CLASS_ID_REQUIRED', 'draftCanonClassViaEndpoint requires a non-empty classId.');
233
+ }
234
+ const authorModel = {
235
+ name: modelPin.name || GLIMMER_DEFAULT_MODEL.name,
236
+ repo: modelPin.repo || GLIMMER_DEFAULT_MODEL.repo,
237
+ digest: normalizeSha256(modelPin.digest) || GLIMMER_DEFAULT_MODEL.digest,
238
+ licence_id: modelPin.licence_id || 'Apache-2.0',
239
+ licence_url: modelPin.licence_url || GLIMMER_DEFAULT_MODEL.licence_url,
240
+ date_read: modelPin.date_read || GLIMMER_DEFAULT_MODEL.date_read,
241
+ credit_note: modelPin.credit_note || GLIMMER_CREDIT_NOTE,
242
+ };
243
+ if (authorModel.licence_id !== 'Apache-2.0') {
244
+ throw inputError(
245
+ 'LANE2_LICENCE_INVALID',
246
+ `Lane 2 canon authoring model "${authorModel.name}" must carry licence_id "Apache-2.0" (got "${authorModel.licence_id}").`
247
+ );
248
+ }
249
+
250
+ const { messages, messagesSha256 } = buildCanonDraftMessages({ classId, brief, canonContext });
251
+
252
+ const response = await fetch(endpoint, {
253
+ method: 'POST',
254
+ headers: { 'Content-Type': 'application/json' },
255
+ body: JSON.stringify({
256
+ model: authorModel.name,
257
+ temperature: 0.2,
258
+ top_p: 0.9,
259
+ messages,
260
+ }),
261
+ });
262
+
263
+ if (!response.ok) {
264
+ throw inputError(
265
+ 'LANE2_ENDPOINT_ERROR',
266
+ `Lane 2 endpoint ${endpoint} returned HTTP ${response.status}.`
267
+ );
268
+ }
269
+
270
+ const body = await response.json();
271
+ const rawContent = body?.choices?.[0]?.message?.content || '{}';
272
+ let parsedContent;
273
+ try {
274
+ const rawParsed = typeof rawContent === 'string' ? JSON.parse(rawContent) : rawContent;
275
+ parsedContent = stripReservedModelKeys(rawParsed);
276
+ } catch {
277
+ parsedContent = {
278
+ prompt: String(rawContent),
279
+ negative_prompt: '',
280
+ canon_entry_draft: { id: classId, notes: String(rawContent) },
281
+ };
282
+ }
283
+
284
+ const rawCanonEntry =
285
+ parsedContent.canon_entry_draft &&
286
+ typeof parsedContent.canon_entry_draft === 'object' &&
287
+ !Array.isArray(parsedContent.canon_entry_draft)
288
+ ? stripReservedModelKeys(parsedContent.canon_entry_draft)
289
+ : { id: classId };
290
+ const canonEntryDraft = {
291
+ ...rawCanonEntry,
292
+ id: rawCanonEntry.id || classId,
293
+ status: 'draft',
294
+ drafted_by_model: authorModel.name,
295
+ author_model: authorModel,
296
+ };
297
+
298
+ const draftBase = {
299
+ draft_id: `draft_${classId}`,
300
+ class_id: classId,
301
+ entry_id: classId,
302
+ status: 'draft',
303
+ judgment: null,
304
+ prompt: parsedContent.prompt || '',
305
+ negative_prompt: parsedContent.negative_prompt || '',
306
+ canon_entry_draft: canonEntryDraft,
307
+ drafted_by_model: authorModel.name,
308
+ author_model: authorModel,
309
+ messages_sha256: messagesSha256,
310
+ created_at: createdAt,
311
+ };
312
+
313
+ const draftSha256 = computeDraftContentSha256(draftBase);
314
+ return {
315
+ ...draftBase,
316
+ draft_sha256: draftSha256,
317
+ };
318
+ }
319
+
320
+ /**
321
+ * Write `prompts.jsonl` and its companion `prompts.jsonl.sha256` (§7).
322
+ *
323
+ * @param {string} promptsPath
324
+ * @param {Array<Object>} drafts
325
+ * @returns {{ promptsPath: string, sha256Path: string, sha256: string }}
326
+ */
327
+ export function writePromptsJsonl(promptsPath, drafts) {
328
+ mkdirSync(dirname(promptsPath), { recursive: true });
329
+ const content = drafts.map((d) => JSON.stringify(d)).join('\n') + '\n';
330
+ writeFileSync(promptsPath, content, 'utf-8');
331
+ const digest = sha256String(content);
332
+ const sha256Path = `${promptsPath}.sha256`;
333
+ writeFileSync(sha256Path, `${digest} prompts.jsonl\n`, 'utf-8');
334
+ return { promptsPath, sha256Path, sha256: digest };
335
+ }
336
+
337
+ /**
338
+ * Read `prompts.jsonl` rows from disk.
339
+ *
340
+ * @param {string} promptsPath
341
+ * @returns {Array<Object>}
342
+ */
343
+ export function readPromptsJsonl(promptsPath) {
344
+ const raw = readFileSync(promptsPath, 'utf-8');
345
+ return raw
346
+ .split(/\r?\n/)
347
+ .map((l) => l.trim())
348
+ .filter(Boolean)
349
+ .map((line) => JSON.parse(line));
350
+ }
@@ -91,12 +91,34 @@ export async function runBuild(opts) {
91
91
  const entries = [];
92
92
  const schemaChangeTriggersFullRebuild = await detectSchemaChange(projectRoot, schemas, full);
93
93
 
94
+ // Entity ids must be unique across the ENTIRE canon tree, not just within
95
+ // one schema's subdirectory: they key prompts/<id>.j2, context/<id>.md, and
96
+ // manifest.per_entity_hashes, all of which are flat maps keyed by id alone.
97
+ // Two entries sharing an id (e.g. a forgotten `id:` edit after cloning an
98
+ // existing file) would silently overwrite each other's prompt/context
99
+ // output and collide in the manifest (last write wins), while
100
+ // dataset/all.jsonl would keep rows for both under the one id — so a row
101
+ // permanently points at the wrong entry's content. Catch this at discovery
102
+ // time, before any output is written.
103
+ const seenIds = new Map(); // id -> first-seen filePath
104
+
94
105
  for (const [schemaName, subdir] of Object.entries(config.entity_dirs)) {
95
106
  const dir = join(config.canon_root, subdir);
96
107
  const loaded = await loadCanonEntriesInDir(dir);
97
108
  for (const entry of loaded) {
98
109
  const id = entryId(entry);
99
110
  if (only && !only.includes(id)) continue;
111
+ if (seenIds.has(id)) {
112
+ throw inputError(
113
+ 'CANON_DUPLICATE_ENTITY_ID',
114
+ `Duplicate canon entity id "${id}": both ${seenIds.get(id)} and ${entry.filePath} ` +
115
+ `resolve to this id.`,
116
+ `Give one of these entries a unique "id:" field in its frontmatter. IDs must be ` +
117
+ `unique across the whole canon tree — they key prompts/, context/, and ` +
118
+ `manifest.per_entity_hashes, all flat maps with no per-schema namespacing.`,
119
+ );
120
+ }
121
+ seenIds.set(id, entry.filePath);
100
122
  entries.push({ entry, schemaName, id });
101
123
  }
102
124
  }
@@ -166,9 +188,17 @@ export async function runBuild(opts) {
166
188
  || entry.frontmatter.reference_plate_uri
167
189
  || null;
168
190
 
169
- // Build the row
191
+ // Build the row. `id` is the RESOLVED id (frontmatter.id, falling back to
192
+ // the filename stem — see entryId() in load-entry.js), which is what the
193
+ // prompts/<id>.j2 and context/<id>.md files below are actually named
194
+ // with. canonEntryToRow() (lib/rows.js) reads `entry.id` for entity_id —
195
+ // passing the raw entry.frontmatter would leave entity_id undefined for
196
+ // any entry that relies on the filename-stem fallback (no explicit
197
+ // `id:` field), and serializeRow() drops undefined fields entirely, so
198
+ // the row would orphan itself from its own correctly-named prompt/context
199
+ // files. Overlay the resolved id so the row always carries it.
170
200
  const row = canonEntryToRow({
171
- entry: entry.frontmatter,
201
+ entry: { ...entry.frontmatter, id },
172
202
  schemaKind: rendered.schemaKind,
173
203
  assetPath: refPath,
174
204
  lane,
@@ -212,104 +242,127 @@ export async function runBuild(opts) {
212
242
  }
213
243
 
214
244
  // Write to staging, then atomic rename.
245
+ //
246
+ // M2: everything from here through the rename is wrapped in try/catch.
247
+ // Before this fix, a throw ANYWHERE in this block (most notably
248
+ // readFreezeStatus() below, which throws CANON_FREEZE_STATUS_INVALID on a
249
+ // malformed freeze.status) left `stagingDir` — by then already populated
250
+ // with dataset/prompts/context — orphaned on disk forever: the only
251
+ // cleanup is the `rm(stagingDir, ...)` at the top of THIS function, and
252
+ // that runs against a NEW `${generatedFromDir}.tmp-${process.pid}` path on
253
+ // the next invocation (a different pid), so it can never match the
254
+ // orphan. The catch below removes the in-progress staging directory
255
+ // before rethrowing, and does not touch anything once rename() has
256
+ // succeeded (stagingDir no longer exists at that point, so the block
257
+ // exits normally and the catch never runs).
215
258
  await rm(stagingDir, { recursive: true, force: true });
216
- await mkdir(join(stagingDir, 'dataset'), { recursive: true });
217
- await mkdir(join(stagingDir, 'prompts'), { recursive: true });
218
- await mkdir(join(stagingDir, 'context'), { recursive: true });
219
-
220
- // Write prompts + context
221
- for (const { entry } of entries) {
222
- const r = entry._rendered;
223
- const id = entry._id;
224
- await writeFile(join(stagingDir, 'prompts', `${id}.j2`), r.prompt, 'utf-8');
225
- await writeFile(join(stagingDir, 'context', `${id}.md`), r.context, 'utf-8');
226
- }
259
+ // Declared here (not inside the try block) because they're read again
260
+ // after it, in writeLatestSchemaVersions() and the return value below.
261
+ let frozenEntriesHashes = {};
262
+ let drift = [];
263
+ let manifest;
264
+ try {
265
+ await mkdir(join(stagingDir, 'dataset'), { recursive: true });
266
+ await mkdir(join(stagingDir, 'prompts'), { recursive: true });
267
+ await mkdir(join(stagingDir, 'context'), { recursive: true });
268
+
269
+ // Write prompts + context
270
+ for (const { entry } of entries) {
271
+ const r = entry._rendered;
272
+ const id = entry._id;
273
+ await writeFile(join(stagingDir, 'prompts', `${id}.j2`), r.prompt, 'utf-8');
274
+ await writeFile(join(stagingDir, 'context', `${id}.md`), r.context, 'utf-8');
275
+ }
227
276
 
228
- // Write dataset.jsonl (all + per-lane-per-partition)
229
- const allLines = rows.map((r) => serializeRow(r)).join('\n') + (rows.length ? '\n' : '');
230
- await writeFile(join(stagingDir, 'dataset', 'all.jsonl'), allLines, 'utf-8');
277
+ // Write dataset.jsonl (all + per-lane-per-partition)
278
+ const allLines = rows.map((r) => serializeRow(r)).join('\n') + (rows.length ? '\n' : '');
279
+ await writeFile(join(stagingDir, 'dataset', 'all.jsonl'), allLines, 'utf-8');
231
280
 
232
- // Group by (lane × partition)
233
- const byLaneAndPartition = {};
234
- for (const row of rows) {
235
- const key = `${row.lane}-${row.partition}`;
236
- if (!byLaneAndPartition[key]) byLaneAndPartition[key] = [];
237
- byLaneAndPartition[key].push(row);
238
- }
239
- for (const [key, laneRows] of Object.entries(byLaneAndPartition)) {
240
- const content = laneRows.map((r) => serializeRow(r)).join('\n') + '\n';
241
- await writeFile(join(stagingDir, 'dataset', `${key}.jsonl`), content, 'utf-8');
242
- }
281
+ // Group by (lane × partition)
282
+ const byLaneAndPartition = {};
283
+ for (const row of rows) {
284
+ const key = `${row.lane}-${row.partition}`;
285
+ if (!byLaneAndPartition[key]) byLaneAndPartition[key] = [];
286
+ byLaneAndPartition[key].push(row);
287
+ }
288
+ for (const [key, laneRows] of Object.entries(byLaneAndPartition)) {
289
+ const content = laneRows.map((r) => serializeRow(r)).join('\n') + '\n';
290
+ await writeFile(join(stagingDir, 'dataset', `${key}.jsonl`), content, 'utf-8');
291
+ }
243
292
 
244
- // Freeze drift pass — stamp frozen_entries_hashes (D1 witness chain).
245
- // For every entry with a non-auto freeze status, compute the watch-hash
246
- // of its currently-watched fields. Drift is computed by later builds or
247
- // the `sdlab canon drift` command by diffing their hash against this.
248
- const frozenEntriesHashes = {};
249
- const drift = [];
250
- const priorManifest = await loadPreviousManifest(projectRoot, outputDir);
251
- for (const { entry, schemaName, id } of entries) {
252
- const status = readFreezeStatus(entry.frontmatter);
253
- if (status === 'auto') continue;
254
- const watchFields = resolveWatchFields(entry.frontmatter, schemaName, config);
255
- const watchHash = computeWatchHash(
256
- entry.frontmatter,
257
- watchFields,
258
- schemas[schemaName].version,
259
- );
260
- frozenEntriesHashes[id] = {
261
- status,
262
- watch_hash: watchHash,
263
- watch_fields: watchFields,
264
- locked_at_build: entry.frontmatter.freeze?.locked_at_build || null,
265
- };
266
- // Compute drift against prior build's stamp, if any.
267
- const priorHash = priorManifest?.frozen_entries_hashes?.[id]?.watch_hash;
268
- if (priorHash && priorHash !== watchHash) {
269
- drift.push({
270
- entity_id: id,
271
- schema_kind: schemaName.replace('.schema.json', ''),
293
+ // Freeze drift pass — stamp frozen_entries_hashes (D1 witness chain).
294
+ // For every entry with a non-auto freeze status, compute the watch-hash
295
+ // of its currently-watched fields. Drift is computed by later builds or
296
+ // the `sdlab canon drift` command by diffing their hash against this.
297
+ const priorManifest = await loadPreviousManifest(projectRoot, outputDir);
298
+ for (const { entry, schemaName, id } of entries) {
299
+ const status = readFreezeStatus(entry.frontmatter);
300
+ if (status === 'auto') continue;
301
+ const watchFields = resolveWatchFields(entry.frontmatter, schemaName, config);
302
+ const watchHash = computeWatchHash(
303
+ entry.frontmatter,
304
+ watchFields,
305
+ schemas[schemaName].version,
306
+ );
307
+ frozenEntriesHashes[id] = {
272
308
  status,
273
- prior_hash: priorHash,
274
- current_hash: watchHash,
275
- });
309
+ watch_hash: watchHash,
310
+ watch_fields: watchFields,
311
+ locked_at_build: entry.frontmatter.freeze?.locked_at_build || null,
312
+ };
313
+ // Compute drift against prior build's stamp, if any.
314
+ const priorHash = priorManifest?.frozen_entries_hashes?.[id]?.watch_hash;
315
+ if (priorHash && priorHash !== watchHash) {
316
+ drift.push({
317
+ entity_id: id,
318
+ schema_kind: schemaName.replace('.schema.json', ''),
319
+ status,
320
+ prior_hash: priorHash,
321
+ current_hash: watchHash,
322
+ });
323
+ }
276
324
  }
277
- }
278
325
 
279
- // Write manifest
280
- const manifest = {
281
- schema_version: BUILD_SCHEMA_VERSION,
282
- row_schema_version: ROW_SCHEMA_VERSION,
283
- generated_from: generatedFrom,
284
- built_at: new Date().toISOString(),
285
- project_id: config.project_id,
286
- schema_versions: Object.fromEntries(
287
- Object.entries(schemas).map(([k, v]) => [k, v.version])
288
- ),
289
- build_config_hash: buildConfigHash,
290
- project_config_fingerprint: projectFingerprint,
291
- per_entity_hashes: Object.fromEntries(entries.map(({ entry, id }) => [id, entry._entryHash])),
292
- frozen_entries_hashes: frozenEntriesHashes,
293
- drift_against_prior: drift,
294
- stats: {
295
- entities_total: entries.length,
296
- entities_cached: cached,
297
- entities_rebuilt: rebuilt,
298
- dataset_rows: rows.length,
299
- frozen_entries: Object.keys(frozenEntriesHashes).length,
300
- drifted_entries: drift.length,
301
- schema_change_full_rebuild: schemaChangeTriggersFullRebuild,
302
- },
303
- };
304
- await writeFile(
305
- join(stagingDir, 'manifest.json'),
306
- JSON.stringify(manifest, null, 2) + '\n',
307
- 'utf-8'
308
- );
309
-
310
- // Atomic swap: remove old output, rename staging → output.
311
- await rm(outputDir, { recursive: true, force: true });
312
- await rename(stagingDir, outputDir);
326
+ // Write manifest
327
+ manifest = {
328
+ schema_version: BUILD_SCHEMA_VERSION,
329
+ row_schema_version: ROW_SCHEMA_VERSION,
330
+ generated_from: generatedFrom,
331
+ built_at: new Date().toISOString(),
332
+ project_id: config.project_id,
333
+ schema_versions: Object.fromEntries(
334
+ Object.entries(schemas).map(([k, v]) => [k, v.version])
335
+ ),
336
+ build_config_hash: buildConfigHash,
337
+ project_config_fingerprint: projectFingerprint,
338
+ per_entity_hashes: Object.fromEntries(entries.map(({ entry, id }) => [id, entry._entryHash])),
339
+ frozen_entries_hashes: frozenEntriesHashes,
340
+ drift_against_prior: drift,
341
+ stats: {
342
+ entities_total: entries.length,
343
+ entities_cached: cached,
344
+ entities_rebuilt: rebuilt,
345
+ dataset_rows: rows.length,
346
+ frozen_entries: Object.keys(frozenEntriesHashes).length,
347
+ drifted_entries: drift.length,
348
+ schema_change_full_rebuild: schemaChangeTriggersFullRebuild,
349
+ },
350
+ };
351
+ await writeFile(
352
+ join(stagingDir, 'manifest.json'),
353
+ JSON.stringify(manifest, null, 2) + '\n',
354
+ 'utf-8'
355
+ );
356
+
357
+ // Atomic swap: remove old output, rename staging → output.
358
+ await rm(outputDir, { recursive: true, force: true });
359
+ await rename(stagingDir, outputDir);
360
+ } catch (err) {
361
+ try {
362
+ await rm(stagingDir, { recursive: true, force: true });
363
+ } catch { /* best-effort cleanup — surface the original error below regardless */ }
364
+ throw err;
365
+ }
313
366
 
314
367
  // Update the "latest" schema-version snapshot for change detection next run.
315
368
  await writeLatestSchemaVersions(projectRoot, manifest.schema_versions);