@mcptoolshop/style-dataset-lab 3.3.0 → 3.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/README.es.md +117 -55
- package/README.fr.md +125 -65
- package/README.hi.md +117 -68
- package/README.it.md +119 -58
- package/README.ja.md +124 -74
- package/README.md +73 -7
- package/README.pt-BR.md +128 -63
- package/README.zh.md +116 -65
- package/bin/sdlab.js +100 -4
- package/lib/adapters/ai-toolkit.js +224 -35
- package/lib/adapters/comfyui-runner.js +79 -26
- package/lib/adapters/comfyui-workflows.js +94 -11
- package/lib/adapters/diffusers-lora.js +37 -5
- package/lib/adapters/generic-image-caption.js +43 -5
- package/lib/args.js +16 -1
- package/lib/asset-ingest.js +575 -0
- package/lib/asset-masks.js +323 -0
- package/lib/asset-source.js +1176 -0
- package/lib/batch-modes.js +29 -8
- package/lib/batch-runs.js +29 -1
- package/lib/batch-sheet-render.js +13 -3
- package/lib/brief-compiler.js +19 -0
- package/lib/candidate-sheet.js +327 -0
- package/lib/canon-author.js +350 -0
- package/lib/canon-build/build.js +146 -93
- package/lib/canon-review-gate.js +205 -0
- package/lib/captions.js +57 -0
- package/lib/card.js +74 -2
- package/lib/colab-launcher-guards.js +285 -0
- package/lib/comfyui.js +116 -18
- package/lib/config.js +116 -12
- package/lib/critique-engine.js +36 -2
- package/lib/eligibility.js +121 -9
- package/lib/eval-pack.js +14 -14
- package/lib/eval-runs.js +33 -0
- package/lib/export.js +122 -34
- package/lib/freeze-gate.js +16 -3
- package/lib/freeze-stamp.js +28 -3
- package/lib/generated-provenance.js +194 -6
- package/lib/implementation-packs.js +95 -17
- package/lib/ingest.js +396 -0
- package/lib/judgment-provenance.js +220 -0
- package/lib/lane1-render.js +668 -0
- package/lib/measure.js +422 -0
- package/lib/memory-store.js +663 -0
- package/lib/model-licences.js +305 -0
- package/lib/npy-meta.js +130 -0
- package/lib/paths.js +196 -7
- package/lib/png-meta.js +512 -0
- package/lib/records.js +54 -8
- package/lib/reingest-selected.js +113 -17
- package/lib/reingest.js +27 -3
- package/lib/run-manifest.js +159 -25
- package/lib/run-summary.js +12 -6
- package/lib/runtime-runs.js +45 -4
- package/lib/snapshot.js +114 -15
- package/lib/split.js +364 -48
- package/lib/train-runs.js +214 -0
- package/lib/training-manifests.js +114 -21
- package/lib/training-packages.js +147 -15
- package/lib/verifier.js +3 -0
- package/lib/verify-chain.js +367 -0
- package/lib/workflow-profiles.js +24 -3
- package/package.json +4 -3
- package/runtime/comfyui/txt2img-qwen.json +38 -0
- package/runtime/comfyui/txt2img-standard.json +21 -0
- package/scripts/_resume.js +62 -0
- package/scripts/_shared-args.js +44 -0
- package/scripts/asset-ingest.js +160 -0
- package/scripts/brief-compile.js +2 -1
- package/scripts/brief-show.js +2 -1
- package/scripts/canon-bind.js +2 -2
- package/scripts/canon-build.js +27 -20
- package/scripts/canon-draft.js +82 -0
- package/scripts/canon-drift.js +16 -7
- package/scripts/canon-freeze-status.js +14 -6
- package/scripts/canon-freeze.js +31 -11
- package/scripts/canon-unfreeze.js +17 -9
- package/scripts/card.js +2 -2
- package/scripts/colab/REAL_SERVICE_ASSUMPTIONS.md +14 -0
- package/scripts/colab/ai-toolkit-override-py315.txt +45 -0
- package/scripts/colab/colab_window.sh +587 -0
- package/scripts/colab/lane1_render.py +970 -0
- package/scripts/colab/launch_guards.py +564 -0
- package/scripts/colab/launch_lane1_stub.sh +309 -0
- package/scripts/colab/launch_lane3_stub.sh +366 -0
- package/scripts/colab/remote_runner.py +59 -0
- package/scripts/compare.js +2 -2
- package/scripts/critique.js +16 -1
- package/scripts/curate.js +2 -2
- package/scripts/doctor.js +182 -55
- package/scripts/eligibility.js +42 -2
- package/scripts/eval-pack.js +2 -2
- package/scripts/eval-run.js +2 -2
- package/scripts/export.js +2 -2
- package/scripts/generate-controlnet.js +45 -25
- package/scripts/generate-identity.js +53 -23
- package/scripts/generate-ipadapter.js +44 -23
- package/scripts/generate.js +86 -17
- package/scripts/implementation-pack.js +2 -2
- package/scripts/ingest.js +145 -0
- package/scripts/init.js +73 -13
- package/scripts/measure.js +170 -0
- package/scripts/measure_image.py +458 -0
- package/scripts/migrate-records.js +142 -0
- package/scripts/painterly-test.js +4 -3
- package/scripts/painterly.js +7 -3
- package/scripts/qwen_generate.py +138 -20
- package/scripts/reingest.js +2 -2
- package/scripts/sheet.js +142 -0
- package/scripts/snapshot.js +2 -2
- package/scripts/split.js +2 -2
- package/scripts/training-manifest.js +2 -2
- package/scripts/training-package.js +2 -2
- package/scripts/training-profile.js +2 -2
- package/scripts/verify.js +67 -0
- package/scripts/workflow-list.js +2 -1
- package/scripts/workflow-show.js +2 -1
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lane 2 canon and prompt authoring adapter (DESIGN-v3 §3.2, §9.2 item 7, §11, G10).
|
|
3
|
+
*
|
|
4
|
+
* Calls a local loopback OpenAI-compatible HTTP endpoint (e.g., llama-server or
|
|
5
|
+
* Ollama serving Muse-Glimmer-30B-A3B-GGUF under Apache-2.0) to draft visual
|
|
6
|
+
* canon entries and per-class prompts + negatives.
|
|
7
|
+
*
|
|
8
|
+
* Every model-authored output starts with `status: "draft"` and `judgment: null`,
|
|
9
|
+
* carrying the model name, digest, SPDX licence (`Apache-2.0`), credit note,
|
|
10
|
+
* `messages_sha256`, and `draft_sha256`.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { createHash } from 'node:crypto';
|
|
14
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
15
|
+
import { dirname } from 'node:path';
|
|
16
|
+
import { canonicalize } from './freeze-stamp.js';
|
|
17
|
+
import { inputError as rawInputError } from './errors.js';
|
|
18
|
+
|
|
19
|
+
function inputError(code, msg) {
|
|
20
|
+
return rawInputError(code, `${code}: ${msg}`);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export const MIN_LLAMA_SERVER_BUILD = 10353;
|
|
24
|
+
|
|
25
|
+
export const GLIMMER_CREDIT_NOTE =
|
|
26
|
+
'Canon and prompt drafting assisted by Muse-Glimmer-30B-A3B (GlimmerLabs, Apache-2.0); vision inspection via Qwen3-VL mmproj (Apache-2.0).';
|
|
27
|
+
|
|
28
|
+
export const GLIMMER_DEFAULT_MODEL = Object.freeze({
|
|
29
|
+
name: 'Muse-Glimmer-30B-A3B-Q4_K_M.gguf',
|
|
30
|
+
repo: 'GlimmerLabs/Muse-Glimmer-30B-A3B-GGUF',
|
|
31
|
+
digest: 'sha256:' + '3'.repeat(64),
|
|
32
|
+
licence_id: 'Apache-2.0',
|
|
33
|
+
licence_url: 'https://huggingface.co/GlimmerLabs/Muse-Glimmer-30B-A3B-GGUF/blob/main/LICENSE',
|
|
34
|
+
date_read: '2026-11-05',
|
|
35
|
+
credit_note: GLIMMER_CREDIT_NOTE,
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
export const LOOPBACK_HOSTS = new Set(['127.0.0.1', 'localhost', '::1', '[::1]']);
|
|
39
|
+
|
|
40
|
+
function sha256String(str) {
|
|
41
|
+
return 'sha256:' + createHash('sha256').update(str, 'utf-8').digest('hex');
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function sha256Buffer(buf) {
|
|
45
|
+
return 'sha256:' + createHash('sha256').update(buf).digest('hex');
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function normalizeSha256(val) {
|
|
49
|
+
if (typeof val !== 'string') return null;
|
|
50
|
+
const trimmed = val.trim().toLowerCase();
|
|
51
|
+
const hex = trimmed.startsWith('sha256:') ? trimmed.slice(7) : trimmed;
|
|
52
|
+
if (!/^[0-9a-f]{64}$/.test(hex)) return null;
|
|
53
|
+
return `sha256:${hex}`;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Enforce that the Lane 2 LLM endpoint is strictly loopback-bound (§3.2, §6, §11.3).
|
|
58
|
+
*
|
|
59
|
+
* @param {string} endpointUrl
|
|
60
|
+
* @returns {URL}
|
|
61
|
+
*/
|
|
62
|
+
export function assertLoopbackEndpoint(endpointUrl) {
|
|
63
|
+
let parsed;
|
|
64
|
+
try {
|
|
65
|
+
parsed = new URL(endpointUrl);
|
|
66
|
+
} catch {
|
|
67
|
+
throw inputError(
|
|
68
|
+
'LANE2_NON_LOOPBACK_ENDPOINT',
|
|
69
|
+
`Invalid Lane 2 endpoint URL "${endpointUrl}". Must be http://127.0.0.1:<port>/...`
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
const host = parsed.hostname.toLowerCase();
|
|
73
|
+
if (!LOOPBACK_HOSTS.has(host)) {
|
|
74
|
+
throw inputError(
|
|
75
|
+
'LANE2_NON_LOOPBACK_ENDPOINT',
|
|
76
|
+
`Lane 2 endpoint "${endpointUrl}" binds to non-loopback host "${host}". Must bind to 127.0.0.1.`
|
|
77
|
+
);
|
|
78
|
+
}
|
|
79
|
+
return parsed;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Pre-check the llama.cpp `llama-server` build (`>= b10353` for Qwen3.5 /
|
|
84
|
+
* Muse-Glimmer-30B-A3B) and optional `mmproj` vision projector (§9.2 item 7, §11.3).
|
|
85
|
+
*
|
|
86
|
+
* @param {Object} opts
|
|
87
|
+
* @param {string|number} opts.llamaServerBuild — e.g. 'b10353' or 10400
|
|
88
|
+
* @param {boolean} [opts.vision=false] — whether vision inspection / plate reading is active
|
|
89
|
+
* @param {string} [opts.mmprojPath]
|
|
90
|
+
* @param {string} [opts.mmprojSha256]
|
|
91
|
+
* @param {boolean} [opts.verifyFiles=false]
|
|
92
|
+
* @returns {{ ok: true, buildNumber: number, vision: boolean, mmprojSha256: string|null }}
|
|
93
|
+
*/
|
|
94
|
+
export function verifyGlimmerRuntime({
|
|
95
|
+
llamaServerBuild,
|
|
96
|
+
vision = false,
|
|
97
|
+
mmprojPath = null,
|
|
98
|
+
mmprojSha256 = null,
|
|
99
|
+
verifyFiles = false,
|
|
100
|
+
} = {}) {
|
|
101
|
+
const rawBuild = String(llamaServerBuild ?? '').trim();
|
|
102
|
+
const match = /(\d{4,6})/.exec(rawBuild);
|
|
103
|
+
const buildNumber = match ? Number(match[1]) : NaN;
|
|
104
|
+
if (!Number.isFinite(buildNumber) || buildNumber < MIN_LLAMA_SERVER_BUILD) {
|
|
105
|
+
throw inputError(
|
|
106
|
+
'LANE2_LLAMA_SERVER_TOO_OLD',
|
|
107
|
+
`Muse-Glimmer-30B-A3B (Qwen3.5 architecture) requires llama-server >= b${MIN_LLAMA_SERVER_BUILD}; got "${rawBuild}".`
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
let normalizedMmprojSha = null;
|
|
112
|
+
if (vision) {
|
|
113
|
+
normalizedMmprojSha = normalizeSha256(mmprojSha256);
|
|
114
|
+
if (!mmprojPath || !normalizedMmprojSha) {
|
|
115
|
+
throw inputError(
|
|
116
|
+
'LANE2_MMPROJ_MISSING',
|
|
117
|
+
'Vision mode on Muse-Glimmer-30B-A3B requires --mmproj (mmproj-F16.gguf or mmproj-BF16.gguf) with a pinned 64-hex sha256.'
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
if (verifyFiles) {
|
|
121
|
+
if (!existsSync(mmprojPath)) {
|
|
122
|
+
throw inputError('LANE2_MMPROJ_MISSING', `mmproj file not found at "${mmprojPath}".`);
|
|
123
|
+
}
|
|
124
|
+
const actualSha = sha256Buffer(readFileSync(mmprojPath));
|
|
125
|
+
if (actualSha !== normalizedMmprojSha) {
|
|
126
|
+
throw inputError(
|
|
127
|
+
'LANE2_MMPROJ_SHA_MISMATCH',
|
|
128
|
+
`mmproj sha256 mismatch: got ${actualSha}, expected ${normalizedMmprojSha}.`
|
|
129
|
+
);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return {
|
|
135
|
+
ok: true,
|
|
136
|
+
buildNumber,
|
|
137
|
+
vision: Boolean(vision),
|
|
138
|
+
mmprojSha256: normalizedMmprojSha,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Compute the canonical content hash of a canon/prompt draft so any edit after
|
|
144
|
+
* Director ratification is detected by the freeze gate.
|
|
145
|
+
*
|
|
146
|
+
* @param {Object} draft
|
|
147
|
+
* @returns {string}
|
|
148
|
+
*/
|
|
149
|
+
const RESERVED_MODEL_KEYS = Object.freeze([
|
|
150
|
+
'ratification',
|
|
151
|
+
'judgment',
|
|
152
|
+
'status',
|
|
153
|
+
'drafted_by_model',
|
|
154
|
+
'author_model',
|
|
155
|
+
]);
|
|
156
|
+
|
|
157
|
+
function stripReservedModelKeys(obj) {
|
|
158
|
+
if (!obj || typeof obj !== 'object' || Array.isArray(obj)) return {};
|
|
159
|
+
const cleaned = { ...obj };
|
|
160
|
+
for (const key of RESERVED_MODEL_KEYS) {
|
|
161
|
+
delete cleaned[key];
|
|
162
|
+
}
|
|
163
|
+
return cleaned;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export function computeDraftContentSha256(draft) {
|
|
167
|
+
const payload = {
|
|
168
|
+
class_id: draft.class_id || draft.entry_id || null,
|
|
169
|
+
prompt: draft.prompt || '',
|
|
170
|
+
negative_prompt: draft.negative_prompt || '',
|
|
171
|
+
canon_entry_draft: draft.canon_entry_draft || null,
|
|
172
|
+
messages_sha256: draft.messages_sha256 || null,
|
|
173
|
+
};
|
|
174
|
+
return sha256String(canonicalize(payload));
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Build the request messages for drafting a visual canon class and prompt.
|
|
179
|
+
*
|
|
180
|
+
* @param {Object} opts
|
|
181
|
+
* @param {string} opts.classId
|
|
182
|
+
* @param {string} opts.brief
|
|
183
|
+
* @param {Object} [opts.canonContext]
|
|
184
|
+
* @returns {{ messages: Array<Object>, messagesSha256: string }}
|
|
185
|
+
*/
|
|
186
|
+
export function buildCanonDraftMessages({ classId, brief, canonContext = {} }) {
|
|
187
|
+
const messages = [
|
|
188
|
+
{
|
|
189
|
+
role: 'system',
|
|
190
|
+
content:
|
|
191
|
+
'You are the sdlab Lane 2 canon and prompt drafter (Muse-Glimmer-30B-A3B, Apache-2.0). ' +
|
|
192
|
+
'Return strict JSON with keys: class_id, canon_entry_draft, prompt, negative_prompt.',
|
|
193
|
+
},
|
|
194
|
+
{
|
|
195
|
+
role: 'user',
|
|
196
|
+
content: JSON.stringify({
|
|
197
|
+
class_id: classId,
|
|
198
|
+
brief,
|
|
199
|
+
canon_context: canonContext,
|
|
200
|
+
}),
|
|
201
|
+
},
|
|
202
|
+
];
|
|
203
|
+
const messagesSha256 = sha256String(canonicalize(messages));
|
|
204
|
+
return { messages, messagesSha256 };
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Draft a visual canon entry and class prompt via a loopback OpenAI-compatible
|
|
209
|
+
* endpoint (`http://127.0.0.1:<port>/v1/chat/completions`) (G10, §3.2, §11).
|
|
210
|
+
*
|
|
211
|
+
* Always emits `status: "draft"` and `judgment: null`.
|
|
212
|
+
*
|
|
213
|
+
* @param {Object} opts
|
|
214
|
+
* @param {string} opts.endpoint
|
|
215
|
+
* @param {string} opts.classId
|
|
216
|
+
* @param {string} opts.brief
|
|
217
|
+
* @param {Object} [opts.canonContext]
|
|
218
|
+
* @param {Object} [opts.modelPin]
|
|
219
|
+
* @param {string} [opts.createdAt]
|
|
220
|
+
* @returns {Promise<Object>}
|
|
221
|
+
*/
|
|
222
|
+
export async function draftCanonClassViaEndpoint({
|
|
223
|
+
endpoint,
|
|
224
|
+
classId,
|
|
225
|
+
brief,
|
|
226
|
+
canonContext = {},
|
|
227
|
+
modelPin = GLIMMER_DEFAULT_MODEL,
|
|
228
|
+
createdAt = new Date().toISOString(),
|
|
229
|
+
}) {
|
|
230
|
+
assertLoopbackEndpoint(endpoint);
|
|
231
|
+
if (!classId || typeof classId !== 'string') {
|
|
232
|
+
throw inputError('LANE2_CLASS_ID_REQUIRED', 'draftCanonClassViaEndpoint requires a non-empty classId.');
|
|
233
|
+
}
|
|
234
|
+
const authorModel = {
|
|
235
|
+
name: modelPin.name || GLIMMER_DEFAULT_MODEL.name,
|
|
236
|
+
repo: modelPin.repo || GLIMMER_DEFAULT_MODEL.repo,
|
|
237
|
+
digest: normalizeSha256(modelPin.digest) || GLIMMER_DEFAULT_MODEL.digest,
|
|
238
|
+
licence_id: modelPin.licence_id || 'Apache-2.0',
|
|
239
|
+
licence_url: modelPin.licence_url || GLIMMER_DEFAULT_MODEL.licence_url,
|
|
240
|
+
date_read: modelPin.date_read || GLIMMER_DEFAULT_MODEL.date_read,
|
|
241
|
+
credit_note: modelPin.credit_note || GLIMMER_CREDIT_NOTE,
|
|
242
|
+
};
|
|
243
|
+
if (authorModel.licence_id !== 'Apache-2.0') {
|
|
244
|
+
throw inputError(
|
|
245
|
+
'LANE2_LICENCE_INVALID',
|
|
246
|
+
`Lane 2 canon authoring model "${authorModel.name}" must carry licence_id "Apache-2.0" (got "${authorModel.licence_id}").`
|
|
247
|
+
);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
const { messages, messagesSha256 } = buildCanonDraftMessages({ classId, brief, canonContext });
|
|
251
|
+
|
|
252
|
+
const response = await fetch(endpoint, {
|
|
253
|
+
method: 'POST',
|
|
254
|
+
headers: { 'Content-Type': 'application/json' },
|
|
255
|
+
body: JSON.stringify({
|
|
256
|
+
model: authorModel.name,
|
|
257
|
+
temperature: 0.2,
|
|
258
|
+
top_p: 0.9,
|
|
259
|
+
messages,
|
|
260
|
+
}),
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
if (!response.ok) {
|
|
264
|
+
throw inputError(
|
|
265
|
+
'LANE2_ENDPOINT_ERROR',
|
|
266
|
+
`Lane 2 endpoint ${endpoint} returned HTTP ${response.status}.`
|
|
267
|
+
);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
const body = await response.json();
|
|
271
|
+
const rawContent = body?.choices?.[0]?.message?.content || '{}';
|
|
272
|
+
let parsedContent;
|
|
273
|
+
try {
|
|
274
|
+
const rawParsed = typeof rawContent === 'string' ? JSON.parse(rawContent) : rawContent;
|
|
275
|
+
parsedContent = stripReservedModelKeys(rawParsed);
|
|
276
|
+
} catch {
|
|
277
|
+
parsedContent = {
|
|
278
|
+
prompt: String(rawContent),
|
|
279
|
+
negative_prompt: '',
|
|
280
|
+
canon_entry_draft: { id: classId, notes: String(rawContent) },
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
const rawCanonEntry =
|
|
285
|
+
parsedContent.canon_entry_draft &&
|
|
286
|
+
typeof parsedContent.canon_entry_draft === 'object' &&
|
|
287
|
+
!Array.isArray(parsedContent.canon_entry_draft)
|
|
288
|
+
? stripReservedModelKeys(parsedContent.canon_entry_draft)
|
|
289
|
+
: { id: classId };
|
|
290
|
+
const canonEntryDraft = {
|
|
291
|
+
...rawCanonEntry,
|
|
292
|
+
id: rawCanonEntry.id || classId,
|
|
293
|
+
status: 'draft',
|
|
294
|
+
drafted_by_model: authorModel.name,
|
|
295
|
+
author_model: authorModel,
|
|
296
|
+
};
|
|
297
|
+
|
|
298
|
+
const draftBase = {
|
|
299
|
+
draft_id: `draft_${classId}`,
|
|
300
|
+
class_id: classId,
|
|
301
|
+
entry_id: classId,
|
|
302
|
+
status: 'draft',
|
|
303
|
+
judgment: null,
|
|
304
|
+
prompt: parsedContent.prompt || '',
|
|
305
|
+
negative_prompt: parsedContent.negative_prompt || '',
|
|
306
|
+
canon_entry_draft: canonEntryDraft,
|
|
307
|
+
drafted_by_model: authorModel.name,
|
|
308
|
+
author_model: authorModel,
|
|
309
|
+
messages_sha256: messagesSha256,
|
|
310
|
+
created_at: createdAt,
|
|
311
|
+
};
|
|
312
|
+
|
|
313
|
+
const draftSha256 = computeDraftContentSha256(draftBase);
|
|
314
|
+
return {
|
|
315
|
+
...draftBase,
|
|
316
|
+
draft_sha256: draftSha256,
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Write `prompts.jsonl` and its companion `prompts.jsonl.sha256` (§7).
|
|
322
|
+
*
|
|
323
|
+
* @param {string} promptsPath
|
|
324
|
+
* @param {Array<Object>} drafts
|
|
325
|
+
* @returns {{ promptsPath: string, sha256Path: string, sha256: string }}
|
|
326
|
+
*/
|
|
327
|
+
export function writePromptsJsonl(promptsPath, drafts) {
|
|
328
|
+
mkdirSync(dirname(promptsPath), { recursive: true });
|
|
329
|
+
const content = drafts.map((d) => JSON.stringify(d)).join('\n') + '\n';
|
|
330
|
+
writeFileSync(promptsPath, content, 'utf-8');
|
|
331
|
+
const digest = sha256String(content);
|
|
332
|
+
const sha256Path = `${promptsPath}.sha256`;
|
|
333
|
+
writeFileSync(sha256Path, `${digest} prompts.jsonl\n`, 'utf-8');
|
|
334
|
+
return { promptsPath, sha256Path, sha256: digest };
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* Read `prompts.jsonl` rows from disk.
|
|
339
|
+
*
|
|
340
|
+
* @param {string} promptsPath
|
|
341
|
+
* @returns {Array<Object>}
|
|
342
|
+
*/
|
|
343
|
+
export function readPromptsJsonl(promptsPath) {
|
|
344
|
+
const raw = readFileSync(promptsPath, 'utf-8');
|
|
345
|
+
return raw
|
|
346
|
+
.split(/\r?\n/)
|
|
347
|
+
.map((l) => l.trim())
|
|
348
|
+
.filter(Boolean)
|
|
349
|
+
.map((line) => JSON.parse(line));
|
|
350
|
+
}
|
package/lib/canon-build/build.js
CHANGED
|
@@ -91,12 +91,34 @@ export async function runBuild(opts) {
|
|
|
91
91
|
const entries = [];
|
|
92
92
|
const schemaChangeTriggersFullRebuild = await detectSchemaChange(projectRoot, schemas, full);
|
|
93
93
|
|
|
94
|
+
// Entity ids must be unique across the ENTIRE canon tree, not just within
|
|
95
|
+
// one schema's subdirectory: they key prompts/<id>.j2, context/<id>.md, and
|
|
96
|
+
// manifest.per_entity_hashes, all of which are flat maps keyed by id alone.
|
|
97
|
+
// Two entries sharing an id (e.g. a forgotten `id:` edit after cloning an
|
|
98
|
+
// existing file) would silently overwrite each other's prompt/context
|
|
99
|
+
// output and collide in the manifest (last write wins), while
|
|
100
|
+
// dataset/all.jsonl would keep rows for both under the one id — so a row
|
|
101
|
+
// permanently points at the wrong entry's content. Catch this at discovery
|
|
102
|
+
// time, before any output is written.
|
|
103
|
+
const seenIds = new Map(); // id -> first-seen filePath
|
|
104
|
+
|
|
94
105
|
for (const [schemaName, subdir] of Object.entries(config.entity_dirs)) {
|
|
95
106
|
const dir = join(config.canon_root, subdir);
|
|
96
107
|
const loaded = await loadCanonEntriesInDir(dir);
|
|
97
108
|
for (const entry of loaded) {
|
|
98
109
|
const id = entryId(entry);
|
|
99
110
|
if (only && !only.includes(id)) continue;
|
|
111
|
+
if (seenIds.has(id)) {
|
|
112
|
+
throw inputError(
|
|
113
|
+
'CANON_DUPLICATE_ENTITY_ID',
|
|
114
|
+
`Duplicate canon entity id "${id}": both ${seenIds.get(id)} and ${entry.filePath} ` +
|
|
115
|
+
`resolve to this id.`,
|
|
116
|
+
`Give one of these entries a unique "id:" field in its frontmatter. IDs must be ` +
|
|
117
|
+
`unique across the whole canon tree — they key prompts/, context/, and ` +
|
|
118
|
+
`manifest.per_entity_hashes, all flat maps with no per-schema namespacing.`,
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
seenIds.set(id, entry.filePath);
|
|
100
122
|
entries.push({ entry, schemaName, id });
|
|
101
123
|
}
|
|
102
124
|
}
|
|
@@ -166,9 +188,17 @@ export async function runBuild(opts) {
|
|
|
166
188
|
|| entry.frontmatter.reference_plate_uri
|
|
167
189
|
|| null;
|
|
168
190
|
|
|
169
|
-
// Build the row
|
|
191
|
+
// Build the row. `id` is the RESOLVED id (frontmatter.id, falling back to
|
|
192
|
+
// the filename stem — see entryId() in load-entry.js), which is what the
|
|
193
|
+
// prompts/<id>.j2 and context/<id>.md files below are actually named
|
|
194
|
+
// with. canonEntryToRow() (lib/rows.js) reads `entry.id` for entity_id —
|
|
195
|
+
// passing the raw entry.frontmatter would leave entity_id undefined for
|
|
196
|
+
// any entry that relies on the filename-stem fallback (no explicit
|
|
197
|
+
// `id:` field), and serializeRow() drops undefined fields entirely, so
|
|
198
|
+
// the row would orphan itself from its own correctly-named prompt/context
|
|
199
|
+
// files. Overlay the resolved id so the row always carries it.
|
|
170
200
|
const row = canonEntryToRow({
|
|
171
|
-
entry: entry.frontmatter,
|
|
201
|
+
entry: { ...entry.frontmatter, id },
|
|
172
202
|
schemaKind: rendered.schemaKind,
|
|
173
203
|
assetPath: refPath,
|
|
174
204
|
lane,
|
|
@@ -212,104 +242,127 @@ export async function runBuild(opts) {
|
|
|
212
242
|
}
|
|
213
243
|
|
|
214
244
|
// Write to staging, then atomic rename.
|
|
245
|
+
//
|
|
246
|
+
// M2: everything from here through the rename is wrapped in try/catch.
|
|
247
|
+
// Before this fix, a throw ANYWHERE in this block (most notably
|
|
248
|
+
// readFreezeStatus() below, which throws CANON_FREEZE_STATUS_INVALID on a
|
|
249
|
+
// malformed freeze.status) left `stagingDir` — by then already populated
|
|
250
|
+
// with dataset/prompts/context — orphaned on disk forever: the only
|
|
251
|
+
// cleanup is the `rm(stagingDir, ...)` at the top of THIS function, and
|
|
252
|
+
// that runs against a NEW `${generatedFromDir}.tmp-${process.pid}` path on
|
|
253
|
+
// the next invocation (a different pid), so it can never match the
|
|
254
|
+
// orphan. The catch below removes the in-progress staging directory
|
|
255
|
+
// before rethrowing, and does not touch anything once rename() has
|
|
256
|
+
// succeeded (stagingDir no longer exists at that point, so the block
|
|
257
|
+
// exits normally and the catch never runs).
|
|
215
258
|
await rm(stagingDir, { recursive: true, force: true });
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
await
|
|
225
|
-
|
|
226
|
-
|
|
259
|
+
// Declared here (not inside the try block) because they're read again
|
|
260
|
+
// after it, in writeLatestSchemaVersions() and the return value below.
|
|
261
|
+
let frozenEntriesHashes = {};
|
|
262
|
+
let drift = [];
|
|
263
|
+
let manifest;
|
|
264
|
+
try {
|
|
265
|
+
await mkdir(join(stagingDir, 'dataset'), { recursive: true });
|
|
266
|
+
await mkdir(join(stagingDir, 'prompts'), { recursive: true });
|
|
267
|
+
await mkdir(join(stagingDir, 'context'), { recursive: true });
|
|
268
|
+
|
|
269
|
+
// Write prompts + context
|
|
270
|
+
for (const { entry } of entries) {
|
|
271
|
+
const r = entry._rendered;
|
|
272
|
+
const id = entry._id;
|
|
273
|
+
await writeFile(join(stagingDir, 'prompts', `${id}.j2`), r.prompt, 'utf-8');
|
|
274
|
+
await writeFile(join(stagingDir, 'context', `${id}.md`), r.context, 'utf-8');
|
|
275
|
+
}
|
|
227
276
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
277
|
+
// Write dataset.jsonl (all + per-lane-per-partition)
|
|
278
|
+
const allLines = rows.map((r) => serializeRow(r)).join('\n') + (rows.length ? '\n' : '');
|
|
279
|
+
await writeFile(join(stagingDir, 'dataset', 'all.jsonl'), allLines, 'utf-8');
|
|
231
280
|
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
281
|
+
// Group by (lane × partition)
|
|
282
|
+
const byLaneAndPartition = {};
|
|
283
|
+
for (const row of rows) {
|
|
284
|
+
const key = `${row.lane}-${row.partition}`;
|
|
285
|
+
if (!byLaneAndPartition[key]) byLaneAndPartition[key] = [];
|
|
286
|
+
byLaneAndPartition[key].push(row);
|
|
287
|
+
}
|
|
288
|
+
for (const [key, laneRows] of Object.entries(byLaneAndPartition)) {
|
|
289
|
+
const content = laneRows.map((r) => serializeRow(r)).join('\n') + '\n';
|
|
290
|
+
await writeFile(join(stagingDir, 'dataset', `${key}.jsonl`), content, 'utf-8');
|
|
291
|
+
}
|
|
243
292
|
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
);
|
|
260
|
-
frozenEntriesHashes[id] = {
|
|
261
|
-
status,
|
|
262
|
-
watch_hash: watchHash,
|
|
263
|
-
watch_fields: watchFields,
|
|
264
|
-
locked_at_build: entry.frontmatter.freeze?.locked_at_build || null,
|
|
265
|
-
};
|
|
266
|
-
// Compute drift against prior build's stamp, if any.
|
|
267
|
-
const priorHash = priorManifest?.frozen_entries_hashes?.[id]?.watch_hash;
|
|
268
|
-
if (priorHash && priorHash !== watchHash) {
|
|
269
|
-
drift.push({
|
|
270
|
-
entity_id: id,
|
|
271
|
-
schema_kind: schemaName.replace('.schema.json', ''),
|
|
293
|
+
// Freeze drift pass — stamp frozen_entries_hashes (D1 witness chain).
|
|
294
|
+
// For every entry with a non-auto freeze status, compute the watch-hash
|
|
295
|
+
// of its currently-watched fields. Drift is computed by later builds or
|
|
296
|
+
// the `sdlab canon drift` command by diffing their hash against this.
|
|
297
|
+
const priorManifest = await loadPreviousManifest(projectRoot, outputDir);
|
|
298
|
+
for (const { entry, schemaName, id } of entries) {
|
|
299
|
+
const status = readFreezeStatus(entry.frontmatter);
|
|
300
|
+
if (status === 'auto') continue;
|
|
301
|
+
const watchFields = resolveWatchFields(entry.frontmatter, schemaName, config);
|
|
302
|
+
const watchHash = computeWatchHash(
|
|
303
|
+
entry.frontmatter,
|
|
304
|
+
watchFields,
|
|
305
|
+
schemas[schemaName].version,
|
|
306
|
+
);
|
|
307
|
+
frozenEntriesHashes[id] = {
|
|
272
308
|
status,
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
309
|
+
watch_hash: watchHash,
|
|
310
|
+
watch_fields: watchFields,
|
|
311
|
+
locked_at_build: entry.frontmatter.freeze?.locked_at_build || null,
|
|
312
|
+
};
|
|
313
|
+
// Compute drift against prior build's stamp, if any.
|
|
314
|
+
const priorHash = priorManifest?.frozen_entries_hashes?.[id]?.watch_hash;
|
|
315
|
+
if (priorHash && priorHash !== watchHash) {
|
|
316
|
+
drift.push({
|
|
317
|
+
entity_id: id,
|
|
318
|
+
schema_kind: schemaName.replace('.schema.json', ''),
|
|
319
|
+
status,
|
|
320
|
+
prior_hash: priorHash,
|
|
321
|
+
current_hash: watchHash,
|
|
322
|
+
});
|
|
323
|
+
}
|
|
276
324
|
}
|
|
277
|
-
}
|
|
278
325
|
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
326
|
+
// Write manifest
|
|
327
|
+
manifest = {
|
|
328
|
+
schema_version: BUILD_SCHEMA_VERSION,
|
|
329
|
+
row_schema_version: ROW_SCHEMA_VERSION,
|
|
330
|
+
generated_from: generatedFrom,
|
|
331
|
+
built_at: new Date().toISOString(),
|
|
332
|
+
project_id: config.project_id,
|
|
333
|
+
schema_versions: Object.fromEntries(
|
|
334
|
+
Object.entries(schemas).map(([k, v]) => [k, v.version])
|
|
335
|
+
),
|
|
336
|
+
build_config_hash: buildConfigHash,
|
|
337
|
+
project_config_fingerprint: projectFingerprint,
|
|
338
|
+
per_entity_hashes: Object.fromEntries(entries.map(({ entry, id }) => [id, entry._entryHash])),
|
|
339
|
+
frozen_entries_hashes: frozenEntriesHashes,
|
|
340
|
+
drift_against_prior: drift,
|
|
341
|
+
stats: {
|
|
342
|
+
entities_total: entries.length,
|
|
343
|
+
entities_cached: cached,
|
|
344
|
+
entities_rebuilt: rebuilt,
|
|
345
|
+
dataset_rows: rows.length,
|
|
346
|
+
frozen_entries: Object.keys(frozenEntriesHashes).length,
|
|
347
|
+
drifted_entries: drift.length,
|
|
348
|
+
schema_change_full_rebuild: schemaChangeTriggersFullRebuild,
|
|
349
|
+
},
|
|
350
|
+
};
|
|
351
|
+
await writeFile(
|
|
352
|
+
join(stagingDir, 'manifest.json'),
|
|
353
|
+
JSON.stringify(manifest, null, 2) + '\n',
|
|
354
|
+
'utf-8'
|
|
355
|
+
);
|
|
356
|
+
|
|
357
|
+
// Atomic swap: remove old output, rename staging → output.
|
|
358
|
+
await rm(outputDir, { recursive: true, force: true });
|
|
359
|
+
await rename(stagingDir, outputDir);
|
|
360
|
+
} catch (err) {
|
|
361
|
+
try {
|
|
362
|
+
await rm(stagingDir, { recursive: true, force: true });
|
|
363
|
+
} catch { /* best-effort cleanup — surface the original error below regardless */ }
|
|
364
|
+
throw err;
|
|
365
|
+
}
|
|
313
366
|
|
|
314
367
|
// Update the "latest" schema-version snapshot for change detection next run.
|
|
315
368
|
await writeLatestSchemaVersions(projectRoot, manifest.schema_versions);
|