@mcptoolshop/style-dataset-lab 3.1.0 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/CHANGELOG.md +37 -0
  2. package/README.es.md +84 -62
  3. package/README.fr.md +85 -63
  4. package/README.hi.md +97 -77
  5. package/README.it.md +86 -64
  6. package/README.ja.md +94 -72
  7. package/README.md +22 -0
  8. package/README.pt-BR.md +83 -61
  9. package/README.zh.md +88 -66
  10. package/bin/sdlab.js +71 -0
  11. package/lib/adapters/ai-toolkit.js +209 -0
  12. package/lib/adapters/comfyui-workflows.js +133 -2
  13. package/lib/adapters/diffusers-lora.js +46 -66
  14. package/lib/adapters/generic-image-caption.js +22 -50
  15. package/lib/canon-build/build.js +450 -0
  16. package/lib/canon-build/emitters/base.js +122 -0
  17. package/lib/canon-build/emitters/character.js +167 -0
  18. package/lib/canon-build/emitters/deity.js +165 -0
  19. package/lib/canon-build/emitters/faction.js +161 -0
  20. package/lib/canon-build/emitters/index.js +38 -0
  21. package/lib/canon-build/emitters/location.js +153 -0
  22. package/lib/canon-build/emitters/monster.js +131 -0
  23. package/lib/canon-build/emitters/relic.js +142 -0
  24. package/lib/canon-build/emitters/ship.js +154 -0
  25. package/lib/canon-build/emitters/species.js +165 -0
  26. package/lib/canon-build/frontmatter.js +76 -0
  27. package/lib/canon-build/lane-resolve.js +64 -0
  28. package/lib/canon-build/load-config.js +117 -0
  29. package/lib/canon-build/load-entry.js +62 -0
  30. package/lib/canon-build/load-schema.js +46 -0
  31. package/lib/captions.js +197 -0
  32. package/lib/config.js +19 -0
  33. package/lib/freeze-events.js +109 -0
  34. package/lib/freeze-gate.js +165 -0
  35. package/lib/freeze-stamp.js +161 -0
  36. package/lib/rows.js +228 -0
  37. package/lib/training-adapters.js +5 -0
  38. package/lib/training-packages.js +24 -18
  39. package/lib/training-profiles.js +33 -0
  40. package/package.json +12 -3
  41. package/scripts/_batch_cut.py +71 -0
  42. package/scripts/_seam_cutout.py +74 -0
  43. package/scripts/canon-build.js +88 -0
  44. package/scripts/canon-drift.js +193 -0
  45. package/scripts/canon-freeze-status.js +85 -0
  46. package/scripts/canon-freeze.js +211 -0
  47. package/scripts/canon-unfreeze.js +137 -0
  48. package/scripts/curate.js +12 -0
  49. package/scripts/generate-controlnet.js +12 -0
  50. package/scripts/generate-identity.js +20 -4
  51. package/scripts/generate-ipadapter.js +11 -0
  52. package/scripts/generate.js +238 -102
  53. package/scripts/qwen_generate.py +124 -0
@@ -0,0 +1,165 @@
1
+ /**
2
+ * Freeze enforcement gate (D3).
3
+ *
4
+ * Called from CLI commands before any write to a canon-bound asset or record.
5
+ * Semantics:
6
+ * - frozen → block outright (CANON_ENTRY_FROZEN)
7
+ * - soft-advisory → block unless --i-know + --reason; on override, append
8
+ * a 'bypass' event to freeze-events.jsonl and proceed
9
+ * - auto → no block
10
+ * - on-canon-change → no block at write time; canon-build surfaces drift
11
+ * in its summary output (enforcement lives at the build
12
+ * boundary, not the CLI boundary, because drift is a
13
+ * build-observable condition)
14
+ *
15
+ * The canon reverse-map helpers resolve an action's identifier (subject id,
16
+ * asset path) to a canon entry so the gate can look up its freeze state.
17
+ * Actions without a canon binding (prompt-pack-driven generate, reingest of
18
+ * un-bound generated outputs) skip the gate naturally: no entry = nothing
19
+ * to protect.
20
+ */
21
+
22
+ import { join } from 'node:path';
23
+ import { existsSync } from 'node:fs';
24
+ import { loadBuildConfig } from './canon-build/load-config.js';
25
+ import { loadCanonEntriesInDir, entryId } from './canon-build/load-entry.js';
26
+ import { readFreezeStatus } from './freeze-stamp.js';
27
+ import { appendEvent } from './freeze-events.js';
28
+ import { inputError } from './errors.js';
29
+
30
+ /**
31
+ * Load every canon entry in a project, keyed by schema.
32
+ *
33
+ * Returns a flat array of `{ entry, schemaName }` objects.
34
+ * Returns [] (no-op) when the project has no canon-build config — meaning
35
+ * the project isn't canon-integrated yet, so the gate has nothing to enforce.
36
+ */
37
+ export async function loadAllCanonEntries(projectRoot) {
38
+ const configPath = join(projectRoot, 'canon-build', 'config.json');
39
+ if (!existsSync(configPath)) return [];
40
+ let config;
41
+ try {
42
+ config = await loadBuildConfig(configPath, projectRoot);
43
+ } catch {
44
+ // Invalid config → degrade to "no gate" rather than blocking every write.
45
+ // The operator will hit the config error on `sdlab canon build`.
46
+ return [];
47
+ }
48
+ const entries = [];
49
+ for (const [schemaName, subdir] of Object.entries(config.entity_dirs)) {
50
+ const dir = join(config.canon_root, subdir);
51
+ const loaded = await loadCanonEntriesInDir(dir);
52
+ for (const entry of loaded) {
53
+ entries.push({ entry, schemaName });
54
+ }
55
+ }
56
+ return entries;
57
+ }
58
+
59
+ /**
60
+ * Find the canon entry whose `id` matches the given subject identifier.
61
+ * Returns { entry, schemaName } or null.
62
+ */
63
+ export async function resolveEntryBySubject(projectRoot, subjectId) {
64
+ if (!subjectId) return null;
65
+ const all = await loadAllCanonEntries(projectRoot);
66
+ const hit = all.find(({ entry }) => entryId(entry) === subjectId);
67
+ return hit || null;
68
+ }
69
+
70
+ /**
71
+ * Find the canon entry whose `visual.reference_plate_uri` (or top-level
72
+ * `reference_plate_uri`, for monsters) equals the given asset path.
73
+ * Returns { entry, schemaName } or null.
74
+ */
75
+ export async function resolveEntryByAssetPath(projectRoot, assetPath) {
76
+ if (!assetPath) return null;
77
+ const all = await loadAllCanonEntries(projectRoot);
78
+ const normalized = assetPath.replace(/\\/g, '/');
79
+ const hit = all.find(({ entry }) => {
80
+ const fm = entry.frontmatter;
81
+ const plate = (fm.visual?.reference_plate_uri || fm.reference_plate_uri || '').replace(/\\/g, '/');
82
+ return plate && plate === normalized;
83
+ });
84
+ return hit || null;
85
+ }
86
+
87
+ /**
88
+ * Core gate. Throws CANON_ENTRY_FROZEN on block; on soft-advisory bypass,
89
+ * appends a 'bypass' event and returns without throwing.
90
+ *
91
+ * @param {Object} args
92
+ * @param {string} args.projectRoot
93
+ * @param {Object|null} args.resolved — { entry, schemaName } or null
94
+ * @param {string} args.action — short verb for error context ("generate", "painterly", "curate", ...)
95
+ * @param {boolean} [args.allowSoftAdvisoryBypass] — the command supports --i-know
96
+ * @param {string} [args.bypassReason] — value of --reason when bypassing
97
+ * @param {string} [args.by] — who is invoking (default 'mike')
98
+ */
99
+ export async function assertNotFrozen({
100
+ projectRoot,
101
+ resolved,
102
+ action,
103
+ allowSoftAdvisoryBypass = false,
104
+ bypassReason = null,
105
+ by = 'mike',
106
+ }) {
107
+ if (!resolved || !resolved.entry) return; // nothing canon-bound; no gate
108
+ const fm = resolved.entry.frontmatter;
109
+ const status = readFreezeStatus(fm);
110
+ const id = entryId(resolved.entry);
111
+
112
+ if (status === 'frozen') {
113
+ throw inputError(
114
+ 'CANON_ENTRY_FROZEN',
115
+ `Canon entry "${id}" is frozen; "${action}" refuses to proceed.`,
116
+ `Unfreeze with: sdlab canon unfreeze ${id} --reason "<why>". The override will be appended to canon-build/freeze-events.jsonl as an audit record.`,
117
+ );
118
+ }
119
+
120
+ if (status === 'soft-advisory') {
121
+ if (!allowSoftAdvisoryBypass) {
122
+ throw inputError(
123
+ 'CANON_ENTRY_FROZEN',
124
+ `Canon entry "${id}" is soft-advisory frozen; "${action}" refuses to proceed.`,
125
+ `This command does not support a --i-know bypass. Run "sdlab canon unfreeze ${id} --reason \\"<why>\\"" for a proper unfreeze, or pick a command that accepts --i-know.`,
126
+ );
127
+ }
128
+ if (!bypassReason) {
129
+ throw inputError(
130
+ 'CANON_ENTRY_FROZEN',
131
+ `Canon entry "${id}" is soft-advisory frozen; "${action}" requires --i-know AND --reason to bypass.`,
132
+ `Re-run with: --i-know --reason "<short reason>". The bypass will be logged to canon-build/freeze-events.jsonl.`,
133
+ );
134
+ }
135
+ // Bypass allowed — log and proceed.
136
+ await appendEvent(projectRoot, {
137
+ type: 'bypass',
138
+ entity_id: id,
139
+ schema_kind: resolved.schemaName?.replace('.schema.json', ''),
140
+ by,
141
+ reason: bypassReason,
142
+ prior_status: 'soft-advisory',
143
+ new_status: 'soft-advisory',
144
+ });
145
+ return;
146
+ }
147
+
148
+ // auto and on-canon-change: no write-side block.
149
+ }
150
+
151
+ /**
152
+ * Convenience wrapper: resolve by subject id, then gate.
153
+ */
154
+ export async function assertNotFrozenBySubject(projectRoot, subjectId, options = {}) {
155
+ const resolved = await resolveEntryBySubject(projectRoot, subjectId);
156
+ return assertNotFrozen({ projectRoot, resolved, ...options });
157
+ }
158
+
159
+ /**
160
+ * Convenience wrapper: resolve by asset_path, then gate.
161
+ */
162
+ export async function assertNotFrozenByAssetPath(projectRoot, assetPath, options = {}) {
163
+ const resolved = await resolveEntryByAssetPath(projectRoot, assetPath);
164
+ return assertNotFrozen({ projectRoot, resolved, ...options });
165
+ }
@@ -0,0 +1,161 @@
1
+ /**
2
+ * Freeze block helpers — read/write the `freeze` block on a canon entry,
3
+ * compute watch-field hashes for the witness chain (D1).
4
+ *
5
+ * The `freeze` block (schema v2.0.0+) is the sole freeze surface on canon
6
+ * entries; the legacy `visual_locked_at` + `regen_policy` fields have been
7
+ * removed per post-research D10. See
8
+ * memory/freeze-tooling-research-2026-04-24.md for the full contract.
9
+ */
10
+
11
+ import { createHash } from 'node:crypto';
12
+ import { getByPath } from './canon-build/load-config.js';
13
+
14
+ /**
15
+ * The four legal freeze statuses. Entries without a `freeze` block behave
16
+ * as `auto` (permissive default — see post-research D8).
17
+ */
18
+ export const FREEZE_STATUSES = Object.freeze(['auto', 'frozen', 'soft-advisory', 'on-canon-change']);
19
+
20
+ /**
21
+ * Read the normalized freeze status for an entry frontmatter.
22
+ * Missing block / missing status / invalid value all collapse to `auto`.
23
+ */
24
+ export function readFreezeStatus(frontmatter) {
25
+ const status = frontmatter?.freeze?.status;
26
+ if (typeof status !== 'string') return 'auto';
27
+ if (!FREEZE_STATUSES.includes(status)) return 'auto';
28
+ return status;
29
+ }
30
+
31
+ /**
32
+ * Whether an entry's freeze status blocks a regen-like action outright.
33
+ * `frozen` blocks. `soft-advisory` blocks unless the caller overrides via
34
+ * --i-know + --reason. `auto` / `on-canon-change` do not block.
35
+ */
36
+ export function freezeBlocksAction(status) {
37
+ return status === 'frozen';
38
+ }
39
+
40
+ /**
41
+ * Whether an entry's freeze status warrants a soft-advisory bypass path.
42
+ */
43
+ export function freezeIsAdvisory(status) {
44
+ return status === 'soft-advisory';
45
+ }
46
+
47
+ /**
48
+ * Stable canonical JSON serialization for an object used as hash input.
49
+ *
50
+ * Sorts object keys; arrays preserve order (arrays are meaningful sequences).
51
+ * Normalizes `undefined` → omitted. Bit-stability matters because the witness
52
+ * chain compares hashes across builds — the same semantic value must hash
53
+ * identically regardless of key insertion order or JSON formatting drift.
54
+ */
55
+ export function canonicalize(value) {
56
+ if (value === null) return 'null';
57
+ if (value === undefined) return 'null';
58
+ if (typeof value === 'number') return Number.isFinite(value) ? String(value) : 'null';
59
+ if (typeof value === 'boolean') return value ? 'true' : 'false';
60
+ if (typeof value === 'string') return JSON.stringify(value);
61
+ if (Array.isArray(value)) {
62
+ return '[' + value.map(canonicalize).join(',') + ']';
63
+ }
64
+ if (typeof value === 'object') {
65
+ const keys = Object.keys(value).sort();
66
+ const parts = [];
67
+ for (const k of keys) {
68
+ if (value[k] === undefined) continue;
69
+ parts.push(JSON.stringify(k) + ':' + canonicalize(value[k]));
70
+ }
71
+ return '{' + parts.join(',') + '}';
72
+ }
73
+ return 'null';
74
+ }
75
+
76
+ /**
77
+ * Compute the watch-hash for an entry given its frontmatter, the list of
78
+ * watched fields, and the schema version.
79
+ *
80
+ * The hash is sha256 of a canonical JSON representation of the watched
81
+ * field values + the schema version. Any change to a watched field, or a
82
+ * schema version bump, invalidates the hash.
83
+ *
84
+ * @param {Object} frontmatter — entry frontmatter object
85
+ * @param {Array<string>} watchFields — dotted paths (e.g. "visual.silhouette_cue")
86
+ * @param {string} schemaVersion — schema `version` string
87
+ * @returns {string} 64-char hex sha256 digest
88
+ */
89
+ export function computeWatchHash(frontmatter, watchFields, schemaVersion) {
90
+ const payload = {
91
+ schema_version: schemaVersion,
92
+ watched: {},
93
+ };
94
+ for (const field of watchFields) {
95
+ payload.watched[field] = getByPath(frontmatter, field) ?? null;
96
+ }
97
+ return createHash('sha256').update(canonicalize(payload)).digest('hex');
98
+ }
99
+
100
+ /**
101
+ * Resolve the effective watch-field list for an entry.
102
+ *
103
+ * Precedence (entry override wins over build-config default):
104
+ * 1. `frontmatter.freeze.watch_fields` if present + non-empty
105
+ * 2. `buildConfig.freeze_watch_defaults[schemaName]` if present
106
+ * 3. [] (empty — a freeze with no watch-fields stamps only the schema
107
+ * version, useful for "freeze but don't drift-check" intent)
108
+ */
109
+ export function resolveWatchFields(frontmatter, schemaName, buildConfig) {
110
+ const entryOverride = frontmatter?.freeze?.watch_fields;
111
+ if (Array.isArray(entryOverride) && entryOverride.length > 0) {
112
+ return entryOverride;
113
+ }
114
+ const defaults = buildConfig?.freeze_watch_defaults?.[schemaName];
115
+ if (Array.isArray(defaults) && defaults.length > 0) {
116
+ return defaults;
117
+ }
118
+ return [];
119
+ }
120
+
121
+ /**
122
+ * Build a fresh freeze block for writing to an entry (canon freeze command).
123
+ *
124
+ * @param {Object} args
125
+ * @param {string} args.status — 'frozen' | 'soft-advisory' | 'on-canon-change'
126
+ * @param {string} args.lockedAtBuild — generated_from of the witness build
127
+ * @param {string} [args.lockedAtCanonVersion]
128
+ * @param {string} args.frozenBy
129
+ * @param {string} args.frozenReason
130
+ * @param {Array<string>} [args.watchFields]
131
+ * @param {Array} [args.overrides] — existing override history (preserved on re-freeze)
132
+ */
133
+ export function buildFreezeBlock({
134
+ status,
135
+ lockedAtBuild,
136
+ lockedAtCanonVersion,
137
+ frozenBy,
138
+ frozenReason,
139
+ watchFields,
140
+ overrides,
141
+ }) {
142
+ const block = { status };
143
+ if (lockedAtBuild) block.locked_at_build = lockedAtBuild;
144
+ if (lockedAtCanonVersion) block.locked_at_canon_version = lockedAtCanonVersion;
145
+ if (frozenBy) block.frozen_by = frozenBy;
146
+ if (frozenReason) block.frozen_reason = frozenReason;
147
+ if (Array.isArray(watchFields) && watchFields.length) block.watch_fields = watchFields;
148
+ if (Array.isArray(overrides) && overrides.length) block.overrides = overrides;
149
+ return block;
150
+ }
151
+
152
+ /**
153
+ * Append an override record to an entry's freeze.overrides[] list.
154
+ * Denormalizes the event into the entry for glance-reading; the authoritative
155
+ * source of truth is the append-only log at canon-build/freeze-events.jsonl.
156
+ */
157
+ export function appendOverrideToFreezeBlock(freeze, override) {
158
+ const next = { ...freeze };
159
+ next.overrides = Array.isArray(freeze?.overrides) ? [...freeze.overrides, override] : [override];
160
+ return next;
161
+ }
package/lib/rows.js ADDED
@@ -0,0 +1,228 @@
1
+ /**
2
+ * Canonical dataset row shape.
3
+ *
4
+ * The single source-of-truth shape that adapters (generic-image-caption,
5
+ * diffusers-lora, ai-toolkit) consume. Two producers feed this shape:
6
+ *
7
+ * 1. `canon-build` — emits rows to disk at
8
+ * `<project>/canon-build/<canon_sha>/dataset/<lane>-<partition>.jsonl`
9
+ * from the canon entity store (research D1, D2).
10
+ * 2. `training-packages` — converts legacy records-path `{record, lane, group}`
11
+ * entries into rows in memory so the records flow keeps working while
12
+ * canon authoring catches up.
13
+ *
14
+ * Both paths produce rows the adapter cannot distinguish. This is D6:
15
+ * adapters have one input contract, not two.
16
+ *
17
+ * The research deliverable pins the row schema. See
18
+ * `memory/three-projection-build-research-2026-04-24.md` §Projection contracts.
19
+ */
20
+
21
+ import { readFile } from 'node:fs/promises';
22
+ import { createHash } from 'node:crypto';
23
+ import { buildCaption, deriveStyleTrigger } from './captions.js';
24
+
25
+ export const ROW_SCHEMA_VERSION = 'canon-build-dataset-1.0';
26
+
27
+ /**
28
+ * Compute SHA-256 of a deterministic serialization of the inputs.
29
+ * Used for `entry_hash` (cache key) when building rows from canon entries.
30
+ */
31
+ export function hashRowInputs({ entryBody, schemaVersion, buildConfigJson, projectFingerprint }) {
32
+ const h = createHash('sha256');
33
+ h.update(entryBody || '');
34
+ h.update('\0');
35
+ h.update(schemaVersion || '');
36
+ h.update('\0');
37
+ h.update(buildConfigJson || '');
38
+ h.update('\0');
39
+ h.update(projectFingerprint || '');
40
+ return h.digest('hex');
41
+ }
42
+
43
+ /**
44
+ * Build a row from a legacy record object.
45
+ *
46
+ * The records flow (star-freight today) produces `{ record, lane, group }`
47
+ * tuples via split + snapshot + canon-binding. Each tuple becomes one row.
48
+ *
49
+ * @param {Object} record — the records/<id>.json object
50
+ * @param {string|null} lane — resolved lane (costume, environment, ...)
51
+ * @param {string|null} group — resolved group (faction, region, ...)
52
+ * @param {Object} profile — training profile (for caption + trigger)
53
+ * @param {string} partition — train | val | test
54
+ * @returns {Object|null} row, or null if the record has no asset_path
55
+ */
56
+ export function recordToRow(record, lane, group, profile, partition) {
57
+ if (!record?.id || !record.asset_path) return null;
58
+
59
+ const trigger = deriveStyleTrigger(profile) || null;
60
+ const caption = buildCaption(record, lane, group, profile);
61
+
62
+ const row = {
63
+ schema_version: ROW_SCHEMA_VERSION,
64
+ generated_from: null, // records path: no canon commit
65
+ entity_id: record.id,
66
+ schema_kind: 'record', // legacy marker: not a canon entity
67
+ entity_type: record.identity?.kind || null,
68
+ lane,
69
+ partition,
70
+ asset_path: record.asset_path,
71
+ caption,
72
+ trigger,
73
+ subject_filter_key: record.identity?.subject_name || record.id,
74
+ forbidden_tokens: record.canon?.forbidden_inputs || [],
75
+ pass_ratio: record.canon?.assertion_count > 0
76
+ ? +(record.canon.pass_count / record.canon.assertion_count).toFixed(3)
77
+ : null,
78
+ group,
79
+ };
80
+
81
+ return row;
82
+ }
83
+
84
+ /**
85
+ * Build a row from a canon entity + approved reference plate.
86
+ *
87
+ * The canon-build flow produces rows from canon entries. Each entry may
88
+ * contribute one or more rows (one per approved reference plate × lane).
89
+ *
90
+ * @param {Object} args
91
+ * @param {Object} args.entry — parsed canon entry (frontmatter object)
92
+ * @param {string} args.schemaKind — monster | character | deity | location | relic
93
+ * @param {string} args.assetPath — project-relative path to the approved plate
94
+ * @param {string} args.lane — resolved lane
95
+ * @param {string} args.caption — pre-rendered via buildCaption
96
+ * @param {string|null} args.trigger — from the training profile that drove the plate
97
+ * @param {string} args.partition — train | val | test (canon-build defaults to train)
98
+ * @param {string} args.entryHash — SHA-256 of entry + schema_version + build_config + project_fingerprint
99
+ * @param {string|null} args.generatedFrom — canon commit SHA or "content-sha:..." fallback
100
+ */
101
+ export function canonEntryToRow({
102
+ entry,
103
+ schemaKind,
104
+ assetPath,
105
+ lane,
106
+ caption,
107
+ trigger,
108
+ partition,
109
+ entryHash,
110
+ generatedFrom,
111
+ }) {
112
+ const entityType = entry.kind
113
+ || entry.species_tag
114
+ || entry.location_type
115
+ || entry.relic_type
116
+ || entry.generation
117
+ || null;
118
+
119
+ return {
120
+ schema_version: ROW_SCHEMA_VERSION,
121
+ generated_from: generatedFrom,
122
+ entity_id: entry.id,
123
+ schema_kind: schemaKind,
124
+ entity_type: entityType,
125
+ lane,
126
+ partition,
127
+ asset_path: assetPath,
128
+ caption,
129
+ trigger,
130
+ subject_filter_key: entry.id,
131
+ forbidden_tokens: entry.forbidden_inputs || [],
132
+ entry_hash: entryHash,
133
+ };
134
+ }
135
+
136
+ /**
137
+ * Read rows from a dataset.jsonl file.
138
+ *
139
+ * Parses each non-empty line as JSON; rejects the file if any row's
140
+ * `schema_version` does not match ROW_SCHEMA_VERSION.
141
+ *
142
+ * @param {string} path — absolute path to dataset.jsonl
143
+ * @returns {Promise<Array<Object>>} rows
144
+ */
145
+ export async function readDatasetJsonl(path) {
146
+ const text = await readFile(path, 'utf-8');
147
+ const rows = [];
148
+ const lines = text.split('\n');
149
+ for (let i = 0; i < lines.length; i++) {
150
+ const line = lines[i].trim();
151
+ if (!line) continue;
152
+ let row;
153
+ try {
154
+ row = JSON.parse(line);
155
+ } catch (err) {
156
+ throw new Error(`readDatasetJsonl: line ${i + 1} of ${path} is not valid JSON (${err.message})`);
157
+ }
158
+ if (row.schema_version !== ROW_SCHEMA_VERSION) {
159
+ throw new Error(
160
+ `readDatasetJsonl: line ${i + 1} of ${path} has schema_version "${row.schema_version}" (expected "${ROW_SCHEMA_VERSION}")`
161
+ );
162
+ }
163
+ rows.push(row);
164
+ }
165
+ return rows;
166
+ }
167
+
168
+ /**
169
+ * Group rows by partition (train | val | test).
170
+ * Rows with missing/unknown partition are grouped under an "other" key.
171
+ */
172
+ export function groupRowsByPartition(rows) {
173
+ const out = { train: [], val: [], test: [] };
174
+ for (const row of rows) {
175
+ const p = row.partition || 'other';
176
+ if (!out[p]) out[p] = [];
177
+ out[p].push(row);
178
+ }
179
+ return out;
180
+ }
181
+
182
+ /**
183
+ * Filter rows for a training profile.
184
+ *
185
+ * Two filter axes:
186
+ * - lane-based: include row if row.lane ∈ profile.eligible_lanes
187
+ * - entity-based: include row if row.entity_id === profile.entity_id_scope
188
+ * (when entity_id_scope is set — per-character LoRA case, D8)
189
+ *
190
+ * Both filters compose: a per-character profile with both eligible_lanes and
191
+ * entity_id_scope set applies both as AND. Zero filters = pass everything.
192
+ */
193
+ export function filterRowsForProfile(rows, profile) {
194
+ const eligibleLanes = profile.eligible_lanes;
195
+ const entityScope = profile.entity_id_scope;
196
+
197
+ return rows.filter((row) => {
198
+ if (Array.isArray(eligibleLanes) && eligibleLanes.length > 0) {
199
+ if (!eligibleLanes.includes(row.lane)) return false;
200
+ }
201
+ if (entityScope) {
202
+ if (row.entity_id !== entityScope && row.subject_filter_key !== entityScope) {
203
+ return false;
204
+ }
205
+ }
206
+ return true;
207
+ });
208
+ }
209
+
210
+ /**
211
+ * Serialize a row to a stable JSONL line.
212
+ * Key order is enforced by emitting known fields first, then alphabetized extras.
213
+ */
214
+ export function serializeRow(row) {
215
+ const knownOrder = [
216
+ 'schema_version', 'generated_from', 'entity_id', 'schema_kind', 'entity_type',
217
+ 'lane', 'partition', 'asset_path', 'caption', 'trigger',
218
+ 'subject_filter_key', 'forbidden_tokens', 'entry_hash', 'pass_ratio', 'group',
219
+ ];
220
+ const ordered = {};
221
+ for (const key of knownOrder) {
222
+ if (row[key] !== undefined) ordered[key] = row[key];
223
+ }
224
+ for (const key of Object.keys(row).sort()) {
225
+ if (!knownOrder.includes(key)) ordered[key] = row[key];
226
+ }
227
+ return JSON.stringify(ordered);
228
+ }
@@ -27,6 +27,11 @@ export const ADAPTER_REGISTRY = Object.freeze({
27
27
  description: 'Diffusers fine-tuning layout: image + caption .txt sidecars + per-partition JSONL',
28
28
  module: './adapters/diffusers-lora.js',
29
29
  },
30
+ 'ai-toolkit': {
31
+ id: 'ai-toolkit',
32
+ description: 'Ostris ai-toolkit Flux training package: image + .txt sidecars + ai-toolkit-config.yaml at root',
33
+ module: './adapters/ai-toolkit.js',
34
+ },
30
35
  });
31
36
 
32
37
  /**
@@ -18,6 +18,7 @@ import { loadProjectConfig, detectLane, detectGroup } from './config.js';
18
18
  import { inputError } from './errors.js';
19
19
  import { SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
20
20
  import { loadAdapter } from './training-adapters.js';
21
+ import { recordToRow, filterRowsForProfile } from './rows.js';
21
22
 
22
23
  /**
23
24
  * Generate a training package ID: tp-YYYYMMDD-HHMMSS-XXXX
@@ -61,38 +62,43 @@ export async function buildTrainingPackage(projectRoot, manifestId, options = {}
61
62
  );
62
63
  }
63
64
 
64
- // Load split partitions
65
- const partitions = {};
66
- let totalRecords = 0;
67
- let totalImages = 0;
65
+ // Convert split partitions → canonical rows (D6 adapter input shape).
66
+ // Adapters only understand rows — records-flow callers go through this
67
+ // converter, canon-build-flow callers produce rows directly. One input
68
+ // contract, two producers.
69
+ const rows = [];
70
+ const partitionCounts = { train: 0, val: 0, test: 0 };
68
71
 
69
72
  for (const partition of ['train', 'val', 'test']) {
70
73
  const entries = await loadSplitPartition(projectRoot, manifest.source_split_id, partition);
71
- const records = [];
72
74
 
73
75
  for (const entry of entries) {
74
76
  const record = await loadRecord(recordsDir, entry.record_id);
75
77
  if (!record) continue;
76
78
 
77
- // Filter by profile's eligible lanes
78
79
  const prompt = record.provenance?.prompt || '';
79
80
  const lane = detectLane(record.id, prompt, config.lanes);
80
- if (profile.eligible_lanes.length > 0 && !profile.eligible_lanes.includes(lane)) {
81
- continue;
82
- }
83
-
84
81
  const group = detectGroup(record.id, prompt, config.terminology);
85
- records.push({ record, lane, group });
82
+
83
+ const row = recordToRow(record, lane, group, profile, partition);
84
+ if (row) rows.push(row);
86
85
  }
86
+ }
87
87
 
88
- partitions[partition] = records;
89
- totalRecords += records.length;
88
+ // Apply profile filters (eligible_lanes + entity_id_scope — D8).
89
+ // filterRowsForProfile returns rows where lane ∈ eligible_lanes AND
90
+ // entity_id === entity_id_scope (when set). Zero filters = identity.
91
+ const filteredRows = filterRowsForProfile(rows, profile);
92
+ for (const row of filteredRows) {
93
+ if (partitionCounts[row.partition] !== undefined) partitionCounts[row.partition]++;
90
94
  }
95
+ const totalRecords = filteredRows.length;
96
+ let totalImages = 0;
91
97
 
92
- // Let the adapter build the package layout
98
+ // Let the adapter build the package layout from the canonical rows.
93
99
  const adapterResult = await adapter.buildPackage({
94
100
  packageDir,
95
- partitions,
101
+ rows: filteredRows,
96
102
  profile,
97
103
  manifest,
98
104
  config,
@@ -118,9 +124,9 @@ export async function buildTrainingPackage(projectRoot, manifestId, options = {}
118
124
  config_fingerprint: manifest.config_fingerprint,
119
125
  counts: {
120
126
  total_records: totalRecords,
121
- train: partitions.train.length,
122
- val: partitions.val.length,
123
- test: partitions.test.length,
127
+ train: partitionCounts.train,
128
+ val: partitionCounts.val,
129
+ test: partitionCounts.test,
124
130
  images: totalImages,
125
131
  checksums: 0, // updated after checksum pass below
126
132
  },
@@ -13,6 +13,13 @@ import { isRegisteredAdapter, listAdapters } from './training-adapters.js';
13
13
 
14
14
  const REQUIRED_FIELDS = ['profile_id', 'label', 'target_family', 'asset_type', 'eligible_lanes', 'adapter_targets'];
15
15
 
16
+ // trigger_override constraints (research ref: two-LoRA stack D3, 2026-04-24).
17
+ // T5 tokenizes on SentencePiece Unigram with `_` as the space sentinel; hyphens
18
+ // fragment unpredictably and uppercase is not canonicalized. Generic suffixes
19
+ // alone collide across projects — a game-slug prefix is the documented defense.
20
+ const TRIGGER_OVERRIDE_RE = /^[a-z0-9_]+$/;
21
+ const TRIGGER_GENERIC_SUFFIX_DENYLIST = ['style', 'character', 'anime', 'realistic'];
22
+
16
23
  /**
17
24
  * Validate a training profile object.
18
25
  * @returns {{ valid: boolean, errors: string[] }}
@@ -42,6 +49,32 @@ export function validateProfile(profile) {
42
49
  }
43
50
  }
44
51
  }
52
+ // trigger_override is optional. When present, enforce SentencePiece-safe
53
+ // format and reject bare generic suffixes that collide across projects.
54
+ if (profile.trigger_override !== undefined && profile.trigger_override !== null) {
55
+ if (typeof profile.trigger_override !== 'string') {
56
+ errors.push('trigger_override must be a string');
57
+ } else if (profile.trigger_override.length === 0) {
58
+ errors.push('trigger_override must be a non-empty string (or omit the field)');
59
+ } else if (!TRIGGER_OVERRIDE_RE.test(profile.trigger_override)) {
60
+ errors.push(`trigger_override "${profile.trigger_override}" must match ^[a-z0-9_]+$ (lowercase, digits, underscores — no hyphens, no uppercase)`);
61
+ } else if (TRIGGER_GENERIC_SUFFIX_DENYLIST.includes(profile.trigger_override)) {
62
+ errors.push(`trigger_override "${profile.trigger_override}" is a generic suffix; prefix with a game slug (e.g. "sf_${profile.trigger_override}")`);
63
+ }
64
+ }
65
+ // entity_id_scope is optional (post-research D8). Same SentencePiece-safe
66
+ // format as trigger_override — no hyphens, no uppercase, lowercase slug.
67
+ // Used by per-character LoRA profiles to row-filter canonical dataset.jsonl
68
+ // to a single subject at packaging time.
69
+ if (profile.entity_id_scope !== undefined && profile.entity_id_scope !== null) {
70
+ if (typeof profile.entity_id_scope !== 'string') {
71
+ errors.push('entity_id_scope must be a string');
72
+ } else if (profile.entity_id_scope.length === 0) {
73
+ errors.push('entity_id_scope must be a non-empty string (or omit the field)');
74
+ } else if (!TRIGGER_OVERRIDE_RE.test(profile.entity_id_scope)) {
75
+ errors.push(`entity_id_scope "${profile.entity_id_scope}" must match ^[a-z0-9_]+$ (lowercase, digits, underscores — no hyphens, no uppercase)`);
76
+ }
77
+ }
45
78
  return { valid: errors.length === 0, errors };
46
79
  }
47
80