@mcptoolshop/style-dataset-lab 3.1.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.es.md +84 -62
- package/README.fr.md +85 -63
- package/README.hi.md +97 -77
- package/README.it.md +86 -64
- package/README.ja.md +94 -72
- package/README.md +22 -0
- package/README.pt-BR.md +83 -61
- package/README.zh.md +88 -66
- package/bin/sdlab.js +71 -0
- package/lib/adapters/ai-toolkit.js +209 -0
- package/lib/adapters/comfyui-workflows.js +133 -2
- package/lib/adapters/diffusers-lora.js +46 -66
- package/lib/adapters/generic-image-caption.js +22 -50
- package/lib/canon-build/build.js +450 -0
- package/lib/canon-build/emitters/base.js +122 -0
- package/lib/canon-build/emitters/character.js +167 -0
- package/lib/canon-build/emitters/deity.js +165 -0
- package/lib/canon-build/emitters/faction.js +161 -0
- package/lib/canon-build/emitters/index.js +38 -0
- package/lib/canon-build/emitters/location.js +153 -0
- package/lib/canon-build/emitters/monster.js +131 -0
- package/lib/canon-build/emitters/relic.js +142 -0
- package/lib/canon-build/emitters/ship.js +154 -0
- package/lib/canon-build/emitters/species.js +165 -0
- package/lib/canon-build/frontmatter.js +76 -0
- package/lib/canon-build/lane-resolve.js +64 -0
- package/lib/canon-build/load-config.js +117 -0
- package/lib/canon-build/load-entry.js +62 -0
- package/lib/canon-build/load-schema.js +46 -0
- package/lib/captions.js +197 -0
- package/lib/config.js +19 -0
- package/lib/freeze-events.js +109 -0
- package/lib/freeze-gate.js +165 -0
- package/lib/freeze-stamp.js +161 -0
- package/lib/rows.js +228 -0
- package/lib/training-adapters.js +5 -0
- package/lib/training-packages.js +24 -18
- package/lib/training-profiles.js +33 -0
- package/package.json +12 -3
- package/scripts/_batch_cut.py +71 -0
- package/scripts/_seam_cutout.py +74 -0
- package/scripts/canon-build.js +88 -0
- package/scripts/canon-drift.js +193 -0
- package/scripts/canon-freeze-status.js +85 -0
- package/scripts/canon-freeze.js +211 -0
- package/scripts/canon-unfreeze.js +137 -0
- package/scripts/curate.js +12 -0
- package/scripts/generate-controlnet.js +12 -0
- package/scripts/generate-identity.js +20 -4
- package/scripts/generate-ipadapter.js +11 -0
- package/scripts/generate.js +238 -102
- package/scripts/qwen_generate.py +124 -0
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Freeze enforcement gate (D3).
|
|
3
|
+
*
|
|
4
|
+
* Called from CLI commands before any write to a canon-bound asset or record.
|
|
5
|
+
* Semantics:
|
|
6
|
+
* - frozen → block outright (CANON_ENTRY_FROZEN)
|
|
7
|
+
* - soft-advisory → block unless --i-know + --reason; on override, append
|
|
8
|
+
* a 'bypass' event to freeze-events.jsonl and proceed
|
|
9
|
+
* - auto → no block
|
|
10
|
+
* - on-canon-change → no block at write time; canon-build surfaces drift
|
|
11
|
+
* in its summary output (enforcement lives at the build
|
|
12
|
+
* boundary, not the CLI boundary, because drift is a
|
|
13
|
+
* build-observable condition)
|
|
14
|
+
*
|
|
15
|
+
* The canon reverse-map helpers resolve an action's identifier (subject id,
|
|
16
|
+
* asset path) to a canon entry so the gate can look up its freeze state.
|
|
17
|
+
* Actions without a canon binding (prompt-pack-driven generate, reingest of
|
|
18
|
+
* un-bound generated outputs) skip the gate naturally: no entry = nothing
|
|
19
|
+
* to protect.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { join } from 'node:path';
|
|
23
|
+
import { existsSync } from 'node:fs';
|
|
24
|
+
import { loadBuildConfig } from './canon-build/load-config.js';
|
|
25
|
+
import { loadCanonEntriesInDir, entryId } from './canon-build/load-entry.js';
|
|
26
|
+
import { readFreezeStatus } from './freeze-stamp.js';
|
|
27
|
+
import { appendEvent } from './freeze-events.js';
|
|
28
|
+
import { inputError } from './errors.js';
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Load every canon entry in a project, keyed by schema.
|
|
32
|
+
*
|
|
33
|
+
* Returns a flat array of `{ entry, schemaName }` objects.
|
|
34
|
+
* Returns [] (no-op) when the project has no canon-build config — meaning
|
|
35
|
+
* the project isn't canon-integrated yet, so the gate has nothing to enforce.
|
|
36
|
+
*/
|
|
37
|
+
export async function loadAllCanonEntries(projectRoot) {
|
|
38
|
+
const configPath = join(projectRoot, 'canon-build', 'config.json');
|
|
39
|
+
if (!existsSync(configPath)) return [];
|
|
40
|
+
let config;
|
|
41
|
+
try {
|
|
42
|
+
config = await loadBuildConfig(configPath, projectRoot);
|
|
43
|
+
} catch {
|
|
44
|
+
// Invalid config → degrade to "no gate" rather than blocking every write.
|
|
45
|
+
// The operator will hit the config error on `sdlab canon build`.
|
|
46
|
+
return [];
|
|
47
|
+
}
|
|
48
|
+
const entries = [];
|
|
49
|
+
for (const [schemaName, subdir] of Object.entries(config.entity_dirs)) {
|
|
50
|
+
const dir = join(config.canon_root, subdir);
|
|
51
|
+
const loaded = await loadCanonEntriesInDir(dir);
|
|
52
|
+
for (const entry of loaded) {
|
|
53
|
+
entries.push({ entry, schemaName });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return entries;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Find the canon entry whose `id` matches the given subject identifier.
|
|
61
|
+
* Returns { entry, schemaName } or null.
|
|
62
|
+
*/
|
|
63
|
+
export async function resolveEntryBySubject(projectRoot, subjectId) {
|
|
64
|
+
if (!subjectId) return null;
|
|
65
|
+
const all = await loadAllCanonEntries(projectRoot);
|
|
66
|
+
const hit = all.find(({ entry }) => entryId(entry) === subjectId);
|
|
67
|
+
return hit || null;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Find the canon entry whose `visual.reference_plate_uri` (or top-level
|
|
72
|
+
* `reference_plate_uri`, for monsters) equals the given asset path.
|
|
73
|
+
* Returns { entry, schemaName } or null.
|
|
74
|
+
*/
|
|
75
|
+
export async function resolveEntryByAssetPath(projectRoot, assetPath) {
|
|
76
|
+
if (!assetPath) return null;
|
|
77
|
+
const all = await loadAllCanonEntries(projectRoot);
|
|
78
|
+
const normalized = assetPath.replace(/\\/g, '/');
|
|
79
|
+
const hit = all.find(({ entry }) => {
|
|
80
|
+
const fm = entry.frontmatter;
|
|
81
|
+
const plate = (fm.visual?.reference_plate_uri || fm.reference_plate_uri || '').replace(/\\/g, '/');
|
|
82
|
+
return plate && plate === normalized;
|
|
83
|
+
});
|
|
84
|
+
return hit || null;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Core gate. Throws CANON_ENTRY_FROZEN on block; on soft-advisory bypass,
|
|
89
|
+
* appends a 'bypass' event and returns without throwing.
|
|
90
|
+
*
|
|
91
|
+
* @param {Object} args
|
|
92
|
+
* @param {string} args.projectRoot
|
|
93
|
+
* @param {Object|null} args.resolved — { entry, schemaName } or null
|
|
94
|
+
* @param {string} args.action — short verb for error context ("generate", "painterly", "curate", ...)
|
|
95
|
+
* @param {boolean} [args.allowSoftAdvisoryBypass] — the command supports --i-know
|
|
96
|
+
* @param {string} [args.bypassReason] — value of --reason when bypassing
|
|
97
|
+
* @param {string} [args.by] — who is invoking (default 'mike')
|
|
98
|
+
*/
|
|
99
|
+
export async function assertNotFrozen({
|
|
100
|
+
projectRoot,
|
|
101
|
+
resolved,
|
|
102
|
+
action,
|
|
103
|
+
allowSoftAdvisoryBypass = false,
|
|
104
|
+
bypassReason = null,
|
|
105
|
+
by = 'mike',
|
|
106
|
+
}) {
|
|
107
|
+
if (!resolved || !resolved.entry) return; // nothing canon-bound; no gate
|
|
108
|
+
const fm = resolved.entry.frontmatter;
|
|
109
|
+
const status = readFreezeStatus(fm);
|
|
110
|
+
const id = entryId(resolved.entry);
|
|
111
|
+
|
|
112
|
+
if (status === 'frozen') {
|
|
113
|
+
throw inputError(
|
|
114
|
+
'CANON_ENTRY_FROZEN',
|
|
115
|
+
`Canon entry "${id}" is frozen; "${action}" refuses to proceed.`,
|
|
116
|
+
`Unfreeze with: sdlab canon unfreeze ${id} --reason "<why>". The override will be appended to canon-build/freeze-events.jsonl as an audit record.`,
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
if (status === 'soft-advisory') {
|
|
121
|
+
if (!allowSoftAdvisoryBypass) {
|
|
122
|
+
throw inputError(
|
|
123
|
+
'CANON_ENTRY_FROZEN',
|
|
124
|
+
`Canon entry "${id}" is soft-advisory frozen; "${action}" refuses to proceed.`,
|
|
125
|
+
`This command does not support a --i-know bypass. Run "sdlab canon unfreeze ${id} --reason \\"<why>\\"" for a proper unfreeze, or pick a command that accepts --i-know.`,
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
if (!bypassReason) {
|
|
129
|
+
throw inputError(
|
|
130
|
+
'CANON_ENTRY_FROZEN',
|
|
131
|
+
`Canon entry "${id}" is soft-advisory frozen; "${action}" requires --i-know AND --reason to bypass.`,
|
|
132
|
+
`Re-run with: --i-know --reason "<short reason>". The bypass will be logged to canon-build/freeze-events.jsonl.`,
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
// Bypass allowed — log and proceed.
|
|
136
|
+
await appendEvent(projectRoot, {
|
|
137
|
+
type: 'bypass',
|
|
138
|
+
entity_id: id,
|
|
139
|
+
schema_kind: resolved.schemaName?.replace('.schema.json', ''),
|
|
140
|
+
by,
|
|
141
|
+
reason: bypassReason,
|
|
142
|
+
prior_status: 'soft-advisory',
|
|
143
|
+
new_status: 'soft-advisory',
|
|
144
|
+
});
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// auto and on-canon-change: no write-side block.
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Convenience wrapper: resolve by subject id, then gate.
|
|
153
|
+
*/
|
|
154
|
+
export async function assertNotFrozenBySubject(projectRoot, subjectId, options = {}) {
|
|
155
|
+
const resolved = await resolveEntryBySubject(projectRoot, subjectId);
|
|
156
|
+
return assertNotFrozen({ projectRoot, resolved, ...options });
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Convenience wrapper: resolve by asset_path, then gate.
|
|
161
|
+
*/
|
|
162
|
+
export async function assertNotFrozenByAssetPath(projectRoot, assetPath, options = {}) {
|
|
163
|
+
const resolved = await resolveEntryByAssetPath(projectRoot, assetPath);
|
|
164
|
+
return assertNotFrozen({ projectRoot, resolved, ...options });
|
|
165
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Freeze block helpers — read/write the `freeze` block on a canon entry,
|
|
3
|
+
* compute watch-field hashes for the witness chain (D1).
|
|
4
|
+
*
|
|
5
|
+
* The `freeze` block (schema v2.0.0+) is the sole freeze surface on canon
|
|
6
|
+
* entries; the legacy `visual_locked_at` + `regen_policy` fields have been
|
|
7
|
+
* removed per post-research D10. See
|
|
8
|
+
* memory/freeze-tooling-research-2026-04-24.md for the full contract.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { createHash } from 'node:crypto';
|
|
12
|
+
import { getByPath } from './canon-build/load-config.js';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* The four legal freeze statuses. Entries without a `freeze` block behave
|
|
16
|
+
* as `auto` (permissive default — see post-research D8).
|
|
17
|
+
*/
|
|
18
|
+
export const FREEZE_STATUSES = Object.freeze(['auto', 'frozen', 'soft-advisory', 'on-canon-change']);
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Read the normalized freeze status for an entry frontmatter.
|
|
22
|
+
* Missing block / missing status / invalid value all collapse to `auto`.
|
|
23
|
+
*/
|
|
24
|
+
export function readFreezeStatus(frontmatter) {
|
|
25
|
+
const status = frontmatter?.freeze?.status;
|
|
26
|
+
if (typeof status !== 'string') return 'auto';
|
|
27
|
+
if (!FREEZE_STATUSES.includes(status)) return 'auto';
|
|
28
|
+
return status;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Whether an entry's freeze status blocks a regen-like action outright.
|
|
33
|
+
* `frozen` blocks. `soft-advisory` blocks unless the caller overrides via
|
|
34
|
+
* --i-know + --reason. `auto` / `on-canon-change` do not block.
|
|
35
|
+
*/
|
|
36
|
+
export function freezeBlocksAction(status) {
|
|
37
|
+
return status === 'frozen';
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Whether an entry's freeze status warrants a soft-advisory bypass path.
|
|
42
|
+
*/
|
|
43
|
+
export function freezeIsAdvisory(status) {
|
|
44
|
+
return status === 'soft-advisory';
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Stable canonical JSON serialization for an object used as hash input.
|
|
49
|
+
*
|
|
50
|
+
* Sorts object keys; arrays preserve order (arrays are meaningful sequences).
|
|
51
|
+
* Normalizes `undefined` → omitted. Bit-stability matters because the witness
|
|
52
|
+
* chain compares hashes across builds — the same semantic value must hash
|
|
53
|
+
* identically regardless of key insertion order or JSON formatting drift.
|
|
54
|
+
*/
|
|
55
|
+
export function canonicalize(value) {
|
|
56
|
+
if (value === null) return 'null';
|
|
57
|
+
if (value === undefined) return 'null';
|
|
58
|
+
if (typeof value === 'number') return Number.isFinite(value) ? String(value) : 'null';
|
|
59
|
+
if (typeof value === 'boolean') return value ? 'true' : 'false';
|
|
60
|
+
if (typeof value === 'string') return JSON.stringify(value);
|
|
61
|
+
if (Array.isArray(value)) {
|
|
62
|
+
return '[' + value.map(canonicalize).join(',') + ']';
|
|
63
|
+
}
|
|
64
|
+
if (typeof value === 'object') {
|
|
65
|
+
const keys = Object.keys(value).sort();
|
|
66
|
+
const parts = [];
|
|
67
|
+
for (const k of keys) {
|
|
68
|
+
if (value[k] === undefined) continue;
|
|
69
|
+
parts.push(JSON.stringify(k) + ':' + canonicalize(value[k]));
|
|
70
|
+
}
|
|
71
|
+
return '{' + parts.join(',') + '}';
|
|
72
|
+
}
|
|
73
|
+
return 'null';
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Compute the watch-hash for an entry given its frontmatter, the list of
|
|
78
|
+
* watched fields, and the schema version.
|
|
79
|
+
*
|
|
80
|
+
* The hash is sha256 of a canonical JSON representation of the watched
|
|
81
|
+
* field values + the schema version. Any change to a watched field, or a
|
|
82
|
+
* schema version bump, invalidates the hash.
|
|
83
|
+
*
|
|
84
|
+
* @param {Object} frontmatter — entry frontmatter object
|
|
85
|
+
* @param {Array<string>} watchFields — dotted paths (e.g. "visual.silhouette_cue")
|
|
86
|
+
* @param {string} schemaVersion — schema `version` string
|
|
87
|
+
* @returns {string} 64-char hex sha256 digest
|
|
88
|
+
*/
|
|
89
|
+
export function computeWatchHash(frontmatter, watchFields, schemaVersion) {
|
|
90
|
+
const payload = {
|
|
91
|
+
schema_version: schemaVersion,
|
|
92
|
+
watched: {},
|
|
93
|
+
};
|
|
94
|
+
for (const field of watchFields) {
|
|
95
|
+
payload.watched[field] = getByPath(frontmatter, field) ?? null;
|
|
96
|
+
}
|
|
97
|
+
return createHash('sha256').update(canonicalize(payload)).digest('hex');
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Resolve the effective watch-field list for an entry.
|
|
102
|
+
*
|
|
103
|
+
* Precedence (entry override wins over build-config default):
|
|
104
|
+
* 1. `frontmatter.freeze.watch_fields` if present + non-empty
|
|
105
|
+
* 2. `buildConfig.freeze_watch_defaults[schemaName]` if present
|
|
106
|
+
* 3. [] (empty — a freeze with no watch-fields stamps only the schema
|
|
107
|
+
* version, useful for "freeze but don't drift-check" intent)
|
|
108
|
+
*/
|
|
109
|
+
export function resolveWatchFields(frontmatter, schemaName, buildConfig) {
|
|
110
|
+
const entryOverride = frontmatter?.freeze?.watch_fields;
|
|
111
|
+
if (Array.isArray(entryOverride) && entryOverride.length > 0) {
|
|
112
|
+
return entryOverride;
|
|
113
|
+
}
|
|
114
|
+
const defaults = buildConfig?.freeze_watch_defaults?.[schemaName];
|
|
115
|
+
if (Array.isArray(defaults) && defaults.length > 0) {
|
|
116
|
+
return defaults;
|
|
117
|
+
}
|
|
118
|
+
return [];
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Build a fresh freeze block for writing to an entry (canon freeze command).
|
|
123
|
+
*
|
|
124
|
+
* @param {Object} args
|
|
125
|
+
* @param {string} args.status — 'frozen' | 'soft-advisory' | 'on-canon-change'
|
|
126
|
+
* @param {string} args.lockedAtBuild — generated_from of the witness build
|
|
127
|
+
* @param {string} [args.lockedAtCanonVersion]
|
|
128
|
+
* @param {string} args.frozenBy
|
|
129
|
+
* @param {string} args.frozenReason
|
|
130
|
+
* @param {Array<string>} [args.watchFields]
|
|
131
|
+
* @param {Array} [args.overrides] — existing override history (preserved on re-freeze)
|
|
132
|
+
*/
|
|
133
|
+
export function buildFreezeBlock({
|
|
134
|
+
status,
|
|
135
|
+
lockedAtBuild,
|
|
136
|
+
lockedAtCanonVersion,
|
|
137
|
+
frozenBy,
|
|
138
|
+
frozenReason,
|
|
139
|
+
watchFields,
|
|
140
|
+
overrides,
|
|
141
|
+
}) {
|
|
142
|
+
const block = { status };
|
|
143
|
+
if (lockedAtBuild) block.locked_at_build = lockedAtBuild;
|
|
144
|
+
if (lockedAtCanonVersion) block.locked_at_canon_version = lockedAtCanonVersion;
|
|
145
|
+
if (frozenBy) block.frozen_by = frozenBy;
|
|
146
|
+
if (frozenReason) block.frozen_reason = frozenReason;
|
|
147
|
+
if (Array.isArray(watchFields) && watchFields.length) block.watch_fields = watchFields;
|
|
148
|
+
if (Array.isArray(overrides) && overrides.length) block.overrides = overrides;
|
|
149
|
+
return block;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Append an override record to an entry's freeze.overrides[] list.
|
|
154
|
+
* Denormalizes the event into the entry for glance-reading; the authoritative
|
|
155
|
+
* source of truth is the append-only log at canon-build/freeze-events.jsonl.
|
|
156
|
+
*/
|
|
157
|
+
export function appendOverrideToFreezeBlock(freeze, override) {
|
|
158
|
+
const next = { ...freeze };
|
|
159
|
+
next.overrides = Array.isArray(freeze?.overrides) ? [...freeze.overrides, override] : [override];
|
|
160
|
+
return next;
|
|
161
|
+
}
|
package/lib/rows.js
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical dataset row shape.
|
|
3
|
+
*
|
|
4
|
+
* The single source-of-truth shape that adapters (generic-image-caption,
|
|
5
|
+
* diffusers-lora, ai-toolkit) consume. Two producers feed this shape:
|
|
6
|
+
*
|
|
7
|
+
* 1. `canon-build` — emits rows to disk at
|
|
8
|
+
* `<project>/canon-build/<canon_sha>/dataset/<lane>-<partition>.jsonl`
|
|
9
|
+
* from the canon entity store (research D1, D2).
|
|
10
|
+
* 2. `training-packages` — converts legacy records-path `{record, lane, group}`
|
|
11
|
+
* entries into rows in memory so the records flow keeps working while
|
|
12
|
+
* canon authoring catches up.
|
|
13
|
+
*
|
|
14
|
+
* Both paths produce rows the adapter cannot distinguish. This is D6:
|
|
15
|
+
* adapters have one input contract, not two.
|
|
16
|
+
*
|
|
17
|
+
* The research deliverable pins the row schema. See
|
|
18
|
+
* `memory/three-projection-build-research-2026-04-24.md` §Projection contracts.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { readFile } from 'node:fs/promises';
|
|
22
|
+
import { createHash } from 'node:crypto';
|
|
23
|
+
import { buildCaption, deriveStyleTrigger } from './captions.js';
|
|
24
|
+
|
|
25
|
+
export const ROW_SCHEMA_VERSION = 'canon-build-dataset-1.0';
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Compute SHA-256 of a deterministic serialization of the inputs.
|
|
29
|
+
* Used for `entry_hash` (cache key) when building rows from canon entries.
|
|
30
|
+
*/
|
|
31
|
+
export function hashRowInputs({ entryBody, schemaVersion, buildConfigJson, projectFingerprint }) {
|
|
32
|
+
const h = createHash('sha256');
|
|
33
|
+
h.update(entryBody || '');
|
|
34
|
+
h.update('\0');
|
|
35
|
+
h.update(schemaVersion || '');
|
|
36
|
+
h.update('\0');
|
|
37
|
+
h.update(buildConfigJson || '');
|
|
38
|
+
h.update('\0');
|
|
39
|
+
h.update(projectFingerprint || '');
|
|
40
|
+
return h.digest('hex');
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Build a row from a legacy record object.
|
|
45
|
+
*
|
|
46
|
+
* The records flow (star-freight today) produces `{ record, lane, group }`
|
|
47
|
+
* tuples via split + snapshot + canon-binding. Each tuple becomes one row.
|
|
48
|
+
*
|
|
49
|
+
* @param {Object} record — the records/<id>.json object
|
|
50
|
+
* @param {string|null} lane — resolved lane (costume, environment, ...)
|
|
51
|
+
* @param {string|null} group — resolved group (faction, region, ...)
|
|
52
|
+
* @param {Object} profile — training profile (for caption + trigger)
|
|
53
|
+
* @param {string} partition — train | val | test
|
|
54
|
+
* @returns {Object|null} row, or null if the record has no asset_path
|
|
55
|
+
*/
|
|
56
|
+
export function recordToRow(record, lane, group, profile, partition) {
|
|
57
|
+
if (!record?.id || !record.asset_path) return null;
|
|
58
|
+
|
|
59
|
+
const trigger = deriveStyleTrigger(profile) || null;
|
|
60
|
+
const caption = buildCaption(record, lane, group, profile);
|
|
61
|
+
|
|
62
|
+
const row = {
|
|
63
|
+
schema_version: ROW_SCHEMA_VERSION,
|
|
64
|
+
generated_from: null, // records path: no canon commit
|
|
65
|
+
entity_id: record.id,
|
|
66
|
+
schema_kind: 'record', // legacy marker: not a canon entity
|
|
67
|
+
entity_type: record.identity?.kind || null,
|
|
68
|
+
lane,
|
|
69
|
+
partition,
|
|
70
|
+
asset_path: record.asset_path,
|
|
71
|
+
caption,
|
|
72
|
+
trigger,
|
|
73
|
+
subject_filter_key: record.identity?.subject_name || record.id,
|
|
74
|
+
forbidden_tokens: record.canon?.forbidden_inputs || [],
|
|
75
|
+
pass_ratio: record.canon?.assertion_count > 0
|
|
76
|
+
? +(record.canon.pass_count / record.canon.assertion_count).toFixed(3)
|
|
77
|
+
: null,
|
|
78
|
+
group,
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
return row;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Build a row from a canon entity + approved reference plate.
|
|
86
|
+
*
|
|
87
|
+
* The canon-build flow produces rows from canon entries. Each entry may
|
|
88
|
+
* contribute one or more rows (one per approved reference plate × lane).
|
|
89
|
+
*
|
|
90
|
+
* @param {Object} args
|
|
91
|
+
* @param {Object} args.entry — parsed canon entry (frontmatter object)
|
|
92
|
+
* @param {string} args.schemaKind — monster | character | deity | location | relic
|
|
93
|
+
* @param {string} args.assetPath — project-relative path to the approved plate
|
|
94
|
+
* @param {string} args.lane — resolved lane
|
|
95
|
+
* @param {string} args.caption — pre-rendered via buildCaption
|
|
96
|
+
* @param {string|null} args.trigger — from the training profile that drove the plate
|
|
97
|
+
* @param {string} args.partition — train | val | test (canon-build defaults to train)
|
|
98
|
+
* @param {string} args.entryHash — SHA-256 of entry + schema_version + build_config + project_fingerprint
|
|
99
|
+
* @param {string|null} args.generatedFrom — canon commit SHA or "content-sha:..." fallback
|
|
100
|
+
*/
|
|
101
|
+
export function canonEntryToRow({
|
|
102
|
+
entry,
|
|
103
|
+
schemaKind,
|
|
104
|
+
assetPath,
|
|
105
|
+
lane,
|
|
106
|
+
caption,
|
|
107
|
+
trigger,
|
|
108
|
+
partition,
|
|
109
|
+
entryHash,
|
|
110
|
+
generatedFrom,
|
|
111
|
+
}) {
|
|
112
|
+
const entityType = entry.kind
|
|
113
|
+
|| entry.species_tag
|
|
114
|
+
|| entry.location_type
|
|
115
|
+
|| entry.relic_type
|
|
116
|
+
|| entry.generation
|
|
117
|
+
|| null;
|
|
118
|
+
|
|
119
|
+
return {
|
|
120
|
+
schema_version: ROW_SCHEMA_VERSION,
|
|
121
|
+
generated_from: generatedFrom,
|
|
122
|
+
entity_id: entry.id,
|
|
123
|
+
schema_kind: schemaKind,
|
|
124
|
+
entity_type: entityType,
|
|
125
|
+
lane,
|
|
126
|
+
partition,
|
|
127
|
+
asset_path: assetPath,
|
|
128
|
+
caption,
|
|
129
|
+
trigger,
|
|
130
|
+
subject_filter_key: entry.id,
|
|
131
|
+
forbidden_tokens: entry.forbidden_inputs || [],
|
|
132
|
+
entry_hash: entryHash,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Read rows from a dataset.jsonl file.
|
|
138
|
+
*
|
|
139
|
+
* Parses each non-empty line as JSON; rejects the file if any row's
|
|
140
|
+
* `schema_version` does not match ROW_SCHEMA_VERSION.
|
|
141
|
+
*
|
|
142
|
+
* @param {string} path — absolute path to dataset.jsonl
|
|
143
|
+
* @returns {Promise<Array<Object>>} rows
|
|
144
|
+
*/
|
|
145
|
+
export async function readDatasetJsonl(path) {
|
|
146
|
+
const text = await readFile(path, 'utf-8');
|
|
147
|
+
const rows = [];
|
|
148
|
+
const lines = text.split('\n');
|
|
149
|
+
for (let i = 0; i < lines.length; i++) {
|
|
150
|
+
const line = lines[i].trim();
|
|
151
|
+
if (!line) continue;
|
|
152
|
+
let row;
|
|
153
|
+
try {
|
|
154
|
+
row = JSON.parse(line);
|
|
155
|
+
} catch (err) {
|
|
156
|
+
throw new Error(`readDatasetJsonl: line ${i + 1} of ${path} is not valid JSON (${err.message})`);
|
|
157
|
+
}
|
|
158
|
+
if (row.schema_version !== ROW_SCHEMA_VERSION) {
|
|
159
|
+
throw new Error(
|
|
160
|
+
`readDatasetJsonl: line ${i + 1} of ${path} has schema_version "${row.schema_version}" (expected "${ROW_SCHEMA_VERSION}")`
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
rows.push(row);
|
|
164
|
+
}
|
|
165
|
+
return rows;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Group rows by partition (train | val | test).
|
|
170
|
+
* Rows with missing/unknown partition are grouped under an "other" key.
|
|
171
|
+
*/
|
|
172
|
+
export function groupRowsByPartition(rows) {
|
|
173
|
+
const out = { train: [], val: [], test: [] };
|
|
174
|
+
for (const row of rows) {
|
|
175
|
+
const p = row.partition || 'other';
|
|
176
|
+
if (!out[p]) out[p] = [];
|
|
177
|
+
out[p].push(row);
|
|
178
|
+
}
|
|
179
|
+
return out;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Filter rows for a training profile.
|
|
184
|
+
*
|
|
185
|
+
* Two filter axes:
|
|
186
|
+
* - lane-based: include row if row.lane ∈ profile.eligible_lanes
|
|
187
|
+
* - entity-based: include row if row.entity_id === profile.entity_id_scope
|
|
188
|
+
* (when entity_id_scope is set — per-character LoRA case, D8)
|
|
189
|
+
*
|
|
190
|
+
* Both filters compose: a per-character profile with both eligible_lanes and
|
|
191
|
+
* entity_id_scope set applies both as AND. Zero filters = pass everything.
|
|
192
|
+
*/
|
|
193
|
+
export function filterRowsForProfile(rows, profile) {
|
|
194
|
+
const eligibleLanes = profile.eligible_lanes;
|
|
195
|
+
const entityScope = profile.entity_id_scope;
|
|
196
|
+
|
|
197
|
+
return rows.filter((row) => {
|
|
198
|
+
if (Array.isArray(eligibleLanes) && eligibleLanes.length > 0) {
|
|
199
|
+
if (!eligibleLanes.includes(row.lane)) return false;
|
|
200
|
+
}
|
|
201
|
+
if (entityScope) {
|
|
202
|
+
if (row.entity_id !== entityScope && row.subject_filter_key !== entityScope) {
|
|
203
|
+
return false;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return true;
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Serialize a row to a stable JSONL line.
|
|
212
|
+
* Key order is enforced by emitting known fields first, then alphabetized extras.
|
|
213
|
+
*/
|
|
214
|
+
export function serializeRow(row) {
|
|
215
|
+
const knownOrder = [
|
|
216
|
+
'schema_version', 'generated_from', 'entity_id', 'schema_kind', 'entity_type',
|
|
217
|
+
'lane', 'partition', 'asset_path', 'caption', 'trigger',
|
|
218
|
+
'subject_filter_key', 'forbidden_tokens', 'entry_hash', 'pass_ratio', 'group',
|
|
219
|
+
];
|
|
220
|
+
const ordered = {};
|
|
221
|
+
for (const key of knownOrder) {
|
|
222
|
+
if (row[key] !== undefined) ordered[key] = row[key];
|
|
223
|
+
}
|
|
224
|
+
for (const key of Object.keys(row).sort()) {
|
|
225
|
+
if (!knownOrder.includes(key)) ordered[key] = row[key];
|
|
226
|
+
}
|
|
227
|
+
return JSON.stringify(ordered);
|
|
228
|
+
}
|
package/lib/training-adapters.js
CHANGED
|
@@ -27,6 +27,11 @@ export const ADAPTER_REGISTRY = Object.freeze({
|
|
|
27
27
|
description: 'Diffusers fine-tuning layout: image + caption .txt sidecars + per-partition JSONL',
|
|
28
28
|
module: './adapters/diffusers-lora.js',
|
|
29
29
|
},
|
|
30
|
+
'ai-toolkit': {
|
|
31
|
+
id: 'ai-toolkit',
|
|
32
|
+
description: 'Ostris ai-toolkit Flux training package: image + .txt sidecars + ai-toolkit-config.yaml at root',
|
|
33
|
+
module: './adapters/ai-toolkit.js',
|
|
34
|
+
},
|
|
30
35
|
});
|
|
31
36
|
|
|
32
37
|
/**
|
package/lib/training-packages.js
CHANGED
|
@@ -18,6 +18,7 @@ import { loadProjectConfig, detectLane, detectGroup } from './config.js';
|
|
|
18
18
|
import { inputError } from './errors.js';
|
|
19
19
|
import { SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
|
|
20
20
|
import { loadAdapter } from './training-adapters.js';
|
|
21
|
+
import { recordToRow, filterRowsForProfile } from './rows.js';
|
|
21
22
|
|
|
22
23
|
/**
|
|
23
24
|
* Generate a training package ID: tp-YYYYMMDD-HHMMSS-XXXX
|
|
@@ -61,38 +62,43 @@ export async function buildTrainingPackage(projectRoot, manifestId, options = {}
|
|
|
61
62
|
);
|
|
62
63
|
}
|
|
63
64
|
|
|
64
|
-
//
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
65
|
+
// Convert split partitions → canonical rows (D6 adapter input shape).
|
|
66
|
+
// Adapters only understand rows — records-flow callers go through this
|
|
67
|
+
// converter, canon-build-flow callers produce rows directly. One input
|
|
68
|
+
// contract, two producers.
|
|
69
|
+
const rows = [];
|
|
70
|
+
const partitionCounts = { train: 0, val: 0, test: 0 };
|
|
68
71
|
|
|
69
72
|
for (const partition of ['train', 'val', 'test']) {
|
|
70
73
|
const entries = await loadSplitPartition(projectRoot, manifest.source_split_id, partition);
|
|
71
|
-
const records = [];
|
|
72
74
|
|
|
73
75
|
for (const entry of entries) {
|
|
74
76
|
const record = await loadRecord(recordsDir, entry.record_id);
|
|
75
77
|
if (!record) continue;
|
|
76
78
|
|
|
77
|
-
// Filter by profile's eligible lanes
|
|
78
79
|
const prompt = record.provenance?.prompt || '';
|
|
79
80
|
const lane = detectLane(record.id, prompt, config.lanes);
|
|
80
|
-
if (profile.eligible_lanes.length > 0 && !profile.eligible_lanes.includes(lane)) {
|
|
81
|
-
continue;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
81
|
const group = detectGroup(record.id, prompt, config.terminology);
|
|
85
|
-
|
|
82
|
+
|
|
83
|
+
const row = recordToRow(record, lane, group, profile, partition);
|
|
84
|
+
if (row) rows.push(row);
|
|
86
85
|
}
|
|
86
|
+
}
|
|
87
87
|
|
|
88
|
-
|
|
89
|
-
|
|
88
|
+
// Apply profile filters (eligible_lanes + entity_id_scope — D8).
|
|
89
|
+
// filterRowsForProfile returns rows where lane ∈ eligible_lanes AND
|
|
90
|
+
// entity_id === entity_id_scope (when set). Zero filters = identity.
|
|
91
|
+
const filteredRows = filterRowsForProfile(rows, profile);
|
|
92
|
+
for (const row of filteredRows) {
|
|
93
|
+
if (partitionCounts[row.partition] !== undefined) partitionCounts[row.partition]++;
|
|
90
94
|
}
|
|
95
|
+
const totalRecords = filteredRows.length;
|
|
96
|
+
let totalImages = 0;
|
|
91
97
|
|
|
92
|
-
// Let the adapter build the package layout
|
|
98
|
+
// Let the adapter build the package layout from the canonical rows.
|
|
93
99
|
const adapterResult = await adapter.buildPackage({
|
|
94
100
|
packageDir,
|
|
95
|
-
|
|
101
|
+
rows: filteredRows,
|
|
96
102
|
profile,
|
|
97
103
|
manifest,
|
|
98
104
|
config,
|
|
@@ -118,9 +124,9 @@ export async function buildTrainingPackage(projectRoot, manifestId, options = {}
|
|
|
118
124
|
config_fingerprint: manifest.config_fingerprint,
|
|
119
125
|
counts: {
|
|
120
126
|
total_records: totalRecords,
|
|
121
|
-
train:
|
|
122
|
-
val:
|
|
123
|
-
test:
|
|
127
|
+
train: partitionCounts.train,
|
|
128
|
+
val: partitionCounts.val,
|
|
129
|
+
test: partitionCounts.test,
|
|
124
130
|
images: totalImages,
|
|
125
131
|
checksums: 0, // updated after checksum pass below
|
|
126
132
|
},
|
package/lib/training-profiles.js
CHANGED
|
@@ -13,6 +13,13 @@ import { isRegisteredAdapter, listAdapters } from './training-adapters.js';
|
|
|
13
13
|
|
|
14
14
|
const REQUIRED_FIELDS = ['profile_id', 'label', 'target_family', 'asset_type', 'eligible_lanes', 'adapter_targets'];
|
|
15
15
|
|
|
16
|
+
// trigger_override constraints (research ref: two-LoRA stack D3, 2026-04-24).
|
|
17
|
+
// T5 tokenizes on SentencePiece Unigram with `_` as the space sentinel; hyphens
|
|
18
|
+
// fragment unpredictably and uppercase is not canonicalized. Generic suffixes
|
|
19
|
+
// alone collide across projects — a game-slug prefix is the documented defense.
|
|
20
|
+
const TRIGGER_OVERRIDE_RE = /^[a-z0-9_]+$/;
|
|
21
|
+
const TRIGGER_GENERIC_SUFFIX_DENYLIST = ['style', 'character', 'anime', 'realistic'];
|
|
22
|
+
|
|
16
23
|
/**
|
|
17
24
|
* Validate a training profile object.
|
|
18
25
|
* @returns {{ valid: boolean, errors: string[] }}
|
|
@@ -42,6 +49,32 @@ export function validateProfile(profile) {
|
|
|
42
49
|
}
|
|
43
50
|
}
|
|
44
51
|
}
|
|
52
|
+
// trigger_override is optional. When present, enforce SentencePiece-safe
|
|
53
|
+
// format and reject bare generic suffixes that collide across projects.
|
|
54
|
+
if (profile.trigger_override !== undefined && profile.trigger_override !== null) {
|
|
55
|
+
if (typeof profile.trigger_override !== 'string') {
|
|
56
|
+
errors.push('trigger_override must be a string');
|
|
57
|
+
} else if (profile.trigger_override.length === 0) {
|
|
58
|
+
errors.push('trigger_override must be a non-empty string (or omit the field)');
|
|
59
|
+
} else if (!TRIGGER_OVERRIDE_RE.test(profile.trigger_override)) {
|
|
60
|
+
errors.push(`trigger_override "${profile.trigger_override}" must match ^[a-z0-9_]+$ (lowercase, digits, underscores — no hyphens, no uppercase)`);
|
|
61
|
+
} else if (TRIGGER_GENERIC_SUFFIX_DENYLIST.includes(profile.trigger_override)) {
|
|
62
|
+
errors.push(`trigger_override "${profile.trigger_override}" is a generic suffix; prefix with a game slug (e.g. "sf_${profile.trigger_override}")`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
// entity_id_scope is optional (post-research D8). Same SentencePiece-safe
|
|
66
|
+
// format as trigger_override — no hyphens, no uppercase, lowercase slug.
|
|
67
|
+
// Used by per-character LoRA profiles to row-filter canonical dataset.jsonl
|
|
68
|
+
// to a single subject at packaging time.
|
|
69
|
+
if (profile.entity_id_scope !== undefined && profile.entity_id_scope !== null) {
|
|
70
|
+
if (typeof profile.entity_id_scope !== 'string') {
|
|
71
|
+
errors.push('entity_id_scope must be a string');
|
|
72
|
+
} else if (profile.entity_id_scope.length === 0) {
|
|
73
|
+
errors.push('entity_id_scope must be a non-empty string (or omit the field)');
|
|
74
|
+
} else if (!TRIGGER_OVERRIDE_RE.test(profile.entity_id_scope)) {
|
|
75
|
+
errors.push(`entity_id_scope "${profile.entity_id_scope}" must match ^[a-z0-9_]+$ (lowercase, digits, underscores — no hyphens, no uppercase)`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
45
78
|
return { valid: errors.length === 0, errors };
|
|
46
79
|
}
|
|
47
80
|
|