@mcptoolshop/style-dataset-lab 3.1.0 → 3.2.0

package/lib/canon-build/lane-resolve.js

@@ -0,0 +1,64 @@
+/**
+ * Lane resolution for canon entries (D3 + D9).
+ *
+ * The build config declares, per schema, how to resolve an entry's lane:
+ *   - { source: "constant", value: "creature" } → always that value
+ *   - { source: "field", field: "visual.art_lane" } → read that dotted path
+ *     from the entry's frontmatter
+ *
+ * Unmapped entries throw a structured error so an operator can fix the
+ * source of truth rather than letting the build silently drop a row.
+ */
+
+import { inputError } from '../errors.js';
+import { getByPath } from './load-config.js';
+
+/**
+ * Resolve the lane for a single entry.
+ *
+ * @param {Object} entry — loaded canon entry ({ frontmatter, filePath, ... })
+ * @param {string} schemaName — e.g. "monster.schema.json"
+ * @param {Object} schemaToLane — config.schema_to_lane rule object
+ * @returns {string} resolved lane
+ * @throws SdlabError CANON_LANE_UNMAPPED when resolution fails
+ */
+export function resolveLane(entry, schemaName, schemaToLane) {
+  const rule = schemaToLane[schemaName];
+  if (!rule) {
+    throw inputError(
+      'CANON_LANE_UNMAPPED',
+      `canon-build config has no schema_to_lane rule for "${schemaName}" (entry ${entry.filePath})`,
+      `Add a rule to canon-build/config.json — e.g. { "source": "field", "field": "visual.art_lane" } or { "source": "constant", "value": "<lane>" }.`,
+    );
+  }
+
+  if (rule.source === 'constant') {
+    return rule.value;
+  }
+
+  if (rule.source === 'field') {
+    const value = getByPath(entry.frontmatter, rule.field);
+    if (value === undefined || value === null || value === '') {
+      throw inputError(
+        'CANON_LANE_UNMAPPED',
+        `entry ${entry.filePath} has no value at "${rule.field}" (required to resolve lane for schema "${schemaName}")`,
+        `Add ${rule.field} to the entry's frontmatter, or switch the config rule to source: "constant" if the schema should use a fixed lane.`,
+      );
+    }
+    if (typeof value !== 'string') {
+      throw inputError(
+        'CANON_LANE_UNMAPPED',
+        `entry ${entry.filePath} value at "${rule.field}" is not a string (got ${typeof value})`,
+        'Lane resolution expects a string. Check the schema definition for this field.',
+      );
+    }
+    return value;
+  }
+
+  // load-config.js already rejects unknown sources; this is defense-in-depth.
+  throw inputError(
+    'CANON_LANE_UNMAPPED',
+    `schema_to_lane rule for "${schemaName}" has unsupported source "${rule.source}"`,
+    'Valid sources: "field" or "constant".',
+  );
+}

package/lib/canon-build/load-config.js

@@ -0,0 +1,117 @@
+/**
+ * canon-build/config.json loader + validator.
+ *
+ * The build config declares:
+ *   - canon_root: absolute (or project-relative) path to the canon directory
+ *   - schema_dir: absolute path to the schemas directory
+ *   - entity_dirs: schema filename → subdirectory under canon_root
+ *   - schema_to_lane: per-schema lane resolution (D3 / D9)
+ *   - context_limits: per-schema max_lines (default 300)
+ *   - profile_id: training profile id that supplies trigger + caption strategy
+ *     (canon-build delegates to captions.js just like the adapters do).
+ *   - freeze_watch_defaults: per-schema default watch_fields for the freeze
+ *     witness chain (D5). Consumed by lib/freeze-stamp.js:resolveWatchFields.
+ */
+
+import { readFile } from 'node:fs/promises';
+import { isAbsolute, resolve } from 'node:path';
+import { existsSync } from 'node:fs';
+import { inputError } from '../errors.js';
+
+/**
+ * Load and validate a canon-build config.
+ *
+ * @param {string} configPath — absolute path to canon-build/config.json
+ * @param {string} projectRoot — for resolving project-relative paths
+ */
+export async function loadBuildConfig(configPath, projectRoot) {
+  if (!existsSync(configPath)) {
+    throw inputError(
+      'CANON_BUILD_CONFIG_NOT_FOUND',
+      `canon-build config not found at ${configPath}`,
+      `Create the file with a project_id, canon_root, schema_dir, entity_dirs, and schema_to_lane map. See the handbook page "Canon Build" for the schema.`,
+    );
+  }
+
+  const raw = await readFile(configPath, 'utf-8');
+  let config;
+  try {
+    config = JSON.parse(raw);
+  } catch (err) {
+    throw inputError(
+      'CANON_BUILD_CONFIG_INVALID_JSON',
+      `canon-build config at ${configPath} is not valid JSON (${err.message})`,
+      'Fix the JSON syntax; this file is the lane-resolution source of truth for the build.',
+    );
+  }
+
+  const errors = [];
+
+  if (!config.project_id) errors.push('missing required field: project_id');
+  if (!config.canon_root) errors.push('missing required field: canon_root');
+  if (!config.schema_dir) errors.push('missing required field: schema_dir');
+  if (!config.entity_dirs || typeof config.entity_dirs !== 'object') {
+    errors.push('missing or invalid entity_dirs (expected object mapping schema filename to subdirectory)');
+  }
+  if (!config.schema_to_lane || typeof config.schema_to_lane !== 'object') {
+    errors.push('missing or invalid schema_to_lane (expected object)');
+  } else {
+    for (const [schemaName, rule] of Object.entries(config.schema_to_lane)) {
+      if (!rule || typeof rule !== 'object') {
+        errors.push(`schema_to_lane["${schemaName}"] must be an object`);
+        continue;
+      }
+      if (rule.source === 'field') {
+        if (!rule.field || typeof rule.field !== 'string') {
+          errors.push(`schema_to_lane["${schemaName}"].source="field" requires a "field" string (dot-path)`);
+        }
+      } else if (rule.source === 'constant') {
+        if (!rule.value || typeof rule.value !== 'string') {
+          errors.push(`schema_to_lane["${schemaName}"].source="constant" requires a "value" string`);
+        }
+      } else {
+        errors.push(`schema_to_lane["${schemaName}"].source must be "field" or "constant" (got "${rule.source}")`);
+      }
+    }
+  }
+
+  if (errors.length) {
+    throw inputError(
+      'CANON_BUILD_CONFIG_INVALID',
+      `canon-build config at ${configPath} is invalid:\n  ${errors.join('\n  ')}`,
+      'Fix the listed fields and retry.',
+    );
+  }
+
+  // Resolve relative paths against the project root.
+  const canonRoot = isAbsolute(config.canon_root) ? config.canon_root : resolve(projectRoot, config.canon_root);
+  const schemaDir = isAbsolute(config.schema_dir) ? config.schema_dir : resolve(projectRoot, config.schema_dir);
+
+  return {
+    project_id: config.project_id,
+    canon_root: canonRoot,
+    schema_dir: schemaDir,
+    entity_dirs: config.entity_dirs,
+    schema_to_lane: config.schema_to_lane,
+    context_limits: config.context_limits || {},
+    profile_id: config.profile_id || null,
+    freeze_watch_defaults: config.freeze_watch_defaults || {},
+    // Preserve raw text for hashing — any change to the config invalidates cache.
+    raw_text: raw,
+  };
+}
+
+/**
+ * Read a nested value from an object via a dot-path (e.g. "visual.art_lane").
+ * Returns undefined if any segment is missing.
+ */
+export function getByPath(obj, path) {
+  if (!obj || !path) return undefined;
+  const parts = path.split('.');
+  let cur = obj;
+  for (const p of parts) {
+    if (cur == null || typeof cur !== 'object') return undefined;
+    cur = cur[p];
+  }
+  return cur;
+}

package/lib/canon-build/load-entry.js

@@ -0,0 +1,62 @@
+/**
+ * Canon entry loader.
+ *
+ * Reads a canon MD file, splits frontmatter from body, returns a normalized
+ * entry object. Validation against the schema happens later in the
+ * orchestrator (after the schema has been loaded).
+ */
+
+import { readFile, readdir } from 'node:fs/promises';
+import { join, extname, basename } from 'node:path';
+import { existsSync } from 'node:fs';
+import { parseFrontmatter } from './frontmatter.js';
+
+/**
+ * Load a single canon entry file.
+ *
+ * @param {string} filePath — absolute path to a .md file
+ * @returns {Promise<{ frontmatter: Object, body: string, rawYaml: string, rawText: string, filePath: string }>}
+ */
+export async function loadCanonEntry(filePath) {
+  const text = await readFile(filePath, 'utf-8');
+  const parsed = parseFrontmatter(text, filePath);
+  return {
+    frontmatter: parsed.frontmatter,
+    body: parsed.body,
+    rawYaml: parsed.rawYaml,
+    rawText: text,
+    filePath,
+  };
+}
+
+/**
+ * Load all canon entries in a directory.
+ *
+ * Non-.md files are skipped; .md files that fail to parse throw with the
+ * filename included so the operator can fix the offending entry.
+ *
+ * @param {string} dir — absolute path to a canon-type directory
+ * @returns {Promise<Array<{frontmatter, body, rawYaml, rawText, filePath}>>}
+ */
+export async function loadCanonEntriesInDir(dir) {
+  if (!existsSync(dir)) return [];
+  const files = await readdir(dir);
+  const entries = [];
+  for (const file of files.sort()) {
+    if (extname(file) !== '.md') continue;
+    const filePath = join(dir, file);
+    const entry = await loadCanonEntry(filePath);
+    entries.push(entry);
+  }
+  return entries;
+}
+
+/**
+ * Derive an entity id from a canon entry.
+ * Prefers frontmatter.id; falls back to filename stem for entries that
+ * haven't added the id field yet (validation will reject those downstream).
+ */
+export function entryId(entry) {
+  if (entry.frontmatter?.id) return entry.frontmatter.id;
+  return basename(entry.filePath, '.md');
+}

package/lib/canon-build/load-schema.js

@@ -0,0 +1,46 @@
+/**
+ * Schema loader for canon-build.
+ *
+ * Reads a JSON Schema file and returns { schema, version }.
+ *
+ * `version` resolution (per post-research D7):
+ *   1. If the schema's top-level `version` field is a string, use it.
+ *   2. Otherwise fall back to SHA-256 of the schema file contents, prefixed
+ *      with `content-sha:`. Any textual change (including whitespace)
+ *      invalidates the fallback. Documented as a defensive bridge for
+ *      projects whose schemas haven't been version-stamped yet.
+ */
+
+import { readFile } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { inputError } from '../errors.js';
+
+/**
+ * Load and resolve a single schema file.
+ *
+ * @param {string} schemaPath — absolute path to <name>.schema.json
+ * @returns {Promise<{ schema: Object, version: string, fileBytes: Buffer }>}
+ */
+export async function loadSchema(schemaPath) {
+  const buf = await readFile(schemaPath);
+  let schema;
+  try {
+    schema = JSON.parse(buf.toString('utf-8'));
+  } catch (err) {
+    throw inputError(
+      'CANON_SCHEMA_INVALID_JSON',
+      `Schema file is not valid JSON: ${schemaPath} (${err.message})`,
+      'Validate the schema file with a JSON linter; malformed schemas cannot be used to validate canon entries.',
+    );
+  }
+
+  let version;
+  if (typeof schema.version === 'string' && schema.version.length > 0) {
+    version = schema.version;
+  } else {
+    const hash = createHash('sha256').update(buf).digest('hex');
+    version = `content-sha:${hash}`;
+  }
+
+  return { schema, version, fileBytes: buf };
+}

package/lib/captions.js

@@ -0,0 +1,197 @@
+/**
+ * Caption building — shared between training adapters.
+ *
+ * Research-backed strategies for producing LoRA-training captions from
+ * approved canon records. Previously, both adapters carried near-duplicate
+ * buildCaption stubs that spliced `provenance.prompt` into the caption —
+ * a prompt-bleed antipattern that bakes generation-time style vocabulary
+ * (e.g. "oil painting, semi-realistic painterly, muted dusty palette...")
+ * back into the trigger token and corrupts downstream training.
+ *
+ * Strategies (from `profile.caption_strategy`):
+ *   - 'legacy' (default when unset): preserves prior behavior exactly —
+ *      trigger, group, lane, provenance.prompt[0..200]. Kept so existing
+ *      training manifests reproduce bit-identical.
+ *   - 'structured-metadata': trigger, faction, group, lane,
+ *      judgment.explanation. No generation prompt. Research-backed for
+ *      SDXL-style LoRAs where the trigger should absorb the style and
+ *      captions describe scene content as comma-separated tags.
+ *   - 'flux-natural-language': trigger + subject noun phrase + scene as
+ *      a natural-language description. Recommended when
+ *      `profile.target_family === 'flux'` because Flux's T5 text encoder
+ *      holds identity across natural-language prompts much better than
+ *      SDXL's CLIP does on comma-separated tags. See research notes.
+ *
+ * References:
+ *   - Apatero LoRA Best Practices 2025 — trigger-first tag ordering
+ *   - alvdansen HF blog — caption format variety
+ *   - Ye et al. 2025 (arXiv 2510.09475) — multi-token DreamBooth+LoRA
+ *   - Pelayo Arbues — Flux caption template (NL + scene variations)
+ *   - Civitai Flux Character Caption Diary (article 6868) — natural
+ *     language outperforms booru-style tags on Flux
+ */
+
+/**
+ * Derive the style trigger token from a profile.
+ *
+ * If `profile.trigger_override` is a non-empty string, use it verbatim.
+ * Otherwise fall back to the hyphenated-id derivation:
+ * `character-style-lora` → `character_style_lora`.
+ *
+ * `trigger_override` exists so profile IDs can stay descriptive (e.g.
+ * `character-style-lora-flux`) while triggers stay compact, spec-compliant
+ * (`^[a-z0-9_]+$`), and game-prefixed. Validation of the override value
+ * (format + generic-suffix deny-list) lives in training-profiles.js so
+ * captions are never the gate for a malformed trigger.
+ *
+ * @param {{profile_id?: string, trigger_override?: string}} profile
+ * @returns {string|null}
+ */
+export function deriveStyleTrigger(profile) {
+  if (!profile) return null;
+  if (typeof profile.trigger_override === 'string' && profile.trigger_override.length > 0) {
+    return profile.trigger_override;
+  }
+  if (!profile.profile_id) return null;
+  return profile.profile_id.replace(/-/g, '_');
+}
+
+/**
+ * Build a training caption from a record + context + profile.
+ *
+ * @param {Object} record    — approved canon record (see schema in projects/**)
+ * @param {string|null} lane — partition lane (costume, equipment, ...)
+ * @param {string|null} group — partition group (faction, region, ...)
+ * @param {Object} profile   — loaded training profile JSON
+ * @returns {string}
+ */
+export function buildCaption(record, lane, group, profile) {
+  const strategy = profile?.caption_strategy || 'legacy';
+
+  switch (strategy) {
+    case 'structured-metadata':
+      return buildStructuredMetadataCaption(record, lane, group, profile);
+
+    case 'flux-natural-language':
+      return buildFluxNaturalLanguageCaption(record, lane, group, profile);
+
+    case 'legacy':
+    default:
+      return buildLegacyCaption(record, lane, group, profile);
+  }
+}
+
+/**
+ * Legacy caption — preserved verbatim from pre-refactor adapters so existing
+ * training manifests remain reproducible.
+ *
+ * Shape: `<trigger>, <group>, <lane>, <provenance.prompt sliced to 200>`
+ *
+ * WARNING: This strategy splices the raw generation prompt into the caption,
+ * which causes prompt bleed during training. Kept for backward compatibility
+ * only. New profiles should use `structured-metadata`.
+ */
+function buildLegacyCaption(record, lane, group, profile) {
+  const parts = [];
+
+  if (profile?.prompt_strategy === 'trigger-word' && profile?.profile_id) {
+    parts.push(profile.profile_id.replace(/-/g, '_'));
+  }
+
+  if (group) parts.push(group);
+  if (lane) parts.push(lane);
+
+  const prompt = record?.provenance?.prompt;
+  if (prompt) {
+    parts.push(prompt.slice(0, 200));
+  } else if (record?.id) {
+    parts.push(record.id.replace(/_/g, ' '));
+  }
+
+  return parts.join(', ');
+}
+
+/**
+ * Structured-metadata caption.
+ *
+ * Builds captions from clean, canon-anchored fields only. Never touches
+ * `provenance.prompt` (which contains style vocabulary that causes bleed).
+ *
+ * Order follows the research rule that earlier tokens are weighted higher:
+ *   1. Style trigger (profile-derived)    — absorbs the house style
+ *   2. Faction / group (canon-derived)    — primary discriminator
+ *   3. Lane                                — category context
+ *   4. Scene description from judgment    — short human scene notes
+ *
+ * Dedupes faction vs group when they're the same value. Trims empty
+ * segments. No segment exceeds a reasonable bound individually; full
+ * caption is expected to stay well under the 75-token practical cap.
+ */
+function buildStructuredMetadataCaption(record, lane, group, profile) {
+  const parts = [];
+
+  const trigger = deriveStyleTrigger(profile);
+  if (trigger) parts.push(trigger);
+
+  const faction = record?.canon?.faction;
+  if (faction) parts.push(`${faction} faction`);
+
+  if (group && group !== faction) parts.push(group);
+
+  if (lane) parts.push(lane);
+
+  const scene = record?.judgment?.explanation;
+  if (scene) parts.push(scene.trim());
+
+  return parts.join(', ');
+}
+
+/**
+ * Flux natural-language caption.
+ *
+ * Flux's T5 text encoder holds identity across long, natural-language
+ * prompts where SDXL's CLIP loses identity on comma-separated tag lists.
+ * For Flux LoRAs the recommended caption shape is a full sentence:
+ * trigger-style descriptor, then a subject noun phrase, then a short
+ * scene sentence.
+ *
+ * Template (from the research): `[Trigger] style, a [Subject], [Scene].`
+ *
+ * Unlike structured-metadata, this strategy is prose-shaped — sentences
+ * with articles ("a", "the") rather than bare comma-joined tags. But it
+ * still uses only clean canon fields (faction, lane, judgment.explanation).
+ * Never touches `provenance.prompt`.
+ *
+ * Example (anchor_03_officer_v3):
+ *   "character_style_lora style, a compact faction costume, Steel-blue
+ *    uniform, high collar, ship corridor. Clean and institutional."
+ *
+ * Richer captions (hair / eyes / clothing / pose breakouts) require
+ * structured visual fields on the canon record that don't exist yet.
+ * This strategy is the floor — on-contract Flux captions from the fields
+ * we have. Expand the canon schema first, then enrich this strategy.
+ */
+function buildFluxNaturalLanguageCaption(record, lane, group, profile) {
+  const segments = [];
+
+  const trigger = deriveStyleTrigger(profile);
+  if (trigger) segments.push(`${trigger} style`);
+
+  const faction = record?.canon?.faction;
+  const subjectParts = [];
+  if (faction) subjectParts.push(`${faction} faction`);
+  if (lane) {
+    subjectParts.push(lane);
+  } else if (group && group !== faction) {
+    subjectParts.push(group);
+  }
+
+  if (subjectParts.length > 0) {
+    segments.push(`a ${subjectParts.join(' ')}`);
+  }
+
+  const scene = record?.judgment?.explanation;
+  if (scene) segments.push(scene.trim());
+
+  return segments.join(', ');
+}

package/lib/config.js

@@ -88,6 +88,25 @@ export function loadProjectMeta(projectRoot) {
   return data;
 }
 
+/**
+ * Default GPU label written into provenance when a project doesn't pin one.
+ * This rig is the Robot rig (Omen 45L, RTX 5090 32GB); older records say
+ * "RTX 5080" because the generator hardcoded the prior card.
+ */
+export const DEFAULT_GPU_MODEL = 'RTX 5090';
+
+/**
+ * Resolve the GPU label for a generation provenance record.
+ * Precedence: project.json `defaults.gpu_model` → DEFAULT_GPU_MODEL.
+ *
+ * @param {Object} [projectDefaults] — the `defaults` block from project.json
+ * @returns {string}
+ */
+export function resolveGpuModel(projectDefaults) {
+  const v = projectDefaults?.gpu_model;
+  return typeof v === 'string' && v.trim() ? v.trim() : DEFAULT_GPU_MODEL;
+}
+
 /**
  * Load constitution.json — array of rule objects.
  */

package/lib/freeze-events.js

@@ -0,0 +1,109 @@
+/**
+ * Append-only event log for freeze transitions (D4).
+ *
+ * Lives at `<project>/canon-build/freeze-events.jsonl`. One line per event,
+ * JSON, stable key order. The jsonl is the audit source of truth; the
+ * `freeze.overrides[]` denormalization on the entry is a glance cache.
+ *
+ * Atomic append: we write the new event as a line-append operation at the
+ * OS level. Node's `appendFile` is atomic per-line on POSIX but not
+ * guaranteed on Windows when two processes race. For single-operator
+ * workflow this is fine; for a paranoid hardening pass later we can add
+ * a lockfile. Documented as a known hazard.
+ */
+
+import { readFile, appendFile, mkdir } from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { existsSync } from 'node:fs';
+
+export const FREEZE_EVENT_TYPES = Object.freeze(['freeze', 'unfreeze', 'override', 'bypass']);
+
+export function eventsPath(projectRoot) {
+  return join(projectRoot, 'canon-build', 'freeze-events.jsonl');
+}
+
+/**
+ * Append one event to the log. Creates the parent dir + file on first write.
+ *
+ * @param {string} projectRoot — absolute path to the sdlab project root
+ * @param {Object} event — at minimum: {type, entity_id, by, reason}
+ */
+export async function appendEvent(projectRoot, event) {
+  if (!FREEZE_EVENT_TYPES.includes(event.type)) {
+    throw new Error(`appendEvent: unknown event type "${event.type}" (allowed: ${FREEZE_EVENT_TYPES.join(', ')})`);
+  }
+  if (!event.entity_id) throw new Error('appendEvent: entity_id required');
+  if (!event.by) throw new Error('appendEvent: by required');
+  if (!event.reason) throw new Error('appendEvent: reason required');
+
+  const path = eventsPath(projectRoot);
+  await mkdir(dirname(path), { recursive: true });
+
+  // Serialize with a stable key order so `diff`-style reading stays readable.
+  const ordered = {
+    at: event.at || new Date().toISOString(),
+    build_hash: event.build_hash ?? null,
+    type: event.type,
+    entity_id: event.entity_id,
+    schema_kind: event.schema_kind ?? null,
+    by: event.by,
+    reason: event.reason,
+    prior_status: event.prior_status ?? null,
+    new_status: event.new_status ?? null,
+    watch_fields: event.watch_fields ?? null,
+    prior_build_hash: event.prior_build_hash ?? null,
+  };
+
+  await appendFile(path, JSON.stringify(ordered) + '\n', 'utf-8');
+  return ordered;
+}
+
+/**
+ * Read all events from the log, in append order.
+ */
+export async function readEvents(projectRoot) {
+  const path = eventsPath(projectRoot);
+  if (!existsSync(path)) return [];
+  const text = await readFile(path, 'utf-8');
+  const events = [];
+  const lines = text.split('\n');
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (!line) continue;
+    try {
+      events.push(JSON.parse(line));
+    } catch (err) {
+      throw new Error(`readEvents: line ${i + 1} of ${path} is not valid JSON (${err.message})`);
+    }
+  }
+  return events;
+}
+
+/**
+ * Return events for a specific entity, in append order.
+ */
+export async function readEventsFor(projectRoot, entityId) {
+  const all = await readEvents(projectRoot);
+  return all.filter((e) => e.entity_id === entityId);
+}
+
+/**
+ * Return events newer than a given build_hash. Used by `sdlab canon drift`
+ * to compute "overrides since last build."
+ *
+ * Semantics: events are "since X" when their `build_hash` field differs
+ * from X. Since the log is append-only, that approximates temporal ordering.
+ */
+export async function readEventsSince(projectRoot, buildHash) {
+  if (!buildHash) return readEvents(projectRoot);
+  const all = await readEvents(projectRoot);
+  // Find last event with matching build_hash; return everything after it.
+  let cutoff = -1;
+  for (let i = all.length - 1; i >= 0; i--) {
+    if (all[i].build_hash === buildHash) {
+      cutoff = i;
+      break;
+    }
+  }
+  return cutoff < 0 ? all : all.slice(cutoff + 1);
+}