@mcptoolshop/style-dataset-lab 3.1.0 → 3.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mcptoolshop/style-dataset-lab",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "description": "Canon-aligned dataset production and generation workbench — define visual rules, build versioned training data, compile production briefs, run local workflows, batch-produce, select winners, and re-ingest into your corpus",
   "type": "module",
   "bin": {
@@ -18,7 +18,9 @@
     "generate:identity": "node scripts/generate-identity.js"
   },
   "c8": {
-    "include": ["lib/**/*.js"],
+    "include": [
+      "lib/**/*.js"
+    ],
     "exclude": [
       "tests/**",
       "scripts/**",
@@ -27,7 +29,11 @@
       "templates/**",
       "**/*.test.js"
     ],
-    "reporter": ["text", "lcov", "text-summary"],
+    "reporter": [
+      "text",
+      "lcov",
+      "text-summary"
+    ],
     "report-dir": "coverage",
     "all": true
   },
@@ -76,5 +82,8 @@
   "license": "MIT",
   "devDependencies": {
     "c8": "^10.1.3"
+  },
+  "dependencies": {
+    "yaml": "^2.8.3"
   }
 }

package/scripts/_batch_cut.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Batch alpha-cut cast turnaround views via ComfyUI's native BiRefNet (RGBA out).
+For each character, cuts its 8 restylized views -> out/<char>/<view>.png on transparency.
+Free Qwen (POST /free) before running so BiRefNet doesn't co-reside with the 28 GB transformer.
+
+    python _batch_cut.py --restylized <dir-of-per-char> --out <dir> --chars rustgrave,hearthframe,dustwhisper
+"""
+from __future__ import annotations
+import argparse, json, shutil, time, urllib.request, uuid
+from pathlib import Path
+
+COMFY = "http://127.0.0.1:8188"
+COMFY_OUT = Path(r"E:/AI-Models/ComfyUI_windows_portable/ComfyUI/output")
+COMFY_IN = Path(r"E:/AI-Models/ComfyUI_windows_portable/ComfyUI/input")
+MODEL = "birefnet.safetensors"
+
+
+def graph(image_filename: str, out_prefix: str) -> dict:
+    return {
+        "1": {"class_type": "LoadImage", "inputs": {"image": image_filename}},
+        "2": {"class_type": "LoadBackgroundRemovalModel", "inputs": {"bg_removal_name": MODEL}},
+        "3": {"class_type": "RemoveBackground", "inputs": {"image": ["1", 0], "bg_removal_model": ["2", 0]}},
+        "4": {"class_type": "InvertMask", "inputs": {"mask": ["3", 0]}},
+        "5": {"class_type": "JoinImageWithAlpha", "inputs": {"image": ["1", 0], "alpha": ["4", 0]}},
+        "6": {"class_type": "SaveImage", "inputs": {"filename_prefix": out_prefix, "images": ["5", 0]}},
+    }
+
+
+def post(path: str, payload: dict):
+    return json.loads(urllib.request.urlopen(urllib.request.Request(
+        f"{COMFY}{path}", json.dumps(payload).encode(), {"Content-Type": "application/json"}), timeout=120).read())
+
+
+def cut_one(src: Path, staged_name: str, dst: Path):
+    shutil.copyfile(src, COMFY_IN / f"{staged_name}.png")
+    g = graph(f"{staged_name}.png", f"cut_{staged_name}")
+    pid = post("/prompt", {"prompt": g, "client_id": uuid.uuid4().hex})["prompt_id"]
+    for _ in range(600):
+        hist = json.loads(urllib.request.urlopen(f"{COMFY}/history/{pid}", timeout=30).read())
+        if pid in hist and hist[pid].get("outputs"):
+            img = hist[pid]["outputs"]["6"]["images"][0]
+            srcp = COMFY_OUT / img["subfolder"] / img["filename"]
+            for _w in range(120):
+                if srcp.exists():
+                    dst.parent.mkdir(parents=True, exist_ok=True)
+                    shutil.copyfile(srcp, dst)
+                    return
+                time.sleep(0.5)
+        time.sleep(0.5)
+    raise RuntimeError(f"cut timed out: {src}")
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--restylized", required=True)
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--chars", required=True)
+    args = ap.parse_args()
+    rdir = Path(args.restylized); odir = Path(args.out)
+    n = 0
+    for c in args.chars.split(","):
+        c = c.strip()
+        for v in sorted((rdir / c).glob("*.png")):
+            cut_one(v, f"{c}_{v.stem}", odir / c / f"{v.stem}.png")
+            n += 1
+            print(f"  cut {c}/{v.stem}", flush=True)
+    print(f"BATCH CUT DONE: {n} sprites -> {odir}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/_seam_cutout.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""Alpha-cut sprites via ComfyUI's native BiRefNet background removal, for the A1 seam-test.
+
+Replicates the shipped "Remove Background (BiRefNet)" subgraph:
+  LoadImage -> LoadBackgroundRemovalModel -> RemoveBackground -> InvertMask -> JoinImageWithAlpha -> SaveImage
+Outputs RGBA PNGs (figure on transparency) and copies them to the Godot seam-test sprites dir.
+
+Inputs must already be in ComfyUI/input. Run with ComfyUI's embedded python:
+    python _seam_cutout.py seam_front seam_pitch30 seam_pitch40
+"""
+from __future__ import annotations
+
+import json
+import shutil
+import sys
+import time
+import urllib.request
+import uuid
+from pathlib import Path
+
+COMFY = "http://127.0.0.1:8188"
+COMFY_OUT = Path(r"E:/AI-Models/ComfyUI_windows_portable/ComfyUI/output")
+GODOT_SPRITES = Path(r"E:/AI/hesperia/game/seam_test/sprites")
+MODEL = "birefnet.safetensors"
+
+
+def graph(image_filename: str, out_prefix: str) -> dict:
+    return {
+        "1": {"class_type": "LoadImage", "inputs": {"image": image_filename}},
+        "2": {"class_type": "LoadBackgroundRemovalModel", "inputs": {"bg_removal_name": MODEL}},
+        "3": {"class_type": "RemoveBackground", "inputs": {"image": ["1", 0], "bg_removal_model": ["2", 0]}},
+        "4": {"class_type": "InvertMask", "inputs": {"mask": ["3", 0]}},
+        "5": {"class_type": "JoinImageWithAlpha", "inputs": {"image": ["1", 0], "alpha": ["4", 0]}},
+        "6": {"class_type": "SaveImage", "inputs": {"filename_prefix": out_prefix, "images": ["5", 0]}},
+    }
+
+
+def post(path: str, payload: dict):
+    return json.loads(urllib.request.urlopen(urllib.request.Request(
+        f"{COMFY}{path}", json.dumps(payload).encode(), {"Content-Type": "application/json"}), timeout=120).read())
+
+
+def run_one(name: str) -> Path:
+    g = graph(f"{name}.png", f"seamcut_{name}")
+    pid = post("/prompt", {"prompt": g, "client_id": uuid.uuid4().hex})["prompt_id"]
+    for _ in range(600):
+        hist = json.loads(urllib.request.urlopen(f"{COMFY}/history/{pid}", timeout=30).read())
+        if pid in hist and hist[pid].get("outputs"):
+            img = hist[pid]["outputs"]["6"]["images"][0]
+            src = COMFY_OUT / img["subfolder"] / img["filename"]
+            for _w in range(120):
+                if src.exists():
+                    GODOT_SPRITES.mkdir(parents=True, exist_ok=True)
+                    dst = GODOT_SPRITES / f"{name}.png"
+                    shutil.copyfile(src, dst)
+                    return dst
+                time.sleep(0.5)
+        time.sleep(0.5)
+    raise RuntimeError(f"cutout timed out for {name}")
+
+
+def main() -> None:
+    names = sys.argv[1:]
+    if not names:
+        print("usage: _seam_cutout.py <name> [<name> ...]  (names are ComfyUI/input stems, no .png)")
+        sys.exit(2)
+    for n in names:
+        dst = run_one(n)
+        print(f"  cut {n} -> {dst}", flush=True)
+    print(f"\nSEAMCUT DONE: {len(names)} sprite(s) -> {GODOT_SPRITES}")
+
+
+if __name__ == "__main__":
+    main()

package/scripts/canon-build.js

@@ -0,0 +1,88 @@
+/**
+ * `sdlab canon build` — CLI entry point for the three-projection build.
+ *
+ * Delegates orchestration to lib/canon-build/build.js. Parses flags, prints
+ * a summary, exits with codes per the errors.js inputError/runtimeError
+ * model (1 = user error, 2 = runtime error).
+ */
+
+import { parseArgs } from 'node:util';
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { runBuild } from '../lib/canon-build/build.js';
+import { inputError } from '../lib/errors.js';
+
+export async function run(argv) {
+  const parsed = parseArgs({
+    args: argv,
+    options: {
+      project:  { type: 'string' },
+      game:     { type: 'string' },           // deprecated alias
+      full:     { type: 'boolean', default: false },
+      'no-cache': { type: 'boolean', default: false },
+      'dry-run':  { type: 'boolean', default: false },
+      only:     { type: 'string' },
+      json:     { type: 'boolean', default: false },
+      quiet:    { type: 'boolean', default: false },
+    },
+    allowPositionals: true,
+    strict: false,
+  });
+
+  const projectName = parsed.values.project || parsed.values.game;
+  if (!projectName) {
+    throw inputError(
+      'INPUT_MISSING_FLAG',
+      'sdlab canon build requires --project <name>',
+      'Example: sdlab canon build --project greek-rpg',
+    );
+  }
+
+  const projectRoot = join(process.cwd(), 'projects', projectName);
+  if (!existsSync(projectRoot)) {
+    throw inputError(
+      'INPUT_PROJECT_NOT_FOUND',
+      `Project directory not found: ${projectRoot}`,
+      `Run from the style-dataset-lab repo root, or scaffold the project with "sdlab init ${projectName}".`,
+    );
+  }
+
+  const only = parsed.values.only
+    ? parsed.values.only.split(',').map((s) => s.trim()).filter(Boolean)
+    : null;
+
+  const result = await runBuild({
+    projectRoot,
+    full: parsed.values.full,
+    noCache: parsed.values['no-cache'],
+    dryRun: parsed.values['dry-run'],
+    only,
+  });
+
+  if (parsed.values.json) {
+    console.log(JSON.stringify(result, null, 2));
+    return;
+  }
+
+  if (!parsed.values.quiet) {
+    if (result.dryRun) {
+      console.log(`[dry-run] canon build would process ${result.entities_total} entities (${result.entities_cached} cached, ${result.entities_rebuilt} would rebuild) → ${result.rows} dataset rows.`);
+      console.log(`[dry-run] output would land at: ${result.output_dir}`);
+    } else {
+      console.log(`canon build complete: ${result.entities_total} entities (${result.entities_cached} cached, ${result.entities_rebuilt} rebuilt), ${result.rows} dataset rows.`);
+      console.log(`output: ${result.output_dir}`);
+      console.log(`generated_from: ${result.generated_from}`);
+      if (result.frozen_entries > 0) {
+        console.log(`frozen entries: ${result.frozen_entries} stamped with witness hashes`);
+      }
+      if (Array.isArray(result.drift) && result.drift.length > 0) {
+        console.log('');
+        console.log(`drift detected on ${result.drift.length} frozen entr${result.drift.length === 1 ? 'y' : 'ies'} since the previous build:`);
+        for (const d of result.drift) {
+          console.log(`  ${d.schema_kind}:${d.entity_id} (${d.status})`);
+        }
+        console.log('Run `sdlab canon drift` for the full report.');
+      }
+    }
+  }
+}

package/scripts/canon-drift.js

@@ -0,0 +1,193 @@
+/**
+ * `sdlab canon drift` — drift report (D7 command).
+ *
+ * For every frozen or on-canon-change entry, computes the current watch-hash
+ * and compares against the hash stamped in the latest canon-build manifest.
+ * Surfaces:
+ *   - Frozen entries whose current projection drifts from their witness.
+ *   - on-canon-change entries where the watch-fields have changed since freeze.
+ *   - Overrides since a given build hash (or since the last clean build).
+ */
+
+import { parseArgs } from 'node:util';
+import { readFile, readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { loadBuildConfig } from '../lib/canon-build/load-config.js';
+import { loadCanonEntriesInDir } from '../lib/canon-build/load-entry.js';
+import { loadSchema } from '../lib/canon-build/load-schema.js';
+import { readFreezeStatus, resolveWatchFields, computeWatchHash } from '../lib/freeze-stamp.js';
+import { readEventsSince } from '../lib/freeze-events.js';
+import { inputError } from '../lib/errors.js';
+
+export async function run(argv) {
+  const parsed = parseArgs({
+    args: argv,
+    options: {
+      project: { type: 'string' },
+      game:    { type: 'string' },
+      since:   { type: 'string' },
+      json:    { type: 'boolean', default: false },
+    },
+    allowPositionals: true,
+    strict: false,
+  });
+
+  const projectName = parsed.values.project || parsed.values.game;
+  if (!projectName) throw inputError('INPUT_MISSING_FLAG', '--project required');
+
+  const projectRoot = join(process.cwd(), 'projects', projectName);
+  const config = await loadBuildConfig(join(projectRoot, 'canon-build', 'config.json'), projectRoot);
+
+  // Load latest build manifest for the stamped watch-hashes
+  const latestManifest = await findLatestBuildManifest(projectRoot);
+
+  // Load schemas for version info
+  const schemas = {};
+  for (const schemaName of Object.keys(config.entity_dirs)) {
+    const path = join(config.schema_dir, schemaName);
+    schemas[schemaName] = await loadSchema(path);
+  }
+
+  // Walk entries, compute drift
+  const drifted = [];
+  const cleanFrozen = [];
+  let totalEntries = 0;
+  let autoCount = 0;
+
+  for (const [schemaName, subdir] of Object.entries(config.entity_dirs)) {
+    const dir = join(config.canon_root, subdir);
+    const entries = await loadCanonEntriesInDir(dir);
+    for (const entry of entries) {
+      totalEntries++;
+      const status = readFreezeStatus(entry.frontmatter);
+      if (status === 'auto') { autoCount++; continue; }
+
+      const watchFields = resolveWatchFields(entry.frontmatter, schemaName, config);
+      const currentHash = computeWatchHash(
+        entry.frontmatter,
+        watchFields,
+        schemas[schemaName].version,
+      );
+
+      // Look up stamped hash from latest manifest. The manifest stores a
+      // richer stamp object (status / watch_hash / watch_fields /
+      // locked_at_build); the drift comparison only needs `.watch_hash`.
+      const entityId = entry.frontmatter.id;
+      const stamp = latestManifest?.frozen_entries_hashes?.[entityId] || null;
+      const stampedHash = stamp?.watch_hash || null;
+
+      if (stampedHash && stampedHash !== currentHash) {
+        drifted.push({
+          entity_id: entityId,
+          schema_kind: schemaName.replace('.schema.json', ''),
+          status,
+          locked_at_build: stamp?.locked_at_build || entry.frontmatter.freeze?.locked_at_build || null,
+          witness_hash: stampedHash,
+          current_hash: currentHash,
+          watch_fields: watchFields,
+        });
+      } else if (stampedHash) {
+        cleanFrozen.push({
+          entity_id: entityId,
+          status,
+          witness_hash: stampedHash,
+        });
+      } else {
+        // Entry is frozen now but was not in the last build manifest (e.g.
+        // newly frozen since). Flag as "no witness yet" rather than mis-
+        // classifying as either drifted or clean.
+        cleanFrozen.push({
+          entity_id: entityId,
+          status,
+          witness_hash: null,
+          note: 'no witness in latest build — rebuild to stamp',
+        });
+      }
+    }
+  }
+
+  // Events since a reference build hash
+  const sinceHash = parsed.values.since || latestManifest?.generated_from || null;
+  const eventsSince = sinceHash
+    ? await readEventsSince(projectRoot, sinceHash)
+    : [];
+
+  const report = {
+    project_id: config.project_id,
+    latest_build: latestManifest?.generated_from || null,
+    latest_built_at: latestManifest?.built_at || null,
+    entities_total: totalEntries,
+    entities_auto: autoCount,
+    entities_frozen_clean: cleanFrozen.length,
+    entities_drifted: drifted.length,
+    drifted,
+    overrides_since: eventsSince.filter((e) => e.type !== 'freeze'),
+  };
+
+  if (parsed.values.json) {
+    console.log(JSON.stringify(report, null, 2));
+    return;
+  }
+
+  renderHumanReport(report);
+}
+
+function renderHumanReport(r) {
+  console.log(`Drift report for ${r.project_id}`);
+  console.log(`  latest build:    ${r.latest_build || '(no build output found)'}`);
+  console.log(`  latest built_at: ${r.latest_built_at || '(unknown)'}`);
+  console.log('');
+  console.log(`  auto entries:            ${r.entities_auto}`);
+  console.log(`  frozen entries (clean):  ${r.entities_frozen_clean}`);
+  console.log(`  frozen entries (drift):  ${r.entities_drifted}`);
+  console.log('');
+
+  if (r.drifted.length > 0) {
+    console.log('Drifted frozen entries:');
+    for (const d of r.drifted) {
+      console.log(`  ${d.schema_kind}:${d.entity_id} (${d.status})`);
+      console.log(`    witness hash:  ${d.witness_hash.slice(0, 16)}...`);
+      console.log(`    current hash:  ${d.current_hash.slice(0, 16)}...`);
+      console.log(`    watch fields:  ${d.watch_fields.join(', ') || '(none)'}`);
+    }
+    console.log('');
+  } else if (r.entities_frozen_clean > 0) {
+    console.log('No drift detected on frozen entries.');
+    console.log('');
+  }
+
+  if (r.overrides_since.length > 0) {
+    console.log(`Overrides since last build (${r.overrides_since.length}):`);
+    for (const e of r.overrides_since) {
+      console.log(`  ${e.type.padEnd(10)} ${e.entity_id} by ${e.by} — ${e.reason}`);
+    }
+  }
+}
+
+async function findLatestBuildManifest(projectRoot) {
+  const canonBuildDir = join(projectRoot, 'canon-build');
+  if (!existsSync(canonBuildDir)) return null;
+  const entries = await readdir(canonBuildDir, { withFileTypes: true });
+  const candidates = entries
+    .filter((e) => e.isDirectory() && !e.name.startsWith('.') && !e.name.includes('.tmp'))
+    .map((e) => e.name);
+
+  let latest = null;
+  let latestTime = 0;
+  for (const name of candidates) {
+    const manifestPath = join(canonBuildDir, name, 'manifest.json');
+    if (!existsSync(manifestPath)) continue;
+    try {
+      const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
+      const built = new Date(manifest.built_at || 0).getTime();
+      if (built > latestTime) {
+        latestTime = built;
+        latest = manifest;
+      }
+    } catch {
+      continue;
+    }
+  }
+  return latest;
+}

package/scripts/canon-freeze-status.js

@@ -0,0 +1,85 @@
+/**
+ * `sdlab canon freeze-status <entity_id>` — read-only glance at an entry's
+ * freeze state. Human-readable by default; --json for machine consumption.
+ */
+
+import { parseArgs } from 'node:util';
+import { join } from 'node:path';
+import { loadBuildConfig } from '../lib/canon-build/load-config.js';
+import { loadCanonEntriesInDir } from '../lib/canon-build/load-entry.js';
+import { readFreezeStatus, resolveWatchFields } from '../lib/freeze-stamp.js';
+import { readEventsFor } from '../lib/freeze-events.js';
+import { inputError } from '../lib/errors.js';
+
+export async function run(argv) {
+  const parsed = parseArgs({
+    args: argv,
+    options: {
+      project: { type: 'string' },
+      game:    { type: 'string' },
+      json:    { type: 'boolean', default: false },
+    },
+    allowPositionals: true,
+    strict: false,
+  });
+
+  const entityIdArg = (parsed.positionals || [])[0];
+  if (!entityIdArg) throw inputError('INPUT_MISSING_ARG', 'sdlab canon freeze-status requires an entity id.');
+  const projectName = parsed.values.project || parsed.values.game;
+  if (!projectName) throw inputError('INPUT_MISSING_FLAG', '--project required');
+
+  const projectRoot = join(process.cwd(), 'projects', projectName);
+  const config = await loadBuildConfig(join(projectRoot, 'canon-build', 'config.json'), projectRoot);
+
+  let resolved = null;
+  for (const [schemaName, subdir] of Object.entries(config.entity_dirs)) {
+    const dir = join(config.canon_root, subdir);
+    const entries = await loadCanonEntriesInDir(dir);
+    const hit = entries.find((e) => e.frontmatter?.id === entityIdArg);
+    if (hit) { resolved = { entry: hit, schemaName }; break; }
+  }
+  if (!resolved) throw inputError('CANON_ENTRY_NOT_FOUND', `No canon entry "${entityIdArg}".`);
+
+  const fm = resolved.entry.frontmatter;
+  const status = readFreezeStatus(fm);
+  const block = fm.freeze || {};
+  const events = await readEventsFor(projectRoot, entityIdArg);
+
+  // Entry override OR config default — the same resolution used by the
+  // build, so a reader sees the actual watched fields, not just what's
+  // literally in the entry frontmatter.
+  const resolvedWatchFields = resolveWatchFields(fm, resolved.schemaName, config);
+  const watchFieldsSource = Array.isArray(block.watch_fields) && block.watch_fields.length
+    ? 'entry-override'
+    : (resolvedWatchFields.length ? 'config-default' : 'none');
+
+  const result = {
+    entity_id: entityIdArg,
+    schema_kind: resolved.schemaName.replace('.schema.json', ''),
+    status,
+    locked_at_build: block.locked_at_build || null,
+    frozen_by: block.frozen_by || null,
+    frozen_reason: block.frozen_reason || null,
+    watch_fields: resolvedWatchFields,
+    watch_fields_source: watchFieldsSource,
+    overrides_count: Array.isArray(block.overrides) ? block.overrides.length : 0,
+    event_count: events.length,
+  };
+
+  if (parsed.values.json) {
+    console.log(JSON.stringify(result, null, 2));
+    return;
+  }
+
+  console.log(`${entityIdArg} (${result.schema_kind})`);
+  console.log(`  status:          ${status}`);
+  if (status !== 'auto') {
+    console.log(`  locked_at_build: ${result.locked_at_build || '(unset)'}`);
+    console.log(`  frozen_by:       ${result.frozen_by || '(unset)'}`);
+    console.log(`  frozen_reason:   ${result.frozen_reason || '(unset)'}`);
+    const fieldsDisplay = result.watch_fields.length ? result.watch_fields.join(', ') : '(none)';
+    console.log(`  watch_fields:    ${fieldsDisplay} [${result.watch_fields_source}]`);
+  }
+  console.log(`  overrides[]:     ${result.overrides_count} recorded on entry`);
+  console.log(`  event log:       ${result.event_count} events total`);
+}