@mcptoolshop/style-dataset-lab 3.1.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -0
- package/README.es.md +96 -63
- package/README.fr.md +97 -64
- package/README.hi.md +108 -77
- package/README.it.md +97 -64
- package/README.ja.md +105 -72
- package/README.md +33 -0
- package/README.pt-BR.md +94 -61
- package/README.zh.md +98 -65
- package/bin/sdlab.js +71 -0
- package/lib/adapters/ai-toolkit.js +209 -0
- package/lib/adapters/comfyui-runner.js +36 -2
- package/lib/adapters/comfyui-workflows.js +133 -2
- package/lib/adapters/diffusers-lora.js +46 -66
- package/lib/adapters/generic-image-caption.js +22 -50
- package/lib/canon-build/build.js +450 -0
- package/lib/canon-build/emitters/base.js +122 -0
- package/lib/canon-build/emitters/character.js +167 -0
- package/lib/canon-build/emitters/deity.js +165 -0
- package/lib/canon-build/emitters/faction.js +161 -0
- package/lib/canon-build/emitters/index.js +38 -0
- package/lib/canon-build/emitters/location.js +153 -0
- package/lib/canon-build/emitters/monster.js +131 -0
- package/lib/canon-build/emitters/relic.js +142 -0
- package/lib/canon-build/emitters/ship.js +154 -0
- package/lib/canon-build/emitters/species.js +165 -0
- package/lib/canon-build/frontmatter.js +76 -0
- package/lib/canon-build/lane-resolve.js +64 -0
- package/lib/canon-build/load-config.js +117 -0
- package/lib/canon-build/load-entry.js +62 -0
- package/lib/canon-build/load-schema.js +46 -0
- package/lib/captions.js +197 -0
- package/lib/config.js +19 -0
- package/lib/critique-engine.js +46 -1
- package/lib/critique-render.js +45 -0
- package/lib/freeze-events.js +109 -0
- package/lib/freeze-gate.js +208 -0
- package/lib/freeze-stamp.js +161 -0
- package/lib/rows.js +228 -0
- package/lib/run-manifest.js +308 -0
- package/lib/snapshot.js +7 -1
- package/lib/training-adapters.js +5 -0
- package/lib/training-packages.js +24 -18
- package/lib/training-profiles.js +33 -0
- package/lib/verifier.js +67 -0
- package/package.json +12 -3
- package/scripts/_batch_cut.py +71 -0
- package/scripts/_seam_cutout.py +74 -0
- package/scripts/canon-build.js +88 -0
- package/scripts/canon-drift.js +193 -0
- package/scripts/canon-freeze-status.js +85 -0
- package/scripts/canon-freeze.js +211 -0
- package/scripts/canon-unfreeze.js +137 -0
- package/scripts/critique.js +32 -7
- package/scripts/curate.js +33 -0
- package/scripts/generate-controlnet.js +12 -0
- package/scripts/generate-identity.js +20 -4
- package/scripts/generate-ipadapter.js +11 -0
- package/scripts/generate.js +262 -102
- package/scripts/qwen_generate.py +327 -0
- package/scripts/run-generate.js +3 -0
package/lib/training-profiles.js
CHANGED
|
@@ -13,6 +13,13 @@ import { isRegisteredAdapter, listAdapters } from './training-adapters.js';
|
|
|
13
13
|
|
|
14
14
|
const REQUIRED_FIELDS = ['profile_id', 'label', 'target_family', 'asset_type', 'eligible_lanes', 'adapter_targets'];
|
|
15
15
|
|
|
16
|
+
// trigger_override constraints (research ref: two-LoRA stack D3, 2026-04-24).
|
|
17
|
+
// T5 tokenizes on SentencePiece Unigram with `_` as the space sentinel; hyphens
|
|
18
|
+
// fragment unpredictably and uppercase is not canonicalized. Generic suffixes
|
|
19
|
+
// alone collide across projects — a game-slug prefix is the documented defense.
|
|
20
|
+
const TRIGGER_OVERRIDE_RE = /^[a-z0-9_]+$/;
|
|
21
|
+
const TRIGGER_GENERIC_SUFFIX_DENYLIST = ['style', 'character', 'anime', 'realistic'];
|
|
22
|
+
|
|
16
23
|
/**
|
|
17
24
|
* Validate a training profile object.
|
|
18
25
|
* @returns {{ valid: boolean, errors: string[] }}
|
|
@@ -42,6 +49,32 @@ export function validateProfile(profile) {
|
|
|
42
49
|
}
|
|
43
50
|
}
|
|
44
51
|
}
|
|
52
|
+
// trigger_override is optional. When present, enforce SentencePiece-safe
|
|
53
|
+
// format and reject bare generic suffixes that collide across projects.
|
|
54
|
+
if (profile.trigger_override !== undefined && profile.trigger_override !== null) {
|
|
55
|
+
if (typeof profile.trigger_override !== 'string') {
|
|
56
|
+
errors.push('trigger_override must be a string');
|
|
57
|
+
} else if (profile.trigger_override.length === 0) {
|
|
58
|
+
errors.push('trigger_override must be a non-empty string (or omit the field)');
|
|
59
|
+
} else if (!TRIGGER_OVERRIDE_RE.test(profile.trigger_override)) {
|
|
60
|
+
errors.push(`trigger_override "${profile.trigger_override}" must match ^[a-z0-9_]+$ (lowercase, digits, underscores — no hyphens, no uppercase)`);
|
|
61
|
+
} else if (TRIGGER_GENERIC_SUFFIX_DENYLIST.includes(profile.trigger_override)) {
|
|
62
|
+
errors.push(`trigger_override "${profile.trigger_override}" is a generic suffix; prefix with a game slug (e.g. "sf_${profile.trigger_override}")`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
// entity_id_scope is optional (post-research D8). Same SentencePiece-safe
|
|
66
|
+
// format as trigger_override — no hyphens, no uppercase, lowercase slug.
|
|
67
|
+
// Used by per-character LoRA profiles to row-filter canonical dataset.jsonl
|
|
68
|
+
// to a single subject at packaging time.
|
|
69
|
+
if (profile.entity_id_scope !== undefined && profile.entity_id_scope !== null) {
|
|
70
|
+
if (typeof profile.entity_id_scope !== 'string') {
|
|
71
|
+
errors.push('entity_id_scope must be a string');
|
|
72
|
+
} else if (profile.entity_id_scope.length === 0) {
|
|
73
|
+
errors.push('entity_id_scope must be a non-empty string (or omit the field)');
|
|
74
|
+
} else if (!TRIGGER_OVERRIDE_RE.test(profile.entity_id_scope)) {
|
|
75
|
+
errors.push(`entity_id_scope "${profile.entity_id_scope}" must match ^[a-z0-9_]+$ (lowercase, digits, underscores — no hyphens, no uppercase)`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
45
78
|
return { valid: errors.length === 0, errors };
|
|
46
79
|
}
|
|
47
80
|
|
package/lib/verifier.js
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EXTERNAL_VERIFIER support (Kambhampati arXiv:2402.01817; Huang / DeepMind
|
|
3
|
+
* arXiv:2310.01798): a judgment records WHO judged it (`judged_by_model`) and
|
|
4
|
+
* WHAT produced the artifact (`generator_model`), and we WARN when they are the
|
|
5
|
+
* same — the self-verification failure mode a verifier must never fall into.
|
|
6
|
+
*
|
|
7
|
+
* Today the two judges are `human` (curate) and a rule-based engine
|
|
8
|
+
* (critique-engine) — both a different artifact class from the image generator,
|
|
9
|
+
* so the warning does not fire in practice and the standard's spirit is already
|
|
10
|
+
* met. Recording the fields + wiring the warning installs the muscle: the moment
|
|
11
|
+
* an LLM critique mode enters the loop sharing the generator's base, the WARN
|
|
12
|
+
* fires, and the standard's remediation flips it from a warning to a hard gate.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { warn } from './log.js';
|
|
16
|
+
|
|
17
|
+
/** The rule-based critique engine's judge id. Bump vN when the rule set changes. */
|
|
18
|
+
export const RULE_BASED_JUDGE = 'rule-based:sdlab-critique-v1';
|
|
19
|
+
|
|
20
|
+
/** The human curator's judge id (curate.js reviewer is "human:mike"). */
|
|
21
|
+
export const HUMAN_JUDGE = 'human';
|
|
22
|
+
|
|
23
|
+
const MODEL_EXT_RE = /\.(safetensors|ckpt|pt|pth|bin|gguf)$/i;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Derive a stable generator model id from a record `provenance` (or any object
|
|
27
|
+
* carrying {base, unet|checkpoint}): `<base>:<model-basename-without-ext>`, e.g.
|
|
28
|
+
* "qwen-image:qwen_image_fp8_e4m3fn" or "sdxl:dreamshaperXL". Returns null when
|
|
29
|
+
* neither a base nor a model file is known (legacy/hand-migrated records).
|
|
30
|
+
*/
|
|
31
|
+
export function deriveGeneratorModel(provenance) {
|
|
32
|
+
if (!provenance || typeof provenance !== 'object') return null;
|
|
33
|
+
const base = provenance.base || null;
|
|
34
|
+
const modelFile = provenance.unet || provenance.checkpoint || null;
|
|
35
|
+
const stripped = modelFile ? String(modelFile).replace(MODEL_EXT_RE, '') : null;
|
|
36
|
+
if (base && stripped) return `${base}:${stripped}`;
|
|
37
|
+
return base || stripped || null;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Derive the generator model id from a run manifest (comfyui-runner shape).
|
|
42
|
+
* Prefers the pinning block's model identity, falling back to the top-level
|
|
43
|
+
* checkpoint. Qwen manifests carry pinning.models.unet; SDXL carry checkpoint.
|
|
44
|
+
*/
|
|
45
|
+
export function generatorModelFromManifest(manifest) {
|
|
46
|
+
if (!manifest || typeof manifest !== 'object') return null;
|
|
47
|
+
const unet = manifest.pinning?.models?.unet?.name;
|
|
48
|
+
const checkpoint = manifest.checkpoint || manifest.pinning?.models?.checkpoint?.name;
|
|
49
|
+
const base = unet ? 'qwen-image' : (checkpoint ? 'sdxl' : null);
|
|
50
|
+
return deriveGeneratorModel({ base, unet, checkpoint });
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Emit a WARN when the judge and the generator are the same model — the
|
|
55
|
+
* self-verification failure mode. No-op when either is null or they differ.
|
|
56
|
+
* Returns true when the warning fired, so callers/tests can assert on it without
|
|
57
|
+
* capturing stderr.
|
|
58
|
+
*/
|
|
59
|
+
export function warnIfSelfVerification(judgedByModel, generatorModel, context = '') {
|
|
60
|
+
if (!judgedByModel || !generatorModel) return false;
|
|
61
|
+
if (judgedByModel !== generatorModel) return false;
|
|
62
|
+
warn(
|
|
63
|
+
`EXTERNAL_VERIFIER: judged_by_model === generator_model ("${judgedByModel}")${context ? ` for ${context}` : ''} — ` +
|
|
64
|
+
'a model must not verify its own output. Use a different-family judge (ai-eyes-mcp or a cross-family LLM).',
|
|
65
|
+
);
|
|
66
|
+
return true;
|
|
67
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mcptoolshop/style-dataset-lab",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.3.0",
|
|
4
4
|
"description": "Canon-aligned dataset production and generation workbench — define visual rules, build versioned training data, compile production briefs, run local workflows, batch-produce, select winners, and re-ingest into your corpus",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -18,7 +18,9 @@
|
|
|
18
18
|
"generate:identity": "node scripts/generate-identity.js"
|
|
19
19
|
},
|
|
20
20
|
"c8": {
|
|
21
|
-
"include": [
|
|
21
|
+
"include": [
|
|
22
|
+
"lib/**/*.js"
|
|
23
|
+
],
|
|
22
24
|
"exclude": [
|
|
23
25
|
"tests/**",
|
|
24
26
|
"scripts/**",
|
|
@@ -27,7 +29,11 @@
|
|
|
27
29
|
"templates/**",
|
|
28
30
|
"**/*.test.js"
|
|
29
31
|
],
|
|
30
|
-
"reporter": [
|
|
32
|
+
"reporter": [
|
|
33
|
+
"text",
|
|
34
|
+
"lcov",
|
|
35
|
+
"text-summary"
|
|
36
|
+
],
|
|
31
37
|
"report-dir": "coverage",
|
|
32
38
|
"all": true
|
|
33
39
|
},
|
|
@@ -76,5 +82,8 @@
|
|
|
76
82
|
"license": "MIT",
|
|
77
83
|
"devDependencies": {
|
|
78
84
|
"c8": "^10.1.3"
|
|
85
|
+
},
|
|
86
|
+
"dependencies": {
|
|
87
|
+
"yaml": "^2.8.3"
|
|
79
88
|
}
|
|
80
89
|
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Batch alpha-cut cast turnaround views via ComfyUI's native BiRefNet (RGBA out).
|
|
3
|
+
For each character, cuts its 8 restylized views -> out/<char>/<view>.png on transparency.
|
|
4
|
+
Free Qwen (POST /free) before running so BiRefNet doesn't co-reside with the 28 GB transformer.
|
|
5
|
+
|
|
6
|
+
python _batch_cut.py --restylized <dir-of-per-char> --out <dir> --chars rustgrave,hearthframe,dustwhisper
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
import argparse, json, shutil, time, urllib.request, uuid
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
COMFY = "http://127.0.0.1:8188"
|
|
13
|
+
COMFY_OUT = Path(r"E:/AI-Models/ComfyUI_windows_portable/ComfyUI/output")
|
|
14
|
+
COMFY_IN = Path(r"E:/AI-Models/ComfyUI_windows_portable/ComfyUI/input")
|
|
15
|
+
MODEL = "birefnet.safetensors"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def graph(image_filename: str, out_prefix: str) -> dict:
|
|
19
|
+
return {
|
|
20
|
+
"1": {"class_type": "LoadImage", "inputs": {"image": image_filename}},
|
|
21
|
+
"2": {"class_type": "LoadBackgroundRemovalModel", "inputs": {"bg_removal_name": MODEL}},
|
|
22
|
+
"3": {"class_type": "RemoveBackground", "inputs": {"image": ["1", 0], "bg_removal_model": ["2", 0]}},
|
|
23
|
+
"4": {"class_type": "InvertMask", "inputs": {"mask": ["3", 0]}},
|
|
24
|
+
"5": {"class_type": "JoinImageWithAlpha", "inputs": {"image": ["1", 0], "alpha": ["4", 0]}},
|
|
25
|
+
"6": {"class_type": "SaveImage", "inputs": {"filename_prefix": out_prefix, "images": ["5", 0]}},
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def post(path: str, payload: dict):
|
|
30
|
+
return json.loads(urllib.request.urlopen(urllib.request.Request(
|
|
31
|
+
f"{COMFY}{path}", json.dumps(payload).encode(), {"Content-Type": "application/json"}), timeout=120).read())
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def cut_one(src: Path, staged_name: str, dst: Path):
|
|
35
|
+
shutil.copyfile(src, COMFY_IN / f"{staged_name}.png")
|
|
36
|
+
g = graph(f"{staged_name}.png", f"cut_{staged_name}")
|
|
37
|
+
pid = post("/prompt", {"prompt": g, "client_id": uuid.uuid4().hex})["prompt_id"]
|
|
38
|
+
for _ in range(600):
|
|
39
|
+
hist = json.loads(urllib.request.urlopen(f"{COMFY}/history/{pid}", timeout=30).read())
|
|
40
|
+
if pid in hist and hist[pid].get("outputs"):
|
|
41
|
+
img = hist[pid]["outputs"]["6"]["images"][0]
|
|
42
|
+
srcp = COMFY_OUT / img["subfolder"] / img["filename"]
|
|
43
|
+
for _w in range(120):
|
|
44
|
+
if srcp.exists():
|
|
45
|
+
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
46
|
+
shutil.copyfile(srcp, dst)
|
|
47
|
+
return
|
|
48
|
+
time.sleep(0.5)
|
|
49
|
+
time.sleep(0.5)
|
|
50
|
+
raise RuntimeError(f"cut timed out: {src}")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def main():
|
|
54
|
+
ap = argparse.ArgumentParser()
|
|
55
|
+
ap.add_argument("--restylized", required=True)
|
|
56
|
+
ap.add_argument("--out", required=True)
|
|
57
|
+
ap.add_argument("--chars", required=True)
|
|
58
|
+
args = ap.parse_args()
|
|
59
|
+
rdir = Path(args.restylized); odir = Path(args.out)
|
|
60
|
+
n = 0
|
|
61
|
+
for c in args.chars.split(","):
|
|
62
|
+
c = c.strip()
|
|
63
|
+
for v in sorted((rdir / c).glob("*.png")):
|
|
64
|
+
cut_one(v, f"{c}_{v.stem}", odir / c / f"{v.stem}.png")
|
|
65
|
+
n += 1
|
|
66
|
+
print(f" cut {c}/{v.stem}", flush=True)
|
|
67
|
+
print(f"BATCH CUT DONE: {n} sprites -> {odir}", flush=True)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
if __name__ == "__main__":
|
|
71
|
+
main()
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Alpha-cut sprites via ComfyUI's native BiRefNet background removal, for the A1 seam-test.
|
|
3
|
+
|
|
4
|
+
Replicates the shipped "Remove Background (BiRefNet)" subgraph:
|
|
5
|
+
LoadImage -> LoadBackgroundRemovalModel -> RemoveBackground -> InvertMask -> JoinImageWithAlpha -> SaveImage
|
|
6
|
+
Outputs RGBA PNGs (figure on transparency) and copies them to the Godot seam-test sprites dir.
|
|
7
|
+
|
|
8
|
+
Inputs must already be in ComfyUI/input. Run with ComfyUI's embedded python:
|
|
9
|
+
python _seam_cutout.py seam_front seam_pitch30 seam_pitch40
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import json
|
|
14
|
+
import shutil
|
|
15
|
+
import sys
|
|
16
|
+
import time
|
|
17
|
+
import urllib.request
|
|
18
|
+
import uuid
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
COMFY = "http://127.0.0.1:8188"
|
|
22
|
+
COMFY_OUT = Path(r"E:/AI-Models/ComfyUI_windows_portable/ComfyUI/output")
|
|
23
|
+
GODOT_SPRITES = Path(r"E:/AI/hesperia/game/seam_test/sprites")
|
|
24
|
+
MODEL = "birefnet.safetensors"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def graph(image_filename: str, out_prefix: str) -> dict:
|
|
28
|
+
return {
|
|
29
|
+
"1": {"class_type": "LoadImage", "inputs": {"image": image_filename}},
|
|
30
|
+
"2": {"class_type": "LoadBackgroundRemovalModel", "inputs": {"bg_removal_name": MODEL}},
|
|
31
|
+
"3": {"class_type": "RemoveBackground", "inputs": {"image": ["1", 0], "bg_removal_model": ["2", 0]}},
|
|
32
|
+
"4": {"class_type": "InvertMask", "inputs": {"mask": ["3", 0]}},
|
|
33
|
+
"5": {"class_type": "JoinImageWithAlpha", "inputs": {"image": ["1", 0], "alpha": ["4", 0]}},
|
|
34
|
+
"6": {"class_type": "SaveImage", "inputs": {"filename_prefix": out_prefix, "images": ["5", 0]}},
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def post(path: str, payload: dict):
|
|
39
|
+
return json.loads(urllib.request.urlopen(urllib.request.Request(
|
|
40
|
+
f"{COMFY}{path}", json.dumps(payload).encode(), {"Content-Type": "application/json"}), timeout=120).read())
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def run_one(name: str) -> Path:
|
|
44
|
+
g = graph(f"{name}.png", f"seamcut_{name}")
|
|
45
|
+
pid = post("/prompt", {"prompt": g, "client_id": uuid.uuid4().hex})["prompt_id"]
|
|
46
|
+
for _ in range(600):
|
|
47
|
+
hist = json.loads(urllib.request.urlopen(f"{COMFY}/history/{pid}", timeout=30).read())
|
|
48
|
+
if pid in hist and hist[pid].get("outputs"):
|
|
49
|
+
img = hist[pid]["outputs"]["6"]["images"][0]
|
|
50
|
+
src = COMFY_OUT / img["subfolder"] / img["filename"]
|
|
51
|
+
for _w in range(120):
|
|
52
|
+
if src.exists():
|
|
53
|
+
GODOT_SPRITES.mkdir(parents=True, exist_ok=True)
|
|
54
|
+
dst = GODOT_SPRITES / f"{name}.png"
|
|
55
|
+
shutil.copyfile(src, dst)
|
|
56
|
+
return dst
|
|
57
|
+
time.sleep(0.5)
|
|
58
|
+
time.sleep(0.5)
|
|
59
|
+
raise RuntimeError(f"cutout timed out for {name}")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def main() -> None:
|
|
63
|
+
names = sys.argv[1:]
|
|
64
|
+
if not names:
|
|
65
|
+
print("usage: _seam_cutout.py <name> [<name> ...] (names are ComfyUI/input stems, no .png)")
|
|
66
|
+
sys.exit(2)
|
|
67
|
+
for n in names:
|
|
68
|
+
dst = run_one(n)
|
|
69
|
+
print(f" cut {n} -> {dst}", flush=True)
|
|
70
|
+
print(f"\nSEAMCUT DONE: {len(names)} sprite(s) -> {GODOT_SPRITES}")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
if __name__ == "__main__":
|
|
74
|
+
main()
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `sdlab canon build` — CLI entry point for the three-projection build.
|
|
3
|
+
*
|
|
4
|
+
* Delegates orchestration to lib/canon-build/build.js. Parses flags, prints
|
|
5
|
+
* a summary, exits with codes per the errors.js inputError/runtimeError
|
|
6
|
+
* model (1 = user error, 2 = runtime error).
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { parseArgs } from 'node:util';
|
|
10
|
+
import { join } from 'node:path';
|
|
11
|
+
import { existsSync } from 'node:fs';
|
|
12
|
+
import { runBuild } from '../lib/canon-build/build.js';
|
|
13
|
+
import { inputError } from '../lib/errors.js';
|
|
14
|
+
|
|
15
|
+
export async function run(argv) {
|
|
16
|
+
const parsed = parseArgs({
|
|
17
|
+
args: argv,
|
|
18
|
+
options: {
|
|
19
|
+
project: { type: 'string' },
|
|
20
|
+
game: { type: 'string' }, // deprecated alias
|
|
21
|
+
full: { type: 'boolean', default: false },
|
|
22
|
+
'no-cache': { type: 'boolean', default: false },
|
|
23
|
+
'dry-run': { type: 'boolean', default: false },
|
|
24
|
+
only: { type: 'string' },
|
|
25
|
+
json: { type: 'boolean', default: false },
|
|
26
|
+
quiet: { type: 'boolean', default: false },
|
|
27
|
+
},
|
|
28
|
+
allowPositionals: true,
|
|
29
|
+
strict: false,
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
const projectName = parsed.values.project || parsed.values.game;
|
|
33
|
+
if (!projectName) {
|
|
34
|
+
throw inputError(
|
|
35
|
+
'INPUT_MISSING_FLAG',
|
|
36
|
+
'sdlab canon build requires --project <name>',
|
|
37
|
+
'Example: sdlab canon build --project greek-rpg',
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const projectRoot = join(process.cwd(), 'projects', projectName);
|
|
42
|
+
if (!existsSync(projectRoot)) {
|
|
43
|
+
throw inputError(
|
|
44
|
+
'INPUT_PROJECT_NOT_FOUND',
|
|
45
|
+
`Project directory not found: ${projectRoot}`,
|
|
46
|
+
`Run from the style-dataset-lab repo root, or scaffold the project with "sdlab init ${projectName}".`,
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const only = parsed.values.only
|
|
51
|
+
? parsed.values.only.split(',').map((s) => s.trim()).filter(Boolean)
|
|
52
|
+
: null;
|
|
53
|
+
|
|
54
|
+
const result = await runBuild({
|
|
55
|
+
projectRoot,
|
|
56
|
+
full: parsed.values.full,
|
|
57
|
+
noCache: parsed.values['no-cache'],
|
|
58
|
+
dryRun: parsed.values['dry-run'],
|
|
59
|
+
only,
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
if (parsed.values.json) {
|
|
63
|
+
console.log(JSON.stringify(result, null, 2));
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (!parsed.values.quiet) {
|
|
68
|
+
if (result.dryRun) {
|
|
69
|
+
console.log(`[dry-run] canon build would process ${result.entities_total} entities (${result.entities_cached} cached, ${result.entities_rebuilt} would rebuild) → ${result.rows} dataset rows.`);
|
|
70
|
+
console.log(`[dry-run] output would land at: ${result.output_dir}`);
|
|
71
|
+
} else {
|
|
72
|
+
console.log(`canon build complete: ${result.entities_total} entities (${result.entities_cached} cached, ${result.entities_rebuilt} rebuilt), ${result.rows} dataset rows.`);
|
|
73
|
+
console.log(`output: ${result.output_dir}`);
|
|
74
|
+
console.log(`generated_from: ${result.generated_from}`);
|
|
75
|
+
if (result.frozen_entries > 0) {
|
|
76
|
+
console.log(`frozen entries: ${result.frozen_entries} stamped with witness hashes`);
|
|
77
|
+
}
|
|
78
|
+
if (Array.isArray(result.drift) && result.drift.length > 0) {
|
|
79
|
+
console.log('');
|
|
80
|
+
console.log(`drift detected on ${result.drift.length} frozen entr${result.drift.length === 1 ? 'y' : 'ies'} since the previous build:`);
|
|
81
|
+
for (const d of result.drift) {
|
|
82
|
+
console.log(` ${d.schema_kind}:${d.entity_id} (${d.status})`);
|
|
83
|
+
}
|
|
84
|
+
console.log('Run `sdlab canon drift` for the full report.');
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `sdlab canon drift` — drift report (D7 command).
|
|
3
|
+
*
|
|
4
|
+
* For every frozen or on-canon-change entry, computes the current watch-hash
|
|
5
|
+
* and compares against the hash stamped in the latest canon-build manifest.
|
|
6
|
+
* Surfaces:
|
|
7
|
+
* - Frozen entries whose current projection drifts from their witness.
|
|
8
|
+
* - on-canon-change entries where the watch-fields have changed since freeze.
|
|
9
|
+
* - Overrides since a given build hash (or since the last clean build).
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { parseArgs } from 'node:util';
|
|
13
|
+
import { readFile, readdir } from 'node:fs/promises';
|
|
14
|
+
import { join } from 'node:path';
|
|
15
|
+
import { existsSync } from 'node:fs';
|
|
16
|
+
import { loadBuildConfig } from '../lib/canon-build/load-config.js';
|
|
17
|
+
import { loadCanonEntriesInDir } from '../lib/canon-build/load-entry.js';
|
|
18
|
+
import { loadSchema } from '../lib/canon-build/load-schema.js';
|
|
19
|
+
import { readFreezeStatus, resolveWatchFields, computeWatchHash } from '../lib/freeze-stamp.js';
|
|
20
|
+
import { readEventsSince } from '../lib/freeze-events.js';
|
|
21
|
+
import { inputError } from '../lib/errors.js';
|
|
22
|
+
|
|
23
|
+
export async function run(argv) {
|
|
24
|
+
const parsed = parseArgs({
|
|
25
|
+
args: argv,
|
|
26
|
+
options: {
|
|
27
|
+
project: { type: 'string' },
|
|
28
|
+
game: { type: 'string' },
|
|
29
|
+
since: { type: 'string' },
|
|
30
|
+
json: { type: 'boolean', default: false },
|
|
31
|
+
},
|
|
32
|
+
allowPositionals: true,
|
|
33
|
+
strict: false,
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
const projectName = parsed.values.project || parsed.values.game;
|
|
37
|
+
if (!projectName) throw inputError('INPUT_MISSING_FLAG', '--project required');
|
|
38
|
+
|
|
39
|
+
const projectRoot = join(process.cwd(), 'projects', projectName);
|
|
40
|
+
const config = await loadBuildConfig(join(projectRoot, 'canon-build', 'config.json'), projectRoot);
|
|
41
|
+
|
|
42
|
+
// Load latest build manifest for the stamped watch-hashes
|
|
43
|
+
const latestManifest = await findLatestBuildManifest(projectRoot);
|
|
44
|
+
|
|
45
|
+
// Load schemas for version info
|
|
46
|
+
const schemas = {};
|
|
47
|
+
for (const schemaName of Object.keys(config.entity_dirs)) {
|
|
48
|
+
const path = join(config.schema_dir, schemaName);
|
|
49
|
+
schemas[schemaName] = await loadSchema(path);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// Walk entries, compute drift
|
|
53
|
+
const drifted = [];
|
|
54
|
+
const cleanFrozen = [];
|
|
55
|
+
let totalEntries = 0;
|
|
56
|
+
let autoCount = 0;
|
|
57
|
+
|
|
58
|
+
for (const [schemaName, subdir] of Object.entries(config.entity_dirs)) {
|
|
59
|
+
const dir = join(config.canon_root, subdir);
|
|
60
|
+
const entries = await loadCanonEntriesInDir(dir);
|
|
61
|
+
for (const entry of entries) {
|
|
62
|
+
totalEntries++;
|
|
63
|
+
const status = readFreezeStatus(entry.frontmatter);
|
|
64
|
+
if (status === 'auto') { autoCount++; continue; }
|
|
65
|
+
|
|
66
|
+
const watchFields = resolveWatchFields(entry.frontmatter, schemaName, config);
|
|
67
|
+
const currentHash = computeWatchHash(
|
|
68
|
+
entry.frontmatter,
|
|
69
|
+
watchFields,
|
|
70
|
+
schemas[schemaName].version,
|
|
71
|
+
);
|
|
72
|
+
|
|
73
|
+
// Look up stamped hash from latest manifest. The manifest stores a
|
|
74
|
+
// richer stamp object (status / watch_hash / watch_fields /
|
|
75
|
+
// locked_at_build); the drift comparison only needs `.watch_hash`.
|
|
76
|
+
const entityId = entry.frontmatter.id;
|
|
77
|
+
const stamp = latestManifest?.frozen_entries_hashes?.[entityId] || null;
|
|
78
|
+
const stampedHash = stamp?.watch_hash || null;
|
|
79
|
+
|
|
80
|
+
if (stampedHash && stampedHash !== currentHash) {
|
|
81
|
+
drifted.push({
|
|
82
|
+
entity_id: entityId,
|
|
83
|
+
schema_kind: schemaName.replace('.schema.json', ''),
|
|
84
|
+
status,
|
|
85
|
+
locked_at_build: stamp?.locked_at_build || entry.frontmatter.freeze?.locked_at_build || null,
|
|
86
|
+
witness_hash: stampedHash,
|
|
87
|
+
current_hash: currentHash,
|
|
88
|
+
watch_fields: watchFields,
|
|
89
|
+
});
|
|
90
|
+
} else if (stampedHash) {
|
|
91
|
+
cleanFrozen.push({
|
|
92
|
+
entity_id: entityId,
|
|
93
|
+
status,
|
|
94
|
+
witness_hash: stampedHash,
|
|
95
|
+
});
|
|
96
|
+
} else {
|
|
97
|
+
// Entry is frozen now but was not in the last build manifest (e.g.
|
|
98
|
+
// newly frozen since). Flag as "no witness yet" rather than mis-
|
|
99
|
+
// classifying as either drifted or clean.
|
|
100
|
+
cleanFrozen.push({
|
|
101
|
+
entity_id: entityId,
|
|
102
|
+
status,
|
|
103
|
+
witness_hash: null,
|
|
104
|
+
note: 'no witness in latest build — rebuild to stamp',
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Events since a reference build hash
|
|
111
|
+
const sinceHash = parsed.values.since || latestManifest?.generated_from || null;
|
|
112
|
+
const eventsSince = sinceHash
|
|
113
|
+
? await readEventsSince(projectRoot, sinceHash)
|
|
114
|
+
: [];
|
|
115
|
+
|
|
116
|
+
const report = {
|
|
117
|
+
project_id: config.project_id,
|
|
118
|
+
latest_build: latestManifest?.generated_from || null,
|
|
119
|
+
latest_built_at: latestManifest?.built_at || null,
|
|
120
|
+
entities_total: totalEntries,
|
|
121
|
+
entities_auto: autoCount,
|
|
122
|
+
entities_frozen_clean: cleanFrozen.length,
|
|
123
|
+
entities_drifted: drifted.length,
|
|
124
|
+
drifted,
|
|
125
|
+
overrides_since: eventsSince.filter((e) => e.type !== 'freeze'),
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
if (parsed.values.json) {
|
|
129
|
+
console.log(JSON.stringify(report, null, 2));
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
renderHumanReport(report);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function renderHumanReport(r) {
|
|
137
|
+
console.log(`Drift report for ${r.project_id}`);
|
|
138
|
+
console.log(` latest build: ${r.latest_build || '(no build output found)'}`);
|
|
139
|
+
console.log(` latest built_at: ${r.latest_built_at || '(unknown)'}`);
|
|
140
|
+
console.log('');
|
|
141
|
+
console.log(` auto entries: ${r.entities_auto}`);
|
|
142
|
+
console.log(` frozen entries (clean): ${r.entities_frozen_clean}`);
|
|
143
|
+
console.log(` frozen entries (drift): ${r.entities_drifted}`);
|
|
144
|
+
console.log('');
|
|
145
|
+
|
|
146
|
+
if (r.drifted.length > 0) {
|
|
147
|
+
console.log('Drifted frozen entries:');
|
|
148
|
+
for (const d of r.drifted) {
|
|
149
|
+
console.log(` ${d.schema_kind}:${d.entity_id} (${d.status})`);
|
|
150
|
+
console.log(` witness hash: ${d.witness_hash.slice(0, 16)}...`);
|
|
151
|
+
console.log(` current hash: ${d.current_hash.slice(0, 16)}...`);
|
|
152
|
+
console.log(` watch fields: ${d.watch_fields.join(', ') || '(none)'}`);
|
|
153
|
+
}
|
|
154
|
+
console.log('');
|
|
155
|
+
} else if (r.entities_frozen_clean > 0) {
|
|
156
|
+
console.log('No drift detected on frozen entries.');
|
|
157
|
+
console.log('');
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
if (r.overrides_since.length > 0) {
|
|
161
|
+
console.log(`Overrides since last build (${r.overrides_since.length}):`);
|
|
162
|
+
for (const e of r.overrides_since) {
|
|
163
|
+
console.log(` ${e.type.padEnd(10)} ${e.entity_id} by ${e.by} — ${e.reason}`);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
async function findLatestBuildManifest(projectRoot) {
|
|
169
|
+
const canonBuildDir = join(projectRoot, 'canon-build');
|
|
170
|
+
if (!existsSync(canonBuildDir)) return null;
|
|
171
|
+
const entries = await readdir(canonBuildDir, { withFileTypes: true });
|
|
172
|
+
const candidates = entries
|
|
173
|
+
.filter((e) => e.isDirectory() && !e.name.startsWith('.') && !e.name.includes('.tmp'))
|
|
174
|
+
.map((e) => e.name);
|
|
175
|
+
|
|
176
|
+
let latest = null;
|
|
177
|
+
let latestTime = 0;
|
|
178
|
+
for (const name of candidates) {
|
|
179
|
+
const manifestPath = join(canonBuildDir, name, 'manifest.json');
|
|
180
|
+
if (!existsSync(manifestPath)) continue;
|
|
181
|
+
try {
|
|
182
|
+
const manifest = JSON.parse(await readFile(manifestPath, 'utf-8'));
|
|
183
|
+
const built = new Date(manifest.built_at || 0).getTime();
|
|
184
|
+
if (built > latestTime) {
|
|
185
|
+
latestTime = built;
|
|
186
|
+
latest = manifest;
|
|
187
|
+
}
|
|
188
|
+
} catch {
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
return latest;
|
|
193
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `sdlab canon freeze-status <entity_id>` — read-only glance at an entry's
|
|
3
|
+
* freeze state. Human-readable by default; --json for machine consumption.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { parseArgs } from 'node:util';
|
|
7
|
+
import { join } from 'node:path';
|
|
8
|
+
import { loadBuildConfig } from '../lib/canon-build/load-config.js';
|
|
9
|
+
import { loadCanonEntriesInDir } from '../lib/canon-build/load-entry.js';
|
|
10
|
+
import { readFreezeStatus, resolveWatchFields } from '../lib/freeze-stamp.js';
|
|
11
|
+
import { readEventsFor } from '../lib/freeze-events.js';
|
|
12
|
+
import { inputError } from '../lib/errors.js';
|
|
13
|
+
|
|
14
|
+
export async function run(argv) {
|
|
15
|
+
const parsed = parseArgs({
|
|
16
|
+
args: argv,
|
|
17
|
+
options: {
|
|
18
|
+
project: { type: 'string' },
|
|
19
|
+
game: { type: 'string' },
|
|
20
|
+
json: { type: 'boolean', default: false },
|
|
21
|
+
},
|
|
22
|
+
allowPositionals: true,
|
|
23
|
+
strict: false,
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
const entityIdArg = (parsed.positionals || [])[0];
|
|
27
|
+
if (!entityIdArg) throw inputError('INPUT_MISSING_ARG', 'sdlab canon freeze-status requires an entity id.');
|
|
28
|
+
const projectName = parsed.values.project || parsed.values.game;
|
|
29
|
+
if (!projectName) throw inputError('INPUT_MISSING_FLAG', '--project required');
|
|
30
|
+
|
|
31
|
+
const projectRoot = join(process.cwd(), 'projects', projectName);
|
|
32
|
+
const config = await loadBuildConfig(join(projectRoot, 'canon-build', 'config.json'), projectRoot);
|
|
33
|
+
|
|
34
|
+
let resolved = null;
|
|
35
|
+
for (const [schemaName, subdir] of Object.entries(config.entity_dirs)) {
|
|
36
|
+
const dir = join(config.canon_root, subdir);
|
|
37
|
+
const entries = await loadCanonEntriesInDir(dir);
|
|
38
|
+
const hit = entries.find((e) => e.frontmatter?.id === entityIdArg);
|
|
39
|
+
if (hit) { resolved = { entry: hit, schemaName }; break; }
|
|
40
|
+
}
|
|
41
|
+
if (!resolved) throw inputError('CANON_ENTRY_NOT_FOUND', `No canon entry "${entityIdArg}".`);
|
|
42
|
+
|
|
43
|
+
const fm = resolved.entry.frontmatter;
|
|
44
|
+
const status = readFreezeStatus(fm);
|
|
45
|
+
const block = fm.freeze || {};
|
|
46
|
+
const events = await readEventsFor(projectRoot, entityIdArg);
|
|
47
|
+
|
|
48
|
+
// Entry override OR config default — the same resolution used by the
|
|
49
|
+
// build, so a reader sees the actual watched fields, not just what's
|
|
50
|
+
// literally in the entry frontmatter.
|
|
51
|
+
const resolvedWatchFields = resolveWatchFields(fm, resolved.schemaName, config);
|
|
52
|
+
const watchFieldsSource = Array.isArray(block.watch_fields) && block.watch_fields.length
|
|
53
|
+
? 'entry-override'
|
|
54
|
+
: (resolvedWatchFields.length ? 'config-default' : 'none');
|
|
55
|
+
|
|
56
|
+
const result = {
|
|
57
|
+
entity_id: entityIdArg,
|
|
58
|
+
schema_kind: resolved.schemaName.replace('.schema.json', ''),
|
|
59
|
+
status,
|
|
60
|
+
locked_at_build: block.locked_at_build || null,
|
|
61
|
+
frozen_by: block.frozen_by || null,
|
|
62
|
+
frozen_reason: block.frozen_reason || null,
|
|
63
|
+
watch_fields: resolvedWatchFields,
|
|
64
|
+
watch_fields_source: watchFieldsSource,
|
|
65
|
+
overrides_count: Array.isArray(block.overrides) ? block.overrides.length : 0,
|
|
66
|
+
event_count: events.length,
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
if (parsed.values.json) {
|
|
70
|
+
console.log(JSON.stringify(result, null, 2));
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
console.log(`${entityIdArg} (${result.schema_kind})`);
|
|
75
|
+
console.log(` status: ${status}`);
|
|
76
|
+
if (status !== 'auto') {
|
|
77
|
+
console.log(` locked_at_build: ${result.locked_at_build || '(unset)'}`);
|
|
78
|
+
console.log(` frozen_by: ${result.frozen_by || '(unset)'}`);
|
|
79
|
+
console.log(` frozen_reason: ${result.frozen_reason || '(unset)'}`);
|
|
80
|
+
const fieldsDisplay = result.watch_fields.length ? result.watch_fields.join(', ') : '(none)';
|
|
81
|
+
console.log(` watch_fields: ${fieldsDisplay} [${result.watch_fields_source}]`);
|
|
82
|
+
}
|
|
83
|
+
console.log(` overrides[]: ${result.overrides_count} recorded on entry`);
|
|
84
|
+
console.log(` event log: ${result.event_count} events total`);
|
|
85
|
+
}
|