@mcptoolshop/style-dataset-lab 3.2.0 → 3.4.0

package/scripts/ingest.js

@@ -0,0 +1,145 @@
+#!/usr/bin/env node
+
+/**
+ * ingest.js — Bring a directory of pre-existing images into the pipeline
+ * as bare, uncurated candidate records.
+ *
+ * Closes the gap where generation happened OUTSIDE sdlab entirely (a hand
+ * batch, external Python against Comfy Cloud, salvage from somewhere else)
+ * and there was no CLI path to get those images into records/ for review.
+ * Unlike `sdlab reingest generated` (which requires a trained LoRA's
+ * training manifest) or `sdlab reingest selected` (which requires a
+ * selection id only the production loop can mint), this takes any folder
+ * of images and any project — no pipeline history required.
+ *
+ * Usage:
+ *   sdlab ingest <dir> --project <name>
+ *   sdlab ingest <dir> --project <name> --wave inputs/prompts/wave-v2.json
+ *   sdlab ingest <dir> --project <name> --dry-run
+ *   sdlab ingest <dir> --project <name> --json
+ */
+
+import { parseArgs, getProjectName } from '../lib/args.js';
+import { REPO_ROOT, getProjectRoot, resolveSafeProjectPath } from '../lib/paths.js';
+import { inputError, handleCliError } from '../lib/errors.js';
+import { ingestDirectory } from '../lib/ingest.js';
+import { result } from '../lib/log.js';
+
+const USAGE = `Usage: sdlab ingest <dir> --project <name> [--wave <wave.json>] [--dry-run] [--json]
+
+Scan <dir> for images and create bare candidate records (judgment: null,
+canon: null, provenance.source: "external") ready for "sdlab curate".
+
+Positional:
+  dir                  Directory of images to ingest (png/jpg/jpeg/webp)
+
+Flags:
+  --project <name>     Project to operate on
+  --wave <path>        Optional wave JSON (inputs/prompts/*.json shape) to
+                        attach real generation provenance (seed, prompt, ...)
+                        by matching each image's filename stem to an item id.
+                        A sibling receipts.jsonl next to <dir> is picked up
+                        automatically when present (prompt_id, seconds, status).
+  --dry-run            Preview what would be created — no files written
+  --json               Emit the result as JSON on stdout
+
+Next steps after ingesting:
+  1. Review: sdlab curate <id> approved/rejected "explanation"
+  2. Bind:   sdlab bind --project <name>`;
+
+export async function run(argv = process.argv.slice(2)) {
+  if (argv.includes('--help') || argv.includes('-h')) {
+    console.log(USAGE);
+    return;
+  }
+
+  const { flags, positionals } = parseArgs(argv, {
+    flags: {
+      project: { type: 'string' },
+      wave: { type: 'string' },
+      'dry-run': { type: 'boolean', default: false },
+      json: { type: 'boolean', default: false },
+    },
+    deprecated: { game: 'project' },
+  });
+
+  const dirArg = positionals[0];
+  if (!dirArg) {
+    throw inputError(
+      'INPUT_MISSING_ARGS',
+      'A source directory is required.',
+      USAGE
+    );
+  }
+
+  const projectName = flags.project || getProjectName(argv);
+  const projectRoot = getProjectRoot(projectName);
+  const dryRun = flags['dry-run'] || flags.dryRun;
+
+  // Mirrors scripts/reingest.js's --source handling: the source directory
+  // of pre-existing images is allowed to live anywhere under the workspace
+  // root, not just inside this specific project (e.g. a shared staging
+  // inbox), while still refusing to escape the workspace entirely.
+  const safeSourceDir = resolveSafeProjectPath(projectRoot, dirArg, { baseRoot: REPO_ROOT, flagName: 'dir' });
+
+  // Mirrors scripts/generate.js's pack-path handling: a --wave file is
+  // expected to live inside THIS project (inputs/prompts/*.json), so its
+  // containment root defaults to the project root, not the whole workspace.
+  const safeWavePath = flags.wave
+    ? resolveSafeProjectPath(projectRoot, flags.wave, { flagName: 'wave' })
+    : null;
+
+  if (!flags.json) {
+    console.log(`\x1b[1msdlab ingest\x1b[0m — ${projectName}`);
+    console.log(`  Source: ${dirArg}`);
+    if (flags.wave) console.log(`  Wave:   ${flags.wave}`);
+    if (dryRun) console.log(`  Mode:   DRY RUN (no files written)`);
+    console.log('');
+  }
+
+  const { created, skipped, rejected, receiptsFile } = await ingestDirectory(projectRoot, safeSourceDir, {
+    wavePath: safeWavePath,
+    dryRun,
+  });
+
+  if (flags.json) {
+    console.log(JSON.stringify({ created, skipped, rejected, receiptsFile, dryRun }, null, 2));
+    return;
+  }
+
+  if (receiptsFile) {
+    console.log(`  \x1b[2mreceipts: ${receiptsFile}\x1b[0m`);
+  }
+
+  console.log(`  \x1b[32m✓\x1b[0m Created: ${created.length} record(s)${dryRun ? ' (dry run — nothing written)' : ''}`);
+  if (created.length > 0 && created.length <= 25) {
+    for (const id of created) console.log(`    ✓ ${id}`);
+  }
+
+  if (skipped.length > 0) {
+    console.log(`  Skipped: ${skipped.length} (record already exists)`);
+    if (skipped.length <= 25) {
+      for (const id of skipped) console.log(`    ○ ${id}`);
+    }
+  }
+
+  if (rejected.length > 0) {
+    console.log(`  \x1b[33m⚠\x1b[0m Rejected: ${rejected.length} (unsafe id derived from filename — rename and re-run)`);
+    for (const r of rejected) console.log(`    ✗ ${r.file}: ${r.reason}`);
+  }
+
+  if (!dryRun && created.length > 0) {
+    console.log('');
+    result(`Candidates: outputs/candidates/`);
+    result(`Records: records/`);
+    console.log('');
+    console.log('  Next steps:');
+    console.log('    1. Review: sdlab curate <id> approved/rejected "explanation"');
+    console.log('    2. Bind:   sdlab bind --project ' + projectName);
+  }
+}
+
+// Direct execution guard
+if (process.argv[1] && (process.argv[1].endsWith('ingest.js') || process.argv[1].endsWith('ingest'))) {
+  run().catch(handleCliError);
+}

package/scripts/init.js

@@ -12,10 +12,16 @@
 import { mkdir, readdir, readFile, writeFile, copyFile } from 'node:fs/promises';
 import { join, basename } from 'node:path';
 import { existsSync } from 'node:fs';
-import { REPO_ROOT } from '../lib/paths.js';
+import { REPO_ROOT, getPackageRoot } from '../lib/paths.js';
 import { inputError, handleCliError } from '../lib/errors.js';
+import { takeFlagValue } from '../lib/args.js';
 
-const TEMPLATES_DIR = join(REPO_ROOT, 'templates');
+// Two DIFFERENT roots — see getPackageRoot() in lib/paths.js.
+// `templates/` ships inside the npm tarball, so it is package-relative;
+// `projects/` is the user's own work, so it is workspace-relative. In a repo
+// checkout these coincide, which is why resolving both against REPO_ROOT
+// passed every in-repo test while breaking `init` for every installed user.
+const TEMPLATES_DIR = join(getPackageRoot(), 'templates');
 const PROJECTS_DIR = join(REPO_ROOT, 'projects');
 
 const PROJECT_DIRS = [
@@ -64,11 +70,15 @@ async function listDomains() {
 export async function run(argv = process.argv.slice(2)) {
   const projectName = argv.find(a => !a.startsWith('--'));
 
-  // Parse --domain
+  // Parse --domain. H4: previously a hand-rolled `argv.indexOf('--domain')`
+  // that only recognized the space-separated form — `sdlab init foo
+  // --domain=game-art` silently fell through to the 'generic' default
+  // because no argv token was EXACTLY "--domain". takeFlagValue() (already
+  // hardened for both `--flag value` and `--flag=value`) closes that gap.
   let domain = 'generic';
-  const domainIdx = argv.indexOf('--domain');
-  if (domainIdx >= 0 && argv[domainIdx + 1]) {
-    domain = argv[domainIdx + 1];
+  const domainValue = takeFlagValue(argv, 'domain');
+  if (domainValue !== undefined) {
+    domain = domainValue;
   }
 
   if (!projectName) {
@@ -133,6 +143,34 @@ export async function run(argv = process.argv.slice(2)) {
   }
   console.log(`  \x1b[32m✓\x1b[0m Created ${PROJECT_DIRS.length} directories`);
 
+  // SDL-M6: the generate → curate → reingest loop fills inbox/generated/ and
+  // the outputs/ subtrees with PNGs (see lib/reingest-selected.js), but this
+  // repo's OWN .gitignore (which happens to cover those paths in dev) is not
+  // in the npm `files` allow-list and ships to nobody. Without a per-project
+  // .gitignore, every downstream `sdlab init` user stages regenerable blobs
+  // by construction — measured at 55 MB of untracked PNGs in this repo before
+  // this fix. Never clobber a .gitignore the user already created by hand.
+  const gitignorePath = join(projectDir, '.gitignore');
+  if (!existsSync(gitignorePath)) {
+    const gitignoreLines = [
+      '# Generated by `sdlab init` — regenerable blobs for this project.',
+      '# Records/config JSON are tracked; only pixels and derived artifacts are not.',
+      '',
+      'inbox/**/*.png',
+      'inbox/**/*.jpg',
+      'inbox/**/*.jpeg',
+      'inbox/**/*.webp',
+      'outputs/**',
+      'exports/',
+      'snapshots/',
+      'splits/',
+      'canon-build/*/',
+      '',
+    ];
+    await writeFile(gitignorePath, gitignoreLines.join('\n'));
+    console.log(`  \x1b[32m✓\x1b[0m .gitignore (regenerable outputs/inbox/exports/snapshots/splits)`);
+  }
+
   // Copy config files from domain template or generate generic ones
   const configFiles = ['project.json', 'constitution.json', 'lanes.json', 'rubric.json', 'terminology.json'];
 
@@ -331,21 +369,43 @@ export async function run(argv = process.argv.slice(2)) {
   console.log('');
   console.log(`\x1b[32m✓\x1b[0m Project "${projectName}" initialized`);
   console.log('');
+  // Two DIFFERENT path bases are printed in this block, for two DIFFERENT
+  // audiences — do not collapse them back into one helper:
+  //   - "Edit <file>" / "Full quick-start" lines are opened by the user in
+  //     an editor/file-browser from wherever they ran `init` (the workspace
+  //     root), so they need workspace-relative paths: projects/<name>/....
+  //   - The `sdlab generate <pack>` line's argument is consumed by
+  //     generate.js via resolveSafeProjectPath(GAME_ROOT, packPath) — GAME_ROOT
+  //     IS projects/<name>, so that path must be PROJECT-relative
+  //     (inputs/prompts/...), not workspace-relative. Printing the
+  //     workspace-relative form here doubles the path
+  //     (projects/<name>/projects/<name>/...) and generate.js 404s with
+  //     INPUT_FILE_NOT_FOUND. Regression introduced in 27c42c6 while fixing
+  //     the sibling "Edit" lines (which legitimately needed the workspace-
+  //     relative switch) — see tests/cli-scripts/init-generate-path.test.js.
+  const rel = (...segments) => ['projects', projectName, ...segments].join('/');
+  const projectRel = (...segments) => segments.join('/');
+
   console.log('Next steps:');
   let step = 1;
   if (wroteCanon['constitution.md']) {
-    console.log(`  ${step++}. Edit ${projectName}/canon/constitution.md — define your visual rules`);
+    console.log(`  ${step++}. Edit ${rel('canon', 'constitution.md')}`);
+    console.log(`     Describe your visual rules in prose — this is the canon the pipeline binds to.`);
   }
-  console.log(`  ${step++}. Edit ${projectName}/constitution.json — encode rules for canon-bind`);
+  console.log(`  ${step++}. Edit ${rel('constitution.json')}`);
+  console.log(`     Encode those rules as machine-checkable entries for canon-bind.`);
   if (wroteExamplePack) {
-    console.log(`  ${step++}. Run: sdlab generate inputs/prompts/example-wave.json --project ${projectName}`);
+    console.log(`  ${step++}. Generate your first candidates:`);
+    console.log(`     sdlab generate ${projectRel('inputs', 'prompts', 'example-wave.json')} --project ${projectName}`);
   } else {
-    console.log(`  ${step++}. Create prompt packs under ${projectName}/inputs/prompts/`);
-    console.log(`  ${step++}. Run: sdlab generate <pack> --project ${projectName}`);
+    console.log(`  ${step++}. Add a prompt pack under ${rel('inputs', 'prompts')}/`);
+    console.log(`  ${step++}. Generate your first candidates:`);
+    console.log(`     sdlab generate <pack> --project ${projectName}`);
   }
-  console.log(`  ${step++}. Run: sdlab project doctor --project ${projectName}`);
+  console.log(`  ${step++}. Check the project is wired correctly:`);
+  console.log(`     sdlab project doctor --project ${projectName}`);
   console.log('');
-  console.log(`See ${projectName}/README.md for the full quick-start.`);
+  console.log(`Full quick-start: ${rel('README.md')}`);
 }
 
 // Direct execution guard

package/scripts/measure.js

@@ -0,0 +1,170 @@
+#!/usr/bin/env node
+
+/**
+ * measure.js — Attach deterministic pixel measurements to candidate records.
+ *
+ * Until now nothing in this tool ever opened an image. `lib/critique-engine.js`
+ * builds its entire drift report from brief and workflow TEXT — keyword
+ * matching against prompt strings — and stamps every candidate
+ * `overall_fit: 'usable'` without looking at a single pixel. Every question
+ * about what an image actually looks like was answered by eyeballing it.
+ *
+ * The instruments here are a port, not an invention. They come from the
+ * measurement scripts written for the salt-road project
+ * (`projects/salt-road/inputs/prompts/measure-palette.py`, `measure-texture.py`),
+ * which were built and validated against real production waves and carry
+ * their own written rationale for why simpler proxies were rejected.
+ *
+ * MEASUREMENT, NOT VERDICT. This command attaches numbers. It never sets
+ * `judgment`, never sets `overall_fit`, never approves or rejects. That line
+ * is deliberate and load-bearing: this repo deleted three scripts in its
+ * Stage A health pass for minting judgments nobody made, and 948 of
+ * star-freight's 1,182 records still carry that damage. Numbers inform a
+ * human; they do not replace one.
+ *
+ * Usage:
+ *   sdlab measure <dir-or-record-glob> --project <name>
+ *   sdlab measure outputs/approved --project salt-road
+ *   sdlab measure "styleset_p*" --project salt-road --anchors canon/anchors.json
+ *   sdlab measure outputs/candidates --project salt-road --dry-run
+ */
+
+import { parseArgs, getProjectName } from '../lib/args.js';
+import { getProjectRoot } from '../lib/paths.js';
+import { handleCliError } from '../lib/errors.js';
+import { warn, result } from '../lib/log.js';
+import {
+  assertMeasurementPrereqs,
+  resolveMeasureTargets,
+  loadAnchors,
+  runPythonMeasurement,
+  applyMeasurements,
+} from '../lib/measure.js';
+
+const USAGE = `Usage: sdlab measure <dir-or-record-glob> --project <name> [--anchors <path>] [--dry-run] [--json]
+
+Measure candidate images (palette conformance, texture character) and attach
+the numbers to their records. Measurement only — never sets a judgment.
+
+Positional:
+  dir-or-record-glob   A directory of images, or a record-id glob
+                       (quote globs so your shell does not expand them)
+
+Flags:
+  --project <name>     Project to operate on
+  --anchors <path>     Palette anchor definitions, relative to the project
+                       root. Without it, palette measures that need anchors
+                       are skipped rather than guessed.
+  --dry-run            Measure and report, write nothing
+  --json               Emit the raw result as JSON
+
+Requires Python 3.9+ with Pillow, numpy and scipy (see scripts/measure_image.py).
+Set SDLAB_PYTHON to pin an explicit interpreter.`;
+
+export async function run(argv = process.argv.slice(2)) {
+  const { flags, positionals } = parseArgs(argv, {
+    flags: {
+      project: { type: 'string' },
+      anchors: { type: 'string' },
+      'dry-run': { type: 'boolean' },
+      json: { type: 'boolean' },
+      help: { type: 'boolean' },
+    },
+    deprecated: { game: 'project' },
+    allowUnknown: true,
+  });
+
+  if (flags.help) {
+    console.log(USAGE);
+    return;
+  }
+
+  const projectName = flags.project || getProjectName(argv);
+  const projectRoot = getProjectRoot(projectName);
+  const dryRun = Boolean(flags['dry-run']);
+
+  // Fail on a missing interpreter or package BEFORE resolving targets, so an
+  // operator without the Python side gets one clear actionable error rather
+  // than a successful scan followed by a confusing failure.
+  const python = assertMeasurementPrereqs();
+
+  const { targets, skipped } = resolveMeasureTargets(projectRoot, positionals[0]);
+
+  console.log(`\x1b[1msdlab measure\x1b[0m — ${projectName}`);
+  console.log(`  Target:  ${positionals[0]}`);
+  console.log(`  Images:  ${targets.length}${dryRun ? '  (dry run — nothing will be written)' : ''}`);
+
+  const anchorSpec = loadAnchors(flags.anchors, projectRoot);
+  if (anchorSpec.source) {
+    console.log(`  Anchors: ${anchorSpec.anchors.length} from ${flags.anchors}`);
+  } else {
+    // Say so plainly. A palette measure with no anchors is not a failed
+    // measurement, but it is a narrower one, and silently returning fewer
+    // fields than the operator expected is how a tool loses trust.
+    console.log(`  Anchors: none — pass --anchors <path> for palette-conformance measures`);
+  }
+  console.log('');
+
+  if (skipped.length > 0) {
+    warn(`${skipped.length} target(s) skipped: ${skipped.slice(0, 3).map((s) => `${s.id} (${s.reason})`).join(', ')}${skipped.length > 3 ? ', …' : ''}`);
+  }
+
+  if (targets.length === 0) {
+    console.log('  Nothing to measure.');
+    return;
+  }
+
+  const pyResult = runPythonMeasurement(python, {
+    images: targets.map((t) => ({ id: t.id, path: t.imagePath })),
+    anchors: anchorSpec.anchors,
+    hue_tolerance_deg: anchorSpec.hue_tolerance_deg,
+    sat_min: anchorSpec.sat_min,
+    val_min: anchorSpec.val_min,
+  });
+
+  const { applied, failed } = await applyMeasurements(targets, pyResult, {
+    anchorsSource: anchorSpec.source,
+    dryRun,
+  });
+
+  if (flags.json) {
+    result(JSON.stringify({ applied, failed, skipped }, null, 2));
+    return;
+  }
+
+  console.log(`  \x1b[32m✓\x1b[0m Measured: ${applied.length}`);
+  if (failed.length > 0) {
+    console.log(`  \x1b[31m✗\x1b[0m Failed:   ${failed.length}`);
+    for (const f of failed.slice(0, 5)) {
+      console.log(`      ${f.id}: ${f.error}`);
+    }
+    if (failed.length > 5) console.log(`      … and ${failed.length - 5} more`);
+  }
+
+  if (applied.length > 0) {
+    // Show one real measurement rather than only a count — a number the
+    // operator can sanity-check against the image in front of them is worth
+    // more than a success tally.
+    const sample = applied[0];
+    const keys = Object.keys(sample.measurements || {}).filter((k) => !k.startsWith('_'));
+    if (keys.length > 0) {
+      console.log('');
+      console.log(`  Sample (${sample.id}):`);
+      for (const k of keys.slice(0, 6)) {
+        const v = sample.measurements[k];
+        const shown = typeof v === 'number' ? v.toFixed(4) : JSON.stringify(v);
+        console.log(`    ${k.padEnd(28)} ${shown}`);
+      }
+    }
+  }
+
+  if (dryRun) {
+    console.log('');
+    console.log('  (dry run — no records were modified)');
+  }
+}
+
+// Direct execution guard
+if (process.argv[1] && (process.argv[1].endsWith('measure.js') || process.argv[1].endsWith('measure'))) {
+  run().catch(handleCliError);
+}