@mcptoolshop/style-dataset-lab 3.1.0 → 3.3.0

package/lib/rows.js

@@ -0,0 +1,228 @@
+/**
+ * Canonical dataset row shape.
+ *
+ * The single source-of-truth shape that adapters (generic-image-caption,
+ * diffusers-lora, ai-toolkit) consume. Two producers feed this shape:
+ *
+ *   1. `canon-build` — emits rows to disk at
+ *      `<project>/canon-build/<canon_sha>/dataset/<lane>-<partition>.jsonl`
+ *      from the canon entity store (research D1, D2).
+ *   2. `training-packages` — converts legacy records-path `{record, lane, group}`
+ *      entries into rows in memory so the records flow keeps working while
+ *      canon authoring catches up.
+ *
+ * Both paths produce rows the adapter cannot distinguish. This is D6:
+ * adapters have one input contract, not two.
+ *
+ * The research deliverable pins the row schema. See
+ * `memory/three-projection-build-research-2026-04-24.md` §Projection contracts.
+ */
+
+import { readFile } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { buildCaption, deriveStyleTrigger } from './captions.js';
+
+export const ROW_SCHEMA_VERSION = 'canon-build-dataset-1.0';
+
+/**
+ * Compute SHA-256 of a deterministic serialization of the inputs.
+ * Used for `entry_hash` (cache key) when building rows from canon entries.
+ */
+export function hashRowInputs({ entryBody, schemaVersion, buildConfigJson, projectFingerprint }) {
+  const h = createHash('sha256');
+  h.update(entryBody || '');
+  h.update('\0');
+  h.update(schemaVersion || '');
+  h.update('\0');
+  h.update(buildConfigJson || '');
+  h.update('\0');
+  h.update(projectFingerprint || '');
+  return h.digest('hex');
+}
+
+/**
+ * Build a row from a legacy record object.
+ *
+ * The records flow (star-freight today) produces `{ record, lane, group }`
+ * tuples via split + snapshot + canon-binding. Each tuple becomes one row.
+ *
+ * @param {Object} record — the records/<id>.json object
+ * @param {string|null} lane — resolved lane (costume, environment, ...)
+ * @param {string|null} group — resolved group (faction, region, ...)
+ * @param {Object} profile — training profile (for caption + trigger)
+ * @param {string} partition — train | val | test
+ * @returns {Object|null} row, or null if the record has no asset_path
+ */
+export function recordToRow(record, lane, group, profile, partition) {
+  if (!record?.id || !record.asset_path) return null;
+
+  const trigger = deriveStyleTrigger(profile) || null;
+  const caption = buildCaption(record, lane, group, profile);
+
+  const row = {
+    schema_version: ROW_SCHEMA_VERSION,
+    generated_from: null,               // records path: no canon commit
+    entity_id: record.id,
+    schema_kind: 'record',              // legacy marker: not a canon entity
+    entity_type: record.identity?.kind || null,
+    lane,
+    partition,
+    asset_path: record.asset_path,
+    caption,
+    trigger,
+    subject_filter_key: record.identity?.subject_name || record.id,
+    forbidden_tokens: record.canon?.forbidden_inputs || [],
+    pass_ratio: record.canon?.assertion_count > 0
+      ? +(record.canon.pass_count / record.canon.assertion_count).toFixed(3)
+      : null,
+    group,
+  };
+
+  return row;
+}
+
+/**
+ * Build a row from a canon entity + approved reference plate.
+ *
+ * The canon-build flow produces rows from canon entries. Each entry may
+ * contribute one or more rows (one per approved reference plate × lane).
+ *
+ * @param {Object} args
+ * @param {Object} args.entry — parsed canon entry (frontmatter object)
+ * @param {string} args.schemaKind — monster | character | deity | location | relic
+ * @param {string} args.assetPath — project-relative path to the approved plate
+ * @param {string} args.lane — resolved lane
+ * @param {string} args.caption — pre-rendered via buildCaption
+ * @param {string|null} args.trigger — from the training profile that drove the plate
+ * @param {string} args.partition — train | val | test (canon-build defaults to train)
+ * @param {string} args.entryHash — SHA-256 of entry + schema_version + build_config + project_fingerprint
+ * @param {string|null} args.generatedFrom — canon commit SHA or "content-sha:..." fallback
+ */
+export function canonEntryToRow({
+  entry,
+  schemaKind,
+  assetPath,
+  lane,
+  caption,
+  trigger,
+  partition,
+  entryHash,
+  generatedFrom,
+}) {
+  const entityType = entry.kind
+    || entry.species_tag
+    || entry.location_type
+    || entry.relic_type
+    || entry.generation
+    || null;
+
+  return {
+    schema_version: ROW_SCHEMA_VERSION,
+    generated_from: generatedFrom,
+    entity_id: entry.id,
+    schema_kind: schemaKind,
+    entity_type: entityType,
+    lane,
+    partition,
+    asset_path: assetPath,
+    caption,
+    trigger,
+    subject_filter_key: entry.id,
+    forbidden_tokens: entry.forbidden_inputs || [],
+    entry_hash: entryHash,
+  };
+}
+
+/**
+ * Read rows from a dataset.jsonl file.
+ *
+ * Parses each non-empty line as JSON; rejects the file if any row's
+ * `schema_version` does not match ROW_SCHEMA_VERSION.
+ *
+ * @param {string} path — absolute path to dataset.jsonl
+ * @returns {Promise<Array<Object>>} rows
+ */
+export async function readDatasetJsonl(path) {
+  const text = await readFile(path, 'utf-8');
+  const rows = [];
+  const lines = text.split('\n');
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    if (!line) continue;
+    let row;
+    try {
+      row = JSON.parse(line);
+    } catch (err) {
+      throw new Error(`readDatasetJsonl: line ${i + 1} of ${path} is not valid JSON (${err.message})`);
+    }
+    if (row.schema_version !== ROW_SCHEMA_VERSION) {
+      throw new Error(
+        `readDatasetJsonl: line ${i + 1} of ${path} has schema_version "${row.schema_version}" (expected "${ROW_SCHEMA_VERSION}")`
+      );
+    }
+    rows.push(row);
+  }
+  return rows;
+}
+
+/**
+ * Group rows by partition (train | val | test).
+ * Rows with missing/unknown partition are grouped under an "other" key.
+ */
+export function groupRowsByPartition(rows) {
+  const out = { train: [], val: [], test: [] };
+  for (const row of rows) {
+    const p = row.partition || 'other';
+    if (!out[p]) out[p] = [];
+    out[p].push(row);
+  }
+  return out;
+}
+
+/**
+ * Filter rows for a training profile.
+ *
+ * Two filter axes:
+ *   - lane-based: include row if row.lane ∈ profile.eligible_lanes
+ *   - entity-based: include row if row.entity_id === profile.entity_id_scope
+ *     (when entity_id_scope is set — per-character LoRA case, D8)
+ *
+ * Both filters compose: a per-character profile with both eligible_lanes and
+ * entity_id_scope set applies both as AND. Zero filters = pass everything.
+ */
+export function filterRowsForProfile(rows, profile) {
+  const eligibleLanes = profile.eligible_lanes;
+  const entityScope = profile.entity_id_scope;
+
+  return rows.filter((row) => {
+    if (Array.isArray(eligibleLanes) && eligibleLanes.length > 0) {
+      if (!eligibleLanes.includes(row.lane)) return false;
+    }
+    if (entityScope) {
+      if (row.entity_id !== entityScope && row.subject_filter_key !== entityScope) {
+        return false;
+      }
+    }
+    return true;
+  });
+}
+
+/**
+ * Serialize a row to a stable JSONL line.
+ * Key order is enforced by emitting known fields first, then alphabetized extras.
+ */
+export function serializeRow(row) {
+  const knownOrder = [
+    'schema_version', 'generated_from', 'entity_id', 'schema_kind', 'entity_type',
+    'lane', 'partition', 'asset_path', 'caption', 'trigger',
+    'subject_filter_key', 'forbidden_tokens', 'entry_hash', 'pass_ratio', 'group',
+  ];
+  const ordered = {};
+  for (const key of knownOrder) {
+    if (row[key] !== undefined) ordered[key] = row[key];
+  }
+  for (const key of Object.keys(row).sort()) {
+    if (!knownOrder.includes(key)) ordered[key] = row[key];
+  }
+  return JSON.stringify(ordered);
+}

package/lib/run-manifest.js

@@ -0,0 +1,308 @@
+/**
+ * Run-manifest pinning contract (PIN_PER_STEP).
+ *
+ * ONE source of truth for the fields that make a generation wave byte-for-byte
+ * replayable. Three writers emit provenance today — `scripts/generate.js`
+ * (record `provenance`), `lib/adapters/comfyui-runner.js` (run `manifest.json`),
+ * and `scripts/qwen_generate.py` (the wave `generation.json` receipt) — and they
+ * MUST all pin the same way. This module is the field-name authority; the
+ * Python receipt mirrors the identical field contract (see qwen_generate.py).
+ *
+ * Design (DECOMPOSE_BY_SECRETS, Parnas 1972): the pinning field names + the
+ * best-effort model hasher live here so the writers can never drift apart
+ * silently. When the model changes, this secret-hiding boundary survives.
+ *
+ * The pinning block closes three PIN_PER_STEP gaps that the pre-3.3.0 manifests
+ * left open (base/sampler/scheduler/shift/seed/steps/cfg were already recorded):
+ *   1. comfy_workflow_sha — the exact ComfyUI graph that ran, as a hash.
+ *   2. model/LoRA content identity — filenames upgraded to {name,size_bytes,sha256}.
+ *   3. seed_policy — HOW seeds were chosen (intent), not just the values.
+ *
+ * Every field is optional and additive. Legacy manifests with no `pinning`
+ * block load fine; readers nullish-coalesce (`m.pinning?.comfy_workflow_sha ?? null`).
+ */
+
+import { createHash } from 'node:crypto';
+import { openSync, readSync, closeSync, statSync, existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/**
+ * Contract version for the `pinning` block itself. Bump when the block's shape
+ * changes (a new field, a renamed field). Independent of the manifest
+ * SCHEMA_VERSION (lib/snapshot.js) — that versions the snapshot/split/export
+ * manifest family; this versions the generation-provenance pin. Mirrored as a
+ * literal in scripts/qwen_generate.py — keep the two in lockstep.
+ */
+export const PINNING_VERSION = '1.0.0';
+
+/**
+ * Seed policies — the INTENT behind seed selection, recorded alongside the
+ * concrete per-item seed values so a reader knows how to regenerate.
+ */
+export const SEED_POLICIES = Object.freeze({
+  EXPLICIT_PER_ITEM: 'explicit-per-item', // each item's seed is recorded verbatim (qwen_generate.py wave receipt)
+  BASE_INCREMENT: 'base+increment',        // seed = base_seed + job_index (generate.js / batch path)
+  FIXED: 'fixed',                          // every item shares one seed
+  RANDOM: 'random',                        // first seed fixed, rest sampled
+});
+
+/**
+ * Map a runtime seed_mode ('increment' | 'fixed' | 'random') to a seed_policy.
+ * `increment` is the base+index scheme; unknown modes fall back to base+increment.
+ */
+export function seedPolicyFromMode(seedMode) {
+  switch (seedMode) {
+    case 'fixed': return SEED_POLICIES.FIXED;
+    case 'random': return SEED_POLICIES.RANDOM;
+    case 'increment':
+    default: return SEED_POLICIES.BASE_INCREMENT;
+  }
+}
+
+// ─── Stable serialization ────────────────────────────────────────
+
+/**
+ * Deterministic JSON: object keys sorted, arrays kept in order (arrays are
+ * meaningful sequences — a LoRA chain, a node's input wiring). Guarantees the
+ * same semantic graph hashes identically regardless of key insertion order.
+ */
+export function stableStringify(value) {
+  if (value === null || value === undefined) return 'null';
+  if (typeof value === 'number') return Number.isFinite(value) ? String(value) : 'null';
+  if (typeof value === 'boolean') return value ? 'true' : 'false';
+  if (typeof value === 'string') return JSON.stringify(value);
+  if (Array.isArray(value)) return '[' + value.map(stableStringify).join(',') + ']';
+  if (typeof value === 'object') {
+    const keys = Object.keys(value).sort();
+    const parts = [];
+    for (const k of keys) {
+      if (value[k] === undefined) continue;
+      parts.push(JSON.stringify(k) + ':' + stableStringify(value[k]));
+    }
+    return '{' + parts.join(',') + '}';
+  }
+  return 'null';
+}
+
+/**
+ * Return a copy of a ComfyUI graph with per-ITEM nondeterminism removed, so the
+ * whole wave shares ONE comfy_workflow_sha. Normalized out (recorded elsewhere,
+ * per item): KSampler seed / noise_seed, CLIPTextEncode prompt text, SaveImage
+ * filename_prefix. What SURVIVES the hash is the pipeline skeleton — node
+ * topology, model loaders, sampler/scheduler/shift, LoRA chain + weights,
+ * dims/steps/cfg — i.e. exactly the part that must match to replay the wave.
+ */
+export function normalizeGraphForHash(graph) {
+  const out = {};
+  for (const [id, node] of Object.entries(graph || {})) {
+    const inputs = { ...(node?.inputs || {}) };
+    const cls = node?.class_type;
+    if (cls === 'KSampler' || cls === 'KSamplerAdvanced') {
+      if ('seed' in inputs) inputs.seed = 0;
+      if ('noise_seed' in inputs) inputs.noise_seed = 0;
+    } else if (cls === 'CLIPTextEncode') {
+      if ('text' in inputs) inputs.text = '';
+    } else if (cls === 'SaveImage') {
+      if ('filename_prefix' in inputs) inputs.filename_prefix = '';
+    }
+    out[id] = { class_type: cls, inputs };
+  }
+  return out;
+}
+
+/**
+ * SHA-256 of the exact ComfyUI graph submitted for the wave (per-item seed +
+ * prompt normalized out — see normalizeGraphForHash). Returns a `sha256:`-prefixed
+ * digest, or null for an empty/absent graph.
+ */
+export function comfyWorkflowSha(graph) {
+  if (!graph || Object.keys(graph).length === 0) return null;
+  const normalized = normalizeGraphForHash(graph);
+  return 'sha256:' + createHash('sha256').update(stableStringify(normalized)).digest('hex');
+}
+
+// ─── Model file resolution + hashing (best-effort) ───────────────
+
+/**
+ * Default ComfyUI models directory on the rig. Override with SDLAB_MODELS_DIR.
+ * Hashing is opt-in and best-effort, so a wrong/absent path degrades cleanly to
+ * `sha256: null` — it never fails a generation.
+ */
+export function defaultModelsDir() {
+  return process.env.SDLAB_MODELS_DIR
+    || 'E:/AI-Models/ComfyUI_windows_portable/ComfyUI/models';
+}
+
+// Kind → candidate subdirectories under the models dir, in search order.
+const KIND_SUBDIRS = {
+  unet: ['diffusion_models', 'unet'],
+  clip: ['text_encoders', 'clip'],
+  vae: ['vae'],
+  checkpoint: ['checkpoints'],
+  lora: ['loras'],
+};
+
+/**
+ * Resolve a model filename to an absolute path under the models dir. Searches
+ * the kind-appropriate subdirs first, then every known subdir, then the root.
+ * Returns null when nothing matches (unresolvable → sha256 null with a note).
+ */
+export function resolveModelPath(name, kind, modelsDir = defaultModelsDir()) {
+  if (!name || !modelsDir) return null;
+  const tried = new Set();
+  const subdirs = [
+    ...(KIND_SUBDIRS[kind] || []),
+    ...Object.values(KIND_SUBDIRS).flat(),
+    '.',
+  ];
+  for (const sub of subdirs) {
+    const candidate = join(modelsDir, sub, name);
+    if (tried.has(candidate)) continue;
+    tried.add(candidate);
+    if (existsSync(candidate)) return candidate;
+  }
+  return null;
+}
+
+/**
+ * Streaming SHA-256 of a file — 1 MiB chunks, synchronous, never loads the whole
+ * (multi-GB) file into memory. Returns a `sha256:`-prefixed digest.
+ */
+export function sha256File(path) {
+  const hash = createHash('sha256');
+  const fd = openSync(path, 'r');
+  try {
+    const buf = Buffer.allocUnsafe(1 << 20);
+    let bytes;
+    while ((bytes = readSync(fd, buf, 0, buf.length, null)) > 0) {
+      hash.update(bytes === buf.length ? buf : buf.subarray(0, bytes));
+    }
+  } finally {
+    closeSync(fd);
+  }
+  return 'sha256:' + hash.digest('hex');
+}
+
+/**
+ * Best-effort content identity for one model/LoRA file: {name, size_bytes, sha256}
+ * (+ a short `hash_note` when sha256 can't be produced). size_bytes is always
+ * filled when the file resolves (a cheap stat); sha256 is gated behind
+ * `doHash` (the --hash-models opt-in) and cached by (name, size, mtime) so
+ * multi-GB checkpoints aren't re-hashed every wave.
+ *
+ * @param {string} name — model filename as it appears in the graph
+ * @param {Object} [opts]
+ * @param {'unet'|'clip'|'vae'|'checkpoint'|'lora'} [opts.kind]
+ * @param {string} [opts.modelsDir]
+ * @param {boolean} [opts.doHash=false] — compute sha256 (else record name+size only)
+ * @param {{map: Map, cacheFile?: string, dirty: boolean}} [opts.cache]
+ */
+export function hashModelFile(name, { kind, modelsDir = defaultModelsDir(), doHash = false, cache } = {}) {
+  if (!name) return { name: name ?? null, size_bytes: null, sha256: null, hash_note: 'no filename' };
+  const resolved = resolveModelPath(name, kind, modelsDir);
+  if (!resolved) return { name, size_bytes: null, sha256: null, hash_note: `unresolved under ${modelsDir}` };
+  let size_bytes = null;
+  let mtimeMs = 0;
+  try {
+    const st = statSync(resolved);
+    size_bytes = st.size;
+    mtimeMs = st.mtimeMs;
+  } catch {
+    return { name, size_bytes: null, sha256: null, hash_note: 'stat failed' };
+  }
+  if (!doHash) return { name, size_bytes, sha256: null, hash_note: 'hash skipped (pass --hash-models)' };
+  const key = `${name}:${size_bytes}:${Math.round(mtimeMs)}`;
+  if (cache?.map?.has(key)) return { name, size_bytes, sha256: cache.map.get(key) };
+  let sha256;
+  try {
+    sha256 = sha256File(resolved);
+  } catch {
+    return { name, size_bytes, sha256: null, hash_note: 'hash failed (read error)' };
+  }
+  if (cache?.map) {
+    cache.map.set(key, sha256);
+    cache.dirty = true;
+  }
+  return { name, size_bytes, sha256 };
+}
+
+/**
+ * Content-identity block for the model loaders of a wave. `modelSpec` maps a
+ * role to a filename — {unet, clip, vae} for the Qwen DiT path, {checkpoint}
+ * for the SDXL path. Only provided roles appear in the result.
+ */
+export function buildModelHashes(modelSpec = {}, opts = {}) {
+  const roleKind = { unet: 'unet', clip: 'clip', vae: 'vae', checkpoint: 'checkpoint' };
+  const out = {};
+  for (const [role, kind] of Object.entries(roleKind)) {
+    const name = modelSpec[role];
+    if (!name) continue;
+    out[role] = hashModelFile(name, { ...opts, kind });
+  }
+  return out;
+}
+
+// ─── Persistent hash cache (best-effort) ─────────────────────────
+
+/**
+ * Create a hash cache. Loads the on-disk JSON map (name:size:mtime → sha256) if
+ * present, so hashes survive across waves. `cacheFile` defaults to a dotfile in
+ * the models dir; override with SDLAB_HASH_CACHE. All I/O is best-effort — a
+ * missing/unwritable cache never fails a run.
+ */
+export function createHashCache(cacheFile) {
+  const file = cacheFile
+    || process.env.SDLAB_HASH_CACHE
+    || join(defaultModelsDir(), '.sdlab-hash-cache.json');
+  const map = new Map();
+  try {
+    if (existsSync(file)) {
+      const obj = JSON.parse(readFileSync(file, 'utf-8'));
+      for (const [k, v] of Object.entries(obj)) map.set(k, v);
+    }
+  } catch { /* best-effort: ignore a corrupt/unreadable cache */ }
+  return { map, cacheFile: file, dirty: false };
+}
+
+/**
+ * Persist a hash cache to disk if it grew. Best-effort — swallows write errors.
+ */
+export function flushHashCache(cache) {
+  if (!cache || !cache.dirty || !cache.cacheFile) return;
+  try {
+    const obj = Object.fromEntries(cache.map);
+    writeFileSync(cache.cacheFile, JSON.stringify(obj, null, 2));
+    cache.dirty = false;
+  } catch { /* best-effort: ignore */ }
+}
+
+// ─── The pinning block ───────────────────────────────────────────
+
+/**
+ * Build the pinning block for a wave/item. Identical shape across all three
+ * writers (see module header). Every writer calls this so the contract is
+ * enforced in ONE place.
+ *
+ * @param {Object} args
+ * @param {Object} [args.graph] — the ComfyUI node graph submitted (for comfy_workflow_sha)
+ * @param {Object} [args.models] — {unet?, clip?, vae?, checkpoint?} filenames
+ * @param {Array<{name:string, weight?:number}>} [args.loras]
+ * @param {string} [args.seedPolicy] — one of SEED_POLICIES
+ * @param {boolean} [args.hashModels=false] — opt-in sha256 of model/LoRA files
+ * @param {string} [args.modelsDir]
+ * @param {{map: Map, cacheFile?: string, dirty: boolean}} [args.cache]
+ * @returns {Object} pinning block
+ */
+export function buildPinning({ graph, models = {}, loras = [], seedPolicy = null, hashModels = false, modelsDir = defaultModelsDir(), cache } = {}) {
+  const opts = { modelsDir, doHash: hashModels, cache };
+  return {
+    pinning_version: PINNING_VERSION,
+    comfy_workflow_sha: comfyWorkflowSha(graph),
+    seed_policy: seedPolicy,
+    models: buildModelHashes(models, opts),
+    loras: (loras || []).map((l) => ({
+      ...hashModelFile(l.name, { ...opts, kind: 'lora' }),
+      weight: l.weight ?? 1.0,
+    })),
+  };
+}

package/lib/snapshot.js

@@ -19,8 +19,14 @@ import { warn } from './log.js';
  * snapshot/split/export/eval-pack/eval-run/training-manifest/training-package/
  * impl-pack stamps this version; readers log a WARNING on version mismatch
  * (best-effort compatibility).
+ *
+ * 2.3.0 (v3.3.0): PIN_PER_STEP — the ComfyUI run manifest now also stamps this
+ * version and carries a `pinning` block (comfy_workflow_sha + model/LoRA content
+ * hashes + seed_policy; see lib/run-manifest.js). The block is additive and
+ * optional, so a legacy manifest with no schema_version / no pinning still loads
+ * (checkManifestVersion only warns; it never throws).
  */
-export const SCHEMA_VERSION = '2.2.0';
+export const SCHEMA_VERSION = '2.3.0';
 
 /**
  * Warn if a loaded manifest's schema_version does not match the reader.

package/lib/training-adapters.js

@@ -27,6 +27,11 @@ export const ADAPTER_REGISTRY = Object.freeze({
     description: 'Diffusers fine-tuning layout: image + caption .txt sidecars + per-partition JSONL',
     module: './adapters/diffusers-lora.js',
   },
+  'ai-toolkit': {
+    id: 'ai-toolkit',
+    description: 'Ostris ai-toolkit Flux training package: image + .txt sidecars + ai-toolkit-config.yaml at root',
+    module: './adapters/ai-toolkit.js',
+  },
 });
 
 /**

package/lib/training-packages.js

@@ -18,6 +18,7 @@ import { loadProjectConfig, detectLane, detectGroup } from './config.js';
 import { inputError } from './errors.js';
 import { SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
 import { loadAdapter } from './training-adapters.js';
+import { recordToRow, filterRowsForProfile } from './rows.js';
 
 /**
  * Generate a training package ID: tp-YYYYMMDD-HHMMSS-XXXX
@@ -61,38 +62,43 @@ export async function buildTrainingPackage(projectRoot, manifestId, options = {}
     );
   }
 
-  // Load split partitions
-  const partitions = {};
-  let totalRecords = 0;
-  let totalImages = 0;
+  // Convert split partitions → canonical rows (D6 adapter input shape).
+  // Adapters only understand rows — records-flow callers go through this
+  // converter, canon-build-flow callers produce rows directly. One input
+  // contract, two producers.
+  const rows = [];
+  const partitionCounts = { train: 0, val: 0, test: 0 };
 
   for (const partition of ['train', 'val', 'test']) {
     const entries = await loadSplitPartition(projectRoot, manifest.source_split_id, partition);
-    const records = [];
 
     for (const entry of entries) {
       const record = await loadRecord(recordsDir, entry.record_id);
       if (!record) continue;
 
-      // Filter by profile's eligible lanes
       const prompt = record.provenance?.prompt || '';
       const lane = detectLane(record.id, prompt, config.lanes);
-      if (profile.eligible_lanes.length > 0 && !profile.eligible_lanes.includes(lane)) {
-        continue;
-      }
-
       const group = detectGroup(record.id, prompt, config.terminology);
-      records.push({ record, lane, group });
+
+      const row = recordToRow(record, lane, group, profile, partition);
+      if (row) rows.push(row);
     }
+  }
 
-    partitions[partition] = records;
-    totalRecords += records.length;
+  // Apply profile filters (eligible_lanes + entity_id_scope — D8).
+  // filterRowsForProfile returns rows where lane ∈ eligible_lanes AND
+  // entity_id === entity_id_scope (when set). Zero filters = identity.
+  const filteredRows = filterRowsForProfile(rows, profile);
+  for (const row of filteredRows) {
+    if (partitionCounts[row.partition] !== undefined) partitionCounts[row.partition]++;
   }
+  const totalRecords = filteredRows.length;
+  let totalImages = 0;
 
-  // Let the adapter build the package layout
+  // Let the adapter build the package layout from the canonical rows.
   const adapterResult = await adapter.buildPackage({
     packageDir,
-    partitions,
+    rows: filteredRows,
     profile,
     manifest,
     config,
@@ -118,9 +124,9 @@ export async function buildTrainingPackage(projectRoot, manifestId, options = {}
     config_fingerprint: manifest.config_fingerprint,
     counts: {
       total_records: totalRecords,
-      train: partitions.train.length,
-      val: partitions.val.length,
-      test: partitions.test.length,
+      train: partitionCounts.train,
+      val: partitionCounts.val,
+      test: partitionCounts.test,
       images: totalImages,
       checksums: 0, // updated after checksum pass below
     },