@mcptoolshop/style-dataset-lab 3.3.0 → 3.5.0

package/lib/generated-provenance.js

@@ -3,12 +3,117 @@
  * attached to records created from selected run/batch outputs.
  *
  * Normalizes source context (run vs batch) into a single shape
- * that traces back through brief → workflow → seed → config.
+ * that traces back through brief → workflow → seed → config,
+ * plus G4 (model licences), G5 (owner + CC-BY-4.0), and G6 (canon link).
  */
 
 import { readFileSync, existsSync } from 'node:fs';
 import { join } from 'node:path';
 import { createHash } from 'node:crypto';
+import { warn } from './log.js';
+import { STUDIO_OWNER, STUDIO_LICENCE } from './records.js';
+
+/**
+ * Resolve `{ canon_sha, entry_id, entry_sha256 }` from explicit metadata or
+ * the project's `canon-build/<canon_sha>/manifest.json` (`per_entity_hashes`).
+ */
+export function resolveCanonLink(
+  projectRoot,
+  {
+    canonSha = null,
+    entryId = null,
+    entrySha256 = null,
+    subjectId = null,
+    runManifest = null,
+    batchManifest = null,
+    item = null,
+    brief = null,
+  } = {},
+) {
+  let resolvedCanonSha =
+    canonSha ||
+    item?.canon_sha ||
+    runManifest?.canon_sha ||
+    batchManifest?.canon_sha ||
+    brief?.canon_sha ||
+    null;
+
+  if (!resolvedCanonSha && projectRoot) {
+    const currentPath = join(projectRoot, 'canon-build', 'CURRENT');
+    if (existsSync(currentPath)) {
+      try {
+        const raw = readFileSync(currentPath, 'utf-8').trim();
+        if (raw) resolvedCanonSha = raw;
+      } catch {
+        // best-effort read of CURRENT pointer
+      }
+    }
+  }
+
+  let resolvedEntryId =
+    entryId ||
+    item?.entry_id ||
+    runManifest?.entry_id ||
+    batchManifest?.entry_id ||
+    brief?.entry_id ||
+    subjectId ||
+    item?.subject_id ||
+    runManifest?.subject_id ||
+    batchManifest?.subject_id ||
+    brief?.subject_id ||
+    null;
+
+  let resolvedEntrySha =
+    entrySha256 ||
+    item?.entry_sha256 ||
+    runManifest?.entry_sha256 ||
+    batchManifest?.entry_sha256 ||
+    brief?.entry_sha256 ||
+    null;
+
+  if (resolvedCanonSha && projectRoot) {
+    const dirName = String(resolvedCanonSha).replace(/:/g, '-');
+    const manifestPath = existsSync(join(projectRoot, 'canon-build', String(resolvedCanonSha), 'manifest.json'))
+      ? join(projectRoot, 'canon-build', String(resolvedCanonSha), 'manifest.json')
+      : join(projectRoot, 'canon-build', dirName, 'manifest.json');
+    if (existsSync(manifestPath)) {
+      try {
+        const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8'));
+        const hashes = manifest.per_entity_hashes || {};
+        if (resolvedEntryId) {
+          if (hashes[resolvedEntryId]) {
+            resolvedEntrySha = resolvedEntrySha || hashes[resolvedEntryId];
+          } else {
+            const suffix = `/${resolvedEntryId}`;
+            const matchKey = Object.keys(hashes).find((k) => k.endsWith(suffix));
+            if (matchKey) {
+              resolvedEntryId = matchKey;
+              resolvedEntrySha = resolvedEntrySha || hashes[matchKey];
+            }
+          }
+        } else {
+          const keys = Object.keys(hashes);
+          if (keys.length === 1) {
+            resolvedEntryId = keys[0];
+            resolvedEntrySha = resolvedEntrySha || hashes[keys[0]];
+          }
+        }
+      } catch {
+        // best-effort manifest lookup
+      }
+    }
+  }
+
+  if (!resolvedCanonSha && !resolvedEntryId && !resolvedEntrySha) {
+    return null;
+  }
+
+  return {
+    canon_sha: resolvedCanonSha,
+    entry_id: resolvedEntryId,
+    entry_sha256: resolvedEntrySha,
+  };
+}
 
 /**
  * Build a generation provenance payload for a selected output.
@@ -20,6 +125,9 @@ import { createHash } from 'node:crypto';
  * @param {Object} [opts.batchManifest] — batch manifest if source is batch
  * @param {Object} opts.item — the selection item (slot_or_output, filename, seed)
  * @param {string} opts.projectRoot
+ * @param {string} [opts.canonSha]
+ * @param {string} [opts.entryId]
+ * @param {string} [opts.entrySha256]
  * @returns {Object} generation_provenance block
  */
 export function buildGeneratedProvenance({
@@ -29,8 +137,13 @@ export function buildGeneratedProvenance({
   batchManifest,
   item,
   projectRoot,
+  canonSha = null,
+  entryId = null,
+  entrySha256 = null,
 }) {
   const prov = {
+    owner: STUDIO_OWNER,
+    licence: STUDIO_LICENCE,
     source_type: sourceType,
     source_id: sourceId,
     run_id: runManifest.run_id,
@@ -44,12 +157,55 @@ export function buildGeneratedProvenance({
     prov.batch_id = batchManifest.batch_id;
   }
 
+  const matchedOutput = runManifest.outputs
+    ? runManifest.outputs.find((o) => o.filename === item.filename)
+    : null;
+
   // Seed from the item or from the run manifest outputs
   if (item.seed !== undefined) {
     prov.seed = item.seed;
-  } else if (runManifest.outputs) {
-    const match = runManifest.outputs.find(o => o.filename === item.filename);
-    if (match?.seed !== undefined) prov.seed = match.seed;
+  } else if (matchedOutput?.seed !== undefined) {
+    prov.seed = matchedOutput.seed;
+  }
+
+  // Prompt and prompt_sha256 when available on the item or output
+  const promptText = item.prompt ?? matchedOutput?.prompt ?? null;
+  if (typeof promptText === 'string' && promptText.length > 0) {
+    prov.prompt = promptText;
+    prov.prompt_sha256 = 'sha256:' + createHash('sha256').update(promptText).digest('hex');
+  } else if (item.prompt_sha256 || matchedOutput?.prompt_sha256) {
+    prov.prompt_sha256 = item.prompt_sha256 || matchedOutput.prompt_sha256;
+  }
+
+  // Comfy workflow sha and model licences from run manifest pinning (G4, G9)
+  const wfSha = runManifest.pinning?.comfy_workflow_sha ?? runManifest.comfy_workflow_sha ?? null;
+  if (wfSha) {
+    prov.comfy_workflow_sha = wfSha;
+  }
+  if (runManifest.pinning?.models || runManifest.pinning?.loras) {
+    prov.model_licences = {
+      cc_by_4_eligible: runManifest.pinning.cc_by_4_eligible ?? null,
+      models: Object.fromEntries(
+        Object.entries(runManifest.pinning.models || {}).map(([role, m]) => [
+          role,
+          {
+            name: m.name,
+            sha256: m.sha256 ?? null,
+            licence_id: m.licence_id ?? m.licence?.spdx_id ?? null,
+            licence_url: m.licence_url ?? m.licence?.url ?? null,
+            date_read: m.date_read ?? m.licence?.verified_at ?? null,
+          },
+        ]),
+      ),
+      loras: (runManifest.pinning.loras || []).map((l) => ({
+        name: l.name,
+        weight: l.weight ?? 1.0,
+        sha256: l.sha256 ?? null,
+        licence_id: l.licence_id ?? l.licence?.spdx_id ?? null,
+        licence_url: l.licence_url ?? l.licence?.url ?? null,
+        date_read: l.date_read ?? l.licence?.verified_at ?? null,
+      })),
+    };
   }
 
   // Adapter target from run manifest
@@ -57,21 +213,53 @@ export function buildGeneratedProvenance({
     prov.adapter_target = runManifest.adapter_target;
   }
 
+  let parsedBrief = null;
   // Subject from run brief or batch manifest
   if (batchManifest?.subject_id) {
     prov.subject_id = batchManifest.subject_id;
-  } else {
+  } else if (projectRoot && runManifest.brief_id) {
     // Try to read subject from brief
     const briefPath = join(projectRoot, 'briefs', `${runManifest.brief_id}.json`);
     if (existsSync(briefPath)) {
       try {
         const brief = JSON.parse(readFileSync(briefPath, 'utf-8'));
+        parsedBrief = brief;
         if (brief.subject_id) prov.subject_id = brief.subject_id;
         if (brief.parent_brief_id) prov.parent_brief_id = brief.parent_brief_id;
-      } catch { /* skip */ }
+      } catch (err) {
+        // B08: the brief exists but couldn't be read/parsed (e.g. a
+        // concurrent mid-write) — lineage is genuinely UNKNOWN here, not
+        // merely absent. Surface it instead of silently shipping
+        // provenance with no lineage and no signal that anything failed.
+        warn(`generated-provenance: brief ${briefPath} could not be read (${err.message}) — lineage_incomplete`);
+        prov.lineage_incomplete = true;
+      }
+    } else {
+      // B08: no brief on disk at all for this run — the same "genuinely
+      // unknown" lineage state as an unreadable brief, not a normal
+      // "this brief simply has no subject_id" case (that one is NOT
+      // flagged — see the brief-read-fine-but-no-subject_id branch above).
+      warn(`generated-provenance: brief not found at ${briefPath} — lineage_incomplete`);
+      prov.lineage_incomplete = true;
     }
   }
 
+  const canonLink = resolveCanonLink(projectRoot, {
+    canonSha,
+    entryId,
+    entrySha256,
+    subjectId: prov.subject_id,
+    runManifest,
+    batchManifest,
+    item,
+    brief: parsedBrief,
+  });
+  if (canonLink) {
+    prov.canon_sha = canonLink.canon_sha;
+    prov.entry_id = canonLink.entry_id;
+    prov.entry_sha256 = canonLink.entry_sha256;
+  }
+
   // Config fingerprint — hash the run manifest for reproducibility tracing
   prov.config_fingerprint = createHash('sha256')
     .update(JSON.stringify({

package/lib/implementation-packs.js

@@ -8,7 +8,7 @@
  * Every implementation pack is tied to a training manifest.
  */
 
-import { readFile, writeFile, mkdir, readdir } from 'node:fs/promises';
+import { readFile, writeFile, readdir } from 'node:fs/promises';
 import { join } from 'node:path';
 import { existsSync } from 'node:fs';
 import { randomBytes } from 'node:crypto';
@@ -17,8 +17,8 @@ import { loadTrainingProfile } from './training-profiles.js';
 import { loadProjectConfig, detectLane, detectGroup } from './config.js';
 import { loadSplitPartition } from './split.js';
 import { loadRecord } from './records.js';
-import { inputError } from './errors.js';
-import { SCHEMA_VERSION, checkManifestVersion } from './snapshot.js';
+import { SCHEMA_VERSION, checkManifestVersion, claimIdDir } from './snapshot.js';
+import { warn } from './log.js';
 
 function generateImplId() {
   const now = new Date();
@@ -46,13 +46,27 @@ export async function buildImplementationPack(projectRoot, manifestId) {
   // Also load some train records for broader coverage
   const trainEntries = await loadSplitPartition(projectRoot, manifest.source_split_id, 'train');
 
+  // F3: a record the split promised (via source_split_id's train/test
+  // partitions) whose file was later deleted from records/ must not
+  // silently drop out of the implementation pack with no counter, no
+  // warning, no id recorded. loadRecord() returns null on ENOENT
+  // (lib/records.js); previously `if (!record) continue` was the entire
+  // handling at both loops below. WARN + ledger (not hard-fail): an
+  // implementation pack is an illustrative/example artifact, not the
+  // Law-4 reproducibility-guaranteeing one (export.js and
+  // training-packages.js are, and hard-fail on the identical situation).
+  const missingRecords = [];
+
   // Build prompt examples — per-lane best records
   const promptExamples = [];
   const laneExamples = new Map();
 
   for (const entry of [...testEntries, ...trainEntries.slice(0, 50)]) {
     const record = await loadRecord(recordsDir, entry.record_id);
-    if (!record) continue;
+    if (!record) {
+      missingRecords.push({ record_id: entry.record_id, stage: 'implementation-pack', source: 'prompt-examples' });
+      continue;
+    }
     if (record.judgment?.status !== 'approved') continue;
 
     const prompt = record.provenance?.prompt || '';
@@ -85,10 +99,26 @@ export async function buildImplementationPack(projectRoot, manifestId) {
   const knownFailures = [];
   const allFiles = (await readdir(recordsDir)).filter(f => f.endsWith('.json')).sort();
   let failCount = 0;
+  // F4: adopt the DB-003 pattern already proven in snapshot.js's
+  // createSnapshot (and mirrored in eval-pack.js's buildEvalPack) — this
+  // loop was the one place left in the codebase still doing a bare
+  // JSON.parse per record file with no try/catch. One truncated/corrupt
+  // record file threw a filename-free SyntaxError here and discarded the
+  // prompt-examples work (the loop above) and the subject-continuity work
+  // (the loop below) this function had already computed, since every
+  // artifact is written to disk only at the very end of
+  // buildImplementationPack — skip, count, warn, continue instead.
+  const unreadable = [];
 
   for (const file of allFiles) {
     if (failCount >= 10) break;
-    const record = JSON.parse(await readFile(join(recordsDir, file), 'utf-8'));
+    let record;
+    try {
+      record = JSON.parse(await readFile(join(recordsDir, file), 'utf-8'));
+    } catch (err) {
+      unreadable.push({ record_id: file.replace(/\.json$/, ''), file, error: err.message });
+      continue;
+    }
     if (record.judgment?.status !== 'rejected') continue;
 
     const prompt = record.provenance?.prompt || '';
@@ -107,6 +137,15 @@ export async function buildImplementationPack(projectRoot, manifestId) {
     });
     failCount++;
   }
+  // F4: surface unreadable records loudly (DB-003 parity) — same shape,
+  // same warn() visibility as snapshot.js's createSnapshot.
+  if (unreadable.length > 0) {
+    const sample = unreadable.slice(0, 5).map(u => u.file).join(', ');
+    warn(
+      `implementation-pack: ${unreadable.length} record file(s) were unreadable and skipped` +
+      (unreadable.length > 5 ? ` (first 5: ${sample}, ...)` : ` (${sample})`),
+    );
+  }
 
   // Build subject continuity examples
   const subjectExamples = [];
@@ -114,12 +153,33 @@ export async function buildImplementationPack(projectRoot, manifestId) {
 
   for (const entry of [...testEntries, ...trainEntries]) {
     const record = await loadRecord(recordsDir, entry.record_id);
-    if (!record?.identity?.subject_name) continue;
+    // F3: keep "record file is missing" (a data-integrity problem worth a
+    // warning) distinct from "record loaded fine but simply has no
+    // identity.subject_name" (the normal, expected case for most records —
+    // not a missing-record situation). The original combined
+    // `!record?.identity?.subject_name` check could not tell these apart.
+    if (!record) {
+      missingRecords.push({ record_id: entry.record_id, stage: 'implementation-pack', source: 'subject-continuity' });
+      continue;
+    }
+    if (!record.identity?.subject_name) continue;
     const name = record.identity.subject_name;
     if (!bySubject.has(name)) bySubject.set(name, []);
     bySubject.get(name).push(record);
   }
 
+  // F3: warn once for both loops combined — the ledger records which
+  // loop(s) actually hit the miss via `source`, so the summary line stays
+  // useful even though it's a single warning for the whole pack build.
+  if (missingRecords.length > 0) {
+    const sample = missingRecords.slice(0, 10).map(m => `${m.record_id} (${m.source})`);
+    warn(
+      `implementation-pack: ${missingRecords.length} record(s) referenced by split "${manifest.source_split_id}" ` +
+      `could not be loaded from ${recordsDir}` +
+      (missingRecords.length > 10 ? ` — first 10: ${sample.join(', ')}, ...` : ` — ${sample.join(', ')}`),
+    );
+  }
+
   for (const [name, records] of [...bySubject].sort((a, b) => b[1].length - a[1].length).slice(0, 5)) {
     subjectExamples.push({
       subject_name: name,
@@ -138,17 +198,18 @@ export async function buildImplementationPack(projectRoot, manifestId) {
   };
 
   // Write implementation pack
-  const implId = generateImplId();
-  const implDir = join(projectRoot, 'training', 'implementations', implId);
-  // D-005: refuse to silently overwrite on ID collision.
-  if (existsSync(implDir)) {
-    throw inputError(
-      'IMPL_PACK_ID_COLLISION',
-      `Implementation pack directory already exists: ${implDir}`,
-      'Retry to generate a fresh implementation pack ID.',
-    );
-  }
-  await mkdir(implDir, { recursive: true });
+  // SDL-M10: atomically claim the implementation-pack id/dir — closes the
+  // check-then-act race the old existsSync+recursive-mkdir pattern had
+  // (D-005's intent, now actually race-proof; see claimIdDir in snapshot.js).
+  const { id: implId, dir: implDir } = await claimIdDir(
+    join(projectRoot, 'training', 'implementations'),
+    generateImplId,
+    {
+      code: 'IMPL_PACK_ID_COLLISION',
+      message: 'Could not claim a unique implementation-pack ID — every candidate collided with an existing implementation-pack directory.',
+      hint: 'Retry; if this persists, another sdlab process may be creating implementation packs in a tight loop.',
+    },
+  );
 
   const implManifest = {
     // DB-001 / DB-006: stamp manifest schema version
@@ -163,7 +224,24 @@ export async function buildImplementationPack(projectRoot, manifestId) {
       known_failures: knownFailures.length,
       subject_examples: subjectExamples.length,
       lanes_covered: laneExamples.size,
+      // F3: how many split-referenced records could not be loaded while
+      // building this pack. 0 in the common case; see missing_records
+      // below for the actual ids.
+      missing_records: missingRecords.length,
+      // F4: how many record files (from the known-failures scan over ALL
+      // of records/) existed but threw on JSON.parse — distinct from
+      // missing_records above, which is "no file at all". 0 in the common
+      // case; see unreadable_records below for the actual files.
+      unreadable_records: unreadable.length,
     },
+    // F3: ledger of every record_id that failed to load while building
+    // this pack (mirrors the images_failed pattern export.js established
+    // at export time) — record_id + stage + which loop hit it, so an
+    // operator can trace it without re-diffing the split against records/.
+    missing_records: missingRecords,
+    // F4: ledger of record files that existed but failed to JSON.parse
+    // (DB-003 parity with snapshot.js's `errors` field).
+    unreadable_records: unreadable,
   };
 
   await writeFile(join(implDir, 'manifest.json'), JSON.stringify(implManifest, null, 2) + '\n');