@mcptoolshop/style-dataset-lab 3.4.0 → 3.5.0

package/lib/colab-launcher-guards.js

@@ -0,0 +1,285 @@
+/**
+ * C2 Colab launcher guards (DESIGN-v3 §9.2 item 6 — lessons v4k, v4p, v4q).
+ *
+ * Implements the four required launcher guards:
+ *   1. Independent wall watcher (`evaluateWallGuard`, v4k): honours `wallS` and
+ *      `graceS` in ALL modes including `stub` mode.
+ *   2. Independent cap watcher (`evaluateCapGuard`, v4k): honours `cap` and
+ *      `margin` in ALL modes including `stub` mode (`spend = max(drop, rate * elapsed_h)`).
+ *   3. `vitals.log` parser & `DEAD HARNESS` / `STALL` watcher (`parseVitalsLog`,
+ *      `evaluateVitalsAndStallGuard`, v4p): trips `DEAD_HARNESS_TRIPPED` when 2
+ *      consecutive `phase: run` samples have 0 active worker processes, and trips
+ *      `STALL_GUARD_TRIPPED` when neither rows nor `vitals.log` advance in `stallS`.
+ *   4. Host-RAM / cgroup limit pre-load projection check & between-item fetch
+ *      (`checkHostRamProjection`, `runBetweenItemFetch`, v4q): refuses any step
+ *      whose projected anonymous host RAM (`projectedMb * 1.05 > limitMb`) would
+ *      trigger a kernel SIGKILL (`rc=-9`), and copies completed artifacts listed
+ *      in `current.json` before `current.json` is overwritten for the next item.
+ */
+
+import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { inputError as rawInputError } from './errors.js';
+
+function inputError(code, msg) {
+  return rawInputError(code, `${code}: ${msg}`);
+}
+
+const WORKER_PROC_RE = /\b(comfy|main\.py|ai-toolkit|run\.py|llama-server|ollama|lane1_render\.py)\b/i;
+
+/**
+ * 1. Independent wall-clock watcher (v4k).
+ * Never disabled in `stub` mode: trips when `elapsedS >= wallS - graceS`.
+ *
+ * @param {Object} opts
+ * @param {string} [opts.mode='live']
+ * @param {number} opts.elapsedS
+ * @param {number} [opts.wallS=3000]
+ * @param {number} [opts.graceS=60]
+ * @returns {{ mode: string, tripped: boolean, code: string|null, remainingS: number, thresholdS: number }}
+ */
+export function evaluateWallGuard({
+  mode = 'live',
+  elapsedS,
+  wallS = 3000,
+  graceS = 60,
+} = {}) {
+  const thresholdS = Math.max(0, Number(wallS) - Number(graceS));
+  const remainingS = thresholdS - Number(elapsedS);
+  const tripped = Number(elapsedS) >= thresholdS;
+  return {
+    mode,
+    tripped,
+    code: tripped ? 'WALL_GUARD_TRIPPED' : null,
+    remainingS,
+    thresholdS,
+  };
+}
+
+/**
+ * 2. Independent CCU spend cap watcher (v4k).
+ * Never disabled in `stub` mode: computes `spend = max(startBalance - currentBalance, rateCcuPerHour * elapsedH)`
+ * and trips when `spend >= cap - margin`.
+ *
+ * @param {Object} opts
+ * @param {string} [opts.mode='live']
+ * @param {number} [opts.cap=0.05]
+ * @param {number} [opts.margin=0.01]
+ * @param {number|null} [opts.startBalance=null]
+ * @param {number|null} [opts.currentBalance=null]
+ * @param {number} [opts.rateCcuPerHour=0]
+ * @param {number} [opts.elapsedS=0]
+ * @returns {{ mode: string, tripped: boolean, code: string|null, spendCcu: number, ceilingCcu: number }}
+ */
+export function evaluateCapGuard({
+  mode = 'live',
+  cap = 0.05,
+  margin = 0.01,
+  startBalance = null,
+  currentBalance = null,
+  rateCcuPerHour = 0,
+  elapsedS = 0,
+} = {}) {
+  const balanceDrop =
+    startBalance !== null && currentBalance !== null
+      ? Math.max(0, Number(startBalance) - Number(currentBalance))
+      : 0;
+  const rateEst = Math.max(0, (Number(rateCcuPerHour) * Number(elapsedS)) / 3600);
+  const spendCcu = Math.max(balanceDrop, rateEst);
+  const ceilingCcu = Math.max(0, Number(cap) - Number(margin));
+  const tripped = spendCcu >= ceilingCcu;
+  return {
+    mode,
+    tripped,
+    code: tripped ? 'CAP_GUARD_TRIPPED' : null,
+    spendCcu: Number(spendCcu.toFixed(6)),
+    ceilingCcu: Number(ceilingCcu.toFixed(6)),
+  };
+}
+
+/**
+ * Parse `vitals.log` blocks (`=== VITALS <iso> ===`) emitted by the VM vitals sampler (v4p).
+ *
+ * @param {string} vitalsText
+ * @returns {Array<{ timestamp: string, phase: string|null, workerProcCount: number, rawBlock: string }>}
+ */
+export function parseVitalsLog(vitalsText = '') {
+  const blocks = String(vitalsText)
+    .split(/^===\s*VITALS\s+/m)
+    .map((b) => b.trim())
+    .filter(Boolean);
+
+  const samples = [];
+  for (const block of blocks) {
+    const lines = block.split(/\r?\n/);
+    const header = lines[0] || '';
+    const tsMatch = /^([^\s=]+)/.exec(header);
+    const timestamp = tsMatch ? tsMatch[1] : '';
+
+    let phase = null;
+    let workerProcCount = 0;
+    let inProc = false;
+
+    for (let i = 1; i < lines.length; i++) {
+      const line = lines[i];
+      if (line.startsWith('PROC:')) {
+        inProc = true;
+        continue;
+      }
+      if (line.startsWith('HARNESS:')) {
+        inProc = false;
+        const payload = line.slice('HARNESS:'.length).trim();
+        try {
+          const parsed = JSON.parse(payload);
+          phase = parsed.phase || null;
+        } catch {
+          const m = /"phase"\s*:\s*"([^"]+)"/.exec(payload) || /phase:\s*([a-z0-9_-]+)/i.exec(payload);
+          phase = m ? m[1] : null;
+        }
+        continue;
+      }
+      if (inProc && WORKER_PROC_RE.test(line)) {
+        workerProcCount++;
+      }
+    }
+
+    samples.push({
+      timestamp,
+      phase,
+      workerProcCount,
+      rawBlock: block,
+    });
+  }
+  return samples;
+}
+
+/**
+ * 3. Evaluate the `DEAD HARNESS` and `STALL` watcher over `vitals.log` + idle timer (v4p).
+ *
+ * Trips `DEAD_HARNESS_TRIPPED` when the last 2 vitals samples both show
+ * `phase === 'run'` (or `phase.startsWith('run')`) and `workerProcCount === 0`.
+ * Otherwise trips `STALL_GUARD_TRIPPED` when `idleSeconds >= stallS`.
+ *
+ * @param {Object} opts
+ * @param {string} opts.vitalsText
+ * @param {number} [opts.idleSeconds=0]
+ * @param {number} [opts.stallS=300]
+ * @returns {{ tripped: boolean, code: string|null, reason: string|null, samplesCount: number }}
+ */
+export function evaluateVitalsAndStallGuard({
+  vitalsText = '',
+  idleSeconds = 0,
+  stallS = 300,
+} = {}) {
+  const samples = parseVitalsLog(vitalsText);
+  if (samples.length >= 2) {
+    const lastTwo = samples.slice(-2);
+    const bothDeadInRun = lastTwo.every(
+      (s) => typeof s.phase === 'string' && s.phase.startsWith('run') && s.workerProcCount === 0
+    );
+    if (bothDeadInRun) {
+      return {
+        tripped: true,
+        code: 'DEAD_HARNESS_TRIPPED',
+        reason: '2 consecutive vitals samples in phase=run with 0 active worker processes',
+        samplesCount: samples.length,
+      };
+    }
+  }
+
+  if (Number(idleSeconds) >= Number(stallS)) {
+    return {
+      tripped: true,
+      code: 'STALL_GUARD_TRIPPED',
+      reason: `No output row or vitals progress for ${idleSeconds}s (stallS=${stallS}s)`,
+      samplesCount: samples.length,
+    };
+  }
+
+  return {
+    tripped: false,
+    code: null,
+    reason: null,
+    samplesCount: samples.length,
+  };
+}
+
+/**
+ * 4a. Host-RAM / cgroup limit pre-load projection check (v4q).
+ * Refuses any step whose projected anonymous host RAM (`(currentAnonMb + projectedDeltaMb) * safetyFactor`)
+ * exceeds `min(memTotalMb, cgroupMaxMb)`.
+ *
+ * @param {Object} opts
+ * @param {number} opts.currentAnonMb
+ * @param {number} opts.projectedDeltaMb
+ * @param {number} opts.memTotalMb
+ * @param {number|null} [opts.cgroupMaxMb=null]
+ * @param {number} [opts.safetyFactor=1.05]
+ * @returns {{ ok: true, projectedMb: number, limitMb: number }}
+ */
+export function checkHostRamProjection({
+  currentAnonMb,
+  projectedDeltaMb,
+  memTotalMb,
+  cgroupMaxMb = null,
+  safetyFactor = 1.05,
+}) {
+  const limits = [Number(memTotalMb)];
+  if (cgroupMaxMb !== null && cgroupMaxMb !== undefined && Number.isFinite(Number(cgroupMaxMb))) {
+    limits.push(Number(cgroupMaxMb));
+  }
+  const limitMb = Math.min(...limits);
+  const rawProjected = Number(currentAnonMb) + Number(projectedDeltaMb);
+  const projectedWithMargin = rawProjected * Number(safetyFactor);
+
+  if (projectedWithMargin > limitMb) {
+    throw inputError(
+      'HOST_RAM_PROJECTION_EXCEEDED',
+      `Projected anonymous host RAM ${projectedWithMargin.toFixed(1)} MB (${rawProjected} MB * ${safetyFactor}) exceeds host/cgroup ceiling ${limitMb} MB (v4q guard against kernel SIGKILL rc=-9).`
+    );
+  }
+
+  return {
+    ok: true,
+    projectedMb: Number(projectedWithMargin.toFixed(1)),
+    limitMb,
+  };
+}
+
+/**
+ * 4b. Between-item fetch (v4q): copies every completed file listed in `current.json`
+ * from `remoteDir` to `localDir` BEFORE `current.json` is overwritten for `nextItem`.
+ *
+ * @param {Object} opts
+ * @param {string} opts.remoteDir
+ * @param {string} opts.localDir
+ * @param {Object|null} [opts.nextItem=null]
+ * @returns {{ copiedFiles: string[], previousItem: Object|null }}
+ */
+export function runBetweenItemFetch({ remoteDir, localDir, nextItem = null }) {
+  mkdirSync(localDir, { recursive: true });
+  const currentJsonPath = join(remoteDir, 'current.json');
+  const copiedFiles = [];
+  let previousItem = null;
+
+  if (existsSync(currentJsonPath)) {
+    previousItem = JSON.parse(readFileSync(currentJsonPath, 'utf-8'));
+    const files = Array.isArray(previousItem?.files) ? previousItem.files : [];
+    for (const relFile of files) {
+      const src = join(remoteDir, relFile);
+      if (existsSync(src)) {
+        const dst = join(localDir, relFile);
+        mkdirSync(join(dst, '..'), { recursive: true });
+        copyFileSync(src, dst);
+        copiedFiles.push(relFile);
+      }
+    }
+    copyFileSync(currentJsonPath, join(localDir, 'current.json'));
+  }
+
+  if (nextItem) {
+    writeFileSync(currentJsonPath, JSON.stringify(nextItem, null, 2) + '\n', 'utf-8');
+  }
+
+  return { copiedFiles, previousItem };
+}

package/lib/export.js

@@ -40,7 +40,7 @@ async function sha256File(filePath) {
  * SDL-H6: validate that a record-relative asset_path does not escape
  * projectRoot. Rejects absolute paths outright (ASSET_PATH_INVALID) and
  * rejects any relative path that resolves outside projectRoot
- * (ASSET_PATH_TRAVERSAL — e.g. "../../../../Users/victim/.ssh/id_rsa").
+ * (ASSET_PATH_TRAVERSAL — e.g. "../../../../etc/passwd").
  * Returns the resolved absolute path on success.
  *
  * This was previously inlined here only (D-007) and never ported to the

package/lib/freeze-gate.js

@@ -27,6 +27,9 @@ import { readFreezeStatus } from './freeze-stamp.js';
 import { appendEvent, readEventsFor } from './freeze-events.js';
 import { inputError } from './errors.js';
 import { warn } from './log.js';
+import { assertDirectorRatifiedForFreeze } from './canon-review-gate.js';
+
+export { assertDirectorRatifiedForFreeze };
 
 /**
  * Best-effort freeze date for an entry: the `at` of its most recent freeze /

package/lib/generated-provenance.js

@@ -3,13 +3,117 @@
  * attached to records created from selected run/batch outputs.
  *
  * Normalizes source context (run vs batch) into a single shape
- * that traces back through brief → workflow → seed → config.
+ * that traces back through brief → workflow → seed → config,
+ * plus G4 (model licences), G5 (owner + CC-BY-4.0), and G6 (canon link).
  */
 
 import { readFileSync, existsSync } from 'node:fs';
 import { join } from 'node:path';
 import { createHash } from 'node:crypto';
 import { warn } from './log.js';
+import { STUDIO_OWNER, STUDIO_LICENCE } from './records.js';
+
+/**
+ * Resolve `{ canon_sha, entry_id, entry_sha256 }` from explicit metadata or
+ * the project's `canon-build/<canon_sha>/manifest.json` (`per_entity_hashes`).
+ */
+export function resolveCanonLink(
+  projectRoot,
+  {
+    canonSha = null,
+    entryId = null,
+    entrySha256 = null,
+    subjectId = null,
+    runManifest = null,
+    batchManifest = null,
+    item = null,
+    brief = null,
+  } = {},
+) {
+  let resolvedCanonSha =
+    canonSha ||
+    item?.canon_sha ||
+    runManifest?.canon_sha ||
+    batchManifest?.canon_sha ||
+    brief?.canon_sha ||
+    null;
+
+  if (!resolvedCanonSha && projectRoot) {
+    const currentPath = join(projectRoot, 'canon-build', 'CURRENT');
+    if (existsSync(currentPath)) {
+      try {
+        const raw = readFileSync(currentPath, 'utf-8').trim();
+        if (raw) resolvedCanonSha = raw;
+      } catch {
+        // best-effort read of CURRENT pointer
+      }
+    }
+  }
+
+  let resolvedEntryId =
+    entryId ||
+    item?.entry_id ||
+    runManifest?.entry_id ||
+    batchManifest?.entry_id ||
+    brief?.entry_id ||
+    subjectId ||
+    item?.subject_id ||
+    runManifest?.subject_id ||
+    batchManifest?.subject_id ||
+    brief?.subject_id ||
+    null;
+
+  let resolvedEntrySha =
+    entrySha256 ||
+    item?.entry_sha256 ||
+    runManifest?.entry_sha256 ||
+    batchManifest?.entry_sha256 ||
+    brief?.entry_sha256 ||
+    null;
+
+  if (resolvedCanonSha && projectRoot) {
+    const dirName = String(resolvedCanonSha).replace(/:/g, '-');
+    const manifestPath = existsSync(join(projectRoot, 'canon-build', String(resolvedCanonSha), 'manifest.json'))
+      ? join(projectRoot, 'canon-build', String(resolvedCanonSha), 'manifest.json')
+      : join(projectRoot, 'canon-build', dirName, 'manifest.json');
+    if (existsSync(manifestPath)) {
+      try {
+        const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8'));
+        const hashes = manifest.per_entity_hashes || {};
+        if (resolvedEntryId) {
+          if (hashes[resolvedEntryId]) {
+            resolvedEntrySha = resolvedEntrySha || hashes[resolvedEntryId];
+          } else {
+            const suffix = `/${resolvedEntryId}`;
+            const matchKey = Object.keys(hashes).find((k) => k.endsWith(suffix));
+            if (matchKey) {
+              resolvedEntryId = matchKey;
+              resolvedEntrySha = resolvedEntrySha || hashes[matchKey];
+            }
+          }
+        } else {
+          const keys = Object.keys(hashes);
+          if (keys.length === 1) {
+            resolvedEntryId = keys[0];
+            resolvedEntrySha = resolvedEntrySha || hashes[keys[0]];
+          }
+        }
+      } catch {
+        // best-effort manifest lookup
+      }
+    }
+  }
+
+  if (!resolvedCanonSha && !resolvedEntryId && !resolvedEntrySha) {
+    return null;
+  }
+
+  return {
+    canon_sha: resolvedCanonSha,
+    entry_id: resolvedEntryId,
+    entry_sha256: resolvedEntrySha,
+  };
+}
 
 /**
  * Build a generation provenance payload for a selected output.
@@ -21,6 +125,9 @@ import { warn } from './log.js';
  * @param {Object} [opts.batchManifest] — batch manifest if source is batch
  * @param {Object} opts.item — the selection item (slot_or_output, filename, seed)
  * @param {string} opts.projectRoot
+ * @param {string} [opts.canonSha]
+ * @param {string} [opts.entryId]
+ * @param {string} [opts.entrySha256]
  * @returns {Object} generation_provenance block
  */
 export function buildGeneratedProvenance({
@@ -30,8 +137,13 @@ export function buildGeneratedProvenance({
   batchManifest,
   item,
   projectRoot,
+  canonSha = null,
+  entryId = null,
+  entrySha256 = null,
 }) {
   const prov = {
+    owner: STUDIO_OWNER,
+    licence: STUDIO_LICENCE,
     source_type: sourceType,
     source_id: sourceId,
     run_id: runManifest.run_id,
@@ -45,12 +157,55 @@ export function buildGeneratedProvenance({
     prov.batch_id = batchManifest.batch_id;
   }
 
+  const matchedOutput = runManifest.outputs
+    ? runManifest.outputs.find((o) => o.filename === item.filename)
+    : null;
+
   // Seed from the item or from the run manifest outputs
   if (item.seed !== undefined) {
     prov.seed = item.seed;
-  } else if (runManifest.outputs) {
-    const match = runManifest.outputs.find(o => o.filename === item.filename);
-    if (match?.seed !== undefined) prov.seed = match.seed;
+  } else if (matchedOutput?.seed !== undefined) {
+    prov.seed = matchedOutput.seed;
+  }
+
+  // Prompt and prompt_sha256 when available on the item or output
+  const promptText = item.prompt ?? matchedOutput?.prompt ?? null;
+  if (typeof promptText === 'string' && promptText.length > 0) {
+    prov.prompt = promptText;
+    prov.prompt_sha256 = 'sha256:' + createHash('sha256').update(promptText).digest('hex');
+  } else if (item.prompt_sha256 || matchedOutput?.prompt_sha256) {
+    prov.prompt_sha256 = item.prompt_sha256 || matchedOutput.prompt_sha256;
+  }
+
+  // Comfy workflow sha and model licences from run manifest pinning (G4, G9)
+  const wfSha = runManifest.pinning?.comfy_workflow_sha ?? runManifest.comfy_workflow_sha ?? null;
+  if (wfSha) {
+    prov.comfy_workflow_sha = wfSha;
+  }
+  if (runManifest.pinning?.models || runManifest.pinning?.loras) {
+    prov.model_licences = {
+      cc_by_4_eligible: runManifest.pinning.cc_by_4_eligible ?? null,
+      models: Object.fromEntries(
+        Object.entries(runManifest.pinning.models || {}).map(([role, m]) => [
+          role,
+          {
+            name: m.name,
+            sha256: m.sha256 ?? null,
+            licence_id: m.licence_id ?? m.licence?.spdx_id ?? null,
+            licence_url: m.licence_url ?? m.licence?.url ?? null,
+            date_read: m.date_read ?? m.licence?.verified_at ?? null,
+          },
+        ]),
+      ),
+      loras: (runManifest.pinning.loras || []).map((l) => ({
+        name: l.name,
+        weight: l.weight ?? 1.0,
+        sha256: l.sha256 ?? null,
+        licence_id: l.licence_id ?? l.licence?.spdx_id ?? null,
+        licence_url: l.licence_url ?? l.licence?.url ?? null,
+        date_read: l.date_read ?? l.licence?.verified_at ?? null,
+      })),
+    };
   }
 
   // Adapter target from run manifest
@@ -58,15 +213,17 @@ export function buildGeneratedProvenance({
     prov.adapter_target = runManifest.adapter_target;
   }
 
+  let parsedBrief = null;
   // Subject from run brief or batch manifest
   if (batchManifest?.subject_id) {
     prov.subject_id = batchManifest.subject_id;
-  } else {
+  } else if (projectRoot && runManifest.brief_id) {
     // Try to read subject from brief
     const briefPath = join(projectRoot, 'briefs', `${runManifest.brief_id}.json`);
     if (existsSync(briefPath)) {
       try {
         const brief = JSON.parse(readFileSync(briefPath, 'utf-8'));
+        parsedBrief = brief;
         if (brief.subject_id) prov.subject_id = brief.subject_id;
         if (brief.parent_brief_id) prov.parent_brief_id = brief.parent_brief_id;
       } catch (err) {
@@ -87,6 +244,22 @@ export function buildGeneratedProvenance({
     }
   }
 
+  const canonLink = resolveCanonLink(projectRoot, {
+    canonSha,
+    entryId,
+    entrySha256,
+    subjectId: prov.subject_id,
+    runManifest,
+    batchManifest,
+    item,
+    brief: parsedBrief,
+  });
+  if (canonLink) {
+    prov.canon_sha = canonLink.canon_sha;
+    prov.entry_id = canonLink.entry_id;
+    prov.entry_sha256 = canonLink.entry_sha256;
+  }
+
   // Config fingerprint — hash the run manifest for reproducibility tracing
   prov.config_fingerprint = createHash('sha256')
     .update(JSON.stringify({