@aifabrix/builder 2.53.1 → 2.55.0

package/lib/role-assistant/test-case-expected-catalog.js

@@ -0,0 +1,135 @@
+/**
+ * Nested expected-field catalog for Role Assistant YAML cases.
+ * @fileoverview Fail-closed coverage walk after AJV nested schema
+ */
+
+'use strict';
+
+const HONESTY_KEYS = [
+  'requireWorkResult',
+  'schemaVersion',
+  'requireCompletion',
+  'requireContributionObservation',
+  'minResourceChanges',
+  'softResourceChanges',
+  'forbidBuilderKeysInObjective'
+];
+const SEARCH_KEYS = [
+  'statusCode',
+  'operation',
+  'resultMode',
+  'minRows',
+  'errorContains',
+  'graphDepth',
+  'relatedTypes',
+  'related'
+];
+const SEARCH_RELATED_KEYS = ['minRows', 'resultMode'];
+const SKILLS_KEYS = ['resourceType', 'operations', 'filterFields', 'operationRequired'];
+const EVIDENCE_KEYS = ['requiredKeys', 'roleKey', 'resourceType', 'requireStatus'];
+const BID_KEYS = [
+  'schemaName',
+  'schemaVersion',
+  'requireDefinitionHash',
+  'requireSuiteRunId',
+  'forbidGoldPaths',
+  'minDatasources',
+  'requireCapability'
+];
+const EXPECTED_KEYS = [
+  'soft',
+  'softNotes',
+  'honesty',
+  'lifecycleState',
+  'capabilities',
+  'search',
+  'skills',
+  'evidence',
+  'bid',
+  'facts',
+  'prohibited'
+];
+const CAPABILITY_KEYS = ['allowed', 'required', 'forbidden'];
+const NESTED = {
+  'expected.honesty': HONESTY_KEYS,
+  'expected.search': SEARCH_KEYS,
+  'expected.skills': SKILLS_KEYS,
+  'expected.evidence': EVIDENCE_KEYS,
+  'expected.bid': BID_KEYS,
+  'expected.capabilities': CAPABILITY_KEYS,
+  'preflight.evidence': EVIDENCE_KEYS
+};
+
+function objectKeys(value) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return [];
+  }
+  return Object.keys(value);
+}
+
+function unknownKeyError(filePath, pointer, key, allowed) {
+  const allowedText = allowed.join(', ');
+  return (
+    `${filePath}: unknown field ${pointer}.${key}. ` +
+    `Allowed properties: ${allowedText}. ` +
+    'Remove the field or rename it to a listed property.'
+  );
+}
+
+function collectUnknownInObject(filePath, pointer, value, allowed, errors) {
+  for (const key of objectKeys(value)) {
+    if (!allowed.includes(key)) {
+      errors.push(unknownKeyError(filePath, pointer, key, allowed));
+    }
+  }
+}
+
+function collectSearchRelatedUnknown(filePath, related, errors) {
+  for (const resourceType of objectKeys(related)) {
+    collectUnknownInObject(
+      filePath,
+      `expected.search.related.${resourceType}`,
+      related[resourceType],
+      SEARCH_RELATED_KEYS,
+      errors
+    );
+  }
+}
+
+function collectNestedUnknown(filePath, parsed, errors) {
+  const expected = parsed.expected && typeof parsed.expected === 'object' ? parsed.expected : {};
+  collectUnknownInObject(filePath, 'expected', expected, EXPECTED_KEYS, errors);
+  collectUnknownInObject(filePath, 'expected.honesty', expected.honesty, HONESTY_KEYS, errors);
+  collectUnknownInObject(filePath, 'expected.search', expected.search, SEARCH_KEYS, errors);
+  collectSearchRelatedUnknown(filePath, expected.search && expected.search.related, errors);
+  collectUnknownInObject(filePath, 'expected.skills', expected.skills, SKILLS_KEYS, errors);
+  collectUnknownInObject(filePath, 'expected.evidence', expected.evidence, EVIDENCE_KEYS, errors);
+  collectUnknownInObject(filePath, 'expected.bid', expected.bid, BID_KEYS, errors);
+  collectUnknownInObject(
+    filePath,
+    'expected.capabilities',
+    expected.capabilities,
+    CAPABILITY_KEYS,
+    errors
+  );
+  const preflight = parsed.preflight && typeof parsed.preflight === 'object' ? parsed.preflight : {};
+  collectUnknownInObject(filePath, 'preflight.evidence', preflight.evidence, EVIDENCE_KEYS, errors);
+}
+
+function assertExpectedCoverage(parsed, filePath) {
+  const errors = [];
+  collectNestedUnknown(filePath, parsed || {}, errors);
+  return errors;
+}
+
+module.exports = {
+  HONESTY_KEYS,
+  SEARCH_KEYS,
+  SEARCH_RELATED_KEYS,
+  SKILLS_KEYS,
+  EVIDENCE_KEYS,
+  BID_KEYS,
+  EXPECTED_KEYS,
+  NESTED,
+  assertExpectedCoverage
+};

package/lib/role-assistant/test-case-validate.js

@@ -0,0 +1,88 @@
+/**
+ * Offline Role Assistant test-case schema validation.
+ * @fileoverview AJV check for YAML cases before auth or package publish
+ */
+
+'use strict';
+
+const path = require('path');
+const { parseJsonFileContent } = require('../validation/validate-external-file');
+const { loadRoleAssistantTestCaseSchema } = require('../utils/schema-loader');
+const { formatValidationErrors } = require('../utils/error-formatter');
+const { assertExpectedCoverage } = require('./test-case-expected-catalog');
+
+const TEST_CASE_TYPE = 'role-assistant-test-case';
+
+function formatCaseErrors(filePath, ajvErrors) {
+  return formatValidationErrors(ajvErrors, { dedupe: true }).map((msg) => {
+    const fix = /additional properties/i.test(msg)
+      ? ' Remove the field or rename it to a listed property.'
+      : '';
+    return `${filePath}: ${msg}${fix}`;
+  });
+}
+
+function validateTestCaseDocument(parsed, filePath) {
+  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+    return {
+      valid: false,
+      errors: [`${filePath}: Test case must be an object`],
+      warnings: []
+    };
+  }
+  const validate = loadRoleAssistantTestCaseSchema();
+  const ok = validate(parsed);
+  const errors = ok
+    ? assertExpectedCoverage(parsed, filePath)
+    : formatCaseErrors(filePath, validate.errors);
+  return { valid: errors.length === 0, errors, warnings: [] };
+}
+
+function validateTestCaseFile(filePath) {
+  const parseResult = parseJsonFileContent(filePath);
+  if (parseResult.error) {
+    return { ...parseResult.error, file: filePath, type: TEST_CASE_TYPE };
+  }
+  const result = validateTestCaseDocument(parseResult.parsed, filePath);
+  return { ...result, file: filePath, type: TEST_CASE_TYPE };
+}
+
+function assertTestCaseDocument(parsed, filePath) {
+  const result = validateTestCaseDocument(parsed, filePath);
+  if (!result.valid) {
+    throw new Error(result.errors.join('\n'));
+  }
+}
+
+function throwIfCaseFileInvalid(filePath) {
+  const result = validateTestCaseFile(filePath);
+  if (!result.valid) {
+    throw new Error(result.errors.join('\n'));
+  }
+}
+
+function assertLocalCasesBeforeAuth(systemKey, opts) {
+  if (!opts || opts.fromDataplane || opts.listCases) {
+    return;
+  }
+  const { listTestCaseFiles, loadRoleAssistantTestCase } = require('./test-cases');
+  if (opts.case) {
+    loadRoleAssistantTestCase(systemKey, opts.case, { testsDir: opts.testsDir || null });
+    return;
+  }
+  if (!opts.suite) {
+    return;
+  }
+  const testsDir = opts.testsDir || path.join(process.cwd(), 'role-assistant', systemKey, 'tests');
+  for (const filePath of listTestCaseFiles(testsDir)) {
+    throwIfCaseFileInvalid(filePath);
+  }
+}
+
+module.exports = {
+  TEST_CASE_TYPE,
+  validateTestCaseDocument,
+  validateTestCaseFile,
+  assertTestCaseDocument,
+  assertLocalCasesBeforeAuth
+};

package/lib/role-assistant/test-cases-bid.js

@@ -5,6 +5,8 @@
 
 'use strict';
 
+const { skipRow } = require('./test-case-assertion-row');
+
 function asArray(value) {
   return Array.isArray(value) ? value : [];
 }
@@ -71,10 +73,71 @@ function collectBidSchemaAssertions(expectedBid, observed, assertions, soft) {
   }
 }
 
-function assertDefinitionHash(expectedBid, observed, assertions, soft) {
-  if (expectedBid.requireDefinitionHash !== true) {
+function isDatasourceRecord(value) {
+  const key = String(value.key || value.datasourceKey || '').trim();
+  return Boolean(key && (Array.isArray(value.capabilities) || value.entityType || value.resourceType));
+}
+
+function nestedBidContainers(value) {
+  const nested = [];
+  if (Array.isArray(value.datasources)) {
+    nested.push(value.datasources);
+  }
+  if (value.spec && typeof value.spec === 'object') {
+    nested.push(value.spec);
+  }
+  if (value.emitted && typeof value.emitted === 'object') {
+    nested.push(value.emitted);
+  }
+  if (value.bid && typeof value.bid === 'object') {
+    nested.push(value.bid);
+  }
+  return nested;
+}
+
+function collectBidObjects(value, acc) {
+  if (Array.isArray(value)) {
+    value.forEach((item) => collectBidObjects(item, acc));
     return;
   }
+  if (!value || typeof value !== 'object') {
+    return;
+  }
+  nestedBidContainers(value).forEach((child) => collectBidObjects(child, acc));
+  if (isDatasourceRecord(value)) {
+    acc.push(value);
+  }
+}
+
+function bidBlob(observed) {
+  return observed.emitted || observed.emitRow || observed.applyRow || {};
+}
+
+function datasourceKeysFromBid(observed) {
+  const objects = [];
+  collectBidObjects(bidBlob(observed), objects);
+  return new Set(objects.map((row) => String(row.key || row.datasourceKey || '').trim()).filter(Boolean));
+}
+
+function datasourceHasCapability(observed, datasource, capKey) {
+  const objects = [];
+  collectBidObjects(bidBlob(observed), objects);
+  const match = objects.find((row) => String(row.key || row.datasourceKey || '') === datasource);
+  const caps = asArray(match && match.capabilities);
+  return caps.some((cap) => String((cap && cap.key) || cap || '') === capKey);
+}
+
+function pushBooleanBidFlag(expectedBid, field, observed, assertions, soft, assertFn) {
+  if (expectedBid[field] === false) {
+    assertions.push(skipRow(`expected.bid.${field}`, `${field} is false — check skipped.`));
+    return;
+  }
+  if (expectedBid[field] === true) {
+    assertFn(expectedBid, observed, assertions, soft);
+  }
+}
+
+function assertDefinitionHash(expectedBid, observed, assertions, soft) {
   const { emitRow, applyRow, emitted } = observed;
   const hash =
     (emitted && emitted.definitionHash) ||
@@ -112,11 +175,60 @@ function assertSuiteRunId(expectedBid, observed, assertions, soft) {
 }
 
 function collectBidProvenanceAssertions(expectedBid, observed, assertions, soft) {
-  assertDefinitionHash(expectedBid, observed, assertions, soft);
-  assertSuiteRunId(expectedBid, observed, assertions, soft);
+  pushBooleanBidFlag(expectedBid, 'requireDefinitionHash', observed, assertions, soft, assertDefinitionHash);
+  pushBooleanBidFlag(expectedBid, 'requireSuiteRunId', observed, assertions, soft, assertSuiteRunId);
+}
+
+function collectBidDatasourceAssertions(expectedBid, observed, assertions, soft) {
+  if (Array.isArray(expectedBid.minDatasources)) {
+    collectMinDatasources(expectedBid.minDatasources, observed, assertions, soft);
+  }
+  if (expectedBid.requireCapability && typeof expectedBid.requireCapability === 'object') {
+    collectRequireCapability(expectedBid.requireCapability, observed, assertions, soft);
+  }
+}
+
+function collectMinDatasources(requiredKeys, observed, assertions, soft) {
+  if (requiredKeys.length === 0) {
+    assertions.push(skipRow('expected.bid.minDatasources', 'minDatasources is empty — nothing to assert.'));
+    return;
+  }
+  const actual = datasourceKeysFromBid(observed);
+  for (const raw of requiredKeys) {
+    const key = String(raw || '').trim();
+    const passed = hasEmitOrApply(observed) && actual.has(key);
+    pushBidAssertion(
+      assertions,
+      soft,
+      `expected.bid.minDatasources:${key}`,
+      passed,
+      `BID lists datasource "${key}".`,
+      `BID does not list required datasource "${key}".`
+    );
+  }
+}
+
+function collectRequireCapability(spec, observed, assertions, soft) {
+  const datasource = String(spec.datasource || '').trim();
+  const capKey = String(spec.key || '').trim();
+  const passed = hasEmitOrApply(observed) && datasourceHasCapability(observed, datasource, capKey);
+  pushBidAssertion(
+    assertions,
+    soft,
+    'expected.bid.requireCapability',
+    passed,
+    `BID datasource "${datasource}" lists capability "${capKey}".`,
+    `BID datasource "${datasource}" does not list capability "${capKey}".`
+  );
 }
 
 function collectBidPathAssertions(expectedBid, observed, assertions, soft) {
+  if (expectedBid.forbidGoldPaths === false) {
+    assertions.push(
+      skipRow('expected.bid.forbidGoldPaths', 'forbidGoldPaths is false — gold-path check skipped.')
+    );
+    return;
+  }
   if (expectedBid.forbidGoldPaths === true) {
     const { emitRow, applyRow, emitted } = observed;
     const blob = JSON.stringify(emitted || emitRow || applyRow || {});
@@ -146,6 +258,7 @@ function collectBidAssertions(expectedBid, observation, options = {}) {
   const soft = options.soft === true;
   collectBidSchemaAssertions(expectedBid, observed, assertions, soft);
   collectBidProvenanceAssertions(expectedBid, observed, assertions, soft);
+  collectBidDatasourceAssertions(expectedBid, observed, assertions, soft);
   collectBidPathAssertions(expectedBid, observed, assertions, soft);
   return assertions;
 }

package/lib/role-assistant/test-cases-evaluate.js

@@ -10,12 +10,49 @@ const {
   evaluateHonestyExpectations
 } = require('./test-cases-honesty');
 const { collectBidAssertions } = require('./test-cases-bid');
+const {
+  evaluateEvidenceExpectations,
+  evaluateForbiddenCapabilities
+} = require('./test-cases-evidence');
+const {
+  evaluateSearchExpectations,
+  evaluateSkillsExpectations
+} = require('./test-cases-search');
+const { notesRow, withKind } = require('./test-case-assertion-row');
+
+const UNSAFE_FACT_SEGMENTS = new Set(['__proto__', 'prototype', 'constructor']);
+const WILDCARD_FACT_SEGMENTS = new Set(['*', '**']);
+
+function splitDottedPath(dottedPath) {
+  return String(dottedPath || '').split('.').filter((part) => part.length > 0);
+}
+
+function factPathSafetyMessage(parts) {
+  for (const part of parts) {
+    if (UNSAFE_FACT_SEGMENTS.has(part)) {
+      return `Fact path rejects prototype segment "${part}" before lookup.`;
+    }
+    if (WILDCARD_FACT_SEGMENTS.has(part)) {
+      return `Fact path rejects wildcard segment "${part}".`;
+    }
+    if (part === 'latest') {
+      return 'Fact path rejects latest-global record lookup.';
+    }
+  }
+  return null;
+}
 
 function getValueAtPath(obj, dottedPath) {
-  const parts = String(dottedPath || '').split('.').filter(Boolean);
+  const parts = splitDottedPath(dottedPath);
+  if (factPathSafetyMessage(parts)) {
+    return undefined;
+  }
   let cursor = obj;
   for (const key of parts) {
-    if (!cursor || typeof cursor !== 'object' || !(key in cursor)) {
+    if (cursor === null || cursor === undefined || typeof cursor !== 'object') {
+      return undefined;
+    }
+    if (!Object.prototype.hasOwnProperty.call(cursor, key)) {
       return undefined;
     }
     cursor = cursor[key];
@@ -23,6 +60,10 @@ function getValueAtPath(obj, dottedPath) {
   return cursor;
 }
 
+function stringifyObserved(value) {
+  return value === undefined ? 'missing' : JSON.stringify(value);
+}
+
 function evaluateLifecycleStateExpectation(expectedState, observedState) {
   if (!expectedState) {
     return null;
@@ -38,7 +79,7 @@ function evaluateLifecycleStateExpectation(expectedState, observedState) {
   };
 }
 
-function evaluateCapabilityExpectations(requiredList, observedCapabilities) {
+function evaluateCapabilityExpectations(kind, requiredList, observedCapabilities) {
   if (!Array.isArray(requiredList) || requiredList.length === 0) {
     return [];
   }
@@ -47,29 +88,39 @@ function evaluateCapabilityExpectations(requiredList, observedCapabilities) {
     const key = String(capability || '').trim();
     const passed = observedSet.has(key);
     return {
-      id: `expected.capabilities.required:${key}`,
+      id: `expected.capabilities.${kind}:${key}`,
       passed,
       message: passed
-        ? `Required capability "${key}" observed.`
-        : `Required capability "${key}" was not observed.`
+        ? `${kind === 'allowed' ? 'Allowed' : 'Required'} capability "${key}" observed.`
+        : `${kind === 'allowed' ? 'Allowed' : 'Required'} capability "${key}" was not observed.`
     };
   });
 }
 
+function evaluateOneFact(factPath, expectedValue, observation) {
+  const parts = splitDottedPath(factPath);
+  const safety = factPathSafetyMessage(parts);
+  const id = `expected.facts:${factPath}`;
+  if (safety) {
+    return { id, passed: false, message: safety };
+  }
+  const actualValue = getValueAtPath(observation, factPath);
+  const passed = JSON.stringify(actualValue) === JSON.stringify(expectedValue);
+  return {
+    id,
+    passed,
+    message: passed
+      ? `Fact "${factPath}" matched expected value.`
+      : `Fact "${factPath}" expected ${JSON.stringify(expectedValue)} but got ${stringifyObserved(actualValue)}.`
+  };
+}
+
 function evaluateFactExpectations(expectedFacts, observation) {
   if (!expectedFacts || typeof expectedFacts !== 'object' || Array.isArray(expectedFacts)) {
     return [];
   }
   return Object.entries(expectedFacts).map(([factPath, expectedValue]) => {
-    const actualValue = getValueAtPath(observation, factPath);
-    const passed = JSON.stringify(actualValue) === JSON.stringify(expectedValue);
-    return {
-      id: `expected.facts:${factPath}`,
-      passed,
-      message: passed
-        ? `Fact "${factPath}" matched expected value.`
-        : `Fact "${factPath}" expected ${JSON.stringify(expectedValue)} but got ${JSON.stringify(actualValue)}.`
-    };
+    return evaluateOneFact(factPath, expectedValue, observation);
   });
 }
 
@@ -78,6 +129,11 @@ function evaluateProhibitedExpectations(prohibited, observation) {
     return [];
   }
   return prohibited.map((factPath) => {
+    const parts = splitDottedPath(factPath);
+    const safety = factPathSafetyMessage(parts);
+    if (safety) {
+      return { id: `expected.prohibited:${factPath}`, passed: false, message: safety };
+    }
     const actualValue = getValueAtPath(observation, factPath);
     const passed = actualValue === undefined || actualValue === null || actualValue === false;
     return {
@@ -85,7 +141,7 @@ function evaluateProhibitedExpectations(prohibited, observation) {
       passed,
       message: passed
         ? `Prohibited marker "${factPath}" was not observed.`
-        : `Prohibited marker "${factPath}" observed: ${JSON.stringify(actualValue)}.`
+        : `Prohibited marker "${factPath}" observed: ${stringifyObserved(actualValue)}.`
     };
   });
 }
@@ -111,6 +167,10 @@ function buildCaseObservation(result, lifecycleVerdict, portalUrl) {
     workResult,
     workResultError: result?.workResultError || null,
     honesty: buildHonestyObservation(workResult),
+    evidencePreflight: result?.evidencePreflight || null,
+    search: result?.search || null,
+    skills: result?.skills || null,
+    roleRef: result?.roleRef || null,
     interactions: Array.isArray(result?.interactions)
       ? result.interactions
       : Array.isArray(result?.workInteractions)
@@ -136,30 +196,103 @@ function softWrapLifecycleAssertion(lifecycleAssertion, soft) {
   };
 }
 
-function collectCaseAssertions(expected, observation) {
+function softWrapFailedAssertion(assertion, soft) {
+  return softWrapLifecycleAssertion(assertion, soft);
+}
+
+function collectSoftModifierRows(expected) {
+  const rows = [];
+  if (Object.prototype.hasOwnProperty.call(expected, 'soft')) {
+    rows.push(
+      notesRow(
+        'expected.soft',
+        expected.soft === true
+          ? 'soft: true — failures become warnings'
+          : 'soft: false — failures are blocking'
+      )
+    );
+  }
+  const notes = Array.isArray(expected.softNotes) ? expected.softNotes : [];
+  notes.forEach((item, index) => {
+    const text = typeof item === 'string'
+      ? item
+      : String((item && (item.description || item.note)) || JSON.stringify(item));
+    rows.push(notesRow(`expected.softNotes[${index}]`, text));
+  });
+  return rows;
+}
+
+function collectEvidenceContextRows(spec, prefix) {
+  if (!spec || typeof spec !== 'object' || Array.isArray(spec)) {
+    return [];
+  }
+  const rows = [];
+  if (spec.roleKey) {
+    rows.push(notesRow(`${prefix}.roleKey`, `Evidence roleKey "${spec.roleKey}" (preflight context).`));
+  }
+  if (spec.resourceType) {
+    rows.push(
+      notesRow(`${prefix}.resourceType`, `Evidence resourceType "${spec.resourceType}" (preflight context).`)
+    );
+  }
+  if (spec.requireStatus) {
+    rows.push(
+      notesRow(`${prefix}.requireStatus`, `Evidence requireStatus "${spec.requireStatus}" (preflight context).`)
+    );
+  }
+  return rows;
+}
+
+function collectCapabilityAssertions(capabilities, observedCapabilities, soft) {
+  const spec = capabilities && typeof capabilities === 'object' ? capabilities : {};
+  const rows = [
+    ...evaluateCapabilityExpectations('required', spec.required, observedCapabilities),
+    ...evaluateCapabilityExpectations('allowed', spec.allowed, observedCapabilities),
+    ...evaluateForbiddenCapabilities(spec.forbidden, observedCapabilities)
+  ];
+  return rows.map((row) => softWrapFailedAssertion(row, soft));
+}
+
+function collectCaseAssertions(expected, observation, preflight = {}) {
   const soft = expected.soft === true;
   const observedCapabilities = observation.capabilities?.observed || [];
   const assertions = [];
-  const lifecycleAssertion = softWrapLifecycleAssertion(
+  const lifecycleAssertion = softWrapFailedAssertion(
     evaluateLifecycleStateExpectation(expected.lifecycleState, observation.lifecycleState),
     soft
   );
   if (lifecycleAssertion) {
     assertions.push(lifecycleAssertion);
   }
+  assertions.push(...collectSoftModifierRows(expected));
+  assertions.push(...collectCapabilityAssertions(expected.capabilities, observedCapabilities, soft));
   assertions.push(
-    ...evaluateCapabilityExpectations(expected?.capabilities?.required, observedCapabilities)
+    ...evaluateFactExpectations(expected.facts, observation).map((row) =>
+      softWrapFailedAssertion(row, soft)
+    )
+  );
+  assertions.push(
+    ...evaluateProhibitedExpectations(expected.prohibited, observation).map((row) =>
+      softWrapFailedAssertion(row, soft)
+    )
   );
-  assertions.push(...evaluateFactExpectations(expected.facts, observation));
-  assertions.push(...evaluateProhibitedExpectations(expected.prohibited, observation));
   assertions.push(...evaluateHonestyExpectations(expected.honesty, observation, { soft }));
   assertions.push(...collectBidAssertions(expected.bid, observation, { soft }));
-  return assertions;
+  assertions.push(...evaluateEvidenceExpectations(expected.evidence, observation));
+  assertions.push(...collectEvidenceContextRows(expected.evidence, 'expected.evidence'));
+  assertions.push(...collectEvidenceContextRows(preflight.evidence, 'preflight.evidence'));
+  assertions.push(...evaluateSearchExpectations(expected.search, observation));
+  assertions.push(...evaluateSkillsExpectations(expected.skills, observation));
+  return assertions.map(withKind);
 }
 
 function evaluateRoleAssistantTestCase(testCase, result, lifecycleVerdict, portalUrl) {
   const observation = buildCaseObservation(result, lifecycleVerdict, portalUrl);
-  const assertions = collectCaseAssertions(testCase.expected || {}, observation);
+  const assertions = collectCaseAssertions(
+    testCase.expected || {},
+    observation,
+    testCase.preflight || {}
+  );
   const failed = assertions.filter((row) => !row.passed);
   return {
     caseId: testCase.caseId,
@@ -174,5 +307,7 @@ function evaluateRoleAssistantTestCase(testCase, result, lifecycleVerdict, porta
 }
 
 module.exports = {
-  evaluateRoleAssistantTestCase
+  evaluateRoleAssistantTestCase,
+  evaluateFactExpectations,
+  getValueAtPath
 };