@aifabrix/builder 2.51.0 → 2.51.1

package/lib/role-assistant/test-cases-evaluate.js

@@ -0,0 +1,170 @@
+/**
+ * Role assistant test-case expectation evaluation.
+ * @fileoverview Evaluate lifecycle/capability/fact/honesty expectations for a case result
+ */
+
+'use strict';
+
+const {
+  buildHonestyObservation,
+  evaluateHonestyExpectations
+} = require('./test-cases-honesty');
+
+function getValueAtPath(obj, dottedPath) {
+  const parts = String(dottedPath || '').split('.').filter(Boolean);
+  let cursor = obj;
+  for (const key of parts) {
+    if (!cursor || typeof cursor !== 'object' || !(key in cursor)) {
+      return undefined;
+    }
+    cursor = cursor[key];
+  }
+  return cursor;
+}
+
+function evaluateLifecycleStateExpectation(expectedState, observedState) {
+  if (!expectedState) {
+    return null;
+  }
+  const expected = String(expectedState).trim();
+  const passed = expected === observedState;
+  return {
+    id: 'expected.lifecycleState',
+    passed,
+    message: passed
+      ? `Lifecycle state matched expected value "${expected}".`
+      : `Lifecycle state expected "${expected}" but got "${observedState}".`
+  };
+}
+
+function evaluateCapabilityExpectations(requiredList, observedCapabilities) {
+  if (!Array.isArray(requiredList) || requiredList.length === 0) {
+    return [];
+  }
+  const observedSet = new Set(observedCapabilities);
+  return requiredList.map((capability) => {
+    const key = String(capability || '').trim();
+    const passed = observedSet.has(key);
+    return {
+      id: `expected.capabilities.required:${key}`,
+      passed,
+      message: passed
+        ? `Required capability "${key}" observed.`
+        : `Required capability "${key}" was not observed.`
+    };
+  });
+}
+
+function evaluateFactExpectations(expectedFacts, observation) {
+  if (!expectedFacts || typeof expectedFacts !== 'object' || Array.isArray(expectedFacts)) {
+    return [];
+  }
+  return Object.entries(expectedFacts).map(([factPath, expectedValue]) => {
+    const actualValue = getValueAtPath(observation, factPath);
+    const passed = JSON.stringify(actualValue) === JSON.stringify(expectedValue);
+    return {
+      id: `expected.facts:${factPath}`,
+      passed,
+      message: passed
+        ? `Fact "${factPath}" matched expected value.`
+        : `Fact "${factPath}" expected ${JSON.stringify(expectedValue)} but got ${JSON.stringify(actualValue)}.`
+    };
+  });
+}
+
+function evaluateProhibitedExpectations(prohibited, observation) {
+  if (!Array.isArray(prohibited) || prohibited.length === 0) {
+    return [];
+  }
+  return prohibited.map((factPath) => {
+    const actualValue = getValueAtPath(observation, factPath);
+    const passed = actualValue === undefined || actualValue === null || actualValue === false;
+    return {
+      id: `expected.prohibited:${factPath}`,
+      passed,
+      message: passed
+        ? `Prohibited marker "${factPath}" was not observed.`
+        : `Prohibited marker "${factPath}" observed: ${JSON.stringify(actualValue)}.`
+    };
+  });
+}
+
+function buildCaseObservation(result, lifecycleVerdict, portalUrl) {
+  const {
+    deriveLifecycleState,
+    collectObservedCapabilities
+  } = require('./test-cases');
+  const observedState = deriveLifecycleState(result, lifecycleVerdict);
+  const observedCapabilities = collectObservedCapabilities(result);
+  const workResult =
+    result?.workResult && typeof result.workResult === 'object' ? result.workResult : null;
+  return {
+    lifecycleState: observedState,
+    verdict: lifecycleVerdict,
+    openApiPublished: result?.openApiPublished === true,
+    portalUrl,
+    report: result?.report || {},
+    stepsRun: Array.isArray(result?.stepsRun) ? result.stepsRun : [],
+    capabilities: { observed: observedCapabilities },
+    processExecutionId: result?.processExecutionId || null,
+    workResult,
+    workResultError: result?.workResultError || null,
+    honesty: buildHonestyObservation(workResult)
+  };
+}
+
+function softWrapLifecycleAssertion(lifecycleAssertion, soft) {
+  if (!lifecycleAssertion) {
+    return null;
+  }
+  if (!(soft && !lifecycleAssertion.passed)) {
+    return lifecycleAssertion;
+  }
+  return {
+    ...lifecycleAssertion,
+    passed: true,
+    soft: true,
+    warn: true,
+    message: `${lifecycleAssertion.message} (soft — warning only)`
+  };
+}
+
+function collectCaseAssertions(expected, observation) {
+  const soft = expected.soft === true;
+  const observedCapabilities = observation.capabilities?.observed || [];
+  const assertions = [];
+  const lifecycleAssertion = softWrapLifecycleAssertion(
+    evaluateLifecycleStateExpectation(expected.lifecycleState, observation.lifecycleState),
+    soft
+  );
+  if (lifecycleAssertion) {
+    assertions.push(lifecycleAssertion);
+  }
+  assertions.push(
+    ...evaluateCapabilityExpectations(expected?.capabilities?.required, observedCapabilities)
+  );
+  assertions.push(...evaluateFactExpectations(expected.facts, observation));
+  assertions.push(...evaluateProhibitedExpectations(expected.prohibited, observation));
+  assertions.push(...evaluateHonestyExpectations(expected.honesty, observation, { soft }));
+  return assertions;
+}
+
+function evaluateRoleAssistantTestCase(testCase, result, lifecycleVerdict, portalUrl) {
+  const observation = buildCaseObservation(result, lifecycleVerdict, portalUrl);
+  const assertions = collectCaseAssertions(testCase.expected || {}, observation);
+  const failed = assertions.filter((row) => !row.passed);
+  return {
+    caseId: testCase.caseId,
+    sourceFile: testCase.sourceFile,
+    passed: failed.length === 0,
+    lifecycleState: observation.lifecycleState,
+    assertions,
+    failedAssertions: failed,
+    warnings: assertions.filter((row) => row.warn === true),
+    observation
+  };
+}
+
+module.exports = {
+  evaluateRoleAssistantTestCase
+};

package/lib/role-assistant/test-cases-honesty.js

@@ -0,0 +1,184 @@
+/**
+ * Soft / structural honesty assertions for role-assistant test cases.
+ */
+
+'use strict';
+
+const HONESTY_COMPLETION_STATES = new Set([
+  'notEvaluated',
+  'waiting',
+  'completed',
+  'failed',
+  'cancelled',
+  'safeStopped'
+]);
+const HONESTY_GOAL_STATUSES = new Set([
+  'unverified',
+  'partiallySatisfied',
+  'satisfied',
+  'notSatisfied'
+]);
+const HONESTY_CONTRIB_STATUSES = new Set([
+  'notDefined',
+  'expected',
+  'pendingVerification',
+  'realized',
+  'notRealized',
+  'partial'
+]);
+
+function emptyHonestyObservation() {
+  return {
+    present: false,
+    schemaVersion: null,
+    effectiveDepth: null,
+    completionState: null,
+    goalStatus: null,
+    contributionStatus: null,
+    resourceChangeCount: 0,
+    relationshipChangeCount: 0
+  };
+}
+
+function asObject(value) {
+  return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
+}
+
+function buildHonestyObservation(workResult) {
+  if (!workResult || typeof workResult !== 'object') {
+    return emptyHonestyObservation();
+  }
+  const completion = asObject(workResult.completion);
+  const measurements = asObject(workResult.measurements);
+  const contribution = asObject(measurements.contributionObservation);
+  const businessChanges = asObject(workResult.businessChanges);
+  const resources = Array.isArray(businessChanges.resources) ? businessChanges.resources : [];
+  const relationships = Array.isArray(businessChanges.relationships)
+    ? businessChanges.relationships
+    : [];
+  return {
+    present: true,
+    schemaVersion: workResult.schemaVersion || null,
+    effectiveDepth: workResult.effectiveDepth || null,
+    completionState: completion.state || null,
+    goalStatus: completion.goalStatus || null,
+    contributionStatus: contribution.status || null,
+    resourceChangeCount: resources.length,
+    relationshipChangeCount: relationships.length
+  };
+}
+
+function pushHonestyAssertion(assertions, id, passed, message) {
+  assertions.push({ id, passed, message });
+}
+
+function assertWorkResultPresent(assertions, honesty, observation) {
+  const present = honesty.present === true;
+  const missingSuffix = observation.workResultError
+    ? `: ${observation.workResultError}`
+    : '.';
+  pushHonestyAssertion(
+    assertions,
+    'expected.honesty.requireWorkResult',
+    present,
+    present
+      ? 'work.result Output was present.'
+      : `work.result Output missing${missingSuffix}`
+  );
+}
+
+function assertSchemaVersion(assertions, honestySpec, honesty) {
+  if (!honestySpec.schemaVersion) {
+    return;
+  }
+  const expected = String(honestySpec.schemaVersion);
+  const passed = String(honesty.schemaVersion || '') === expected;
+  pushHonestyAssertion(
+    assertions,
+    'expected.honesty.schemaVersion',
+    passed,
+    passed
+      ? `schemaVersion matched "${expected}".`
+      : `schemaVersion expected "${expected}" but got ${JSON.stringify(honesty.schemaVersion)}.`
+  );
+}
+
+function assertCompletion(assertions, honestySpec, honesty) {
+  if (honestySpec.requireCompletion === false) {
+    return;
+  }
+  const stateOk = HONESTY_COMPLETION_STATES.has(String(honesty.completionState || ''));
+  const goalOk = HONESTY_GOAL_STATUSES.has(String(honesty.goalStatus || ''));
+  pushHonestyAssertion(
+    assertions,
+    'expected.honesty.completion',
+    stateOk && goalOk,
+    stateOk && goalOk
+      ? `completion.state/goalStatus honest (${honesty.completionState}/${honesty.goalStatus}).`
+      : `completion missing or invalid: state=${JSON.stringify(honesty.completionState)} goalStatus=${JSON.stringify(honesty.goalStatus)}.`
+  );
+}
+
+function assertContribution(assertions, honestySpec, honesty) {
+  if (honestySpec.requireContributionObservation === false) {
+    return;
+  }
+  const ok = HONESTY_CONTRIB_STATUSES.has(String(honesty.contributionStatus || ''));
+  pushHonestyAssertion(
+    assertions,
+    'expected.honesty.contributionObservation',
+    ok,
+    ok
+      ? `contributionObservation.status honest (${honesty.contributionStatus}).`
+      : `contributionObservation.status missing or invalid: ${JSON.stringify(honesty.contributionStatus)}.`
+  );
+}
+
+function assertMinResourceChanges(assertions, honestySpec, honesty, soft) {
+  if (typeof honestySpec.minResourceChanges !== 'number') {
+    return;
+  }
+  const count = Number(honesty.resourceChangeCount || 0);
+  const passed = count >= honestySpec.minResourceChanges;
+  const softChange = soft || honestySpec.softResourceChanges === true;
+  pushHonestyAssertion(
+    assertions,
+    'expected.honesty.minResourceChanges',
+    softChange ? true : passed,
+    passed
+      ? `resourceChangeCount ${count} >= ${honestySpec.minResourceChanges}.`
+      : `resourceChangeCount ${count} < ${honestySpec.minResourceChanges}${
+        softChange ? ' (soft — warning only)' : ''
+      }.`
+  );
+  if (softChange && !passed) {
+    assertions[assertions.length - 1].warn = true;
+    assertions[assertions.length - 1].soft = true;
+  }
+}
+
+function evaluateHonestyExpectations(honestySpec, observation, options = {}) {
+  if (!honestySpec || typeof honestySpec !== 'object' || Array.isArray(honestySpec)) {
+    return [];
+  }
+  const soft = options.soft === true;
+  const honesty = observation.honesty || {};
+  const assertions = [];
+  if (honestySpec.requireWorkResult !== false) {
+    assertWorkResultPresent(assertions, honesty, observation);
+  }
+  if (!honesty.present) {
+    // Missing Output is always a hard failure when required — soft does not waive it.
+    return assertions;
+  }
+  assertSchemaVersion(assertions, honestySpec, honesty);
+  assertCompletion(assertions, honestySpec, honesty);
+  assertContribution(assertions, honestySpec, honesty);
+  assertMinResourceChanges(assertions, honestySpec, honesty, soft);
+  return assertions;
+}
+
+module.exports = {
+  buildHonestyObservation,
+  evaluateHonestyExpectations
+};