outcome-cli 1.0.0

package/src/eval/evaluateOutcome.property.test.ts

@@ -0,0 +1,1174 @@
+/**
+ * Property-based tests for Evaluation System
+ *
+ * **Feature: earnd-bounty-engine, Property 3: Evaluation Determinism**
+ * **Validates: Requirements 1.3**
+ *
+ * **Feature: earnd-bounty-engine, Property 10: Evaluation Result Structure**
+ * **Validates: Requirements 5.1, 5.2, 5.3**
+ *
+ * **Feature: earnd-bounty-engine, Property 19: Qualified Sales Interest Payout**
+ * **Validates: Requirements 8.6**
+ *
+ * Property 3: Evaluation Determinism
+ * *For any* outcome and artifact pair, evaluating the same inputs multiple times
+ * SHALL always produce the same result (SUCCESS or FAILURE with identical reason).
+ *
+ * Property 10: Evaluation Result Structure
+ * *For any* evaluation result, the status SHALL be exactly 'SUCCESS' or 'FAILURE',
+ * FAILURE results SHALL have a non-empty reason, and SUCCESS results SHALL have
+ * verificationDetails populated.
+ *
+ * Property 19: Qualified Sales Interest Payout
+ * *For any* artifact that passes all 5 criteria (buying intent, company size, role,
+ * message length, email), the evaluation SHALL return SUCCESS with payoutAmount of 250.
+ */
+
+import { describe, test, expect } from 'vitest';
+import * as fc from 'fast-check';
+import { evaluateOutcome, type AgentArtifact } from './evaluateOutcome.js';
+import type { Outcome, SuccessCriterion } from '../outcomes/outcome.schema.js';
+
+/**
+ * Arbitrary for generating valid SuccessCriterion objects.
+ * Uses validators that exist in the validatorMap.
+ */
+const successCriterionArb = fc.oneof(
+  fc.record({
+    name: fc.string({ minLength: 1 }),
+    validator: fc.constant('validateBuyingIntent'),
+    params: fc.record({
+      keywords: fc.array(fc.string({ minLength: 1 }), { minLength: 1, maxLength: 5 }),
+    }),
+  }),
+  fc.record({
+    name: fc.string({ minLength: 1 }),
+    validator: fc.constant('validateCompanySize'),
+    params: fc.record({
+      minimum: fc.integer({ min: 1, max: 10000 }),
+    }),
+  }),
+  fc.record({
+    name: fc.string({ minLength: 1 }),
+    validator: fc.constant('validateRole'),
+    params: fc.record({
+      excludedRoles: fc.array(fc.string({ minLength: 1 }), { minLength: 1, maxLength: 5 }),
+    }),
+  }),
+  fc.record({
+    name: fc.string({ minLength: 1 }),
+    validator: fc.constant('validateMessageLength'),
+    params: fc.record({
+      minWords: fc.integer({ min: 1, max: 100 }),
+    }),
+  }),
+  fc.record({
+    name: fc.string({ minLength: 1 }),
+    validator: fc.constant('validateEmail'),
+    params: fc.constant({}),
+  })
+) as fc.Arbitrary<SuccessCriterion>;
+
+/**
+ * Arbitrary for generating valid Outcome objects.
+ */
+const outcomeArb: fc.Arbitrary<Outcome> = fc.record({
+  name: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+  description: fc.string(),
+  payoutAmount: fc.double({ min: 0.01, max: 10000, noNaN: true }),
+  maxAttempts: fc.integer({ min: 1, max: 100 }),
+  timeLimitMs: fc.integer({ min: 1, max: 3600000 }),
+  successCriteria: fc.array(successCriterionArb, { minLength: 1, maxLength: 5 }),
+  failureReasons: fc.array(fc.string(), { minLength: 0, maxLength: 10 }),
+});
+
+/**
+ * Arbitrary for generating valid AgentArtifact objects.
+ */
+const artifactContentArb = fc.record({
+  message: fc.string(),
+  targetEmail: fc.string(),
+  targetCompany: fc.string(),
+  targetCompanySize: fc.integer({ min: 0, max: 1000000 }),
+  targetRole: fc.string(),
+});
+
+/**
+ * Generate valid ISO timestamp strings using integer timestamps.
+ * Avoids invalid date issues with fc.date().
+ */
+const validTimestampArb = fc
+  .integer({ min: 1577836800000, max: 1924905600000 }) // 2020-01-01 to 2030-12-31
+  .map((ts) => new Date(ts).toISOString());
+
+const agentArtifactArb = (outcomeName: string): fc.Arbitrary<AgentArtifact> =>
+  fc.record({
+    agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+    outcomeId: fc.constant(outcomeName),
+    attemptNumber: fc.integer({ min: 1, max: 100 }),
+    content: artifactContentArb,
+    timestamp: validTimestampArb,
+  });
+
+/**
+ * Arbitrary for generating outcome and matching artifact pairs.
+ */
+const outcomeArtifactPairArb = outcomeArb.chain((outcome) =>
+  agentArtifactArb(outcome.name).map((artifact) => ({ outcome, artifact }))
+);
+
+describe('Evaluation Determinism - Property Tests', () => {
+  // **Feature: earnd-bounty-engine, Property 3: Evaluation Determinism**
+  test('same inputs produce identical results on multiple evaluations', async () => {
+    await fc.assert(
+      fc.asyncProperty(outcomeArtifactPairArb, async ({ outcome, artifact }) => {
+        // Evaluate the same inputs multiple times
+        const result1 = await evaluateOutcome(outcome, artifact);
+        const result2 = await evaluateOutcome(outcome, artifact);
+        const result3 = await evaluateOutcome(outcome, artifact);
+
+        // All results must have identical status
+        expect(result1.status).toBe(result2.status);
+        expect(result2.status).toBe(result3.status);
+
+        // All results must have identical reason
+        expect(result1.reason).toBe(result2.reason);
+        expect(result2.reason).toBe(result3.reason);
+
+        // All results must have identical criteriaResults
+        expect(result1.criteriaResults).toEqual(result2.criteriaResults);
+        expect(result2.criteriaResults).toEqual(result3.criteriaResults);
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 3: Evaluation Determinism**
+  test('evaluation result is always binary (SUCCESS or FAILURE)', async () => {
+    await fc.assert(
+      fc.asyncProperty(outcomeArtifactPairArb, async ({ outcome, artifact }) => {
+        const result = await evaluateOutcome(outcome, artifact);
+
+        // Status must be exactly SUCCESS or FAILURE
+        expect(['SUCCESS', 'FAILURE']).toContain(result.status);
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 3: Evaluation Determinism**
+  test('evaluation result structure is consistent', async () => {
+    await fc.assert(
+      fc.asyncProperty(outcomeArtifactPairArb, async ({ outcome, artifact }) => {
+        const result = await evaluateOutcome(outcome, artifact);
+
+        // Result must have required fields
+        expect(typeof result.status).toBe('string');
+        expect(typeof result.reason).toBe('string');
+        expect(Array.isArray(result.criteriaResults)).toBe(true);
+
+        // Reason must not be empty
+        expect(result.reason.length).toBeGreaterThan(0);
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 3: Evaluation Determinism**
+  test('SUCCESS results include verificationDetails', async () => {
+    // Generate artifacts that are likely to pass all criteria
+    const passingArtifactArb = (outcomeName: string): fc.Arbitrary<AgentArtifact> =>
+      fc.record({
+        agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+        outcomeId: fc.constant(outcomeName),
+        attemptNumber: fc.integer({ min: 1, max: 100 }),
+        content: fc.record({
+          // Include buying intent keywords
+          message: fc
+            .array(fc.string({ minLength: 1 }).filter((s) => !/\s/.test(s)), {
+              minLength: 20,
+              maxLength: 50,
+            })
+            .map((words) => [...words, 'pricing', 'demo'].join(' ')),
+          // Valid email format
+          targetEmail: fc.constant('test@example.com'),
+          targetCompany: fc.string({ minLength: 1 }),
+          // Large company size
+          targetCompanySize: fc.integer({ min: 100, max: 10000 }),
+          // Valid role (not intern or student)
+          targetRole: fc.constantFrom('Manager', 'Director', 'VP', 'CEO', 'Engineer'),
+        }),
+        timestamp: validTimestampArb,
+      });
+
+    // Create a simple outcome with known validators
+    const simpleOutcome: Outcome = {
+      name: 'test_outcome',
+      description: 'Test outcome for determinism',
+      payoutAmount: 100,
+      maxAttempts: 3,
+      timeLimitMs: 60000,
+      successCriteria: [
+        {
+          name: 'buying_intent',
+          validator: 'validateBuyingIntent',
+          params: { keywords: ['pricing', 'demo', 'next steps'] },
+        },
+        {
+          name: 'company_size',
+          validator: 'validateCompanySize',
+          params: { minimum: 50 },
+        },
+        {
+          name: 'valid_role',
+          validator: 'validateRole',
+          params: { excludedRoles: ['intern', 'student'] },
+        },
+        {
+          name: 'message_length',
+          validator: 'validateMessageLength',
+          params: { minWords: 20 },
+        },
+        {
+          name: 'valid_email',
+          validator: 'validateEmail',
+          params: {},
+        },
+      ],
+      failureReasons: ['No buying intent', 'Company too small'],
+    };
+
+    await fc.assert(
+      fc.asyncProperty(passingArtifactArb(simpleOutcome.name), async (artifact) => {
+        const result = await evaluateOutcome(simpleOutcome, artifact);
+
+        if (result.status === 'SUCCESS') {
+          // SUCCESS results must have verificationDetails
+          expect(result.verificationDetails).toBeDefined();
+          expect(typeof result.verificationDetails).toBe('object');
+        }
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 3: Evaluation Determinism**
+  test('FAILURE results do not include verificationDetails', async () => {
+    // Generate artifacts that will fail (invalid email)
+    const failingArtifactArb = (outcomeName: string): fc.Arbitrary<AgentArtifact> =>
+      fc.record({
+        agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+        outcomeId: fc.constant(outcomeName),
+        attemptNumber: fc.integer({ min: 1, max: 100 }),
+        content: fc.record({
+          message: fc.constant('short'),
+          targetEmail: fc.constant('invalid-email'),
+          targetCompany: fc.string({ minLength: 1 }),
+          targetCompanySize: fc.integer({ min: 0, max: 10 }),
+          targetRole: fc.constant('intern'),
+        }),
+        timestamp: validTimestampArb,
+      });
+
+    const simpleOutcome: Outcome = {
+      name: 'test_outcome',
+      description: 'Test outcome',
+      payoutAmount: 100,
+      maxAttempts: 3,
+      timeLimitMs: 60000,
+      successCriteria: [
+        {
+          name: 'valid_email',
+          validator: 'validateEmail',
+          params: {},
+        },
+      ],
+      failureReasons: ['Invalid email'],
+    };
+
+    await fc.assert(
+      fc.asyncProperty(failingArtifactArb(simpleOutcome.name), async (artifact) => {
+        const result = await evaluateOutcome(simpleOutcome, artifact);
+
+        if (result.status === 'FAILURE') {
+          // FAILURE results should not have verificationDetails
+          expect(result.verificationDetails).toBeUndefined();
+        }
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 3: Evaluation Determinism**
+  test('criteriaResults count matches successCriteria count for valid artifacts', async () => {
+    await fc.assert(
+      fc.asyncProperty(outcomeArtifactPairArb, async ({ outcome, artifact }) => {
+        const result = await evaluateOutcome(outcome, artifact);
+
+        // If artifact schema is valid and outcome matches, criteriaResults should match criteria count
+        if (
+          result.reason.indexOf('Invalid artifact') === -1 &&
+          result.reason.indexOf('outcome mismatch') === -1
+        ) {
+          expect(result.criteriaResults.length).toBe(outcome.successCriteria.length);
+        }
+      }),
+      { numRuns: 100 }
+    );
+  });
+});
+
+
+/**
+ * Property 10: Evaluation Result Structure
+ *
+ * **Feature: earnd-bounty-engine, Property 10: Evaluation Result Structure**
+ * **Validates: Requirements 5.1, 5.2, 5.3**
+ *
+ * *For any* evaluation result, the status SHALL be exactly 'SUCCESS' or 'FAILURE',
+ * FAILURE results SHALL have a non-empty reason, and SUCCESS results SHALL have
+ * verificationDetails populated.
+ */
+describe('Evaluation Result Structure - Property Tests', () => {
+  /**
+   * Arbitrary for generating artifacts that will definitely pass all criteria.
+   * Used to test SUCCESS result structure.
+   */
+  const guaranteedPassingArtifactArb = (outcomeName: string): fc.Arbitrary<AgentArtifact> =>
+    fc.record({
+      agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+      outcomeId: fc.constant(outcomeName),
+      attemptNumber: fc.integer({ min: 1, max: 100 }),
+      content: fc.record({
+        // Message with 25+ words including buying intent keywords
+        message: fc
+          .array(fc.constantFrom('word', 'test', 'content', 'data', 'info'), {
+            minLength: 22,
+            maxLength: 30,
+          })
+          .map((words) => [...words, 'pricing', 'demo', 'next steps'].join(' ')),
+        // Valid email format
+        targetEmail: fc.constantFrom('user@example.com', 'test@company.org', 'lead@business.net'),
+        targetCompany: fc.string({ minLength: 1 }),
+        // Company size >= 50
+        targetCompanySize: fc.integer({ min: 50, max: 10000 }),
+        // Valid role (not intern or student)
+        targetRole: fc.constantFrom('Manager', 'Director', 'VP', 'CEO', 'Engineer', 'Developer'),
+      }),
+      timestamp: validTimestampArb,
+    });
+
+  /**
+   * Arbitrary for generating artifacts that will definitely fail at least one criterion.
+   * Used to test FAILURE result structure.
+   */
+  const guaranteedFailingArtifactArb = (outcomeName: string): fc.Arbitrary<AgentArtifact> =>
+    fc.record({
+      agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+      outcomeId: fc.constant(outcomeName),
+      attemptNumber: fc.integer({ min: 1, max: 100 }),
+      content: fc.record({
+        // Short message without keywords - will fail message length and buying intent
+        message: fc.constant('short message'),
+        // Invalid email format
+        targetEmail: fc.constant('not-an-email'),
+        targetCompany: fc.string({ minLength: 1 }),
+        // Company size < 50
+        targetCompanySize: fc.integer({ min: 0, max: 49 }),
+        // Invalid role
+        targetRole: fc.constantFrom('intern', 'student'),
+      }),
+      timestamp: validTimestampArb,
+    });
+
+  /**
+   * Standard outcome with all 5 validators for comprehensive testing.
+   */
+  const standardOutcome: Outcome = {
+    name: 'test_qualified_sales',
+    description: 'Test outcome for result structure validation',
+    payoutAmount: 250,
+    maxAttempts: 5,
+    timeLimitMs: 300000,
+    successCriteria: [
+      {
+        name: 'buying_intent',
+        validator: 'validateBuyingIntent',
+        params: { keywords: ['pricing', 'demo', 'next steps'] },
+      },
+      {
+        name: 'company_size',
+        validator: 'validateCompanySize',
+        params: { minimum: 50 },
+      },
+      {
+        name: 'valid_role',
+        validator: 'validateRole',
+        params: { excludedRoles: ['intern', 'student'] },
+      },
+      {
+        name: 'message_length',
+        validator: 'validateMessageLength',
+        params: { minWords: 20 },
+      },
+      {
+        name: 'valid_email',
+        validator: 'validateEmail',
+        params: {},
+      },
+    ],
+    failureReasons: ['No buying intent', 'Company too small', 'Invalid role', 'Message too short', 'Invalid email'],
+  };
+
+  // **Feature: earnd-bounty-engine, Property 10: Evaluation Result Structure**
+  // Validates: Requirement 5.1 - status SHALL be exactly 'SUCCESS' or 'FAILURE'
+  test('status is exactly SUCCESS or FAILURE for any evaluation', async () => {
+    await fc.assert(
+      fc.asyncProperty(outcomeArtifactPairArb, async ({ outcome, artifact }) => {
+        const result = await evaluateOutcome(outcome, artifact);
+
+        // Status must be exactly one of the two valid values
+        expect(result.status === 'SUCCESS' || result.status === 'FAILURE').toBe(true);
+        // Ensure it's not any other string
+        expect(['SUCCESS', 'FAILURE']).toContain(result.status);
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 10: Evaluation Result Structure**
+  // Validates: Requirement 5.2 - FAILURE results SHALL have a non-empty reason
+  test('FAILURE results have non-empty reason', async () => {
+    await fc.assert(
+      fc.asyncProperty(
+        guaranteedFailingArtifactArb(standardOutcome.name),
+        async (artifact) => {
+          const result = await evaluateOutcome(standardOutcome, artifact);
+
+          // This artifact is designed to fail
+          expect(result.status).toBe('FAILURE');
+
+          // FAILURE must have a non-empty reason
+          expect(typeof result.reason).toBe('string');
+          expect(result.reason.length).toBeGreaterThan(0);
+          expect(result.reason.trim().length).toBeGreaterThan(0);
+        }
+      ),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 10: Evaluation Result Structure**
+  // Validates: Requirement 5.3 - SUCCESS results SHALL have verificationDetails populated
+  test('SUCCESS results have verificationDetails populated', async () => {
+    await fc.assert(
+      fc.asyncProperty(
+        guaranteedPassingArtifactArb(standardOutcome.name),
+        async (artifact) => {
+          const result = await evaluateOutcome(standardOutcome, artifact);
+
+          // This artifact is designed to pass
+          expect(result.status).toBe('SUCCESS');
+
+          // SUCCESS must have verificationDetails
+          expect(result.verificationDetails).toBeDefined();
+          expect(result.verificationDetails).not.toBeNull();
+          expect(typeof result.verificationDetails).toBe('object');
+
+          // verificationDetails should contain expected fields
+          expect(result.verificationDetails).toHaveProperty('outcomeId');
+          expect(result.verificationDetails).toHaveProperty('payoutAmount');
+          expect(result.verificationDetails).toHaveProperty('agentId');
+          expect(result.verificationDetails).toHaveProperty('attemptNumber');
+          expect(result.verificationDetails).toHaveProperty('evaluatedAt');
+          expect(result.verificationDetails).toHaveProperty('criteriaCount');
+        }
+      ),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 10: Evaluation Result Structure**
+  // Combined test: validates all three requirements together
+  test('evaluation result structure is complete and correct', async () => {
+    await fc.assert(
+      fc.asyncProperty(outcomeArtifactPairArb, async ({ outcome, artifact }) => {
+        const result = await evaluateOutcome(outcome, artifact);
+
+        // Requirement 5.1: Status is exactly SUCCESS or FAILURE
+        expect(['SUCCESS', 'FAILURE']).toContain(result.status);
+
+        // Result always has reason and criteriaResults
+        expect(typeof result.reason).toBe('string');
+        expect(Array.isArray(result.criteriaResults)).toBe(true);
+
+        if (result.status === 'FAILURE') {
+          // Requirement 5.2: FAILURE has non-empty reason
+          expect(result.reason.length).toBeGreaterThan(0);
+          // FAILURE should not have verificationDetails
+          expect(result.verificationDetails).toBeUndefined();
+        }
+
+        if (result.status === 'SUCCESS') {
+          // Requirement 5.3: SUCCESS has verificationDetails populated
+          expect(result.verificationDetails).toBeDefined();
+          expect(typeof result.verificationDetails).toBe('object');
+        }
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 10: Evaluation Result Structure**
+  // Edge case: invalid artifact schema results in FAILURE with reason
+  test('invalid artifact schema results in FAILURE with structured reason', async () => {
+    const invalidArtifactArb = fc.record({
+      agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+      outcomeId: fc.constant(standardOutcome.name),
+      attemptNumber: fc.integer({ min: 1, max: 100 }),
+      content: fc.record({
+        // Missing or invalid fields
+        message: fc.constant(undefined as unknown as string),
+        targetEmail: fc.string(),
+        targetCompany: fc.string(),
+        targetCompanySize: fc.integer(),
+        targetRole: fc.string(),
+      }),
+      timestamp: validTimestampArb,
+    }) as fc.Arbitrary<AgentArtifact>;
+
+    await fc.assert(
+      fc.asyncProperty(invalidArtifactArb, async (artifact) => {
+        const result = await evaluateOutcome(standardOutcome, artifact);
+
+        // Invalid artifacts should result in FAILURE
+        expect(result.status).toBe('FAILURE');
+        // Should have a reason explaining the failure
+        expect(result.reason.length).toBeGreaterThan(0);
+        // Should not have verificationDetails
+        expect(result.verificationDetails).toBeUndefined();
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 10: Evaluation Result Structure**
+  // Edge case: outcome mismatch results in FAILURE with reason
+  test('outcome mismatch results in FAILURE with structured reason', async () => {
+    const mismatchedArtifactArb = fc.record({
+      agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+      // Different outcome ID than the one being evaluated
+      outcomeId: fc.constant('different_outcome_id'),
+      attemptNumber: fc.integer({ min: 1, max: 100 }),
+      content: fc.record({
+        message: fc.string(),
+        targetEmail: fc.string(),
+        targetCompany: fc.string(),
+        targetCompanySize: fc.integer({ min: 0, max: 1000000 }),
+        targetRole: fc.string(),
+      }),
+      timestamp: validTimestampArb,
+    });
+
+    await fc.assert(
+      fc.asyncProperty(mismatchedArtifactArb, async (artifact) => {
+        const result = await evaluateOutcome(standardOutcome, artifact);
+
+        // Mismatched outcome should result in FAILURE
+        expect(result.status).toBe('FAILURE');
+        // Should have a reason mentioning the mismatch
+        expect(result.reason).toContain('mismatch');
+        // Should not have verificationDetails
+        expect(result.verificationDetails).toBeUndefined();
+      }),
+      { numRuns: 100 }
+    );
+  });
+});
+
+
+/**
+ * Property 11: Fail-Closed Evaluation
+ *
+ * **Feature: earnd-bounty-engine, Property 11: Fail-Closed Evaluation**
+ * **Validates: Requirements 5.4**
+ *
+ * *For any* artifact with invalid or unverifiable data, evaluation SHALL return FAILURE.
+ *
+ * This property ensures the system fails safely - when data is invalid, malformed,
+ * or cannot be verified, the evaluation returns FAILURE rather than SUCCESS or
+ * throwing an exception. This prevents accidental payouts on bad data.
+ */
+describe('Fail-Closed Evaluation - Property Tests', () => {
+  /**
+   * Standard outcome with all 5 validators for comprehensive testing.
+   */
+  const standardOutcome: Outcome = {
+    name: 'test_qualified_sales',
+    description: 'Test outcome for fail-closed validation',
+    payoutAmount: 250,
+    maxAttempts: 5,
+    timeLimitMs: 300000,
+    successCriteria: [
+      {
+        name: 'buying_intent',
+        validator: 'validateBuyingIntent',
+        params: { keywords: ['pricing', 'demo', 'next steps'] },
+      },
+      {
+        name: 'company_size',
+        validator: 'validateCompanySize',
+        params: { minimum: 50 },
+      },
+      {
+        name: 'valid_role',
+        validator: 'validateRole',
+        params: { excludedRoles: ['intern', 'student'] },
+      },
+      {
+        name: 'message_length',
+        validator: 'validateMessageLength',
+        params: { minWords: 20 },
+      },
+      {
+        name: 'valid_email',
+        validator: 'validateEmail',
+        params: {},
+      },
+    ],
+    failureReasons: ['No buying intent', 'Company too small', 'Invalid role', 'Message too short', 'Invalid email'],
+  };
+
+  // **Feature: earnd-bounty-engine, Property 11: Fail-Closed Evaluation**
+  // Test: Artifacts with missing required fields return FAILURE
+  test('artifacts with missing agentId return FAILURE', async () => {
+    const missingAgentIdArb = fc.record({
+      agentId: fc.constantFrom('', '   ', '\t', '\n'), // Empty or whitespace-only
+      outcomeId: fc.constant(standardOutcome.name),
+      attemptNumber: fc.integer({ min: 1, max: 100 }),
+      content: fc.record({
+        message: fc.string(),
+        targetEmail: fc.string(),
+        targetCompany: fc.string(),
+        targetCompanySize: fc.integer({ min: 0, max: 1000000 }),
+        targetRole: fc.string(),
+      }),
+      timestamp: validTimestampArb,
+    });
+
+    await fc.assert(
+      fc.asyncProperty(missingAgentIdArb, async (artifact) => {
+        const result = await evaluateOutcome(standardOutcome, artifact);
+
+        // Invalid artifact must return FAILURE (fail closed)
+        expect(result.status).toBe('FAILURE');
+        expect(result.reason).toContain('Invalid artifact');
+        expect(result.verificationDetails).toBeUndefined();
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 11: Fail-Closed Evaluation**
+  // Test: Artifacts with missing outcomeId return FAILURE
+  test('artifacts with missing outcomeId return FAILURE', async () => {
+    const missingOutcomeIdArb = fc.record({
+      agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+      outcomeId: fc.constantFrom('', '   ', '\t', '\n'), // Empty or whitespace-only
+      attemptNumber: fc.integer({ min: 1, max: 100 }),
+      content: fc.record({
+        message: fc.string(),
+        targetEmail: fc.string(),
+        targetCompany: fc.string(),
+        targetCompanySize: fc.integer({ min: 0, max: 1000000 }),
+        targetRole: fc.string(),
+      }),
+      timestamp: validTimestampArb,
+    });
+
+    await fc.assert(
+      fc.asyncProperty(missingOutcomeIdArb, async (artifact) => {
+        const result = await evaluateOutcome(standardOutcome, artifact);
+
+        // Invalid artifact must return FAILURE (fail closed)
+        expect(result.status).toBe('FAILURE');
+        expect(result.verificationDetails).toBeUndefined();
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 11: Fail-Closed Evaluation**
+  // Test: Artifacts with invalid attemptNumber return FAILURE
+  test('artifacts with invalid attemptNumber return FAILURE', async () => {
+    const invalidAttemptNumberArb = fc.record({
+      agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+      outcomeId: fc.constant(standardOutcome.name),
+      attemptNumber: fc.constantFrom(0, -1, -100, 1.5, NaN), // Invalid attempt numbers
+      content: fc.record({
+        message: fc.string(),
+        targetEmail: fc.string(),
+        targetCompany: fc.string(),
+        targetCompanySize: fc.integer({ min: 0, max: 1000000 }),
+        targetRole: fc.string(),
+      }),
+      timestamp: validTimestampArb,
+    });
+
+    await fc.assert(
+      fc.asyncProperty(invalidAttemptNumberArb, async (artifact) => {
+        const result = await evaluateOutcome(standardOutcome, artifact);
+
+        // Invalid artifact must return FAILURE (fail closed)
+        expect(result.status).toBe('FAILURE');
+        expect(result.verificationDetails).toBeUndefined();
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 11: Fail-Closed Evaluation**
+  // Test: Artifacts with missing content fields return FAILURE
+  test('artifacts with null content return FAILURE', async () => {
+    const nullContentArb = fc.record({
+      agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+      outcomeId: fc.constant(standardOutcome.name),
+      attemptNumber: fc.integer({ min: 1, max: 100 }),
+      content: fc.constant(null as unknown as AgentArtifact['content']),
+      timestamp: validTimestampArb,
+    });
+
+    await fc.assert(
+      fc.asyncProperty(nullContentArb, async (artifact) => {
+        const result = await evaluateOutcome(standardOutcome, artifact);
+
+        // Invalid artifact must return FAILURE (fail closed)
+        expect(result.status).toBe('FAILURE');
+        expect(result.reason).toContain('Invalid artifact');
+        expect(result.verificationDetails).toBeUndefined();
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 11: Fail-Closed Evaluation**
+  // Test: Artifacts with wrong content field types return FAILURE
+  test('artifacts with wrong content field types return FAILURE', async () => {
+    const wrongTypeContentArb = fc.record({
+      agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+      outcomeId: fc.constant(standardOutcome.name),
+      attemptNumber: fc.integer({ min: 1, max: 100 }),
+      content: fc.record({
+        message: fc.constant(123 as unknown as string), // Wrong type: number instead of string
+        targetEmail: fc.string(),
+        targetCompany: fc.string(),
+        targetCompanySize: fc.integer({ min: 0, max: 1000000 }),
+        targetRole: fc.string(),
+      }),
+      timestamp: validTimestampArb,
+    });
+
+    await fc.assert(
+      fc.asyncProperty(wrongTypeContentArb, async (artifact) => {
+        const result = await evaluateOutcome(standardOutcome, artifact);
+
+        // Invalid artifact must return FAILURE (fail closed)
+        expect(result.status).toBe('FAILURE');
+        expect(result.verificationDetails).toBeUndefined();
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 11: Fail-Closed Evaluation**
+  // Test: Artifacts with unknown validator references return FAILURE
+  test('outcomes with unknown validators return FAILURE for criteria', async () => {
+    const outcomeWithUnknownValidator: Outcome = {
+      name: 'test_unknown_validator',
+      description: 'Test outcome with unknown validator',
+      payoutAmount: 100,
+      maxAttempts: 3,
+      timeLimitMs: 60000,
+      successCriteria: [
+        {
+          name: 'unknown_criterion',
+          validator: 'validateNonExistent', // Unknown validator
+          params: {},
+        },
+      ],
+      failureReasons: ['Unknown validator'],
+    };
+
+    const validArtifactArb = fc.record({
+      agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+      outcomeId: fc.constant(outcomeWithUnknownValidator.name),
+      attemptNumber: fc.integer({ min: 1, max: 100 }),
+      content: fc.record({
+        message: fc.string(),
+        targetEmail: fc.string(),
+        targetCompany: fc.string(),
+        targetCompanySize: fc.integer({ min: 0, max: 1000000 }),
+        targetRole: fc.string(),
+      }),
+      timestamp: validTimestampArb,
+    });
+
+    await fc.assert(
+      fc.asyncProperty(validArtifactArb, async (artifact) => {
+        const result = await evaluateOutcome(outcomeWithUnknownValidator, artifact);
+
+        // Unknown validator must return FAILURE (fail closed)
+        expect(result.status).toBe('FAILURE');
+        expect(result.verificationDetails).toBeUndefined();
+        // Should have a criterion result indicating the unknown validator
+        const unknownCriterion = result.criteriaResults.find(
+          (c) => c.name === 'unknown_criterion'
+        );
+        expect(unknownCriterion).toBeDefined();
+        expect(unknownCriterion?.passed).toBe(false);
+        expect(unknownCriterion?.reason).toContain('Unknown validator');
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 11: Fail-Closed Evaluation**
+  // Test: Outcome mismatch returns FAILURE (unverifiable data)
+  test('outcome ID mismatch returns FAILURE', async () => {
+    const mismatchedOutcomeArb = fc.record({
+      agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+      outcomeId: fc.string({ minLength: 1 }).filter(
+        (s) => s.trim().length > 0 && s !== standardOutcome.name
+      ),
+      attemptNumber: fc.integer({ min: 1, max: 100 }),
+      content: fc.record({
+        message: fc.string(),
+        targetEmail: fc.string(),
+        targetCompany: fc.string(),
+        targetCompanySize: fc.integer({ min: 0, max: 1000000 }),
+        targetRole: fc.string(),
+      }),
+      timestamp: validTimestampArb,
+    });
+
+    await fc.assert(
+      fc.asyncProperty(mismatchedOutcomeArb, async (artifact) => {
+        const result = await evaluateOutcome(standardOutcome, artifact);
+
+        // Mismatched outcome must return FAILURE (fail closed)
+        expect(result.status).toBe('FAILURE');
+        expect(result.reason).toContain('mismatch');
+        expect(result.verificationDetails).toBeUndefined();
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 11: Fail-Closed Evaluation**
+  // Test: Completely malformed artifacts return FAILURE
+  test('completely malformed artifacts return FAILURE', async () => {
+    // Generate various malformed artifact shapes
+    const malformedArtifactArb = fc.oneof(
+      // Null artifact
+      fc.constant(null as unknown as AgentArtifact),
+      // Undefined artifact
+      fc.constant(undefined as unknown as AgentArtifact),
+      // Empty object
+      fc.constant({} as unknown as AgentArtifact),
+      // Array instead of object
+      fc.constant([] as unknown as AgentArtifact),
+      // String instead of object
+      fc.constant('not an artifact' as unknown as AgentArtifact),
+      // Number instead of object
+      fc.constant(42 as unknown as AgentArtifact)
+    );
+
+    await fc.assert(
+      fc.asyncProperty(malformedArtifactArb, async (artifact) => {
+        const result = await evaluateOutcome(standardOutcome, artifact);
+
+        // Malformed artifact must return FAILURE (fail closed)
+        expect(result.status).toBe('FAILURE');
+        expect(result.verificationDetails).toBeUndefined();
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 11: Fail-Closed Evaluation**
+  // Test: Evaluation never throws exceptions - always returns FAILURE for bad data
+  test('evaluation never throws exceptions for any input', async () => {
+    // Generate completely random inputs
+    const anyInputArb = fc.anything();
+
+    await fc.assert(
+      fc.asyncProperty(anyInputArb, async (randomInput) => {
+        // Should never throw, always return a result
+        let result: Awaited<ReturnType<typeof evaluateOutcome>>;
+        let threw = false;
+
+        try {
+          result = await evaluateOutcome(
+            standardOutcome,
+            randomInput as AgentArtifact
+          );
+        } catch {
+          threw = true;
+          result = { status: 'FAILURE', reason: 'Exception thrown', criteriaResults: [] };
+        }
+
+        // Evaluation should not throw exceptions
+        expect(threw).toBe(false);
+
+        // If input is invalid, result should be FAILURE
+        if (
+          typeof randomInput !== 'object' ||
+          randomInput === null ||
+          !('agentId' in randomInput) ||
+          !('content' in randomInput)
+        ) {
+          expect(result.status).toBe('FAILURE');
+          expect(result.verificationDetails).toBeUndefined();
+        }
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 11: Fail-Closed Evaluation**
+  // Test: Missing timestamp returns FAILURE
+  test('artifacts with missing timestamp return FAILURE', async () => {
+    const missingTimestampArb = fc.record({
+      agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+      outcomeId: fc.constant(standardOutcome.name),
+      attemptNumber: fc.integer({ min: 1, max: 100 }),
+      content: fc.record({
+        message: fc.string(),
+        targetEmail: fc.string(),
+        targetCompany: fc.string(),
+        targetCompanySize: fc.integer({ min: 0, max: 1000000 }),
+        targetRole: fc.string(),
+      }),
+      timestamp: fc.constantFrom('', '   ', '\t', '\n'), // Empty or whitespace-only
+    });
+
+    await fc.assert(
+      fc.asyncProperty(missingTimestampArb, async (artifact) => {
+        const result = await evaluateOutcome(standardOutcome, artifact);
+
+        // Invalid artifact must return FAILURE (fail closed)
+        expect(result.status).toBe('FAILURE');
+        expect(result.verificationDetails).toBeUndefined();
+      }),
+      { numRuns: 100 }
+    );
+  });
+});
+
+
+/**
+ * Property 19: Qualified Sales Interest Payout
+ *
+ * **Feature: earnd-bounty-engine, Property 19: Qualified Sales Interest Payout**
+ * **Validates: Requirements 8.6**
+ *
+ * *For any* artifact that passes all 5 criteria (buying intent, company size, role,
+ * message length, email), the evaluation SHALL return SUCCESS with payoutAmount of 250.
+ */
+describe('Qualified Sales Interest Payout - Property Tests', () => {
+  /**
+   * Import the qualified_sales_interest outcome for testing.
+   * This is the demo outcome with $250 payout.
+   */
+  const qualifiedSalesInterestOutcome: Outcome = {
+    name: 'qualified_sales_interest',
+    description: 'Generate a qualified sales lead',
+    payoutAmount: 250,
+    maxAttempts: 5,
+    timeLimitMs: 300000,
+    successCriteria: [
+      {
+        name: 'buying_intent',
+        validator: 'validateBuyingIntent',
+        params: { keywords: ['pricing', 'demo', 'next steps'] },
+      },
+      {
+        name: 'company_size',
+        validator: 'validateCompanySize',
+        params: { minimum: 50 },
+      },
+      {
+        name: 'valid_role',
+        validator: 'validateRole',
+        params: { excludedRoles: ['intern', 'student'] },
+      },
+      {
+        name: 'message_length',
+        validator: 'validateMessageLength',
+        params: { minWords: 20 },
+      },
+      {
+        name: 'valid_email',
+        validator: 'validateEmail',
+        params: {},
+      },
+    ],
+    failureReasons: [
+      'No buying intent detected',
+      'Company too small',
+      'Invalid role',
+      'Message too short',
+      'Invalid email format',
+    ],
+  };
+
+  /**
+   * Arbitrary for generating artifacts that pass ALL 5 criteria:
+   * 1. Message contains buying intent keywords (pricing, demo, next steps)
+   * 2. Company size >= 50
+   * 3. Role is not intern or student
+   * 4. Message length >= 20 words
+   * 5. Email is syntactically valid
+   */
+  const passingAllCriteriaArtifactArb: fc.Arbitrary<AgentArtifact> = fc.record({
+    agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+    outcomeId: fc.constant('qualified_sales_interest'),
+    attemptNumber: fc.integer({ min: 1, max: 100 }),
+    content: fc.record({
+      // Message with 20+ words AND at least one buying intent keyword
+      message: fc
+        .tuple(
+          // Generate 17+ filler words (non-whitespace)
+          fc.array(
+            fc.string({ minLength: 1, maxLength: 10 }).filter((s) => !/\s/.test(s) && s.length > 0),
+            { minLength: 17, maxLength: 30 }
+          ),
+          // Pick at least one buying intent keyword
+          fc.constantFrom('pricing', 'demo', 'next steps')
+        )
+        .map(([words, keyword]) => [...words, keyword, 'additional', 'words'].join(' ')),
+      // Valid email format
+      targetEmail: fc.tuple(
+        fc.string({ minLength: 1, maxLength: 10 }).filter((s) => /^[a-zA-Z0-9]+$/.test(s)),
+        fc.constantFrom('example.com', 'company.org', 'business.net')
+      ).map(([local, domain]) => `${local}@${domain}`),
+      targetCompany: fc.string({ minLength: 1 }),
+      // Company size >= 50
+      targetCompanySize: fc.integer({ min: 50, max: 100000 }),
+      // Valid role (not intern or student)
+      targetRole: fc.constantFrom(
+        'Manager',
+        'Director',
+        'VP',
+        'CEO',
+        'CTO',
+        'Engineer',
+        'Developer',
+        'Analyst',
+        'Consultant',
+        'Lead'
+      ),
+    }),
+    timestamp: validTimestampArb,
+  });
+
+  // **Feature: earnd-bounty-engine, Property 19: Qualified Sales Interest Payout**
+  test('artifacts passing all 5 criteria return SUCCESS with payoutAmount of 250', async () => {
+    await fc.assert(
+      fc.asyncProperty(passingAllCriteriaArtifactArb, async (artifact) => {
+        const result = await evaluateOutcome(qualifiedSalesInterestOutcome, artifact);
+
+        // Must return SUCCESS
+        expect(result.status).toBe('SUCCESS');
+
+        // Must have verificationDetails
+        expect(result.verificationDetails).toBeDefined();
+
+        // verificationDetails must contain payoutAmount of 250
+        expect(result.verificationDetails?.payoutAmount).toBe(250);
+
+        // All 5 criteria must have passed
+        expect(result.criteriaResults.length).toBe(5);
+        expect(result.criteriaResults.every((c) => c.passed)).toBe(true);
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 19: Qualified Sales Interest Payout**
+  test('payoutAmount in verificationDetails matches outcome definition', async () => {
+    await fc.assert(
+      fc.asyncProperty(passingAllCriteriaArtifactArb, async (artifact) => {
+        const result = await evaluateOutcome(qualifiedSalesInterestOutcome, artifact);
+
+        if (result.status === 'SUCCESS') {
+          // The payoutAmount in verificationDetails must match the outcome's payoutAmount
+          expect(result.verificationDetails?.payoutAmount).toBe(
+            qualifiedSalesInterestOutcome.payoutAmount
+          );
+          // Specifically for qualified_sales_interest, this must be $250
+          expect(result.verificationDetails?.payoutAmount).toBe(250);
+        }
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 19: Qualified Sales Interest Payout**
+  test('SUCCESS result includes all required verification fields', async () => {
+    await fc.assert(
+      fc.asyncProperty(passingAllCriteriaArtifactArb, async (artifact) => {
+        const result = await evaluateOutcome(qualifiedSalesInterestOutcome, artifact);
+
+        expect(result.status).toBe('SUCCESS');
+        expect(result.verificationDetails).toBeDefined();
+
+        // Verify all required fields are present
+        expect(result.verificationDetails).toHaveProperty('outcomeId', 'qualified_sales_interest');
+        expect(result.verificationDetails).toHaveProperty('payoutAmount', 250);
+        expect(result.verificationDetails).toHaveProperty('agentId', artifact.agentId);
+        expect(result.verificationDetails).toHaveProperty('attemptNumber', artifact.attemptNumber);
+        expect(result.verificationDetails).toHaveProperty('evaluatedAt');
+        expect(result.verificationDetails).toHaveProperty('criteriaCount', 5);
+      }),
+      { numRuns: 100 }
+    );
+  });
+
+  // **Feature: earnd-bounty-engine, Property 19: Qualified Sales Interest Payout**
+  test('failing any single criterion prevents payout', async () => {
+    // Generate artifacts that fail exactly one criterion
+    const failingBuyingIntentArb: fc.Arbitrary<AgentArtifact> = fc.record({
+      agentId: fc.string({ minLength: 1 }).filter((s) => s.trim().length > 0),
+      outcomeId: fc.constant('qualified_sales_interest'),
+      attemptNumber: fc.integer({ min: 1, max: 100 }),
+      content: fc.record({
+        // Message WITHOUT buying intent keywords but with 20+ words
+        message: fc
+          .array(
+            fc.constantFrom('hello', 'world', 'test', 'data', 'content', 'info', 'text', 'word'),
+            { minLength: 20, maxLength: 30 }
+          )
+          .map((words) => words.join(' ')),
+        targetEmail: fc.constant('valid@example.com'),
+        targetCompany: fc.string({ minLength: 1 }),
+        targetCompanySize: fc.integer({ min: 50, max: 100000 }),
+        targetRole: fc.constantFrom('Manager', 'Director', 'VP'),
+      }),
+      timestamp: validTimestampArb,
+    });
+
+    await fc.assert(
+      fc.asyncProperty(failingBuyingIntentArb, async (artifact) => {
+        const result = await evaluateOutcome(qualifiedSalesInterestOutcome, artifact);
+
+        // Must return FAILURE when any criterion fails
+        expect(result.status).toBe('FAILURE');
+
+        // No payout should be triggered
+        expect(result.verificationDetails).toBeUndefined();
+      }),
+      { numRuns: 100 }
+    );
+  });
+});