@qianxude/tem 0.3.0 → 0.4.2

package/src/core/worker.ts

@@ -1,5 +1,5 @@
 import * as i from '../interfaces/index.js';
-import { TaskService } from '../services/task.js';
+import { TaskService, BatchInterruptionService } from '../services/index.js';
 import { ConcurrencyController, RateLimiter, type RateLimitConfig } from '../utils/index.js';
 
 /**
@@ -18,6 +18,10 @@ export interface WorkerConfig {
   concurrency: number;
   pollIntervalMs: number;
   rateLimit?: RateLimitConfig;
+  /** Specific batch ID to process (optional - if set, only processes this batch) */
+  batchId?: string;
+  /** Interruption service for checking batch status */
+  interruptionService?: BatchInterruptionService;
 }
 
 export class Worker {
@@ -28,6 +32,13 @@ export class Worker {
   private pollIntervalMs: number;
   private abortController: AbortController;
   private inFlightTasks: Set<Promise<void>> = new Set();
+  private batchId?: string;
+  private interruptionService?: BatchInterruptionService;
+
+  // Track failure context for interruption decisions
+  private consecutiveFailures = 0;
+  private rateLimitHits = 0;
+  private concurrencyErrors = 0;
 
   constructor(
     private taskService: TaskService,
@@ -36,6 +47,8 @@ export class Worker {
     this.concurrency = new ConcurrencyController(config.concurrency);
     this.pollIntervalMs = config.pollIntervalMs;
     this.abortController = new AbortController();
+    this.batchId = config.batchId;
+    this.interruptionService = config.interruptionService;
 
     if (config.rateLimit) {
       this.rateLimiter = new RateLimiter(config.rateLimit);
@@ -93,8 +106,18 @@ export class Worker {
           break;
         }
 
+        // For batch-specific workers: check batch is still active
+        if (this.batchId && this.interruptionService) {
+          const isActive = await this.interruptionService.isBatchActive(this.batchId);
+          if (!isActive) {
+            this.concurrency.release();
+            this.stop();
+            break;
+          }
+        }
+
         // Claim a task while holding the concurrency slot
-        const task = await this.taskService.claim();
+        const task = await this.taskService.claim(this.batchId);
 
         if (!task) {
           // No task available, release the slot and sleep
@@ -123,6 +146,8 @@ export class Worker {
    * Note: Assumes concurrency slot has already been acquired.
    */
   private async execute(task: i.Task): Promise<void> {
+    const taskStartTime = Date.now();
+
     try {
       if (this.rateLimiter) {
         await this.rateLimiter.acquire();
@@ -134,6 +159,8 @@ export class Worker {
       }
 
       const payload = JSON.parse(task.payload);
+
+      // Build context with optional deadline
       const context: i.TaskContext = {
         taskId: task.id,
         batchId: task.batchId,
@@ -141,10 +168,22 @@ export class Worker {
         signal: this.abortController.signal,
       };
 
+      // If we have interruption service and batchId, set deadline from criteria
+      if (this.interruptionService && task.batchId) {
+        const { criteria } = await this.interruptionService['batchService'].getWithCriteria(task.batchId);
+        if (criteria?.taskTimeoutMs) {
+          context.deadline = new Date(taskStartTime + criteria.taskTimeoutMs);
+        }
+      }
+
       const result = await handler(payload, context);
       await this.taskService.complete(task.id, result);
+
+      // Reset consecutive failures on success
+      this.consecutiveFailures = 0;
     } catch (error) {
-      await this.handleError(task, error);
+      const taskRuntimeMs = Date.now() - taskStartTime;
+      await this.handleError(task, error, taskRuntimeMs);
     } finally {
       this.concurrency.release();
     }
@@ -153,16 +192,49 @@ export class Worker {
   /**
    * Handle task execution errors.
    */
-  private async handleError(task: i.Task, error: unknown): Promise<void> {
+  private async handleError(task: i.Task, error: unknown, taskRuntimeMs?: number): Promise<void> {
     const isRetryable = !(error instanceof NonRetryableError);
     const shouldRetry = isRetryable && task.attempt < task.maxAttempt;
 
+    // Track failure type for interruption decisions
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    if (this.isRateLimitError(errorMessage)) {
+      this.rateLimitHits++;
+    } else if (this.isConcurrencyError(errorMessage)) {
+      this.concurrencyErrors++;
+    }
+
     if (shouldRetry) {
       // Reset to pending for automatic retry (attempt already incremented by claim)
       await this.taskService.retry(task.id);
+      this.consecutiveFailures++;
     } else {
-      const message = error instanceof Error ? error.message : String(error);
-      await this.taskService.fail(task.id, message);
+      await this.taskService.fail(task.id, errorMessage);
+      this.consecutiveFailures++;
+
+      // Check if batch should be interrupted
+      if (task.batchId && this.interruptionService) {
+        const interrupted = await this.interruptionService.checkAndInterruptIfNeeded(
+          task.batchId,
+          {
+            consecutiveFailures: this.consecutiveFailures,
+            rateLimitHits: this.rateLimitHits,
+            concurrencyErrors: this.concurrencyErrors,
+            currentTaskRuntimeMs: taskRuntimeMs,
+          }
+        );
+        if (interrupted) {
+          this.stop();
+        }
+      }
     }
   }
+
+  private isRateLimitError(message: string): boolean {
+    return message.includes('429') || message.toLowerCase().includes('rate limit');
+  }
+
+  private isConcurrencyError(message: string): boolean {
+    return message.includes('502') || message.includes('503') || message.toLowerCase().includes('bad gateway') || message.toLowerCase().includes('service unavailable');
+  }
 }

package/src/database/index.ts

@@ -33,13 +33,20 @@ export class Database implements i.DatabaseConnection {
       )
     `);
 
-    // Check if initial schema needs to be applied
-    const migrationCount = this.db
-      .query('SELECT COUNT(*) as count FROM _migration WHERE name = $name')
-      .get({ $name: '001_initial_schema' }) as { count: number };
-
-    if (migrationCount.count === 0) {
-      this.applyInitialSchema();
+    // Check and apply migrations in order
+    const migrations = [
+      { name: '001_initial_schema', apply: () => this.applyInitialSchema() },
+      { name: '002_batch_interruption', apply: () => this.applyBatchInterruptionMigration() },
+    ];
+
+    for (const migration of migrations) {
+      const migrationCount = this.db
+        .query('SELECT COUNT(*) as count FROM _migration WHERE name = $name')
+        .get({ $name: migration.name }) as { count: number };
+
+      if (migrationCount.count === 0) {
+        migration.apply();
+      }
     }
   }
 
@@ -90,6 +97,39 @@ export class Database implements i.DatabaseConnection {
     });
   }
 
+  private applyBatchInterruptionMigration(): void {
+    const migration = `
+      -- Add status to batch table
+      ALTER TABLE batch ADD COLUMN status TEXT NOT NULL DEFAULT 'active'
+        CHECK(status IN ('active', 'interrupted', 'completed'));
+
+      -- Add interruption criteria storage (JSON)
+      ALTER TABLE batch ADD COLUMN interruption_criteria TEXT;
+
+      -- Index for quickly finding active batches
+      CREATE INDEX IF NOT EXISTS idx_batch_status ON batch(status);
+
+      -- New table: interruption log
+      CREATE TABLE IF NOT EXISTS batch_interrupt_log (
+        id TEXT PRIMARY KEY,
+        batch_id TEXT NOT NULL REFERENCES batch(id) ON DELETE CASCADE,
+        reason TEXT NOT NULL,
+        message TEXT NOT NULL,
+        stats_snapshot TEXT NOT NULL, -- JSON of BatchStats
+        created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_interrupt_log_batch_id ON batch_interrupt_log(batch_id);
+    `;
+
+    this.transaction(() => {
+      this.db.exec(migration);
+      this.db
+        .query('INSERT INTO _migration (name) VALUES ($name)')
+        .run({ $name: '002_batch_interruption' });
+    });
+  }
+
   query<T = unknown>(sql: string, params?: SQLQueryBindings[]): T[] {
     const stmt = this.db.prepare(sql);
     const results = stmt.all(...(params ?? []));

package/src/index.ts

@@ -1,7 +1,7 @@
 // Main exports for TEM framework
 export * as interfaces from './interfaces/index.js';
 export { Database, type DatabaseOptions } from './database/index.js';
-export { BatchService, TaskService } from './services/index.js';
+export { BatchService, TaskService, BatchInterruptionService } from './services/index.js';
 export {
   ConcurrencyController,
   RateLimiter,

package/src/interfaces/index.ts

@@ -6,18 +6,48 @@
 // ============================================================================
 
 export type TaskStatus = 'pending' | 'running' | 'completed' | 'failed';
+export type BatchStatus = 'active' | 'interrupted' | 'completed';
+
+export type BatchInterruptionReason =
+  | 'error_rate_exceeded'
+  | 'failed_tasks_exceeded'
+  | 'consecutive_failures_exceeded'
+  | 'rate_limit_hits_exceeded'
+  | 'concurrency_errors_exceeded'
+  | 'task_timeout'
+  | 'batch_runtime_exceeded'
+  | 'manual';
 
 // ============================================================================
 // Entity Types
 // ============================================================================
 
+export interface BatchInterruptionCriteria {
+  /** Max error rate (0-1, e.g., 0.1 = 10%) */
+  maxErrorRate?: number;
+  /** Max absolute number of failed tasks */
+  maxFailedTasks?: number;
+  /** Max consecutive failures before interruption */
+  maxConsecutiveFailures?: number;
+  /** Max rate limit (429) hits before interruption */
+  maxRateLimitHits?: number;
+  /** Max concurrency errors (502/503) before interruption - indicates too aggressive concurrency */
+  maxConcurrencyErrors?: number;
+  /** Max runtime for a single task in ms */
+  taskTimeoutMs?: number;
+  /** Max total batch runtime in ms */
+  maxBatchRuntimeMs?: number;
+}
+
 export interface Batch {
   id: string;
   code: string;
   type: string;
+  status: BatchStatus;
   createdAt: Date;
   completedAt: Date | null;
   metadata: Record<string, unknown> | null;
+  interruptionCriteria: BatchInterruptionCriteria | null;
 }
 
 export interface BatchStats {
@@ -29,6 +59,14 @@ export interface BatchStats {
   failed: number;
 }
 
+export interface BatchInterruption {
+  batchId: string;
+  reason: BatchInterruptionReason;
+  message: string;
+  statsAtInterruption: BatchStats;
+  createdAt: Date;
+}
+
 export interface Task {
   id: string;
   batchId: string | null;
@@ -88,6 +126,8 @@ export interface DetectOptions {
   maxConcurrencyToTest?: number;
   /** Duration to run rate limit tests (default: 10000) */
   rateLimitTestDurationMs?: number;
+  /** Maximum number of requests to send during rate limit detection (default: 200) */
+  maxRateLimitTestRequests?: number;
 }
 
 export interface DetectedConfig {
@@ -118,6 +158,8 @@ export interface TaskContext {
   batchId: string | null;
   attempt: number;
   signal: AbortSignal;
+  /** Deadline for task execution (for timeout enforcement) */
+  deadline?: Date;
 }
 
 // ============================================================================
@@ -144,6 +186,7 @@ export interface CreateBatchInput {
   code: string;
   type: string;
   metadata?: Record<string, unknown>;
+  interruptionCriteria?: BatchInterruptionCriteria;
 }
 
 export interface CreateTaskInput {
@@ -162,6 +205,23 @@ export interface BatchService {
   complete(id: string): Promise<void>;
   resume(id: string): Promise<number>;
   retryFailed(id: string): Promise<number>;
+  updateStatus(id: string, status: BatchStatus): Promise<void>;
+  getWithCriteria(id: string): Promise<{ batch: Batch; criteria: BatchInterruptionCriteria | null }>;
+}
+
+export interface BatchInterruptionService {
+  checkAndInterruptIfNeeded(
+    batchId: string,
+    context: {
+      consecutiveFailures?: number;
+      rateLimitHits?: number;
+      concurrencyErrors?: number;
+      currentTaskRuntimeMs?: number;
+    }
+  ): Promise<boolean>;
+  interrupt(batchId: string, reason: BatchInterruptionReason, message: string): Promise<void>;
+  isBatchActive(batchId: string): Promise<boolean>;
+  getInterruptionLog(batchId: string): Promise<BatchInterruption[]>;
 }
 
 export interface TaskService {

package/src/mock-server/README.md

@@ -1,14 +1,15 @@
 # Mock Server
 
-A lightweight HTTP server for simulating external API services with configurable concurrency and rate limiting constraints. Used for testing TEM's task execution capabilities under various load conditions.
+A lightweight HTTP server for simulating external API services with configurable concurrency, rate limiting, and error simulation. Used for testing TEM's task execution capabilities under various load conditions.
 
 ## Overview
 
 The mock server provides a controlled environment to test how TEM handles:
 
-- **Concurrency limits** - Simulate services that reject requests when too many are in flight
-- **Rate limiting** - Test backoff and retry behavior against rate-limited endpoints
-- **Processing delays** - Verify timeout handling and async processing
+- **Concurrency limits** — Simulate services that reject requests when too many are in flight (503 errors)
+- **Rate limiting** — Test backoff and retry behavior against rate-limited endpoints (429 errors)
+- **Error simulation** — Verify resilience with configurable random failure rates
+- **Processing delays** — Verify timeout handling and async processing
 
 ## Architecture
 
@@ -30,6 +31,8 @@ The mock server provides a controlled environment to test how TEM handles:
 | Component | File | Purpose |
 |-----------|------|---------|
 | `startMockServer` | `server.ts` | Server lifecycle management |
+| `createMockService` | `server.ts` | Client helper to create services |
+| `createErrorSimulation` | `server.ts` | Helper to create error simulation config |
 | `createRouter` | `router.ts` | HTTP routing and request handling |
 | `MockService` | `service.ts` | Per-service concurrency and rate limiting |
 | `RejectingRateLimiter` | `service.ts` | Token bucket rate limiter with immediate reject |
@@ -49,7 +52,8 @@ startMockServer({
   defaultService: {
     maxConcurrency: 3,
     rateLimit: { limit: 10, windowMs: 1000 },
-    delayMs: [10, 50]
+    delayMs: [10, 50],
+    errorSimulation: { rate: 0.1, statusCode: 500 }
   }
 });
 ```
@@ -61,10 +65,7 @@ Dynamic service creation and management. Each service has its own concurrency/ra
 **Use case:** Complex tests with multiple services having different constraints.
 
 ```typescript
-startMockServer({
-  port: 8080,
-  mode: 'multi'
-});
+startMockServer({ port: 8080, mode: 'multi' });
 
 // Create services dynamically via HTTP API
 ```
@@ -75,7 +76,7 @@ startMockServer({
 
 | Method | Path | Description |
 |--------|------|-------------|
-| `GET` | `/` | Access the default service |
+| `GET` / `POST` | `/` | Access the default service |
 | `POST` | `/shutdown` | Shutdown the server |
 
 ### Multi Mode Endpoints
@@ -84,7 +85,7 @@ startMockServer({
 |--------|------|-------------|
 | `POST` | `/service/:name` | Create or replace a service |
 | `DELETE` | `/service/:name` | Delete a service |
-| `GET` | `/mock/:name` | Access a service |
+| `GET` / `POST` | `/mock/:name` | Access a service |
 | `POST` | `/shutdown` | Shutdown the server |
 
 ### Create Service (Multi Mode Only)
@@ -99,7 +100,12 @@ Content-Type: application/json
     "limit": 10,                 // Requests per window (required)
     "windowMs": 1000             // Window size in ms (required)
   },
-  "delayMs": [10, 200]           // [min, max] processing delay (optional, default: [10, 200])
+  "delayMs": [10, 200],          // [min, max] processing delay (optional, default: [10, 200])
+  "errorSimulation": {           // Optional error simulation config
+    "rate": 0.1,                 // Error rate 0-1 (10% = 0.1)
+    "statusCode": 503,           // HTTP status to return (default: 500)
+    "errorMessage": "simulated_error"  // Error message (default: "internal_server_error")
+  }
 }
 ```
 
@@ -114,11 +120,30 @@ Content-Type: application/json
 }
 ```
 
+### Delete Service (Multi Mode Only)
+
+```http
+DELETE /service/:name
+```
+
+**Response:**
+```http
+HTTP/1.1 200 OK
+Content-Type: application/json
+
+{
+  "service": "test1",
+  "status": "deleted"
+}
+```
+
 ### Access Service
 
 ```http
 GET /mock/:name        # Multi mode
+POST /mock/:name       # Multi mode (body ignored)
 GET /                  # Single mode
+POST /                 # Single mode (body ignored)
 ```
 
 **Success Response (200):**
@@ -152,6 +177,8 @@ Content-Type: application/json
 }
 ```
 
+The server will stop accepting new connections and exit the process after a brief delay.
+
 ## Error Responses
 
 | Status | Error Code | Description |
@@ -163,6 +190,21 @@ Content-Type: application/json
 | `429` | `rate_limit_exceeded` | Rate limit reached |
 | `503` | `concurrency_limit_exceeded` | Concurrency limit reached |
 
+### Error Simulation
+
+When `errorSimulation` is configured, requests may randomly fail with:
+
+```http
+HTTP/1.1 500 Internal Server Error  // Or configured statusCode
+Content-Type: application/json
+
+{
+  "error": "internal_server_error"  // Or configured errorMessage
+}
+```
+
+The error is checked **before** acquiring concurrency/rate limit resources, so simulated errors don't count against limits.
+
 ## Configuration
 
 ### ServerConfig
@@ -185,6 +227,21 @@ interface ServiceConfig {
     windowMs: number;              // Window duration in milliseconds
   };
   delayMs: [number, number];       // [min, max] simulated processing delay
+  errorSimulation?: {              // Optional error simulation
+    rate: number;                  // Error rate 0-1
+    statusCode?: number;           // HTTP status code (default: 500)
+    errorMessage?: string;         // Error message (default: "internal_server_error")
+  };
+}
+```
+
+### ErrorSimulationConfig
+
+```typescript
+interface ErrorSimulationConfig {
+  rate: number;        // Error rate 0-1 (e.g., 0.1 = 10% error rate)
+  statusCode?: number; // HTTP status code to return (default: 500)
+  errorMessage?: string; // Error message (default: "internal_server_error")
 }
 ```
 
@@ -315,6 +372,29 @@ console.log(r3.status);  // 429
 console.log(await r3.json());  // { error: 'rate_limit_exceeded' }
 ```
 
+### Testing Error Simulation
+
+```typescript
+import { startMockServer, stopMockServer, createErrorSimulation } from './src/mock-server';
+
+startMockServer({
+  port: 8080,
+  mode: 'single',
+  defaultService: {
+    maxConcurrency: 10,
+    rateLimit: { limit: 100, windowMs: 1000 },
+    delayMs: [10, 50],
+    errorSimulation: createErrorSimulation(0.3, 503, "service_unavailable")
+  }
+});
+
+// Approximately 30% of requests will fail with 503
+for (let i = 0; i < 10; i++) {
+  const res = await fetch('http://localhost:8080/');
+  console.log(res.status);  // Mix of 200 and 503
+}
+```
+
 ## Rate Limiting Algorithm
 
 The mock server uses a **token bucket** algorithm for rate limiting:
@@ -332,15 +412,31 @@ Concurrency is tracked per-service:
 - Decrements when request completes (in `finally` block)
 - If `currentConcurrency >= maxConcurrency`, new requests get 503
 
+## Error Simulation
+
+Error simulation is checked before acquiring resources:
+
+```typescript
+// Pseudocode
+if (Math.random() < errorSimulation.rate) {
+  return errorResponse;  // Doesn't consume concurrency/rate limit tokens
+}
+// Continue with normal request handling...
+```
+
+This ensures that simulated errors don't deplete your concurrency slots or rate limit budget.
+
 ## Programmatic API
 
+### Server Lifecycle
+
 ```typescript
 import { startMockServer, stopMockServer, getServerState } from './src/mock-server';
 
 // Start server
 startMockServer(config: ServerConfig): void
 
-// Stop server programmatically
+// Stop server programmatically (for testing)
 stopMockServer(): void
 
 // Get current state (for testing)
@@ -350,3 +446,74 @@ getServerState(): {
   hasDefaultService: boolean;
 }
 ```
+
+### Client Helpers
+
+```typescript
+import { createMockService, createErrorSimulation } from './src/mock-server';
+
+// Create a service programmatically (wraps HTTP call)
+createMockService(
+  name: string,
+  config: CreateServiceRequest,
+  mockUrl?: string  // defaults to http://localhost:19999
+): Promise<Response>
+
+// Create error simulation config with validation
+createErrorSimulation(
+  rate: number,           // 0-1 error rate
+  statusCode?: number,    // HTTP status (default: 500)
+  errorMessage?: string   // Error message
+): ErrorSimulationConfig
+```
+
+Example using client helpers:
+
+```typescript
+import { createMockService, createErrorSimulation } from './src/mock-server';
+
+// Create service with error simulation
+await createMockService('flaky-api', {
+  maxConcurrency: 5,
+  rateLimit: { limit: 10, windowMs: 1000 },
+  delayMs: [50, 100],
+  errorSimulation: createErrorSimulation(0.2, 503)
+});
+
+// Use the service
+const res = await fetch('http://localhost:19999/mock/flaky-api');
+```
+
+## Testing with TEM
+
+The mock server is designed to integrate seamlessly with TEM for testing retry and error handling:
+
+```typescript
+import { TEM } from '@qianxude/tem';
+import { startMockServer, createMockService, createErrorSimulation } from './src/mock-server';
+
+// Start mock server with flaky service
+startMockServer({ port: 19999, mode: 'multi' });
+
+await createMockService('api', {
+  maxConcurrency: 3,
+  rateLimit: { limit: 10, windowMs: 1000 },
+  errorSimulation: createErrorSimulation(0.2)  // 20% failure rate
+});
+
+// Configure TEM to match mock server limits
+const tem = new TEM({
+  databasePath: ':memory:',
+  concurrency: 3,
+  rateLimit: { requests: 10, windowMs: 1000 }
+});
+
+// Register handler that calls mock server
+tem.worker.register('test', async (payload) => {
+  const res = await fetch('http://localhost:19999/mock/api');
+  if (!res.ok) throw new Error(`HTTP ${res.status}`);
+  return res.json();
+});
+
+// TEM's retry mechanism will handle the 20% failure rate
+```