@qianxude/tem 0.2.0 → 0.4.0

package/src/services/batch-interruption.ts

@@ -0,0 +1,192 @@
+import * as i from '../interfaces/index.js';
+import type { Database } from '../database/index.js';
+import type { BatchService } from './batch.js';
+
+export interface BatchInterruptionRow {
+  batch_id: string;
+  reason: i.BatchInterruptionReason;
+  message: string;
+  stats_snapshot: string;
+  created_at: string;
+}
+
+export class BatchInterruptionService implements i.BatchInterruptionService {
+  constructor(
+    private db: Database,
+    private batchService: BatchService
+  ) {}
+
+  /**
+   * Check if batch should be interrupted based on current stats.
+   * Called after each task failure or periodically.
+   */
+  async checkAndInterruptIfNeeded(
+    batchId: string,
+    context: {
+      consecutiveFailures?: number;
+      rateLimitHits?: number;
+      concurrencyErrors?: number;
+      currentTaskRuntimeMs?: number;
+    }
+  ): Promise<boolean> {
+    // Fetch batch with its interruption criteria
+    const { batch, criteria } = await this.batchService.getWithCriteria(batchId);
+
+    // If already interrupted or completed, no need to check
+    if (batch.status !== 'active') {
+      return false;
+    }
+
+    // If no criteria set, never interrupt
+    if (!criteria) {
+      return false;
+    }
+
+    // Get current stats
+    const stats = await this.batchService.getStats(batchId);
+
+    // Check each criterion in order of severity
+
+    // 1. Check maxBatchRuntimeMs - total batch runtime
+    if (criteria.maxBatchRuntimeMs) {
+      const batchRuntimeMs = Date.now() - batch.createdAt.getTime();
+      if (batchRuntimeMs > criteria.maxBatchRuntimeMs) {
+        await this.interrupt(
+          batchId,
+          'batch_runtime_exceeded',
+          `Batch runtime (${batchRuntimeMs}ms) exceeded maximum (${criteria.maxBatchRuntimeMs}ms)`
+        );
+        return true;
+      }
+    }
+
+    // 2. Check taskTimeoutMs - single task runtime
+    if (criteria.taskTimeoutMs && context.currentTaskRuntimeMs) {
+      if (context.currentTaskRuntimeMs > criteria.taskTimeoutMs) {
+        await this.interrupt(
+          batchId,
+          'task_timeout',
+          `Task runtime (${context.currentTaskRuntimeMs}ms) exceeded maximum (${criteria.taskTimeoutMs}ms)`
+        );
+        return true;
+      }
+    }
+
+    // 3. Check maxConsecutiveFailures
+    if (criteria.maxConsecutiveFailures && context.consecutiveFailures) {
+      if (context.consecutiveFailures >= criteria.maxConsecutiveFailures) {
+        await this.interrupt(
+          batchId,
+          'consecutive_failures_exceeded',
+          `Consecutive failures (${context.consecutiveFailures}) exceeded maximum (${criteria.maxConsecutiveFailures})`
+        );
+        return true;
+      }
+    }
+
+    // 4. Check maxRateLimitHits
+    if (criteria.maxRateLimitHits && context.rateLimitHits) {
+      if (context.rateLimitHits >= criteria.maxRateLimitHits) {
+        await this.interrupt(
+          batchId,
+          'rate_limit_hits_exceeded',
+          `Rate limit hits (${context.rateLimitHits}) exceeded maximum (${criteria.maxRateLimitHits})`
+        );
+        return true;
+      }
+    }
+
+    // 5. Check maxConcurrencyErrors (502/503 errors)
+    if (criteria.maxConcurrencyErrors && context.concurrencyErrors) {
+      if (context.concurrencyErrors >= criteria.maxConcurrencyErrors) {
+        await this.interrupt(
+          batchId,
+          'concurrency_errors_exceeded',
+          `Concurrency errors (${context.concurrencyErrors}) exceeded maximum (${criteria.maxConcurrencyErrors})`
+        );
+        return true;
+      }
+    }
+
+    // 6. Check maxFailedTasks (absolute count)
+    if (criteria.maxFailedTasks) {
+      if (stats.failed >= criteria.maxFailedTasks) {
+        await this.interrupt(
+          batchId,
+          'failed_tasks_exceeded',
+          `Failed tasks (${stats.failed}) exceeded maximum (${criteria.maxFailedTasks})`
+        );
+        return true;
+      }
+    }
+
+    // 7. Check maxErrorRate (percentage)
+    if (criteria.maxErrorRate && stats.total > 0) {
+      const errorRate = stats.failed / stats.total;
+      if (errorRate > criteria.maxErrorRate) {
+        await this.interrupt(
+          batchId,
+          'error_rate_exceeded',
+          `Error rate (${(errorRate * 100).toFixed(1)}%) exceeded maximum (${(criteria.maxErrorRate * 100).toFixed(1)}%)`
+        );
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+  /**
+   * Interrupt a batch atomically.
+   */
+  async interrupt(
+    batchId: string,
+    reason: i.BatchInterruptionReason,
+    message: string
+  ): Promise<void> {
+    // Get current stats for the log
+    const stats = await this.batchService.getStats(batchId);
+
+    // Update batch status to 'interrupted'
+    await this.batchService.updateStatus(batchId, 'interrupted');
+
+    // Log the interruption event
+    const id = crypto.randomUUID();
+    const now = new Date().toISOString();
+
+    this.db.run(
+      `INSERT INTO batch_interrupt_log (id, batch_id, reason, message, stats_snapshot, created_at)
+       VALUES (?, ?, ?, ?, ?, ?)`,
+      [id, batchId, reason, message, JSON.stringify(stats), now]
+    );
+  }
+
+  /**
+   * Check if a batch is active (can claim tasks from it).
+   */
+  async isBatchActive(batchId: string): Promise<boolean> {
+    const batch = await this.batchService.getById(batchId);
+    return batch?.status === 'active';
+  }
+
+  /**
+   * Get interruption history for a batch.
+   */
+  async getInterruptionLog(batchId: string): Promise<i.BatchInterruption[]> {
+    const rows = this.db.query<BatchInterruptionRow>(
+      `SELECT batch_id, reason, message, stats_snapshot, created_at
+       FROM batch_interrupt_log
+       WHERE batch_id = ?
+       ORDER BY created_at DESC`,
+      [batchId]
+    );
+
+    return rows.map((row) => ({
+      batchId: row.batch_id,
+      reason: row.reason,
+      message: row.message,
+      statsAtInterruption: JSON.parse(row.stats_snapshot) as i.BatchStats,
+      createdAt: new Date(row.created_at),
+    }));
+  }
+}

package/src/services/batch.ts

@@ -5,9 +5,11 @@ export interface BatchRow {
   id: string;
   code: string;
   type: string;
+  status: i.BatchStatus;
   created_at: string;
   completed_at: string | null;
   metadata: string | null;
+  interruption_criteria: string | null;
 }
 
 function rowToBatch(row: BatchRow): i.Batch {
@@ -15,9 +17,13 @@ function rowToBatch(row: BatchRow): i.Batch {
     id: row.id,
     code: row.code,
     type: row.type,
+    status: row.status ?? 'active',
     createdAt: new Date(row.created_at),
     completedAt: row.completed_at ? new Date(row.completed_at) : null,
     metadata: row.metadata ? (JSON.parse(row.metadata) as Record<string, unknown>) : null,
+    interruptionCriteria: row.interruption_criteria
+      ? (JSON.parse(row.interruption_criteria) as i.BatchInterruptionCriteria)
+      : null,
   };
 }
 
@@ -29,9 +35,16 @@ export class BatchService implements i.BatchService {
     const now = new Date().toISOString();
 
     this.db.run(
-      `INSERT INTO batch (id, code, type, created_at, metadata)
-       VALUES (?, ?, ?, ?, ?)`,
-      [id, input.code, input.type, now, input.metadata ? JSON.stringify(input.metadata) : null]
+      `INSERT INTO batch (id, code, type, created_at, metadata, interruption_criteria)
+       VALUES (?, ?, ?, ?, ?, ?)`,
+      [
+        id,
+        input.code,
+        input.type,
+        now,
+        input.metadata ? JSON.stringify(input.metadata) : null,
+        input.interruptionCriteria ? JSON.stringify(input.interruptionCriteria) : null,
+      ]
     );
 
     const rows = this.db.query<BatchRow>('SELECT * FROM batch WHERE id = ?', [id]);
@@ -60,7 +73,7 @@ export class BatchService implements i.BatchService {
   async complete(id: string): Promise<void> {
     const now = new Date().toISOString();
     this.db.run(
-      `UPDATE batch SET completed_at = ? WHERE id = ?`,
+      `UPDATE batch SET completed_at = ?, status = 'completed' WHERE id = ?`,
       [now, id]
     );
   }
@@ -118,4 +131,19 @@ export class BatchService implements i.BatchService {
     );
     return result.changes ?? 0;
   }
+
+  async updateStatus(id: string, status: i.BatchStatus): Promise<void> {
+    this.db.run(
+      `UPDATE batch SET status = ? WHERE id = ?`,
+      [status, id]
+    );
+  }
+
+  async getWithCriteria(id: string): Promise<{ batch: i.Batch; criteria: i.BatchInterruptionCriteria | null }> {
+    const batch = await this.getById(id);
+    if (!batch) {
+      throw new Error(`Batch not found: ${id}`);
+    }
+    return { batch, criteria: batch.interruptionCriteria };
+  }
 }

package/src/services/index.ts

@@ -1,2 +1,3 @@
 export { BatchService } from './batch.js';
 export { TaskService } from './task.js';
+export { BatchInterruptionService } from './batch-interruption.js';

package/src/utils/auto-detect.ts

@@ -6,6 +6,7 @@ export interface DetectOptions {
   timeoutMs?: number;
   maxConcurrencyToTest?: number;
   rateLimitTestDurationMs?: number;
+  maxRateLimitTestRequests?: number;
 }
 
 export interface DetectedConfig {
@@ -48,6 +49,7 @@ export async function detectConstraints(options: DetectOptions): Promise<Detecte
   const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
   const maxConcurrency = options.maxConcurrencyToTest ?? DEFAULT_MAX_CONCURRENCY;
   const rateLimitDurationMs = options.rateLimitTestDurationMs ?? DEFAULT_RATE_LIMIT_TEST_DURATION_MS;
+  const maxRateLimitTestRequests = options.maxRateLimitTestRequests ?? MAX_RATE_LIMIT_TEST_REQUESTS;
 
   const requestOptions: RequestOptions = {
     url: options.url,
@@ -64,7 +66,7 @@ export async function detectConstraints(options: DetectOptions): Promise<Detecte
 
   // Phase 2: Detect rate limit using safe concurrency (80% of detected)
   const safeConcurrency = Math.max(1, Math.floor(detectedConcurrency * 0.8));
-  const rateLimitResult = await detectRateLimit(requestOptions, safeConcurrency, rateLimitDurationMs, notes);
+  const rateLimitResult = await detectRateLimit(requestOptions, safeConcurrency, rateLimitDurationMs, maxRateLimitTestRequests, notes);
 
   // Calculate confidence
   const confidence = calculateConfidence(detectedConcurrency, rateLimitResult, notes);
@@ -206,6 +208,7 @@ async function detectRateLimit(
   options: RequestOptions,
   safeConcurrency: number,
   durationMs: number,
+  maxRequests: number,
   notes: string[]
 ): Promise<{ requests: number; windowMs: number }> {
   const startTime = Date.now();
@@ -214,7 +217,7 @@ async function detectRateLimit(
   let totalRequests = 0;
 
   // Send requests as fast as possible at safe concurrency
-  while (Date.now() - startTime < durationMs && totalRequests < MAX_RATE_LIMIT_TEST_REQUESTS) {
+  while (Date.now() - startTime < durationMs && totalRequests < maxRequests) {
     const batchStart = Date.now();
     const results = await sendBatch(options, safeConcurrency);