@qianxude/tem 0.2.0

package/src/database/index.ts

@@ -0,0 +1,114 @@
+import { Database as SQLiteDatabase, type SQLQueryBindings } from 'bun:sqlite';
+import * as i from '../interfaces/index.js';
+
+export interface DatabaseOptions {
+  path: string;
+  busyTimeout?: number;
+}
+
+export class Database implements i.DatabaseConnection {
+  private db: SQLiteDatabase;
+
+  constructor(options: DatabaseOptions) {
+    this.db = new SQLiteDatabase(options.path);
+
+    // Enable WAL mode for better concurrency
+    this.db.exec('PRAGMA journal_mode = WAL;');
+
+    // Set busy timeout for concurrent access safety (default 5 seconds)
+    const timeout = options.busyTimeout ?? 5000;
+    this.db.exec(`PRAGMA busy_timeout = ${timeout};`);
+
+    // Run migrations
+    this.migrate();
+  }
+
+  private migrate(): void {
+    // Create migration tracking table first
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS _migration (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        name TEXT NOT NULL UNIQUE,
+        applied_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+      )
+    `);
+
+    // Check if initial schema needs to be applied
+    const migrationCount = this.db
+      .query('SELECT COUNT(*) as count FROM _migration WHERE name = $name')
+      .get({ $name: '001_initial_schema' }) as { count: number };
+
+    if (migrationCount.count === 0) {
+      this.applyInitialSchema();
+    }
+  }
+
+  private applyInitialSchema(): void {
+    const schema = `
+      -- Batch: Groups of related tasks
+      CREATE TABLE IF NOT EXISTS batch (
+        id TEXT PRIMARY KEY,
+        code TEXT NOT NULL UNIQUE,
+        type TEXT NOT NULL,
+        created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+        completed_at DATETIME,
+        metadata TEXT
+      );
+
+      -- Task: Individual units of work
+      CREATE TABLE IF NOT EXISTS task (
+        id TEXT PRIMARY KEY,
+        batch_id TEXT REFERENCES batch(id) ON DELETE CASCADE,
+        type TEXT NOT NULL,
+        status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'completed', 'failed')),
+        payload TEXT NOT NULL,
+        result TEXT,
+        error TEXT,
+        attempt INTEGER NOT NULL DEFAULT 0,
+        max_attempt INTEGER NOT NULL DEFAULT 3,
+        claimed_at DATETIME,
+        completed_at DATETIME,
+        version INTEGER NOT NULL DEFAULT 0,
+        created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+      );
+
+      -- Indexes for performance
+      CREATE INDEX IF NOT EXISTS idx_batch_code ON batch(code);
+      CREATE INDEX IF NOT EXISTS idx_batch_type ON batch(type);
+      CREATE INDEX IF NOT EXISTS idx_task_batch_id ON task(batch_id);
+      CREATE INDEX IF NOT EXISTS idx_task_status ON task(status);
+      CREATE INDEX IF NOT EXISTS idx_task_type ON task(type);
+      CREATE INDEX IF NOT EXISTS idx_task_claim ON task(status, claimed_at);
+      CREATE INDEX IF NOT EXISTS idx_task_pending ON task(status, created_at) WHERE status = 'pending';
+    `;
+
+    this.transaction(() => {
+      this.db.exec(schema);
+      this.db
+        .query('INSERT INTO _migration (name) VALUES ($name)')
+        .run({ $name: '001_initial_schema' });
+    });
+  }
+
+  query<T = unknown>(sql: string, params?: SQLQueryBindings[]): T[] {
+    const stmt = this.db.prepare(sql);
+    const results = stmt.all(...(params ?? []));
+    stmt.finalize();
+    return results as T[];
+  }
+
+  run(sql: string, params?: SQLQueryBindings[]): { changes: number } {
+    const stmt = this.db.prepare(sql);
+    const result = stmt.run(...(params ?? []));
+    stmt.finalize();
+    return { changes: result.changes };
+  }
+
+  transaction<T>(fn: () => T): T {
+    return this.db.transaction(fn)();
+  }
+
+  close(): void {
+    this.db.close();
+  }
+}

package/src/database/schema.sql

@@ -0,0 +1,45 @@
+-- TEM Database Schema
+-- SQLite with WAL mode
+
+-- Migration tracking
+CREATE TABLE IF NOT EXISTS _migration (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  name TEXT NOT NULL UNIQUE,
+  applied_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Batch: Groups of related tasks
+CREATE TABLE IF NOT EXISTS batch (
+  id TEXT PRIMARY KEY,
+  code TEXT NOT NULL UNIQUE,
+  type TEXT NOT NULL,
+  created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  completed_at DATETIME,
+  metadata TEXT -- JSON object
+);
+
+-- Task: Individual units of work
+CREATE TABLE IF NOT EXISTS task (
+  id TEXT PRIMARY KEY,
+  batch_id TEXT REFERENCES batch(id) ON DELETE CASCADE,
+  type TEXT NOT NULL,
+  status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'completed', 'failed')),
+  payload TEXT NOT NULL, -- JSON object (opaque)
+  result TEXT, -- JSON object (opaque)
+  error TEXT,
+  attempt INTEGER NOT NULL DEFAULT 0,
+  max_attempt INTEGER NOT NULL DEFAULT 3,
+  claimed_at DATETIME,
+  completed_at DATETIME,
+  version INTEGER NOT NULL DEFAULT 0,
+  created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Indexes for performance
+CREATE INDEX IF NOT EXISTS idx_batch_code ON batch(code);
+CREATE INDEX IF NOT EXISTS idx_batch_type ON batch(type);
+CREATE INDEX IF NOT EXISTS idx_task_batch_id ON task(batch_id);
+CREATE INDEX IF NOT EXISTS idx_task_status ON task(status);
+CREATE INDEX IF NOT EXISTS idx_task_type ON task(type);
+CREATE INDEX IF NOT EXISTS idx_task_claim ON task(status, claimed_at);
+CREATE INDEX IF NOT EXISTS idx_task_pending ON task(status, created_at) WHERE status = 'pending';

package/src/index.ts

@@ -0,0 +1,19 @@
+// Main exports for TEM framework
+export * as interfaces from './interfaces/index.js';
+export { Database, type DatabaseOptions } from './database/index.js';
+export { BatchService, TaskService } from './services/index.js';
+export {
+  ConcurrencyController,
+  RateLimiter,
+  printDetectedConfig,
+  type RateLimitConfig,
+} from './utils/index.js';
+export {
+  TEM,
+  Worker,
+  NonRetryableError,
+  type TEMConfig,
+  type WorkerConfig,
+  type DetectOptions,
+  type DetectedConfig,
+} from './core/index.js';

package/src/interfaces/index.ts

@@ -0,0 +1,186 @@
+// Public API types for TEM
+// Import as: import * as i from './interfaces'
+
+// ============================================================================
+// Enums
+// ============================================================================
+
+export type TaskStatus = 'pending' | 'running' | 'completed' | 'failed';
+
+// ============================================================================
+// Entity Types
+// ============================================================================
+
+export interface Batch {
+  id: string;
+  code: string;
+  type: string;
+  createdAt: Date;
+  completedAt: Date | null;
+  metadata: Record<string, unknown> | null;
+}
+
+export interface BatchStats {
+  batchId: string;
+  total: number;
+  pending: number;
+  running: number;
+  completed: number;
+  failed: number;
+}
+
+export interface Task {
+  id: string;
+  batchId: string | null;
+  type: string;
+  status: TaskStatus;
+  payload: string; // JSON string - opaque to framework
+  result: string | null; // JSON string - opaque to framework
+  error: string | null;
+  attempt: number;
+  maxAttempt: number;
+  claimedAt: Date | null;
+  completedAt: Date | null;
+  version: number;
+  createdAt: Date;
+}
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+export interface TEMConfig {
+  // Database
+  databasePath: string;
+
+  // Concurrency
+  concurrency: number;
+
+  // Rate limiting
+  rateLimit?: {
+    requests: number;
+    windowMs: number;
+  };
+
+  // Retry
+  defaultMaxAttempts: number;
+
+  // Polling
+  pollIntervalMs: number;
+}
+
+// ============================================================================
+// Auto-Detect Configuration
+// ============================================================================
+
+export interface DetectOptions {
+  /** Target URL to test */
+  url: string;
+  /** HTTP method to use (default: GET) */
+  method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH';
+  /** Request headers to include */
+  headers?: Record<string, string>;
+  /** Request body (will be JSON stringified for POST/PUT/PATCH) */
+  body?: unknown;
+  /** Request timeout in milliseconds (default: 30000) */
+  timeoutMs?: number;
+  /** Maximum concurrency level to test (default: 100) */
+  maxConcurrencyToTest?: number;
+  /** Duration to run rate limit tests (default: 10000) */
+  rateLimitTestDurationMs?: number;
+}
+
+export interface DetectedConfig {
+  /** Recommended concurrency (80% of detected max) */
+  concurrency: number;
+  /** Recommended rate limit (90% of detected limit) */
+  rateLimit: {
+    requests: number;
+    windowMs: number;
+  };
+  /** Confidence level in the detection results */
+  confidence: 'high' | 'medium' | 'low';
+  /** Notes about the detection process and findings */
+  notes: string[];
+}
+
+// ============================================================================
+// Task Handler
+// ============================================================================
+
+export type TaskHandler<TInput = unknown, TOutput = unknown> = (
+  payload: TInput,
+  context: TaskContext
+) => Promise<TOutput>;
+
+export interface TaskContext {
+  taskId: string;
+  batchId: string | null;
+  attempt: number;
+  signal: AbortSignal;
+}
+
+// ============================================================================
+// Error Types
+// ============================================================================
+
+/**
+ * Error class to mark errors as non-retryable.
+ * When thrown from a task handler, the task will fail immediately
+ * without retry attempts.
+ */
+export class NonRetryableError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'NonRetryableError';
+  }
+}
+
+// ============================================================================
+// Service Interfaces
+// ============================================================================
+
+export interface CreateBatchInput {
+  code: string;
+  type: string;
+  metadata?: Record<string, unknown>;
+}
+
+export interface CreateTaskInput {
+  batchId?: string;
+  type: string;
+  payload: unknown;
+  maxAttempt?: number;
+}
+
+export interface BatchService {
+  create(input: CreateBatchInput): Promise<Batch>;
+  getById(id: string): Promise<Batch | null>;
+  getByCode(code: string): Promise<Batch | null>;
+  list(filter?: { type?: string }): Promise<Batch[]>;
+  getStats(id: string): Promise<BatchStats>;
+  complete(id: string): Promise<void>;
+  resume(id: string): Promise<number>;
+  retryFailed(id: string): Promise<number>;
+}
+
+export interface TaskService {
+  create(input: CreateTaskInput): Promise<Task>;
+  createMany(inputs: CreateTaskInput[]): Promise<Task[]>;
+  getById(id: string): Promise<Task | null>;
+  claim(batchId?: string): Promise<Task | null>;
+  complete(id: string, result: unknown): Promise<void>;
+  fail(id: string, error: string): Promise<void>;
+  retry(id: string): Promise<void>;
+}
+
+// ============================================================================
+// Database
+// ============================================================================
+
+export interface DatabaseConnection {
+  query<T = unknown>(sql: string, params?: unknown[]): T[];
+  run(sql: string, params?: unknown[]): { changes: number };
+  transaction<T>(fn: () => T): T;
+  close(): void;
+}