dialekt 0.1.0

package/src/benchmark/metrics.test.ts

@@ -0,0 +1,180 @@
+import { describe, expect, it } from 'vitest';
+import { Effect } from 'effect';
+import { summarizeBenchmarkResults, runBenchmarkedChunk } from './metrics.js';
+import type { ChunkBenchmarkResult } from './metrics.js';
+import type { TranslationStrategy, TranslationContext } from '../translation/types.js';
+import { TranslationFailedError } from '../translation/types.js';
+
+describe('summarizeBenchmarkResults', () => {
+  it('computes totals and averages correctly', () => {
+    const results = [
+      { strategyName: 'one-shot' as const, chunkKeyCount: 2, durationMs: 100, attemptCount: 1, succeeded: true },
+      { strategyName: 'one-shot' as const, chunkKeyCount: 2, durationMs: 200, attemptCount: 1, succeeded: true },
+      { strategyName: 'one-shot' as const, chunkKeyCount: 1, durationMs: 300, attemptCount: 1, succeeded: false, errorMessage: 'oops' },
+    ];
+    const summary = summarizeBenchmarkResults(results);
+    expect(summary.totalChunks).toBe(3);
+    expect(summary.succeededChunks).toBe(2);
+    expect(summary.failedChunks).toBe(1);
+    expect(summary.totalDurationMs).toBe(600);
+    expect(summary.averageDurationMsPerChunk).toBe(200);
+    expect(summary.totalAttempts).toBe(3);
+  });
+
+  it('handles empty results', () => {
+    const summary = summarizeBenchmarkResults([]);
+    expect(summary.totalChunks).toBe(0);
+    expect(summary.succeededChunks).toBe(0);
+    expect(summary.failedChunks).toBe(0);
+    expect(summary.totalDurationMs).toBe(0);
+    expect(summary.averageDurationMsPerChunk).toBe(0);
+    expect(summary.totalAttempts).toBe(0);
+    expect(summary.strategyName).toBe('one-shot');
+  });
+
+  it('handles single result', () => {
+    const results = [
+      { strategyName: 'tool-loop-agent' as const, chunkKeyCount: 5, durationMs: 150, attemptCount: 2, succeeded: true },
+    ];
+    const summary = summarizeBenchmarkResults(results);
+    expect(summary.totalChunks).toBe(1);
+    expect(summary.succeededChunks).toBe(1);
+    expect(summary.failedChunks).toBe(0);
+    expect(summary.averageDurationMsPerChunk).toBe(150);
+    expect(summary.totalAttempts).toBe(2);
+  });
+
+  it('handles all failures', () => {
+    const results = [
+      { strategyName: 'one-shot' as const, chunkKeyCount: 1, durationMs: 50, attemptCount: 1, succeeded: false, errorMessage: 'a' },
+      { strategyName: 'one-shot' as const, chunkKeyCount: 1, durationMs: 60, attemptCount: 1, succeeded: false, errorMessage: 'b' },
+    ];
+    const summary = summarizeBenchmarkResults(results);
+    expect(summary.succeededChunks).toBe(0);
+    expect(summary.failedChunks).toBe(2);
+    expect(summary.totalDurationMs).toBe(110);
+  });
+
+  it('handles zero-duration results', () => {
+    const results = [
+      { strategyName: 'one-shot' as const, chunkKeyCount: 1, durationMs: 0, attemptCount: 1, succeeded: true },
+    ];
+    const summary = summarizeBenchmarkResults(results);
+    expect(summary.totalDurationMs).toBe(0);
+    expect(summary.averageDurationMsPerChunk).toBe(0);
+  });
+
+  it('preserves strategyName from first result', () => {
+    const results = [
+      { strategyName: 'tool-loop-agent' as const, chunkKeyCount: 1, durationMs: 100, attemptCount: 1, succeeded: true },
+    ];
+    const summary = summarizeBenchmarkResults(results);
+    expect(summary.strategyName).toBe('tool-loop-agent');
+  });
+
+  it('handles very large durations without overflow', () => {
+    const results = [
+      { strategyName: 'one-shot' as const, chunkKeyCount: 1, durationMs: 1_000_000, attemptCount: 1, succeeded: true },
+      { strategyName: 'one-shot' as const, chunkKeyCount: 1, durationMs: 2_000_000, attemptCount: 1, succeeded: true },
+    ];
+    const summary = summarizeBenchmarkResults(results);
+    expect(summary.totalDurationMs).toBe(3_000_000);
+    expect(summary.averageDurationMsPerChunk).toBe(1_500_000);
+  });
+});
+
+describe('runBenchmarkedChunk', () => {
+  it('measures duration and returns success', async () => {
+    const strategy: TranslationStrategy = {
+      name: 'one-shot',
+      translateChunk: () => Effect.succeed({ hello: 'Hallo' }),
+    };
+    const ctx: TranslationContext = {
+      sourceLocale: 'en',
+      targetLocale: 'de',
+      sourceMap: { hello: 'Hello' },
+      targetMap: {},
+      keys: ['hello'],
+    };
+    const result = await Effect.runPromise(runBenchmarkedChunk(strategy, ctx)) as ChunkBenchmarkResult;
+    expect(result.succeeded).toBe(true);
+    expect(result.chunkKeyCount).toBe(1);
+    expect(result.durationMs).toBeGreaterThanOrEqual(0);
+    expect(result.strategyName).toBe('one-shot');
+  });
+
+  it('records failure without propagating', async () => {
+    const strategy: TranslationStrategy = {
+      name: 'tool-loop-agent',
+      translateChunk: () =>
+        Effect.fail(new TranslationFailedError({ keys: ['a'], cause: 'boom' })),
+    };
+    const ctx: TranslationContext = {
+      sourceLocale: 'en',
+      targetLocale: 'de',
+      sourceMap: { a: 'A' },
+      targetMap: {},
+      keys: ['a'],
+    };
+    const result = await Effect.runPromise(runBenchmarkedChunk(strategy, ctx)) as ChunkBenchmarkResult;
+    expect(result.succeeded).toBe(false);
+    expect(result.errorMessage).toBe('boom');
+    expect(result.chunkKeyCount).toBe(1);
+  });
+
+  it('handles empty key list', async () => {
+    const strategy: TranslationStrategy = {
+      name: 'one-shot',
+      translateChunk: () => Effect.succeed({}),
+    };
+    const ctx: TranslationContext = {
+      sourceLocale: 'en',
+      targetLocale: 'de',
+      sourceMap: {},
+      targetMap: {},
+      keys: [],
+    };
+    const result = await Effect.runPromise(runBenchmarkedChunk(strategy, ctx)) as ChunkBenchmarkResult;
+    expect(result.succeeded).toBe(true);
+    expect(result.chunkKeyCount).toBe(0);
+  });
+
+  it('measures duration for slow strategies', async () => {
+    const strategy: TranslationStrategy = {
+      name: 'tool-loop-agent',
+      translateChunk: () =>
+        Effect.gen(function* () {
+          yield* Effect.sleep('50 millis');
+          return { k: 'v' };
+        }),
+    };
+    const ctx: TranslationContext = {
+      sourceLocale: 'en',
+      targetLocale: 'de',
+      sourceMap: { k: 'K' },
+      targetMap: {},
+      keys: ['k'],
+    };
+    const result = await Effect.runPromise(runBenchmarkedChunk(strategy, ctx)) as ChunkBenchmarkResult;
+    expect(result.succeeded).toBe(true);
+    expect(result.durationMs).toBeGreaterThanOrEqual(30);
+  });
+
+  it('captures Error cause in errorMessage', async () => {
+    const strategy: TranslationStrategy = {
+      name: 'one-shot',
+      translateChunk: () =>
+        Effect.fail(new TranslationFailedError({ keys: ['x'], cause: new Error('deep error') })),
+    };
+    const ctx: TranslationContext = {
+      sourceLocale: 'en',
+      targetLocale: 'de',
+      sourceMap: { x: 'X' },
+      targetMap: {},
+      keys: ['x'],
+    };
+    const result = await Effect.runPromise(runBenchmarkedChunk(strategy, ctx)) as ChunkBenchmarkResult;
+    expect(result.succeeded).toBe(false);
+    expect(result.errorMessage).toContain('deep error');
+  });
+});

package/src/benchmark/metrics.ts

@@ -0,0 +1,69 @@
+import { Effect } from 'effect';
+import type { TranslationStrategy, TranslationContext } from '../translation/types.js';
+
+export interface ChunkBenchmarkResult {
+  readonly strategyName: 'one-shot' | 'tool-loop-agent';
+  readonly chunkKeyCount: number;
+  readonly durationMs: number;
+  readonly attemptCount: number;
+  readonly succeeded: boolean;
+  readonly errorMessage?: string | undefined;
+}
+
+export interface StrategyBenchmarkSummary {
+  readonly strategyName: 'one-shot' | 'tool-loop-agent';
+  readonly totalChunks: number;
+  readonly succeededChunks: number;
+  readonly failedChunks: number;
+  readonly totalDurationMs: number;
+  readonly averageDurationMsPerChunk: number;
+  readonly totalAttempts: number;
+}
+
+export function summarizeBenchmarkResults(
+  results: readonly ChunkBenchmarkResult[],
+): StrategyBenchmarkSummary {
+  const totalChunks = results.length;
+  const succeededChunks = results.filter((r) => r.succeeded).length;
+  const failedChunks = totalChunks - succeededChunks;
+  const totalDurationMs = results.reduce((sum, r) => sum + r.durationMs, 0);
+  const totalAttempts = results.reduce((sum, r) => sum + r.attemptCount, 0);
+  return {
+    strategyName: results[0]?.strategyName ?? 'one-shot',
+    totalChunks,
+    succeededChunks,
+    failedChunks,
+    totalDurationMs,
+    averageDurationMsPerChunk: totalChunks > 0 ? totalDurationMs / totalChunks : 0,
+    totalAttempts,
+  };
+}
+
+export function runBenchmarkedChunk(
+  strategy: TranslationStrategy,
+  ctx: TranslationContext,
+): Effect.Effect<ChunkBenchmarkResult, never> {
+  return Effect.gen(function* () {
+    const start = Date.now();
+    const result = yield* Effect.either(strategy.translateChunk(ctx));
+    const durationMs = Date.now() - start;
+    if (result._tag === 'Right') {
+      return {
+        strategyName: strategy.name,
+        chunkKeyCount: ctx.keys.length,
+        durationMs,
+        attemptCount: 1,
+        succeeded: true as const,
+        errorMessage: undefined,
+      };
+    }
+    return {
+      strategyName: strategy.name,
+      chunkKeyCount: ctx.keys.length,
+      durationMs,
+      attemptCount: 1,
+      succeeded: false as const,
+      errorMessage: String((result.left as { cause?: unknown }).cause),
+    };
+  });
+}

package/src/benchmark/report.test.ts

@@ -0,0 +1,129 @@
+import { describe, expect, it } from 'vitest';
+import { formatBenchmarkReport } from './report.js';
+import type { StrategyBenchmarkSummary } from './metrics.js';
+
+describe('formatBenchmarkReport', () => {
+  it('produces valid JSON round-trip', () => {
+    const summaries: StrategyBenchmarkSummary[] = [
+      {
+        strategyName: 'one-shot',
+        totalChunks: 3,
+        succeededChunks: 3,
+        failedChunks: 0,
+        totalDurationMs: 300,
+        averageDurationMsPerChunk: 100,
+        totalAttempts: 3,
+      },
+    ];
+    const json = formatBenchmarkReport(summaries, 'json');
+    expect(JSON.parse(json)).toEqual(summaries);
+  });
+
+  it('produces human-readable table with key metrics', () => {
+    const summaries: StrategyBenchmarkSummary[] = [
+      {
+        strategyName: 'one-shot',
+        totalChunks: 3,
+        succeededChunks: 3,
+        failedChunks: 0,
+        totalDurationMs: 300,
+        averageDurationMsPerChunk: 100,
+        totalAttempts: 3,
+      },
+    ];
+    const table = formatBenchmarkReport(summaries, 'table');
+    expect(table).toContain('one-shot');
+    expect(table).toContain('300');
+    expect(table).toContain('100.0');
+    expect(table).toContain('3');
+  });
+
+  it('handles multiple strategies in table format', () => {
+    const summaries: StrategyBenchmarkSummary[] = [
+      {
+        strategyName: 'one-shot',
+        totalChunks: 5,
+        succeededChunks: 5,
+        failedChunks: 0,
+        totalDurationMs: 500,
+        averageDurationMsPerChunk: 100,
+        totalAttempts: 5,
+      },
+      {
+        strategyName: 'tool-loop-agent',
+        totalChunks: 5,
+        succeededChunks: 4,
+        failedChunks: 1,
+        totalDurationMs: 750,
+        averageDurationMsPerChunk: 150,
+        totalAttempts: 6,
+      },
+    ];
+    const table = formatBenchmarkReport(summaries, 'table');
+    expect(table).toContain('one-shot');
+    expect(table).toContain('tool-loop-agent');
+    expect(table).toContain('4 ok, 1 failed');
+  });
+
+  it('handles empty summaries in JSON format', () => {
+    const json = formatBenchmarkReport([], 'json');
+    expect(JSON.parse(json)).toEqual([]);
+  });
+
+  it('handles empty summaries in table format', () => {
+    const table = formatBenchmarkReport([], 'table');
+    expect(table).toContain('Benchmark Results');
+  });
+
+  it('handles all-failed strategy in table', () => {
+    const summaries: StrategyBenchmarkSummary[] = [
+      {
+        strategyName: 'one-shot',
+        totalChunks: 3,
+        succeededChunks: 0,
+        failedChunks: 3,
+        totalDurationMs: 300,
+        averageDurationMsPerChunk: 100,
+        totalAttempts: 3,
+      },
+    ];
+    const table = formatBenchmarkReport(summaries, 'table');
+    expect(table).toContain('0 ok, 3 failed');
+  });
+
+  it('handles zero-duration strategy', () => {
+    const summaries: StrategyBenchmarkSummary[] = [
+      {
+        strategyName: 'one-shot',
+        totalChunks: 1,
+        succeededChunks: 1,
+        failedChunks: 0,
+        totalDurationMs: 0,
+        averageDurationMsPerChunk: 0,
+        totalAttempts: 1,
+      },
+    ];
+    const table = formatBenchmarkReport(summaries, 'table');
+    expect(table).toContain('0ms');
+    expect(table).toContain('0.0ms');
+  });
+
+  it('JSON format includes all numeric fields', () => {
+    const summaries: StrategyBenchmarkSummary[] = [
+      {
+        strategyName: 'tool-loop-agent',
+        totalChunks: 2,
+        succeededChunks: 1,
+        failedChunks: 1,
+        totalDurationMs: 1234,
+        averageDurationMsPerChunk: 617,
+        totalAttempts: 4,
+      },
+    ];
+    const json = formatBenchmarkReport(summaries, 'json');
+    const parsed = JSON.parse(json) as StrategyBenchmarkSummary[];
+    expect(parsed[0]!.totalChunks).toBe(2);
+    expect(parsed[0]!.totalAttempts).toBe(4);
+    expect(parsed[0]!.averageDurationMsPerChunk).toBe(617);
+  });
+});

package/src/benchmark/report.ts

@@ -0,0 +1,21 @@
+import type { StrategyBenchmarkSummary } from './metrics.js';
+
+export function formatBenchmarkReport(
+  summaries: readonly StrategyBenchmarkSummary[],
+  format: 'table' | 'json',
+): string {
+  if (format === 'json') {
+    return JSON.stringify(summaries, null, 2);
+  }
+
+  const lines: string[] = ['Benchmark Results', '================='];
+  for (const s of summaries) {
+    lines.push(`Strategy: ${s.strategyName}`);
+    lines.push(`  Chunks: ${s.totalChunks} (${s.succeededChunks} ok, ${s.failedChunks} failed)`);
+    lines.push(`  Total duration: ${s.totalDurationMs.toFixed(0)}ms`);
+    lines.push(`  Avg per chunk: ${s.averageDurationMsPerChunk.toFixed(1)}ms`);
+    lines.push(`  Total attempts: ${s.totalAttempts}`);
+    lines.push('');
+  }
+  return lines.join('\n');
+}

package/src/benchmark/runner.test.ts

@@ -0,0 +1,162 @@
+import { describe, expect, it } from 'vitest';
+import { Effect } from 'effect';
+import { runBenchmark } from './runner.js';
+import type { StrategyBenchmarkSummary } from './metrics.js';
+import type { TranslationStrategy, TranslationContext } from '../translation/types.js';
+import { TranslationFailedError } from '../translation/types.js';
+
+describe('runBenchmark', () => {
+  it('returns summaries for both strategies', async () => {
+    const a: TranslationStrategy = {
+      name: 'one-shot',
+      translateChunk: () => Effect.succeed({ k: 'a' }),
+    };
+    const b: TranslationStrategy = {
+      name: 'tool-loop-agent',
+      translateChunk: () => Effect.succeed({ k: 'b' }),
+    };
+    const chunks: TranslationContext[] = [
+      {
+        sourceLocale: 'en',
+        targetLocale: 'de',
+        sourceMap: { k: 'K' },
+        targetMap: {},
+        keys: ['k'],
+      },
+    ];
+    const result = await Effect.runPromise(
+      runBenchmark({ strategies: [a, b], chunks, concurrency: 1 }),
+    ) as StrategyBenchmarkSummary[];
+    expect(result).toHaveLength(2);
+    expect(result[0]!.strategyName).toBe('one-shot');
+    expect(result[1]!.strategyName).toBe('tool-loop-agent');
+  });
+
+  it('does not abort when one strategy fails every chunk', async () => {
+    const a: TranslationStrategy = {
+      name: 'one-shot',
+      translateChunk: () => Effect.succeed({ k: 'a' }),
+    };
+    const b: TranslationStrategy = {
+      name: 'tool-loop-agent',
+      translateChunk: () =>
+        Effect.fail(new TranslationFailedError({ keys: ['k'], cause: 'boom' })),
+    };
+    const chunks: TranslationContext[] = [
+      {
+        sourceLocale: 'en',
+        targetLocale: 'de',
+        sourceMap: { k: 'K' },
+        targetMap: {},
+        keys: ['k'],
+      },
+    ];
+    const result = await Effect.runPromise(
+      runBenchmark({ strategies: [a, b], chunks, concurrency: 1 }),
+    ) as StrategyBenchmarkSummary[];
+    expect(result[0]!.succeededChunks).toBe(1);
+    expect(result[1]!.failedChunks).toBe(1);
+  });
+
+  it('handles empty chunks', async () => {
+    const a: TranslationStrategy = {
+      name: 'one-shot',
+      translateChunk: () => Effect.succeed({}),
+    };
+    const result = await Effect.runPromise(
+      runBenchmark({ strategies: [a], chunks: [], concurrency: 1 }),
+    ) as StrategyBenchmarkSummary[];
+    expect(result).toHaveLength(1);
+    expect(result[0]!.totalChunks).toBe(0);
+    expect(result[0]!.succeededChunks).toBe(0);
+  });
+
+  it('handles single strategy', async () => {
+    const a: TranslationStrategy = {
+      name: 'one-shot',
+      translateChunk: () => Effect.succeed({ a: 'A', b: 'B' }),
+    };
+    const chunks: TranslationContext[] = [
+      {
+        sourceLocale: 'en',
+        targetLocale: 'de',
+        sourceMap: { a: 'A', b: 'B' },
+        targetMap: {},
+        keys: ['a', 'b'],
+      },
+    ];
+    const result = await Effect.runPromise(
+      runBenchmark({ strategies: [a], chunks, concurrency: 1 }),
+    ) as StrategyBenchmarkSummary[];
+    expect(result).toHaveLength(1);
+    expect(result[0]!.totalChunks).toBe(1);
+    expect(result[0]!.succeededChunks).toBe(1);
+  });
+
+  it('handles multiple chunks with concurrency', async () => {
+    const a: TranslationStrategy = {
+      name: 'one-shot',
+      translateChunk: () => Effect.succeed({ k: 'v' }),
+    };
+    const chunks: TranslationContext[] = Array.from({ length: 5 }, (_, i) => ({
+      sourceLocale: 'en',
+      targetLocale: 'de',
+      sourceMap: { [`k${i}`]: `V${i}` },
+      targetMap: {},
+      keys: [`k${i}`],
+    }));
+    const result = await Effect.runPromise(
+      runBenchmark({ strategies: [a], chunks, concurrency: 3 }),
+    ) as StrategyBenchmarkSummary[];
+    expect(result[0]!.totalChunks).toBe(5);
+    expect(result[0]!.succeededChunks).toBe(5);
+  });
+
+  it('handles all strategies failing', async () => {
+    const a: TranslationStrategy = {
+      name: 'one-shot',
+      translateChunk: () =>
+        Effect.fail(new TranslationFailedError({ keys: ['k'], cause: 'fail' })),
+    };
+    const chunks: TranslationContext[] = [
+      {
+        sourceLocale: 'en',
+        targetLocale: 'de',
+        sourceMap: { k: 'K' },
+        targetMap: {},
+        keys: ['k'],
+      },
+    ];
+    const result = await Effect.runPromise(
+      runBenchmark({ strategies: [a], chunks, concurrency: 1 }),
+    ) as StrategyBenchmarkSummary[];
+    expect(result[0]!.failedChunks).toBe(1);
+    expect(result[0]!.succeededChunks).toBe(0);
+  });
+
+  it('handles mixed success and failure across chunks', async () => {
+    let callCount = 0;
+    const a: TranslationStrategy = {
+      name: 'one-shot',
+      translateChunk: () => {
+        callCount++;
+        return callCount % 2 === 1
+          ? Effect.succeed({ k: 'v' })
+          : Effect.fail(new TranslationFailedError({ keys: ['k'], cause: 'odd' }));
+      },
+    };
+    const chunks: TranslationContext[] = Array.from({ length: 4 }, () => ({
+      sourceLocale: 'en',
+      targetLocale: 'de',
+      sourceMap: { k: 'K' },
+      targetMap: {},
+      keys: ['k'],
+    }));
+    const result = await Effect.runPromise(
+      runBenchmark({ strategies: [a], chunks, concurrency: 1 }),
+    ) as StrategyBenchmarkSummary[];
+    expect(result[0]!.totalChunks).toBe(4);
+    expect(result[0]!.succeededChunks).toBe(2);
+    expect(result[0]!.failedChunks).toBe(2);
+  });
+});

package/src/benchmark/runner.ts

@@ -0,0 +1,27 @@
+import { Effect } from 'effect';
+import type { TranslationStrategy, TranslationContext } from '../translation/types.js';
+import type { StrategyBenchmarkSummary } from './metrics.js';
+import { runBenchmarkedChunk, summarizeBenchmarkResults } from './metrics.js';
+
+export interface BenchmarkConfig {
+  readonly strategies: readonly TranslationStrategy[];
+  readonly chunks: readonly TranslationContext[];
+  readonly concurrency: number;
+}
+
+export function runBenchmark(
+  config: BenchmarkConfig,
+): Effect.Effect<readonly StrategyBenchmarkSummary[], never> {
+  return Effect.gen(function* () {
+    const summaries: StrategyBenchmarkSummary[] = [];
+    for (const strategy of config.strategies) {
+      const results = yield* Effect.forEach(
+        config.chunks,
+        (chunk: TranslationContext) => runBenchmarkedChunk(strategy, chunk),
+        { concurrency: config.concurrency },
+      );
+      summaries.push(summarizeBenchmarkResults(results));
+    }
+    return summaries;
+  });
+}