dialekt 0.1.0 → 0.1.1

package/src/cli/commands/benchmark.test.ts

@@ -1,18 +1,18 @@
-import { describe, expect, it } from 'vitest';
-import { Effect, Option } from 'effect';
-import { runBenchmarkCommand, benchmarkCommand } from './benchmark.js';
-import type { DialektConfig } from '../../config/types.js';
-import type { TranslationAdapter, ResourceRef } from '../../adapter/types.js';
-import type { TranslationStrategy, TranslationContext } from '../../translation/types.js';
-import type { StrategyBenchmarkSummary } from '../../benchmark/metrics.js';
-
-describe('runBenchmarkCommand', () => {
+import { describe, expect, it } from "vitest";
+import { Effect, Option } from "effect";
+import { runBenchmarkCommand, benchmarkCommand } from "./benchmark.js";
+import type { DialektConfig } from "../../config/types.js";
+import type { TranslationAdapter, ResourceRef } from "../../adapter/types.js";
+import type { TranslationStrategy, TranslationContext } from "../../translation/types.js";
+import type { StrategyBenchmarkSummary } from "../../benchmark/metrics.js";
+
+describe("runBenchmarkCommand", () => {
   const baseConfig: DialektConfig = {
-    sourceLocale: 'en',
-    targetLocales: ['de'],
-    strategy: 'one-shot',
-    model: { provider: 'openai', modelId: 'gpt-4o' },
-    fastModel: { provider: 'openai', modelId: 'gpt-4o-mini' },
+    sourceLocale: "en",
+    targetLocales: ["de"],
+    strategy: "one-shot",
+    model: { provider: "openai", modelId: "gpt-4o" },
+    fastModel: { provider: "openai", modelId: "gpt-4o-mini" },
     chunking: { maxTokens: 3000, charsPerToken: 3.0, concurrency: 3 },
     retry: { maxAttempts: 3, baseDelayMs: 1000 },
     adapters: [],
@@ -29,10 +29,11 @@ describe('runBenchmarkCommand', () => {
     return {
       name: opts.name,
       capabilities: { canCreateResource: true, unusedKeyDetection: false },
-      listLocales: () => Effect.succeed(opts.locales ?? ['en', 'de']),
-      listResources: () => Effect.succeed(opts.resources ?? [{ key: 'messages', label: 'messages' }]),
+      listLocales: () => Effect.succeed(opts.locales ?? ["en", "de"]),
+      listResources: () =>
+        Effect.succeed(opts.resources ?? [{ key: "messages", label: "messages" }]),
       readResource: (_locale: string) =>
-        Effect.succeed(_locale === 'en' ? (opts.sourceMap ?? {}) : (opts.targetMap ?? {})),
+        Effect.succeed(_locale === "en" ? (opts.sourceMap ?? {}) : (opts.targetMap ?? {})),
       writeResource: () => Effect.void,
     };
   }
@@ -44,11 +45,18 @@ describe('runBenchmarkCommand', () => {
       configLoader: () => Effect.succeed(baseConfig),
       modelResolver: () => Effect.succeed({} as unknown),
       missingKeysComputer: () =>
-        Effect.succeed([{ adapter: 'test', locale: 'de', resource: { key: 'messages', label: 'messages' } as ResourceRef, missing: ['hello'] as readonly string[] }]),
+        Effect.succeed([
+          {
+            adapter: "test",
+            locale: "de",
+            resource: { key: "messages", label: "messages" } as ResourceRef,
+            missing: ["hello"] as readonly string[],
+          },
+        ]),
       benchmarkRunner: () =>
         Effect.succeed([
           {
-            strategyName: 'one-shot',
+            strategyName: "one-shot",
             totalChunks: 1,
             succeededChunks: 1,
             failedChunks: 0,
@@ -63,7 +71,7 @@ describe('runBenchmarkCommand', () => {
     };
   }
 
-  it('logs a cost warning', async () => {
+  it("logs a cost warning", async () => {
     const errors: string[] = [];
     const deps = makeDeps({
       errorLogger: (msg) => Effect.sync(() => errors.push(msg)),
@@ -71,7 +79,7 @@ describe('runBenchmarkCommand', () => {
 
     const program = runBenchmarkCommand(
       {
-        config: './config.ts',
+        config: "./config.ts",
         adapter: Option.none(),
         strategies: Option.none(),
         sampleSize: Option.none(),
@@ -80,10 +88,10 @@ describe('runBenchmarkCommand', () => {
     );
 
     await Effect.runPromise(program);
-    expect(errors.some((e) => e.includes('Warning') && e.includes('cost'))).toBe(true);
+    expect(errors.some((e) => e.includes("Warning") && e.includes("cost"))).toBe(true);
   });
 
-  it('uses default strategies when none specified', async () => {
+  it("uses default strategies when none specified", async () => {
     let usedStrategies: readonly string[] | undefined;
     const deps = makeDeps({
       benchmarkRunner: (opts) =>
@@ -95,7 +103,7 @@ describe('runBenchmarkCommand', () => {
 
     const program = runBenchmarkCommand(
       {
-        config: './config.ts',
+        config: "./config.ts",
         adapter: Option.none(),
         strategies: Option.none(),
         sampleSize: Option.none(),
@@ -104,10 +112,10 @@ describe('runBenchmarkCommand', () => {
     );
 
     await Effect.runPromise(program);
-    expect(usedStrategies).toEqual(['one-shot', 'tool-loop-agent']);
+    expect(usedStrategies).toEqual(["one-shot", "tool-loop-agent"]);
   });
 
-  it('uses custom strategies when --strategies is passed', async () => {
+  it("uses custom strategies when --strategies is passed", async () => {
     let usedStrategies: readonly string[] | undefined;
     const deps = makeDeps({
       benchmarkRunner: (opts) =>
@@ -119,26 +127,26 @@ describe('runBenchmarkCommand', () => {
 
     const program = runBenchmarkCommand(
       {
-        config: './config.ts',
+        config: "./config.ts",
         adapter: Option.none(),
-        strategies: Option.some('tool-loop-agent'),
+        strategies: Option.some("tool-loop-agent"),
         sampleSize: Option.none(),
       },
       deps,
     );
 
     await Effect.runPromise(program);
-    expect(usedStrategies).toEqual(['tool-loop-agent']);
+    expect(usedStrategies).toEqual(["tool-loop-agent"]);
   });
 
-  it('limits sample size with --sample-size', async () => {
+  it("limits sample size with --sample-size", async () => {
     let usedChunks: readonly TranslationContext[] | undefined;
     const adapter = makeAdapter({
-      name: 'test',
-      sourceMap: { a: 'A', b: 'B', c: 'C' },
+      name: "test",
+      sourceMap: { a: "A", b: "B", c: "C" },
       targetMap: {},
     });
-    const config = { ...baseConfig, adapters: [adapter] as unknown as DialektConfig['adapters'] };
+    const config = { ...baseConfig, adapters: [adapter] as unknown as DialektConfig["adapters"] };
     const deps = makeDeps({
       configLoader: () => Effect.succeed(config),
       benchmarkRunner: (opts) =>
@@ -148,13 +156,18 @@ describe('runBenchmarkCommand', () => {
         }),
       missingKeysComputer: () =>
         Effect.succeed([
-          { adapter: 'test', locale: 'de', resource: { key: 'messages', label: 'messages' } as ResourceRef, missing: ['a', 'b', 'c'] as readonly string[] },
+          {
+            adapter: "test",
+            locale: "de",
+            resource: { key: "messages", label: "messages" } as ResourceRef,
+            missing: ["a", "b", "c"] as readonly string[],
+          },
         ]),
     });
 
     const program = runBenchmarkCommand(
       {
-        config: './config.ts',
+        config: "./config.ts",
         adapter: Option.none(),
         strategies: Option.none(),
         sampleSize: Option.some(1),
@@ -166,7 +179,7 @@ describe('runBenchmarkCommand', () => {
     expect(usedChunks).toHaveLength(1);
   });
 
-  it('outputs JSON report when --format json is passed', async () => {
+  it("outputs JSON report when --format json is passed", async () => {
     const logs: string[] = [];
     const deps = makeDeps({
       logger: (msg) => Effect.sync(() => logs.push(msg)),
@@ -174,11 +187,11 @@ describe('runBenchmarkCommand', () => {
 
     const program = runBenchmarkCommand(
       {
-        config: './config.ts',
+        config: "./config.ts",
         adapter: Option.none(),
         strategies: Option.none(),
         sampleSize: Option.none(),
-        format: Option.some('json'),
+        format: Option.some("json"),
       },
       deps,
     );
@@ -187,10 +200,10 @@ describe('runBenchmarkCommand', () => {
     expect(logs).toHaveLength(1);
     const parsed = JSON.parse(logs[0]!);
     expect(parsed).toBeInstanceOf(Array);
-    expect(parsed[0]).toMatchObject({ strategyName: 'one-shot' });
+    expect(parsed[0]).toMatchObject({ strategyName: "one-shot" });
   });
 
-  it('outputs benchmark data by default', async () => {
+  it("outputs benchmark data by default", async () => {
     const logs: string[] = [];
     const deps = makeDeps({
       logger: (msg) => Effect.sync(() => logs.push(msg)),
@@ -198,7 +211,7 @@ describe('runBenchmarkCommand', () => {
 
     const program = runBenchmarkCommand(
       {
-        config: './config.ts',
+        config: "./config.ts",
         adapter: Option.none(),
         strategies: Option.none(),
         sampleSize: Option.none(),
@@ -212,11 +225,11 @@ describe('runBenchmarkCommand', () => {
     expect(parsed).toBeInstanceOf(Array);
   });
 
-  it('filters adapters by --adapter flag', async () => {
+  it("filters adapters by --adapter flag", async () => {
     let queriedAdapter: string | undefined;
-    const a1 = makeAdapter({ name: 'a1' });
-    const a2 = makeAdapter({ name: 'a2' });
-    const config = { ...baseConfig, adapters: [a1, a2] as unknown as DialektConfig['adapters'] };
+    const a1 = makeAdapter({ name: "a1" });
+    const a2 = makeAdapter({ name: "a2" });
+    const config = { ...baseConfig, adapters: [a1, a2] as unknown as DialektConfig["adapters"] };
     const deps = makeDeps({
       configLoader: () => Effect.succeed(config),
       missingKeysComputer: (a) =>
@@ -228,8 +241,8 @@ describe('runBenchmarkCommand', () => {
 
     const program = runBenchmarkCommand(
       {
-        config: './config.ts',
-        adapter: Option.some('a2'),
+        config: "./config.ts",
+        adapter: Option.some("a2"),
         strategies: Option.none(),
         sampleSize: Option.none(),
       },
@@ -237,17 +250,17 @@ describe('runBenchmarkCommand', () => {
     );
 
     await Effect.runPromise(program);
-    expect(queriedAdapter).toBe('a2');
+    expect(queriedAdapter).toBe("a2");
   });
 
-  it('fails when configLoader fails', async () => {
+  it("fails when configLoader fails", async () => {
     const deps = makeDeps({
-      configLoader: () => Effect.fail(new Error('Config not found')),
+      configLoader: () => Effect.fail(new Error("Config not found")),
     });
 
     const program = runBenchmarkCommand(
       {
-        config: './missing.ts',
+        config: "./missing.ts",
         adapter: Option.none(),
         strategies: Option.none(),
         sampleSize: Option.none(),
@@ -255,17 +268,17 @@ describe('runBenchmarkCommand', () => {
       deps,
     );
 
-    await expect(Effect.runPromise(program)).rejects.toThrow('Config not found');
+    await expect(Effect.runPromise(program)).rejects.toThrow("Config not found");
   });
 
-  it('fails when modelResolver fails', async () => {
+  it("fails when modelResolver fails", async () => {
     const deps = makeDeps({
-      modelResolver: () => Effect.fail(new Error('Model unavailable')),
+      modelResolver: () => Effect.fail(new Error("Model unavailable")),
     });
 
     const program = runBenchmarkCommand(
       {
-        config: './config.ts',
+        config: "./config.ts",
         adapter: Option.none(),
         strategies: Option.none(),
         sampleSize: Option.none(),
@@ -273,17 +286,17 @@ describe('runBenchmarkCommand', () => {
       deps,
     );
 
-    await expect(Effect.runPromise(program)).rejects.toThrow('Model unavailable');
+    await expect(Effect.runPromise(program)).rejects.toThrow("Model unavailable");
   });
 
-  it('fails when benchmarkRunner fails', async () => {
+  it("fails when benchmarkRunner fails", async () => {
     const deps = makeDeps({
-      benchmarkRunner: () => Effect.fail(new Error('Benchmark crashed')),
+      benchmarkRunner: () => Effect.fail(new Error("Benchmark crashed")),
     });
 
     const program = runBenchmarkCommand(
       {
-        config: './config.ts',
+        config: "./config.ts",
         adapter: Option.none(),
         strategies: Option.none(),
         sampleSize: Option.none(),
@@ -291,10 +304,10 @@ describe('runBenchmarkCommand', () => {
       deps,
     );
 
-    await expect(Effect.runPromise(program)).rejects.toThrow('Benchmark crashed');
+    await expect(Effect.runPromise(program)).rejects.toThrow("Benchmark crashed");
   });
 
-  it('handles empty adapter list', async () => {
+  it("handles empty adapter list", async () => {
     const logs: string[] = [];
     const config = { ...baseConfig, adapters: [] };
     const deps = makeDeps({
@@ -305,7 +318,7 @@ describe('runBenchmarkCommand', () => {
 
     const program = runBenchmarkCommand(
       {
-        config: './config.ts',
+        config: "./config.ts",
         adapter: Option.none(),
         strategies: Option.none(),
         sampleSize: Option.none(),
@@ -319,7 +332,7 @@ describe('runBenchmarkCommand', () => {
     expect(parsed).toEqual([]);
   });
 
-  it('handles adapter with no missing keys', async () => {
+  it("handles adapter with no missing keys", async () => {
     let usedChunks: readonly TranslationContext[] | undefined;
     const deps = makeDeps({
       missingKeysComputer: () => Effect.succeed([]),
@@ -332,7 +345,7 @@ describe('runBenchmarkCommand', () => {
 
     const program = runBenchmarkCommand(
       {
-        config: './config.ts',
+        config: "./config.ts",
         adapter: Option.none(),
         strategies: Option.none(),
         sampleSize: Option.none(),

package/src/cli/commands/benchmark.ts

@@ -1,19 +1,19 @@
-import { Command, Options } from '@effect/cli';
-import { Effect, Console, Option } from 'effect';
-import { loadConfig } from '../../config/load-config.js';
-import { resolveEffectiveConfig } from '../config-resolution.js';
-import { resolveModel } from '../../translation/model-registry.js';
-import { createOneShotStrategy } from '../../translation/one-shot-strategy.js';
-import { createToolLoopStrategy } from '../../translation/tool-loop-strategy.js';
-import { chunkKeys } from '../../translation/chunking.js';
-import { computeMissingKeys } from '../../translation/missing-keys.js';
-import { runBenchmark } from '../../benchmark/runner.js';
-import { detectFormat, type OutputFormat } from '../format.js';
-import { formatBenchmark, formatError } from '../formatters.js';
-import type { DialektConfig } from '../../config/types.js';
-import type { TranslationAdapter, ResourceRef } from '../../adapter/types.js';
-import type { TranslationStrategy, TranslationContext } from '../../translation/types.js';
-import type { StrategyBenchmarkSummary } from '../../benchmark/metrics.js';
+import { Command, Options } from "@effect/cli";
+import { Effect, Console, Option } from "effect";
+import { loadConfig } from "../../config/load-config.js";
+import { resolveEffectiveConfig } from "../config-resolution.js";
+import { resolveModel } from "../../translation/model-registry.js";
+import { createOneShotStrategy } from "../../translation/one-shot-strategy.js";
+import { createToolLoopStrategy } from "../../translation/tool-loop-strategy.js";
+import { chunkKeys } from "../../translation/chunking.js";
+import { computeMissingKeys } from "../../translation/missing-keys.js";
+import { runBenchmark } from "../../benchmark/runner.js";
+import { detectFormat, type OutputFormat } from "../format.js";
+import { formatBenchmark, formatError } from "../formatters.js";
+import type { DialektConfig } from "../../config/types.js";
+import type { TranslationAdapter, ResourceRef } from "../../adapter/types.js";
+import type { TranslationStrategy, TranslationContext } from "../../translation/types.js";
+import type { StrategyBenchmarkSummary } from "../../benchmark/metrics.js";
 
 export interface BenchmarkFlags {
   readonly config: string;
@@ -25,18 +25,32 @@ export interface BenchmarkFlags {
 
 export interface BenchmarkDeps {
   readonly configLoader: (path: string) => Effect.Effect<DialektConfig, unknown>;
-  readonly modelResolver: (config: { provider: string; modelId: string }) => Effect.Effect<unknown, unknown>;
+  readonly modelResolver: (config: {
+    provider: string;
+    modelId: string;
+  }) => Effect.Effect<unknown, unknown>;
   readonly missingKeysComputer: (
     adapter: TranslationAdapter,
     sourceLocale: string,
     targetLocales: readonly string[],
-  ) => Effect.Effect<readonly { adapter: string; locale: string; resource: ResourceRef; missing: readonly string[] }[], unknown>;
+  ) => Effect.Effect<
+    readonly {
+      adapter: string;
+      locale: string;
+      resource: ResourceRef;
+      missing: readonly string[];
+    }[],
+    unknown
+  >;
   readonly benchmarkRunner: (opts: {
     strategies: readonly TranslationStrategy[];
     chunks: readonly TranslationContext[];
     concurrency: number;
   }) => Effect.Effect<readonly StrategyBenchmarkSummary[], unknown>;
-  readonly reportFormatter?: (summaries: readonly StrategyBenchmarkSummary[], format: 'table' | 'json') => string;
+  readonly reportFormatter?: (
+    summaries: readonly StrategyBenchmarkSummary[],
+    format: "table" | "json",
+  ) => string;
   readonly logger: (msg: string) => Effect.Effect<void>;
   readonly errorLogger: (msg: string) => Effect.Effect<void>;
 }
@@ -48,7 +62,7 @@ export function runBenchmarkCommand(
   return Effect.gen(function* () {
     yield* deps.errorLogger(
       formatError(
-        'Warning: This will make real API calls to the configured model provider(s) and may incur cost.',
+        "Warning: This will make real API calls to the configured model provider(s) and may incur cost.",
         detectFormat(
           flags.format !== undefined
             ? (Option.getOrUndefined(flags.format) as OutputFormat | undefined)
@@ -63,14 +77,17 @@ export function runBenchmarkCommand(
       loaded,
     );
 
-    const strategyNames = Option.getOrElse(flags.strategies, () => 'one-shot,tool-loop-agent')
-      .split(',')
-      .map((s: string) => s.trim()) as Array<'one-shot' | 'tool-loop-agent'>;
+    const strategyNames = Option.getOrElse(flags.strategies, () => "one-shot,tool-loop-agent")
+      .split(",")
+      .map((s: string) => s.trim()) as Array<"one-shot" | "tool-loop-agent">;
 
-    const model = yield* deps.modelResolver(effective.model) as Effect.Effect<import('ai').LanguageModel, unknown>;
+    const model = yield* deps.modelResolver(effective.model) as Effect.Effect<
+      import("ai").LanguageModel,
+      unknown
+    >;
 
     const strategyList = strategyNames.map((name) =>
-      name === 'tool-loop-agent'
+      name === "tool-loop-agent"
         ? createToolLoopStrategy({ model, retry: effective.retry })
         : createOneShotStrategy({ model, retry: effective.retry }),
     );
@@ -102,7 +119,10 @@ export function runBenchmarkCommand(
       }
     }
 
-    const sampled = allChunks.slice(0, Option.getOrElse(flags.sampleSize, () => 20));
+    const sampled = allChunks.slice(
+      0,
+      Option.getOrElse(flags.sampleSize, () => 20),
+    );
 
     const summaries = yield* deps.benchmarkRunner({
       strategies: strategyList,
@@ -130,19 +150,22 @@ export function runBenchmarkCommand(
   });
 }
 
-export const benchmarkCommand = Command.make('benchmark', {
-  config: Options.text('config').pipe(Options.withDefault('./dialekt.config.ts')),
-  adapter: Options.optional(Options.text('adapter')),
-  strategies: Options.optional(Options.text('strategies')),
-  sampleSize: Options.optional(Options.integer('sample-size')),
-  format: Options.optional(Options.text('format')),
-}, (flags) =>
-  runBenchmarkCommand(flags, {
-    configLoader: loadConfig,
-    modelResolver: resolveModel,
-    missingKeysComputer: computeMissingKeys as unknown as BenchmarkDeps['missingKeysComputer'],
-    benchmarkRunner: runBenchmark,
-    logger: (msg: string) => Console.log(msg),
-    errorLogger: (msg: string) => Console.error(msg),
-  }),
+export const benchmarkCommand = Command.make(
+  "benchmark",
+  {
+    config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
+    adapter: Options.optional(Options.text("adapter")),
+    strategies: Options.optional(Options.text("strategies")),
+    sampleSize: Options.optional(Options.integer("sample-size")),
+    format: Options.optional(Options.text("format")),
+  },
+  (flags) =>
+    runBenchmarkCommand(flags, {
+      configLoader: loadConfig,
+      modelResolver: resolveModel,
+      missingKeysComputer: computeMissingKeys as unknown as BenchmarkDeps["missingKeysComputer"],
+      benchmarkRunner: runBenchmark,
+      logger: (msg: string) => Console.log(msg),
+      errorLogger: (msg: string) => Console.error(msg),
+    }),
 );