dialekt 0.1.0

package/src/translation/chunking.test.ts

@@ -0,0 +1,118 @@
+import { describe, expect, it } from 'vitest';
+import { chunkKeys } from './chunking.js';
+
+describe('chunkKeys', () => {
+  it('keeps a small key set in a single chunk', () => {
+    const source = { a: 'A', b: 'B' };
+    const target = {};
+    const chunks = chunkKeys(['a', 'b'], source, target, { maxTokens: 3000, charsPerToken: 3.0 });
+    expect(chunks).toHaveLength(1);
+    expect(chunks[0]).toEqual(['a', 'b']);
+  });
+
+  it('splits a large key set into multiple chunks', () => {
+    const source: Record<string, string> = {};
+    for (let i = 0; i < 150; i++) {
+      source[`key_${i}`] = `This is the value number ${i} with some extra text to make it longer`;
+    }
+    const target = {};
+    const keys = Object.keys(source);
+    const chunks = chunkKeys(keys, source, target, { maxTokens: 3000, charsPerToken: 3.0 });
+    expect(chunks.length).toBeGreaterThan(1);
+    const allKeys = chunks.flat();
+    expect(allKeys).toHaveLength(keys.length);
+    expect(new Set(allKeys).size).toBe(keys.length);
+  });
+
+  it('preserves all keys across chunks without loss or duplication', () => {
+    const source: Record<string, string> = {};
+    for (let i = 0; i < 60; i++) {
+      source[`k${i}`] = 'word '.repeat(10);
+    }
+    const target = {};
+    const keys = Object.keys(source);
+    const chunks = chunkKeys(keys, source, target, { maxTokens: 3000, charsPerToken: 3.0 });
+    const flattened = chunks.flat();
+    expect(flattened.sort()).toEqual([...keys].sort());
+  });
+
+  it('keeps a single oversized key in its own chunk', () => {
+    const source = { long: 'x'.repeat(5000) };
+    const target = {};
+    const chunks = chunkKeys(['long'], source, target, { maxTokens: 3000, charsPerToken: 3.0 });
+    expect(chunks).toHaveLength(1);
+    expect(chunks[0]).toEqual(['long']);
+  });
+
+  it('reduces effective chunk size when file context is huge', () => {
+    const source: Record<string, string> = {};
+    for (let i = 0; i < 26; i++) {
+      source[String.fromCharCode(97 + i)] = 'word '.repeat(50);
+    }
+    const target = {};
+    const keys = Object.keys(source).slice(0, 10);
+    const chunks = chunkKeys(keys, source, target, { maxTokens: 3000, charsPerToken: 3.0 });
+    expect(chunks.length).toBeGreaterThan(0);
+  });
+
+  it('falls back to one key per chunk when the context exceeds the limit', () => {
+    const source: Record<string, string> = {};
+    for (let i = 0; i < 100; i++) {
+      source[`key_${i}`] = 'word '.repeat(100);
+    }
+    const target = { ...source };
+    const chunks = chunkKeys(['key_0', 'key_1'], source, target, { maxTokens: 10, charsPerToken: 3.0 });
+    expect(chunks[0]).toEqual(['key_0']);
+    expect(chunks[1]).toEqual(['key_1']);
+  });
+
+  it('returns [] for empty keys', () => {
+    expect(chunkKeys([], {}, {}, { maxTokens: 3000, charsPerToken: 3.0 })).toEqual([]);
+  });
+
+  it('still includes keys not present in source (uses empty string as value)', () => {
+    const source = { a: 'A' };
+    const chunks = chunkKeys(['missing'], source, {}, { maxTokens: 3000, charsPerToken: 3.0 });
+    expect(chunks).toEqual([['missing']]);
+  });
+
+  it('handles keys with empty values', () => {
+    const source = { a: '', b: '' };
+    const chunks = chunkKeys(['a', 'b'], source, {}, { maxTokens: 3000, charsPerToken: 3.0 });
+    expect(chunks).toHaveLength(1);
+    expect(chunks[0]).toEqual(['a', 'b']);
+  });
+
+  it('handles single key', () => {
+    const source = { only: 'value' };
+    const chunks = chunkKeys(['only'], source, {}, { maxTokens: 3000, charsPerToken: 3.0 });
+    expect(chunks).toHaveLength(1);
+    expect(chunks[0]).toEqual(['only']);
+  });
+
+  it('handles very large number of keys', () => {
+    const source: Record<string, string> = {};
+    for (let i = 0; i < 1000; i++) {
+      source[`key_${i}`] = 'v';
+    }
+    const keys = Object.keys(source);
+    const chunks = chunkKeys(keys, source, {}, { maxTokens: 3000, charsPerToken: 3.0 });
+    const all = chunks.flat();
+    expect(all).toHaveLength(1000);
+  });
+
+  it('handles keys with special characters', () => {
+    const source = { 'key.with.dots': 'value', 'key-with-dashes': 'value2' };
+    const chunks = chunkKeys(['key.with.dots', 'key-with-dashes'], source, {}, { maxTokens: 3000, charsPerToken: 3.0 });
+    expect(chunks).toHaveLength(1);
+    expect(chunks[0]).toEqual(['key.with.dots', 'key-with-dashes']);
+  });
+
+  it('respects MIN_EFFECTIVE_MAX_CHARS even with zero maxTokens', () => {
+    const source = { a: 'A', b: 'B' };
+    const chunks = chunkKeys(['a', 'b'], source, {}, { maxTokens: 0, charsPerToken: 3.0 });
+    // MIN_EFFECTIVE_MAX_CHARS is 200, so both small keys fit in one chunk
+    expect(chunks).toHaveLength(1);
+    expect(chunks[0]).toEqual(['a', 'b']);
+  });
+});

package/src/translation/chunking.ts

@@ -0,0 +1,57 @@
+export interface ChunkingConfig {
+  readonly maxTokens: number;
+  readonly charsPerToken: number;
+}
+
+const PROMPT_OVERHEAD = 600;
+const ITEM_JSON_OVERHEAD = 20;
+const MIN_EFFECTIVE_MAX_CHARS = 200;
+
+export function chunkKeys(
+  keys: readonly string[],
+  sourceMap: Readonly<Record<string, string>>,
+  targetMap: Readonly<Record<string, string>>,
+  config: ChunkingConfig,
+): string[][] {
+  const maxChars = config.maxTokens * config.charsPerToken;
+  const sourceJson = JSON.stringify(sourceMap);
+  const targetJson = JSON.stringify(targetMap);
+  const contextOverhead = (sourceJson?.length ?? 0) + (targetJson?.length ?? 0) + PROMPT_OVERHEAD;
+  const effectiveMaxChars = Math.max(MIN_EFFECTIVE_MAX_CHARS, maxChars - contextOverhead);
+
+  const chunks: string[][] = [];
+  let currentChunk: string[] = [];
+  let currentChars = 0;
+
+  for (const key of keys) {
+    const value = sourceMap[key] ?? '';
+    const itemChars = key.length + value.length + ITEM_JSON_OVERHEAD;
+
+    // If a single key is larger than the entire chunk, force it through alone.
+    if (itemChars > effectiveMaxChars && currentChunk.length === 0) {
+      chunks.push([key]);
+      continue;
+    }
+
+    if (currentChunk.length > 0 && currentChars + itemChars > effectiveMaxChars) {
+      chunks.push(currentChunk);
+      currentChunk = [];
+      currentChars = 0;
+    }
+
+    currentChunk.push(key);
+    currentChars += itemChars;
+  }
+
+  if (currentChunk.length > 0) {
+    chunks.push(currentChunk);
+  }
+
+  // Safety valve: if the file context is so large that no keys fit,
+  // process keys one-by-one so translation still proceeds.
+  if (chunks.length === 0 && keys.length > 0) {
+    return keys.map((key) => [key]);
+  }
+
+  return chunks;
+}

package/src/translation/missing-keys.test.ts

@@ -0,0 +1,179 @@
+import { describe, expect, it } from 'vitest';
+import { Effect } from 'effect';
+import { computeMissingKeys } from './missing-keys.js';
+import type { TranslationAdapter } from '../adapter/types.js';
+
+describe('computeMissingKeys', () => {
+  it('lists missing keys per resource/locale', async () => {
+    const adapter: TranslationAdapter = {
+      name: 'test',
+      capabilities: { canCreateResource: true, unusedKeyDetection: false },
+      listLocales: () => Effect.succeed(['en', 'de']),
+      listResources: () => Effect.succeed([{ key: 'messages', label: 'messages' }]),
+      readResource: (locale: string) =>
+        Effect.succeed(
+          locale === 'en' ? { hello: 'Hello', bye: 'Bye' } : { hello: 'Hallo' },
+        ),
+      writeResource: () => Effect.void,
+    };
+
+    const result = await Effect.runPromise(
+      computeMissingKeys(adapter, 'en', ['de']),
+    ) as ReadonlyArray<{ missing: readonly string[] }>;
+    expect(result).toHaveLength(1);
+    expect(result[0]!.missing).toEqual(['bye']);
+  });
+
+  it('returns empty when nothing is missing', async () => {
+    const adapter: TranslationAdapter = {
+      name: 'test',
+      capabilities: { canCreateResource: true, unusedKeyDetection: false },
+      listLocales: () => Effect.succeed(['en', 'de']),
+      listResources: () => Effect.succeed([{ key: 'messages', label: 'messages' }]),
+      readResource: () => Effect.succeed({ hello: 'Hello' }),
+      writeResource: () => Effect.void,
+    };
+
+    const result = await Effect.runPromise(
+      computeMissingKeys(adapter, 'en', ['de']),
+    ) as ReadonlyArray<{ missing: readonly string[] }>;
+    expect(result).toHaveLength(0);
+  });
+
+  it('handles multiple resources', async () => {
+    const adapter: TranslationAdapter = {
+      name: 'test',
+      capabilities: { canCreateResource: true, unusedKeyDetection: false },
+      listLocales: () => Effect.succeed(['en', 'de']),
+      listResources: () =>
+        Effect.succeed([
+          { key: 'auth', label: 'auth' },
+          { key: 'validation', label: 'validation' },
+        ]),
+      readResource: (locale: string, resource) =>
+        Effect.succeed(
+          locale === 'en'
+            ? resource.key === 'auth'
+              ? { login: 'Login' }
+              : { email: 'Email' }
+            : resource.key === 'auth'
+              ? { login: 'Anmelden' }
+              : {},
+        ),
+      writeResource: () => Effect.void,
+    };
+
+    const result = await Effect.runPromise(
+      computeMissingKeys(adapter, 'en', ['de']),
+    ) as ReadonlyArray<{ resource: { label: string }; missing: readonly string[] }>;
+    expect(result).toHaveLength(1);
+    expect(result[0]!.resource.label).toBe('validation');
+    expect(result[0]!.missing).toEqual(['email']);
+  });
+
+  it('handles multiple target locales', async () => {
+    const adapter: TranslationAdapter = {
+      name: 'test',
+      capabilities: { canCreateResource: true, unusedKeyDetection: false },
+      listLocales: () => Effect.succeed(['en', 'de', 'fr']),
+      listResources: () => Effect.succeed([{ key: 'messages', label: 'messages' }]),
+      readResource: (locale: string) =>
+        Effect.succeed(
+          locale === 'en' ? { hello: 'Hello' } : {},
+        ),
+      writeResource: () => Effect.void,
+    };
+
+    const result = await Effect.runPromise(
+      computeMissingKeys(adapter, 'en', ['de', 'fr']),
+    ) as ReadonlyArray<{ locale: string; missing: readonly string[] }>;
+    expect(result).toHaveLength(2);
+    const locales = result.map((r) => r.locale);
+    expect(locales).toContain('de');
+    expect(locales).toContain('fr');
+  });
+
+  it('handles empty target locales', async () => {
+    const adapter: TranslationAdapter = {
+      name: 'test',
+      capabilities: { canCreateResource: true, unusedKeyDetection: false },
+      listLocales: () => Effect.succeed(['en']),
+      listResources: () => Effect.succeed([{ key: 'messages', label: 'messages' }]),
+      readResource: () => Effect.succeed({}),
+      writeResource: () => Effect.void,
+    };
+
+    const result = await Effect.runPromise(
+      computeMissingKeys(adapter, 'en', []),
+    ) as ReadonlyArray<{ missing: readonly string[] }>;
+    expect(result).toHaveLength(0);
+  });
+
+  it('handles empty resources', async () => {
+    const adapter: TranslationAdapter = {
+      name: 'test',
+      capabilities: { canCreateResource: true, unusedKeyDetection: false },
+      listLocales: () => Effect.succeed(['en', 'de']),
+      listResources: () => Effect.succeed([]),
+      readResource: () => Effect.succeed({}),
+      writeResource: () => Effect.void,
+    };
+
+    const result = await Effect.runPromise(
+      computeMissingKeys(adapter, 'en', ['de']),
+    ) as ReadonlyArray<{ missing: readonly string[] }>;
+    expect(result).toHaveLength(0);
+  });
+
+  it('handles adapter readResource failure', async () => {
+    const adapter: TranslationAdapter = {
+      name: 'broken',
+      capabilities: { canCreateResource: true, unusedKeyDetection: false },
+      listLocales: () => Effect.succeed(['en', 'de']),
+      listResources: () => Effect.succeed([{ key: 'messages', label: 'messages' }]),
+      readResource: () => Effect.fail(new Error('read failed') as never),
+      writeResource: () => Effect.void,
+    };
+
+    await expect(Effect.runPromise(computeMissingKeys(adapter, 'en', ['de']))).rejects.toThrow('read failed');
+  });
+
+  it('handles all keys missing', async () => {
+    const adapter: TranslationAdapter = {
+      name: 'test',
+      capabilities: { canCreateResource: true, unusedKeyDetection: false },
+      listLocales: () => Effect.succeed(['en', 'de']),
+      listResources: () => Effect.succeed([{ key: 'messages', label: 'messages' }]),
+      readResource: (locale: string) =>
+        Effect.succeed(
+          locale === 'en' ? { a: 'A', b: 'B', c: 'C' } : {},
+        ),
+      writeResource: () => Effect.void,
+    };
+
+    const result = await Effect.runPromise(
+      computeMissingKeys(adapter, 'en', ['de']),
+    ) as ReadonlyArray<{ missing: readonly string[] }>;
+    expect(result).toHaveLength(1);
+    expect(result[0]!.missing).toEqual(['a', 'b', 'c']);
+  });
+
+  it('ignores target keys not present in source', async () => {
+    const adapter: TranslationAdapter = {
+      name: 'test',
+      capabilities: { canCreateResource: true, unusedKeyDetection: false },
+      listLocales: () => Effect.succeed(['en', 'de']),
+      listResources: () => Effect.succeed([{ key: 'messages', label: 'messages' }]),
+      readResource: (locale: string) =>
+        Effect.succeed(
+          locale === 'en' ? { a: 'A' } : { a: 'A-de', b: 'B-de' },
+        ),
+      writeResource: () => Effect.void,
+    };
+
+    const result = await Effect.runPromise(
+      computeMissingKeys(adapter, 'en', ['de']),
+    ) as ReadonlyArray<{ missing: readonly string[] }>;
+    expect(result).toHaveLength(0);
+  });
+});

package/src/translation/missing-keys.ts

@@ -0,0 +1,36 @@
+import { Effect } from 'effect';
+import type { TranslationAdapter, ResourceRef, AdapterReadError } from '../adapter/types.js';
+import { diffKeys } from '../keys/flatten.js';
+
+export interface MissingKeyEntry {
+  readonly adapter: string;
+  readonly locale: string;
+  readonly resource: ResourceRef;
+  readonly missing: readonly string[];
+}
+
+export function computeMissingKeys(
+  adapter: TranslationAdapter,
+  sourceLocale: string,
+  targetLocales: readonly string[],
+): Effect.Effect<readonly MissingKeyEntry[], AdapterReadError> {
+  return Effect.gen(function* () {
+    const resources = yield* adapter.listResources(sourceLocale);
+    const entries = yield* Effect.forEach(resources, (resource: ResourceRef) =>
+      Effect.gen(function* () {
+        const sourceMap = yield* adapter.readResource(sourceLocale, resource);
+        const localeEntries = yield* Effect.forEach(targetLocales, (locale: string) =>
+          Effect.gen(function* () {
+            const targetMap = yield* adapter.readResource(locale, resource);
+            const missing = diffKeys(sourceMap, targetMap);
+            return missing.length > 0
+              ? [{ adapter: adapter.name, locale, resource, missing }]
+              : [];
+          }),
+        );
+        return localeEntries.flat();
+      }),
+    );
+    return entries.flat();
+  });
+}

package/src/translation/model-registry.test.ts

@@ -0,0 +1,54 @@
+import { describe, expect, it } from 'vitest';
+import { Effect, Either } from 'effect';
+import { resolveModel, UnknownProviderError } from './model-registry.js';
+
+describe('resolveModel', () => {
+  it('returns UnknownProviderError for unknown provider', async () => {
+    const program = resolveModel({ provider: 'unknown', modelId: 'x' });
+    const exit = await Effect.runPromise(Effect.either(program)) as Either.Either<unknown, UnknownProviderError>;
+    if (exit._tag === 'Left') {
+      expect(exit.left._tag).toBe('UnknownProviderError');
+      expect(exit.left.provider).toBe('unknown');
+    } else {
+      throw new Error('Expected Left');
+    }
+  });
+
+  it('returns UnknownProviderError for empty provider string', async () => {
+    const program = resolveModel({ provider: '', modelId: 'x' });
+    const exit = await Effect.runPromise(Effect.either(program)) as Either.Either<unknown, UnknownProviderError>;
+    if (exit._tag === 'Left') {
+      expect(exit.left._tag).toBe('UnknownProviderError');
+      expect(exit.left.provider).toBe('');
+    } else {
+      throw new Error('Expected Left');
+    }
+  });
+
+  it('resolves openai provider when package is available', async () => {
+    const program = resolveModel({ provider: 'openai', modelId: 'gpt-4o' });
+    const exit = await Effect.runPromise(Effect.either(program)) as Either.Either<unknown, UnknownProviderError>;
+    // Package is installed in this repo, so it should succeed
+    expect(exit._tag).toBe('Right');
+  });
+
+  it('UnknownProviderError preserves provider in message', () => {
+    const err = new UnknownProviderError({ provider: 'foo' });
+    expect(err.provider).toBe('foo');
+    expect(err._tag).toBe('UnknownProviderError');
+  });
+
+  it('accepts all known providers without type error', () => {
+    const providers = ['openai', 'anthropic', 'google'] as const;
+    for (const provider of providers) {
+      // Should compile without error
+      const config = { provider, modelId: 'test' };
+      expect(config.provider).toBe(provider);
+    }
+  });
+
+  it('modelId is passed through', () => {
+    const config = { provider: 'openai' as const, modelId: 'gpt-4o' };
+    expect(config.modelId).toBe('gpt-4o');
+  });
+});

package/src/translation/model-registry.ts

@@ -0,0 +1,43 @@
+import { Effect, Data } from 'effect';
+import type { LanguageModel } from 'ai';
+
+export class UnknownProviderError extends Data.TaggedError('UnknownProviderError')<{
+  readonly provider: string;
+}> {}
+
+export interface ModelConfig {
+  readonly provider: string;
+  readonly modelId: string;
+}
+
+/**
+ * The one file in the entire codebase allowed to import AI SDK provider packages.
+ */
+export function resolveModel(
+  config: ModelConfig,
+): Effect.Effect<LanguageModel, UnknownProviderError> {
+  return Effect.tryPromise({
+    try: async () => {
+      switch (config.provider) {
+        case 'openai': {
+          const { openai } = await import('@ai-sdk/openai');
+          return openai(config.modelId);
+        }
+        case 'anthropic': {
+          const { anthropic } = await import('@ai-sdk/anthropic');
+          return anthropic(config.modelId);
+        }
+        case 'google': {
+          const { google } = await import('@ai-sdk/google');
+          return google(config.modelId);
+        }
+        default:
+          throw new UnknownProviderError({ provider: config.provider });
+      }
+    },
+    catch: (cause) =>
+      cause instanceof UnknownProviderError
+        ? cause
+        : new UnknownProviderError({ provider: config.provider }),
+  });
+}