dialekt 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -0
- package/TESTING.md +66 -0
- package/dist/cli/main.d.mts +1 -0
- package/dist/cli/main.mjs +412 -0
- package/dist/formatters-De4Q-X1d.mjs +577 -0
- package/dist/index.d.mts +329 -0
- package/dist/index.mjs +60 -0
- package/package.json +39 -0
- package/pnpm-workspace.yaml +7 -0
- package/src/adapter/types.test.ts +98 -0
- package/src/adapter/types.ts +73 -0
- package/src/benchmark/metrics.test.ts +180 -0
- package/src/benchmark/metrics.ts +69 -0
- package/src/benchmark/report.test.ts +129 -0
- package/src/benchmark/report.ts +21 -0
- package/src/benchmark/runner.test.ts +162 -0
- package/src/benchmark/runner.ts +27 -0
- package/src/cli/commands/add.test.ts +267 -0
- package/src/cli/commands/add.ts +123 -0
- package/src/cli/commands/benchmark.test.ts +346 -0
- package/src/cli/commands/benchmark.ts +148 -0
- package/src/cli/commands/languages.test.ts +127 -0
- package/src/cli/commands/languages.ts +42 -0
- package/src/cli/commands/missing.test.ts +256 -0
- package/src/cli/commands/missing.ts +88 -0
- package/src/cli/commands/translate.test.ts +384 -0
- package/src/cli/commands/translate.ts +106 -0
- package/src/cli/commands/unused.test.ts +192 -0
- package/src/cli/commands/unused.ts +87 -0
- package/src/cli/commands/validate.test.ts +245 -0
- package/src/cli/commands/validate.ts +96 -0
- package/src/cli/config-resolution.test.ts +99 -0
- package/src/cli/config-resolution.ts +29 -0
- package/src/cli/format.test.ts +117 -0
- package/src/cli/format.ts +205 -0
- package/src/cli/formatters.test.ts +186 -0
- package/src/cli/formatters.ts +350 -0
- package/src/cli/main.ts +31 -0
- package/src/config/define-config.test.ts +66 -0
- package/src/config/define-config.ts +5 -0
- package/src/config/load-config.test.ts +35 -0
- package/src/config/load-config.ts +21 -0
- package/src/config/types.test.ts +101 -0
- package/src/config/types.ts +28 -0
- package/src/index.ts +56 -0
- package/src/keys/flatten.test.ts +111 -0
- package/src/keys/flatten.ts +41 -0
- package/src/sdk/file-io.test.ts +139 -0
- package/src/sdk/file-io.ts +21 -0
- package/src/sdk/node-layer.test.ts +54 -0
- package/src/sdk/node-layer.ts +10 -0
- package/src/sdk/php-array-reader.test.ts +114 -0
- package/src/sdk/php-array-reader.ts +26 -0
- package/src/translation/chunking.test.ts +118 -0
- package/src/translation/chunking.ts +57 -0
- package/src/translation/missing-keys.test.ts +179 -0
- package/src/translation/missing-keys.ts +36 -0
- package/src/translation/model-registry.test.ts +54 -0
- package/src/translation/model-registry.ts +43 -0
- package/src/translation/one-shot-strategy.test.ts +259 -0
- package/src/translation/one-shot-strategy.ts +48 -0
- package/src/translation/orchestrator.test.ts +276 -0
- package/src/translation/orchestrator.ts +83 -0
- package/src/translation/prompt.test.ts +149 -0
- package/src/translation/prompt.ts +42 -0
- package/src/translation/tool-loop-strategy.test.ts +279 -0
- package/src/translation/tool-loop-strategy.ts +68 -0
- package/src/translation/types.test.ts +37 -0
- package/src/translation/types.ts +21 -0
- package/tsconfig.json +9 -0
- package/tsconfig.tsbuildinfo +1 -0
- package/tsdown.config.ts +7 -0
- package/vitest.config.ts +8 -0
package/README.md
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
<div>
|
|
2
|
+
<img src="../../resources/icon.svg" align="left" width="175">
|
|
3
|
+
</div>
|
|
4
|
+
|
|
5
|
+
# `dialekt`
|
|
6
|
+
|
|
7
|
+
**Dialekt** [*diˈalɛkt, German for "dialect"*] uses large language models to translate your app's strings, with retries when the model garbles the output, chunking for large files, and a typed config file you check into git.
|
|
8
|
+
|
|
9
|
+
<br>
|
|
10
|
+
|
|
11
|
+
## Quick start
|
|
12
|
+
|
|
13
|
+
### 1. Install
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
npm install -D dialekt @dialekt/adapter-laravel
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Install only the adapters you use. If you also have a Paraglide frontend:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
npm install -D @dialekt/adapter-paraglide
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
### 2. Configure
|
|
26
|
+
|
|
27
|
+
Create `dialekt.config.ts` in your project root:
|
|
28
|
+
|
|
29
|
+
```ts
|
|
30
|
+
import { defineConfig } from 'dialekt';
|
|
31
|
+
import { laravel } from '@dialekt/adapter-laravel';
|
|
32
|
+
|
|
33
|
+
export default defineConfig({
|
|
34
|
+
sourceLocale: 'en',
|
|
35
|
+
targetLocales: ['de', 'fr', 'es'],
|
|
36
|
+
strategy: 'one-shot',
|
|
37
|
+
model: { provider: 'openai', modelId: 'gpt-4o' },
|
|
38
|
+
fastModel: { provider: 'openai', modelId: 'gpt-4o-mini' },
|
|
39
|
+
chunking: { maxTokens: 3000, charsPerToken: 3.0, concurrency: 3 },
|
|
40
|
+
retry: { maxAttempts: 3, baseDelayMs: 1000 },
|
|
41
|
+
adapters: [
|
|
42
|
+
laravel({ langDir: './lang', scanPaths: ['./app', './resources/views'] }),
|
|
43
|
+
],
|
|
44
|
+
});
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
### 3. Set your API key
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
export OPENAI_API_KEY=sk-...
|
|
51
|
+
# or ANTHROPIC_API_KEY, GOOGLE_GENERATIVE_AI_API_KEY
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
### 4. Translate
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
npx dialekt translate
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Full documentation
|
|
61
|
+
|
|
62
|
+
See the [GitHub README](https://github.com/mateffy/dialekt) for the full guide, command reference, translation strategies, adapter docs, and the programmatic API.
|
package/TESTING.md
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# dialekt Testing Guide
|
|
2
|
+
|
|
3
|
+
## Tested Areas Map
|
|
4
|
+
|
|
5
|
+
| Export | Test File | Status |
|
|
6
|
+
|--------|-----------|--------|
|
|
7
|
+
| `AdapterReadError` / `AdapterWriteError` | `src/adapter/types.test.ts` | ✅ |
|
|
8
|
+
| `flattenObject` / `unflattenObject` / `diffKeys` | `src/keys/flatten.test.ts` | ✅ |
|
|
9
|
+
| `chunkKeys` | `src/translation/chunking.test.ts` | ✅ |
|
|
10
|
+
| `computeMissingKeys` | `src/translation/missing-keys.test.ts` | ✅ |
|
|
11
|
+
| `resolveModel` / `UnknownProviderError` | `src/translation/model-registry.test.ts` | ✅ |
|
|
12
|
+
| `createOneShotStrategy` | `src/translation/one-shot-strategy.test.ts` | ✅ |
|
|
13
|
+
| `createToolLoopStrategy` | `src/translation/tool-loop-strategy.test.ts` | ✅ |
|
|
14
|
+
| `runTranslation` | `src/translation/orchestrator.test.ts` | ✅ |
|
|
15
|
+
| `buildSystemPrompt` / `buildUserPrompt` | `src/translation/prompt.test.ts` | ✅ |
|
|
16
|
+
| `TranslationFailedError` | `src/translation/types.test.ts` | ✅ |
|
|
17
|
+
| `runBenchmarkedChunk` / `summarizeBenchmarkResults` | `src/benchmark/metrics.test.ts` | ✅ |
|
|
18
|
+
| `formatBenchmarkReport` | `src/benchmark/report.test.ts` | ✅ |
|
|
19
|
+
| `runBenchmark` | `src/benchmark/runner.test.ts` | ✅ |
|
|
20
|
+
| `translateCommand` / `runTranslate` | `src/cli/commands/translate.test.ts` | ✅ |
|
|
21
|
+
| `validateCommand` / `runValidate` | `src/cli/commands/validate.test.ts` | ✅ |
|
|
22
|
+
| `addCommand` / `runAdd` / `parseAddTokens` | `src/cli/commands/add.test.ts` | ✅ |
|
|
23
|
+
| `missingCommand` / `runMissing` | `src/cli/commands/missing.test.ts` | ✅ |
|
|
24
|
+
| `unusedCommand` / `runUnused` | `src/cli/commands/unused.test.ts` | ✅ |
|
|
25
|
+
| `languagesCommand` / `runLanguages` | `src/cli/commands/languages.test.ts` | ✅ |
|
|
26
|
+
| `benchmarkCommand` / `runBenchmarkCommand` | `src/cli/commands/benchmark.test.ts` | ✅ |
|
|
27
|
+
| `resolveEffectiveConfig` | `src/cli/config-resolution.test.ts` | ✅ |
|
|
28
|
+
| `loadConfig` / `ConfigLoadError` | `src/config/load-config.test.ts` | ✅ |
|
|
29
|
+
| `defineConfig` | `src/config/define-config.test.ts` | ✅ |
|
|
30
|
+
| config types | `src/config/types.test.ts` | ✅ |
|
|
31
|
+
| `readFileIfExists` / `writeFileEnsuringDir` | `src/sdk/file-io.test.ts` | ✅ |
|
|
32
|
+
| `NodePlatformLayer` | `src/sdk/node-layer.test.ts` | ✅ |
|
|
33
|
+
| `readPhpArrayAsJson` / `PhpExecutionError` | `src/sdk/php-array-reader.test.ts` | ✅ |
|
|
34
|
+
|
|
35
|
+
## Known Coverage Gaps
|
|
36
|
+
|
|
37
|
+
- `src/cli/main.ts` — CLI entrypoint wiring is not directly tested (covered via integration through command handlers)
|
|
38
|
+
- AI provider resolution for `anthropic` and `google` — tests only verify `openai` works because packages are installed
|
|
39
|
+
- `ChunkingConfig` / `RetryConfig` runtime validation — types enforce structure but no runtime bounds checking tests exist
|
|
40
|
+
|
|
41
|
+
## Specialties & Watch-Outs
|
|
42
|
+
|
|
43
|
+
- **Effect.gen generators are NOT async** — never use `await` inside `Effect.gen`. Import tags at top level.
|
|
44
|
+
- **CLI commands** — handlers extracted into `runXxx` functions for testability. The `Command.make` objects are thin wrappers.
|
|
45
|
+
- **PHP tests** skip when `php` binary is unavailable (`it.skipIf(!hasPhpBinary())`).
|
|
46
|
+
- **MockLanguageModelV3** from `ai/test` is used for strategy tests — it simulates the full AI SDK v7 response shape.
|
|
47
|
+
- **NodeNext `.js` extensions** — all imports must end in `.js` even for `.ts` source files.
|
|
48
|
+
|
|
49
|
+
## Test Utilities & Helpers
|
|
50
|
+
|
|
51
|
+
- `makeFsLayer` in `sdk/file-io.test.ts` — creates a `Layer.succeed(FileSystem, stub)` for in-memory file system testing.
|
|
52
|
+
- `makeAdapter` pattern — repeated across CLI tests to create lightweight `TranslationAdapter` stubs.
|
|
53
|
+
- `hasPhpBinary()` guard — used in `php-array-reader.test.ts` and `adapter-laravel` tests to conditionally run PHP-dependent tests.
|
|
54
|
+
|
|
55
|
+
## Running Tests
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
# All packages
|
|
59
|
+
pnpm -r run test
|
|
60
|
+
|
|
61
|
+
# Single package
|
|
62
|
+
pnpm --filter dialekt test
|
|
63
|
+
|
|
64
|
+
# Single file
|
|
65
|
+
npx vitest run src/translation/chunking.test.ts
|
|
66
|
+
```
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { };
|
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { A as resolveModel, C as runTranslation, S as computeMissingKeys, T as createOneShotStrategy, a as formatMissingKeys, c as formatValidate, d as detectFormat, i as formatLanguages, j as chunkKeys, n as formatBenchmark, o as formatTranslate, r as formatError, s as formatUnusedKeys, t as formatAdd, w as createToolLoopStrategy, x as loadConfig } from "../formatters-De4Q-X1d.mjs";
|
|
3
|
+
import { Console, Effect, Option } from "effect";
|
|
4
|
+
import { NodeContext, NodeRuntime } from "@effect/platform-node";
|
|
5
|
+
import { Command, Options } from "@effect/cli";
|
|
6
|
+
//#region src/cli/config-resolution.ts
|
|
7
|
+
function resolveEffectiveConfig(flags, loaded) {
|
|
8
|
+
return {
|
|
9
|
+
...loaded,
|
|
10
|
+
sourceLocale: flags.baseLanguage ?? loaded.sourceLocale,
|
|
11
|
+
targetLocales: flags.language && flags.language.length > 0 ? flags.language : loaded.targetLocales,
|
|
12
|
+
strategy: flags.strategy ?? loaded.strategy,
|
|
13
|
+
adapters: flags.adapter ? loaded.adapters.filter((a) => a.name === flags.adapter) : loaded.adapters
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
//#endregion
|
|
17
|
+
//#region src/cli/commands/translate.ts
|
|
18
|
+
function runTranslate(flags, configLoader = loadConfig, modelResolver = resolveModel, translationRunner = runTranslation, logger = (msg) => Console.log(msg)) {
|
|
19
|
+
return Effect.gen(function* () {
|
|
20
|
+
const loaded = yield* configLoader(flags.config);
|
|
21
|
+
const effective = resolveEffectiveConfig({
|
|
22
|
+
baseLanguage: Option.getOrUndefined(flags.baseLanguage),
|
|
23
|
+
language: Option.isSome(flags.language) ? [flags.language.value] : void 0,
|
|
24
|
+
adapter: Option.getOrUndefined(flags.adapter),
|
|
25
|
+
strategy: Option.getOrUndefined(flags.strategy) === "one-shot" || Option.getOrUndefined(flags.strategy) === "tool-loop-agent" ? Option.getOrUndefined(flags.strategy) : void 0
|
|
26
|
+
}, loaded);
|
|
27
|
+
const model = yield* modelResolver(flags.fast ? effective.fastModel : effective.model);
|
|
28
|
+
const translationStrategy = effective.strategy === "tool-loop-agent" ? createToolLoopStrategy({
|
|
29
|
+
model,
|
|
30
|
+
retry: effective.retry
|
|
31
|
+
}) : createOneShotStrategy({
|
|
32
|
+
model,
|
|
33
|
+
retry: effective.retry
|
|
34
|
+
});
|
|
35
|
+
yield* translationRunner({
|
|
36
|
+
adapters: effective.adapters,
|
|
37
|
+
strategy: translationStrategy,
|
|
38
|
+
sourceLocale: effective.sourceLocale,
|
|
39
|
+
targetLocales: effective.targetLocales ?? [],
|
|
40
|
+
chunking: effective.chunking
|
|
41
|
+
});
|
|
42
|
+
const format = detectFormat(flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0);
|
|
43
|
+
yield* logger(formatTranslate({
|
|
44
|
+
success: true,
|
|
45
|
+
message: "Translation complete.",
|
|
46
|
+
stats: {
|
|
47
|
+
adaptersProcessed: effective.adapters.length,
|
|
48
|
+
localesTranslated: (effective.targetLocales ?? []).length,
|
|
49
|
+
keysTranslated: 0
|
|
50
|
+
}
|
|
51
|
+
}, format));
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
const translateCommand = Command.make("translate", {
|
|
55
|
+
config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
|
|
56
|
+
adapter: Options.optional(Options.text("adapter")),
|
|
57
|
+
strategy: Options.optional(Options.text("strategy")),
|
|
58
|
+
baseLanguage: Options.optional(Options.text("base-language")),
|
|
59
|
+
language: Options.optional(Options.text("language")),
|
|
60
|
+
name: Options.optional(Options.text("name")),
|
|
61
|
+
skipNames: Options.boolean("skip-names"),
|
|
62
|
+
skipLanguages: Options.boolean("skip-languages"),
|
|
63
|
+
fast: Options.boolean("fast"),
|
|
64
|
+
format: Options.optional(Options.text("format"))
|
|
65
|
+
}, (flags) => runTranslate(flags));
|
|
66
|
+
//#endregion
|
|
67
|
+
//#region src/cli/commands/validate.ts
|
|
68
|
+
function runValidate(flags, configLoader = loadConfig, missingKeysComputer = computeMissingKeys, logger = (msg) => Console.log(msg)) {
|
|
69
|
+
return Effect.gen(function* () {
|
|
70
|
+
const loaded = yield* configLoader(flags.config);
|
|
71
|
+
const effective = resolveEffectiveConfig({
|
|
72
|
+
baseLanguage: Option.getOrUndefined(flags.baseLanguage),
|
|
73
|
+
language: Option.isSome(flags.language) ? [flags.language.value] : void 0,
|
|
74
|
+
adapter: Option.getOrUndefined(flags.adapter)
|
|
75
|
+
}, loaded);
|
|
76
|
+
const entries = [];
|
|
77
|
+
for (const a of effective.adapters) {
|
|
78
|
+
const locales = yield* a.listLocales();
|
|
79
|
+
const sourceLocale = effective.sourceLocale;
|
|
80
|
+
const missingEntries = yield* missingKeysComputer(a, sourceLocale, locales.filter((l) => l !== sourceLocale));
|
|
81
|
+
for (const entry of missingEntries) entries.push({
|
|
82
|
+
adapter: entry.adapter,
|
|
83
|
+
locale: entry.locale,
|
|
84
|
+
resource: entry.resource.label,
|
|
85
|
+
count: entry.missing.length
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
const format = detectFormat(flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0);
|
|
89
|
+
const passing = entries.length === 0;
|
|
90
|
+
yield* logger(formatValidate({
|
|
91
|
+
passing,
|
|
92
|
+
entries
|
|
93
|
+
}, format));
|
|
94
|
+
if (!passing) yield* Effect.sync(() => {
|
|
95
|
+
process.exitCode = 1;
|
|
96
|
+
});
|
|
97
|
+
}).pipe(Effect.mapError((e) => e));
|
|
98
|
+
}
|
|
99
|
+
const validateCommand = Command.make("validate", {
|
|
100
|
+
config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
|
|
101
|
+
adapter: Options.optional(Options.text("adapter")),
|
|
102
|
+
baseLanguage: Options.optional(Options.text("base-language")),
|
|
103
|
+
language: Options.optional(Options.text("language")),
|
|
104
|
+
format: Options.optional(Options.text("format"))
|
|
105
|
+
}, (flags) => runValidate(flags));
|
|
106
|
+
//#endregion
|
|
107
|
+
//#region src/cli/commands/add.ts
|
|
108
|
+
function parseAddTokens(tokens, errorLogger) {
|
|
109
|
+
return Effect.gen(function* () {
|
|
110
|
+
const entriesByResource = {};
|
|
111
|
+
for (const token of tokens) {
|
|
112
|
+
const eqIdx = token.indexOf("=");
|
|
113
|
+
if (eqIdx === -1) {
|
|
114
|
+
yield* errorLogger(`Invalid token (missing '='): ${token}`);
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
const key = token.slice(0, eqIdx);
|
|
118
|
+
const value = token.slice(eqIdx + 1);
|
|
119
|
+
const dotIdx = key.indexOf(".");
|
|
120
|
+
if (dotIdx === -1) {
|
|
121
|
+
yield* errorLogger(`Invalid key (no resource segment): ${key}`);
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
const resource = key.slice(0, dotIdx);
|
|
125
|
+
const subKey = key.slice(dotIdx + 1);
|
|
126
|
+
if (!entriesByResource[resource]) entriesByResource[resource] = {};
|
|
127
|
+
entriesByResource[resource][subKey] = value;
|
|
128
|
+
}
|
|
129
|
+
return entriesByResource;
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
function runAdd(flags, tokens, configLoader = loadConfig, modelResolver = resolveModel, translationRunner = runTranslation, logger = (msg) => Console.log(msg), errorLogger = (msg) => Console.error(msg)) {
|
|
133
|
+
return Effect.gen(function* () {
|
|
134
|
+
const effective = resolveEffectiveConfig({}, yield* configLoader(flags.config));
|
|
135
|
+
const entriesByResource = yield* parseAddTokens(tokens, errorLogger);
|
|
136
|
+
const addedResources = [];
|
|
137
|
+
for (const adapter of effective.adapters) for (const [resourceKey, entries] of Object.entries(entriesByResource)) {
|
|
138
|
+
const resourceRef = {
|
|
139
|
+
key: resourceKey,
|
|
140
|
+
label: resourceKey
|
|
141
|
+
};
|
|
142
|
+
yield* adapter.writeResource(effective.sourceLocale, resourceRef, entries);
|
|
143
|
+
addedResources.push(`${adapter.name}/${effective.sourceLocale}/${resourceKey}`);
|
|
144
|
+
}
|
|
145
|
+
const modelConfig = effective.model;
|
|
146
|
+
const model = yield* modelResolver(modelConfig);
|
|
147
|
+
const translationStrategy = effective.strategy === "tool-loop-agent" ? createToolLoopStrategy({
|
|
148
|
+
model,
|
|
149
|
+
retry: effective.retry
|
|
150
|
+
}) : createOneShotStrategy({
|
|
151
|
+
model,
|
|
152
|
+
retry: effective.retry
|
|
153
|
+
});
|
|
154
|
+
yield* translationRunner({
|
|
155
|
+
adapters: effective.adapters,
|
|
156
|
+
strategy: translationStrategy,
|
|
157
|
+
sourceLocale: effective.sourceLocale,
|
|
158
|
+
targetLocales: (effective.targetLocales ?? []).filter((l) => l !== effective.sourceLocale),
|
|
159
|
+
chunking: effective.chunking
|
|
160
|
+
});
|
|
161
|
+
const format = detectFormat(flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0);
|
|
162
|
+
yield* logger(formatAdd({
|
|
163
|
+
success: true,
|
|
164
|
+
message: "Add + translate complete.",
|
|
165
|
+
addedResources
|
|
166
|
+
}, format));
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
const addCommand = Command.make("add", {
|
|
170
|
+
config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
|
|
171
|
+
create: Options.boolean("create"),
|
|
172
|
+
format: Options.optional(Options.text("format"))
|
|
173
|
+
}, ({ config, create, format }) => {
|
|
174
|
+
const rawTokens = process.argv.slice(3).filter((t) => !t.startsWith("--") && !t.startsWith("-"));
|
|
175
|
+
return runAdd({
|
|
176
|
+
config,
|
|
177
|
+
create,
|
|
178
|
+
format
|
|
179
|
+
}, rawTokens);
|
|
180
|
+
});
|
|
181
|
+
//#endregion
|
|
182
|
+
//#region src/cli/commands/missing.ts
|
|
183
|
+
function runMissing(flags, configLoader = loadConfig, missingKeysComputer = computeMissingKeys, logger = (msg) => Console.log(msg)) {
|
|
184
|
+
return Effect.gen(function* () {
|
|
185
|
+
const loaded = yield* configLoader(flags.config);
|
|
186
|
+
const effective = resolveEffectiveConfig({
|
|
187
|
+
baseLanguage: Option.getOrUndefined(flags.baseLanguage),
|
|
188
|
+
language: Option.isSome(flags.language) ? [flags.language.value] : void 0,
|
|
189
|
+
adapter: Option.getOrUndefined(flags.adapter)
|
|
190
|
+
}, loaded);
|
|
191
|
+
const allEntries = [];
|
|
192
|
+
for (const a of effective.adapters) {
|
|
193
|
+
const locales = yield* a.listLocales();
|
|
194
|
+
const sourceLocale = effective.sourceLocale;
|
|
195
|
+
const entries = yield* missingKeysComputer(a, sourceLocale, locales.filter((l) => l !== sourceLocale));
|
|
196
|
+
for (const entry of entries) for (const key of entry.missing) allEntries.push({
|
|
197
|
+
adapter: entry.adapter,
|
|
198
|
+
locale: entry.locale,
|
|
199
|
+
resource: entry.resource.label,
|
|
200
|
+
key
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
yield* logger(formatMissingKeys(allEntries, detectFormat(flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0)));
|
|
204
|
+
}).pipe(Effect.mapError((e) => e));
|
|
205
|
+
}
|
|
206
|
+
const missingCommand = Command.make("missing", {
|
|
207
|
+
config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
|
|
208
|
+
adapter: Options.optional(Options.text("adapter")),
|
|
209
|
+
baseLanguage: Options.optional(Options.text("base-language")),
|
|
210
|
+
language: Options.optional(Options.text("language")),
|
|
211
|
+
format: Options.optional(Options.text("format"))
|
|
212
|
+
}, (flags) => runMissing(flags));
|
|
213
|
+
//#endregion
|
|
214
|
+
//#region src/cli/commands/unused.ts
|
|
215
|
+
function runUnused(flags, configLoader = loadConfig, logger = (msg) => Console.log(msg), errorLogger = (msg) => Console.error(msg)) {
|
|
216
|
+
return Effect.gen(function* () {
|
|
217
|
+
const loaded = yield* configLoader(flags.config);
|
|
218
|
+
const effective = resolveEffectiveConfig({
|
|
219
|
+
baseLanguage: Option.getOrUndefined(flags.baseLanguage),
|
|
220
|
+
adapter: Option.getOrUndefined(flags.adapter)
|
|
221
|
+
}, loaded);
|
|
222
|
+
const allEntries = [];
|
|
223
|
+
for (const a of effective.adapters) {
|
|
224
|
+
if (!a.capabilities.unusedKeyDetection) {
|
|
225
|
+
yield* errorLogger(formatError(`Adapter '${a.name}' does not support unused-key detection.`, detectFormat(flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0)));
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
yield* a.listLocales();
|
|
229
|
+
const sourceLocale = effective.sourceLocale;
|
|
230
|
+
const resources = yield* a.listResources(sourceLocale);
|
|
231
|
+
for (const resource of resources) {
|
|
232
|
+
const unused = yield* a.findUnusedKeys(sourceLocale, resource);
|
|
233
|
+
for (const key of unused) allEntries.push({
|
|
234
|
+
adapter: a.name,
|
|
235
|
+
locale: sourceLocale,
|
|
236
|
+
resource: resource.label,
|
|
237
|
+
key
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
yield* logger(formatUnusedKeys(allEntries, detectFormat(flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0)));
|
|
242
|
+
}).pipe(Effect.mapError((e) => e));
|
|
243
|
+
}
|
|
244
|
+
const unusedCommand = Command.make("unused", {
|
|
245
|
+
config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
|
|
246
|
+
adapter: Options.optional(Options.text("adapter")),
|
|
247
|
+
baseLanguage: Options.optional(Options.text("base-language")),
|
|
248
|
+
format: Options.optional(Options.text("format"))
|
|
249
|
+
}, (flags) => runUnused(flags));
|
|
250
|
+
//#endregion
|
|
251
|
+
//#region src/cli/commands/languages.ts
|
|
252
|
+
function runLanguages(flags, configLoader = loadConfig, logger = (msg) => Console.log(msg)) {
|
|
253
|
+
return Effect.gen(function* () {
|
|
254
|
+
const effective = resolveEffectiveConfig({}, yield* configLoader(flags.config));
|
|
255
|
+
const entries = [];
|
|
256
|
+
for (const adapter of effective.adapters) {
|
|
257
|
+
const locales = yield* adapter.listLocales();
|
|
258
|
+
entries.push({
|
|
259
|
+
adapter: adapter.name,
|
|
260
|
+
locales
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
yield* logger(formatLanguages(entries, detectFormat(flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0)));
|
|
264
|
+
}).pipe(Effect.mapError((e) => e));
|
|
265
|
+
}
|
|
266
|
+
const languagesCommand = Command.make("languages", {
|
|
267
|
+
config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
|
|
268
|
+
format: Options.optional(Options.text("format"))
|
|
269
|
+
}, (flags) => runLanguages(flags));
|
|
270
|
+
//#endregion
|
|
271
|
+
//#region src/benchmark/metrics.ts
|
|
272
|
+
function summarizeBenchmarkResults(results) {
|
|
273
|
+
const totalChunks = results.length;
|
|
274
|
+
const succeededChunks = results.filter((r) => r.succeeded).length;
|
|
275
|
+
const failedChunks = totalChunks - succeededChunks;
|
|
276
|
+
const totalDurationMs = results.reduce((sum, r) => sum + r.durationMs, 0);
|
|
277
|
+
const totalAttempts = results.reduce((sum, r) => sum + r.attemptCount, 0);
|
|
278
|
+
return {
|
|
279
|
+
strategyName: results[0]?.strategyName ?? "one-shot",
|
|
280
|
+
totalChunks,
|
|
281
|
+
succeededChunks,
|
|
282
|
+
failedChunks,
|
|
283
|
+
totalDurationMs,
|
|
284
|
+
averageDurationMsPerChunk: totalChunks > 0 ? totalDurationMs / totalChunks : 0,
|
|
285
|
+
totalAttempts
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
function runBenchmarkedChunk(strategy, ctx) {
|
|
289
|
+
return Effect.gen(function* () {
|
|
290
|
+
const start = Date.now();
|
|
291
|
+
const result = yield* Effect.either(strategy.translateChunk(ctx));
|
|
292
|
+
const durationMs = Date.now() - start;
|
|
293
|
+
if (result._tag === "Right") return {
|
|
294
|
+
strategyName: strategy.name,
|
|
295
|
+
chunkKeyCount: ctx.keys.length,
|
|
296
|
+
durationMs,
|
|
297
|
+
attemptCount: 1,
|
|
298
|
+
succeeded: true,
|
|
299
|
+
errorMessage: void 0
|
|
300
|
+
};
|
|
301
|
+
return {
|
|
302
|
+
strategyName: strategy.name,
|
|
303
|
+
chunkKeyCount: ctx.keys.length,
|
|
304
|
+
durationMs,
|
|
305
|
+
attemptCount: 1,
|
|
306
|
+
succeeded: false,
|
|
307
|
+
errorMessage: String(result.left.cause)
|
|
308
|
+
};
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
//#endregion
|
|
312
|
+
//#region src/benchmark/runner.ts
|
|
313
|
+
function runBenchmark(config) {
|
|
314
|
+
return Effect.gen(function* () {
|
|
315
|
+
const summaries = [];
|
|
316
|
+
for (const strategy of config.strategies) {
|
|
317
|
+
const results = yield* Effect.forEach(config.chunks, (chunk) => runBenchmarkedChunk(strategy, chunk), { concurrency: config.concurrency });
|
|
318
|
+
summaries.push(summarizeBenchmarkResults(results));
|
|
319
|
+
}
|
|
320
|
+
return summaries;
|
|
321
|
+
});
|
|
322
|
+
}
|
|
323
|
+
//#endregion
|
|
324
|
+
//#region src/cli/commands/benchmark.ts
|
|
325
|
+
function runBenchmarkCommand(flags, deps) {
|
|
326
|
+
return Effect.gen(function* () {
|
|
327
|
+
yield* deps.errorLogger(formatError("Warning: This will make real API calls to the configured model provider(s) and may incur cost.", detectFormat(flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0)));
|
|
328
|
+
const loaded = yield* deps.configLoader(flags.config);
|
|
329
|
+
const effective = resolveEffectiveConfig({ adapter: Option.getOrUndefined(flags.adapter) }, loaded);
|
|
330
|
+
const strategyNames = Option.getOrElse(flags.strategies, () => "one-shot,tool-loop-agent").split(",").map((s) => s.trim());
|
|
331
|
+
const model = yield* deps.modelResolver(effective.model);
|
|
332
|
+
const strategyList = strategyNames.map((name) => name === "tool-loop-agent" ? createToolLoopStrategy({
|
|
333
|
+
model,
|
|
334
|
+
retry: effective.retry
|
|
335
|
+
}) : createOneShotStrategy({
|
|
336
|
+
model,
|
|
337
|
+
retry: effective.retry
|
|
338
|
+
}));
|
|
339
|
+
const allChunks = [];
|
|
340
|
+
for (const a of effective.adapters) {
|
|
341
|
+
const locales = yield* a.listLocales();
|
|
342
|
+
const sourceLocale = effective.sourceLocale;
|
|
343
|
+
const targets = locales.filter((l) => l !== sourceLocale);
|
|
344
|
+
const missingEntries = yield* deps.missingKeysComputer(a, sourceLocale, targets);
|
|
345
|
+
for (const entry of missingEntries) {
|
|
346
|
+
const sourceMap = yield* a.readResource(sourceLocale, entry.resource);
|
|
347
|
+
const targetMap = yield* a.readResource(entry.locale, entry.resource);
|
|
348
|
+
const chunks = chunkKeys(entry.missing, sourceMap, targetMap, {
|
|
349
|
+
maxTokens: effective.chunking.maxTokens,
|
|
350
|
+
charsPerToken: effective.chunking.charsPerToken
|
|
351
|
+
});
|
|
352
|
+
for (const keys of chunks) allChunks.push({
|
|
353
|
+
sourceLocale,
|
|
354
|
+
targetLocale: entry.locale,
|
|
355
|
+
sourceMap,
|
|
356
|
+
targetMap,
|
|
357
|
+
keys
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
const sampled = allChunks.slice(0, Option.getOrElse(flags.sampleSize, () => 20));
|
|
362
|
+
const summaries = yield* deps.benchmarkRunner({
|
|
363
|
+
strategies: strategyList,
|
|
364
|
+
chunks: sampled,
|
|
365
|
+
concurrency: effective.chunking.concurrency
|
|
366
|
+
});
|
|
367
|
+
const format = detectFormat(flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0);
|
|
368
|
+
const entries = summaries.map((s) => ({
|
|
369
|
+
strategyName: s.strategyName,
|
|
370
|
+
totalChunks: s.totalChunks,
|
|
371
|
+
succeededChunks: s.succeededChunks,
|
|
372
|
+
failedChunks: s.failedChunks,
|
|
373
|
+
totalDurationMs: s.totalDurationMs,
|
|
374
|
+
averageDurationMsPerChunk: s.averageDurationMsPerChunk,
|
|
375
|
+
totalAttempts: s.totalAttempts
|
|
376
|
+
}));
|
|
377
|
+
yield* deps.logger(formatBenchmark(entries, format));
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
const benchmarkCommand = Command.make("benchmark", {
|
|
381
|
+
config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
|
|
382
|
+
adapter: Options.optional(Options.text("adapter")),
|
|
383
|
+
strategies: Options.optional(Options.text("strategies")),
|
|
384
|
+
sampleSize: Options.optional(Options.integer("sample-size")),
|
|
385
|
+
format: Options.optional(Options.text("format"))
|
|
386
|
+
}, (flags) => runBenchmarkCommand(flags, {
|
|
387
|
+
configLoader: loadConfig,
|
|
388
|
+
modelResolver: resolveModel,
|
|
389
|
+
missingKeysComputer: computeMissingKeys,
|
|
390
|
+
benchmarkRunner: runBenchmark,
|
|
391
|
+
logger: (msg) => Console.log(msg),
|
|
392
|
+
errorLogger: (msg) => Console.error(msg)
|
|
393
|
+
}));
|
|
394
|
+
//#endregion
|
|
395
|
+
//#region src/cli/main.ts
|
|
396
|
+
const rootCommand = Command.make("dialekt").pipe(Command.withSubcommands([
|
|
397
|
+
translateCommand,
|
|
398
|
+
validateCommand,
|
|
399
|
+
addCommand,
|
|
400
|
+
missingCommand,
|
|
401
|
+
unusedCommand,
|
|
402
|
+
languagesCommand,
|
|
403
|
+
benchmarkCommand
|
|
404
|
+
]));
|
|
405
|
+
const cli = Command.run(rootCommand, {
|
|
406
|
+
name: "dialekt",
|
|
407
|
+
version: "0.1.0"
|
|
408
|
+
});
|
|
409
|
+
const program = Effect.provide(cli(process.argv), NodeContext.layer);
|
|
410
|
+
NodeRuntime.runMain(program);
|
|
411
|
+
//#endregion
|
|
412
|
+
export {};
|