dialekt 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -10
- package/TESTING.md +29 -29
- package/dist/cli/main.d.mts +1 -1
- package/dist/cli/main.mjs +549 -362
- package/dist/formatters-De4Q-X1d.mjs +516 -435
- package/dist/index.d.mts +162 -25
- package/dist/index.mjs +119 -34
- package/package.json +3 -3
- package/pnpm-workspace.yaml +3 -3
- package/src/adapter/types.test.ts +57 -57
- package/src/adapter/types.ts +7 -4
- package/src/benchmark/metrics.test.ts +141 -69
- package/src/benchmark/metrics.ts +6 -6
- package/src/benchmark/report.test.ts +38 -38
- package/src/benchmark/report.ts +6 -6
- package/src/benchmark/runner.test.ts +70 -72
- package/src/benchmark/runner.ts +4 -4
- package/src/cli/commands/add.test.ts +90 -109
- package/src/cli/commands/add.ts +40 -28
- package/src/cli/commands/benchmark.test.ts +77 -64
- package/src/cli/commands/benchmark.ts +64 -41
- package/src/cli/commands/languages.test.ts +45 -42
- package/src/cli/commands/languages.ts +16 -12
- package/src/cli/commands/missing.test.ts +143 -92
- package/src/cli/commands/missing.ts +24 -17
- package/src/cli/commands/translate.test.ts +79 -79
- package/src/cli/commands/translate.ts +41 -31
- package/src/cli/commands/unused.test.ts +62 -51
- package/src/cli/commands/unused.ts +18 -14
- package/src/cli/commands/validate.test.ts +130 -72
- package/src/cli/commands/validate.ts +25 -20
- package/src/cli/config-resolution.test.ts +169 -49
- package/src/cli/config-resolution.ts +5 -7
- package/src/cli/format.test.ts +50 -50
- package/src/cli/format.ts +57 -60
- package/src/cli/formatters.test.ts +128 -106
- package/src/cli/formatters.ts +72 -95
- package/src/cli/main.ts +13 -13
- package/src/config/define-config.test.ts +44 -29
- package/src/config/define-config.ts +1 -1
- package/src/config/load-config.test.ts +21 -18
- package/src/config/load-config.ts +5 -5
- package/src/config/types.test.ts +50 -44
- package/src/config/types.ts +2 -2
- package/src/index.ts +22 -26
- package/src/keys/flatten.test.ts +52 -52
- package/src/keys/flatten.ts +7 -9
- package/src/sdk/file-io.test.ts +47 -59
- package/src/sdk/file-io.ts +2 -2
- package/src/sdk/node-layer.test.ts +18 -18
- package/src/sdk/node-layer.ts +2 -2
- package/src/sdk/php-array-reader.test.ts +49 -40
- package/src/sdk/php-array-reader.ts +5 -5
- package/src/translation/chunking.test.ts +52 -44
- package/src/translation/chunking.ts +1 -1
- package/src/translation/missing-keys.test.ts +86 -93
- package/src/translation/missing-keys.ts +4 -6
- package/src/translation/model-registry.test.ts +41 -32
- package/src/translation/model-registry.ts +9 -9
- package/src/translation/one-shot-strategy.test.ts +105 -86
- package/src/translation/one-shot-strategy.ts +10 -12
- package/src/translation/orchestrator.test.ts +90 -101
- package/src/translation/orchestrator.ts +26 -26
- package/src/translation/prompt.test.ts +76 -76
- package/src/translation/prompt.ts +2 -2
- package/src/translation/tool-loop-strategy.test.ts +134 -107
- package/src/translation/tool-loop-strategy.ts +14 -18
- package/src/translation/types.test.ts +22 -22
- package/src/translation/types.ts +3 -3
- package/tsdown.config.ts +3 -3
- package/vitest.config.ts +3 -3
package/dist/cli/main.mjs
CHANGED
|
@@ -1,410 +1,597 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
A as resolveModel,
|
|
4
|
+
C as runTranslation,
|
|
5
|
+
S as computeMissingKeys,
|
|
6
|
+
T as createOneShotStrategy,
|
|
7
|
+
a as formatMissingKeys,
|
|
8
|
+
c as formatValidate,
|
|
9
|
+
d as detectFormat,
|
|
10
|
+
i as formatLanguages,
|
|
11
|
+
j as chunkKeys,
|
|
12
|
+
n as formatBenchmark,
|
|
13
|
+
o as formatTranslate,
|
|
14
|
+
r as formatError,
|
|
15
|
+
s as formatUnusedKeys,
|
|
16
|
+
t as formatAdd,
|
|
17
|
+
w as createToolLoopStrategy,
|
|
18
|
+
x as loadConfig,
|
|
19
|
+
} from "../formatters-De4Q-X1d.mjs";
|
|
3
20
|
import { Console, Effect, Option } from "effect";
|
|
4
21
|
import { NodeContext, NodeRuntime } from "@effect/platform-node";
|
|
5
22
|
import { Command, Options } from "@effect/cli";
|
|
6
23
|
//#region src/cli/config-resolution.ts
|
|
7
24
|
function resolveEffectiveConfig(flags, loaded) {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
25
|
+
return {
|
|
26
|
+
...loaded,
|
|
27
|
+
sourceLocale: flags.baseLanguage ?? loaded.sourceLocale,
|
|
28
|
+
targetLocales:
|
|
29
|
+
flags.language && flags.language.length > 0 ? flags.language : loaded.targetLocales,
|
|
30
|
+
strategy: flags.strategy ?? loaded.strategy,
|
|
31
|
+
adapters: flags.adapter
|
|
32
|
+
? loaded.adapters.filter((a) => a.name === flags.adapter)
|
|
33
|
+
: loaded.adapters,
|
|
34
|
+
};
|
|
15
35
|
}
|
|
16
36
|
//#endregion
|
|
17
37
|
//#region src/cli/commands/translate.ts
|
|
18
|
-
function runTranslate(
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
38
|
+
function runTranslate(
|
|
39
|
+
flags,
|
|
40
|
+
configLoader = loadConfig,
|
|
41
|
+
modelResolver = resolveModel,
|
|
42
|
+
translationRunner = runTranslation,
|
|
43
|
+
logger = (msg) => Console.log(msg),
|
|
44
|
+
) {
|
|
45
|
+
return Effect.gen(function* () {
|
|
46
|
+
const loaded = yield* configLoader(flags.config);
|
|
47
|
+
const effective = resolveEffectiveConfig(
|
|
48
|
+
{
|
|
49
|
+
baseLanguage: Option.getOrUndefined(flags.baseLanguage),
|
|
50
|
+
language: Option.isSome(flags.language) ? [flags.language.value] : void 0,
|
|
51
|
+
adapter: Option.getOrUndefined(flags.adapter),
|
|
52
|
+
strategy:
|
|
53
|
+
Option.getOrUndefined(flags.strategy) === "one-shot" ||
|
|
54
|
+
Option.getOrUndefined(flags.strategy) === "tool-loop-agent"
|
|
55
|
+
? Option.getOrUndefined(flags.strategy)
|
|
56
|
+
: void 0,
|
|
57
|
+
},
|
|
58
|
+
loaded,
|
|
59
|
+
);
|
|
60
|
+
const model = yield* modelResolver(flags.fast ? effective.fastModel : effective.model);
|
|
61
|
+
const translationStrategy =
|
|
62
|
+
effective.strategy === "tool-loop-agent"
|
|
63
|
+
? createToolLoopStrategy({
|
|
64
|
+
model,
|
|
65
|
+
retry: effective.retry,
|
|
66
|
+
})
|
|
67
|
+
: createOneShotStrategy({
|
|
68
|
+
model,
|
|
69
|
+
retry: effective.retry,
|
|
70
|
+
});
|
|
71
|
+
yield* translationRunner({
|
|
72
|
+
adapters: effective.adapters,
|
|
73
|
+
strategy: translationStrategy,
|
|
74
|
+
sourceLocale: effective.sourceLocale,
|
|
75
|
+
targetLocales: effective.targetLocales ?? [],
|
|
76
|
+
chunking: effective.chunking,
|
|
77
|
+
});
|
|
78
|
+
const format = detectFormat(
|
|
79
|
+
flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0,
|
|
80
|
+
);
|
|
81
|
+
yield* logger(
|
|
82
|
+
formatTranslate(
|
|
83
|
+
{
|
|
84
|
+
success: true,
|
|
85
|
+
message: "Translation complete.",
|
|
86
|
+
stats: {
|
|
87
|
+
adaptersProcessed: effective.adapters.length,
|
|
88
|
+
localesTranslated: (effective.targetLocales ?? []).length,
|
|
89
|
+
keysTranslated: 0,
|
|
90
|
+
},
|
|
91
|
+
},
|
|
92
|
+
format,
|
|
93
|
+
),
|
|
94
|
+
);
|
|
95
|
+
});
|
|
53
96
|
}
|
|
54
|
-
const translateCommand = Command.make(
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
97
|
+
const translateCommand = Command.make(
|
|
98
|
+
"translate",
|
|
99
|
+
{
|
|
100
|
+
config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
|
|
101
|
+
adapter: Options.optional(Options.text("adapter")),
|
|
102
|
+
strategy: Options.optional(Options.text("strategy")),
|
|
103
|
+
baseLanguage: Options.optional(Options.text("base-language")),
|
|
104
|
+
language: Options.optional(Options.text("language")),
|
|
105
|
+
name: Options.optional(Options.text("name")),
|
|
106
|
+
skipNames: Options.boolean("skip-names"),
|
|
107
|
+
skipLanguages: Options.boolean("skip-languages"),
|
|
108
|
+
fast: Options.boolean("fast"),
|
|
109
|
+
format: Options.optional(Options.text("format")),
|
|
110
|
+
},
|
|
111
|
+
(flags) => runTranslate(flags),
|
|
112
|
+
);
|
|
66
113
|
//#endregion
|
|
67
114
|
//#region src/cli/commands/validate.ts
|
|
68
|
-
function runValidate(
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
115
|
+
function runValidate(
|
|
116
|
+
flags,
|
|
117
|
+
configLoader = loadConfig,
|
|
118
|
+
missingKeysComputer = computeMissingKeys,
|
|
119
|
+
logger = (msg) => Console.log(msg),
|
|
120
|
+
) {
|
|
121
|
+
return Effect.gen(function* () {
|
|
122
|
+
const loaded = yield* configLoader(flags.config);
|
|
123
|
+
const effective = resolveEffectiveConfig(
|
|
124
|
+
{
|
|
125
|
+
baseLanguage: Option.getOrUndefined(flags.baseLanguage),
|
|
126
|
+
language: Option.isSome(flags.language) ? [flags.language.value] : void 0,
|
|
127
|
+
adapter: Option.getOrUndefined(flags.adapter),
|
|
128
|
+
},
|
|
129
|
+
loaded,
|
|
130
|
+
);
|
|
131
|
+
const entries = [];
|
|
132
|
+
for (const a of effective.adapters) {
|
|
133
|
+
const locales = yield* a.listLocales();
|
|
134
|
+
const sourceLocale = effective.sourceLocale;
|
|
135
|
+
const missingEntries = yield* missingKeysComputer(
|
|
136
|
+
a,
|
|
137
|
+
sourceLocale,
|
|
138
|
+
locales.filter((l) => l !== sourceLocale),
|
|
139
|
+
);
|
|
140
|
+
for (const entry of missingEntries)
|
|
141
|
+
entries.push({
|
|
142
|
+
adapter: entry.adapter,
|
|
143
|
+
locale: entry.locale,
|
|
144
|
+
resource: entry.resource.label,
|
|
145
|
+
count: entry.missing.length,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
const format = detectFormat(
|
|
149
|
+
flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0,
|
|
150
|
+
);
|
|
151
|
+
const passing = entries.length === 0;
|
|
152
|
+
yield* logger(
|
|
153
|
+
formatValidate(
|
|
154
|
+
{
|
|
155
|
+
passing,
|
|
156
|
+
entries,
|
|
157
|
+
},
|
|
158
|
+
format,
|
|
159
|
+
),
|
|
160
|
+
);
|
|
161
|
+
if (!passing)
|
|
162
|
+
yield* Effect.sync(() => {
|
|
163
|
+
process.exitCode = 1;
|
|
164
|
+
});
|
|
165
|
+
}).pipe(Effect.mapError((e) => e));
|
|
98
166
|
}
|
|
99
|
-
const validateCommand = Command.make(
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
167
|
+
const validateCommand = Command.make(
|
|
168
|
+
"validate",
|
|
169
|
+
{
|
|
170
|
+
config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
|
|
171
|
+
adapter: Options.optional(Options.text("adapter")),
|
|
172
|
+
baseLanguage: Options.optional(Options.text("base-language")),
|
|
173
|
+
language: Options.optional(Options.text("language")),
|
|
174
|
+
format: Options.optional(Options.text("format")),
|
|
175
|
+
},
|
|
176
|
+
(flags) => runValidate(flags),
|
|
177
|
+
);
|
|
106
178
|
//#endregion
|
|
107
179
|
//#region src/cli/commands/add.ts
|
|
108
180
|
function parseAddTokens(tokens, errorLogger) {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
181
|
+
return Effect.gen(function* () {
|
|
182
|
+
const entriesByResource = {};
|
|
183
|
+
for (const token of tokens) {
|
|
184
|
+
const eqIdx = token.indexOf("=");
|
|
185
|
+
if (eqIdx === -1) {
|
|
186
|
+
yield* errorLogger(`Invalid token (missing '='): ${token}`);
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
const key = token.slice(0, eqIdx);
|
|
190
|
+
const value = token.slice(eqIdx + 1);
|
|
191
|
+
const dotIdx = key.indexOf(".");
|
|
192
|
+
if (dotIdx === -1) {
|
|
193
|
+
yield* errorLogger(`Invalid key (no resource segment): ${key}`);
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
const resource = key.slice(0, dotIdx);
|
|
197
|
+
const subKey = key.slice(dotIdx + 1);
|
|
198
|
+
if (!entriesByResource[resource]) entriesByResource[resource] = {};
|
|
199
|
+
entriesByResource[resource][subKey] = value;
|
|
200
|
+
}
|
|
201
|
+
return entriesByResource;
|
|
202
|
+
});
|
|
131
203
|
}
|
|
132
|
-
function runAdd(
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
204
|
+
function runAdd(
|
|
205
|
+
flags,
|
|
206
|
+
tokens,
|
|
207
|
+
configLoader = loadConfig,
|
|
208
|
+
modelResolver = resolveModel,
|
|
209
|
+
translationRunner = runTranslation,
|
|
210
|
+
logger = (msg) => Console.log(msg),
|
|
211
|
+
errorLogger = (msg) => Console.error(msg),
|
|
212
|
+
) {
|
|
213
|
+
return Effect.gen(function* () {
|
|
214
|
+
const effective = resolveEffectiveConfig({}, yield* configLoader(flags.config));
|
|
215
|
+
const entriesByResource = yield* parseAddTokens(tokens, errorLogger);
|
|
216
|
+
const addedResources = [];
|
|
217
|
+
for (const adapter of effective.adapters)
|
|
218
|
+
for (const [resourceKey, entries] of Object.entries(entriesByResource)) {
|
|
219
|
+
const resourceRef = {
|
|
220
|
+
key: resourceKey,
|
|
221
|
+
label: resourceKey,
|
|
222
|
+
};
|
|
223
|
+
yield* adapter.writeResource(effective.sourceLocale, resourceRef, entries);
|
|
224
|
+
addedResources.push(`${adapter.name}/${effective.sourceLocale}/${resourceKey}`);
|
|
225
|
+
}
|
|
226
|
+
const modelConfig = effective.model;
|
|
227
|
+
const model = yield* modelResolver(modelConfig);
|
|
228
|
+
const translationStrategy =
|
|
229
|
+
effective.strategy === "tool-loop-agent"
|
|
230
|
+
? createToolLoopStrategy({
|
|
231
|
+
model,
|
|
232
|
+
retry: effective.retry,
|
|
233
|
+
})
|
|
234
|
+
: createOneShotStrategy({
|
|
235
|
+
model,
|
|
236
|
+
retry: effective.retry,
|
|
237
|
+
});
|
|
238
|
+
yield* translationRunner({
|
|
239
|
+
adapters: effective.adapters,
|
|
240
|
+
strategy: translationStrategy,
|
|
241
|
+
sourceLocale: effective.sourceLocale,
|
|
242
|
+
targetLocales: (effective.targetLocales ?? []).filter((l) => l !== effective.sourceLocale),
|
|
243
|
+
chunking: effective.chunking,
|
|
244
|
+
});
|
|
245
|
+
const format = detectFormat(
|
|
246
|
+
flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0,
|
|
247
|
+
);
|
|
248
|
+
yield* logger(
|
|
249
|
+
formatAdd(
|
|
250
|
+
{
|
|
251
|
+
success: true,
|
|
252
|
+
message: "Add + translate complete.",
|
|
253
|
+
addedResources,
|
|
254
|
+
},
|
|
255
|
+
format,
|
|
256
|
+
),
|
|
257
|
+
);
|
|
258
|
+
});
|
|
168
259
|
}
|
|
169
|
-
const addCommand = Command.make(
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
260
|
+
const addCommand = Command.make(
|
|
261
|
+
"add",
|
|
262
|
+
{
|
|
263
|
+
config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
|
|
264
|
+
create: Options.boolean("create"),
|
|
265
|
+
format: Options.optional(Options.text("format")),
|
|
266
|
+
},
|
|
267
|
+
({ config, create, format }) => {
|
|
268
|
+
const rawTokens = process.argv
|
|
269
|
+
.slice(3)
|
|
270
|
+
.filter((t) => !t.startsWith("--") && !t.startsWith("-"));
|
|
271
|
+
return runAdd(
|
|
272
|
+
{
|
|
273
|
+
config,
|
|
274
|
+
create,
|
|
275
|
+
format,
|
|
276
|
+
},
|
|
277
|
+
rawTokens,
|
|
278
|
+
);
|
|
279
|
+
},
|
|
280
|
+
);
|
|
181
281
|
//#endregion
|
|
182
282
|
//#region src/cli/commands/missing.ts
|
|
183
|
-
function runMissing(
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
283
|
+
function runMissing(
|
|
284
|
+
flags,
|
|
285
|
+
configLoader = loadConfig,
|
|
286
|
+
missingKeysComputer = computeMissingKeys,
|
|
287
|
+
logger = (msg) => Console.log(msg),
|
|
288
|
+
) {
|
|
289
|
+
return Effect.gen(function* () {
|
|
290
|
+
const loaded = yield* configLoader(flags.config);
|
|
291
|
+
const effective = resolveEffectiveConfig(
|
|
292
|
+
{
|
|
293
|
+
baseLanguage: Option.getOrUndefined(flags.baseLanguage),
|
|
294
|
+
language: Option.isSome(flags.language) ? [flags.language.value] : void 0,
|
|
295
|
+
adapter: Option.getOrUndefined(flags.adapter),
|
|
296
|
+
},
|
|
297
|
+
loaded,
|
|
298
|
+
);
|
|
299
|
+
const allEntries = [];
|
|
300
|
+
for (const a of effective.adapters) {
|
|
301
|
+
const locales = yield* a.listLocales();
|
|
302
|
+
const sourceLocale = effective.sourceLocale;
|
|
303
|
+
const entries = yield* missingKeysComputer(
|
|
304
|
+
a,
|
|
305
|
+
sourceLocale,
|
|
306
|
+
locales.filter((l) => l !== sourceLocale),
|
|
307
|
+
);
|
|
308
|
+
for (const entry of entries)
|
|
309
|
+
for (const key of entry.missing)
|
|
310
|
+
allEntries.push({
|
|
311
|
+
adapter: entry.adapter,
|
|
312
|
+
locale: entry.locale,
|
|
313
|
+
resource: entry.resource.label,
|
|
314
|
+
key,
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
yield* logger(
|
|
318
|
+
formatMissingKeys(
|
|
319
|
+
allEntries,
|
|
320
|
+
detectFormat(flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0),
|
|
321
|
+
),
|
|
322
|
+
);
|
|
323
|
+
}).pipe(Effect.mapError((e) => e));
|
|
205
324
|
}
|
|
206
|
-
const missingCommand = Command.make(
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
325
|
+
const missingCommand = Command.make(
|
|
326
|
+
"missing",
|
|
327
|
+
{
|
|
328
|
+
config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
|
|
329
|
+
adapter: Options.optional(Options.text("adapter")),
|
|
330
|
+
baseLanguage: Options.optional(Options.text("base-language")),
|
|
331
|
+
language: Options.optional(Options.text("language")),
|
|
332
|
+
format: Options.optional(Options.text("format")),
|
|
333
|
+
},
|
|
334
|
+
(flags) => runMissing(flags),
|
|
335
|
+
);
|
|
213
336
|
//#endregion
|
|
214
337
|
//#region src/cli/commands/unused.ts
|
|
215
|
-
function runUnused(
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
338
|
+
function runUnused(
|
|
339
|
+
flags,
|
|
340
|
+
configLoader = loadConfig,
|
|
341
|
+
logger = (msg) => Console.log(msg),
|
|
342
|
+
errorLogger = (msg) => Console.error(msg),
|
|
343
|
+
) {
|
|
344
|
+
return Effect.gen(function* () {
|
|
345
|
+
const loaded = yield* configLoader(flags.config);
|
|
346
|
+
const effective = resolveEffectiveConfig(
|
|
347
|
+
{
|
|
348
|
+
baseLanguage: Option.getOrUndefined(flags.baseLanguage),
|
|
349
|
+
adapter: Option.getOrUndefined(flags.adapter),
|
|
350
|
+
},
|
|
351
|
+
loaded,
|
|
352
|
+
);
|
|
353
|
+
const allEntries = [];
|
|
354
|
+
for (const a of effective.adapters) {
|
|
355
|
+
if (!a.capabilities.unusedKeyDetection) {
|
|
356
|
+
yield* errorLogger(
|
|
357
|
+
formatError(
|
|
358
|
+
`Adapter '${a.name}' does not support unused-key detection.`,
|
|
359
|
+
detectFormat(flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0),
|
|
360
|
+
),
|
|
361
|
+
);
|
|
362
|
+
continue;
|
|
363
|
+
}
|
|
364
|
+
yield* a.listLocales();
|
|
365
|
+
const sourceLocale = effective.sourceLocale;
|
|
366
|
+
const resources = yield* a.listResources(sourceLocale);
|
|
367
|
+
for (const resource of resources) {
|
|
368
|
+
const unused = yield* a.findUnusedKeys(sourceLocale, resource);
|
|
369
|
+
for (const key of unused)
|
|
370
|
+
allEntries.push({
|
|
371
|
+
adapter: a.name,
|
|
372
|
+
locale: sourceLocale,
|
|
373
|
+
resource: resource.label,
|
|
374
|
+
key,
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
yield* logger(
|
|
379
|
+
formatUnusedKeys(
|
|
380
|
+
allEntries,
|
|
381
|
+
detectFormat(flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0),
|
|
382
|
+
),
|
|
383
|
+
);
|
|
384
|
+
}).pipe(Effect.mapError((e) => e));
|
|
243
385
|
}
|
|
244
|
-
const unusedCommand = Command.make(
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
386
|
+
const unusedCommand = Command.make(
|
|
387
|
+
"unused",
|
|
388
|
+
{
|
|
389
|
+
config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
|
|
390
|
+
adapter: Options.optional(Options.text("adapter")),
|
|
391
|
+
baseLanguage: Options.optional(Options.text("base-language")),
|
|
392
|
+
format: Options.optional(Options.text("format")),
|
|
393
|
+
},
|
|
394
|
+
(flags) => runUnused(flags),
|
|
395
|
+
);
|
|
250
396
|
//#endregion
|
|
251
397
|
//#region src/cli/commands/languages.ts
|
|
252
398
|
function runLanguages(flags, configLoader = loadConfig, logger = (msg) => Console.log(msg)) {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
399
|
+
return Effect.gen(function* () {
|
|
400
|
+
const effective = resolveEffectiveConfig({}, yield* configLoader(flags.config));
|
|
401
|
+
const entries = [];
|
|
402
|
+
for (const adapter of effective.adapters) {
|
|
403
|
+
const locales = yield* adapter.listLocales();
|
|
404
|
+
entries.push({
|
|
405
|
+
adapter: adapter.name,
|
|
406
|
+
locales,
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
yield* logger(
|
|
410
|
+
formatLanguages(
|
|
411
|
+
entries,
|
|
412
|
+
detectFormat(flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0),
|
|
413
|
+
),
|
|
414
|
+
);
|
|
415
|
+
}).pipe(Effect.mapError((e) => e));
|
|
265
416
|
}
|
|
266
|
-
const languagesCommand = Command.make(
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
417
|
+
const languagesCommand = Command.make(
|
|
418
|
+
"languages",
|
|
419
|
+
{
|
|
420
|
+
config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
|
|
421
|
+
format: Options.optional(Options.text("format")),
|
|
422
|
+
},
|
|
423
|
+
(flags) => runLanguages(flags),
|
|
424
|
+
);
|
|
270
425
|
//#endregion
|
|
271
426
|
//#region src/benchmark/metrics.ts
|
|
272
427
|
function summarizeBenchmarkResults(results) {
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
428
|
+
const totalChunks = results.length;
|
|
429
|
+
const succeededChunks = results.filter((r) => r.succeeded).length;
|
|
430
|
+
const failedChunks = totalChunks - succeededChunks;
|
|
431
|
+
const totalDurationMs = results.reduce((sum, r) => sum + r.durationMs, 0);
|
|
432
|
+
const totalAttempts = results.reduce((sum, r) => sum + r.attemptCount, 0);
|
|
433
|
+
return {
|
|
434
|
+
strategyName: results[0]?.strategyName ?? "one-shot",
|
|
435
|
+
totalChunks,
|
|
436
|
+
succeededChunks,
|
|
437
|
+
failedChunks,
|
|
438
|
+
totalDurationMs,
|
|
439
|
+
averageDurationMsPerChunk: totalChunks > 0 ? totalDurationMs / totalChunks : 0,
|
|
440
|
+
totalAttempts,
|
|
441
|
+
};
|
|
287
442
|
}
|
|
288
443
|
function runBenchmarkedChunk(strategy, ctx) {
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
444
|
+
return Effect.gen(function* () {
|
|
445
|
+
const start = Date.now();
|
|
446
|
+
const result = yield* Effect.either(strategy.translateChunk(ctx));
|
|
447
|
+
const durationMs = Date.now() - start;
|
|
448
|
+
if (result._tag === "Right")
|
|
449
|
+
return {
|
|
450
|
+
strategyName: strategy.name,
|
|
451
|
+
chunkKeyCount: ctx.keys.length,
|
|
452
|
+
durationMs,
|
|
453
|
+
attemptCount: 1,
|
|
454
|
+
succeeded: true,
|
|
455
|
+
errorMessage: void 0,
|
|
456
|
+
};
|
|
457
|
+
return {
|
|
458
|
+
strategyName: strategy.name,
|
|
459
|
+
chunkKeyCount: ctx.keys.length,
|
|
460
|
+
durationMs,
|
|
461
|
+
attemptCount: 1,
|
|
462
|
+
succeeded: false,
|
|
463
|
+
errorMessage: String(result.left.cause),
|
|
464
|
+
};
|
|
465
|
+
});
|
|
310
466
|
}
|
|
311
467
|
//#endregion
|
|
312
468
|
//#region src/benchmark/runner.ts
|
|
313
469
|
function runBenchmark(config) {
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
470
|
+
return Effect.gen(function* () {
|
|
471
|
+
const summaries = [];
|
|
472
|
+
for (const strategy of config.strategies) {
|
|
473
|
+
const results = yield* Effect.forEach(
|
|
474
|
+
config.chunks,
|
|
475
|
+
(chunk) => runBenchmarkedChunk(strategy, chunk),
|
|
476
|
+
{ concurrency: config.concurrency },
|
|
477
|
+
);
|
|
478
|
+
summaries.push(summarizeBenchmarkResults(results));
|
|
479
|
+
}
|
|
480
|
+
return summaries;
|
|
481
|
+
});
|
|
322
482
|
}
|
|
323
483
|
//#endregion
|
|
324
484
|
//#region src/cli/commands/benchmark.ts
|
|
325
485
|
function runBenchmarkCommand(flags, deps) {
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
486
|
+
return Effect.gen(function* () {
|
|
487
|
+
yield* deps.errorLogger(
|
|
488
|
+
formatError(
|
|
489
|
+
"Warning: This will make real API calls to the configured model provider(s) and may incur cost.",
|
|
490
|
+
detectFormat(flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0),
|
|
491
|
+
),
|
|
492
|
+
);
|
|
493
|
+
const loaded = yield* deps.configLoader(flags.config);
|
|
494
|
+
const effective = resolveEffectiveConfig(
|
|
495
|
+
{ adapter: Option.getOrUndefined(flags.adapter) },
|
|
496
|
+
loaded,
|
|
497
|
+
);
|
|
498
|
+
const strategyNames = Option.getOrElse(flags.strategies, () => "one-shot,tool-loop-agent")
|
|
499
|
+
.split(",")
|
|
500
|
+
.map((s) => s.trim());
|
|
501
|
+
const model = yield* deps.modelResolver(effective.model);
|
|
502
|
+
const strategyList = strategyNames.map((name) =>
|
|
503
|
+
name === "tool-loop-agent"
|
|
504
|
+
? createToolLoopStrategy({
|
|
505
|
+
model,
|
|
506
|
+
retry: effective.retry,
|
|
507
|
+
})
|
|
508
|
+
: createOneShotStrategy({
|
|
509
|
+
model,
|
|
510
|
+
retry: effective.retry,
|
|
511
|
+
}),
|
|
512
|
+
);
|
|
513
|
+
const allChunks = [];
|
|
514
|
+
for (const a of effective.adapters) {
|
|
515
|
+
const locales = yield* a.listLocales();
|
|
516
|
+
const sourceLocale = effective.sourceLocale;
|
|
517
|
+
const targets = locales.filter((l) => l !== sourceLocale);
|
|
518
|
+
const missingEntries = yield* deps.missingKeysComputer(a, sourceLocale, targets);
|
|
519
|
+
for (const entry of missingEntries) {
|
|
520
|
+
const sourceMap = yield* a.readResource(sourceLocale, entry.resource);
|
|
521
|
+
const targetMap = yield* a.readResource(entry.locale, entry.resource);
|
|
522
|
+
const chunks = chunkKeys(entry.missing, sourceMap, targetMap, {
|
|
523
|
+
maxTokens: effective.chunking.maxTokens,
|
|
524
|
+
charsPerToken: effective.chunking.charsPerToken,
|
|
525
|
+
});
|
|
526
|
+
for (const keys of chunks)
|
|
527
|
+
allChunks.push({
|
|
528
|
+
sourceLocale,
|
|
529
|
+
targetLocale: entry.locale,
|
|
530
|
+
sourceMap,
|
|
531
|
+
targetMap,
|
|
532
|
+
keys,
|
|
533
|
+
});
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
const sampled = allChunks.slice(
|
|
537
|
+
0,
|
|
538
|
+
Option.getOrElse(flags.sampleSize, () => 20),
|
|
539
|
+
);
|
|
540
|
+
const summaries = yield* deps.benchmarkRunner({
|
|
541
|
+
strategies: strategyList,
|
|
542
|
+
chunks: sampled,
|
|
543
|
+
concurrency: effective.chunking.concurrency,
|
|
544
|
+
});
|
|
545
|
+
const format = detectFormat(
|
|
546
|
+
flags.format !== void 0 ? Option.getOrUndefined(flags.format) : void 0,
|
|
547
|
+
);
|
|
548
|
+
const entries = summaries.map((s) => ({
|
|
549
|
+
strategyName: s.strategyName,
|
|
550
|
+
totalChunks: s.totalChunks,
|
|
551
|
+
succeededChunks: s.succeededChunks,
|
|
552
|
+
failedChunks: s.failedChunks,
|
|
553
|
+
totalDurationMs: s.totalDurationMs,
|
|
554
|
+
averageDurationMsPerChunk: s.averageDurationMsPerChunk,
|
|
555
|
+
totalAttempts: s.totalAttempts,
|
|
556
|
+
}));
|
|
557
|
+
yield* deps.logger(formatBenchmark(entries, format));
|
|
558
|
+
});
|
|
379
559
|
}
|
|
380
|
-
const benchmarkCommand = Command.make(
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
560
|
+
const benchmarkCommand = Command.make(
|
|
561
|
+
"benchmark",
|
|
562
|
+
{
|
|
563
|
+
config: Options.text("config").pipe(Options.withDefault("./dialekt.config.ts")),
|
|
564
|
+
adapter: Options.optional(Options.text("adapter")),
|
|
565
|
+
strategies: Options.optional(Options.text("strategies")),
|
|
566
|
+
sampleSize: Options.optional(Options.integer("sample-size")),
|
|
567
|
+
format: Options.optional(Options.text("format")),
|
|
568
|
+
},
|
|
569
|
+
(flags) =>
|
|
570
|
+
runBenchmarkCommand(flags, {
|
|
571
|
+
configLoader: loadConfig,
|
|
572
|
+
modelResolver: resolveModel,
|
|
573
|
+
missingKeysComputer: computeMissingKeys,
|
|
574
|
+
benchmarkRunner: runBenchmark,
|
|
575
|
+
logger: (msg) => Console.log(msg),
|
|
576
|
+
errorLogger: (msg) => Console.error(msg),
|
|
577
|
+
}),
|
|
578
|
+
);
|
|
394
579
|
//#endregion
|
|
395
580
|
//#region src/cli/main.ts
|
|
396
|
-
const rootCommand = Command.make("dialekt").pipe(
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
581
|
+
const rootCommand = Command.make("dialekt").pipe(
|
|
582
|
+
Command.withSubcommands([
|
|
583
|
+
translateCommand,
|
|
584
|
+
validateCommand,
|
|
585
|
+
addCommand,
|
|
586
|
+
missingCommand,
|
|
587
|
+
unusedCommand,
|
|
588
|
+
languagesCommand,
|
|
589
|
+
benchmarkCommand,
|
|
590
|
+
]),
|
|
591
|
+
);
|
|
405
592
|
const cli = Command.run(rootCommand, {
|
|
406
|
-
|
|
407
|
-
|
|
593
|
+
name: "dialekt",
|
|
594
|
+
version: "0.1.0",
|
|
408
595
|
});
|
|
409
596
|
const program = Effect.provide(cli(process.argv), NodeContext.layer);
|
|
410
597
|
NodeRuntime.runMain(program);
|