dialekt 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -10
- package/TESTING.md +29 -29
- package/dist/cli/main.d.mts +1 -1
- package/dist/cli/main.mjs +549 -362
- package/dist/formatters-De4Q-X1d.mjs +516 -435
- package/dist/index.d.mts +162 -25
- package/dist/index.mjs +119 -34
- package/package.json +3 -3
- package/pnpm-workspace.yaml +3 -3
- package/src/adapter/types.test.ts +57 -57
- package/src/adapter/types.ts +7 -4
- package/src/benchmark/metrics.test.ts +141 -69
- package/src/benchmark/metrics.ts +6 -6
- package/src/benchmark/report.test.ts +38 -38
- package/src/benchmark/report.ts +6 -6
- package/src/benchmark/runner.test.ts +70 -72
- package/src/benchmark/runner.ts +4 -4
- package/src/cli/commands/add.test.ts +90 -109
- package/src/cli/commands/add.ts +40 -28
- package/src/cli/commands/benchmark.test.ts +77 -64
- package/src/cli/commands/benchmark.ts +64 -41
- package/src/cli/commands/languages.test.ts +45 -42
- package/src/cli/commands/languages.ts +16 -12
- package/src/cli/commands/missing.test.ts +143 -92
- package/src/cli/commands/missing.ts +24 -17
- package/src/cli/commands/translate.test.ts +79 -79
- package/src/cli/commands/translate.ts +41 -31
- package/src/cli/commands/unused.test.ts +62 -51
- package/src/cli/commands/unused.ts +18 -14
- package/src/cli/commands/validate.test.ts +130 -72
- package/src/cli/commands/validate.ts +25 -20
- package/src/cli/config-resolution.test.ts +169 -49
- package/src/cli/config-resolution.ts +5 -7
- package/src/cli/format.test.ts +50 -50
- package/src/cli/format.ts +57 -60
- package/src/cli/formatters.test.ts +128 -106
- package/src/cli/formatters.ts +72 -95
- package/src/cli/main.ts +13 -13
- package/src/config/define-config.test.ts +44 -29
- package/src/config/define-config.ts +1 -1
- package/src/config/load-config.test.ts +21 -18
- package/src/config/load-config.ts +5 -5
- package/src/config/types.test.ts +50 -44
- package/src/config/types.ts +2 -2
- package/src/index.ts +22 -26
- package/src/keys/flatten.test.ts +52 -52
- package/src/keys/flatten.ts +7 -9
- package/src/sdk/file-io.test.ts +47 -59
- package/src/sdk/file-io.ts +2 -2
- package/src/sdk/node-layer.test.ts +18 -18
- package/src/sdk/node-layer.ts +2 -2
- package/src/sdk/php-array-reader.test.ts +49 -40
- package/src/sdk/php-array-reader.ts +5 -5
- package/src/translation/chunking.test.ts +52 -44
- package/src/translation/chunking.ts +1 -1
- package/src/translation/missing-keys.test.ts +86 -93
- package/src/translation/missing-keys.ts +4 -6
- package/src/translation/model-registry.test.ts +41 -32
- package/src/translation/model-registry.ts +9 -9
- package/src/translation/one-shot-strategy.test.ts +105 -86
- package/src/translation/one-shot-strategy.ts +10 -12
- package/src/translation/orchestrator.test.ts +90 -101
- package/src/translation/orchestrator.ts +26 -26
- package/src/translation/prompt.test.ts +76 -76
- package/src/translation/prompt.ts +2 -2
- package/src/translation/tool-loop-strategy.test.ts +134 -107
- package/src/translation/tool-loop-strategy.ts +14 -18
- package/src/translation/types.test.ts +22 -22
- package/src/translation/types.ts +3 -3
- package/tsdown.config.ts +3 -3
- package/vitest.config.ts +3 -3
package/src/sdk/file-io.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { describe, expect, it } from
|
|
2
|
-
import { Effect } from
|
|
3
|
-
import { FileSystem } from
|
|
4
|
-
import { Path } from
|
|
5
|
-
import { CommandExecutor } from
|
|
6
|
-
import { NodePlatformLayer } from
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { Effect } from "effect";
|
|
3
|
+
import { FileSystem } from "@effect/platform/FileSystem";
|
|
4
|
+
import { Path } from "@effect/platform/Path";
|
|
5
|
+
import { CommandExecutor } from "@effect/platform/CommandExecutor";
|
|
6
|
+
import { NodePlatformLayer } from "./node-layer.js";
|
|
7
7
|
|
|
8
|
-
describe(
|
|
9
|
-
it(
|
|
8
|
+
describe("NodePlatformLayer", () => {
|
|
9
|
+
it("provides a working FileSystem service", async () => {
|
|
10
10
|
const program = Effect.gen(function* () {
|
|
11
11
|
const fs = yield* FileSystem;
|
|
12
12
|
return yield* fs.exists(process.cwd());
|
|
@@ -15,17 +15,17 @@ describe('NodePlatformLayer', () => {
|
|
|
15
15
|
expect(result).toBe(true);
|
|
16
16
|
});
|
|
17
17
|
|
|
18
|
-
it(
|
|
18
|
+
it("provides a working Path service", async () => {
|
|
19
19
|
const program = Effect.gen(function* () {
|
|
20
20
|
const path = yield* Path;
|
|
21
|
-
return path.join(
|
|
21
|
+
return path.join("a", "b");
|
|
22
22
|
});
|
|
23
23
|
const result = await Effect.runPromise(Effect.provide(program, NodePlatformLayer));
|
|
24
|
-
expect(result).toContain(
|
|
25
|
-
expect(result).toContain(
|
|
24
|
+
expect(result).toContain("a");
|
|
25
|
+
expect(result).toContain("b");
|
|
26
26
|
});
|
|
27
27
|
|
|
28
|
-
it(
|
|
28
|
+
it("provides a working CommandExecutor service", async () => {
|
|
29
29
|
const program = Effect.gen(function* () {
|
|
30
30
|
const executor = yield* CommandExecutor;
|
|
31
31
|
return executor !== undefined;
|
|
@@ -34,19 +34,19 @@ describe('NodePlatformLayer', () => {
|
|
|
34
34
|
expect(result).toBe(true);
|
|
35
35
|
});
|
|
36
36
|
|
|
37
|
-
it(
|
|
37
|
+
it("can read a real file", async () => {
|
|
38
38
|
const program = Effect.gen(function* () {
|
|
39
39
|
const fs = yield* FileSystem;
|
|
40
|
-
return yield* fs.readFileString(process.cwd() +
|
|
40
|
+
return yield* fs.readFileString(process.cwd() + "/package.json");
|
|
41
41
|
});
|
|
42
42
|
const result = await Effect.runPromise(Effect.provide(program, NodePlatformLayer));
|
|
43
|
-
expect(result).toContain(
|
|
43
|
+
expect(result).toContain("name");
|
|
44
44
|
});
|
|
45
45
|
|
|
46
|
-
it(
|
|
46
|
+
it("reports missing files correctly", async () => {
|
|
47
47
|
const program = Effect.gen(function* () {
|
|
48
48
|
const fs = yield* FileSystem;
|
|
49
|
-
return yield* fs.exists(
|
|
49
|
+
return yield* fs.exists("/nonexistent/path/that/should/not/exist");
|
|
50
50
|
});
|
|
51
51
|
const result = await Effect.runPromise(Effect.provide(program, NodePlatformLayer));
|
|
52
52
|
expect(result).toBe(false);
|
package/src/sdk/node-layer.ts
CHANGED
|
@@ -1,89 +1,95 @@
|
|
|
1
|
-
import { describe, expect, it } from
|
|
2
|
-
import { Effect, Either } from
|
|
3
|
-
import { CommandExecutor, Command } from
|
|
4
|
-
import { NodePlatformLayer } from
|
|
5
|
-
import { readPhpArrayAsJson, PhpExecutionError } from
|
|
6
|
-
import { writeFileSync, mkdirSync, rmSync } from
|
|
7
|
-
import { join } from
|
|
8
|
-
import { tmpdir } from
|
|
9
|
-
import { execSync } from
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { Effect, Either } from "effect";
|
|
3
|
+
import { CommandExecutor, Command } from "@effect/platform";
|
|
4
|
+
import { NodePlatformLayer } from "./node-layer.js";
|
|
5
|
+
import { readPhpArrayAsJson, PhpExecutionError } from "./php-array-reader.js";
|
|
6
|
+
import { writeFileSync, mkdirSync, rmSync } from "node:fs";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { tmpdir } from "node:os";
|
|
9
|
+
import { execSync } from "node:child_process";
|
|
10
10
|
|
|
11
11
|
function hasPhpBinary(): boolean {
|
|
12
12
|
try {
|
|
13
|
-
execSync(
|
|
13
|
+
execSync("php -v", { stdio: "ignore" });
|
|
14
14
|
return true;
|
|
15
15
|
} catch {
|
|
16
16
|
return false;
|
|
17
17
|
}
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
-
describe(
|
|
20
|
+
describe("readPhpArrayAsJson", () => {
|
|
21
21
|
const testDir = join(tmpdir(), `dialekt-php-test-${Date.now()}`);
|
|
22
22
|
|
|
23
|
-
it.skipIf(!hasPhpBinary())(
|
|
23
|
+
it.skipIf(!hasPhpBinary())("reads a PHP array file as JSON", async () => {
|
|
24
24
|
mkdirSync(testDir, { recursive: true });
|
|
25
|
-
const filePath = join(testDir,
|
|
25
|
+
const filePath = join(testDir, "test.php");
|
|
26
26
|
writeFileSync(filePath, "<?php return ['email' => 'Email address'];");
|
|
27
27
|
|
|
28
28
|
const program = Effect.provide(readPhpArrayAsJson(filePath), NodePlatformLayer);
|
|
29
29
|
const result = await Effect.runPromise(program);
|
|
30
|
-
expect(result).toEqual({ email:
|
|
30
|
+
expect(result).toEqual({ email: "Email address" });
|
|
31
31
|
|
|
32
32
|
rmSync(testDir, { recursive: true, force: true });
|
|
33
33
|
});
|
|
34
34
|
|
|
35
|
-
it.skipIf(!hasPhpBinary())(
|
|
36
|
-
const program = Effect.provide(readPhpArrayAsJson(
|
|
37
|
-
const exit = await Effect.runPromise(Effect.either(program)) as Either.Either<
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
35
|
+
it.skipIf(!hasPhpBinary())("returns PhpExecutionError for a nonexistent file", async () => {
|
|
36
|
+
const program = Effect.provide(readPhpArrayAsJson("/nonexistent/file.php"), NodePlatformLayer);
|
|
37
|
+
const exit = (await Effect.runPromise(Effect.either(program))) as Either.Either<
|
|
38
|
+
unknown,
|
|
39
|
+
PhpExecutionError
|
|
40
|
+
>;
|
|
41
|
+
if (exit._tag === "Left") {
|
|
42
|
+
expect(exit.left._tag).toBe("PhpExecutionError");
|
|
43
|
+
expect(exit.left.path).toBe("/nonexistent/file.php");
|
|
41
44
|
} else {
|
|
42
|
-
throw new Error(
|
|
45
|
+
throw new Error("Expected Left");
|
|
43
46
|
}
|
|
44
47
|
});
|
|
45
48
|
|
|
46
|
-
it.skipIf(!hasPhpBinary())(
|
|
49
|
+
it.skipIf(!hasPhpBinary())("reads nested PHP arrays", async () => {
|
|
47
50
|
mkdirSync(testDir, { recursive: true });
|
|
48
|
-
const filePath = join(testDir,
|
|
49
|
-
writeFileSync(
|
|
51
|
+
const filePath = join(testDir, "nested.php");
|
|
52
|
+
writeFileSync(
|
|
53
|
+
filePath,
|
|
54
|
+
"<?php return ['validation' => ['email' => 'Email address', 'required' => 'Required field']];",
|
|
55
|
+
);
|
|
50
56
|
|
|
51
57
|
const program = Effect.provide(readPhpArrayAsJson(filePath), NodePlatformLayer);
|
|
52
58
|
const result = await Effect.runPromise(program);
|
|
53
59
|
expect(result).toEqual({
|
|
54
|
-
validation: { email:
|
|
60
|
+
validation: { email: "Email address", required: "Required field" },
|
|
55
61
|
});
|
|
56
62
|
|
|
57
63
|
rmSync(testDir, { recursive: true, force: true });
|
|
58
64
|
});
|
|
59
65
|
|
|
60
|
-
it.skipIf(!hasPhpBinary())(
|
|
66
|
+
it.skipIf(!hasPhpBinary())("reads PHP arrays with numeric keys as JS arrays", async () => {
|
|
61
67
|
mkdirSync(testDir, { recursive: true });
|
|
62
|
-
const filePath = join(testDir,
|
|
68
|
+
const filePath = join(testDir, "numeric.php");
|
|
63
69
|
writeFileSync(filePath, "<?php return ['first', 'second', 'third'];");
|
|
64
70
|
|
|
65
71
|
const program = Effect.provide(readPhpArrayAsJson(filePath), NodePlatformLayer);
|
|
66
72
|
const result = await Effect.runPromise(program);
|
|
67
|
-
expect(result).toEqual([
|
|
73
|
+
expect(result).toEqual(["first", "second", "third"]);
|
|
68
74
|
|
|
69
75
|
rmSync(testDir, { recursive: true, force: true });
|
|
70
76
|
});
|
|
71
77
|
|
|
72
|
-
it.skipIf(!hasPhpBinary())(
|
|
78
|
+
it.skipIf(!hasPhpBinary())("reads PHP arrays with unicode values", async () => {
|
|
73
79
|
mkdirSync(testDir, { recursive: true });
|
|
74
|
-
const filePath = join(testDir,
|
|
80
|
+
const filePath = join(testDir, "unicode.php");
|
|
75
81
|
writeFileSync(filePath, "<?php return ['greeting' => 'Héllo 🌍 — 日本語'];");
|
|
76
82
|
|
|
77
83
|
const program = Effect.provide(readPhpArrayAsJson(filePath), NodePlatformLayer);
|
|
78
84
|
const result = await Effect.runPromise(program);
|
|
79
|
-
expect(result).toEqual({ greeting:
|
|
85
|
+
expect(result).toEqual({ greeting: "Héllo 🌍 — 日本語" });
|
|
80
86
|
|
|
81
87
|
rmSync(testDir, { recursive: true, force: true });
|
|
82
88
|
});
|
|
83
89
|
|
|
84
|
-
it.skipIf(!hasPhpBinary())(
|
|
90
|
+
it.skipIf(!hasPhpBinary())("reads empty PHP array as empty JS array", async () => {
|
|
85
91
|
mkdirSync(testDir, { recursive: true });
|
|
86
|
-
const filePath = join(testDir,
|
|
92
|
+
const filePath = join(testDir, "empty.php");
|
|
87
93
|
writeFileSync(filePath, "<?php return [];");
|
|
88
94
|
|
|
89
95
|
const program = Effect.provide(readPhpArrayAsJson(filePath), NodePlatformLayer);
|
|
@@ -93,22 +99,25 @@ describe('readPhpArrayAsJson', () => {
|
|
|
93
99
|
rmSync(testDir, { recursive: true, force: true });
|
|
94
100
|
});
|
|
95
101
|
|
|
96
|
-
it.skipIf(!hasPhpBinary())(
|
|
102
|
+
it.skipIf(!hasPhpBinary())("returns PhpExecutionError for malformed PHP", async () => {
|
|
97
103
|
mkdirSync(testDir, { recursive: true });
|
|
98
|
-
const filePath = join(testDir,
|
|
104
|
+
const filePath = join(testDir, "bad.php");
|
|
99
105
|
writeFileSync(filePath, "<?php this is not valid php");
|
|
100
106
|
|
|
101
107
|
const program = Effect.provide(readPhpArrayAsJson(filePath), NodePlatformLayer);
|
|
102
|
-
const exit = await Effect.runPromise(Effect.either(program)) as Either.Either<
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
108
|
+
const exit = (await Effect.runPromise(Effect.either(program))) as Either.Either<
|
|
109
|
+
unknown,
|
|
110
|
+
PhpExecutionError
|
|
111
|
+
>;
|
|
112
|
+
expect(exit._tag).toBe("Left");
|
|
113
|
+
if (exit._tag === "Left") {
|
|
114
|
+
expect(exit.left._tag).toBe("PhpExecutionError");
|
|
106
115
|
}
|
|
107
116
|
|
|
108
117
|
rmSync(testDir, { recursive: true, force: true });
|
|
109
118
|
});
|
|
110
119
|
|
|
111
|
-
it(
|
|
120
|
+
it("is skipped when php is unavailable", () => {
|
|
112
121
|
expect(hasPhpBinary()).toBe(true); // meta-test: if php IS available, this proves skipIf works
|
|
113
122
|
});
|
|
114
123
|
});
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { Command } from
|
|
2
|
-
import type { CommandExecutor } from
|
|
3
|
-
import { Effect, Data } from
|
|
1
|
+
import { Command } from "@effect/platform";
|
|
2
|
+
import type { CommandExecutor } from "@effect/platform/CommandExecutor";
|
|
3
|
+
import { Effect, Data } from "effect";
|
|
4
4
|
|
|
5
|
-
export class PhpExecutionError extends Data.TaggedError(
|
|
5
|
+
export class PhpExecutionError extends Data.TaggedError("PhpExecutionError")<{
|
|
6
6
|
readonly path: string;
|
|
7
7
|
readonly cause: unknown;
|
|
8
8
|
}> {}
|
|
@@ -14,7 +14,7 @@ export function readPhpArrayAsJson(
|
|
|
14
14
|
absolutePath: string,
|
|
15
15
|
): Effect.Effect<Record<string, unknown>, PhpExecutionError, CommandExecutor> {
|
|
16
16
|
return Effect.gen(function* () {
|
|
17
|
-
const cmd = Command.make(
|
|
17
|
+
const cmd = Command.make("php", "-r", DUMP_SCRIPT, "--", absolutePath);
|
|
18
18
|
const output = yield* Command.string(cmd).pipe(
|
|
19
19
|
Effect.mapError((cause) => new PhpExecutionError({ path: absolutePath, cause })),
|
|
20
20
|
);
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
import { describe, expect, it } from
|
|
2
|
-
import { chunkKeys } from
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { chunkKeys } from "./chunking.js";
|
|
3
3
|
|
|
4
|
-
describe(
|
|
5
|
-
it(
|
|
6
|
-
const source = { a:
|
|
4
|
+
describe("chunkKeys", () => {
|
|
5
|
+
it("keeps a small key set in a single chunk", () => {
|
|
6
|
+
const source = { a: "A", b: "B" };
|
|
7
7
|
const target = {};
|
|
8
|
-
const chunks = chunkKeys([
|
|
8
|
+
const chunks = chunkKeys(["a", "b"], source, target, { maxTokens: 3000, charsPerToken: 3.0 });
|
|
9
9
|
expect(chunks).toHaveLength(1);
|
|
10
|
-
expect(chunks[0]).toEqual([
|
|
10
|
+
expect(chunks[0]).toEqual(["a", "b"]);
|
|
11
11
|
});
|
|
12
12
|
|
|
13
|
-
it(
|
|
13
|
+
it("splits a large key set into multiple chunks", () => {
|
|
14
14
|
const source: Record<string, string> = {};
|
|
15
15
|
for (let i = 0; i < 150; i++) {
|
|
16
16
|
source[`key_${i}`] = `This is the value number ${i} with some extra text to make it longer`;
|
|
@@ -24,10 +24,10 @@ describe('chunkKeys', () => {
|
|
|
24
24
|
expect(new Set(allKeys).size).toBe(keys.length);
|
|
25
25
|
});
|
|
26
26
|
|
|
27
|
-
it(
|
|
27
|
+
it("preserves all keys across chunks without loss or duplication", () => {
|
|
28
28
|
const source: Record<string, string> = {};
|
|
29
29
|
for (let i = 0; i < 60; i++) {
|
|
30
|
-
source[`k${i}`] =
|
|
30
|
+
source[`k${i}`] = "word ".repeat(10);
|
|
31
31
|
}
|
|
32
32
|
const target = {};
|
|
33
33
|
const keys = Object.keys(source);
|
|
@@ -36,18 +36,18 @@ describe('chunkKeys', () => {
|
|
|
36
36
|
expect(flattened.sort()).toEqual([...keys].sort());
|
|
37
37
|
});
|
|
38
38
|
|
|
39
|
-
it(
|
|
40
|
-
const source = { long:
|
|
39
|
+
it("keeps a single oversized key in its own chunk", () => {
|
|
40
|
+
const source = { long: "x".repeat(5000) };
|
|
41
41
|
const target = {};
|
|
42
|
-
const chunks = chunkKeys([
|
|
42
|
+
const chunks = chunkKeys(["long"], source, target, { maxTokens: 3000, charsPerToken: 3.0 });
|
|
43
43
|
expect(chunks).toHaveLength(1);
|
|
44
|
-
expect(chunks[0]).toEqual([
|
|
44
|
+
expect(chunks[0]).toEqual(["long"]);
|
|
45
45
|
});
|
|
46
46
|
|
|
47
|
-
it(
|
|
47
|
+
it("reduces effective chunk size when file context is huge", () => {
|
|
48
48
|
const source: Record<string, string> = {};
|
|
49
49
|
for (let i = 0; i < 26; i++) {
|
|
50
|
-
source[String.fromCharCode(97 + i)] =
|
|
50
|
+
source[String.fromCharCode(97 + i)] = "word ".repeat(50);
|
|
51
51
|
}
|
|
52
52
|
const target = {};
|
|
53
53
|
const keys = Object.keys(source).slice(0, 10);
|
|
@@ -55,45 +55,48 @@ describe('chunkKeys', () => {
|
|
|
55
55
|
expect(chunks.length).toBeGreaterThan(0);
|
|
56
56
|
});
|
|
57
57
|
|
|
58
|
-
it(
|
|
58
|
+
it("falls back to one key per chunk when the context exceeds the limit", () => {
|
|
59
59
|
const source: Record<string, string> = {};
|
|
60
60
|
for (let i = 0; i < 100; i++) {
|
|
61
|
-
source[`key_${i}`] =
|
|
61
|
+
source[`key_${i}`] = "word ".repeat(100);
|
|
62
62
|
}
|
|
63
63
|
const target = { ...source };
|
|
64
|
-
const chunks = chunkKeys([
|
|
65
|
-
|
|
66
|
-
|
|
64
|
+
const chunks = chunkKeys(["key_0", "key_1"], source, target, {
|
|
65
|
+
maxTokens: 10,
|
|
66
|
+
charsPerToken: 3.0,
|
|
67
|
+
});
|
|
68
|
+
expect(chunks[0]).toEqual(["key_0"]);
|
|
69
|
+
expect(chunks[1]).toEqual(["key_1"]);
|
|
67
70
|
});
|
|
68
71
|
|
|
69
|
-
it(
|
|
72
|
+
it("returns [] for empty keys", () => {
|
|
70
73
|
expect(chunkKeys([], {}, {}, { maxTokens: 3000, charsPerToken: 3.0 })).toEqual([]);
|
|
71
74
|
});
|
|
72
75
|
|
|
73
|
-
it(
|
|
74
|
-
const source = { a:
|
|
75
|
-
const chunks = chunkKeys([
|
|
76
|
-
expect(chunks).toEqual([[
|
|
76
|
+
it("still includes keys not present in source (uses empty string as value)", () => {
|
|
77
|
+
const source = { a: "A" };
|
|
78
|
+
const chunks = chunkKeys(["missing"], source, {}, { maxTokens: 3000, charsPerToken: 3.0 });
|
|
79
|
+
expect(chunks).toEqual([["missing"]]);
|
|
77
80
|
});
|
|
78
81
|
|
|
79
|
-
it(
|
|
80
|
-
const source = { a:
|
|
81
|
-
const chunks = chunkKeys([
|
|
82
|
+
it("handles keys with empty values", () => {
|
|
83
|
+
const source = { a: "", b: "" };
|
|
84
|
+
const chunks = chunkKeys(["a", "b"], source, {}, { maxTokens: 3000, charsPerToken: 3.0 });
|
|
82
85
|
expect(chunks).toHaveLength(1);
|
|
83
|
-
expect(chunks[0]).toEqual([
|
|
86
|
+
expect(chunks[0]).toEqual(["a", "b"]);
|
|
84
87
|
});
|
|
85
88
|
|
|
86
|
-
it(
|
|
87
|
-
const source = { only:
|
|
88
|
-
const chunks = chunkKeys([
|
|
89
|
+
it("handles single key", () => {
|
|
90
|
+
const source = { only: "value" };
|
|
91
|
+
const chunks = chunkKeys(["only"], source, {}, { maxTokens: 3000, charsPerToken: 3.0 });
|
|
89
92
|
expect(chunks).toHaveLength(1);
|
|
90
|
-
expect(chunks[0]).toEqual([
|
|
93
|
+
expect(chunks[0]).toEqual(["only"]);
|
|
91
94
|
});
|
|
92
95
|
|
|
93
|
-
it(
|
|
96
|
+
it("handles very large number of keys", () => {
|
|
94
97
|
const source: Record<string, string> = {};
|
|
95
98
|
for (let i = 0; i < 1000; i++) {
|
|
96
|
-
source[`key_${i}`] =
|
|
99
|
+
source[`key_${i}`] = "v";
|
|
97
100
|
}
|
|
98
101
|
const keys = Object.keys(source);
|
|
99
102
|
const chunks = chunkKeys(keys, source, {}, { maxTokens: 3000, charsPerToken: 3.0 });
|
|
@@ -101,18 +104,23 @@ describe('chunkKeys', () => {
|
|
|
101
104
|
expect(all).toHaveLength(1000);
|
|
102
105
|
});
|
|
103
106
|
|
|
104
|
-
it(
|
|
105
|
-
const source = {
|
|
106
|
-
const chunks = chunkKeys(
|
|
107
|
+
it("handles keys with special characters", () => {
|
|
108
|
+
const source = { "key.with.dots": "value", "key-with-dashes": "value2" };
|
|
109
|
+
const chunks = chunkKeys(
|
|
110
|
+
["key.with.dots", "key-with-dashes"],
|
|
111
|
+
source,
|
|
112
|
+
{},
|
|
113
|
+
{ maxTokens: 3000, charsPerToken: 3.0 },
|
|
114
|
+
);
|
|
107
115
|
expect(chunks).toHaveLength(1);
|
|
108
|
-
expect(chunks[0]).toEqual([
|
|
116
|
+
expect(chunks[0]).toEqual(["key.with.dots", "key-with-dashes"]);
|
|
109
117
|
});
|
|
110
118
|
|
|
111
|
-
it(
|
|
112
|
-
const source = { a:
|
|
113
|
-
const chunks = chunkKeys([
|
|
119
|
+
it("respects MIN_EFFECTIVE_MAX_CHARS even with zero maxTokens", () => {
|
|
120
|
+
const source = { a: "A", b: "B" };
|
|
121
|
+
const chunks = chunkKeys(["a", "b"], source, {}, { maxTokens: 0, charsPerToken: 3.0 });
|
|
114
122
|
// MIN_EFFECTIVE_MAX_CHARS is 200, so both small keys fit in one chunk
|
|
115
123
|
expect(chunks).toHaveLength(1);
|
|
116
|
-
expect(chunks[0]).toEqual([
|
|
124
|
+
expect(chunks[0]).toEqual(["a", "b"]);
|
|
117
125
|
});
|
|
118
126
|
});
|
|
@@ -24,7 +24,7 @@ export function chunkKeys(
|
|
|
24
24
|
let currentChars = 0;
|
|
25
25
|
|
|
26
26
|
for (const key of keys) {
|
|
27
|
-
const value = sourceMap[key] ??
|
|
27
|
+
const value = sourceMap[key] ?? "";
|
|
28
28
|
const itemChars = key.length + value.length + ITEM_JSON_OVERHEAD;
|
|
29
29
|
|
|
30
30
|
// If a single key is larger than the entire chunk, force it through alone.
|