@garrix82/reactgenie-dsl 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +436 -0
- package/dist/__test__/dsl-descriptor.test.d.ts +1 -0
- package/dist/__test__/dsl-descriptor.test.js +27 -0
- package/dist/__test__/dsl-descriptor.test.js.map +1 -0
- package/dist/__test__/example_descriptor.d.ts +125 -0
- package/dist/__test__/example_descriptor.js +607 -0
- package/dist/__test__/example_descriptor.js.map +1 -0
- package/dist/__test__/food_descriptor.state.json +1 -0
- package/dist/__test__/food_descriptor.test.d.ts +74 -0
- package/dist/__test__/food_descriptor.test.js +205 -0
- package/dist/__test__/food_descriptor.test.js.map +1 -0
- package/dist/__test__/nl-interpreter-provider-selection.test.d.ts +1 -0
- package/dist/__test__/nl-interpreter-provider-selection.test.js +73 -0
- package/dist/__test__/nl-interpreter-provider-selection.test.js.map +1 -0
- package/dist/__test__/nl-interpreter.test.d.ts +1 -0
- package/dist/__test__/nl-interpreter.test.js +86 -0
- package/dist/__test__/nl-interpreter.test.js.map +1 -0
- package/dist/decorators/__test__/decorators.test.d.ts +1 -0
- package/dist/decorators/__test__/decorators.test.js +182 -0
- package/dist/decorators/__test__/decorators.test.js.map +1 -0
- package/dist/decorators/__test__/inheritance-descriptor.test.d.ts +1 -0
- package/dist/decorators/__test__/inheritance-descriptor.test.js +107 -0
- package/dist/decorators/__test__/inheritance-descriptor.test.js.map +1 -0
- package/dist/decorators/decorators.d.ts +20 -0
- package/dist/decorators/decorators.js +520 -0
- package/dist/decorators/decorators.js.map +1 -0
- package/dist/decorators/index.d.ts +2 -0
- package/dist/decorators/index.js +19 -0
- package/dist/decorators/index.js.map +1 -0
- package/dist/decorators/store.d.ts +12 -0
- package/dist/decorators/store.js +43 -0
- package/dist/decorators/store.js.map +1 -0
- package/dist/dsl/__test__/dsl-interpreter.test.d.ts +1 -0
- package/dist/dsl/__test__/dsl-interpreter.test.js +334 -0
- package/dist/dsl/__test__/dsl-interpreter.test.js.map +1 -0
- package/dist/dsl/__test__/parser.gen.test.d.ts +1 -0
- package/dist/dsl/__test__/parser.gen.test.js +283 -0
- package/dist/dsl/__test__/parser.gen.test.js.map +1 -0
- package/dist/dsl/dsl-interpreter.d.ts +66 -0
- package/dist/dsl/dsl-interpreter.js +767 -0
- package/dist/dsl/dsl-interpreter.js.map +1 -0
- package/dist/dsl/index.d.ts +1 -0
- package/dist/dsl/index.js +18 -0
- package/dist/dsl/index.js.map +1 -0
- package/dist/dsl/parser.gen.d.ts +10 -0
- package/dist/dsl/parser.gen.js +1524 -0
- package/dist/dsl/parser.gen.js.map +1 -0
- package/dist/dsl-descriptor.d.ts +75 -0
- package/dist/dsl-descriptor.js +151 -0
- package/dist/dsl-descriptor.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +22 -0
- package/dist/index.js.map +1 -0
- package/dist/nl/__test__/context-aware-prompt.test.d.ts +1 -0
- package/dist/nl/__test__/context-aware-prompt.test.js +247 -0
- package/dist/nl/__test__/context-aware-prompt.test.js.map +1 -0
- package/dist/nl/__test__/context-selector.test.d.ts +1 -0
- package/dist/nl/__test__/context-selector.test.js +20 -0
- package/dist/nl/__test__/context-selector.test.js.map +1 -0
- package/dist/nl/__test__/nl-parser-groq-transport.test.d.ts +1 -0
- package/dist/nl/__test__/nl-parser-groq-transport.test.js +87 -0
- package/dist/nl/__test__/nl-parser-groq-transport.test.js.map +1 -0
- package/dist/nl/__test__/nl-parser-openai-parity.test.d.ts +1 -0
- package/dist/nl/__test__/nl-parser-openai-parity.test.js +206 -0
- package/dist/nl/__test__/nl-parser-openai-parity.test.js.map +1 -0
- package/dist/nl/__test__/nl-parser-openai-sampling.test.d.ts +1 -0
- package/dist/nl/__test__/nl-parser-openai-sampling.test.js +44 -0
- package/dist/nl/__test__/nl-parser-openai-sampling.test.js.map +1 -0
- package/dist/nl/__test__/nl-parser-openai-transport.test.d.ts +1 -0
- package/dist/nl/__test__/nl-parser-openai-transport.test.js +55 -0
- package/dist/nl/__test__/nl-parser-openai-transport.test.js.map +1 -0
- package/dist/nl/__test__/nl-parser-utils.test.d.ts +1 -0
- package/dist/nl/__test__/nl-parser-utils.test.js +70 -0
- package/dist/nl/__test__/nl-parser-utils.test.js.map +1 -0
- package/dist/nl/__test__/nl-parser.test.d.ts +1 -0
- package/dist/nl/__test__/nl-parser.test.js +64 -0
- package/dist/nl/__test__/nl-parser.test.js.map +1 -0
- package/dist/nl/__test__/parameter-tuning.test.d.ts +1 -0
- package/dist/nl/__test__/parameter-tuning.test.js +95 -0
- package/dist/nl/__test__/parameter-tuning.test.js.map +1 -0
- package/dist/nl/__test__/semantic-parsing-experiment.test.d.ts +1 -0
- package/dist/nl/__test__/semantic-parsing-experiment.test.js +178 -0
- package/dist/nl/__test__/semantic-parsing-experiment.test.js.map +1 -0
- package/dist/nl/context-selector.d.ts +17 -0
- package/dist/nl/context-selector.js +99 -0
- package/dist/nl/context-selector.js.map +1 -0
- package/dist/nl/index.d.ts +6 -0
- package/dist/nl/index.js +15 -0
- package/dist/nl/index.js.map +1 -0
- package/dist/nl/llm-monitoring.d.ts +49 -0
- package/dist/nl/llm-monitoring.js +243 -0
- package/dist/nl/llm-monitoring.js.map +1 -0
- package/dist/nl/llm-monitoring.test.d.ts +5 -0
- package/dist/nl/llm-monitoring.test.js +101 -0
- package/dist/nl/llm-monitoring.test.js.map +1 -0
- package/dist/nl/nl-parser-groq.d.ts +21 -0
- package/dist/nl/nl-parser-groq.js +357 -0
- package/dist/nl/nl-parser-groq.js.map +1 -0
- package/dist/nl/nl-parser-utils.d.ts +101 -0
- package/dist/nl/nl-parser-utils.js +255 -0
- package/dist/nl/nl-parser-utils.js.map +1 -0
- package/dist/nl/nl-parser.d.ts +30 -0
- package/dist/nl/nl-parser.js +433 -0
- package/dist/nl/nl-parser.js.map +1 -0
- package/dist/nl/prompt-gen.d.ts +93 -0
- package/dist/nl/prompt-gen.js +369 -0
- package/dist/nl/prompt-gen.js.map +1 -0
- package/dist/nl/prompt-res.d.ts +16 -0
- package/dist/nl/prompt-res.js +190 -0
- package/dist/nl/prompt-res.js.map +1 -0
- package/dist/nl-interpreter.d.ts +48 -0
- package/dist/nl-interpreter.js +155 -0
- package/dist/nl-interpreter.js.map +1 -0
- package/lib/__test__/dsl-descriptor.test.ts +27 -0
- package/lib/__test__/example_descriptor.ts +762 -0
- package/lib/__test__/food_descriptor.state.json +1 -0
- package/lib/__test__/food_descriptor.test.ts +331 -0
- package/lib/__test__/nl-interpreter-provider-selection.test.ts +126 -0
- package/lib/__test__/nl-interpreter.test.ts +129 -0
- package/lib/decorators/__test__/decorators.test.ts +177 -0
- package/lib/decorators/__test__/inheritance-descriptor.test.ts +92 -0
- package/lib/decorators/decorators.ts +754 -0
- package/lib/decorators/index.ts +2 -0
- package/lib/decorators/store.ts +47 -0
- package/lib/dsl/__test__/dsl-interpreter.test.ts +453 -0
- package/lib/dsl/__test__/parser.gen.test.ts +296 -0
- package/lib/dsl/dsl-interpreter.ts +974 -0
- package/lib/dsl/index.ts +1 -0
- package/lib/dsl/parser.gen.js +1479 -0
- package/lib/dsl/parser.pegjs +130 -0
- package/lib/dsl-descriptor.ts +241 -0
- package/lib/index.ts +5 -0
- package/lib/nl/__test__/context-aware-prompt.test.ts +372 -0
- package/lib/nl/__test__/context-selector.test.ts +27 -0
- package/lib/nl/__test__/nl-parser-groq-transport.test.ts +139 -0
- package/lib/nl/__test__/nl-parser-openai-parity.test.ts +381 -0
- package/lib/nl/__test__/nl-parser-openai-sampling.test.ts +73 -0
- package/lib/nl/__test__/nl-parser-openai-transport.test.ts +79 -0
- package/lib/nl/__test__/nl-parser-utils.test.ts +98 -0
- package/lib/nl/__test__/nl-parser.test.ts +119 -0
- package/lib/nl/__test__/parameter-tuning.test.ts +137 -0
- package/lib/nl/__test__/semantic-parsing-experiment.test.ts +260 -0
- package/lib/nl/context-selector.ts +123 -0
- package/lib/nl/index.ts +19 -0
- package/lib/nl/llm-monitoring.test.ts +136 -0
- package/lib/nl/llm-monitoring.ts +339 -0
- package/lib/nl/nl-parser-groq.ts +510 -0
- package/lib/nl/nl-parser-utils.ts +310 -0
- package/lib/nl/nl-parser.ts +616 -0
- package/lib/nl/prompt-gen.ts +607 -0
- package/lib/nl/prompt-res.ts +207 -0
- package/lib/nl-interpreter.ts +262 -0
- package/package.json +58 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const nl_parser_groq_1 = require("../nl-parser-groq");
|
|
4
|
+
const prompt_gen_1 = require("../prompt-gen");
|
|
5
|
+
const example_descriptor_1 = require("../../__test__/example_descriptor");
|
|
6
|
+
const decorators_1 = require("../../decorators");
|
|
7
|
+
(0, decorators_1.initGenie)();
|
|
8
|
+
jest.setTimeout(30000);
|
|
9
|
+
describe("NlParserGroq offline regression", () => {
|
|
10
|
+
const originalFetch = global.fetch;
|
|
11
|
+
afterEach(() => {
|
|
12
|
+
global.fetch = originalFetch;
|
|
13
|
+
jest.restoreAllMocks();
|
|
14
|
+
});
|
|
15
|
+
test("Parser basics", async () => {
|
|
16
|
+
global.fetch = jest.fn().mockResolvedValue(new Response(JSON.stringify({
|
|
17
|
+
choices: [
|
|
18
|
+
{
|
|
19
|
+
message: {
|
|
20
|
+
content: JSON.stringify({
|
|
21
|
+
command: 'Restaurant.all().matching(field: .address, value: "palo alto").sort(field: .priceGrade, ascending: true)[0]',
|
|
22
|
+
}),
|
|
23
|
+
},
|
|
24
|
+
},
|
|
25
|
+
],
|
|
26
|
+
usage: {},
|
|
27
|
+
}), { status: 200, headers: { "content-type": "application/json" } }));
|
|
28
|
+
const parser = new nl_parser_groq_1.NlParserGroq(new prompt_gen_1.BasicPromptGen(example_descriptor_1.classDescriptions, example_descriptor_1.examples), "test-key", "llama-3.3-70b-versatile");
|
|
29
|
+
await expect(parser.parse("get me the cheapest restaurant in palo alto")).resolves.toBe('Restaurant.all().matching(field: .address, value: "palo alto").sort(field: .priceGrade, ascending: true)[0]');
|
|
30
|
+
});
|
|
31
|
+
test("Parser complex", async () => {
|
|
32
|
+
global.fetch = jest.fn().mockResolvedValue(new Response(JSON.stringify({
|
|
33
|
+
choices: [
|
|
34
|
+
{
|
|
35
|
+
message: {
|
|
36
|
+
content: JSON.stringify({
|
|
37
|
+
command: 'Order.current().addFoods(foods: [Order.all().matching(field: .restaurant, value: Restaurant.all().matching(field: .name, value: "mcDonald")[0]).sort(field: .dateTime, ascending: false)[0].foods.matching(field: .name, value: "burger")[0]])',
|
|
38
|
+
}),
|
|
39
|
+
},
|
|
40
|
+
},
|
|
41
|
+
],
|
|
42
|
+
usage: {},
|
|
43
|
+
}), { status: 200, headers: { "content-type": "application/json" } }));
|
|
44
|
+
const parser = new nl_parser_groq_1.NlParserGroq(new prompt_gen_1.BasicPromptGen(example_descriptor_1.classDescriptions, example_descriptor_1.examples), "test-key", "llama-3.3-70b-versatile");
|
|
45
|
+
await expect(parser.parse("order the same burger that I ordered at mcDonald last time")).resolves.toBe('Order.current().addFoods(foods: [Order.all().matching(field: .restaurant, value: Restaurant.all().matching(field: .name, value: "mcDonald")[0]).sort(field: .dateTime, ascending: false)[0].foods.matching(field: .name, value: "burger")[0]])');
|
|
46
|
+
});
|
|
47
|
+
test("Parser complex voice", async () => {
|
|
48
|
+
global.fetch = jest.fn().mockResolvedValue(new Response(JSON.stringify({
|
|
49
|
+
choices: [
|
|
50
|
+
{
|
|
51
|
+
message: {
|
|
52
|
+
content: JSON.stringify({
|
|
53
|
+
command: 'Order.current().addFoods(foods: Order.all().matching(field: .restaurant, value: Restaurant.all().matching(field: .name, value: "mcDonald")[0]).sort(field: .dateTime, ascending: false)[0].foods)',
|
|
54
|
+
}),
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
],
|
|
58
|
+
usage: {},
|
|
59
|
+
}), { status: 200, headers: { "content-type": "application/json" } }));
|
|
60
|
+
const parser = new nl_parser_groq_1.NlParserGroq(new prompt_gen_1.BasicPromptGen(example_descriptor_1.classDescriptions, example_descriptor_1.examples, '// We utilize voice recognition technology, which may occasionally result in errors. Please consider the possibility of words with similar sounds being misinterpreted. For instance, the word "order" might be mistakenly recognized as "elder".'), "test-key", "llama-3.3-70b-versatile");
|
|
61
|
+
await expect(parser.parseGpt4("elder the same foods that I ordered at mcDonald last time")).resolves.toBe('Order.current().addFoods(foods: Order.all().matching(field: .restaurant, value: Restaurant.all().matching(field: .name, value: "mcDonald")[0]).sort(field: .dateTime, ascending: false)[0].foods)');
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
//# sourceMappingURL=nl-parser.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"nl-parser.test.js","sourceRoot":"","sources":["../../../lib/nl/__test__/nl-parser.test.ts"],"names":[],"mappings":";;AAAA,sDAAiD;AACjD,8CAA+C;AAC/C,0EAAgF;AAChF,iDAA6C;AAE7C,IAAA,sBAAS,GAAE,CAAC;AACZ,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;AAEvB,QAAQ,CAAC,iCAAiC,EAAE,GAAG,EAAE;IAC/C,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,CAAC;IAEnC,SAAS,CAAC,GAAG,EAAE;QACb,MAAM,CAAC,KAAK,GAAG,aAAa,CAAC;QAC7B,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,eAAe,EAAE,KAAK,IAAI,EAAE;QAC/B,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,iBAAiB,CACxC,IAAI,QAAQ,CACV,IAAI,CAAC,SAAS,CAAC;YACb,OAAO,EAAE;gBACP;oBACE,OAAO,EAAE;wBACP,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;4BACtB,OAAO,EACL,6GAA6G;yBAChH,CAAC;qBACH;iBACF;aACF;YACD,KAAK,EAAE,EAAE;SACV,CAAC,EACF,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,EAAE,CACjE,CACK,CAAC;QAET,MAAM,MAAM,GAAG,IAAI,6BAAY,CAC7B,IAAI,2BAAc,CAAC,sCAAiB,EAAE,6BAAQ,CAAC,EAC/C,UAAU,EACV,yBAAyB,CAC1B,CAAC;QAEF,MAAM,MAAM,CACV,MAAM,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAC5D,CAAC,QAAQ,CAAC,IAAI,CACb,6GAA6G,CAC9G,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,gBAAgB,EAAE,KAAK,IAAI,EAAE;QAChC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,iBAAiB,CACxC,IAAI,QAAQ,CACV,IAAI,CAAC,SAAS,CAAC;YACb,OAAO,EAAE;gBACP;oBACE,OAAO,EAAE;wBACP,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;4BACtB,OAAO,EACL,gPAAgP;yBACnP,CAAC;qBACH;iBACF;aACF;YACD,KAAK,EAAE,EAAE;SACV,CAAC,EACF,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,EAAE,CACjE,CACK,CAAC;QAET,MAAM,MAAM,GAAG,IAAI,6BAAY,CAC7B,IAAI,2BAAc,CAAC,sCAAiB,EAAE,6BAAQ,CAAC,EAC/C,UAAU,EACV,yBAAyB,CAC1B,CAAC;QAEF,MAAM,MAAM,CACV,MAAM,CAAC,KAAK,CAAC,4DAA4D,CAAC,CAC3E,CAAC,QAAQ,CAAC,IAAI,CACb,gPAAgP,CACjP,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,IAAI,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;QACtC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,iBAAiB,CACxC,IAAI,QAAQ,CACV,IAAI,CAAC,SAAS,CAAC;YACb,OAAO,EAAE;gBACP;oBACE,OAAO,EAAE;wBACP,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC;4BACtB,OAAO,EACL,mMAAmM;yBACtM,CAAC;qBACH;iBACF;aACF;YACD,KAAK,EAAE,EAAE;SACV,CAAC,EACF,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,EAAE,CACjE,CACK,CAAC;QAET,MAAM,MAAM,GAAG,IAAI,6BAAY,CAC7B,IAAI,2BAAc,CAChB,sCAAiB,EACjB,6BAAQ,EACR,mPAAmP,CACpP,EACD,UAAU,EACV,yBAAyB,CAC1B,CAAC;QAEF,MAAM,MAAM,CACV,MAAM,CAAC,SAAS,CAAC,2DAA2D,CAAC,CAC9E,CAAC,QAAQ,CAAC,IAAI,CACb,mMAAmM,CACpM,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const nl_parser_groq_1 = require("../nl-parser-groq");
|
|
4
|
+
const prompt_gen_1 = require("../prompt-gen");
|
|
5
|
+
const example_descriptor_1 = require("../../__test__/example_descriptor");
|
|
6
|
+
const decorators_1 = require("../../decorators");
|
|
7
|
+
(0, decorators_1.initGenie)();
|
|
8
|
+
jest.setTimeout(180000);
|
|
9
|
+
const testCases = [
|
|
10
|
+
{
|
|
11
|
+
name: 'Basic query',
|
|
12
|
+
input: 'get me the cheapest restaurant in palo alto',
|
|
13
|
+
expected: 'Restaurant.all().matching(field: .address, value: "palo alto").sort(field: .priceGrade, ascending: true)[0]',
|
|
14
|
+
method: 'parse'
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
name: 'Complex query',
|
|
18
|
+
input: 'order the same burger that I ordered at mcDonald last time',
|
|
19
|
+
expected: 'Order.current().addFoods(foods: [Order.all().matching(field: .restaurant, value: Restaurant.all().matching(field: .name, value: "mcDonald")[0]).sort(field: .dateTime, ascending: false)[0].foods.matching(field: .name, value: "burger")[0]])',
|
|
20
|
+
method: 'parse'
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
name: 'Voice recognition query',
|
|
24
|
+
input: 'elder the same foods that I ordered at mcDonald last time',
|
|
25
|
+
expected: 'Order.current().addFoods(foods: Order.all().matching(field: .restaurant, value: Restaurant.all().matching(field: .name, value: "mcDonald")[0]).sort(field: .dateTime, ascending: false)[0].foods)',
|
|
26
|
+
method: 'parseGpt4',
|
|
27
|
+
extraPrompt: '// We utilize voice recognition technology, which may occasionally result in errors. Please consider the possibility of words with similar sounds being misinterpreted. For instance, the word "order" might be mistakenly recognized as "elder".'
|
|
28
|
+
}
|
|
29
|
+
];
|
|
30
|
+
const parameterSets = [
|
|
31
|
+
{ name: 'Baseline (temp=0.2)', temperature: 0.2, top_p: 1, frequency_penalty: 0, presence_penalty: 0 },
|
|
32
|
+
{ name: 'Very deterministic', temperature: 0, top_p: 1, frequency_penalty: 0, presence_penalty: 0 },
|
|
33
|
+
{ name: 'Slightly random', temperature: 0.1, top_p: 1, frequency_penalty: 0, presence_penalty: 0 },
|
|
34
|
+
{ name: 'More creative', temperature: 0.3, top_p: 1, frequency_penalty: 0, presence_penalty: 0 },
|
|
35
|
+
{ name: 'Moderate creative', temperature: 0.5, top_p: 1, frequency_penalty: 0, presence_penalty: 0 },
|
|
36
|
+
{ name: 'Narrow top_p', temperature: 0.2, top_p: 0.8, frequency_penalty: 0, presence_penalty: 0 },
|
|
37
|
+
{ name: 'Very narrow top_p', temperature: 0.2, top_p: 0.5, frequency_penalty: 0, presence_penalty: 0 },
|
|
38
|
+
{ name: 'Tighter sampling', temperature: 0.1, top_p: 0.9, frequency_penalty: 0, presence_penalty: 0 },
|
|
39
|
+
{ name: 'Low freq penalty', temperature: 0.2, top_p: 1, frequency_penalty: 0.2, presence_penalty: 0 },
|
|
40
|
+
{ name: 'Medium freq penalty', temperature: 0.2, top_p: 1, frequency_penalty: 0.5, presence_penalty: 0 },
|
|
41
|
+
{ name: 'High freq penalty', temperature: 0.2, top_p: 1, frequency_penalty: 1.0, presence_penalty: 0 },
|
|
42
|
+
{ name: 'Low pres penalty', temperature: 0.2, top_p: 1, frequency_penalty: 0, presence_penalty: 0.2 },
|
|
43
|
+
{ name: 'Medium pres penalty', temperature: 0.2, top_p: 1, frequency_penalty: 0, presence_penalty: 0.5 },
|
|
44
|
+
{ name: 'Balanced penalties', temperature: 0.2, top_p: 1, frequency_penalty: 0.3, presence_penalty: 0.3 },
|
|
45
|
+
{ name: 'Strong penalties', temperature: 0.1, top_p: 0.9, frequency_penalty: 0.5, presence_penalty: 0.5 },
|
|
46
|
+
{ name: 'Code-optimized 1', temperature: 0, top_p: 0.95, frequency_penalty: 0, presence_penalty: 0 },
|
|
47
|
+
{ name: 'Code-optimized 2', temperature: 0.1, top_p: 0.95, frequency_penalty: 0.1, presence_penalty: 0 },
|
|
48
|
+
{ name: 'Code-optimized 3', temperature: 0.15, top_p: 0.9, frequency_penalty: 0.2, presence_penalty: 0.1 },
|
|
49
|
+
];
|
|
50
|
+
describe('Parameter Tuning Experiments', () => {
|
|
51
|
+
describe.skip('Comprehensive Parameter Testing', () => {
|
|
52
|
+
parameterSets.forEach((params) => {
|
|
53
|
+
describe(`Config: ${params.name}`, () => {
|
|
54
|
+
testCases.forEach((testCase) => {
|
|
55
|
+
test(`${testCase.name}`, async () => {
|
|
56
|
+
const parser = new nl_parser_groq_1.NlParserGroq(new prompt_gen_1.BasicPromptGen(example_descriptor_1.classDescriptions, example_descriptor_1.examples, testCase.extraPrompt), process.env.GROQ_API_KEY || 'test-key', 'llama-3.3-70b-versatile', params);
|
|
57
|
+
let parsed;
|
|
58
|
+
if (testCase.method === 'parseGpt4') {
|
|
59
|
+
parsed = await parser.parseGpt4(testCase.input);
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
parsed = await parser.parse(testCase.input);
|
|
63
|
+
}
|
|
64
|
+
const matches = parsed === testCase.expected;
|
|
65
|
+
console.log(`\n📊 ${params.name} - ${testCase.name}`);
|
|
66
|
+
console.log(` Temperature: ${params.temperature}, Top-p: ${params.top_p}, Freq: ${params.frequency_penalty}, Pres: ${params.presence_penalty}`);
|
|
67
|
+
console.log(` ✓ Match: ${matches ? '✅ EXACT' : '❌ DIFF'}`);
|
|
68
|
+
if (!matches) {
|
|
69
|
+
console.log(` Expected: ${testCase.expected.substring(0, 80)}...`);
|
|
70
|
+
console.log(` Got: ${parsed?.substring(0, 80)}...`);
|
|
71
|
+
}
|
|
72
|
+
expect(parsed).toBeDefined();
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
});
|
|
78
|
+
describe('Quick Parameter Comparison', () => {
|
|
79
|
+
const focusedParams = [
|
|
80
|
+
{ name: 'Baseline', temperature: 0.2, top_p: 1, frequency_penalty: 0, presence_penalty: 0 },
|
|
81
|
+
{ name: 'Deterministic', temperature: 0, top_p: 1, frequency_penalty: 0, presence_penalty: 0 },
|
|
82
|
+
{ name: 'Code-optimized', temperature: 0.1, top_p: 0.95, frequency_penalty: 0.1, presence_penalty: 0 },
|
|
83
|
+
];
|
|
84
|
+
focusedParams.forEach((params) => {
|
|
85
|
+
test(`Quick test: ${params.name}`, async () => {
|
|
86
|
+
const parser = new nl_parser_groq_1.NlParserGroq(new prompt_gen_1.BasicPromptGen(example_descriptor_1.classDescriptions, example_descriptor_1.examples), process.env.GROQ_API_KEY || 'test-key', 'llama-3.3-70b-versatile', params);
|
|
87
|
+
const parsed = await parser.parse('get me the cheapest restaurant in palo alto');
|
|
88
|
+
console.log(`\n🎯 ${params.name}:`);
|
|
89
|
+
console.log(` Result: ${parsed?.substring(0, 100)}...`);
|
|
90
|
+
expect(parsed).toBeDefined();
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
//# sourceMappingURL=parameter-tuning.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parameter-tuning.test.js","sourceRoot":"","sources":["../../../lib/nl/__test__/parameter-tuning.test.ts"],"names":[],"mappings":";;AAAA,sDAAiD;AACjD,8CAA+C;AAC/C,0EAAgF;AAChF,iDAA6C;AAE7C,IAAA,sBAAS,GAAE,CAAC;AACZ,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AAGxB,MAAM,SAAS,GAAG;IAChB;QACE,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,6CAA6C;QACpD,QAAQ,EAAE,6GAA6G;QACvH,MAAM,EAAE,OAAO;KAChB;IACD;QACE,IAAI,EAAE,eAAe;QACrB,KAAK,EAAE,4DAA4D;QACnE,QAAQ,EAAE,gPAAgP;QAC1P,MAAM,EAAE,OAAO;KAChB;IACD;QACE,IAAI,EAAE,yBAAyB;QAC/B,KAAK,EAAE,2DAA2D;QAClE,QAAQ,EAAE,mMAAmM;QAC7M,MAAM,EAAE,WAAW;QACnB,WAAW,EAAE,mPAAmP;KACjQ;CACF,CAAC;AAGF,MAAM,aAAa,GAAG;IAEpB,EAAE,IAAI,EAAE,qBAAqB,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;IAGtG,EAAE,IAAI,EAAE,oBAAoB,EAAE,WAAW,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;IACnG,EAAE,IAAI,EAAE,iBAAiB,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;IAClG,EAAE,IAAI,EAAE,eAAe,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;IAChG,EAAE,IAAI,EAAE,mBAAmB,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;IAGpG,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;IACjG,EAAE,IAAI,EAAE,mBAAmB,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;IACtG,EAAE,IAAI,EAAE,kBAAkB,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;IAGrG,EAAE,IAAI,EAAE,kBAAkB,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,GAAG,EAAE,gBAAgB,EAAE,CAAC,EAAE;IACrG,EAAE,IAAI,EAAE,qBAAqB,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,GAAG,EAAE,gBAAgB,EAAE,CAAC,EAAE;IACxG,EAAE,IAAI,EAAE,mBAAmB,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,GAAG,EAAE,gBAAgB,EAAE,CAAC,EAAE;IAGtG,EAAE,IAAI,EAAE,kBAAkB,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,GAAG,EAAE;IACrG,EAAE,IAAI,EAAE,qBAAqB,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,GAAG,EAAE;IAGxG,EAAE,IAAI,EAAE,oBAAoB,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,EAAE;IACzG,EAAE,IAAI,EAAE,kBAAkB,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,iBAAiB,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,EAAE;IAGzG,EAAE,IAAI,EAAE,kBAAkB,EAAE,WAAW,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;IACpG,EAAE,IAAI,EAAE,kBAAkB,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,gBAAgB,EAAE,CAAC,EAAE;IACxG,EAAE,IAAI,EAAE,kBAAkB,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,iBAAiB,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,EAAE;CAC3G,CAAC;AAEF,QAAQ,CAAC,8BAA8B,EAAE,GAAG,EAAE;IAE5C,QAAQ,CAAC,IAAI,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACpD,aAAa,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAC/B,QAAQ,CAAC,WAAW,MAAM,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE;gBACtC,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;oBAC7B,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,EAAE,KAAK,IAAI,EAAE;wBAClC,MAAM,MAAM,GAAG,IAAI,6BAAY,CAC7B,IAAI,2BAAc,CAChB,sCAAiB,EACjB,6BAAQ,EACR,QAAQ,CAAC,WAAW,CACrB,EACD,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,UAAU,EACtC,yBAAyB,EACzB,MAAM,CACP,CAAC;wBAEF,IAAI,MAAqB,CAAC;wBAC1B,IAAI,QAAQ,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;4BACpC,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;wBAClD,CAAC;6BAAM,CAAC;4BACN,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;wBAC9C,CAAC;wBAED,MAAM,OAAO,GAAG,MAAM,KAAK,QAAQ,CAAC,QAAQ,CAAC;wBAE7C,OAAO,CAAC,GAAG,CAAC,QAAQ,MAAM,CAAC,IAAI,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;wBACtD,OAAO,CAAC,GAAG,CAAC,mBAAmB,MAAM,CAAC,WAAW,YAAY,MAAM,CAAC,KAAK,WAAW,MAAM,CAAC,iBAAiB,WAAW,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;wBAClJ,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;wBAE7D,IAAI,CAAC,OAAO,EAAE,CAAC;4BACb,OAAO,CAAC,GAAG,CAAC,gBAAgB,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;4BACrE,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,EAAE,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;wBAC7D,CAAC;wBAGD,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;oBAC/B,CAAC,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAGH,QAAQ,CAAC,4BAA4B,EAAE,GAAG,EAAE;QAC1C,MAAM,aAAa,GAAG;YACpB,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;YAC3F,EAAE,IAAI,EAAE,eAAe,EAAE,WAAW,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE;YAC9F,EAAE,IAAI,EAAE,gBAAgB,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,gBAAgB,EAAE,CAAC,EAAE;SACvG,CAAC;QAEF,aAAa,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAC/B,IAAI,CAAC,eAAe,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,IAAI,EAAE;gBAC5C,MAAM,MAAM,GAAG,IAAI,6BAAY,CAC7B,IAAI,2BAAc,CAAC,sCAAiB,EAAE,6BAAQ,CAAC,EAC/C,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,UAAU,EACtC,yBAAyB,EACzB,MAAM,CACP,CAAC;gBAEF,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;gBAEjF,OAAO,CAAC,GAAG,CAAC,QAAQ,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,EAAE,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;gBAE1D,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/B,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const nl_parser_groq_1 = require("../nl-parser-groq");
|
|
4
|
+
const prompt_gen_1 = require("../prompt-gen");
|
|
5
|
+
const example_descriptor_1 = require("../../__test__/example_descriptor");
|
|
6
|
+
const decorators_1 = require("../../decorators");
|
|
7
|
+
(0, decorators_1.initGenie)();
|
|
8
|
+
jest.setTimeout(300000);
|
|
9
|
+
const testCases = [
|
|
10
|
+
{
|
|
11
|
+
name: 'Basic query',
|
|
12
|
+
input: 'get me the cheapest restaurant in palo alto',
|
|
13
|
+
expected: 'Restaurant.all().matching(field: .address, value: "palo alto").sort(field: .priceGrade, ascending: true)[0]',
|
|
14
|
+
method: 'parse'
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
name: 'Complex query',
|
|
18
|
+
input: 'order the same burger that I ordered at mcDonald last time',
|
|
19
|
+
expected: 'Order.current().addFoods(foods: [Order.all().matching(field: .restaurant, value: Restaurant.all().matching(field: .name, value: "mcDonald")[0]).sort(field: .dateTime, ascending: false)[0].foods.matching(field: .name, value: "burger")[0]])',
|
|
20
|
+
method: 'parse'
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
name: 'Voice recognition query',
|
|
24
|
+
input: 'elder the same foods that I ordered at mcDonald last time',
|
|
25
|
+
expected: 'Order.current().addFoods(foods: Order.all().matching(field: .restaurant, value: Restaurant.all().matching(field: .name, value: "mcDonald")[0]).sort(field: .dateTime, ascending: false)[0].foods)',
|
|
26
|
+
method: 'parseGpt4',
|
|
27
|
+
extraPrompt: '// We utilize voice recognition technology, which may occasionally result in errors. Please consider the possibility of words with similar sounds being misinterpreted. For instance, the word "order" might be mistakenly recognized as "elder".'
|
|
28
|
+
}
|
|
29
|
+
];
|
|
30
|
+
const experimentConfigs = [
|
|
31
|
+
{ name: 'Current (temp=0.2)', params: { temperature: 0.2, top_p: 1, frequency_penalty: 0, presence_penalty: 0 } },
|
|
32
|
+
{ name: 'Deterministic (temp=0)', params: { temperature: 0, top_p: 1, frequency_penalty: 0, presence_penalty: 0 } },
|
|
33
|
+
{ name: 'Low temp (0.1)', params: { temperature: 0.1, top_p: 1, frequency_penalty: 0, presence_penalty: 0 } },
|
|
34
|
+
{ name: 'Medium temp (0.3)', params: { temperature: 0.3, top_p: 1, frequency_penalty: 0, presence_penalty: 0 } },
|
|
35
|
+
{ name: 'Narrow sampling (top_p=0.9)', params: { temperature: 0.2, top_p: 0.9, frequency_penalty: 0, presence_penalty: 0 } },
|
|
36
|
+
{ name: 'Very narrow (top_p=0.8)', params: { temperature: 0.2, top_p: 0.8, frequency_penalty: 0, presence_penalty: 0 } },
|
|
37
|
+
{ name: 'Code-opt 1', params: { temperature: 0, top_p: 0.95, frequency_penalty: 0, presence_penalty: 0 } },
|
|
38
|
+
{ name: 'Code-opt 2', params: { temperature: 0.1, top_p: 0.95, frequency_penalty: 0.1, presence_penalty: 0 } },
|
|
39
|
+
{ name: 'Code-opt 3', params: { temperature: 0.15, top_p: 0.9, frequency_penalty: 0.2, presence_penalty: 0.1 } },
|
|
40
|
+
{ name: 'With freq penalty', params: { temperature: 0.2, top_p: 1, frequency_penalty: 0.3, presence_penalty: 0 } },
|
|
41
|
+
{ name: 'Balanced penalties', params: { temperature: 0.2, top_p: 1, frequency_penalty: 0.2, presence_penalty: 0.2 } },
|
|
42
|
+
];
|
|
43
|
+
function calculateSimilarity(str1, str2) {
|
|
44
|
+
if (!str1)
|
|
45
|
+
return 0;
|
|
46
|
+
if (str1 === str2)
|
|
47
|
+
return 1.0;
|
|
48
|
+
const longer = str1.length > str2.length ? str1 : str2;
|
|
49
|
+
const shorter = str1.length > str2.length ? str2 : str1;
|
|
50
|
+
if (longer.length === 0)
|
|
51
|
+
return 1.0;
|
|
52
|
+
const editDistance = levenshteinDistance(str1, str2);
|
|
53
|
+
return (longer.length - editDistance) / longer.length;
|
|
54
|
+
}
|
|
55
|
+
function levenshteinDistance(str1, str2) {
|
|
56
|
+
const matrix = [];
|
|
57
|
+
for (let i = 0; i <= str2.length; i++) {
|
|
58
|
+
matrix[i] = [i];
|
|
59
|
+
}
|
|
60
|
+
for (let j = 0; j <= str1.length; j++) {
|
|
61
|
+
matrix[0][j] = j;
|
|
62
|
+
}
|
|
63
|
+
for (let i = 1; i <= str2.length; i++) {
|
|
64
|
+
for (let j = 1; j <= str1.length; j++) {
|
|
65
|
+
if (str2.charAt(i - 1) === str1.charAt(j - 1)) {
|
|
66
|
+
matrix[i][j] = matrix[i - 1][j - 1];
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
matrix[i][j] = Math.min(matrix[i - 1][j - 1] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j] + 1);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return matrix[str2.length][str1.length];
|
|
74
|
+
}
|
|
75
|
+
describe('🔬 Semantic Parsing Parameter Experiments', () => {
|
|
76
|
+
let allResults = [];
|
|
77
|
+
describe('Run experiments', () => {
|
|
78
|
+
experimentConfigs.forEach((config) => {
|
|
79
|
+
describe(`Config: ${config.name}`, () => {
|
|
80
|
+
testCases.forEach((testCase) => {
|
|
81
|
+
test(testCase.name, async () => {
|
|
82
|
+
const parser = new nl_parser_groq_1.NlParserGroq(new prompt_gen_1.BasicPromptGen(example_descriptor_1.classDescriptions, example_descriptor_1.examples, testCase.extraPrompt), process.env.GROQ_API_KEY || 'test-key', 'llama-3.3-70b-versatile', config.params);
|
|
83
|
+
let output;
|
|
84
|
+
if (testCase.method === 'parseGpt4') {
|
|
85
|
+
output = await parser.parseGpt4(testCase.input);
|
|
86
|
+
}
|
|
87
|
+
else {
|
|
88
|
+
output = await parser.parse(testCase.input);
|
|
89
|
+
}
|
|
90
|
+
const exactMatch = output === testCase.expected;
|
|
91
|
+
const similarity = calculateSimilarity(output, testCase.expected);
|
|
92
|
+
const result = {
|
|
93
|
+
configName: config.name,
|
|
94
|
+
params: config.params,
|
|
95
|
+
testName: testCase.name,
|
|
96
|
+
input: testCase.input,
|
|
97
|
+
output: output,
|
|
98
|
+
expected: testCase.expected,
|
|
99
|
+
exactMatch,
|
|
100
|
+
similarityScore: similarity
|
|
101
|
+
};
|
|
102
|
+
allResults.push(result);
|
|
103
|
+
console.log(`\n${exactMatch ? '✅' : '❌'} ${config.name} - ${testCase.name}`);
|
|
104
|
+
console.log(` Params: temp=${config.params.temperature}, top_p=${config.params.top_p}, freq=${config.params.frequency_penalty}, pres=${config.params.presence_penalty}`);
|
|
105
|
+
console.log(` Similarity: ${(similarity * 100).toFixed(1)}%`);
|
|
106
|
+
if (!exactMatch && output) {
|
|
107
|
+
console.log(` Expected: ${testCase.expected.substring(0, 60)}...`);
|
|
108
|
+
console.log(` Got: ${output.substring(0, 60)}...`);
|
|
109
|
+
}
|
|
110
|
+
expect(output).toBeDefined();
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
});
|
|
116
|
+
describe('📊 Summary and Recommendations', () => {
|
|
117
|
+
test('Analyze results', () => {
|
|
118
|
+
console.log('\n\n' + '='.repeat(80));
|
|
119
|
+
console.log('📊 EXPERIMENT SUMMARY');
|
|
120
|
+
console.log('='.repeat(80));
|
|
121
|
+
const configSummaries = new Map();
|
|
122
|
+
allResults.forEach(result => {
|
|
123
|
+
if (!configSummaries.has(result.configName)) {
|
|
124
|
+
configSummaries.set(result.configName, { totalTests: 0, exactMatches: 0, avgSimilarity: 0 });
|
|
125
|
+
}
|
|
126
|
+
const summary = configSummaries.get(result.configName);
|
|
127
|
+
summary.totalTests++;
|
|
128
|
+
if (result.exactMatch)
|
|
129
|
+
summary.exactMatches++;
|
|
130
|
+
summary.avgSimilarity += result.similarityScore;
|
|
131
|
+
});
|
|
132
|
+
const sortedConfigs = Array.from(configSummaries.entries())
|
|
133
|
+
.map(([name, data]) => ({
|
|
134
|
+
name,
|
|
135
|
+
...data,
|
|
136
|
+
avgSimilarity: data.avgSimilarity / data.totalTests,
|
|
137
|
+
successRate: (data.exactMatches / data.totalTests) * 100
|
|
138
|
+
}))
|
|
139
|
+
.sort((a, b) => {
|
|
140
|
+
if (b.exactMatches !== a.exactMatches) {
|
|
141
|
+
return b.exactMatches - a.exactMatches;
|
|
142
|
+
}
|
|
143
|
+
return b.avgSimilarity - a.avgSimilarity;
|
|
144
|
+
});
|
|
145
|
+
console.log('\n🏆 Configuration Rankings:');
|
|
146
|
+
console.log('-'.repeat(80));
|
|
147
|
+
sortedConfigs.forEach((config, index) => {
|
|
148
|
+
const params = allResults.find(r => r.configName === config.name)?.params;
|
|
149
|
+
console.log(`\n${index + 1}. ${config.name}`);
|
|
150
|
+
console.log(` Success Rate: ${config.successRate.toFixed(1)}% (${config.exactMatches}/${config.totalTests} exact matches)`);
|
|
151
|
+
console.log(` Avg Similarity: ${(config.avgSimilarity * 100).toFixed(1)}%`);
|
|
152
|
+
if (params) {
|
|
153
|
+
console.log(` Parameters: temp=${params.temperature}, top_p=${params.top_p}, freq=${params.frequency_penalty}, pres=${params.presence_penalty}`);
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
console.log('\n' + '='.repeat(80));
|
|
157
|
+
console.log('🎯 RECOMMENDATION');
|
|
158
|
+
console.log('='.repeat(80));
|
|
159
|
+
const best = sortedConfigs[0];
|
|
160
|
+
const bestParams = allResults.find(r => r.configName === best.name)?.params;
|
|
161
|
+
console.log(`\n✨ Best Configuration: ${best.name}`);
|
|
162
|
+
console.log(` Success Rate: ${best.successRate.toFixed(1)}%`);
|
|
163
|
+
console.log(` Average Similarity: ${(best.avgSimilarity * 100).toFixed(1)}%`);
|
|
164
|
+
if (bestParams) {
|
|
165
|
+
console.log(`\n Recommended Parameters:`);
|
|
166
|
+
console.log(` {`);
|
|
167
|
+
console.log(` temperature: ${bestParams.temperature},`);
|
|
168
|
+
console.log(` top_p: ${bestParams.top_p},`);
|
|
169
|
+
console.log(` frequency_penalty: ${bestParams.frequency_penalty},`);
|
|
170
|
+
console.log(` presence_penalty: ${bestParams.presence_penalty}`);
|
|
171
|
+
console.log(` }`);
|
|
172
|
+
}
|
|
173
|
+
console.log('\n' + '='.repeat(80) + '\n');
|
|
174
|
+
expect(allResults.length).toBeGreaterThan(0);
|
|
175
|
+
});
|
|
176
|
+
});
|
|
177
|
+
});
|
|
178
|
+
//# sourceMappingURL=semantic-parsing-experiment.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"semantic-parsing-experiment.test.js","sourceRoot":"","sources":["../../../lib/nl/__test__/semantic-parsing-experiment.test.ts"],"names":[],"mappings":";;AASA,sDAAiE;AACjE,8CAA+C;AAC/C,0EAAgF;AAChF,iDAA6C;AAE7C,IAAA,sBAAS,GAAE,CAAC;AACZ,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AAsBxB,MAAM,SAAS,GAAe;IAC5B;QACE,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,6CAA6C;QACpD,QAAQ,EAAE,6GAA6G;QACvH,MAAM,EAAE,OAAO;KAChB;IACD;QACE,IAAI,EAAE,eAAe;QACrB,KAAK,EAAE,4DAA4D;QACnE,QAAQ,EAAE,gPAAgP;QAC1P,MAAM,EAAE,OAAO;KAChB;IACD;QACE,IAAI,EAAE,yBAAyB;QAC/B,KAAK,EAAE,2DAA2D;QAClE,QAAQ,EAAE,mMAAmM;QAC7M,MAAM,EAAE,WAAW;QACnB,WAAW,EAAE,mPAAmP;KACjQ;CACF,CAAC;AAGF,MAAM,iBAAiB,GAAoD;IAEzE,EAAE,IAAI,EAAE,oBAAoB,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE;IAGjH,EAAE,IAAI,EAAE,wBAAwB,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE;IACnH,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE;IAC7G,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE;IAGhH,EAAE,IAAI,EAAE,6BAA6B,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE;IAC5H,EAAE,IAAI,EAAE,yBAAyB,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE;IAGxH,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE;IAC1G,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE;IAC9G,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,iBAAiB,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,EAAE,EAAE;IAGhH,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,GAAG,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE;IAClH,EAAE,IAAI,EAAE,oBAAoB,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,iBAAiB,EAAE,GAAG,EAAE,gBAAgB,EAAE,GAAG,EAAE,EAAE;CACtH,CAAC;AAGF,SAAS,mBAAmB,CAAC,IAAmB,EAAE,IAAY;IAC5D,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO,GAAG,CAAC;IAE9B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAExD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAGpC,MAAM,YAAY,GAAG,mBAAmB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACrD,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,YAAY,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AACxD,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY,EAAE,IAAY;IACrD,MAAM,MAAM,GAAe,EAAE,CAAC;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC9C,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACtC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CACrB,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EACxB,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EACpB,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CACrB,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C,CAAC;AAED,QAAQ,CAAC,2CAA2C,EAAE,GAAG,EAAE;IACzD,IAAI,UAAU,GAAuB,EAAE,CAAC;IAExC,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,iBAAiB,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YACnC,QAAQ,CAAC,WAAW,MAAM,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE;gBACtC,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;oBAC7B,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,IAAI,EAAE;wBAC7B,MAAM,MAAM,GAAG,IAAI,6BAAY,CAC7B,IAAI,2BAAc,CAChB,sCAAiB,EACjB,6BAAQ,EACR,QAAQ,CAAC,WAAW,CACrB,EACD,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,UAAU,EACtC,yBAAyB,EACzB,MAAM,CAAC,MAAM,CACd,CAAC;wBAEF,IAAI,MAAqB,CAAC;wBAC1B,IAAI,QAAQ,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;4BACpC,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;wBAClD,CAAC;6BAAM,CAAC;4BACN,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;wBAC9C,CAAC;wBAED,MAAM,UAAU,GAAG,MAAM,KAAK,QAAQ,CAAC,QAAQ,CAAC;wBAChD,MAAM,UAAU,GAAG,mBAAmB,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;wBAElE,MAAM,MAAM,GAAqB;4BAC/B,UAAU,EAAE,MAAM,CAAC,IAAI;4BACvB,MAAM,EAAE,MAAM,CAAC,MAAM;4BACrB,QAAQ,EAAE,QAAQ,CAAC,IAAI;4BACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;4BACrB,MAAM,EAAE,MAAM;4BACd,QAAQ,EAAE,QAAQ,CAAC,QAAQ;4BAC3B,UAAU;4BACV,eAAe,EAAE,UAAU;yBAC5B,CAAC;wBAEF,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBAGxB,OAAO,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,IAAI,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;wBAC7E,OAAO,CAAC,GAAG,CAAC,mBAAmB,MAAM,CAAC,MAAM,CAAC,WAAW,WAAW,MAAM,CAAC,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,MAAM,CAAC,iBAAiB,UAAU,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;wBAC3K,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;wBAEhE,IAAI,CAAC,UAAU,IAAI,MAAM,EAAE,CAAC;4BAC1B,OAAO,CAAC,GAAG,CAAC,gBAAgB,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;4BACrE,OAAO,CAAC,GAAG,CAAC,gBAAgB,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;wBAC5D,CAAC;wBAED,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;oBAC/B,CAAC,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,gCAAgC,EAAE,GAAG,EAAE;QAC9C,IAAI,CAAC,iBAAiB,EAAE,GAAG,EAAE;YAC3B,OAAO,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAG5B,MAAM,eAAe,GAAG,IAAI,GAAG,EAA+E,CAAC;YAE/G,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE;gBAC1B,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC5C,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC,CAAC;gBAC/F,CAAC;gBAED,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAE,CAAC;gBACxD,OAAO,CAAC,UAAU,EAAE,CAAC;gBACrB,IAAI,MAAM,CAAC,UAAU;oBAAE,OAAO,CAAC,YAAY,EAAE,CAAC;gBAC9C,OAAO,CAAC,aAAa,IAAI,MAAM,CAAC,eAAe,CAAC;YAClD,CAAC,CAAC,CAAC;YAGH,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,CAAC;iBACxD,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtB,IAAI;gBACJ,GAAG,IAAI;gBACP,aAAa,EAAE,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,UAAU;gBACnD,WAAW,EAAE,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG;aACzD,CAAC,CAAC;iBACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBAEb,IAAI,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,YAAY,EAAE,CAAC;oBACtC,OAAO,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC;gBACzC,CAAC;gBACD,OAAO,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,CAAC;YAC3C,CAAC,CAAC,CAAC;YAEL,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5B,aAAa,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;gBACtC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;gBAC1E,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,GAAG,CAAC,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC9C,OAAO,CAAC,GAAG,CAAC,oBAAoB,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,UAAU,iBAAiB,CAAC,CAAC;gBAC9H,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAC9E,IAAI,MAAM,EAAE,CAAC;oBACX,OAAO,CAAC,GAAG,CAAC,uBAAuB,MAAM,CAAC,WAAW,WAAW,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,iBAAiB,UAAU,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;gBACrJ,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5B,MAAM,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YAC9B,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;YAE5E,OAAO,CAAC,GAAG,CAAC,2BAA2B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YACpD,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAChE,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAChF,IAAI,UAAU,EAAE,CAAC;gBACf,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;gBAC5C,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACpB,OAAO,CAAC,GAAG,CAAC,qBAAqB,UAAU,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC5D,OAAO,CAAC,GAAG,CAAC,eAAe,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC;gBAChD,OAAO,CAAC,GAAG,CAAC,2BAA2B,UAAU,CAAC,iBAAiB,GAAG,CAAC,CAAC;gBACxE,OAAO,CAAC,GAAG,CAAC,0BAA0B,UAAU,CAAC,gBAAgB,EAAE,CAAC,CAAC;gBACrE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACtB,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;YAE1C,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export interface SelectionScore {
|
|
2
|
+
score: number;
|
|
3
|
+
index: number;
|
|
4
|
+
}
|
|
5
|
+
export interface RankByQueryOptions {
|
|
6
|
+
allowZeroScoreBackfill?: boolean;
|
|
7
|
+
}
|
|
8
|
+
export declare function scoreTextOverlap(query: string, candidate: string): number;
|
|
9
|
+
export declare function rankByQuery<T>(query: string, items: T[], textExtractor: (item: T) => string, fallbackCount: number, options?: RankByQueryOptions): SelectionScore[];
|
|
10
|
+
export declare class BoundedCache<T> {
|
|
11
|
+
private readonly maxSize;
|
|
12
|
+
private readonly cache;
|
|
13
|
+
constructor(maxSize: number);
|
|
14
|
+
get(key: string): T | undefined;
|
|
15
|
+
set(key: string, value: T): void;
|
|
16
|
+
}
|
|
17
|
+
export declare function normalizeCacheKey(text: string): string;
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.normalizeCacheKey = exports.BoundedCache = exports.rankByQuery = exports.scoreTextOverlap = void 0;
|
|
4
|
+
function splitCompoundBoundaries(text) {
|
|
5
|
+
return text
|
|
6
|
+
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
7
|
+
.replace(/[_]+/g, " ");
|
|
8
|
+
}
|
|
9
|
+
function normalize(text) {
|
|
10
|
+
return splitCompoundBoundaries(text)
|
|
11
|
+
.toLowerCase()
|
|
12
|
+
.replace(/[^a-z0-9\s]/g, " ")
|
|
13
|
+
.replace(/\s+/g, " ")
|
|
14
|
+
.trim();
|
|
15
|
+
}
|
|
16
|
+
function tokenize(text) {
|
|
17
|
+
const normalized = normalize(text);
|
|
18
|
+
if (!normalized) {
|
|
19
|
+
return [];
|
|
20
|
+
}
|
|
21
|
+
return normalized.split(" ").filter((token) => token.length > 1);
|
|
22
|
+
}
|
|
23
|
+
function tokenSet(text) {
|
|
24
|
+
return new Set(tokenize(text));
|
|
25
|
+
}
|
|
26
|
+
function scoreTextOverlap(query, candidate) {
|
|
27
|
+
const queryTokens = tokenSet(query);
|
|
28
|
+
if (queryTokens.size === 0) {
|
|
29
|
+
return 0;
|
|
30
|
+
}
|
|
31
|
+
const candidateTokens = tokenSet(candidate);
|
|
32
|
+
if (candidateTokens.size === 0) {
|
|
33
|
+
return 0;
|
|
34
|
+
}
|
|
35
|
+
let overlap = 0;
|
|
36
|
+
queryTokens.forEach((token) => {
|
|
37
|
+
if (candidateTokens.has(token)) {
|
|
38
|
+
overlap += 1;
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
return overlap / Math.sqrt(queryTokens.size * candidateTokens.size);
|
|
42
|
+
}
|
|
43
|
+
exports.scoreTextOverlap = scoreTextOverlap;
|
|
44
|
+
function rankByQuery(query, items, textExtractor, fallbackCount, options) {
|
|
45
|
+
const scores = items.map((item, index) => ({
|
|
46
|
+
index,
|
|
47
|
+
score: scoreTextOverlap(query, textExtractor(item)),
|
|
48
|
+
}));
|
|
49
|
+
scores.sort((a, b) => {
|
|
50
|
+
if (b.score !== a.score) {
|
|
51
|
+
return b.score - a.score;
|
|
52
|
+
}
|
|
53
|
+
return a.index - b.index;
|
|
54
|
+
});
|
|
55
|
+
const nonZero = scores.filter((entry) => entry.score > 0);
|
|
56
|
+
const allowZeroScoreBackfill = options?.allowZeroScoreBackfill ?? true;
|
|
57
|
+
if (!allowZeroScoreBackfill) {
|
|
58
|
+
return nonZero;
|
|
59
|
+
}
|
|
60
|
+
if (nonZero.length >= fallbackCount) {
|
|
61
|
+
return nonZero;
|
|
62
|
+
}
|
|
63
|
+
return scores;
|
|
64
|
+
}
|
|
65
|
+
exports.rankByQuery = rankByQuery;
|
|
66
|
+
class BoundedCache {
|
|
67
|
+
constructor(maxSize) {
|
|
68
|
+
this.maxSize = maxSize;
|
|
69
|
+
this.cache = new Map();
|
|
70
|
+
}
|
|
71
|
+
get(key) {
|
|
72
|
+
const existing = this.cache.get(key);
|
|
73
|
+
if (existing === undefined) {
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
this.cache.delete(key);
|
|
77
|
+
this.cache.set(key, existing);
|
|
78
|
+
return existing;
|
|
79
|
+
}
|
|
80
|
+
set(key, value) {
|
|
81
|
+
if (this.cache.has(key)) {
|
|
82
|
+
this.cache.delete(key);
|
|
83
|
+
}
|
|
84
|
+
this.cache.set(key, value);
|
|
85
|
+
while (this.cache.size > this.maxSize) {
|
|
86
|
+
const firstKey = this.cache.keys().next().value;
|
|
87
|
+
if (firstKey === undefined) {
|
|
88
|
+
break;
|
|
89
|
+
}
|
|
90
|
+
this.cache.delete(firstKey);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
exports.BoundedCache = BoundedCache;
|
|
95
|
+
function normalizeCacheKey(text) {
|
|
96
|
+
return normalize(text);
|
|
97
|
+
}
|
|
98
|
+
exports.normalizeCacheKey = normalizeCacheKey;
|
|
99
|
+
//# sourceMappingURL=context-selector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context-selector.js","sourceRoot":"","sources":["../../lib/nl/context-selector.ts"],"names":[],"mappings":";;;AASA,SAAS,uBAAuB,CAAC,IAAY;IAC3C,OAAO,IAAI;SACR,OAAO,CAAC,oBAAoB,EAAE,OAAO,CAAC;SACtC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,uBAAuB,CAAC,IAAI,CAAC;SACjC,WAAW,EAAE;SACb,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;SAC5B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,SAAgB,gBAAgB,CAAC,KAAa,EAAE,SAAiB;IAC/D,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,WAAW,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,eAAe,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC5C,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,WAAW,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;QAC5B,IAAI,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;AACtE,CAAC;AAnBD,4CAmBC;AAED,SAAgB,WAAW,CACzB,KAAa,EACb,KAAU,EACV,aAAkC,EAClC,aAAqB,EACrB,OAA4B;IAE5B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;QACzC,KAAK;QACL,KAAK,EAAE,gBAAgB,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;KACpD,CAAC,CAAC,CAAC;IAEJ,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACnB,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;YACxB,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;QAC3B,CAAC;QACD,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC1D,MAAM,sBAAsB,GAAG,OAAO,EAAE,sBAAsB,IAAI,IAAI,CAAC;IACvE,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC5B,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,IAAI,aAAa,EAAE,CAAC;QACpC,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AA9BD,kCA8BC;AAED,MAAa,YAAY;IAGvB,YAA6B,OAAe;QAAf,YAAO,GAAP,OAAO,CAAQ;QAF3B,UAAK,GAAG,IAAI,GAAG,EAAa,CAAC;IAEC,CAAC;IAEhD,GAAG,CAAC,GAAW;QACb,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAC9B,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,GAAG,CAAC,GAAW,EAAE,KAAQ;QACvB,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAE3B,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YAChD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,MAAM;YACR,CAAC;YACD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;CACF;AA9BD,oCA8BC;AAED,SAAgB,iBAAiB,CAAC,IAAY;IAC5C,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAFD,8CAEC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { NlParser } from "./nl-parser";
|
|
2
|
+
export { NlParserGroq } from "./nl-parser-groq";
|
|
3
|
+
export type { SamplingParams } from "./nl-parser";
|
|
4
|
+
export { PromptGen, BasicPromptGen, ExampleParse, DescriptorPromptGen, ContextAwareDescriptorPromptGen, PromptInteractionRecord, PromptInteractionRecorder, PromptSelectionConfig, PromptRuntimeContextSetter, RuntimeUiContext, AgentResponsePayload, ResponseSeverity, } from "./prompt-gen";
|
|
5
|
+
export { getLLMMonitor } from "./llm-monitoring";
|
|
6
|
+
export type { LLMMonitorTelemetryBridge } from "./llm-monitoring";
|
package/dist/nl/index.js
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getLLMMonitor = exports.ContextAwareDescriptorPromptGen = exports.DescriptorPromptGen = exports.ExampleParse = exports.BasicPromptGen = exports.NlParserGroq = exports.NlParser = void 0;
|
|
4
|
+
var nl_parser_1 = require("./nl-parser");
|
|
5
|
+
Object.defineProperty(exports, "NlParser", { enumerable: true, get: function () { return nl_parser_1.NlParser; } });
|
|
6
|
+
var nl_parser_groq_1 = require("./nl-parser-groq");
|
|
7
|
+
Object.defineProperty(exports, "NlParserGroq", { enumerable: true, get: function () { return nl_parser_groq_1.NlParserGroq; } });
|
|
8
|
+
var prompt_gen_1 = require("./prompt-gen");
|
|
9
|
+
Object.defineProperty(exports, "BasicPromptGen", { enumerable: true, get: function () { return prompt_gen_1.BasicPromptGen; } });
|
|
10
|
+
Object.defineProperty(exports, "ExampleParse", { enumerable: true, get: function () { return prompt_gen_1.ExampleParse; } });
|
|
11
|
+
Object.defineProperty(exports, "DescriptorPromptGen", { enumerable: true, get: function () { return prompt_gen_1.DescriptorPromptGen; } });
|
|
12
|
+
Object.defineProperty(exports, "ContextAwareDescriptorPromptGen", { enumerable: true, get: function () { return prompt_gen_1.ContextAwareDescriptorPromptGen; } });
|
|
13
|
+
var llm_monitoring_1 = require("./llm-monitoring");
|
|
14
|
+
Object.defineProperty(exports, "getLLMMonitor", { enumerable: true, get: function () { return llm_monitoring_1.getLLMMonitor; } });
|
|
15
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../lib/nl/index.ts"],"names":[],"mappings":";;;AAAA,yCAAuC;AAA9B,qGAAA,QAAQ,OAAA;AACjB,mDAAgD;AAAvC,8GAAA,YAAY,OAAA;AAErB,2CAasB;AAXrB,4GAAA,cAAc,OAAA;AACd,0GAAA,YAAY,OAAA;AACZ,iHAAA,mBAAmB,OAAA;AACnB,6HAAA,+BAA+B,OAAA;AAShC,mDAAiD;AAAxC,+GAAA,aAAa,OAAA"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
export interface LLMCallMetrics {
|
|
2
|
+
model: string;
|
|
3
|
+
provider: "openai" | "groq";
|
|
4
|
+
promptTokens?: number;
|
|
5
|
+
completionTokens?: number;
|
|
6
|
+
totalTokens?: number;
|
|
7
|
+
inputCost?: number;
|
|
8
|
+
outputCost?: number;
|
|
9
|
+
totalCost?: number;
|
|
10
|
+
prompt: string;
|
|
11
|
+
completion: string | null;
|
|
12
|
+
latency: number;
|
|
13
|
+
timestamp: Date;
|
|
14
|
+
error?: string;
|
|
15
|
+
}
|
|
16
|
+
export type LLMMonitorOptions = {
|
|
17
|
+
apiKey?: string;
|
|
18
|
+
project?: string;
|
|
19
|
+
endpoint?: string;
|
|
20
|
+
telemetryBridge?: LLMMonitorTelemetryBridge;
|
|
21
|
+
};
|
|
22
|
+
export type LLMMonitorTelemetryBridge = {
|
|
23
|
+
captureException?: (error: Error, context?: Record<string, unknown>) => void;
|
|
24
|
+
captureMessage?: (message: string, level?: "error" | "warn" | "info" | "debug", context?: Record<string, unknown>) => void;
|
|
25
|
+
};
|
|
26
|
+
export declare class LLMMonitor {
|
|
27
|
+
private static instance;
|
|
28
|
+
private langsmith;
|
|
29
|
+
private enabled;
|
|
30
|
+
private apiKey?;
|
|
31
|
+
private project?;
|
|
32
|
+
private endpoint?;
|
|
33
|
+
private currentThread?;
|
|
34
|
+
private telemetryBridge;
|
|
35
|
+
private authErrorLogged;
|
|
36
|
+
private constructor();
|
|
37
|
+
static getInstance(options?: LLMMonitorOptions): LLMMonitor;
|
|
38
|
+
private configure;
|
|
39
|
+
setThread(threadId?: string | number | null): void;
|
|
40
|
+
private getProjectName;
|
|
41
|
+
private buildPipelineMetadata;
|
|
42
|
+
private buildUsageMetadata;
|
|
43
|
+
logCall(metrics: LLMCallMetrics): Promise<void>;
|
|
44
|
+
traceCall<T>(name: string, metadata: {
|
|
45
|
+
model: string;
|
|
46
|
+
provider: "openai" | "groq";
|
|
47
|
+
}, fn: () => Promise<T>): Promise<T>;
|
|
48
|
+
}
|
|
49
|
+
export declare function getLLMMonitor(options?: LLMMonitorOptions): LLMMonitor;
|