node-llama-cpp 3.2.0 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bindings/AddonTypes.d.ts +7 -3
- package/dist/bindings/Llama.d.ts +1 -1
- package/dist/bindings/Llama.js +17 -2
- package/dist/bindings/Llama.js.map +1 -1
- package/dist/bindings/getLlama.d.ts +4 -1
- package/dist/bindings/getLlama.js +42 -2
- package/dist/bindings/getLlama.js.map +1 -1
- package/dist/bindings/types.d.ts +3 -1
- package/dist/bindings/types.js +2 -0
- package/dist/bindings/types.js.map +1 -1
- package/dist/bindings/utils/compileLLamaCpp.d.ts +1 -1
- package/dist/bindings/utils/compileLLamaCpp.js +41 -2
- package/dist/bindings/utils/compileLLamaCpp.js.map +1 -1
- package/dist/bindings/utils/testBindingBinary.d.ts +2 -1
- package/dist/bindings/utils/testBindingBinary.js +122 -37
- package/dist/bindings/utils/testBindingBinary.js.map +1 -1
- package/dist/chatWrappers/Llama3_1ChatWrapper.js +3 -2
- package/dist/chatWrappers/Llama3_1ChatWrapper.js.map +1 -1
- package/dist/chatWrappers/Llama3_2LightweightChatWrapper.d.ts +35 -0
- package/dist/chatWrappers/Llama3_2LightweightChatWrapper.js +253 -0
- package/dist/chatWrappers/Llama3_2LightweightChatWrapper.js.map +1 -0
- package/dist/chatWrappers/utils/ChatModelFunctionsDocumentationGenerator.d.ts +13 -0
- package/dist/chatWrappers/utils/ChatModelFunctionsDocumentationGenerator.js +30 -0
- package/dist/chatWrappers/utils/ChatModelFunctionsDocumentationGenerator.js.map +1 -1
- package/dist/chatWrappers/utils/getModelLinageNames.d.ts +2 -0
- package/dist/chatWrappers/utils/getModelLinageNames.js +18 -0
- package/dist/chatWrappers/utils/getModelLinageNames.js.map +1 -0
- package/dist/chatWrappers/utils/isLlama3_2LightweightModel.d.ts +2 -0
- package/dist/chatWrappers/utils/isLlama3_2LightweightModel.js +9 -0
- package/dist/chatWrappers/utils/isLlama3_2LightweightModel.js.map +1 -0
- package/dist/chatWrappers/utils/resolveChatWrapper.d.ts +4 -2
- package/dist/chatWrappers/utils/resolveChatWrapper.js +9 -31
- package/dist/chatWrappers/utils/resolveChatWrapper.js.map +1 -1
- package/dist/cli/commands/ChatCommand.js +1 -1
- package/dist/cli/commands/ChatCommand.js.map +1 -1
- package/dist/cli/commands/InitCommand.d.ts +2 -1
- package/dist/cli/commands/InitCommand.js +32 -13
- package/dist/cli/commands/InitCommand.js.map +1 -1
- package/dist/cli/recommendedModels.js +11 -0
- package/dist/cli/recommendedModels.js.map +1 -1
- package/dist/evaluator/LlamaChat/LlamaChat.js +25 -12
- package/dist/evaluator/LlamaChat/LlamaChat.js.map +1 -1
- package/dist/evaluator/LlamaChatSession/LlamaChatSession.js +140 -132
- package/dist/evaluator/LlamaChatSession/LlamaChatSession.js.map +1 -1
- package/dist/evaluator/LlamaChatSession/utils/defineChatSessionFunction.d.ts +1 -1
- package/dist/evaluator/LlamaCompletion.js +9 -10
- package/dist/evaluator/LlamaCompletion.js.map +1 -1
- package/dist/evaluator/LlamaEmbeddingContext.d.ts +2 -0
- package/dist/evaluator/LlamaEmbeddingContext.js +13 -1
- package/dist/evaluator/LlamaEmbeddingContext.js.map +1 -1
- package/dist/evaluator/LlamaGrammar.js +7 -0
- package/dist/evaluator/LlamaGrammar.js.map +1 -1
- package/dist/evaluator/LlamaJsonSchemaGrammar.d.ts +3 -2
- package/dist/evaluator/LlamaJsonSchemaGrammar.js +3 -0
- package/dist/evaluator/LlamaJsonSchemaGrammar.js.map +1 -1
- package/dist/evaluator/LlamaModel/LlamaModel.d.ts +20 -0
- package/dist/evaluator/LlamaModel/LlamaModel.js +60 -0
- package/dist/evaluator/LlamaModel/LlamaModel.js.map +1 -1
- package/dist/gguf/fileReaders/GgufFileReader.d.ts +1 -1
- package/dist/gguf/fileReaders/GgufFsFileReader.d.ts +1 -1
- package/dist/gguf/fileReaders/GgufNetworkFetchFileReader.d.ts +1 -1
- package/dist/gguf/insights/GgufInsights.d.ts +2 -0
- package/dist/gguf/insights/GgufInsights.js +15 -0
- package/dist/gguf/insights/GgufInsights.js.map +1 -1
- package/dist/gguf/types/GgufMetadataTypes.d.ts +19 -2
- package/dist/gguf/types/GgufMetadataTypes.js +5 -0
- package/dist/gguf/types/GgufMetadataTypes.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/types.d.ts +2 -2
- package/dist/utils/gbnfJson/GbnfGrammarGenerator.d.ts +5 -0
- package/dist/utils/gbnfJson/GbnfGrammarGenerator.js +15 -0
- package/dist/utils/gbnfJson/GbnfGrammarGenerator.js.map +1 -1
- package/dist/utils/gbnfJson/GbnfTerminal.d.ts +3 -0
- package/dist/utils/gbnfJson/GbnfTerminal.js +25 -3
- package/dist/utils/gbnfJson/GbnfTerminal.js.map +1 -1
- package/dist/utils/gbnfJson/getGbnfGrammarForGbnfJsonSchema.d.ts +1 -1
- package/dist/utils/gbnfJson/getGbnfGrammarForGbnfJsonSchema.js.map +1 -1
- package/dist/utils/gbnfJson/terminals/GbnfAnyJson.d.ts +9 -0
- package/dist/utils/gbnfJson/terminals/GbnfAnyJson.js +53 -0
- package/dist/utils/gbnfJson/terminals/GbnfAnyJson.js.map +1 -0
- package/dist/utils/gbnfJson/terminals/GbnfArray.d.ts +11 -2
- package/dist/utils/gbnfJson/terminals/GbnfArray.js +64 -15
- package/dist/utils/gbnfJson/terminals/GbnfArray.js.map +1 -1
- package/dist/utils/gbnfJson/terminals/GbnfBoolean.d.ts +3 -2
- package/dist/utils/gbnfJson/terminals/GbnfBoolean.js +13 -7
- package/dist/utils/gbnfJson/terminals/GbnfBoolean.js.map +1 -1
- package/dist/utils/gbnfJson/terminals/GbnfCommaWhitespace.d.ts +11 -0
- package/dist/utils/gbnfJson/terminals/GbnfCommaWhitespace.js +28 -0
- package/dist/utils/gbnfJson/terminals/GbnfCommaWhitespace.js.map +1 -0
- package/dist/utils/gbnfJson/terminals/GbnfFormatString.d.ts +11 -0
- package/dist/utils/gbnfJson/terminals/GbnfFormatString.js +90 -0
- package/dist/utils/gbnfJson/terminals/GbnfFormatString.js.map +1 -0
- package/dist/utils/gbnfJson/terminals/GbnfGrammar.d.ts +4 -1
- package/dist/utils/gbnfJson/terminals/GbnfGrammar.js +8 -1
- package/dist/utils/gbnfJson/terminals/GbnfGrammar.js.map +1 -1
- package/dist/utils/gbnfJson/terminals/GbnfInsideStringChar.d.ts +5 -0
- package/dist/utils/gbnfJson/terminals/GbnfInsideStringChar.js +24 -0
- package/dist/utils/gbnfJson/terminals/GbnfInsideStringChar.js.map +1 -0
- package/dist/utils/gbnfJson/terminals/GbnfNumber.js +4 -3
- package/dist/utils/gbnfJson/terminals/GbnfNumber.js.map +1 -1
- package/dist/utils/gbnfJson/terminals/GbnfNumberValue.d.ts +3 -1
- package/dist/utils/gbnfJson/terminals/GbnfNumberValue.js +8 -2
- package/dist/utils/gbnfJson/terminals/GbnfNumberValue.js.map +1 -1
- package/dist/utils/gbnfJson/terminals/GbnfObjectMap.d.ts +14 -5
- package/dist/utils/gbnfJson/terminals/GbnfObjectMap.js +67 -9
- package/dist/utils/gbnfJson/terminals/GbnfObjectMap.js.map +1 -1
- package/dist/utils/gbnfJson/terminals/GbnfOr.d.ts +2 -1
- package/dist/utils/gbnfJson/terminals/GbnfOr.js +6 -2
- package/dist/utils/gbnfJson/terminals/GbnfOr.js.map +1 -1
- package/dist/utils/gbnfJson/terminals/GbnfRepetition.d.ts +7 -1
- package/dist/utils/gbnfJson/terminals/GbnfRepetition.js +69 -20
- package/dist/utils/gbnfJson/terminals/GbnfRepetition.js.map +1 -1
- package/dist/utils/gbnfJson/terminals/GbnfString.d.ts +8 -1
- package/dist/utils/gbnfJson/terminals/GbnfString.js +31 -21
- package/dist/utils/gbnfJson/terminals/GbnfString.js.map +1 -1
- package/dist/utils/gbnfJson/terminals/GbnfStringValue.d.ts +2 -0
- package/dist/utils/gbnfJson/terminals/GbnfStringValue.js +7 -4
- package/dist/utils/gbnfJson/terminals/GbnfStringValue.js.map +1 -1
- package/dist/utils/gbnfJson/terminals/GbnfWhitespace.d.ts +2 -1
- package/dist/utils/gbnfJson/terminals/GbnfWhitespace.js +24 -5
- package/dist/utils/gbnfJson/terminals/GbnfWhitespace.js.map +1 -1
- package/dist/utils/gbnfJson/terminals/gbnfConsts.d.ts +16 -1
- package/dist/utils/gbnfJson/terminals/gbnfConsts.js +47 -1
- package/dist/utils/gbnfJson/terminals/gbnfConsts.js.map +1 -1
- package/dist/utils/gbnfJson/types.d.ts +164 -17
- package/dist/utils/gbnfJson/types.js +6 -0
- package/dist/utils/gbnfJson/types.js.map +1 -1
- package/dist/utils/gbnfJson/utils/getGbnfJsonTerminalForGbnfJsonSchema.js +61 -9
- package/dist/utils/gbnfJson/utils/getGbnfJsonTerminalForGbnfJsonSchema.js.map +1 -1
- package/dist/utils/gbnfJson/utils/validateObjectAgainstGbnfSchema.js +99 -8
- package/dist/utils/gbnfJson/utils/validateObjectAgainstGbnfSchema.js.map +1 -1
- package/dist/utils/getTypeScriptTypeStringForGbnfJsonSchema.js +125 -16
- package/dist/utils/getTypeScriptTypeStringForGbnfJsonSchema.js.map +1 -1
- package/dist/utils/includesText.d.ts +1 -0
- package/dist/utils/includesText.js +12 -0
- package/dist/utils/includesText.js.map +1 -0
- package/dist/utils/tokenizeInput.d.ts +1 -1
- package/dist/utils/tokenizeInput.js +3 -1
- package/dist/utils/tokenizeInput.js.map +1 -1
- package/dist/utils/tokenizerUtils.d.ts +12 -0
- package/dist/utils/tokenizerUtils.js +28 -0
- package/dist/utils/tokenizerUtils.js.map +1 -0
- package/dist/utils/wrapAbortSignal.d.ts +1 -1
- package/dist/utils/wrapAbortSignal.js +9 -4
- package/dist/utils/wrapAbortSignal.js.map +1 -1
- package/llama/CMakeLists.txt +10 -90
- package/llama/addon/AddonContext.cpp +2 -1
- package/llama/addon/AddonGrammar.cpp +49 -1
- package/llama/addon/AddonGrammar.h +3 -0
- package/llama/addon/AddonModel.cpp +26 -4
- package/llama/addon/AddonModel.h +3 -0
- package/llama/addon/addon.cpp +17 -0
- package/llama/addon/globals/getGpuInfo.cpp +54 -52
- package/llama/binariesGithubRelease.json +1 -1
- package/llama/gitRelease.bundle +0 -0
- package/llama/gpuInfo/vulkan-gpu-info.cpp +0 -8
- package/llama/gpuInfo/vulkan-gpu-info.h +0 -1
- package/llama/grammars/README.md +2 -2
- package/llama/grammars/english.gbnf +6 -0
- package/llama/llama.cpp.info.json +1 -1
- package/package.json +49 -45
- package/templates/README.md +6 -0
- package/templates/packed/electron-typescript-react.json +1 -1
- package/templates/packed/node-typescript.json +1 -1
- package/llama/gpuInfo/cuda-gpu-info.cu +0 -120
- package/llama/gpuInfo/cuda-gpu-info.h +0 -10
- package/llama/gpuInfo/metal-gpu-info.h +0 -8
- package/llama/gpuInfo/metal-gpu-info.mm +0 -37
|
@@ -1,33 +1,43 @@
|
|
|
1
1
|
import { GbnfTerminal } from "../GbnfTerminal.js";
|
|
2
2
|
import { reservedRuleNames } from "./gbnfConsts.js";
|
|
3
|
+
import { GbnfRepetition } from "./GbnfRepetition.js";
|
|
4
|
+
import { GbnfInsideStringChar } from "./GbnfInsideStringChar.js";
|
|
3
5
|
export class GbnfString extends GbnfTerminal {
|
|
4
|
-
|
|
6
|
+
minLength;
|
|
7
|
+
maxLength;
|
|
8
|
+
constructor({ minLength = 0, maxLength } = {}) {
|
|
9
|
+
super();
|
|
10
|
+
this.minLength = Math.floor(minLength ?? 0);
|
|
11
|
+
this.maxLength = maxLength == null ? undefined : Math.floor(maxLength);
|
|
12
|
+
if (this.minLength < 0)
|
|
13
|
+
this.minLength = 0;
|
|
14
|
+
if (this.maxLength != null && this.maxLength < this.minLength)
|
|
15
|
+
this.maxLength = this.minLength;
|
|
16
|
+
}
|
|
17
|
+
getGrammar(grammarGenerator) {
|
|
18
|
+
if (this.minLength == 0 && this.maxLength == null)
|
|
19
|
+
return [
|
|
20
|
+
'"\\""',
|
|
21
|
+
new GbnfInsideStringChar().resolve(grammarGenerator) + "*",
|
|
22
|
+
'"\\""'
|
|
23
|
+
].join(" ");
|
|
24
|
+
else if (this.minLength == 0 && this.maxLength == 0)
|
|
25
|
+
return '"\\"\\""';
|
|
5
26
|
return [
|
|
6
27
|
'"\\""',
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
"\\x00-\\x1F"
|
|
13
|
-
]),
|
|
14
|
-
// escape sequences
|
|
15
|
-
'"\\\\" ' + or([
|
|
16
|
-
'["\\\\/bfnrt]',
|
|
17
|
-
'"u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]'
|
|
18
|
-
])
|
|
19
|
-
]) + "*",
|
|
28
|
+
new GbnfRepetition({
|
|
29
|
+
value: new GbnfInsideStringChar(),
|
|
30
|
+
minRepetitions: this.minLength,
|
|
31
|
+
maxRepetitions: this.maxLength
|
|
32
|
+
}).getGrammar(grammarGenerator),
|
|
20
33
|
'"\\""'
|
|
21
34
|
].join(" ");
|
|
22
35
|
}
|
|
23
36
|
getRuleName() {
|
|
24
|
-
return reservedRuleNames.string
|
|
37
|
+
return reservedRuleNames.string({
|
|
38
|
+
minLength: this.minLength,
|
|
39
|
+
maxLength: this.maxLength
|
|
40
|
+
});
|
|
25
41
|
}
|
|
26
42
|
}
|
|
27
|
-
function negatedCharacterSet(characterDefinitions) {
|
|
28
|
-
return "[^" + characterDefinitions.join("") + "]";
|
|
29
|
-
}
|
|
30
|
-
function or(definitions) {
|
|
31
|
-
return "(" + definitions.join(" | ") + ")";
|
|
32
|
-
}
|
|
33
43
|
//# sourceMappingURL=GbnfString.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GbnfString.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfString.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"GbnfString.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfString.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAC,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAC,cAAc,EAAC,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAC,oBAAoB,EAAC,MAAM,2BAA2B,CAAC;AAG/D,MAAM,OAAO,UAAW,SAAQ,YAAY;IACxB,SAAS,CAAS;IAClB,SAAS,CAAU;IAEnC,YAAmB,EACf,SAAS,GAAG,CAAC,EACb,SAAS,KAIT,EAAE;QACF,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC;QAC5C,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEvE,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC;YAClB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QAEvB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS;YACzD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IACxC,CAAC;IAEM,UAAU,CAAC,gBAAsC;QACpD,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI;YAC7C,OAAO;gBACH,OAAO;gBACP,IAAI,oBAAoB,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,GAAG;gBAC1D,OAAO;aACV,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;aACX,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC;YAC/C,OAAO,UAAU,CAAC;QAEtB,OAAO;YACH,OAAO;YACP,IAAI,cAAc,CAAC;gBACf,KAAK,EAAE,IAAI,oBAAoB,EAAE;gBACjC,cAAc,EAAE,IAAI,CAAC,SAAS;gBAC9B,cAAc,EAAE,IAAI,CAAC,SAAS;aACjC,CAAC,CAAC,UAAU,CAAC,gBAAgB,CAAC;YAC/B,OAAO;SACV,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,CAAC;IAEkB,WAAW;QAC1B,OAAO,iBAAiB,CAAC,MAAM,CAAC;YAC5B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC5B,CAAC,CAAC;IACP,CAAC;CACJ"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { GbnfTerminal } from "../GbnfTerminal.js";
|
|
2
|
+
import { GbnfGrammarGenerator } from "../GbnfGrammarGenerator.js";
|
|
2
3
|
export declare class GbnfStringValue extends GbnfTerminal {
|
|
3
4
|
readonly value: string;
|
|
4
5
|
constructor(value: string);
|
|
5
6
|
getGrammar(): string;
|
|
7
|
+
protected generateRuleName(grammarGenerator: GbnfGrammarGenerator): string;
|
|
6
8
|
}
|
|
@@ -10,14 +10,17 @@ export class GbnfStringValue extends GbnfTerminal {
|
|
|
10
10
|
'"',
|
|
11
11
|
'\\"',
|
|
12
12
|
this.value
|
|
13
|
-
.replaceAll("\\", "
|
|
14
|
-
.replaceAll("\t", "
|
|
15
|
-
.replaceAll("\r", "
|
|
16
|
-
.replaceAll("\n", "
|
|
13
|
+
.replaceAll("\\", "\\\\\\\\")
|
|
14
|
+
.replaceAll("\t", "\\\\t")
|
|
15
|
+
.replaceAll("\r", "\\\\r")
|
|
16
|
+
.replaceAll("\n", "\\\\n")
|
|
17
17
|
.replaceAll('"', "\\\\" + '\\"'),
|
|
18
18
|
'\\"',
|
|
19
19
|
'"'
|
|
20
20
|
].join("");
|
|
21
21
|
}
|
|
22
|
+
generateRuleName(grammarGenerator) {
|
|
23
|
+
return grammarGenerator.generateRuleNameForLiteralValue(this.value);
|
|
24
|
+
}
|
|
22
25
|
}
|
|
23
26
|
//# sourceMappingURL=GbnfStringValue.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GbnfStringValue.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfStringValue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"GbnfStringValue.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfStringValue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAIhD,MAAM,OAAO,eAAgB,SAAQ,YAAY;IAC7B,KAAK,CAAS;IAE9B,YAAmB,KAAa;QAC5B,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAEe,UAAU;QACtB,OAAO;YACH,GAAG;YACH,KAAK;YACL,IAAI,CAAC,KAAK;iBACL,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC;iBAC5B,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC;iBACzB,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC;iBACzB,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC;iBACzB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,KAAK,CAAC;YACpC,KAAK;YACL,GAAG;SACN,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,CAAC;IAEkB,gBAAgB,CAAC,gBAAsC;QACtE,OAAO,gBAAgB,CAAC,+BAA+B,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxE,CAAC;CACJ"}
|
|
@@ -5,8 +5,9 @@ export declare class GbnfWhitespace extends GbnfTerminal {
|
|
|
5
5
|
readonly newLine: "before" | "after" | false;
|
|
6
6
|
constructor(scopeState: GbnfJsonScopeState, { newLine }?: {
|
|
7
7
|
newLine?: "before" | "after" | false;
|
|
8
|
-
space?: boolean;
|
|
9
8
|
});
|
|
10
9
|
getGrammar(): string;
|
|
10
|
+
protected getGrammarFromResolve(): string;
|
|
11
|
+
private _getGrammar;
|
|
11
12
|
protected getRuleName(): string;
|
|
12
13
|
}
|
|
@@ -10,6 +10,12 @@ export class GbnfWhitespace extends GbnfTerminal {
|
|
|
10
10
|
this.newLine = newLine;
|
|
11
11
|
}
|
|
12
12
|
getGrammar() {
|
|
13
|
+
return this._getGrammar();
|
|
14
|
+
}
|
|
15
|
+
getGrammarFromResolve() {
|
|
16
|
+
return this._getGrammar(false);
|
|
17
|
+
}
|
|
18
|
+
_getGrammar(wrap = true) {
|
|
13
19
|
if (this.scopeState.settings.allowNewLines && this.newLine !== false) {
|
|
14
20
|
const values = [
|
|
15
21
|
...(this.newLine === "before"
|
|
@@ -19,8 +25,8 @@ export class GbnfWhitespace extends GbnfTerminal {
|
|
|
19
25
|
? []
|
|
20
26
|
: [
|
|
21
27
|
or([
|
|
22
|
-
|
|
23
|
-
|
|
28
|
+
verbatimTextRepetition(" ", this.scopeState.currentNestingScope * this.scopeState.settings.scopePadSpaces),
|
|
29
|
+
verbatimTextRepetition("\t", this.scopeState.currentNestingScope)
|
|
24
30
|
])
|
|
25
31
|
]),
|
|
26
32
|
...(this.newLine === "after"
|
|
@@ -30,19 +36,32 @@ export class GbnfWhitespace extends GbnfTerminal {
|
|
|
30
36
|
return or([
|
|
31
37
|
values.join(" "),
|
|
32
38
|
"[ ]?"
|
|
33
|
-
]);
|
|
39
|
+
], wrap);
|
|
34
40
|
}
|
|
35
41
|
return "[ ]?";
|
|
36
42
|
}
|
|
37
43
|
getRuleName() {
|
|
38
44
|
return reservedRuleNames.whitespace({
|
|
39
|
-
newLine: this.
|
|
45
|
+
newLine: this.scopeState.settings.allowNewLines
|
|
46
|
+
? this.newLine
|
|
47
|
+
: false,
|
|
40
48
|
scopeSpaces: this.scopeState.settings.scopePadSpaces,
|
|
41
49
|
nestingScope: this.scopeState.currentNestingScope
|
|
42
50
|
});
|
|
43
51
|
}
|
|
44
52
|
}
|
|
45
|
-
function or(definitions) {
|
|
53
|
+
function or(definitions, wrap = true) {
|
|
54
|
+
if (!wrap)
|
|
55
|
+
return definitions.join(" | ");
|
|
46
56
|
return "(" + definitions.join(" | ") + ")";
|
|
47
57
|
}
|
|
58
|
+
function verbatimTextRepetition(text, count) {
|
|
59
|
+
const textRepetitionGrammar = new GbnfVerbatimText(text.repeat(count)).getGrammar();
|
|
60
|
+
if (count <= 1)
|
|
61
|
+
return textRepetitionGrammar;
|
|
62
|
+
const textRepetitionGrammarWithRepetition = new GbnfVerbatimText(text).getGrammar() + "{" + count + "}";
|
|
63
|
+
if (textRepetitionGrammarWithRepetition.length < textRepetitionGrammar.length)
|
|
64
|
+
return textRepetitionGrammarWithRepetition;
|
|
65
|
+
return textRepetitionGrammar;
|
|
66
|
+
}
|
|
48
67
|
//# sourceMappingURL=GbnfWhitespace.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GbnfWhitespace.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfWhitespace.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAC,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAC,gBAAgB,EAAC,MAAM,uBAAuB,CAAC;AAGvD,MAAM,OAAO,cAAe,SAAQ,YAAY;IAC5B,UAAU,CAAqB;IAC/B,OAAO,CAA6B;IAEpD,YAAmB,UAA8B,EAAE,EAC/C,OAAO,GAAG,QAAQ,
|
|
1
|
+
{"version":3,"file":"GbnfWhitespace.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/GbnfWhitespace.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,oBAAoB,CAAC;AAEhD,OAAO,EAAC,iBAAiB,EAAC,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAC,gBAAgB,EAAC,MAAM,uBAAuB,CAAC;AAGvD,MAAM,OAAO,cAAe,SAAQ,YAAY;IAC5B,UAAU,CAAqB;IAC/B,OAAO,CAA6B;IAEpD,YAAmB,UAA8B,EAAE,EAC/C,OAAO,GAAG,QAAQ,KAGlB,EAAE;QACF,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IAC3B,CAAC;IAEM,UAAU;QACb,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;IAC9B,CAAC;IAEkB,qBAAqB;QACpC,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IAEO,WAAW,CAAC,OAAgB,IAAI;QACpC,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,aAAa,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YACnE,MAAM,MAAM,GAAG;gBACX,GAAG,CACC,IAAI,CAAC,OAAO,KAAK,QAAQ;oBACrB,CAAC,CAAC,CAAC,OAAO,CAAC;oBACX,CAAC,CAAC,EAAE,CACX;gBACD,GAAG,CACC,IAAI,CAAC,UAAU,CAAC,mBAAmB,KAAK,CAAC;oBACrC,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAC;wBACE,EAAE,CAAC;4BACC,sBAAsB,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,mBAAmB,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,cAAc,CAAC;4BAC1G,sBAAsB,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,mBAAmB,CAAC;yBACpE,CAAC;qBACL,CACR;gBACD,GAAG,CACC,IAAI,CAAC,OAAO,KAAK,OAAO;oBACpB,CAAC,CAAC,CAAC,OAAO,CAAC;oBACX,CAAC,CAAC,EAAE,CACX;aACJ,CAAC;YAEF,OAAO,EAAE,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC;gBAChB,MAAM;aACT,EAAE,IAAI,CAAC,CAAC;QACb,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAEkB,WAAW;QAC1B,OAAO,iBAAiB,CAAC,UAAU,CAAC;YAChC,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,aAAa;gBAC3C,CAAC,CAAC,IAAI,CAAC,OAAO;gBACd,CAAC,CAAC,KAAK;YACX,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,cAAc;YACpD,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,mBAAmB;SACpD,CAAC,CAAC;IACP,CAAC;CACJ;AAED,SAAS,EAAE,CAAC,WAAqB,EAAE,OAAgB,IAAI;IACnD,IAAI,CAAC,IAAI;QACL,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAEnC,OAAO,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC;AAC/C,CAAC;AAED,SAAS,sBAAsB,CAAC,IAAY,EAAE,KAAa;IACvD,MAAM,qBAAqB,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC;IAEpF,IAAI,KAAK,IAAI,CAAC;QACV,OAAO,qBAAqB,CAAC;IAEjC,MAAM,mCAAmC,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,GAAG,GAAG,GAAG,KAAK,GAAG,GAAG,CAAC;IACxG,IAAI,mCAAmC,CAAC,MAAM,GAAG,qBAAqB,CAAC,MAAM;QACzE,OAAO,mCAAmC,CAAC;IAE/C,OAAO,qBAAqB,CAAC;AACjC,CAAC"}
|
|
@@ -6,10 +6,25 @@ export declare const reservedRuleNames: {
|
|
|
6
6
|
readonly fractional: "fractional-number-rule";
|
|
7
7
|
readonly integer: "integer-number-rule";
|
|
8
8
|
};
|
|
9
|
-
readonly
|
|
9
|
+
readonly stringChar: "string-char-rule";
|
|
10
|
+
readonly string: ({ minLength, maxLength }: {
|
|
11
|
+
minLength: number;
|
|
12
|
+
maxLength?: number;
|
|
13
|
+
}) => string;
|
|
14
|
+
readonly formatString: (format: string) => string;
|
|
10
15
|
readonly whitespace: ({ newLine, nestingScope, scopeSpaces }: {
|
|
11
16
|
newLine?: "before" | "after" | false;
|
|
12
17
|
nestingScope: number;
|
|
13
18
|
scopeSpaces: number;
|
|
14
19
|
}) => string;
|
|
20
|
+
readonly commaWhitespace: ({ newLine, nestingScope, scopeSpaces }: {
|
|
21
|
+
newLine?: "before" | "after" | false;
|
|
22
|
+
nestingScope: number;
|
|
23
|
+
scopeSpaces: number;
|
|
24
|
+
}) => string;
|
|
25
|
+
readonly anyJson: ({ allowNewLines, nestingScope, scopeSpaces }: {
|
|
26
|
+
allowNewLines: boolean;
|
|
27
|
+
nestingScope: number;
|
|
28
|
+
scopeSpaces: number;
|
|
29
|
+
}) => string;
|
|
15
30
|
};
|
|
@@ -6,7 +6,27 @@ export const reservedRuleNames = {
|
|
|
6
6
|
fractional: "fractional-number-rule",
|
|
7
7
|
integer: "integer-number-rule"
|
|
8
8
|
},
|
|
9
|
-
|
|
9
|
+
stringChar: "string-char-rule",
|
|
10
|
+
string({ minLength, maxLength }) {
|
|
11
|
+
if (minLength === 0 && maxLength == null)
|
|
12
|
+
return "string-rule";
|
|
13
|
+
else if (maxLength == null)
|
|
14
|
+
return [
|
|
15
|
+
"string-",
|
|
16
|
+
minLength,
|
|
17
|
+
"-rule"
|
|
18
|
+
].join("");
|
|
19
|
+
return [
|
|
20
|
+
"string-",
|
|
21
|
+
minLength,
|
|
22
|
+
"-",
|
|
23
|
+
maxLength,
|
|
24
|
+
"-rule"
|
|
25
|
+
].join("");
|
|
26
|
+
},
|
|
27
|
+
formatString(format) {
|
|
28
|
+
return "string-format-" + format + "-rule";
|
|
29
|
+
},
|
|
10
30
|
whitespace({ newLine, nestingScope, scopeSpaces }) {
|
|
11
31
|
if (!newLine)
|
|
12
32
|
return "whitespace-no-new-lines-rule";
|
|
@@ -21,6 +41,32 @@ export const reservedRuleNames = {
|
|
|
21
41
|
"-" + scopeSpaces,
|
|
22
42
|
"-rule"
|
|
23
43
|
].join("");
|
|
44
|
+
},
|
|
45
|
+
commaWhitespace({ newLine, nestingScope, scopeSpaces }) {
|
|
46
|
+
if (!newLine)
|
|
47
|
+
return "comma-whitespace-no-new-lines-rule";
|
|
48
|
+
return [
|
|
49
|
+
"comma-whitespace-",
|
|
50
|
+
newLine === "before"
|
|
51
|
+
? "b"
|
|
52
|
+
: newLine === "after"
|
|
53
|
+
? "a"
|
|
54
|
+
: "n",
|
|
55
|
+
"-" + nestingScope,
|
|
56
|
+
"-" + scopeSpaces,
|
|
57
|
+
"-rule"
|
|
58
|
+
].join("");
|
|
59
|
+
},
|
|
60
|
+
anyJson({ allowNewLines, nestingScope, scopeSpaces }) {
|
|
61
|
+
return [
|
|
62
|
+
"any-json-",
|
|
63
|
+
!allowNewLines
|
|
64
|
+
? "s-"
|
|
65
|
+
: "",
|
|
66
|
+
nestingScope,
|
|
67
|
+
"-" + scopeSpaces,
|
|
68
|
+
"-rule"
|
|
69
|
+
].join("");
|
|
24
70
|
}
|
|
25
71
|
};
|
|
26
72
|
//# sourceMappingURL=gbnfConsts.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gbnfConsts.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/gbnfConsts.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC;AACnC,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC7B,IAAI,EAAE,WAAW;IACjB,OAAO,EAAE,cAAc;IACvB,MAAM,EAAE;QACJ,UAAU,EAAE,wBAAwB;QACpC,OAAO,EAAE,qBAAqB;KACjC;IACD,MAAM,EAAE,aAAa;
|
|
1
|
+
{"version":3,"file":"gbnfConsts.js","sourceRoot":"","sources":["../../../../src/utils/gbnfJson/terminals/gbnfConsts.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAC;AACnC,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC7B,IAAI,EAAE,WAAW;IACjB,OAAO,EAAE,cAAc;IACvB,MAAM,EAAE;QACJ,UAAU,EAAE,wBAAwB;QACpC,OAAO,EAAE,qBAAqB;KACjC;IACD,UAAU,EAAE,kBAAkB;IAC9B,MAAM,CAAC,EAAC,SAAS,EAAE,SAAS,EAA0C;QAClE,IAAI,SAAS,KAAK,CAAC,IAAI,SAAS,IAAI,IAAI;YACpC,OAAO,aAAa,CAAC;aACpB,IAAI,SAAS,IAAI,IAAI;YACtB,OAAO;gBACH,SAAS;gBACT,SAAS;gBACT,OAAO;aACV,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAEf,OAAO;YACH,SAAS;YACT,SAAS;YACT,GAAG;YACH,SAAS;YACT,OAAO;SACV,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,CAAC;IACD,YAAY,CAAC,MAAc;QACvB,OAAO,gBAAgB,GAAG,MAAM,GAAG,OAAO,CAAC;IAC/C,CAAC;IACD,UAAU,CAAC,EAAC,OAAO,EAAE,YAAY,EAAE,WAAW,EAE7C;QACG,IAAI,CAAC,OAAO;YACR,OAAO,8BAA8B,CAAC;QAE1C,OAAO;YACH,aAAa;YACb,OAAO,KAAK,QAAQ;gBAChB,CAAC,CAAC,GAAG;gBACL,CAAC,CAAC,OAAO,KAAK,OAAO;oBACjB,CAAC,CAAC,GAAG;oBACL,CAAC,CAAC,GAAG;YACb,GAAG,GAAG,YAAY;YAClB,GAAG,GAAG,WAAW;YACjB,OAAO;SACV,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,CAAC;IACD,eAAe,CAAC,EAAC,OAAO,EAAE,YAAY,EAAE,WAAW,EAElD;QACG,IAAI,CAAC,OAAO;YACR,OAAO,oCAAoC,CAAC;QAEhD,OAAO;YACH,mBAAmB;YACnB,OAAO,KAAK,QAAQ;gBAChB,CAAC,CAAC,GAAG;gBACL,CAAC,CAAC,OAAO,KAAK,OAAO;oBACjB,CAAC,CAAC,GAAG;oBACL,CAAC,CAAC,GAAG;YACb,GAAG,GAAG,YAAY;YAClB,GAAG,GAAG,WAAW;YACjB,OAAO;SACV,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,CAAC;IACD,OAAO,CAAC,EAAC,aAAa,EAAE,YAAY,EAAE,WAAW,EAEhD;QACG,OAAO;YACH,WAAW;YACX,CAAC,aAAa;gBACV,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,EAAE;YACR,YAAY;YACZ,GAAG,GAAG,WAAW;YACjB,OAAO;SACV,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,CAAC;CACK,CAAC"}
|
|
@@ -1,44 +1,191 @@
|
|
|
1
1
|
export type GbnfJsonSchemaImmutableType = "string" | "number" | "integer" | "boolean" | "null";
|
|
2
|
-
export type GbnfJsonSchema = GbnfJsonBasicSchema | GbnfJsonConstSchema | GbnfJsonEnumSchema | GbnfJsonOneOfSchema | GbnfJsonObjectSchema | GbnfJsonArraySchema;
|
|
2
|
+
export type GbnfJsonSchema = GbnfJsonBasicSchema | GbnfJsonConstSchema | GbnfJsonEnumSchema | GbnfJsonOneOfSchema | GbnfJsonStringSchema | GbnfJsonObjectSchema | GbnfJsonArraySchema;
|
|
3
3
|
export type GbnfJsonBasicSchema = {
|
|
4
|
-
type: GbnfJsonSchemaImmutableType | readonly GbnfJsonSchemaImmutableType[];
|
|
4
|
+
readonly type: GbnfJsonSchemaImmutableType | readonly GbnfJsonSchemaImmutableType[];
|
|
5
|
+
/**
|
|
6
|
+
* A description of what you expect the model to set this value to.
|
|
7
|
+
*
|
|
8
|
+
* Only passed to the model when using function calling, and has no effect when using JSON Schema grammar directly.
|
|
9
|
+
*/
|
|
10
|
+
readonly description?: string;
|
|
5
11
|
};
|
|
6
12
|
export type GbnfJsonConstSchema = {
|
|
7
|
-
const: string | number | boolean | null;
|
|
13
|
+
readonly const: string | number | boolean | null;
|
|
14
|
+
/**
|
|
15
|
+
* A description of what you expect the model to set this value to.
|
|
16
|
+
*
|
|
17
|
+
* Only passed to the model when using function calling, and has no effect when using JSON Schema grammar directly.
|
|
18
|
+
*/
|
|
19
|
+
readonly description?: string;
|
|
8
20
|
};
|
|
9
21
|
export type GbnfJsonEnumSchema = {
|
|
10
|
-
enum: readonly (string | number | boolean | null)[];
|
|
22
|
+
readonly enum: readonly (string | number | boolean | null)[];
|
|
23
|
+
/**
|
|
24
|
+
* A description of what you expect the model to set this value to.
|
|
25
|
+
*
|
|
26
|
+
* Only passed to the model when using function calling, and has no effect when using JSON Schema grammar directly.
|
|
27
|
+
*/
|
|
28
|
+
readonly description?: string;
|
|
11
29
|
};
|
|
12
30
|
export type GbnfJsonOneOfSchema = {
|
|
13
|
-
oneOf: readonly GbnfJsonSchema[];
|
|
31
|
+
readonly oneOf: readonly GbnfJsonSchema[];
|
|
32
|
+
/**
|
|
33
|
+
* A description of what you expect the model to set this value to.
|
|
34
|
+
*
|
|
35
|
+
* Only passed to the model when using function calling, and has no effect when using JSON Schema grammar directly.
|
|
36
|
+
*/
|
|
37
|
+
readonly description?: string;
|
|
38
|
+
};
|
|
39
|
+
export type GbnfJsonStringSchema = GbnfJsonBasicStringSchema | GbnfJsonFormatStringSchema;
|
|
40
|
+
export type GbnfJsonBasicStringSchema = {
|
|
41
|
+
readonly type: "string";
|
|
42
|
+
/**
|
|
43
|
+
* When using `minLength` and/or `maxLength`,
|
|
44
|
+
* ensure to inform the model as part of the prompt what your expectations are regarding the length of the string.
|
|
45
|
+
* Not doing this may lead to hallucinations.
|
|
46
|
+
*/
|
|
47
|
+
readonly minLength?: number;
|
|
48
|
+
/**
|
|
49
|
+
* When using `minLength` and/or `maxLength`,
|
|
50
|
+
* ensure to inform the model as part of the prompt what your expectations are regarding the length of the string.
|
|
51
|
+
* Not doing this may lead to hallucinations.
|
|
52
|
+
*/
|
|
53
|
+
readonly maxLength?: number;
|
|
54
|
+
/**
|
|
55
|
+
* A description of what you expect the model to set this value to.
|
|
56
|
+
*
|
|
57
|
+
* Only passed to the model when using function calling, and has no effect when using JSON Schema grammar directly.
|
|
58
|
+
*/
|
|
59
|
+
readonly description?: string;
|
|
60
|
+
};
|
|
61
|
+
export type GbnfJsonFormatStringSchema = {
|
|
62
|
+
readonly type: "string";
|
|
63
|
+
readonly format: "date-time" | "time" | "date";
|
|
64
|
+
/**
|
|
65
|
+
* A description of what you expect the model to set this value to.
|
|
66
|
+
*
|
|
67
|
+
* Only passed to the model when using function calling, and has no effect when using JSON Schema grammar directly.
|
|
68
|
+
*/
|
|
69
|
+
readonly description?: string;
|
|
14
70
|
};
|
|
15
71
|
export type GbnfJsonObjectSchema<Keys extends string = string> = {
|
|
16
|
-
type: "object";
|
|
17
|
-
properties
|
|
18
|
-
[key in Keys]: GbnfJsonSchema;
|
|
19
|
-
}
|
|
20
|
-
|
|
72
|
+
readonly type: "object";
|
|
73
|
+
readonly properties?: {
|
|
74
|
+
readonly [key in Keys]: GbnfJsonSchema;
|
|
75
|
+
};
|
|
76
|
+
/**
|
|
77
|
+
* Unlike the JSON Schema spec, `additionalProperties` defaults to `false` to avoid breaking existing code.
|
|
78
|
+
*/
|
|
79
|
+
readonly additionalProperties?: boolean | GbnfJsonSchema;
|
|
80
|
+
/**
|
|
81
|
+
* Make sure you define `additionalProperties` for this to have any effect.
|
|
82
|
+
*
|
|
83
|
+
* When using `minProperties` and/or `maxProperties`,
|
|
84
|
+
* ensure to inform the model as part of the prompt what your expectations are regarding the number of keys in the object.
|
|
85
|
+
* Not doing this may lead to hallucinations.
|
|
86
|
+
*/
|
|
87
|
+
readonly minProperties?: number;
|
|
88
|
+
/**
|
|
89
|
+
* Make sure you define `additionalProperties` for this to have any effect.
|
|
90
|
+
*
|
|
91
|
+
* When using `minProperties` and/or `maxProperties`,
|
|
92
|
+
* ensure to inform the model as part of the prompt what your expectations are regarding the number of keys in the object.
|
|
93
|
+
* Not doing this may lead to hallucinations.
|
|
94
|
+
*/
|
|
95
|
+
readonly maxProperties?: number;
|
|
96
|
+
/**
|
|
97
|
+
* `required` is always set to all keys in `properties`, and setting it has no effect.
|
|
98
|
+
*
|
|
99
|
+
* This limitation is due to how the generation works, and may be fixed in the future.
|
|
100
|
+
*
|
|
101
|
+
* This key is part of the type to avoid breaking exiting code (though it was never actually used in the past),
|
|
102
|
+
* and will be removed in the future.
|
|
103
|
+
* @deprecated
|
|
104
|
+
*/
|
|
105
|
+
readonly required?: readonly Keys[];
|
|
106
|
+
/**
|
|
107
|
+
* A description of what you expect the model to set this value to.
|
|
108
|
+
*
|
|
109
|
+
* Only passed to the model when using function calling, and has no effect when using JSON Schema grammar directly.
|
|
110
|
+
*/
|
|
111
|
+
readonly description?: string;
|
|
21
112
|
};
|
|
22
113
|
export type GbnfJsonArraySchema = {
|
|
23
|
-
type: "array";
|
|
24
|
-
items
|
|
114
|
+
readonly type: "array";
|
|
115
|
+
readonly items?: GbnfJsonSchema;
|
|
116
|
+
readonly prefixItems?: readonly GbnfJsonSchema[];
|
|
117
|
+
/**
|
|
118
|
+
* When using `minItems` and/or `maxItems`,
|
|
119
|
+
* ensure to inform the model as part of the prompt what your expectations are regarding the length of the array.
|
|
120
|
+
* Not doing this may lead to hallucinations.
|
|
121
|
+
*/
|
|
122
|
+
readonly minItems?: number;
|
|
123
|
+
/**
|
|
124
|
+
* When using `minItems` and/or `maxItems`,
|
|
125
|
+
* ensure to inform the model as part of the prompt what your expectations are regarding the length of the array.
|
|
126
|
+
* Not doing this may lead to hallucinations.
|
|
127
|
+
*/
|
|
128
|
+
readonly maxItems?: number;
|
|
129
|
+
/**
|
|
130
|
+
* A description of what you expect the model to set this value to.
|
|
131
|
+
*
|
|
132
|
+
* Only passed to the model when using function calling, and has no effect when using JSON Schema grammar directly.
|
|
133
|
+
*/
|
|
134
|
+
readonly description?: string;
|
|
25
135
|
};
|
|
26
136
|
/**
|
|
27
137
|
* Converts a GBNF JSON schema to a TypeScript type
|
|
28
138
|
*/
|
|
29
139
|
export type GbnfJsonSchemaToType<T> = GbnfJsonSchemaToTSType<T>;
|
|
30
|
-
export type GbnfJsonSchemaToTSType<T> = GbnfJsonBasicSchema extends T ? undefined : undefined extends T ? undefined : T extends GbnfJsonBasicSchema ? GbnfJsonBasicSchemaToType<T["type"]> : T extends GbnfJsonConstSchema ? T["const"] : T extends GbnfJsonEnumSchema ? T["enum"][number] : T extends GbnfJsonOneOfSchema ? GbnfJsonSchemaToType<T["oneOf"][number]> : T extends GbnfJsonObjectSchema ? GbnfJsonObjectSchemaToType<T
|
|
31
|
-
type
|
|
32
|
-
type
|
|
33
|
-
type
|
|
34
|
-
|
|
140
|
+
export type GbnfJsonSchemaToTSType<T> = Readonly<GbnfJsonBasicSchema> extends T ? undefined : undefined extends T ? undefined : T extends GbnfJsonBasicStringSchema ? GbnfJsonBasicStringSchemaToType<T> : T extends GbnfJsonFormatStringSchema ? string : T extends GbnfJsonBasicSchema ? GbnfJsonBasicSchemaToType<T["type"]> : T extends GbnfJsonConstSchema ? T["const"] : T extends GbnfJsonEnumSchema ? T["enum"][number] : T extends GbnfJsonOneOfSchema ? GbnfJsonSchemaToType<T["oneOf"][number]> : T extends GbnfJsonObjectSchema ? GbnfJsonObjectSchemaToType<T> : T extends GbnfJsonArraySchema ? ArrayTypeToType<T> : undefined;
|
|
141
|
+
type GbnfJsonBasicStringSchemaToType<T extends GbnfJsonBasicStringSchema> = T["maxLength"] extends 0 ? "" : string;
|
|
142
|
+
type GbnfJsonBasicSchemaToType<T extends GbnfJsonBasicSchema["type"]> = T extends GbnfJsonSchemaImmutableType ? ImmutableTypeToType<T> : T extends GbnfJsonSchemaImmutableType[] ? ImmutableTypeToType<T[number]> : never;
|
|
143
|
+
type ImmutableTypeToType<T extends GbnfJsonSchemaImmutableType> = T extends "string" ? string : T extends "number" ? number : T extends "integer" ? number : T extends "boolean" ? boolean : T extends "null" ? null : never;
|
|
144
|
+
type ArrayTypeToType<T extends GbnfJsonArraySchema, MinItems extends number = T["minItems"] extends number ? T["prefixItems"] extends readonly GbnfJsonSchema[] ? keyof T["prefixItems"] extends T["minItems"] ? T["prefixItems"]["length"] : T["minItems"] : T["minItems"] : T["prefixItems"] extends readonly GbnfJsonSchema[] ? T["prefixItems"]["length"] : 0> = T["prefixItems"] extends readonly GbnfJsonSchema[] ? (MinItems extends T["prefixItems"]["length"] ? (T["maxItems"] extends MinItems ? [
|
|
145
|
+
...GbnfJsonOrderedArrayTypes<T["prefixItems"]>,
|
|
146
|
+
...IndexRangeWithSkip<MinItems, T["prefixItems"]["length"], T["items"] extends GbnfJsonSchema ? GbnfJsonSchemaToType<T["items"]> : GbnfJsonAnyValue>
|
|
147
|
+
] : [
|
|
148
|
+
...GbnfJsonOrderedArrayTypes<T["prefixItems"]>,
|
|
149
|
+
...(T["items"] extends GbnfJsonSchema ? GbnfJsonSchemaToType<T["items"]> : GbnfJsonAnyValue)[]
|
|
150
|
+
]) : T["maxItems"] extends MinItems ? [
|
|
151
|
+
...GbnfJsonOrderedArrayTypes<T["prefixItems"]>,
|
|
152
|
+
...(T["items"] extends GbnfJsonSchema ? IndexRangeWithSkip<T["maxItems"], T["prefixItems"]["length"], GbnfJsonSchemaToType<T["items"]>> : IndexRangeWithSkip<T["maxItems"], T["prefixItems"]["length"], GbnfJsonAnyValue>)
|
|
153
|
+
] : [
|
|
154
|
+
...GbnfJsonOrderedArrayTypes<T["prefixItems"]>,
|
|
155
|
+
...IndexRangeWithSkip<MinItems, T["prefixItems"]["length"], T["items"] extends GbnfJsonSchema ? GbnfJsonSchemaToType<T["items"]> : GbnfJsonAnyValue>,
|
|
156
|
+
...(T["items"] extends GbnfJsonSchema ? GbnfJsonSchemaToType<T["items"]> : GbnfJsonAnyValue)[]
|
|
157
|
+
]) : T["items"] extends GbnfJsonSchema ? (MinItems extends 0 ? GbnfJsonSchemaToType<T["items"]>[] : T["maxItems"] extends MinItems ? IndexRange<T["maxItems"], GbnfJsonSchemaToType<T["items"]>> : [
|
|
158
|
+
...IndexRange<MinItems, GbnfJsonSchemaToType<T["items"]>>,
|
|
159
|
+
...GbnfJsonSchemaToType<T["items"]>[]
|
|
160
|
+
]) : (MinItems extends 0 ? GbnfJsonAnyValue[] : T["maxItems"] extends MinItems ? IndexRange<T["maxItems"], GbnfJsonAnyValue> : [
|
|
161
|
+
...IndexRange<MinItems, GbnfJsonAnyValue>,
|
|
162
|
+
...GbnfJsonAnyValue[]
|
|
163
|
+
]);
|
|
164
|
+
type GbnfJsonObjectSchemaToType<T extends GbnfJsonObjectSchema, Props extends Readonly<Record<string, GbnfJsonSchema>> | undefined = T["properties"], AdditionalProps extends true | false | GbnfJsonSchema | undefined = T["additionalProperties"], PropsMap = Props extends undefined ? {} : {
|
|
165
|
+
-readonly [P in keyof Props]: GbnfJsonSchemaToType<Props[P]>;
|
|
166
|
+
}, Res = AdditionalProps extends undefined | false ? PropsMap : AdditionalProps extends true ? PropsMap & {
|
|
167
|
+
[key: string]: GbnfJsonAnyValue;
|
|
168
|
+
} : AdditionalProps extends GbnfJsonSchema ? PropsMap & {
|
|
169
|
+
[key: string]: GbnfJsonSchemaToType<AdditionalProps>;
|
|
170
|
+
} : PropsMap> = Res;
|
|
171
|
+
type GbnfJsonAnyValue = string | number | boolean | null | GbnfJsonAnyValue[] | {
|
|
172
|
+
[key: string]: GbnfJsonAnyValue;
|
|
35
173
|
};
|
|
36
174
|
export declare function isGbnfJsonConstSchema(schema: GbnfJsonSchema): schema is GbnfJsonConstSchema;
|
|
37
175
|
export declare function isGbnfJsonEnumSchema(schema: GbnfJsonSchema): schema is GbnfJsonEnumSchema;
|
|
38
176
|
export declare function isGbnfJsonOneOfSchema(schema: GbnfJsonSchema): schema is GbnfJsonOneOfSchema;
|
|
177
|
+
export declare function isGbnfJsonBasicStringSchema(schema: GbnfJsonSchema): schema is GbnfJsonBasicStringSchema;
|
|
178
|
+
export declare function isGbnfJsonFormatStringSchema(schema: GbnfJsonSchema): schema is GbnfJsonFormatStringSchema;
|
|
39
179
|
export declare function isGbnfJsonObjectSchema(schema: GbnfJsonSchema): schema is GbnfJsonObjectSchema;
|
|
40
180
|
export declare function isGbnfJsonArraySchema(schema: GbnfJsonSchema): schema is GbnfJsonArraySchema;
|
|
41
181
|
export declare function isGbnfJsonBasicSchemaIncludesType<T extends GbnfJsonSchemaImmutableType>(schema: GbnfJsonBasicSchema, type: T): schema is GbnfJsonBasicSchema & {
|
|
42
182
|
type: T | (T | GbnfJsonSchemaImmutableType)[];
|
|
43
183
|
};
|
|
184
|
+
type IndexRange<Length extends number, FillType = number, Res = _IndexRange<[], Length, FillType>> = Res;
|
|
185
|
+
type _IndexRange<Value extends FillType[], MaxLength extends number, FillType = number> = Value["length"] extends MaxLength ? Value : _IndexRange<[...Value, FillType], MaxLength, FillType>;
|
|
186
|
+
type IndexRangeWithSkip<Length extends number, SkipFirst extends number, FillType, Res = _IndexRangeWithSkip<[], IndexRange<SkipFirst>, Length, FillType>> = Res;
|
|
187
|
+
type _IndexRangeWithSkip<Value extends FillType[], ConditionValue extends number[], MaxLength extends number, FillType> = ConditionValue["length"] extends MaxLength ? Value : _IndexRangeWithSkip<[...Value, FillType], [...ConditionValue, ConditionValue["length"]], MaxLength, FillType>;
|
|
188
|
+
type GbnfJsonOrderedArrayTypes<T extends readonly GbnfJsonSchema[]> = {
|
|
189
|
+
-readonly [P in keyof T]: GbnfJsonSchemaToType<T[P]>;
|
|
190
|
+
};
|
|
44
191
|
export {};
|
|
@@ -7,6 +7,12 @@ export function isGbnfJsonEnumSchema(schema) {
|
|
|
7
7
|
export function isGbnfJsonOneOfSchema(schema) {
|
|
8
8
|
return schema.oneOf != null;
|
|
9
9
|
}
|
|
10
|
+
export function isGbnfJsonBasicStringSchema(schema) {
|
|
11
|
+
return schema.type === "string" && schema.format == null;
|
|
12
|
+
}
|
|
13
|
+
export function isGbnfJsonFormatStringSchema(schema) {
|
|
14
|
+
return schema.type === "string" && schema.format != null;
|
|
15
|
+
}
|
|
10
16
|
export function isGbnfJsonObjectSchema(schema) {
|
|
11
17
|
return schema.type === "object";
|
|
12
18
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/utils/gbnfJson/types.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/utils/gbnfJson/types.ts"],"names":[],"mappings":"AAoTA,MAAM,UAAU,qBAAqB,CAAC,MAAsB;IACxD,OAAQ,MAA8B,CAAC,KAAK,KAAK,SAAS,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAsB;IACvD,OAAQ,MAA6B,CAAC,IAAI,IAAI,IAAI,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,MAAsB;IACxD,OAAQ,MAA8B,CAAC,KAAK,IAAI,IAAI,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,MAAsB;IAC9D,OAAQ,MAAoC,CAAC,IAAI,KAAK,QAAQ,IAAK,MAAqC,CAAC,MAAM,IAAI,IAAI,CAAC;AAC5H,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,MAAsB;IAC/D,OAAQ,MAAqC,CAAC,IAAI,KAAK,QAAQ,IAAK,MAAqC,CAAC,MAAM,IAAI,IAAI,CAAC;AAC7H,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,MAAsB;IACzD,OAAQ,MAA+B,CAAC,IAAI,KAAK,QAAQ,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,MAAsB;IACxD,OAAQ,MAA8B,CAAC,IAAI,KAAK,OAAO,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,iCAAiC,CAC7C,MAA2B,EAAE,IAAO;IAEpC,IAAI,MAAM,CAAC,IAAI,YAAY,KAAK;QAC5B,OAAO,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEtC,OAAO,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC;AAChC,CAAC"}
|
|
@@ -6,7 +6,10 @@ import { GbnfString } from "../terminals/GbnfString.js";
|
|
|
6
6
|
import { GbnfNumber } from "../terminals/GbnfNumber.js";
|
|
7
7
|
import { GbnfBoolean } from "../terminals/GbnfBoolean.js";
|
|
8
8
|
import { GbnfNull } from "../terminals/GbnfNull.js";
|
|
9
|
-
import { isGbnfJsonArraySchema, isGbnfJsonBasicSchemaIncludesType, isGbnfJsonConstSchema, isGbnfJsonEnumSchema, isGbnfJsonObjectSchema, isGbnfJsonOneOfSchema } from "../types.js";
|
|
9
|
+
import { isGbnfJsonArraySchema, isGbnfJsonBasicSchemaIncludesType, isGbnfJsonConstSchema, isGbnfJsonEnumSchema, isGbnfJsonObjectSchema, isGbnfJsonOneOfSchema, isGbnfJsonBasicStringSchema, isGbnfJsonFormatStringSchema } from "../types.js";
|
|
10
|
+
import { getConsoleLogPrefix } from "../../getConsoleLogPrefix.js";
|
|
11
|
+
import { GbnfAnyJson } from "../terminals/GbnfAnyJson.js";
|
|
12
|
+
import { GbnfFormatString } from "../terminals/GbnfFormatString.js";
|
|
10
13
|
import { getGbnfJsonTerminalForLiteral } from "./getGbnfJsonTerminalForLiteral.js";
|
|
11
14
|
import { GbnfJsonScopeState } from "./GbnfJsonScopeState.js";
|
|
12
15
|
export function getGbnfJsonTerminalForGbnfJsonSchema(schema, grammarGenerator, scopeState = new GbnfJsonScopeState()) {
|
|
@@ -22,17 +25,66 @@ export function getGbnfJsonTerminalForGbnfJsonSchema(schema, grammarGenerator, s
|
|
|
22
25
|
return new GbnfOr(schema.enum.map((item) => getGbnfJsonTerminalForLiteral(item)));
|
|
23
26
|
}
|
|
24
27
|
else if (isGbnfJsonObjectSchema(schema)) {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
28
|
+
const propertiesEntries = Object.entries(schema.properties ?? {});
|
|
29
|
+
let maxProperties = schema.maxProperties;
|
|
30
|
+
if (schema.properties != null && maxProperties != null && maxProperties < propertiesEntries.length) {
|
|
31
|
+
console.warn(getConsoleLogPrefix(true, false), `maxProperties (${maxProperties}) must be greater than or equal to ` +
|
|
32
|
+
`properties object keys number (${propertiesEntries.length}). ` +
|
|
33
|
+
"Using properties object keys number as maxProperties.");
|
|
34
|
+
maxProperties = propertiesEntries.length;
|
|
35
|
+
}
|
|
36
|
+
return new GbnfObjectMap({
|
|
37
|
+
fields: propertiesEntries.map(([propName, propSchema]) => {
|
|
38
|
+
return {
|
|
39
|
+
required: true,
|
|
40
|
+
key: new GbnfStringValue(propName),
|
|
41
|
+
value: getGbnfJsonTerminalForGbnfJsonSchema(propSchema, grammarGenerator, scopeState.getForNewScope())
|
|
42
|
+
};
|
|
43
|
+
}),
|
|
44
|
+
additionalProperties: (schema.additionalProperties == null || schema.additionalProperties === false)
|
|
45
|
+
? undefined
|
|
46
|
+
: schema.additionalProperties === true
|
|
47
|
+
? new GbnfAnyJson(scopeState.getForNewScope())
|
|
48
|
+
: getGbnfJsonTerminalForGbnfJsonSchema(schema.additionalProperties, grammarGenerator, scopeState.getForNewScope()),
|
|
49
|
+
minProperties: schema.minProperties,
|
|
50
|
+
maxProperties,
|
|
51
|
+
scopeState
|
|
52
|
+
});
|
|
32
53
|
}
|
|
33
54
|
else if (isGbnfJsonArraySchema(schema)) {
|
|
34
|
-
|
|
55
|
+
let maxItems = schema.maxItems;
|
|
56
|
+
if (schema.prefixItems != null && maxItems != null && maxItems < schema.prefixItems.length) {
|
|
57
|
+
console.warn(getConsoleLogPrefix(true, false), `maxItems (${maxItems}) must be greater than or equal to prefixItems array length (${schema.prefixItems.length}). ` +
|
|
58
|
+
"Using prefixItems length as maxItems.");
|
|
59
|
+
maxItems = schema.prefixItems.length;
|
|
60
|
+
}
|
|
61
|
+
return new GbnfArray({
|
|
62
|
+
items: schema.items == null
|
|
63
|
+
? undefined
|
|
64
|
+
: getGbnfJsonTerminalForGbnfJsonSchema(schema.items, grammarGenerator, scopeState.getForNewScope()),
|
|
65
|
+
prefixItems: schema.prefixItems == null
|
|
66
|
+
? undefined
|
|
67
|
+
: schema.prefixItems.map((item) => (getGbnfJsonTerminalForGbnfJsonSchema(item, grammarGenerator, scopeState.getForNewScope()))),
|
|
68
|
+
minItems: schema.minItems,
|
|
69
|
+
maxItems,
|
|
70
|
+
scopeState
|
|
71
|
+
});
|
|
35
72
|
}
|
|
73
|
+
else if (isGbnfJsonBasicStringSchema(schema)) {
|
|
74
|
+
const minLength = Math.max(0, schema.minLength ?? 0);
|
|
75
|
+
let maxLength = schema.maxLength;
|
|
76
|
+
if (maxLength != null && maxLength < minLength) {
|
|
77
|
+
console.warn(getConsoleLogPrefix(true, false), `maxLength (${maxLength}) must be greater than or equal to minLength (${minLength}). ` +
|
|
78
|
+
"Using minLength as maxLength.");
|
|
79
|
+
maxLength = minLength;
|
|
80
|
+
}
|
|
81
|
+
return new GbnfString({
|
|
82
|
+
minLength,
|
|
83
|
+
maxLength
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
else if (isGbnfJsonFormatStringSchema(schema))
|
|
87
|
+
return new GbnfFormatString(schema.format);
|
|
36
88
|
const terminals = [];
|
|
37
89
|
if (isGbnfJsonBasicSchemaIncludesType(schema, "string"))
|
|
38
90
|
terminals.push(new GbnfString());
|