@probelabs/probe 0.6.0-rc161 → 0.6.0-rc162
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -0
- package/build/agent/ProbeAgent.js +349 -85
- package/build/agent/contextCompactor.js +271 -0
- package/build/agent/index.js +788 -84
- package/build/agent/schemaUtils.js +7 -0
- package/build/agent/xmlParsingUtils.js +24 -4
- package/build/tools/common.js +16 -1
- package/cjs/agent/ProbeAgent.cjs +823 -134
- package/cjs/index.cjs +823 -134
- package/package.json +1 -1
- package/src/agent/ProbeAgent.js +349 -85
- package/src/agent/contextCompactor.js +271 -0
- package/src/agent/index.js +30 -1
- package/src/agent/schemaUtils.js +7 -0
- package/src/agent/xmlParsingUtils.js +24 -4
- package/src/tools/common.js +16 -1
package/cjs/agent/ProbeAgent.cjs
CHANGED
|
@@ -9781,7 +9781,10 @@ var init_CborCodec = __esm({
|
|
|
9781
9781
|
}
|
|
9782
9782
|
} else if (ns.isStructSchema()) {
|
|
9783
9783
|
for (const [key, memberSchema] of ns.structIterator()) {
|
|
9784
|
-
|
|
9784
|
+
const v3 = this.readValue(memberSchema, value[key]);
|
|
9785
|
+
if (v3 != null) {
|
|
9786
|
+
newObject[key] = v3;
|
|
9787
|
+
}
|
|
9785
9788
|
}
|
|
9786
9789
|
}
|
|
9787
9790
|
return newObject;
|
|
@@ -11302,10 +11305,7 @@ var init_JsonShapeSerializer = __esm({
|
|
|
11302
11305
|
if (ns === this.rootSchema) {
|
|
11303
11306
|
return value;
|
|
11304
11307
|
}
|
|
11305
|
-
|
|
11306
|
-
return (0, import_util_base645.toBase64)(value);
|
|
11307
|
-
}
|
|
11308
|
-
return this.serdeContext?.base64Encoder(value);
|
|
11308
|
+
return (this.serdeContext?.base64Encoder ?? import_util_base645.toBase64)(value);
|
|
11309
11309
|
}
|
|
11310
11310
|
if ((ns.isTimestampSchema() || ns.isDocumentSchema()) && value instanceof Date) {
|
|
11311
11311
|
const format2 = determineTimestampFormat(ns, this.settings);
|
|
@@ -12940,7 +12940,13 @@ var init_AwsQueryProtocol = __esm({
|
|
|
12940
12940
|
Code: errorData.Code,
|
|
12941
12941
|
Message: message
|
|
12942
12942
|
};
|
|
12943
|
-
const { errorSchema, errorMetadata } = await this.mixin.getErrorSchemaOrThrowBaseException(errorIdentifier, this.options.defaultNamespace, response, errorData, metadata, (registry, errorName) =>
|
|
12943
|
+
const { errorSchema, errorMetadata } = await this.mixin.getErrorSchemaOrThrowBaseException(errorIdentifier, this.options.defaultNamespace, response, errorData, metadata, (registry, errorName) => {
|
|
12944
|
+
try {
|
|
12945
|
+
return registry.getSchema(errorName);
|
|
12946
|
+
} catch (e3) {
|
|
12947
|
+
return registry.find((schema) => NormalizedSchema.of(schema).getMergedTraits().awsQueryError?.[0] === errorName);
|
|
12948
|
+
}
|
|
12949
|
+
});
|
|
12944
12950
|
const ns = NormalizedSchema.of(errorSchema);
|
|
12945
12951
|
const ErrorCtor = TypeRegistry.for(errorSchema[1]).getErrorCtor(errorSchema) ?? Error;
|
|
12946
12952
|
const exception = new ErrorCtor(message);
|
|
@@ -15306,7 +15312,11 @@ var require_dist_cjs39 = __commonJS({
|
|
|
15306
15312
|
var validRegions = /* @__PURE__ */ new Set();
|
|
15307
15313
|
var checkRegion = (region, check = utilEndpoints.isValidHostLabel) => {
|
|
15308
15314
|
if (!validRegions.has(region) && !check(region)) {
|
|
15309
|
-
|
|
15315
|
+
if (region === "*") {
|
|
15316
|
+
console.warn(`@smithy/config-resolver WARN - Please use the caller region instead of "*". See "sigv4a" in https://github.com/aws/aws-sdk-js-v3/blob/main/supplemental-docs/CLIENTS.md.`);
|
|
15317
|
+
} else {
|
|
15318
|
+
throw new Error(`Region not accepted: region="${region}" is not a valid hostname component.`);
|
|
15319
|
+
}
|
|
15310
15320
|
} else {
|
|
15311
15321
|
validRegions.add(region);
|
|
15312
15322
|
}
|
|
@@ -15504,42 +15514,40 @@ var require_getSSOTokenFromFile = __commonJS({
|
|
|
15504
15514
|
"use strict";
|
|
15505
15515
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
15506
15516
|
exports2.getSSOTokenFromFile = exports2.tokenIntercept = void 0;
|
|
15507
|
-
var
|
|
15517
|
+
var promises_1 = require("fs/promises");
|
|
15508
15518
|
var getSSOTokenFilepath_1 = require_getSSOTokenFilepath();
|
|
15509
|
-
var { readFile: readFile2 } = fs_1.promises;
|
|
15510
15519
|
exports2.tokenIntercept = {};
|
|
15511
15520
|
var getSSOTokenFromFile = async (id) => {
|
|
15512
15521
|
if (exports2.tokenIntercept[id]) {
|
|
15513
15522
|
return exports2.tokenIntercept[id];
|
|
15514
15523
|
}
|
|
15515
15524
|
const ssoTokenFilepath = (0, getSSOTokenFilepath_1.getSSOTokenFilepath)(id);
|
|
15516
|
-
const ssoTokenText = await
|
|
15525
|
+
const ssoTokenText = await (0, promises_1.readFile)(ssoTokenFilepath, "utf8");
|
|
15517
15526
|
return JSON.parse(ssoTokenText);
|
|
15518
15527
|
};
|
|
15519
15528
|
exports2.getSSOTokenFromFile = getSSOTokenFromFile;
|
|
15520
15529
|
}
|
|
15521
15530
|
});
|
|
15522
15531
|
|
|
15523
|
-
// node_modules/@smithy/shared-ini-file-loader/dist-cjs/
|
|
15524
|
-
var
|
|
15525
|
-
"node_modules/@smithy/shared-ini-file-loader/dist-cjs/
|
|
15532
|
+
// node_modules/@smithy/shared-ini-file-loader/dist-cjs/readFile.js
|
|
15533
|
+
var require_readFile = __commonJS({
|
|
15534
|
+
"node_modules/@smithy/shared-ini-file-loader/dist-cjs/readFile.js"(exports2) {
|
|
15526
15535
|
"use strict";
|
|
15527
15536
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
15528
|
-
exports2.
|
|
15529
|
-
var
|
|
15530
|
-
|
|
15531
|
-
exports2.filePromisesHash = {};
|
|
15537
|
+
exports2.readFile = exports2.fileIntercept = exports2.filePromises = void 0;
|
|
15538
|
+
var promises_1 = require("node:fs/promises");
|
|
15539
|
+
exports2.filePromises = {};
|
|
15532
15540
|
exports2.fileIntercept = {};
|
|
15533
|
-
var
|
|
15541
|
+
var readFile2 = (path7, options) => {
|
|
15534
15542
|
if (exports2.fileIntercept[path7] !== void 0) {
|
|
15535
15543
|
return exports2.fileIntercept[path7];
|
|
15536
15544
|
}
|
|
15537
|
-
if (!exports2.
|
|
15538
|
-
exports2.
|
|
15545
|
+
if (!exports2.filePromises[path7] || options?.ignoreCache) {
|
|
15546
|
+
exports2.filePromises[path7] = (0, promises_1.readFile)(path7, "utf8");
|
|
15539
15547
|
}
|
|
15540
|
-
return exports2.
|
|
15548
|
+
return exports2.filePromises[path7];
|
|
15541
15549
|
};
|
|
15542
|
-
exports2.
|
|
15550
|
+
exports2.readFile = readFile2;
|
|
15543
15551
|
}
|
|
15544
15552
|
});
|
|
15545
15553
|
|
|
@@ -15552,7 +15560,7 @@ var require_dist_cjs42 = __commonJS({
|
|
|
15552
15560
|
var getSSOTokenFromFile = require_getSSOTokenFromFile();
|
|
15553
15561
|
var path7 = require("path");
|
|
15554
15562
|
var types2 = require_dist_cjs();
|
|
15555
|
-
var
|
|
15563
|
+
var readFile2 = require_readFile();
|
|
15556
15564
|
var ENV_PROFILE = "AWS_PROFILE";
|
|
15557
15565
|
var DEFAULT_PROFILE = "default";
|
|
15558
15566
|
var getProfileName = (init) => init.profile || process.env[ENV_PROFILE] || DEFAULT_PROFILE;
|
|
@@ -15636,10 +15644,10 @@ var require_dist_cjs42 = __commonJS({
|
|
|
15636
15644
|
resolvedConfigFilepath = path7.join(homeDir, configFilepath.slice(2));
|
|
15637
15645
|
}
|
|
15638
15646
|
const parsedFiles = await Promise.all([
|
|
15639
|
-
|
|
15647
|
+
readFile2.readFile(resolvedConfigFilepath, {
|
|
15640
15648
|
ignoreCache: init.ignoreCache
|
|
15641
15649
|
}).then(parseIni).then(getConfigData).catch(swallowError$1),
|
|
15642
|
-
|
|
15650
|
+
readFile2.readFile(resolvedFilepath, {
|
|
15643
15651
|
ignoreCache: init.ignoreCache
|
|
15644
15652
|
}).then(parseIni).catch(swallowError$1)
|
|
15645
15653
|
]);
|
|
@@ -15650,7 +15658,7 @@ var require_dist_cjs42 = __commonJS({
|
|
|
15650
15658
|
};
|
|
15651
15659
|
var getSsoSessionData = (data2) => Object.entries(data2).filter(([key]) => key.startsWith(types2.IniSectionType.SSO_SESSION + CONFIG_PREFIX_SEPARATOR)).reduce((acc, [key, value]) => ({ ...acc, [key.substring(key.indexOf(CONFIG_PREFIX_SEPARATOR) + 1)]: value }), {});
|
|
15652
15660
|
var swallowError = () => ({});
|
|
15653
|
-
var loadSsoSessionData = async (init = {}) =>
|
|
15661
|
+
var loadSsoSessionData = async (init = {}) => readFile2.readFile(init.configFilepath ?? getConfigFilepath()).then(parseIni).then(getSsoSessionData).catch(swallowError);
|
|
15654
15662
|
var mergeConfigFiles = (...files) => {
|
|
15655
15663
|
const merged = {};
|
|
15656
15664
|
for (const file of files) {
|
|
@@ -15670,10 +15678,10 @@ var require_dist_cjs42 = __commonJS({
|
|
|
15670
15678
|
};
|
|
15671
15679
|
var externalDataInterceptor = {
|
|
15672
15680
|
getFileRecord() {
|
|
15673
|
-
return
|
|
15681
|
+
return readFile2.fileIntercept;
|
|
15674
15682
|
},
|
|
15675
15683
|
interceptFile(path8, contents) {
|
|
15676
|
-
|
|
15684
|
+
readFile2.fileIntercept[path8] = Promise.resolve(contents);
|
|
15677
15685
|
},
|
|
15678
15686
|
getTokenRecord() {
|
|
15679
15687
|
return getSSOTokenFromFile.tokenIntercept;
|
|
@@ -15688,6 +15696,12 @@ var require_dist_cjs42 = __commonJS({
|
|
|
15688
15696
|
return getSSOTokenFromFile.getSSOTokenFromFile;
|
|
15689
15697
|
}
|
|
15690
15698
|
});
|
|
15699
|
+
Object.defineProperty(exports2, "readFile", {
|
|
15700
|
+
enumerable: true,
|
|
15701
|
+
get: function() {
|
|
15702
|
+
return readFile2.readFile;
|
|
15703
|
+
}
|
|
15704
|
+
});
|
|
15691
15705
|
exports2.CONFIG_PREFIX_SEPARATOR = CONFIG_PREFIX_SEPARATOR;
|
|
15692
15706
|
exports2.DEFAULT_PROFILE = DEFAULT_PROFILE;
|
|
15693
15707
|
exports2.ENV_PROFILE = ENV_PROFILE;
|
|
@@ -16827,7 +16841,7 @@ var require_package2 = __commonJS({
|
|
|
16827
16841
|
module2.exports = {
|
|
16828
16842
|
name: "@aws-sdk/client-bedrock-runtime",
|
|
16829
16843
|
description: "AWS SDK for JavaScript Bedrock Runtime Client for Node.js, Browser and React Native",
|
|
16830
|
-
version: "3.
|
|
16844
|
+
version: "3.928.0",
|
|
16831
16845
|
scripts: {
|
|
16832
16846
|
build: "concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types'",
|
|
16833
16847
|
"build:cjs": "node ../../scripts/compilation/inline client-bedrock-runtime",
|
|
@@ -16846,22 +16860,22 @@ var require_package2 = __commonJS({
|
|
|
16846
16860
|
dependencies: {
|
|
16847
16861
|
"@aws-crypto/sha256-browser": "5.2.0",
|
|
16848
16862
|
"@aws-crypto/sha256-js": "5.2.0",
|
|
16849
|
-
"@aws-sdk/core": "3.
|
|
16850
|
-
"@aws-sdk/credential-provider-node": "3.
|
|
16863
|
+
"@aws-sdk/core": "3.928.0",
|
|
16864
|
+
"@aws-sdk/credential-provider-node": "3.928.0",
|
|
16851
16865
|
"@aws-sdk/eventstream-handler-node": "3.922.0",
|
|
16852
16866
|
"@aws-sdk/middleware-eventstream": "3.922.0",
|
|
16853
16867
|
"@aws-sdk/middleware-host-header": "3.922.0",
|
|
16854
16868
|
"@aws-sdk/middleware-logger": "3.922.0",
|
|
16855
16869
|
"@aws-sdk/middleware-recursion-detection": "3.922.0",
|
|
16856
|
-
"@aws-sdk/middleware-user-agent": "3.
|
|
16870
|
+
"@aws-sdk/middleware-user-agent": "3.928.0",
|
|
16857
16871
|
"@aws-sdk/middleware-websocket": "3.922.0",
|
|
16858
|
-
"@aws-sdk/region-config-resolver": "3.
|
|
16859
|
-
"@aws-sdk/token-providers": "3.
|
|
16872
|
+
"@aws-sdk/region-config-resolver": "3.925.0",
|
|
16873
|
+
"@aws-sdk/token-providers": "3.928.0",
|
|
16860
16874
|
"@aws-sdk/types": "3.922.0",
|
|
16861
16875
|
"@aws-sdk/util-endpoints": "3.922.0",
|
|
16862
16876
|
"@aws-sdk/util-user-agent-browser": "3.922.0",
|
|
16863
|
-
"@aws-sdk/util-user-agent-node": "3.
|
|
16864
|
-
"@smithy/config-resolver": "^4.4.
|
|
16877
|
+
"@aws-sdk/util-user-agent-node": "3.928.0",
|
|
16878
|
+
"@smithy/config-resolver": "^4.4.2",
|
|
16865
16879
|
"@smithy/core": "^3.17.2",
|
|
16866
16880
|
"@smithy/eventstream-serde-browser": "^4.2.4",
|
|
16867
16881
|
"@smithy/eventstream-serde-config-resolver": "^4.3.4",
|
|
@@ -16884,7 +16898,7 @@ var require_package2 = __commonJS({
|
|
|
16884
16898
|
"@smithy/util-body-length-browser": "^4.2.0",
|
|
16885
16899
|
"@smithy/util-body-length-node": "^4.2.1",
|
|
16886
16900
|
"@smithy/util-defaults-mode-browser": "^4.3.5",
|
|
16887
|
-
"@smithy/util-defaults-mode-node": "^4.2.
|
|
16901
|
+
"@smithy/util-defaults-mode-node": "^4.2.8",
|
|
16888
16902
|
"@smithy/util-endpoints": "^3.2.4",
|
|
16889
16903
|
"@smithy/util-middleware": "^4.2.4",
|
|
16890
16904
|
"@smithy/util-retry": "^4.2.4",
|
|
@@ -17608,7 +17622,7 @@ var init_package = __esm({
|
|
|
17608
17622
|
"node_modules/@aws-sdk/nested-clients/package.json"() {
|
|
17609
17623
|
package_default = {
|
|
17610
17624
|
name: "@aws-sdk/nested-clients",
|
|
17611
|
-
version: "3.
|
|
17625
|
+
version: "3.928.0",
|
|
17612
17626
|
description: "Nested clients for AWS SDK packages.",
|
|
17613
17627
|
main: "./dist-cjs/index.js",
|
|
17614
17628
|
module: "./dist-es/index.js",
|
|
@@ -17637,17 +17651,17 @@ var init_package = __esm({
|
|
|
17637
17651
|
dependencies: {
|
|
17638
17652
|
"@aws-crypto/sha256-browser": "5.2.0",
|
|
17639
17653
|
"@aws-crypto/sha256-js": "5.2.0",
|
|
17640
|
-
"@aws-sdk/core": "3.
|
|
17654
|
+
"@aws-sdk/core": "3.928.0",
|
|
17641
17655
|
"@aws-sdk/middleware-host-header": "3.922.0",
|
|
17642
17656
|
"@aws-sdk/middleware-logger": "3.922.0",
|
|
17643
17657
|
"@aws-sdk/middleware-recursion-detection": "3.922.0",
|
|
17644
|
-
"@aws-sdk/middleware-user-agent": "3.
|
|
17645
|
-
"@aws-sdk/region-config-resolver": "3.
|
|
17658
|
+
"@aws-sdk/middleware-user-agent": "3.928.0",
|
|
17659
|
+
"@aws-sdk/region-config-resolver": "3.925.0",
|
|
17646
17660
|
"@aws-sdk/types": "3.922.0",
|
|
17647
17661
|
"@aws-sdk/util-endpoints": "3.922.0",
|
|
17648
17662
|
"@aws-sdk/util-user-agent-browser": "3.922.0",
|
|
17649
|
-
"@aws-sdk/util-user-agent-node": "3.
|
|
17650
|
-
"@smithy/config-resolver": "^4.4.
|
|
17663
|
+
"@aws-sdk/util-user-agent-node": "3.928.0",
|
|
17664
|
+
"@smithy/config-resolver": "^4.4.2",
|
|
17651
17665
|
"@smithy/core": "^3.17.2",
|
|
17652
17666
|
"@smithy/fetch-http-handler": "^5.3.5",
|
|
17653
17667
|
"@smithy/hash-node": "^4.2.4",
|
|
@@ -17667,7 +17681,7 @@ var init_package = __esm({
|
|
|
17667
17681
|
"@smithy/util-body-length-browser": "^4.2.0",
|
|
17668
17682
|
"@smithy/util-body-length-node": "^4.2.1",
|
|
17669
17683
|
"@smithy/util-defaults-mode-browser": "^4.3.5",
|
|
17670
|
-
"@smithy/util-defaults-mode-node": "^4.2.
|
|
17684
|
+
"@smithy/util-defaults-mode-node": "^4.2.8",
|
|
17671
17685
|
"@smithy/util-endpoints": "^3.2.4",
|
|
17672
17686
|
"@smithy/util-middleware": "^4.2.4",
|
|
17673
17687
|
"@smithy/util-retry": "^4.2.4",
|
|
@@ -19088,7 +19102,7 @@ var require_package3 = __commonJS({
|
|
|
19088
19102
|
module2.exports = {
|
|
19089
19103
|
name: "@aws-sdk/client-sso",
|
|
19090
19104
|
description: "AWS SDK for JavaScript Sso Client for Node.js, Browser and React Native",
|
|
19091
|
-
version: "3.
|
|
19105
|
+
version: "3.928.0",
|
|
19092
19106
|
scripts: {
|
|
19093
19107
|
build: "concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types'",
|
|
19094
19108
|
"build:cjs": "node ../../scripts/compilation/inline client-sso",
|
|
@@ -19107,17 +19121,17 @@ var require_package3 = __commonJS({
|
|
|
19107
19121
|
dependencies: {
|
|
19108
19122
|
"@aws-crypto/sha256-browser": "5.2.0",
|
|
19109
19123
|
"@aws-crypto/sha256-js": "5.2.0",
|
|
19110
|
-
"@aws-sdk/core": "3.
|
|
19124
|
+
"@aws-sdk/core": "3.928.0",
|
|
19111
19125
|
"@aws-sdk/middleware-host-header": "3.922.0",
|
|
19112
19126
|
"@aws-sdk/middleware-logger": "3.922.0",
|
|
19113
19127
|
"@aws-sdk/middleware-recursion-detection": "3.922.0",
|
|
19114
|
-
"@aws-sdk/middleware-user-agent": "3.
|
|
19115
|
-
"@aws-sdk/region-config-resolver": "3.
|
|
19128
|
+
"@aws-sdk/middleware-user-agent": "3.928.0",
|
|
19129
|
+
"@aws-sdk/region-config-resolver": "3.925.0",
|
|
19116
19130
|
"@aws-sdk/types": "3.922.0",
|
|
19117
19131
|
"@aws-sdk/util-endpoints": "3.922.0",
|
|
19118
19132
|
"@aws-sdk/util-user-agent-browser": "3.922.0",
|
|
19119
|
-
"@aws-sdk/util-user-agent-node": "3.
|
|
19120
|
-
"@smithy/config-resolver": "^4.4.
|
|
19133
|
+
"@aws-sdk/util-user-agent-node": "3.928.0",
|
|
19134
|
+
"@smithy/config-resolver": "^4.4.2",
|
|
19121
19135
|
"@smithy/core": "^3.17.2",
|
|
19122
19136
|
"@smithy/fetch-http-handler": "^5.3.5",
|
|
19123
19137
|
"@smithy/hash-node": "^4.2.4",
|
|
@@ -19137,7 +19151,7 @@ var require_package3 = __commonJS({
|
|
|
19137
19151
|
"@smithy/util-body-length-browser": "^4.2.0",
|
|
19138
19152
|
"@smithy/util-body-length-node": "^4.2.1",
|
|
19139
19153
|
"@smithy/util-defaults-mode-browser": "^4.3.5",
|
|
19140
|
-
"@smithy/util-defaults-mode-node": "^4.2.
|
|
19154
|
+
"@smithy/util-defaults-mode-node": "^4.2.8",
|
|
19141
19155
|
"@smithy/util-endpoints": "^3.2.4",
|
|
19142
19156
|
"@smithy/util-middleware": "^4.2.4",
|
|
19143
19157
|
"@smithy/util-retry": "^4.2.4",
|
|
@@ -26431,6 +26445,162 @@ var init_BytePairEncodingCore = __esm({
|
|
|
26431
26445
|
}
|
|
26432
26446
|
});
|
|
26433
26447
|
|
|
26448
|
+
// node_modules/gpt-tokenizer/esm/functionCalling.js
|
|
26449
|
+
function countMessageTokens(message, countStringTokens) {
|
|
26450
|
+
let tokens = 0;
|
|
26451
|
+
if (message.role) {
|
|
26452
|
+
tokens += countStringTokens(message.role);
|
|
26453
|
+
}
|
|
26454
|
+
if (message.content) {
|
|
26455
|
+
tokens += countStringTokens(message.content);
|
|
26456
|
+
}
|
|
26457
|
+
if (message.name) {
|
|
26458
|
+
tokens += countStringTokens(message.name) + MESSAGE_NAME_TOKEN_OVERHEAD;
|
|
26459
|
+
}
|
|
26460
|
+
if (message.function_call) {
|
|
26461
|
+
const { name: name14, arguments: args } = message.function_call;
|
|
26462
|
+
if (name14) {
|
|
26463
|
+
tokens += countStringTokens(name14);
|
|
26464
|
+
}
|
|
26465
|
+
if (args) {
|
|
26466
|
+
tokens += countStringTokens(args);
|
|
26467
|
+
}
|
|
26468
|
+
tokens += FUNCTION_CALL_METADATA_TOKEN_OVERHEAD;
|
|
26469
|
+
}
|
|
26470
|
+
tokens += MESSAGE_TOKEN_OVERHEAD;
|
|
26471
|
+
if (message.role === "function") {
|
|
26472
|
+
tokens -= FUNCTION_ROLE_TOKEN_DISCOUNT;
|
|
26473
|
+
}
|
|
26474
|
+
return tokens;
|
|
26475
|
+
}
|
|
26476
|
+
function formatObjectProperties(obj, indent, formatType) {
|
|
26477
|
+
if (!obj.properties) {
|
|
26478
|
+
return "";
|
|
26479
|
+
}
|
|
26480
|
+
const lines = [];
|
|
26481
|
+
const requiredParams = new Set(obj.required ?? []);
|
|
26482
|
+
const indentString = " ".repeat(indent);
|
|
26483
|
+
for (const [name14, param] of Object.entries(obj.properties)) {
|
|
26484
|
+
if (param.description && indent < 2) {
|
|
26485
|
+
lines.push(`${indentString}// ${param.description}`);
|
|
26486
|
+
}
|
|
26487
|
+
const isRequired = requiredParams.has(name14);
|
|
26488
|
+
const formattedType = formatType(param, indent);
|
|
26489
|
+
lines.push(`${indentString}${name14}${isRequired ? "" : "?"}: ${formattedType},`);
|
|
26490
|
+
}
|
|
26491
|
+
return lines.join("\n");
|
|
26492
|
+
}
|
|
26493
|
+
function formatFunctionType(param, indent) {
|
|
26494
|
+
switch (param.type) {
|
|
26495
|
+
case "string":
|
|
26496
|
+
return param.enum?.map((value) => JSON.stringify(value)).join(" | ") ?? "string";
|
|
26497
|
+
case "integer":
|
|
26498
|
+
case "number":
|
|
26499
|
+
return param.enum?.map((value) => `${value}`).join(" | ") ?? "number";
|
|
26500
|
+
case "boolean":
|
|
26501
|
+
return "boolean";
|
|
26502
|
+
case "null":
|
|
26503
|
+
return "null";
|
|
26504
|
+
case "array":
|
|
26505
|
+
return param.items ? `${formatFunctionType(param.items, indent)}[]` : "any[]";
|
|
26506
|
+
case "object": {
|
|
26507
|
+
const inner = formatObjectProperties(param, indent + 2, formatFunctionType);
|
|
26508
|
+
const closingIndent = " ".repeat(indent);
|
|
26509
|
+
return `{
|
|
26510
|
+
${inner}
|
|
26511
|
+
${closingIndent}}`;
|
|
26512
|
+
}
|
|
26513
|
+
default:
|
|
26514
|
+
return "any";
|
|
26515
|
+
}
|
|
26516
|
+
}
|
|
26517
|
+
function formatFunctionDefinitions(functions) {
|
|
26518
|
+
const lines = ["namespace functions {", ""];
|
|
26519
|
+
for (const fn of functions) {
|
|
26520
|
+
if (fn.description) {
|
|
26521
|
+
lines.push(`// ${fn.description}`);
|
|
26522
|
+
}
|
|
26523
|
+
const { parameters } = fn;
|
|
26524
|
+
const properties = parameters?.properties;
|
|
26525
|
+
if (!parameters || !properties || Object.keys(properties).length === 0) {
|
|
26526
|
+
lines.push(`type ${fn.name} = () => any;`);
|
|
26527
|
+
} else {
|
|
26528
|
+
lines.push(`type ${fn.name} = (_: {`);
|
|
26529
|
+
const formattedProperties = formatObjectProperties(parameters, 0, formatFunctionType);
|
|
26530
|
+
if (formattedProperties.length > 0) {
|
|
26531
|
+
lines.push(formattedProperties);
|
|
26532
|
+
}
|
|
26533
|
+
lines.push("}) => any;");
|
|
26534
|
+
}
|
|
26535
|
+
lines.push("");
|
|
26536
|
+
}
|
|
26537
|
+
lines.push("} // namespace functions");
|
|
26538
|
+
return lines.join("\n");
|
|
26539
|
+
}
|
|
26540
|
+
function estimateTokensInFunctions(functions, countStringTokens) {
|
|
26541
|
+
const formatted = formatFunctionDefinitions(functions);
|
|
26542
|
+
let tokens = countStringTokens(formatted);
|
|
26543
|
+
tokens += FUNCTION_DEFINITION_TOKEN_OVERHEAD;
|
|
26544
|
+
return tokens;
|
|
26545
|
+
}
|
|
26546
|
+
function padSystemMessage(message, hasFunctions, isSystemPadded) {
|
|
26547
|
+
if (!hasFunctions || isSystemPadded || message.role !== "system") {
|
|
26548
|
+
return message;
|
|
26549
|
+
}
|
|
26550
|
+
if (!message.content || message.content.endsWith(NEWLINE)) {
|
|
26551
|
+
return message;
|
|
26552
|
+
}
|
|
26553
|
+
return {
|
|
26554
|
+
...message,
|
|
26555
|
+
content: `${message.content}${NEWLINE}`
|
|
26556
|
+
};
|
|
26557
|
+
}
|
|
26558
|
+
function computeChatCompletionTokenCount(request, countStringTokens) {
|
|
26559
|
+
const { messages, functions, function_call: functionCall } = request;
|
|
26560
|
+
const hasFunctions = Boolean(functions && functions.length > 0);
|
|
26561
|
+
let paddedSystem = false;
|
|
26562
|
+
let total = 0;
|
|
26563
|
+
for (const message of messages) {
|
|
26564
|
+
const messageToCount = padSystemMessage(message, hasFunctions, paddedSystem);
|
|
26565
|
+
if (messageToCount !== message && message.role === "system") {
|
|
26566
|
+
paddedSystem = true;
|
|
26567
|
+
} else if (message.role === "system" && hasFunctions && !paddedSystem) {
|
|
26568
|
+
paddedSystem = true;
|
|
26569
|
+
}
|
|
26570
|
+
total += countMessageTokens(messageToCount, countStringTokens);
|
|
26571
|
+
}
|
|
26572
|
+
total += COMPLETION_REQUEST_TOKEN_OVERHEAD;
|
|
26573
|
+
if (hasFunctions && functions) {
|
|
26574
|
+
total += estimateTokensInFunctions(functions, countStringTokens);
|
|
26575
|
+
if (messages.some((message) => message.role === "system")) {
|
|
26576
|
+
total -= SYSTEM_FUNCTION_TOKEN_DEDUCTION;
|
|
26577
|
+
}
|
|
26578
|
+
}
|
|
26579
|
+
if (functionCall && functionCall !== "auto") {
|
|
26580
|
+
if (functionCall === "none") {
|
|
26581
|
+
total += FUNCTION_CALL_NONE_TOKEN_OVERHEAD;
|
|
26582
|
+
} else if (typeof functionCall === "object" && functionCall.name) {
|
|
26583
|
+
total += countStringTokens(functionCall.name) + FUNCTION_CALL_NAME_TOKEN_OVERHEAD;
|
|
26584
|
+
}
|
|
26585
|
+
}
|
|
26586
|
+
return total;
|
|
26587
|
+
}
|
|
26588
|
+
var MESSAGE_TOKEN_OVERHEAD, MESSAGE_NAME_TOKEN_OVERHEAD, FUNCTION_ROLE_TOKEN_DISCOUNT, FUNCTION_CALL_METADATA_TOKEN_OVERHEAD, FUNCTION_DEFINITION_TOKEN_OVERHEAD, COMPLETION_REQUEST_TOKEN_OVERHEAD, FUNCTION_CALL_NAME_TOKEN_OVERHEAD, FUNCTION_CALL_NONE_TOKEN_OVERHEAD, SYSTEM_FUNCTION_TOKEN_DEDUCTION, NEWLINE;
|
|
26589
|
+
var init_functionCalling = __esm({
|
|
26590
|
+
"node_modules/gpt-tokenizer/esm/functionCalling.js"() {
|
|
26591
|
+
MESSAGE_TOKEN_OVERHEAD = 3;
|
|
26592
|
+
MESSAGE_NAME_TOKEN_OVERHEAD = 1;
|
|
26593
|
+
FUNCTION_ROLE_TOKEN_DISCOUNT = 2;
|
|
26594
|
+
FUNCTION_CALL_METADATA_TOKEN_OVERHEAD = 3;
|
|
26595
|
+
FUNCTION_DEFINITION_TOKEN_OVERHEAD = 9;
|
|
26596
|
+
COMPLETION_REQUEST_TOKEN_OVERHEAD = 3;
|
|
26597
|
+
FUNCTION_CALL_NAME_TOKEN_OVERHEAD = 4;
|
|
26598
|
+
FUNCTION_CALL_NONE_TOKEN_OVERHEAD = 1;
|
|
26599
|
+
SYSTEM_FUNCTION_TOKEN_DEDUCTION = 4;
|
|
26600
|
+
NEWLINE = "\n";
|
|
26601
|
+
}
|
|
26602
|
+
});
|
|
26603
|
+
|
|
26434
26604
|
// node_modules/gpt-tokenizer/esm/modelsChatEnabled.gen.js
|
|
26435
26605
|
var chatEnabledModels;
|
|
26436
26606
|
var init_modelsChatEnabled_gen = __esm({
|
|
@@ -26444,11 +26614,12 @@ var modelsMap_exports = {};
|
|
|
26444
26614
|
__export(modelsMap_exports, {
|
|
26445
26615
|
cl100k_base: () => cl100k_base,
|
|
26446
26616
|
o200k_base: () => o200k_base,
|
|
26617
|
+
o200k_harmony: () => o200k_harmony,
|
|
26447
26618
|
p50k_base: () => p50k_base,
|
|
26448
26619
|
p50k_edit: () => p50k_edit,
|
|
26449
26620
|
r50k_base: () => r50k_base
|
|
26450
26621
|
});
|
|
26451
|
-
var p50k_base, r50k_base, p50k_edit, cl100k_base, o200k_base;
|
|
26622
|
+
var p50k_base, r50k_base, p50k_edit, cl100k_base, o200k_base, o200k_harmony;
|
|
26452
26623
|
var init_modelsMap = __esm({
|
|
26453
26624
|
"node_modules/gpt-tokenizer/esm/modelsMap.js"() {
|
|
26454
26625
|
p50k_base = [
|
|
@@ -26523,11 +26694,15 @@ var init_modelsMap = __esm({
|
|
|
26523
26694
|
"davinci-002"
|
|
26524
26695
|
];
|
|
26525
26696
|
o200k_base = [];
|
|
26697
|
+
o200k_harmony = [
|
|
26698
|
+
"gpt-oss-20b",
|
|
26699
|
+
"gpt-oss-120b"
|
|
26700
|
+
];
|
|
26526
26701
|
}
|
|
26527
26702
|
});
|
|
26528
26703
|
|
|
26529
26704
|
// node_modules/gpt-tokenizer/esm/specialTokens.js
|
|
26530
|
-
var EndOfText, FimPrefix, FimMiddle, FimSuffix, ImStart, ImEnd, ImSep, EndOfPrompt;
|
|
26705
|
+
var EndOfText, FimPrefix, FimMiddle, FimSuffix, ImStart, ImEnd, ImSep, EndOfPrompt, HarmonyStartOfText, HarmonyStart, HarmonyEnd, HarmonyMessage, HarmonyChannel, HarmonyReturn, HarmonyConstrain, HarmonyCall;
|
|
26531
26706
|
var init_specialTokens = __esm({
|
|
26532
26707
|
"node_modules/gpt-tokenizer/esm/specialTokens.js"() {
|
|
26533
26708
|
EndOfText = "<|endoftext|>";
|
|
@@ -26538,6 +26713,14 @@ var init_specialTokens = __esm({
|
|
|
26538
26713
|
ImEnd = "<|im_end|>";
|
|
26539
26714
|
ImSep = "<|im_sep|>";
|
|
26540
26715
|
EndOfPrompt = "<|endofprompt|>";
|
|
26716
|
+
HarmonyStartOfText = "<|startoftext|>";
|
|
26717
|
+
HarmonyStart = "<|start|>";
|
|
26718
|
+
HarmonyEnd = "<|end|>";
|
|
26719
|
+
HarmonyMessage = "<|message|>";
|
|
26720
|
+
HarmonyChannel = "<|channel|>";
|
|
26721
|
+
HarmonyReturn = "<|return|>";
|
|
26722
|
+
HarmonyConstrain = "<|constrain|>";
|
|
26723
|
+
HarmonyCall = "<|call|>";
|
|
26541
26724
|
}
|
|
26542
26725
|
});
|
|
26543
26726
|
|
|
@@ -26564,11 +26747,16 @@ var init_mapping = __esm({
|
|
|
26564
26747
|
});
|
|
26565
26748
|
|
|
26566
26749
|
// node_modules/gpt-tokenizer/esm/encodingParams/constants.js
|
|
26567
|
-
var R50K_TOKEN_SPLIT_REGEX,
|
|
26750
|
+
var R50K_TOKEN_SPLIT_REGEX, CONTRACTION_SUFFIX_PATTERN, OPTIONAL_CONTRACTION_SUFFIX, CL100K_TOKEN_SPLIT_PATTERN, CL100K_TOKEN_SPLIT_REGEX, O200K_TOKEN_SPLIT_PATTERN, O200K_TOKEN_SPLIT_REGEX;
|
|
26568
26751
|
var init_constants2 = __esm({
|
|
26569
26752
|
"node_modules/gpt-tokenizer/esm/encodingParams/constants.js"() {
|
|
26570
26753
|
R50K_TOKEN_SPLIT_REGEX = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
|
|
26571
|
-
|
|
26754
|
+
CONTRACTION_SUFFIX_PATTERN = String.raw`'(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`;
|
|
26755
|
+
OPTIONAL_CONTRACTION_SUFFIX = String.raw`(?:${CONTRACTION_SUFFIX_PATTERN})?`;
|
|
26756
|
+
CL100K_TOKEN_SPLIT_PATTERN = String.raw`${CONTRACTION_SUFFIX_PATTERN}|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s+$|\s*[\r\n]|\s+(?!\S)|\s`;
|
|
26757
|
+
CL100K_TOKEN_SPLIT_REGEX = new RegExp(CL100K_TOKEN_SPLIT_PATTERN, "gu");
|
|
26758
|
+
O200K_TOKEN_SPLIT_PATTERN = String.raw`[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+${OPTIONAL_CONTRACTION_SUFFIX}|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*${OPTIONAL_CONTRACTION_SUFFIX}|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+`;
|
|
26759
|
+
O200K_TOKEN_SPLIT_REGEX = new RegExp(O200K_TOKEN_SPLIT_PATTERN, "gu");
|
|
26572
26760
|
}
|
|
26573
26761
|
});
|
|
26574
26762
|
|
|
@@ -26585,7 +26773,7 @@ function Cl100KBase(bytePairRankDecoder) {
|
|
|
26585
26773
|
[EndOfPrompt, 100276]
|
|
26586
26774
|
]);
|
|
26587
26775
|
return {
|
|
26588
|
-
tokenSplitRegex:
|
|
26776
|
+
tokenSplitRegex: CL100K_TOKEN_SPLIT_REGEX,
|
|
26589
26777
|
bytePairRankDecoder,
|
|
26590
26778
|
specialTokensEncoder: specialTokenMapping
|
|
26591
26779
|
};
|
|
@@ -26599,26 +26787,69 @@ var init_cl100k_base = __esm({
|
|
|
26599
26787
|
|
|
26600
26788
|
// node_modules/gpt-tokenizer/esm/encodingParams/o200k_base.js
|
|
26601
26789
|
function O200KBase(bytePairRankDecoder) {
|
|
26602
|
-
const specialTokenMapping = /* @__PURE__ */ new Map([
|
|
26603
|
-
[EndOfText, 199999],
|
|
26604
|
-
[FimPrefix, 2e5],
|
|
26605
|
-
[FimMiddle, 200001],
|
|
26606
|
-
[FimSuffix, 200002],
|
|
26607
|
-
[ImStart, 200003],
|
|
26608
|
-
[ImEnd, 200004],
|
|
26609
|
-
[ImSep, 200005],
|
|
26610
|
-
[EndOfPrompt, 200006]
|
|
26611
|
-
]);
|
|
26612
26790
|
return {
|
|
26613
|
-
tokenSplitRegex:
|
|
26791
|
+
tokenSplitRegex: O200K_TOKEN_SPLIT_REGEX,
|
|
26614
26792
|
bytePairRankDecoder,
|
|
26615
|
-
specialTokensEncoder:
|
|
26793
|
+
specialTokensEncoder: createO200KSpecialTokenMap()
|
|
26616
26794
|
};
|
|
26617
26795
|
}
|
|
26796
|
+
var O200K_BASE_SPECIAL_TOKEN_ENTRIES, createO200KSpecialTokenMap;
|
|
26618
26797
|
var init_o200k_base2 = __esm({
|
|
26619
26798
|
"node_modules/gpt-tokenizer/esm/encodingParams/o200k_base.js"() {
|
|
26620
26799
|
init_specialTokens();
|
|
26621
26800
|
init_constants2();
|
|
26801
|
+
O200K_BASE_SPECIAL_TOKEN_ENTRIES = [
|
|
26802
|
+
[EndOfText, 199999],
|
|
26803
|
+
[FimPrefix, 2e5],
|
|
26804
|
+
[FimMiddle, 200001],
|
|
26805
|
+
[FimSuffix, 200002],
|
|
26806
|
+
[ImStart, 200003],
|
|
26807
|
+
[ImEnd, 200004],
|
|
26808
|
+
[ImSep, 200005],
|
|
26809
|
+
[EndOfPrompt, 200006]
|
|
26810
|
+
];
|
|
26811
|
+
createO200KSpecialTokenMap = () => new Map(O200K_BASE_SPECIAL_TOKEN_ENTRIES);
|
|
26812
|
+
}
|
|
26813
|
+
});
|
|
26814
|
+
|
|
26815
|
+
// node_modules/gpt-tokenizer/esm/encodingParams/o200k_harmony.js
|
|
26816
|
+
function O200KHarmony(bytePairRankDecoder) {
|
|
26817
|
+
const specialTokensEncoder = new Map(STATIC_SPECIAL_TOKEN_ENTRIES);
|
|
26818
|
+
for (let tokenId = RESERVED_TOKEN_RANGE_START; tokenId < RESERVED_TOKEN_RANGE_END; tokenId += 1) {
|
|
26819
|
+
specialTokensEncoder.set(`<|reserved_${tokenId}|>`, tokenId);
|
|
26820
|
+
}
|
|
26821
|
+
specialTokensEncoder.set(EndOfPrompt, 200018);
|
|
26822
|
+
return {
|
|
26823
|
+
tokenSplitRegex: O200K_TOKEN_SPLIT_REGEX,
|
|
26824
|
+
bytePairRankDecoder,
|
|
26825
|
+
specialTokensEncoder,
|
|
26826
|
+
chatFormatter: "harmony"
|
|
26827
|
+
};
|
|
26828
|
+
}
|
|
26829
|
+
var RESERVED_TOKEN_RANGE_START, RESERVED_TOKEN_RANGE_END, STATIC_SPECIAL_TOKEN_ENTRIES;
|
|
26830
|
+
var init_o200k_harmony = __esm({
|
|
26831
|
+
"node_modules/gpt-tokenizer/esm/encodingParams/o200k_harmony.js"() {
|
|
26832
|
+
init_specialTokens();
|
|
26833
|
+
init_constants2();
|
|
26834
|
+
RESERVED_TOKEN_RANGE_START = 200013;
|
|
26835
|
+
RESERVED_TOKEN_RANGE_END = 201088;
|
|
26836
|
+
STATIC_SPECIAL_TOKEN_ENTRIES = [
|
|
26837
|
+
[HarmonyStartOfText, 199998],
|
|
26838
|
+
[EndOfText, 199999],
|
|
26839
|
+
["<|reserved_200000|>", 2e5],
|
|
26840
|
+
["<|reserved_200001|>", 200001],
|
|
26841
|
+
[HarmonyReturn, 200002],
|
|
26842
|
+
[HarmonyConstrain, 200003],
|
|
26843
|
+
["<|reserved_200004|>", 200004],
|
|
26844
|
+
[HarmonyChannel, 200005],
|
|
26845
|
+
[HarmonyStart, 200006],
|
|
26846
|
+
[HarmonyEnd, 200007],
|
|
26847
|
+
[HarmonyMessage, 200008],
|
|
26848
|
+
["<|reserved_200009|>", 200009],
|
|
26849
|
+
["<|reserved_200010|>", 200010],
|
|
26850
|
+
["<|reserved_200011|>", 200011],
|
|
26851
|
+
[HarmonyCall, 200012]
|
|
26852
|
+
];
|
|
26622
26853
|
}
|
|
26623
26854
|
});
|
|
26624
26855
|
|
|
@@ -26692,6 +26923,8 @@ function getEncodingParams(encodingName, getMergeableRanks) {
|
|
|
26692
26923
|
return Cl100KBase(mergeableBytePairRanks);
|
|
26693
26924
|
case "o200k_base":
|
|
26694
26925
|
return O200KBase(mergeableBytePairRanks);
|
|
26926
|
+
case "o200k_harmony":
|
|
26927
|
+
return O200KHarmony(mergeableBytePairRanks);
|
|
26695
26928
|
default:
|
|
26696
26929
|
throw new Error(`Unknown encoding name: ${encodingName}`);
|
|
26697
26930
|
}
|
|
@@ -26700,6 +26933,7 @@ var init_modelParams = __esm({
|
|
|
26700
26933
|
"node_modules/gpt-tokenizer/esm/modelParams.js"() {
|
|
26701
26934
|
init_cl100k_base();
|
|
26702
26935
|
init_o200k_base2();
|
|
26936
|
+
init_o200k_harmony();
|
|
26703
26937
|
init_p50k_base();
|
|
26704
26938
|
init_p50k_edit();
|
|
26705
26939
|
init_r50k_base();
|
|
@@ -26712,6 +26946,7 @@ var init_GptEncoding = __esm({
|
|
|
26712
26946
|
"node_modules/gpt-tokenizer/esm/GptEncoding.js"() {
|
|
26713
26947
|
init_BytePairEncodingCore();
|
|
26714
26948
|
init_constants();
|
|
26949
|
+
init_functionCalling();
|
|
26715
26950
|
init_mapping();
|
|
26716
26951
|
init_modelParams();
|
|
26717
26952
|
init_specialTokens();
|
|
@@ -26730,8 +26965,10 @@ var init_GptEncoding = __esm({
|
|
|
26730
26965
|
specialTokensSet;
|
|
26731
26966
|
allSpecialTokenRegex;
|
|
26732
26967
|
defaultSpecialTokenConfig;
|
|
26968
|
+
chatFormatter;
|
|
26969
|
+
countChatCompletionTokens;
|
|
26733
26970
|
vocabularySize;
|
|
26734
|
-
constructor({ bytePairRankDecoder: mergeableBytePairRanks, specialTokensEncoder, expectedVocabularySize, modelName, modelSpec, ...rest }) {
|
|
26971
|
+
constructor({ bytePairRankDecoder: mergeableBytePairRanks, specialTokensEncoder, expectedVocabularySize, modelName, modelSpec, chatFormatter, ...rest }) {
|
|
26735
26972
|
this.specialTokensEncoder = specialTokensEncoder;
|
|
26736
26973
|
this.specialTokensSet = new Set(this.specialTokensEncoder.keys());
|
|
26737
26974
|
this.allSpecialTokenRegex = getSpecialTokenRegex(this.specialTokensSet);
|
|
@@ -26764,8 +27001,69 @@ var init_GptEncoding = __esm({
|
|
|
26764
27001
|
this.setMergeCacheSize = this.setMergeCacheSize.bind(this);
|
|
26765
27002
|
this.clearMergeCache = this.clearMergeCache.bind(this);
|
|
26766
27003
|
this.estimateCost = this.estimateCost.bind(this);
|
|
27004
|
+
if (modelSpec?.supported_features?.includes("function_calling")) {
|
|
27005
|
+
this.countChatCompletionTokens = this.countChatCompletionTokensInternal.bind(this);
|
|
27006
|
+
}
|
|
26767
27007
|
this.modelName = modelName;
|
|
26768
27008
|
this.modelSpec = modelSpec;
|
|
27009
|
+
this.chatFormatter = chatFormatter ?? "chatml";
|
|
27010
|
+
}
|
|
27011
|
+
*encodeHarmonyChatGenerator(chat, encodeOptions) {
|
|
27012
|
+
const harmonyStart = this.specialTokensEncoder.get(HarmonyStart);
|
|
27013
|
+
const harmonyMessage = this.specialTokensEncoder.get(HarmonyMessage);
|
|
27014
|
+
const harmonyEnd = this.specialTokensEncoder.get(HarmonyEnd);
|
|
27015
|
+
const harmonyReturn = this.specialTokensEncoder.get(HarmonyReturn);
|
|
27016
|
+
const harmonyCall = this.specialTokensEncoder.get(HarmonyCall);
|
|
27017
|
+
const harmonyChannel = this.specialTokensEncoder.get(HarmonyChannel);
|
|
27018
|
+
const harmonyConstrain = this.specialTokensEncoder.get(HarmonyConstrain);
|
|
27019
|
+
if (harmonyStart === void 0 || harmonyMessage === void 0 || harmonyEnd === void 0 || harmonyReturn === void 0 || harmonyCall === void 0 || harmonyChannel === void 0 || harmonyConstrain === void 0) {
|
|
27020
|
+
throw new Error("Harmony chat format requires dedicated special tokens.");
|
|
27021
|
+
}
|
|
27022
|
+
const encodeHeaderText = (text) => text.length > 0 ? this.encode(text) : [];
|
|
27023
|
+
const resolveTerminatorToken = (terminator) => {
|
|
27024
|
+
switch (terminator) {
|
|
27025
|
+
case "<|return|>":
|
|
27026
|
+
return harmonyReturn;
|
|
27027
|
+
case "<|call|>":
|
|
27028
|
+
return harmonyCall;
|
|
27029
|
+
// eslint-disable-next-line unicorn/no-useless-switch-case
|
|
27030
|
+
case "<|end|>":
|
|
27031
|
+
default:
|
|
27032
|
+
return harmonyEnd;
|
|
27033
|
+
}
|
|
27034
|
+
};
|
|
27035
|
+
for (const message of chat) {
|
|
27036
|
+
if (message.content === void 0) {
|
|
27037
|
+
throw new Error("Content must be defined for all messages.");
|
|
27038
|
+
}
|
|
27039
|
+
const roleOrName = message.name ?? message.role ?? "user";
|
|
27040
|
+
yield [harmonyStart];
|
|
27041
|
+
yield encodeHeaderText(roleOrName);
|
|
27042
|
+
const recipientInRole = message.recipient && (message.recipientPlacement === "role" || !message.channel);
|
|
27043
|
+
const recipientInChannel = message.recipient && !recipientInRole;
|
|
27044
|
+
if (recipientInRole) {
|
|
27045
|
+
yield encodeHeaderText(` to=${message.recipient}`);
|
|
27046
|
+
}
|
|
27047
|
+
if (message.channel) {
|
|
27048
|
+
yield [harmonyChannel];
|
|
27049
|
+
yield encodeHeaderText(message.channel);
|
|
27050
|
+
if (recipientInChannel) {
|
|
27051
|
+
yield encodeHeaderText(` to=${message.recipient}`);
|
|
27052
|
+
}
|
|
27053
|
+
}
|
|
27054
|
+
if (message.constraint) {
|
|
27055
|
+
yield [harmonyConstrain];
|
|
27056
|
+
yield encodeHeaderText(message.constraint);
|
|
27057
|
+
}
|
|
27058
|
+
yield [harmonyMessage];
|
|
27059
|
+
yield* this.encodeGenerator(message.content, encodeOptions);
|
|
27060
|
+
yield [resolveTerminatorToken(message.terminator)];
|
|
27061
|
+
}
|
|
27062
|
+
const assistantPrime = encodeOptions?.primeWithAssistantResponse ?? "assistant";
|
|
27063
|
+
if (assistantPrime.length > 0) {
|
|
27064
|
+
yield [harmonyStart];
|
|
27065
|
+
yield encodeHeaderText(assistantPrime);
|
|
27066
|
+
}
|
|
26769
27067
|
}
|
|
26770
27068
|
static getEncodingApi(encodingName, getMergeableRanks) {
|
|
26771
27069
|
const modelParams = getEncodingParams(encodingName, getMergeableRanks);
|
|
@@ -26838,9 +27136,16 @@ var init_GptEncoding = __esm({
|
|
|
26838
27136
|
throw new Error("Model name must be provided either during initialization or passed in to the method.");
|
|
26839
27137
|
}
|
|
26840
27138
|
const params = chatModelParams[model];
|
|
27139
|
+
if (!params) {
|
|
27140
|
+
throw new Error(`Model '${model}' does not support chat.`);
|
|
27141
|
+
}
|
|
27142
|
+
if (this.chatFormatter === "harmony") {
|
|
27143
|
+
yield* this.encodeHarmonyChatGenerator(chat, encodeOptions);
|
|
27144
|
+
return;
|
|
27145
|
+
}
|
|
26841
27146
|
const chatStartToken = this.specialTokensEncoder.get(ImStart);
|
|
26842
27147
|
const chatEndToken = this.specialTokensEncoder.get(ImEnd);
|
|
26843
|
-
if (
|
|
27148
|
+
if (chatStartToken === void 0 || chatEndToken === void 0) {
|
|
26844
27149
|
throw new Error(`Model '${model}' does not support chat.`);
|
|
26845
27150
|
}
|
|
26846
27151
|
const allowedSpecial = /* @__PURE__ */ new Set([ImSep]);
|
|
@@ -26863,8 +27168,11 @@ var init_GptEncoding = __esm({
|
|
|
26863
27168
|
yield [chatEndToken];
|
|
26864
27169
|
yield encodedMessageSeparator;
|
|
26865
27170
|
}
|
|
26866
|
-
|
|
26867
|
-
|
|
27171
|
+
const assistantPrime = encodeOptions?.primeWithAssistantResponse ?? "assistant";
|
|
27172
|
+
if (assistantPrime.length > 0) {
|
|
27173
|
+
yield [chatStartToken];
|
|
27174
|
+
yield* this.encodeGenerator(assistantPrime, encodeOptions);
|
|
27175
|
+
}
|
|
26868
27176
|
if (encodedRoleSeparator.length > 0) {
|
|
26869
27177
|
yield encodedRoleSeparator;
|
|
26870
27178
|
}
|
|
@@ -26922,6 +27230,15 @@ var init_GptEncoding = __esm({
|
|
|
26922
27230
|
}
|
|
26923
27231
|
return count;
|
|
26924
27232
|
}
|
|
27233
|
+
countStringTokens(text) {
|
|
27234
|
+
if (!text) {
|
|
27235
|
+
return 0;
|
|
27236
|
+
}
|
|
27237
|
+
return this.bytePairEncodingCoreProcessor.countNative(text);
|
|
27238
|
+
}
|
|
27239
|
+
countChatCompletionTokensInternal(request) {
|
|
27240
|
+
return computeChatCompletionTokenCount(request, (text) => this.countStringTokens(text));
|
|
27241
|
+
}
|
|
26925
27242
|
setMergeCacheSize(size) {
|
|
26926
27243
|
this.bytePairEncodingCoreProcessor.setMergeCacheSize(size);
|
|
26927
27244
|
}
|
|
@@ -33362,7 +33679,15 @@ function parseXmlToolCall(xmlString, validTools = DEFAULT_VALID_TOOLS) {
|
|
|
33362
33679
|
if (openIndex === -1) {
|
|
33363
33680
|
continue;
|
|
33364
33681
|
}
|
|
33365
|
-
let closeIndex
|
|
33682
|
+
let closeIndex;
|
|
33683
|
+
if (toolName === "attempt_completion") {
|
|
33684
|
+
closeIndex = xmlString.lastIndexOf(closeTag);
|
|
33685
|
+
if (closeIndex !== -1 && closeIndex <= openIndex + openTag.length) {
|
|
33686
|
+
closeIndex = -1;
|
|
33687
|
+
}
|
|
33688
|
+
} else {
|
|
33689
|
+
closeIndex = xmlString.indexOf(closeTag, openIndex + openTag.length);
|
|
33690
|
+
}
|
|
33366
33691
|
let hasClosingTag = closeIndex !== -1;
|
|
33367
33692
|
if (closeIndex === -1) {
|
|
33368
33693
|
closeIndex = xmlString.length;
|
|
@@ -42814,10 +43139,22 @@ function extractThinkingContent(xmlString) {
|
|
|
42814
43139
|
return thinkingMatch ? thinkingMatch[1].trim() : null;
|
|
42815
43140
|
}
|
|
42816
43141
|
function checkAttemptCompleteRecovery(cleanedXmlString, validTools = []) {
|
|
42817
|
-
const
|
|
42818
|
-
if (
|
|
42819
|
-
const
|
|
42820
|
-
const
|
|
43142
|
+
const openTagIndex = cleanedXmlString.indexOf("<attempt_completion>");
|
|
43143
|
+
if (openTagIndex !== -1) {
|
|
43144
|
+
const afterOpenTag = cleanedXmlString.substring(openTagIndex + "<attempt_completion>".length);
|
|
43145
|
+
const closeTagIndex = cleanedXmlString.lastIndexOf("</attempt_completion>");
|
|
43146
|
+
let content;
|
|
43147
|
+
let hasClosingTag = false;
|
|
43148
|
+
if (closeTagIndex !== -1 && closeTagIndex >= openTagIndex + "<attempt_completion>".length) {
|
|
43149
|
+
content = cleanedXmlString.substring(
|
|
43150
|
+
openTagIndex + "<attempt_completion>".length,
|
|
43151
|
+
closeTagIndex
|
|
43152
|
+
).trim();
|
|
43153
|
+
hasClosingTag = true;
|
|
43154
|
+
} else {
|
|
43155
|
+
content = afterOpenTag.trim();
|
|
43156
|
+
hasClosingTag = false;
|
|
43157
|
+
}
|
|
42821
43158
|
if (content) {
|
|
42822
43159
|
return {
|
|
42823
43160
|
toolName: "attempt_completion",
|
|
@@ -82322,6 +82659,170 @@ var init_FallbackManager = __esm({
|
|
|
82322
82659
|
}
|
|
82323
82660
|
});
|
|
82324
82661
|
|
|
82662
|
+
// src/agent/contextCompactor.js
|
|
82663
|
+
var contextCompactor_exports = {};
|
|
82664
|
+
__export(contextCompactor_exports, {
|
|
82665
|
+
calculateCompactionStats: () => calculateCompactionStats,
|
|
82666
|
+
compactMessages: () => compactMessages,
|
|
82667
|
+
handleContextLimitError: () => handleContextLimitError,
|
|
82668
|
+
identifyMessageSegments: () => identifyMessageSegments,
|
|
82669
|
+
isContextLimitError: () => isContextLimitError
|
|
82670
|
+
});
|
|
82671
|
+
function isContextLimitError(error2) {
|
|
82672
|
+
if (!error2) return false;
|
|
82673
|
+
const errorMessage = (typeof error2 === "string" ? error2 : error2?.message || "").toLowerCase();
|
|
82674
|
+
const errorString = error2.toString().toLowerCase();
|
|
82675
|
+
for (const pattern of CONTEXT_LIMIT_ERROR_PATTERNS) {
|
|
82676
|
+
const lowerPattern = pattern.toLowerCase();
|
|
82677
|
+
if (errorMessage.includes(lowerPattern) || errorString.includes(lowerPattern)) {
|
|
82678
|
+
return true;
|
|
82679
|
+
}
|
|
82680
|
+
}
|
|
82681
|
+
return false;
|
|
82682
|
+
}
|
|
82683
|
+
function identifyMessageSegments(messages) {
|
|
82684
|
+
const segments = [];
|
|
82685
|
+
let currentSegment = null;
|
|
82686
|
+
for (let i3 = 0; i3 < messages.length; i3++) {
|
|
82687
|
+
const msg = messages[i3];
|
|
82688
|
+
if (msg.role === "system") {
|
|
82689
|
+
continue;
|
|
82690
|
+
}
|
|
82691
|
+
if (msg.role === "user") {
|
|
82692
|
+
const content = typeof msg.content === "string" ? msg.content : "";
|
|
82693
|
+
const isToolResult = content.includes("<tool_result>");
|
|
82694
|
+
if (isToolResult && currentSegment) {
|
|
82695
|
+
currentSegment.finalIndex = i3;
|
|
82696
|
+
segments.push(currentSegment);
|
|
82697
|
+
currentSegment = null;
|
|
82698
|
+
} else {
|
|
82699
|
+
if (currentSegment) {
|
|
82700
|
+
segments.push(currentSegment);
|
|
82701
|
+
}
|
|
82702
|
+
currentSegment = {
|
|
82703
|
+
userIndex: i3,
|
|
82704
|
+
monologueIndices: [],
|
|
82705
|
+
finalIndex: null
|
|
82706
|
+
};
|
|
82707
|
+
}
|
|
82708
|
+
}
|
|
82709
|
+
if (msg.role === "assistant" && currentSegment) {
|
|
82710
|
+
const content = typeof msg.content === "string" ? msg.content : "";
|
|
82711
|
+
if (content.includes("<attempt_completion>") || content.includes("attempt_completion")) {
|
|
82712
|
+
currentSegment.monologueIndices.push(i3);
|
|
82713
|
+
currentSegment.finalIndex = i3;
|
|
82714
|
+
segments.push(currentSegment);
|
|
82715
|
+
currentSegment = null;
|
|
82716
|
+
} else {
|
|
82717
|
+
currentSegment.monologueIndices.push(i3);
|
|
82718
|
+
}
|
|
82719
|
+
}
|
|
82720
|
+
}
|
|
82721
|
+
if (currentSegment) {
|
|
82722
|
+
segments.push(currentSegment);
|
|
82723
|
+
}
|
|
82724
|
+
return segments;
|
|
82725
|
+
}
|
|
82726
|
+
function compactMessages(messages, options = {}) {
|
|
82727
|
+
const {
|
|
82728
|
+
keepLastSegment = true,
|
|
82729
|
+
minSegmentsToKeep = 1
|
|
82730
|
+
} = options;
|
|
82731
|
+
if (!messages || messages.length === 0) {
|
|
82732
|
+
return messages;
|
|
82733
|
+
}
|
|
82734
|
+
const segments = identifyMessageSegments(messages);
|
|
82735
|
+
if (segments.length === 0) {
|
|
82736
|
+
return messages;
|
|
82737
|
+
}
|
|
82738
|
+
const segmentsToPreserve = keepLastSegment ? Math.max(minSegmentsToKeep, 1) : minSegmentsToKeep;
|
|
82739
|
+
const compactableSegments = segments.slice(0, -segmentsToPreserve);
|
|
82740
|
+
const preservedSegments = segments.slice(-segmentsToPreserve);
|
|
82741
|
+
const indicesToKeep = /* @__PURE__ */ new Set();
|
|
82742
|
+
messages.forEach((msg, idx) => {
|
|
82743
|
+
if (msg.role === "system") {
|
|
82744
|
+
indicesToKeep.add(idx);
|
|
82745
|
+
}
|
|
82746
|
+
});
|
|
82747
|
+
compactableSegments.forEach((segment) => {
|
|
82748
|
+
indicesToKeep.add(segment.userIndex);
|
|
82749
|
+
if (segment.finalIndex !== null) {
|
|
82750
|
+
indicesToKeep.add(segment.finalIndex);
|
|
82751
|
+
}
|
|
82752
|
+
});
|
|
82753
|
+
preservedSegments.forEach((segment) => {
|
|
82754
|
+
indicesToKeep.add(segment.userIndex);
|
|
82755
|
+
segment.monologueIndices.forEach((idx) => indicesToKeep.add(idx));
|
|
82756
|
+
if (segment.finalIndex !== null) {
|
|
82757
|
+
indicesToKeep.add(segment.finalIndex);
|
|
82758
|
+
}
|
|
82759
|
+
});
|
|
82760
|
+
const compactedMessages = messages.filter((_2, idx) => indicesToKeep.has(idx));
|
|
82761
|
+
return compactedMessages;
|
|
82762
|
+
}
|
|
82763
|
+
function calculateCompactionStats(originalMessages, compactedMessages) {
|
|
82764
|
+
const originalCount = originalMessages.length;
|
|
82765
|
+
const compactedCount = compactedMessages.length;
|
|
82766
|
+
const removed = originalCount - compactedCount;
|
|
82767
|
+
const reductionPercent = originalCount > 0 ? (removed / originalCount * 100).toFixed(1) : 0;
|
|
82768
|
+
const estimateTokens = (msgs) => {
|
|
82769
|
+
return msgs.reduce((sum, msg) => {
|
|
82770
|
+
const content = typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content);
|
|
82771
|
+
return sum + Math.ceil(content.length / 4);
|
|
82772
|
+
}, 0);
|
|
82773
|
+
};
|
|
82774
|
+
const originalTokens = estimateTokens(originalMessages);
|
|
82775
|
+
const compactedTokens = estimateTokens(compactedMessages);
|
|
82776
|
+
const tokensSaved = originalTokens - compactedTokens;
|
|
82777
|
+
return {
|
|
82778
|
+
originalCount,
|
|
82779
|
+
compactedCount,
|
|
82780
|
+
removed,
|
|
82781
|
+
reductionPercent: parseFloat(reductionPercent),
|
|
82782
|
+
originalTokens,
|
|
82783
|
+
compactedTokens,
|
|
82784
|
+
tokensSaved
|
|
82785
|
+
};
|
|
82786
|
+
}
|
|
82787
|
+
function handleContextLimitError(error2, messages, options = {}) {
|
|
82788
|
+
if (!isContextLimitError(error2)) {
|
|
82789
|
+
return null;
|
|
82790
|
+
}
|
|
82791
|
+
const compactedMessages = compactMessages(messages, options);
|
|
82792
|
+
const stats = calculateCompactionStats(messages, compactedMessages);
|
|
82793
|
+
return {
|
|
82794
|
+
compacted: true,
|
|
82795
|
+
messages: compactedMessages,
|
|
82796
|
+
stats
|
|
82797
|
+
};
|
|
82798
|
+
}
|
|
82799
|
+
var CONTEXT_LIMIT_ERROR_PATTERNS;
|
|
82800
|
+
var init_contextCompactor = __esm({
|
|
82801
|
+
"src/agent/contextCompactor.js"() {
|
|
82802
|
+
"use strict";
|
|
82803
|
+
CONTEXT_LIMIT_ERROR_PATTERNS = [
|
|
82804
|
+
// Anthropic
|
|
82805
|
+
"context_length_exceeded",
|
|
82806
|
+
"prompt is too long",
|
|
82807
|
+
// OpenAI
|
|
82808
|
+
"maximum context length",
|
|
82809
|
+
"context length is",
|
|
82810
|
+
// Google/Gemini
|
|
82811
|
+
"input token count exceeds",
|
|
82812
|
+
"token limit exceeded",
|
|
82813
|
+
// Generic patterns
|
|
82814
|
+
"context window",
|
|
82815
|
+
"too many tokens",
|
|
82816
|
+
"token limit",
|
|
82817
|
+
"context limit",
|
|
82818
|
+
"exceed",
|
|
82819
|
+
// Catches "exceeds", "exceed maximum", etc.
|
|
82820
|
+
"over the limit",
|
|
82821
|
+
"maximum tokens"
|
|
82822
|
+
];
|
|
82823
|
+
}
|
|
82824
|
+
});
|
|
82825
|
+
|
|
82325
82826
|
// src/agent/ProbeAgent.js
|
|
82326
82827
|
var ProbeAgent_exports = {};
|
|
82327
82828
|
__export(ProbeAgent_exports, {
|
|
@@ -82356,6 +82857,7 @@ var init_ProbeAgent = __esm({
|
|
|
82356
82857
|
init_mcp();
|
|
82357
82858
|
init_RetryManager();
|
|
82358
82859
|
init_FallbackManager();
|
|
82860
|
+
init_contextCompactor();
|
|
82359
82861
|
import_dotenv2.default.config();
|
|
82360
82862
|
MAX_TOOL_ITERATIONS = (() => {
|
|
82361
82863
|
const val = parseInt(process.env.MAX_TOOL_ITERATIONS || "30", 10);
|
|
@@ -82391,6 +82893,8 @@ var init_ProbeAgent = __esm({
|
|
|
82391
82893
|
* @param {Array} [options.mcpServers] - Deprecated, use mcpConfig instead
|
|
82392
82894
|
* @param {Object} [options.storageAdapter] - Custom storage adapter for history management
|
|
82393
82895
|
* @param {Object} [options.hooks] - Hook callbacks for events (e.g., {'tool:start': callback})
|
|
82896
|
+
* @param {Array<string>|null} [options.allowedTools] - List of allowed tool names. Use ['*'] for all tools (default), [] or null for no tools (raw AI mode), or specific tool names like ['search', 'query', 'extract']. Supports exclusion with '!' prefix (e.g., ['*', '!bash'])
|
|
82897
|
+
* @param {boolean} [options.disableTools=false] - Convenience flag to disable all tools (equivalent to allowedTools: []). Takes precedence over allowedTools if set.
|
|
82394
82898
|
* @param {Object} [options.retry] - Retry configuration
|
|
82395
82899
|
* @param {number} [options.retry.maxRetries=3] - Maximum retry attempts per provider
|
|
82396
82900
|
* @param {number} [options.retry.initialDelay=1000] - Initial delay in ms
|
|
@@ -82424,6 +82928,8 @@ var init_ProbeAgent = __esm({
|
|
|
82424
82928
|
this.maxIterations = options.maxIterations || null;
|
|
82425
82929
|
this.disableMermaidValidation = !!options.disableMermaidValidation;
|
|
82426
82930
|
this.disableJsonValidation = !!options.disableJsonValidation;
|
|
82931
|
+
const effectiveAllowedTools = options.disableTools ? [] : options.allowedTools;
|
|
82932
|
+
this.allowedTools = this._parseAllowedTools(effectiveAllowedTools);
|
|
82427
82933
|
this.storageAdapter = options.storageAdapter || new InMemoryStorageAdapter();
|
|
82428
82934
|
this.hooks = new HookManager();
|
|
82429
82935
|
if (options.hooks) {
|
|
@@ -82467,6 +82973,61 @@ var init_ProbeAgent = __esm({
|
|
|
82467
82973
|
this.fallbackManager = null;
|
|
82468
82974
|
this.initializeModel();
|
|
82469
82975
|
}
|
|
82976
|
+
/**
|
|
82977
|
+
* Parse allowedTools configuration
|
|
82978
|
+
* @param {Array<string>|null|undefined} allowedTools - Tool filtering configuration
|
|
82979
|
+
* @returns {Object} Parsed configuration with isEnabled method
|
|
82980
|
+
* @private
|
|
82981
|
+
*/
|
|
82982
|
+
_parseAllowedTools(allowedTools) {
|
|
82983
|
+
const matchesPattern2 = (toolName, pattern) => {
|
|
82984
|
+
if (!pattern.includes("*")) {
|
|
82985
|
+
return toolName === pattern;
|
|
82986
|
+
}
|
|
82987
|
+
const regexPattern = pattern.replace(/\*/g, ".*");
|
|
82988
|
+
return new RegExp(`^${regexPattern}$`).test(toolName);
|
|
82989
|
+
};
|
|
82990
|
+
if (!allowedTools || Array.isArray(allowedTools) && allowedTools.includes("*")) {
|
|
82991
|
+
const exclusions = Array.isArray(allowedTools) ? allowedTools.filter((t3) => t3.startsWith("!")).map((t3) => t3.slice(1)) : [];
|
|
82992
|
+
return {
|
|
82993
|
+
mode: "all",
|
|
82994
|
+
exclusions,
|
|
82995
|
+
isEnabled: (toolName) => !exclusions.some((pattern) => matchesPattern2(toolName, pattern))
|
|
82996
|
+
};
|
|
82997
|
+
}
|
|
82998
|
+
if (Array.isArray(allowedTools) && allowedTools.length === 0) {
|
|
82999
|
+
return {
|
|
83000
|
+
mode: "none",
|
|
83001
|
+
isEnabled: () => false
|
|
83002
|
+
};
|
|
83003
|
+
}
|
|
83004
|
+
const allowedPatterns = allowedTools.filter((t3) => !t3.startsWith("!"));
|
|
83005
|
+
return {
|
|
83006
|
+
mode: "whitelist",
|
|
83007
|
+
allowed: allowedPatterns,
|
|
83008
|
+
isEnabled: (toolName) => allowedPatterns.some((pattern) => matchesPattern2(toolName, pattern))
|
|
83009
|
+
};
|
|
83010
|
+
}
|
|
83011
|
+
/**
|
|
83012
|
+
* Check if an MCP tool is allowed based on allowedTools configuration
|
|
83013
|
+
* Uses mcp__ prefix convention (like Claude Code)
|
|
83014
|
+
* @param {string} toolName - The MCP tool name (without mcp__ prefix)
|
|
83015
|
+
* @returns {boolean} - Whether the tool is allowed
|
|
83016
|
+
* @private
|
|
83017
|
+
*/
|
|
83018
|
+
_isMcpToolAllowed(toolName) {
|
|
83019
|
+
const mcpToolName = `mcp__${toolName}`;
|
|
83020
|
+
return this.allowedTools.isEnabled(mcpToolName) || this.allowedTools.isEnabled(toolName);
|
|
83021
|
+
}
|
|
83022
|
+
/**
|
|
83023
|
+
* Filter MCP tools based on allowedTools configuration
|
|
83024
|
+
* @param {string[]} mcpToolNames - Array of MCP tool names
|
|
83025
|
+
* @returns {string[]} - Filtered array of allowed MCP tool names
|
|
83026
|
+
* @private
|
|
83027
|
+
*/
|
|
83028
|
+
_filterMcpTools(mcpToolNames) {
|
|
83029
|
+
return mcpToolNames.filter((toolName) => this._isMcpToolAllowed(toolName));
|
|
83030
|
+
}
|
|
82470
83031
|
/**
|
|
82471
83032
|
* Initialize the agent asynchronously (must be called after constructor)
|
|
82472
83033
|
* This method initializes MCP and merges MCP tools into the tool list, and loads history from storage
|
|
@@ -82493,7 +83054,11 @@ var init_ProbeAgent = __esm({
|
|
|
82493
83054
|
if (this.mcpBridge) {
|
|
82494
83055
|
const mcpTools = this.mcpBridge.mcpTools || {};
|
|
82495
83056
|
for (const [toolName, toolImpl] of Object.entries(mcpTools)) {
|
|
82496
|
-
this.
|
|
83057
|
+
if (this._isMcpToolAllowed(toolName)) {
|
|
83058
|
+
this.toolImplementations[toolName] = toolImpl;
|
|
83059
|
+
} else if (this.debug) {
|
|
83060
|
+
console.error(`[DEBUG] MCP tool '${toolName}' filtered out by allowedTools`);
|
|
83061
|
+
}
|
|
82497
83062
|
}
|
|
82498
83063
|
}
|
|
82499
83064
|
if (this.debug) {
|
|
@@ -83113,7 +83678,11 @@ var init_ProbeAgent = __esm({
|
|
|
83113
83678
|
if (this.mcpBridge) {
|
|
83114
83679
|
const mcpTools = this.mcpBridge.mcpTools || {};
|
|
83115
83680
|
for (const [toolName, toolImpl] of Object.entries(mcpTools)) {
|
|
83116
|
-
this.
|
|
83681
|
+
if (this._isMcpToolAllowed(toolName)) {
|
|
83682
|
+
this.toolImplementations[toolName] = toolImpl;
|
|
83683
|
+
} else if (this.debug) {
|
|
83684
|
+
console.error(`[DEBUG] MCP tool '${toolName}' filtered out by allowedTools`);
|
|
83685
|
+
}
|
|
83117
83686
|
}
|
|
83118
83687
|
}
|
|
83119
83688
|
} catch (error2) {
|
|
@@ -83123,27 +83692,49 @@ var init_ProbeAgent = __esm({
|
|
|
83123
83692
|
}
|
|
83124
83693
|
}
|
|
83125
83694
|
}
|
|
83126
|
-
let toolDefinitions =
|
|
83127
|
-
|
|
83128
|
-
|
|
83129
|
-
|
|
83130
|
-
${listFilesToolDefinition}
|
|
83131
|
-
${searchFilesToolDefinition}
|
|
83132
|
-
${attemptCompletionToolDefinition}
|
|
83695
|
+
let toolDefinitions = "";
|
|
83696
|
+
const isToolAllowed = (toolName) => this.allowedTools.isEnabled(toolName);
|
|
83697
|
+
if (isToolAllowed("search")) {
|
|
83698
|
+
toolDefinitions += `${searchToolDefinition}
|
|
83133
83699
|
`;
|
|
83134
|
-
|
|
83700
|
+
}
|
|
83701
|
+
if (isToolAllowed("query")) {
|
|
83702
|
+
toolDefinitions += `${queryToolDefinition}
|
|
83703
|
+
`;
|
|
83704
|
+
}
|
|
83705
|
+
if (isToolAllowed("extract")) {
|
|
83706
|
+
toolDefinitions += `${extractToolDefinition}
|
|
83707
|
+
`;
|
|
83708
|
+
}
|
|
83709
|
+
if (isToolAllowed("listFiles")) {
|
|
83710
|
+
toolDefinitions += `${listFilesToolDefinition}
|
|
83711
|
+
`;
|
|
83712
|
+
}
|
|
83713
|
+
if (isToolAllowed("searchFiles")) {
|
|
83714
|
+
toolDefinitions += `${searchFilesToolDefinition}
|
|
83715
|
+
`;
|
|
83716
|
+
}
|
|
83717
|
+
if (this.allowEdit && isToolAllowed("implement")) {
|
|
83135
83718
|
toolDefinitions += `${implementToolDefinition}
|
|
83136
83719
|
`;
|
|
83720
|
+
}
|
|
83721
|
+
if (this.allowEdit && isToolAllowed("edit")) {
|
|
83137
83722
|
toolDefinitions += `${editToolDefinition}
|
|
83138
83723
|
`;
|
|
83724
|
+
}
|
|
83725
|
+
if (this.allowEdit && isToolAllowed("create")) {
|
|
83139
83726
|
toolDefinitions += `${createToolDefinition}
|
|
83140
83727
|
`;
|
|
83141
83728
|
}
|
|
83142
|
-
if (this.enableBash) {
|
|
83729
|
+
if (this.enableBash && isToolAllowed("bash")) {
|
|
83143
83730
|
toolDefinitions += `${bashToolDefinition}
|
|
83144
83731
|
`;
|
|
83145
83732
|
}
|
|
83146
|
-
if (
|
|
83733
|
+
if (isToolAllowed("attempt_completion")) {
|
|
83734
|
+
toolDefinitions += `${attemptCompletionToolDefinition}
|
|
83735
|
+
`;
|
|
83736
|
+
}
|
|
83737
|
+
if (this.enableDelegate && isToolAllowed("delegate")) {
|
|
83147
83738
|
toolDefinitions += `${delegateToolDefinition}
|
|
83148
83739
|
`;
|
|
83149
83740
|
}
|
|
@@ -83305,11 +83896,14 @@ ${xmlToolGuidelines}
|
|
|
83305
83896
|
${toolDefinitions}
|
|
83306
83897
|
`;
|
|
83307
83898
|
if (this.mcpBridge && this.mcpBridge.getToolNames().length > 0) {
|
|
83308
|
-
|
|
83899
|
+
const allMcpTools = this.mcpBridge.getToolNames();
|
|
83900
|
+
const allowedMcpTools = this._filterMcpTools(allMcpTools);
|
|
83901
|
+
if (allowedMcpTools.length > 0) {
|
|
83902
|
+
systemMessage += `
|
|
83309
83903
|
## MCP Tools (JSON parameters in <params> tag)
|
|
83310
83904
|
`;
|
|
83311
|
-
|
|
83312
|
-
|
|
83905
|
+
systemMessage += this.mcpBridge.getXmlToolDefinitions(allowedMcpTools);
|
|
83906
|
+
systemMessage += `
|
|
83313
83907
|
|
|
83314
83908
|
For MCP tools, use JSON format within the params tag, e.g.:
|
|
83315
83909
|
<mcp_tool>
|
|
@@ -83318,6 +83912,7 @@ For MCP tools, use JSON format within the params tag, e.g.:
|
|
|
83318
83912
|
</params>
|
|
83319
83913
|
</mcp_tool>
|
|
83320
83914
|
`;
|
|
83915
|
+
}
|
|
83321
83916
|
}
|
|
83322
83917
|
const searchDirectory = this.allowedFolders.length > 0 ? this.allowedFolders[0] : process.cwd();
|
|
83323
83918
|
if (this.debug) {
|
|
@@ -83474,44 +84069,83 @@ You are working with a repository located at: ${searchDirectory}
|
|
|
83474
84069
|
}
|
|
83475
84070
|
}
|
|
83476
84071
|
let assistantResponseContent = "";
|
|
83477
|
-
|
|
83478
|
-
|
|
83479
|
-
|
|
83480
|
-
const
|
|
83481
|
-
|
|
83482
|
-
|
|
83483
|
-
|
|
83484
|
-
|
|
83485
|
-
|
|
83486
|
-
|
|
83487
|
-
|
|
83488
|
-
|
|
83489
|
-
|
|
83490
|
-
|
|
84072
|
+
let compactionAttempted = false;
|
|
84073
|
+
while (true) {
|
|
84074
|
+
try {
|
|
84075
|
+
const executeAIRequest = async () => {
|
|
84076
|
+
const messagesForAI = this.prepareMessagesWithImages(currentMessages);
|
|
84077
|
+
const result = await this.streamTextWithRetryAndFallback({
|
|
84078
|
+
model: this.provider(this.model),
|
|
84079
|
+
messages: messagesForAI,
|
|
84080
|
+
maxTokens: maxResponseTokens,
|
|
84081
|
+
temperature: 0.3
|
|
84082
|
+
});
|
|
84083
|
+
const usagePromise = result.usage;
|
|
84084
|
+
for await (const delta of result.textStream) {
|
|
84085
|
+
assistantResponseContent += delta;
|
|
84086
|
+
if (options.onStream) {
|
|
84087
|
+
options.onStream(delta);
|
|
84088
|
+
}
|
|
84089
|
+
}
|
|
84090
|
+
const usage = await usagePromise;
|
|
84091
|
+
if (usage) {
|
|
84092
|
+
this.tokenCounter.recordUsage(usage, result.experimental_providerMetadata);
|
|
83491
84093
|
}
|
|
84094
|
+
return result;
|
|
84095
|
+
};
|
|
84096
|
+
if (this.tracer) {
|
|
84097
|
+
await this.tracer.withSpan("ai.request", executeAIRequest, {
|
|
84098
|
+
"ai.model": this.model,
|
|
84099
|
+
"ai.provider": this.clientApiProvider || "auto",
|
|
84100
|
+
"iteration": currentIteration,
|
|
84101
|
+
"max_tokens": maxResponseTokens,
|
|
84102
|
+
"temperature": 0.3,
|
|
84103
|
+
"message_count": currentMessages.length
|
|
84104
|
+
});
|
|
84105
|
+
} else {
|
|
84106
|
+
await executeAIRequest();
|
|
83492
84107
|
}
|
|
83493
|
-
|
|
83494
|
-
|
|
83495
|
-
|
|
84108
|
+
break;
|
|
84109
|
+
} catch (error2) {
|
|
84110
|
+
if (!compactionAttempted && handleContextLimitError) {
|
|
84111
|
+
const compactionResult = handleContextLimitError(error2, currentMessages, {
|
|
84112
|
+
keepLastSegment: true,
|
|
84113
|
+
minSegmentsToKeep: 1
|
|
84114
|
+
});
|
|
84115
|
+
if (compactionResult) {
|
|
84116
|
+
const { messages: compactedMessages, stats } = compactionResult;
|
|
84117
|
+
if (stats.removed === 0) {
|
|
84118
|
+
console.error(`[ERROR] Context window exceeded but no messages can be compacted.`);
|
|
84119
|
+
console.error(`[ERROR] The conversation history is already minimal (${stats.originalCount} messages).`);
|
|
84120
|
+
finalResult = `Error: Context window limit exceeded and conversation cannot be compacted further. Consider starting a new session or reducing system message size.`;
|
|
84121
|
+
throw new Error(finalResult);
|
|
84122
|
+
}
|
|
84123
|
+
compactionAttempted = true;
|
|
84124
|
+
console.log(`[INFO] Context window limit exceeded. Compacting conversation...`);
|
|
84125
|
+
console.log(`[INFO] Removed ${stats.removed} messages (${stats.reductionPercent}% reduction)`);
|
|
84126
|
+
console.log(`[INFO] Estimated token savings: ${stats.tokensSaved} tokens`);
|
|
84127
|
+
if (this.debug) {
|
|
84128
|
+
console.log(`[DEBUG] Compaction stats:`, stats);
|
|
84129
|
+
console.log(`[DEBUG] Original message count: ${stats.originalCount}`);
|
|
84130
|
+
console.log(`[DEBUG] Compacted message count: ${stats.compactedCount}`);
|
|
84131
|
+
}
|
|
84132
|
+
currentMessages = [...compactedMessages];
|
|
84133
|
+
if (this.tracer) {
|
|
84134
|
+
this.tracer.addEvent("context.compacted", {
|
|
84135
|
+
"iteration": currentIteration,
|
|
84136
|
+
"original_count": stats.originalCount,
|
|
84137
|
+
"compacted_count": stats.compactedCount,
|
|
84138
|
+
"reduction_percent": stats.reductionPercent,
|
|
84139
|
+
"tokens_saved": stats.tokensSaved
|
|
84140
|
+
});
|
|
84141
|
+
}
|
|
84142
|
+
continue;
|
|
84143
|
+
}
|
|
83496
84144
|
}
|
|
83497
|
-
|
|
83498
|
-
|
|
83499
|
-
|
|
83500
|
-
await this.tracer.withSpan("ai.request", executeAIRequest, {
|
|
83501
|
-
"ai.model": this.model,
|
|
83502
|
-
"ai.provider": this.clientApiProvider || "auto",
|
|
83503
|
-
"iteration": currentIteration,
|
|
83504
|
-
"max_tokens": maxResponseTokens,
|
|
83505
|
-
"temperature": 0.3,
|
|
83506
|
-
"message_count": currentMessages.length
|
|
83507
|
-
});
|
|
83508
|
-
} else {
|
|
83509
|
-
await executeAIRequest();
|
|
84145
|
+
console.error(`Error during streamText (Iter ${currentIteration}):`, error2);
|
|
84146
|
+
finalResult = `Error: Failed to get response from AI model during iteration ${currentIteration}. ${error2.message}`;
|
|
84147
|
+
throw new Error(finalResult);
|
|
83510
84148
|
}
|
|
83511
|
-
} catch (error2) {
|
|
83512
|
-
console.error(`Error during streamText (Iter ${currentIteration}):`, error2);
|
|
83513
|
-
finalResult = `Error: Failed to get response from AI model during iteration ${currentIteration}. ${error2.message}`;
|
|
83514
|
-
throw new Error(finalResult);
|
|
83515
84149
|
}
|
|
83516
84150
|
if (this.debug && assistantResponseContent) {
|
|
83517
84151
|
const assistantPreview = createMessagePreview(assistantResponseContent);
|
|
@@ -83876,7 +84510,6 @@ Convert your previous response content into actual JSON data that follows this s
|
|
|
83876
84510
|
...options,
|
|
83877
84511
|
_schemaFormatted: true
|
|
83878
84512
|
});
|
|
83879
|
-
finalResult = cleanSchemaResponse(finalResult);
|
|
83880
84513
|
if (!this.disableMermaidValidation) {
|
|
83881
84514
|
try {
|
|
83882
84515
|
if (this.debug) {
|
|
@@ -83932,14 +84565,11 @@ Convert your previous response content into actual JSON data that follows this s
|
|
|
83932
84565
|
} else if (this.debug) {
|
|
83933
84566
|
console.log(`[DEBUG] Mermaid validation: Skipped due to disableMermaidValidation option`);
|
|
83934
84567
|
}
|
|
84568
|
+
finalResult = cleanSchemaResponse(finalResult);
|
|
83935
84569
|
if (isJsonSchema(options.schema)) {
|
|
83936
84570
|
if (this.debug) {
|
|
83937
84571
|
console.log(`[DEBUG] JSON validation: Starting validation process for schema response`);
|
|
83938
|
-
console.log(`[DEBUG] JSON validation:
|
|
83939
|
-
}
|
|
83940
|
-
finalResult = cleanSchemaResponse(finalResult);
|
|
83941
|
-
if (this.debug) {
|
|
83942
|
-
console.log(`[DEBUG] JSON validation: After cleaning, length: ${finalResult.length} chars`);
|
|
84572
|
+
console.log(`[DEBUG] JSON validation: Cleaned response length: ${finalResult.length} chars`);
|
|
83943
84573
|
}
|
|
83944
84574
|
if (this.tracer) {
|
|
83945
84575
|
this.tracer.recordJsonValidationEvent("started", {
|
|
@@ -84029,10 +84659,9 @@ Convert your previous response content into actual JSON data that follows this s
|
|
|
84029
84659
|
console.log("[DEBUG] Skipping schema formatting due to max iterations reached without completion");
|
|
84030
84660
|
} else if (completionAttempted && options.schema && !options._schemaFormatted && !options._skipValidation) {
|
|
84031
84661
|
try {
|
|
84032
|
-
finalResult = cleanSchemaResponse(finalResult);
|
|
84033
84662
|
if (!this.disableMermaidValidation) {
|
|
84034
84663
|
if (this.debug) {
|
|
84035
|
-
console.log(`[DEBUG] Mermaid validation: Validating attempt_completion result...`);
|
|
84664
|
+
console.log(`[DEBUG] Mermaid validation: Validating attempt_completion result BEFORE schema cleaning...`);
|
|
84036
84665
|
}
|
|
84037
84666
|
const mermaidValidation = await validateAndFixMermaidResponse(finalResult, {
|
|
84038
84667
|
debug: this.debug,
|
|
@@ -84055,6 +84684,7 @@ Convert your previous response content into actual JSON data that follows this s
|
|
|
84055
84684
|
} else if (this.debug) {
|
|
84056
84685
|
console.log(`[DEBUG] Mermaid validation: Skipped for attempt_completion result due to disableMermaidValidation option`);
|
|
84057
84686
|
}
|
|
84687
|
+
finalResult = cleanSchemaResponse(finalResult);
|
|
84058
84688
|
if (isJsonSchema(options.schema)) {
|
|
84059
84689
|
if (this.debug) {
|
|
84060
84690
|
console.log(`[DEBUG] JSON validation: Starting validation process for attempt_completion result`);
|
|
@@ -84228,6 +84858,56 @@ Convert your previous response content into actual JSON data that follows this s
|
|
|
84228
84858
|
console.log(`[DEBUG] Cleared conversation history and reset counters for session ${this.sessionId}`);
|
|
84229
84859
|
}
|
|
84230
84860
|
}
|
|
84861
|
+
/**
|
|
84862
|
+
* Manually compact conversation history
|
|
84863
|
+
* Removes intermediate monologues from older segments while preserving
|
|
84864
|
+
* user messages, final answers, and the most recent segment
|
|
84865
|
+
*
|
|
84866
|
+
* @param {Object} options - Compaction options
|
|
84867
|
+
* @param {boolean} [options.keepLastSegment=true] - Keep the most recent segment intact
|
|
84868
|
+
* @param {number} [options.minSegmentsToKeep=1] - Number of recent segments to preserve fully
|
|
84869
|
+
* @returns {Object} Compaction statistics
|
|
84870
|
+
*/
|
|
84871
|
+
async compactHistory(options = {}) {
|
|
84872
|
+
const { compactMessages: compactMessages2, calculateCompactionStats: calculateCompactionStats2 } = await Promise.resolve().then(() => (init_contextCompactor(), contextCompactor_exports));
|
|
84873
|
+
if (this.history.length === 0) {
|
|
84874
|
+
if (this.debug) {
|
|
84875
|
+
console.log(`[DEBUG] No history to compact for session ${this.sessionId}`);
|
|
84876
|
+
}
|
|
84877
|
+
return {
|
|
84878
|
+
originalCount: 0,
|
|
84879
|
+
compactedCount: 0,
|
|
84880
|
+
removed: 0,
|
|
84881
|
+
reductionPercent: 0,
|
|
84882
|
+
originalTokens: 0,
|
|
84883
|
+
compactedTokens: 0,
|
|
84884
|
+
tokensSaved: 0
|
|
84885
|
+
};
|
|
84886
|
+
}
|
|
84887
|
+
const compactedMessages = compactMessages2(this.history, options);
|
|
84888
|
+
const stats = calculateCompactionStats2(this.history, compactedMessages);
|
|
84889
|
+
this.history = compactedMessages;
|
|
84890
|
+
try {
|
|
84891
|
+
await this.storageAdapter.clearHistory(this.sessionId);
|
|
84892
|
+
for (const message of compactedMessages) {
|
|
84893
|
+
await this.storageAdapter.saveMessage(this.sessionId, message);
|
|
84894
|
+
}
|
|
84895
|
+
} catch (error2) {
|
|
84896
|
+
console.error(`[ERROR] Failed to save compacted messages to storage:`, error2);
|
|
84897
|
+
}
|
|
84898
|
+
console.log(`[INFO] Manually compacted conversation history`);
|
|
84899
|
+
console.log(`[INFO] Removed ${stats.removed} messages (${stats.reductionPercent}% reduction)`);
|
|
84900
|
+
console.log(`[INFO] Estimated token savings: ${stats.tokensSaved} tokens`);
|
|
84901
|
+
if (this.debug) {
|
|
84902
|
+
console.log(`[DEBUG] Compaction stats:`, stats);
|
|
84903
|
+
}
|
|
84904
|
+
await this.hooks.emit(HOOK_TYPES.STORAGE_SAVE, {
|
|
84905
|
+
sessionId: this.sessionId,
|
|
84906
|
+
compacted: true,
|
|
84907
|
+
stats
|
|
84908
|
+
});
|
|
84909
|
+
return stats;
|
|
84910
|
+
}
|
|
84231
84911
|
/**
|
|
84232
84912
|
* Clone this agent's session to create a new agent with shared conversation history
|
|
84233
84913
|
* @param {Object} options - Clone options
|
|
@@ -84250,6 +84930,14 @@ Convert your previous response content into actual JSON data that follows this s
|
|
|
84250
84930
|
if (stripInternalMessages) {
|
|
84251
84931
|
clonedHistory = this._stripInternalMessages(clonedHistory, keepSystemMessage);
|
|
84252
84932
|
}
|
|
84933
|
+
let allowedToolsArray = null;
|
|
84934
|
+
if (this.allowedTools.mode === "whitelist") {
|
|
84935
|
+
allowedToolsArray = [...this.allowedTools.allowed];
|
|
84936
|
+
} else if (this.allowedTools.mode === "none") {
|
|
84937
|
+
allowedToolsArray = [];
|
|
84938
|
+
} else if (this.allowedTools.mode === "all" && this.allowedTools.exclusions.length > 0) {
|
|
84939
|
+
allowedToolsArray = ["*", ...this.allowedTools.exclusions.map((t3) => "!" + t3)];
|
|
84940
|
+
}
|
|
84253
84941
|
const clonedAgent = new _ProbeAgent({
|
|
84254
84942
|
// Copy current agent's config
|
|
84255
84943
|
customPrompt: this.customPrompt,
|
|
@@ -84267,6 +84955,7 @@ Convert your previous response content into actual JSON data that follows this s
|
|
|
84267
84955
|
maxIterations: this.maxIterations,
|
|
84268
84956
|
disableMermaidValidation: this.disableMermaidValidation,
|
|
84269
84957
|
disableJsonValidation: this.disableJsonValidation,
|
|
84958
|
+
allowedTools: allowedToolsArray,
|
|
84270
84959
|
enableMcp: !!this.mcpBridge,
|
|
84271
84960
|
mcpConfig: this.mcpConfig,
|
|
84272
84961
|
enableBash: this.enableBash,
|