@probelabs/probe 0.6.0-rc161 → 0.6.0-rc162
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +89 -0
- package/build/agent/ProbeAgent.js +349 -85
- package/build/agent/contextCompactor.js +271 -0
- package/build/agent/index.js +788 -84
- package/build/agent/schemaUtils.js +7 -0
- package/build/agent/xmlParsingUtils.js +24 -4
- package/build/tools/common.js +16 -1
- package/cjs/agent/ProbeAgent.cjs +823 -134
- package/cjs/index.cjs +823 -134
- package/package.json +1 -1
- package/src/agent/ProbeAgent.js +349 -85
- package/src/agent/contextCompactor.js +271 -0
- package/src/agent/index.js +30 -1
- package/src/agent/schemaUtils.js +7 -0
- package/src/agent/xmlParsingUtils.js +24 -4
- package/src/tools/common.js +16 -1
package/cjs/index.cjs
CHANGED
|
@@ -11223,7 +11223,10 @@ var init_CborCodec = __esm({
|
|
|
11223
11223
|
}
|
|
11224
11224
|
} else if (ns.isStructSchema()) {
|
|
11225
11225
|
for (const [key, memberSchema] of ns.structIterator()) {
|
|
11226
|
-
|
|
11226
|
+
const v3 = this.readValue(memberSchema, value[key]);
|
|
11227
|
+
if (v3 != null) {
|
|
11228
|
+
newObject[key] = v3;
|
|
11229
|
+
}
|
|
11227
11230
|
}
|
|
11228
11231
|
}
|
|
11229
11232
|
return newObject;
|
|
@@ -12744,10 +12747,7 @@ var init_JsonShapeSerializer = __esm({
|
|
|
12744
12747
|
if (ns === this.rootSchema) {
|
|
12745
12748
|
return value;
|
|
12746
12749
|
}
|
|
12747
|
-
|
|
12748
|
-
return (0, import_util_base645.toBase64)(value);
|
|
12749
|
-
}
|
|
12750
|
-
return this.serdeContext?.base64Encoder(value);
|
|
12750
|
+
return (this.serdeContext?.base64Encoder ?? import_util_base645.toBase64)(value);
|
|
12751
12751
|
}
|
|
12752
12752
|
if ((ns.isTimestampSchema() || ns.isDocumentSchema()) && value instanceof Date) {
|
|
12753
12753
|
const format2 = determineTimestampFormat(ns, this.settings);
|
|
@@ -14382,7 +14382,13 @@ var init_AwsQueryProtocol = __esm({
|
|
|
14382
14382
|
Code: errorData.Code,
|
|
14383
14383
|
Message: message
|
|
14384
14384
|
};
|
|
14385
|
-
const { errorSchema, errorMetadata } = await this.mixin.getErrorSchemaOrThrowBaseException(errorIdentifier, this.options.defaultNamespace, response, errorData, metadata, (registry, errorName) =>
|
|
14385
|
+
const { errorSchema, errorMetadata } = await this.mixin.getErrorSchemaOrThrowBaseException(errorIdentifier, this.options.defaultNamespace, response, errorData, metadata, (registry, errorName) => {
|
|
14386
|
+
try {
|
|
14387
|
+
return registry.getSchema(errorName);
|
|
14388
|
+
} catch (e3) {
|
|
14389
|
+
return registry.find((schema) => NormalizedSchema.of(schema).getMergedTraits().awsQueryError?.[0] === errorName);
|
|
14390
|
+
}
|
|
14391
|
+
});
|
|
14386
14392
|
const ns = NormalizedSchema.of(errorSchema);
|
|
14387
14393
|
const ErrorCtor = TypeRegistry.for(errorSchema[1]).getErrorCtor(errorSchema) ?? Error;
|
|
14388
14394
|
const exception = new ErrorCtor(message);
|
|
@@ -16748,7 +16754,11 @@ var require_dist_cjs39 = __commonJS({
|
|
|
16748
16754
|
var validRegions = /* @__PURE__ */ new Set();
|
|
16749
16755
|
var checkRegion = (region, check = utilEndpoints.isValidHostLabel) => {
|
|
16750
16756
|
if (!validRegions.has(region) && !check(region)) {
|
|
16751
|
-
|
|
16757
|
+
if (region === "*") {
|
|
16758
|
+
console.warn(`@smithy/config-resolver WARN - Please use the caller region instead of "*". See "sigv4a" in https://github.com/aws/aws-sdk-js-v3/blob/main/supplemental-docs/CLIENTS.md.`);
|
|
16759
|
+
} else {
|
|
16760
|
+
throw new Error(`Region not accepted: region="${region}" is not a valid hostname component.`);
|
|
16761
|
+
}
|
|
16752
16762
|
} else {
|
|
16753
16763
|
validRegions.add(region);
|
|
16754
16764
|
}
|
|
@@ -16946,42 +16956,40 @@ var require_getSSOTokenFromFile = __commonJS({
|
|
|
16946
16956
|
"use strict";
|
|
16947
16957
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
16948
16958
|
exports2.getSSOTokenFromFile = exports2.tokenIntercept = void 0;
|
|
16949
|
-
var
|
|
16959
|
+
var promises_1 = require("fs/promises");
|
|
16950
16960
|
var getSSOTokenFilepath_1 = require_getSSOTokenFilepath();
|
|
16951
|
-
var { readFile: readFile2 } = fs_1.promises;
|
|
16952
16961
|
exports2.tokenIntercept = {};
|
|
16953
16962
|
var getSSOTokenFromFile = async (id) => {
|
|
16954
16963
|
if (exports2.tokenIntercept[id]) {
|
|
16955
16964
|
return exports2.tokenIntercept[id];
|
|
16956
16965
|
}
|
|
16957
16966
|
const ssoTokenFilepath = (0, getSSOTokenFilepath_1.getSSOTokenFilepath)(id);
|
|
16958
|
-
const ssoTokenText = await
|
|
16967
|
+
const ssoTokenText = await (0, promises_1.readFile)(ssoTokenFilepath, "utf8");
|
|
16959
16968
|
return JSON.parse(ssoTokenText);
|
|
16960
16969
|
};
|
|
16961
16970
|
exports2.getSSOTokenFromFile = getSSOTokenFromFile;
|
|
16962
16971
|
}
|
|
16963
16972
|
});
|
|
16964
16973
|
|
|
16965
|
-
// node_modules/@smithy/shared-ini-file-loader/dist-cjs/
|
|
16966
|
-
var
|
|
16967
|
-
"node_modules/@smithy/shared-ini-file-loader/dist-cjs/
|
|
16974
|
+
// node_modules/@smithy/shared-ini-file-loader/dist-cjs/readFile.js
|
|
16975
|
+
var require_readFile = __commonJS({
|
|
16976
|
+
"node_modules/@smithy/shared-ini-file-loader/dist-cjs/readFile.js"(exports2) {
|
|
16968
16977
|
"use strict";
|
|
16969
16978
|
Object.defineProperty(exports2, "__esModule", { value: true });
|
|
16970
|
-
exports2.
|
|
16971
|
-
var
|
|
16972
|
-
|
|
16973
|
-
exports2.filePromisesHash = {};
|
|
16979
|
+
exports2.readFile = exports2.fileIntercept = exports2.filePromises = void 0;
|
|
16980
|
+
var promises_1 = require("node:fs/promises");
|
|
16981
|
+
exports2.filePromises = {};
|
|
16974
16982
|
exports2.fileIntercept = {};
|
|
16975
|
-
var
|
|
16983
|
+
var readFile2 = (path7, options) => {
|
|
16976
16984
|
if (exports2.fileIntercept[path7] !== void 0) {
|
|
16977
16985
|
return exports2.fileIntercept[path7];
|
|
16978
16986
|
}
|
|
16979
|
-
if (!exports2.
|
|
16980
|
-
exports2.
|
|
16987
|
+
if (!exports2.filePromises[path7] || options?.ignoreCache) {
|
|
16988
|
+
exports2.filePromises[path7] = (0, promises_1.readFile)(path7, "utf8");
|
|
16981
16989
|
}
|
|
16982
|
-
return exports2.
|
|
16990
|
+
return exports2.filePromises[path7];
|
|
16983
16991
|
};
|
|
16984
|
-
exports2.
|
|
16992
|
+
exports2.readFile = readFile2;
|
|
16985
16993
|
}
|
|
16986
16994
|
});
|
|
16987
16995
|
|
|
@@ -16994,7 +17002,7 @@ var require_dist_cjs42 = __commonJS({
|
|
|
16994
17002
|
var getSSOTokenFromFile = require_getSSOTokenFromFile();
|
|
16995
17003
|
var path7 = require("path");
|
|
16996
17004
|
var types2 = require_dist_cjs();
|
|
16997
|
-
var
|
|
17005
|
+
var readFile2 = require_readFile();
|
|
16998
17006
|
var ENV_PROFILE = "AWS_PROFILE";
|
|
16999
17007
|
var DEFAULT_PROFILE = "default";
|
|
17000
17008
|
var getProfileName = (init) => init.profile || process.env[ENV_PROFILE] || DEFAULT_PROFILE;
|
|
@@ -17078,10 +17086,10 @@ var require_dist_cjs42 = __commonJS({
|
|
|
17078
17086
|
resolvedConfigFilepath = path7.join(homeDir, configFilepath.slice(2));
|
|
17079
17087
|
}
|
|
17080
17088
|
const parsedFiles = await Promise.all([
|
|
17081
|
-
|
|
17089
|
+
readFile2.readFile(resolvedConfigFilepath, {
|
|
17082
17090
|
ignoreCache: init.ignoreCache
|
|
17083
17091
|
}).then(parseIni).then(getConfigData).catch(swallowError$1),
|
|
17084
|
-
|
|
17092
|
+
readFile2.readFile(resolvedFilepath, {
|
|
17085
17093
|
ignoreCache: init.ignoreCache
|
|
17086
17094
|
}).then(parseIni).catch(swallowError$1)
|
|
17087
17095
|
]);
|
|
@@ -17092,7 +17100,7 @@ var require_dist_cjs42 = __commonJS({
|
|
|
17092
17100
|
};
|
|
17093
17101
|
var getSsoSessionData = (data2) => Object.entries(data2).filter(([key]) => key.startsWith(types2.IniSectionType.SSO_SESSION + CONFIG_PREFIX_SEPARATOR)).reduce((acc, [key, value]) => ({ ...acc, [key.substring(key.indexOf(CONFIG_PREFIX_SEPARATOR) + 1)]: value }), {});
|
|
17094
17102
|
var swallowError = () => ({});
|
|
17095
|
-
var loadSsoSessionData = async (init = {}) =>
|
|
17103
|
+
var loadSsoSessionData = async (init = {}) => readFile2.readFile(init.configFilepath ?? getConfigFilepath()).then(parseIni).then(getSsoSessionData).catch(swallowError);
|
|
17096
17104
|
var mergeConfigFiles = (...files) => {
|
|
17097
17105
|
const merged = {};
|
|
17098
17106
|
for (const file of files) {
|
|
@@ -17112,10 +17120,10 @@ var require_dist_cjs42 = __commonJS({
|
|
|
17112
17120
|
};
|
|
17113
17121
|
var externalDataInterceptor = {
|
|
17114
17122
|
getFileRecord() {
|
|
17115
|
-
return
|
|
17123
|
+
return readFile2.fileIntercept;
|
|
17116
17124
|
},
|
|
17117
17125
|
interceptFile(path8, contents) {
|
|
17118
|
-
|
|
17126
|
+
readFile2.fileIntercept[path8] = Promise.resolve(contents);
|
|
17119
17127
|
},
|
|
17120
17128
|
getTokenRecord() {
|
|
17121
17129
|
return getSSOTokenFromFile.tokenIntercept;
|
|
@@ -17130,6 +17138,12 @@ var require_dist_cjs42 = __commonJS({
|
|
|
17130
17138
|
return getSSOTokenFromFile.getSSOTokenFromFile;
|
|
17131
17139
|
}
|
|
17132
17140
|
});
|
|
17141
|
+
Object.defineProperty(exports2, "readFile", {
|
|
17142
|
+
enumerable: true,
|
|
17143
|
+
get: function() {
|
|
17144
|
+
return readFile2.readFile;
|
|
17145
|
+
}
|
|
17146
|
+
});
|
|
17133
17147
|
exports2.CONFIG_PREFIX_SEPARATOR = CONFIG_PREFIX_SEPARATOR;
|
|
17134
17148
|
exports2.DEFAULT_PROFILE = DEFAULT_PROFILE;
|
|
17135
17149
|
exports2.ENV_PROFILE = ENV_PROFILE;
|
|
@@ -18269,7 +18283,7 @@ var require_package2 = __commonJS({
|
|
|
18269
18283
|
module2.exports = {
|
|
18270
18284
|
name: "@aws-sdk/client-bedrock-runtime",
|
|
18271
18285
|
description: "AWS SDK for JavaScript Bedrock Runtime Client for Node.js, Browser and React Native",
|
|
18272
|
-
version: "3.
|
|
18286
|
+
version: "3.928.0",
|
|
18273
18287
|
scripts: {
|
|
18274
18288
|
build: "concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types'",
|
|
18275
18289
|
"build:cjs": "node ../../scripts/compilation/inline client-bedrock-runtime",
|
|
@@ -18288,22 +18302,22 @@ var require_package2 = __commonJS({
|
|
|
18288
18302
|
dependencies: {
|
|
18289
18303
|
"@aws-crypto/sha256-browser": "5.2.0",
|
|
18290
18304
|
"@aws-crypto/sha256-js": "5.2.0",
|
|
18291
|
-
"@aws-sdk/core": "3.
|
|
18292
|
-
"@aws-sdk/credential-provider-node": "3.
|
|
18305
|
+
"@aws-sdk/core": "3.928.0",
|
|
18306
|
+
"@aws-sdk/credential-provider-node": "3.928.0",
|
|
18293
18307
|
"@aws-sdk/eventstream-handler-node": "3.922.0",
|
|
18294
18308
|
"@aws-sdk/middleware-eventstream": "3.922.0",
|
|
18295
18309
|
"@aws-sdk/middleware-host-header": "3.922.0",
|
|
18296
18310
|
"@aws-sdk/middleware-logger": "3.922.0",
|
|
18297
18311
|
"@aws-sdk/middleware-recursion-detection": "3.922.0",
|
|
18298
|
-
"@aws-sdk/middleware-user-agent": "3.
|
|
18312
|
+
"@aws-sdk/middleware-user-agent": "3.928.0",
|
|
18299
18313
|
"@aws-sdk/middleware-websocket": "3.922.0",
|
|
18300
|
-
"@aws-sdk/region-config-resolver": "3.
|
|
18301
|
-
"@aws-sdk/token-providers": "3.
|
|
18314
|
+
"@aws-sdk/region-config-resolver": "3.925.0",
|
|
18315
|
+
"@aws-sdk/token-providers": "3.928.0",
|
|
18302
18316
|
"@aws-sdk/types": "3.922.0",
|
|
18303
18317
|
"@aws-sdk/util-endpoints": "3.922.0",
|
|
18304
18318
|
"@aws-sdk/util-user-agent-browser": "3.922.0",
|
|
18305
|
-
"@aws-sdk/util-user-agent-node": "3.
|
|
18306
|
-
"@smithy/config-resolver": "^4.4.
|
|
18319
|
+
"@aws-sdk/util-user-agent-node": "3.928.0",
|
|
18320
|
+
"@smithy/config-resolver": "^4.4.2",
|
|
18307
18321
|
"@smithy/core": "^3.17.2",
|
|
18308
18322
|
"@smithy/eventstream-serde-browser": "^4.2.4",
|
|
18309
18323
|
"@smithy/eventstream-serde-config-resolver": "^4.3.4",
|
|
@@ -18326,7 +18340,7 @@ var require_package2 = __commonJS({
|
|
|
18326
18340
|
"@smithy/util-body-length-browser": "^4.2.0",
|
|
18327
18341
|
"@smithy/util-body-length-node": "^4.2.1",
|
|
18328
18342
|
"@smithy/util-defaults-mode-browser": "^4.3.5",
|
|
18329
|
-
"@smithy/util-defaults-mode-node": "^4.2.
|
|
18343
|
+
"@smithy/util-defaults-mode-node": "^4.2.8",
|
|
18330
18344
|
"@smithy/util-endpoints": "^3.2.4",
|
|
18331
18345
|
"@smithy/util-middleware": "^4.2.4",
|
|
18332
18346
|
"@smithy/util-retry": "^4.2.4",
|
|
@@ -19050,7 +19064,7 @@ var init_package = __esm({
|
|
|
19050
19064
|
"node_modules/@aws-sdk/nested-clients/package.json"() {
|
|
19051
19065
|
package_default = {
|
|
19052
19066
|
name: "@aws-sdk/nested-clients",
|
|
19053
|
-
version: "3.
|
|
19067
|
+
version: "3.928.0",
|
|
19054
19068
|
description: "Nested clients for AWS SDK packages.",
|
|
19055
19069
|
main: "./dist-cjs/index.js",
|
|
19056
19070
|
module: "./dist-es/index.js",
|
|
@@ -19079,17 +19093,17 @@ var init_package = __esm({
|
|
|
19079
19093
|
dependencies: {
|
|
19080
19094
|
"@aws-crypto/sha256-browser": "5.2.0",
|
|
19081
19095
|
"@aws-crypto/sha256-js": "5.2.0",
|
|
19082
|
-
"@aws-sdk/core": "3.
|
|
19096
|
+
"@aws-sdk/core": "3.928.0",
|
|
19083
19097
|
"@aws-sdk/middleware-host-header": "3.922.0",
|
|
19084
19098
|
"@aws-sdk/middleware-logger": "3.922.0",
|
|
19085
19099
|
"@aws-sdk/middleware-recursion-detection": "3.922.0",
|
|
19086
|
-
"@aws-sdk/middleware-user-agent": "3.
|
|
19087
|
-
"@aws-sdk/region-config-resolver": "3.
|
|
19100
|
+
"@aws-sdk/middleware-user-agent": "3.928.0",
|
|
19101
|
+
"@aws-sdk/region-config-resolver": "3.925.0",
|
|
19088
19102
|
"@aws-sdk/types": "3.922.0",
|
|
19089
19103
|
"@aws-sdk/util-endpoints": "3.922.0",
|
|
19090
19104
|
"@aws-sdk/util-user-agent-browser": "3.922.0",
|
|
19091
|
-
"@aws-sdk/util-user-agent-node": "3.
|
|
19092
|
-
"@smithy/config-resolver": "^4.4.
|
|
19105
|
+
"@aws-sdk/util-user-agent-node": "3.928.0",
|
|
19106
|
+
"@smithy/config-resolver": "^4.4.2",
|
|
19093
19107
|
"@smithy/core": "^3.17.2",
|
|
19094
19108
|
"@smithy/fetch-http-handler": "^5.3.5",
|
|
19095
19109
|
"@smithy/hash-node": "^4.2.4",
|
|
@@ -19109,7 +19123,7 @@ var init_package = __esm({
|
|
|
19109
19123
|
"@smithy/util-body-length-browser": "^4.2.0",
|
|
19110
19124
|
"@smithy/util-body-length-node": "^4.2.1",
|
|
19111
19125
|
"@smithy/util-defaults-mode-browser": "^4.3.5",
|
|
19112
|
-
"@smithy/util-defaults-mode-node": "^4.2.
|
|
19126
|
+
"@smithy/util-defaults-mode-node": "^4.2.8",
|
|
19113
19127
|
"@smithy/util-endpoints": "^3.2.4",
|
|
19114
19128
|
"@smithy/util-middleware": "^4.2.4",
|
|
19115
19129
|
"@smithy/util-retry": "^4.2.4",
|
|
@@ -20530,7 +20544,7 @@ var require_package3 = __commonJS({
|
|
|
20530
20544
|
module2.exports = {
|
|
20531
20545
|
name: "@aws-sdk/client-sso",
|
|
20532
20546
|
description: "AWS SDK for JavaScript Sso Client for Node.js, Browser and React Native",
|
|
20533
|
-
version: "3.
|
|
20547
|
+
version: "3.928.0",
|
|
20534
20548
|
scripts: {
|
|
20535
20549
|
build: "concurrently 'yarn:build:cjs' 'yarn:build:es' 'yarn:build:types'",
|
|
20536
20550
|
"build:cjs": "node ../../scripts/compilation/inline client-sso",
|
|
@@ -20549,17 +20563,17 @@ var require_package3 = __commonJS({
|
|
|
20549
20563
|
dependencies: {
|
|
20550
20564
|
"@aws-crypto/sha256-browser": "5.2.0",
|
|
20551
20565
|
"@aws-crypto/sha256-js": "5.2.0",
|
|
20552
|
-
"@aws-sdk/core": "3.
|
|
20566
|
+
"@aws-sdk/core": "3.928.0",
|
|
20553
20567
|
"@aws-sdk/middleware-host-header": "3.922.0",
|
|
20554
20568
|
"@aws-sdk/middleware-logger": "3.922.0",
|
|
20555
20569
|
"@aws-sdk/middleware-recursion-detection": "3.922.0",
|
|
20556
|
-
"@aws-sdk/middleware-user-agent": "3.
|
|
20557
|
-
"@aws-sdk/region-config-resolver": "3.
|
|
20570
|
+
"@aws-sdk/middleware-user-agent": "3.928.0",
|
|
20571
|
+
"@aws-sdk/region-config-resolver": "3.925.0",
|
|
20558
20572
|
"@aws-sdk/types": "3.922.0",
|
|
20559
20573
|
"@aws-sdk/util-endpoints": "3.922.0",
|
|
20560
20574
|
"@aws-sdk/util-user-agent-browser": "3.922.0",
|
|
20561
|
-
"@aws-sdk/util-user-agent-node": "3.
|
|
20562
|
-
"@smithy/config-resolver": "^4.4.
|
|
20575
|
+
"@aws-sdk/util-user-agent-node": "3.928.0",
|
|
20576
|
+
"@smithy/config-resolver": "^4.4.2",
|
|
20563
20577
|
"@smithy/core": "^3.17.2",
|
|
20564
20578
|
"@smithy/fetch-http-handler": "^5.3.5",
|
|
20565
20579
|
"@smithy/hash-node": "^4.2.4",
|
|
@@ -20579,7 +20593,7 @@ var require_package3 = __commonJS({
|
|
|
20579
20593
|
"@smithy/util-body-length-browser": "^4.2.0",
|
|
20580
20594
|
"@smithy/util-body-length-node": "^4.2.1",
|
|
20581
20595
|
"@smithy/util-defaults-mode-browser": "^4.3.5",
|
|
20582
|
-
"@smithy/util-defaults-mode-node": "^4.2.
|
|
20596
|
+
"@smithy/util-defaults-mode-node": "^4.2.8",
|
|
20583
20597
|
"@smithy/util-endpoints": "^3.2.4",
|
|
20584
20598
|
"@smithy/util-middleware": "^4.2.4",
|
|
20585
20599
|
"@smithy/util-retry": "^4.2.4",
|
|
@@ -27873,6 +27887,162 @@ var init_BytePairEncodingCore = __esm({
|
|
|
27873
27887
|
}
|
|
27874
27888
|
});
|
|
27875
27889
|
|
|
27890
|
+
// node_modules/gpt-tokenizer/esm/functionCalling.js
|
|
27891
|
+
function countMessageTokens(message, countStringTokens) {
|
|
27892
|
+
let tokens = 0;
|
|
27893
|
+
if (message.role) {
|
|
27894
|
+
tokens += countStringTokens(message.role);
|
|
27895
|
+
}
|
|
27896
|
+
if (message.content) {
|
|
27897
|
+
tokens += countStringTokens(message.content);
|
|
27898
|
+
}
|
|
27899
|
+
if (message.name) {
|
|
27900
|
+
tokens += countStringTokens(message.name) + MESSAGE_NAME_TOKEN_OVERHEAD;
|
|
27901
|
+
}
|
|
27902
|
+
if (message.function_call) {
|
|
27903
|
+
const { name: name14, arguments: args } = message.function_call;
|
|
27904
|
+
if (name14) {
|
|
27905
|
+
tokens += countStringTokens(name14);
|
|
27906
|
+
}
|
|
27907
|
+
if (args) {
|
|
27908
|
+
tokens += countStringTokens(args);
|
|
27909
|
+
}
|
|
27910
|
+
tokens += FUNCTION_CALL_METADATA_TOKEN_OVERHEAD;
|
|
27911
|
+
}
|
|
27912
|
+
tokens += MESSAGE_TOKEN_OVERHEAD;
|
|
27913
|
+
if (message.role === "function") {
|
|
27914
|
+
tokens -= FUNCTION_ROLE_TOKEN_DISCOUNT;
|
|
27915
|
+
}
|
|
27916
|
+
return tokens;
|
|
27917
|
+
}
|
|
27918
|
+
function formatObjectProperties(obj, indent, formatType) {
|
|
27919
|
+
if (!obj.properties) {
|
|
27920
|
+
return "";
|
|
27921
|
+
}
|
|
27922
|
+
const lines = [];
|
|
27923
|
+
const requiredParams = new Set(obj.required ?? []);
|
|
27924
|
+
const indentString = " ".repeat(indent);
|
|
27925
|
+
for (const [name14, param] of Object.entries(obj.properties)) {
|
|
27926
|
+
if (param.description && indent < 2) {
|
|
27927
|
+
lines.push(`${indentString}// ${param.description}`);
|
|
27928
|
+
}
|
|
27929
|
+
const isRequired = requiredParams.has(name14);
|
|
27930
|
+
const formattedType = formatType(param, indent);
|
|
27931
|
+
lines.push(`${indentString}${name14}${isRequired ? "" : "?"}: ${formattedType},`);
|
|
27932
|
+
}
|
|
27933
|
+
return lines.join("\n");
|
|
27934
|
+
}
|
|
27935
|
+
function formatFunctionType(param, indent) {
|
|
27936
|
+
switch (param.type) {
|
|
27937
|
+
case "string":
|
|
27938
|
+
return param.enum?.map((value) => JSON.stringify(value)).join(" | ") ?? "string";
|
|
27939
|
+
case "integer":
|
|
27940
|
+
case "number":
|
|
27941
|
+
return param.enum?.map((value) => `${value}`).join(" | ") ?? "number";
|
|
27942
|
+
case "boolean":
|
|
27943
|
+
return "boolean";
|
|
27944
|
+
case "null":
|
|
27945
|
+
return "null";
|
|
27946
|
+
case "array":
|
|
27947
|
+
return param.items ? `${formatFunctionType(param.items, indent)}[]` : "any[]";
|
|
27948
|
+
case "object": {
|
|
27949
|
+
const inner = formatObjectProperties(param, indent + 2, formatFunctionType);
|
|
27950
|
+
const closingIndent = " ".repeat(indent);
|
|
27951
|
+
return `{
|
|
27952
|
+
${inner}
|
|
27953
|
+
${closingIndent}}`;
|
|
27954
|
+
}
|
|
27955
|
+
default:
|
|
27956
|
+
return "any";
|
|
27957
|
+
}
|
|
27958
|
+
}
|
|
27959
|
+
function formatFunctionDefinitions(functions) {
|
|
27960
|
+
const lines = ["namespace functions {", ""];
|
|
27961
|
+
for (const fn of functions) {
|
|
27962
|
+
if (fn.description) {
|
|
27963
|
+
lines.push(`// ${fn.description}`);
|
|
27964
|
+
}
|
|
27965
|
+
const { parameters } = fn;
|
|
27966
|
+
const properties = parameters?.properties;
|
|
27967
|
+
if (!parameters || !properties || Object.keys(properties).length === 0) {
|
|
27968
|
+
lines.push(`type ${fn.name} = () => any;`);
|
|
27969
|
+
} else {
|
|
27970
|
+
lines.push(`type ${fn.name} = (_: {`);
|
|
27971
|
+
const formattedProperties = formatObjectProperties(parameters, 0, formatFunctionType);
|
|
27972
|
+
if (formattedProperties.length > 0) {
|
|
27973
|
+
lines.push(formattedProperties);
|
|
27974
|
+
}
|
|
27975
|
+
lines.push("}) => any;");
|
|
27976
|
+
}
|
|
27977
|
+
lines.push("");
|
|
27978
|
+
}
|
|
27979
|
+
lines.push("} // namespace functions");
|
|
27980
|
+
return lines.join("\n");
|
|
27981
|
+
}
|
|
27982
|
+
function estimateTokensInFunctions(functions, countStringTokens) {
|
|
27983
|
+
const formatted = formatFunctionDefinitions(functions);
|
|
27984
|
+
let tokens = countStringTokens(formatted);
|
|
27985
|
+
tokens += FUNCTION_DEFINITION_TOKEN_OVERHEAD;
|
|
27986
|
+
return tokens;
|
|
27987
|
+
}
|
|
27988
|
+
function padSystemMessage(message, hasFunctions, isSystemPadded) {
|
|
27989
|
+
if (!hasFunctions || isSystemPadded || message.role !== "system") {
|
|
27990
|
+
return message;
|
|
27991
|
+
}
|
|
27992
|
+
if (!message.content || message.content.endsWith(NEWLINE)) {
|
|
27993
|
+
return message;
|
|
27994
|
+
}
|
|
27995
|
+
return {
|
|
27996
|
+
...message,
|
|
27997
|
+
content: `${message.content}${NEWLINE}`
|
|
27998
|
+
};
|
|
27999
|
+
}
|
|
28000
|
+
function computeChatCompletionTokenCount(request, countStringTokens) {
|
|
28001
|
+
const { messages, functions, function_call: functionCall } = request;
|
|
28002
|
+
const hasFunctions = Boolean(functions && functions.length > 0);
|
|
28003
|
+
let paddedSystem = false;
|
|
28004
|
+
let total = 0;
|
|
28005
|
+
for (const message of messages) {
|
|
28006
|
+
const messageToCount = padSystemMessage(message, hasFunctions, paddedSystem);
|
|
28007
|
+
if (messageToCount !== message && message.role === "system") {
|
|
28008
|
+
paddedSystem = true;
|
|
28009
|
+
} else if (message.role === "system" && hasFunctions && !paddedSystem) {
|
|
28010
|
+
paddedSystem = true;
|
|
28011
|
+
}
|
|
28012
|
+
total += countMessageTokens(messageToCount, countStringTokens);
|
|
28013
|
+
}
|
|
28014
|
+
total += COMPLETION_REQUEST_TOKEN_OVERHEAD;
|
|
28015
|
+
if (hasFunctions && functions) {
|
|
28016
|
+
total += estimateTokensInFunctions(functions, countStringTokens);
|
|
28017
|
+
if (messages.some((message) => message.role === "system")) {
|
|
28018
|
+
total -= SYSTEM_FUNCTION_TOKEN_DEDUCTION;
|
|
28019
|
+
}
|
|
28020
|
+
}
|
|
28021
|
+
if (functionCall && functionCall !== "auto") {
|
|
28022
|
+
if (functionCall === "none") {
|
|
28023
|
+
total += FUNCTION_CALL_NONE_TOKEN_OVERHEAD;
|
|
28024
|
+
} else if (typeof functionCall === "object" && functionCall.name) {
|
|
28025
|
+
total += countStringTokens(functionCall.name) + FUNCTION_CALL_NAME_TOKEN_OVERHEAD;
|
|
28026
|
+
}
|
|
28027
|
+
}
|
|
28028
|
+
return total;
|
|
28029
|
+
}
|
|
28030
|
+
var MESSAGE_TOKEN_OVERHEAD, MESSAGE_NAME_TOKEN_OVERHEAD, FUNCTION_ROLE_TOKEN_DISCOUNT, FUNCTION_CALL_METADATA_TOKEN_OVERHEAD, FUNCTION_DEFINITION_TOKEN_OVERHEAD, COMPLETION_REQUEST_TOKEN_OVERHEAD, FUNCTION_CALL_NAME_TOKEN_OVERHEAD, FUNCTION_CALL_NONE_TOKEN_OVERHEAD, SYSTEM_FUNCTION_TOKEN_DEDUCTION, NEWLINE;
|
|
28031
|
+
var init_functionCalling = __esm({
|
|
28032
|
+
"node_modules/gpt-tokenizer/esm/functionCalling.js"() {
|
|
28033
|
+
MESSAGE_TOKEN_OVERHEAD = 3;
|
|
28034
|
+
MESSAGE_NAME_TOKEN_OVERHEAD = 1;
|
|
28035
|
+
FUNCTION_ROLE_TOKEN_DISCOUNT = 2;
|
|
28036
|
+
FUNCTION_CALL_METADATA_TOKEN_OVERHEAD = 3;
|
|
28037
|
+
FUNCTION_DEFINITION_TOKEN_OVERHEAD = 9;
|
|
28038
|
+
COMPLETION_REQUEST_TOKEN_OVERHEAD = 3;
|
|
28039
|
+
FUNCTION_CALL_NAME_TOKEN_OVERHEAD = 4;
|
|
28040
|
+
FUNCTION_CALL_NONE_TOKEN_OVERHEAD = 1;
|
|
28041
|
+
SYSTEM_FUNCTION_TOKEN_DEDUCTION = 4;
|
|
28042
|
+
NEWLINE = "\n";
|
|
28043
|
+
}
|
|
28044
|
+
});
|
|
28045
|
+
|
|
27876
28046
|
// node_modules/gpt-tokenizer/esm/modelsChatEnabled.gen.js
|
|
27877
28047
|
var chatEnabledModels;
|
|
27878
28048
|
var init_modelsChatEnabled_gen = __esm({
|
|
@@ -27886,11 +28056,12 @@ var modelsMap_exports = {};
|
|
|
27886
28056
|
__export(modelsMap_exports, {
|
|
27887
28057
|
cl100k_base: () => cl100k_base,
|
|
27888
28058
|
o200k_base: () => o200k_base,
|
|
28059
|
+
o200k_harmony: () => o200k_harmony,
|
|
27889
28060
|
p50k_base: () => p50k_base,
|
|
27890
28061
|
p50k_edit: () => p50k_edit,
|
|
27891
28062
|
r50k_base: () => r50k_base
|
|
27892
28063
|
});
|
|
27893
|
-
var p50k_base, r50k_base, p50k_edit, cl100k_base, o200k_base;
|
|
28064
|
+
var p50k_base, r50k_base, p50k_edit, cl100k_base, o200k_base, o200k_harmony;
|
|
27894
28065
|
var init_modelsMap = __esm({
|
|
27895
28066
|
"node_modules/gpt-tokenizer/esm/modelsMap.js"() {
|
|
27896
28067
|
p50k_base = [
|
|
@@ -27965,11 +28136,15 @@ var init_modelsMap = __esm({
|
|
|
27965
28136
|
"davinci-002"
|
|
27966
28137
|
];
|
|
27967
28138
|
o200k_base = [];
|
|
28139
|
+
o200k_harmony = [
|
|
28140
|
+
"gpt-oss-20b",
|
|
28141
|
+
"gpt-oss-120b"
|
|
28142
|
+
];
|
|
27968
28143
|
}
|
|
27969
28144
|
});
|
|
27970
28145
|
|
|
27971
28146
|
// node_modules/gpt-tokenizer/esm/specialTokens.js
|
|
27972
|
-
var EndOfText, FimPrefix, FimMiddle, FimSuffix, ImStart, ImEnd, ImSep, EndOfPrompt;
|
|
28147
|
+
var EndOfText, FimPrefix, FimMiddle, FimSuffix, ImStart, ImEnd, ImSep, EndOfPrompt, HarmonyStartOfText, HarmonyStart, HarmonyEnd, HarmonyMessage, HarmonyChannel, HarmonyReturn, HarmonyConstrain, HarmonyCall;
|
|
27973
28148
|
var init_specialTokens = __esm({
|
|
27974
28149
|
"node_modules/gpt-tokenizer/esm/specialTokens.js"() {
|
|
27975
28150
|
EndOfText = "<|endoftext|>";
|
|
@@ -27980,6 +28155,14 @@ var init_specialTokens = __esm({
|
|
|
27980
28155
|
ImEnd = "<|im_end|>";
|
|
27981
28156
|
ImSep = "<|im_sep|>";
|
|
27982
28157
|
EndOfPrompt = "<|endofprompt|>";
|
|
28158
|
+
HarmonyStartOfText = "<|startoftext|>";
|
|
28159
|
+
HarmonyStart = "<|start|>";
|
|
28160
|
+
HarmonyEnd = "<|end|>";
|
|
28161
|
+
HarmonyMessage = "<|message|>";
|
|
28162
|
+
HarmonyChannel = "<|channel|>";
|
|
28163
|
+
HarmonyReturn = "<|return|>";
|
|
28164
|
+
HarmonyConstrain = "<|constrain|>";
|
|
28165
|
+
HarmonyCall = "<|call|>";
|
|
27983
28166
|
}
|
|
27984
28167
|
});
|
|
27985
28168
|
|
|
@@ -28006,11 +28189,16 @@ var init_mapping = __esm({
|
|
|
28006
28189
|
});
|
|
28007
28190
|
|
|
28008
28191
|
// node_modules/gpt-tokenizer/esm/encodingParams/constants.js
|
|
28009
|
-
var R50K_TOKEN_SPLIT_REGEX,
|
|
28192
|
+
var R50K_TOKEN_SPLIT_REGEX, CONTRACTION_SUFFIX_PATTERN, OPTIONAL_CONTRACTION_SUFFIX, CL100K_TOKEN_SPLIT_PATTERN, CL100K_TOKEN_SPLIT_REGEX, O200K_TOKEN_SPLIT_PATTERN, O200K_TOKEN_SPLIT_REGEX;
|
|
28010
28193
|
var init_constants2 = __esm({
|
|
28011
28194
|
"node_modules/gpt-tokenizer/esm/encodingParams/constants.js"() {
|
|
28012
28195
|
R50K_TOKEN_SPLIT_REGEX = /'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+/gu;
|
|
28013
|
-
|
|
28196
|
+
CONTRACTION_SUFFIX_PATTERN = String.raw`'(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`;
|
|
28197
|
+
OPTIONAL_CONTRACTION_SUFFIX = String.raw`(?:${CONTRACTION_SUFFIX_PATTERN})?`;
|
|
28198
|
+
CL100K_TOKEN_SPLIT_PATTERN = String.raw`${CONTRACTION_SUFFIX_PATTERN}|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s+$|\s*[\r\n]|\s+(?!\S)|\s`;
|
|
28199
|
+
CL100K_TOKEN_SPLIT_REGEX = new RegExp(CL100K_TOKEN_SPLIT_PATTERN, "gu");
|
|
28200
|
+
O200K_TOKEN_SPLIT_PATTERN = String.raw`[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+${OPTIONAL_CONTRACTION_SUFFIX}|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*${OPTIONAL_CONTRACTION_SUFFIX}|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+`;
|
|
28201
|
+
O200K_TOKEN_SPLIT_REGEX = new RegExp(O200K_TOKEN_SPLIT_PATTERN, "gu");
|
|
28014
28202
|
}
|
|
28015
28203
|
});
|
|
28016
28204
|
|
|
@@ -28027,7 +28215,7 @@ function Cl100KBase(bytePairRankDecoder) {
|
|
|
28027
28215
|
[EndOfPrompt, 100276]
|
|
28028
28216
|
]);
|
|
28029
28217
|
return {
|
|
28030
|
-
tokenSplitRegex:
|
|
28218
|
+
tokenSplitRegex: CL100K_TOKEN_SPLIT_REGEX,
|
|
28031
28219
|
bytePairRankDecoder,
|
|
28032
28220
|
specialTokensEncoder: specialTokenMapping
|
|
28033
28221
|
};
|
|
@@ -28041,26 +28229,69 @@ var init_cl100k_base = __esm({
|
|
|
28041
28229
|
|
|
28042
28230
|
// node_modules/gpt-tokenizer/esm/encodingParams/o200k_base.js
|
|
28043
28231
|
function O200KBase(bytePairRankDecoder) {
|
|
28044
|
-
const specialTokenMapping = /* @__PURE__ */ new Map([
|
|
28045
|
-
[EndOfText, 199999],
|
|
28046
|
-
[FimPrefix, 2e5],
|
|
28047
|
-
[FimMiddle, 200001],
|
|
28048
|
-
[FimSuffix, 200002],
|
|
28049
|
-
[ImStart, 200003],
|
|
28050
|
-
[ImEnd, 200004],
|
|
28051
|
-
[ImSep, 200005],
|
|
28052
|
-
[EndOfPrompt, 200006]
|
|
28053
|
-
]);
|
|
28054
28232
|
return {
|
|
28055
|
-
tokenSplitRegex:
|
|
28233
|
+
tokenSplitRegex: O200K_TOKEN_SPLIT_REGEX,
|
|
28056
28234
|
bytePairRankDecoder,
|
|
28057
|
-
specialTokensEncoder:
|
|
28235
|
+
specialTokensEncoder: createO200KSpecialTokenMap()
|
|
28058
28236
|
};
|
|
28059
28237
|
}
|
|
28238
|
+
var O200K_BASE_SPECIAL_TOKEN_ENTRIES, createO200KSpecialTokenMap;
|
|
28060
28239
|
var init_o200k_base2 = __esm({
|
|
28061
28240
|
"node_modules/gpt-tokenizer/esm/encodingParams/o200k_base.js"() {
|
|
28062
28241
|
init_specialTokens();
|
|
28063
28242
|
init_constants2();
|
|
28243
|
+
O200K_BASE_SPECIAL_TOKEN_ENTRIES = [
|
|
28244
|
+
[EndOfText, 199999],
|
|
28245
|
+
[FimPrefix, 2e5],
|
|
28246
|
+
[FimMiddle, 200001],
|
|
28247
|
+
[FimSuffix, 200002],
|
|
28248
|
+
[ImStart, 200003],
|
|
28249
|
+
[ImEnd, 200004],
|
|
28250
|
+
[ImSep, 200005],
|
|
28251
|
+
[EndOfPrompt, 200006]
|
|
28252
|
+
];
|
|
28253
|
+
createO200KSpecialTokenMap = () => new Map(O200K_BASE_SPECIAL_TOKEN_ENTRIES);
|
|
28254
|
+
}
|
|
28255
|
+
});
|
|
28256
|
+
|
|
28257
|
+
// node_modules/gpt-tokenizer/esm/encodingParams/o200k_harmony.js
|
|
28258
|
+
function O200KHarmony(bytePairRankDecoder) {
|
|
28259
|
+
const specialTokensEncoder = new Map(STATIC_SPECIAL_TOKEN_ENTRIES);
|
|
28260
|
+
for (let tokenId = RESERVED_TOKEN_RANGE_START; tokenId < RESERVED_TOKEN_RANGE_END; tokenId += 1) {
|
|
28261
|
+
specialTokensEncoder.set(`<|reserved_${tokenId}|>`, tokenId);
|
|
28262
|
+
}
|
|
28263
|
+
specialTokensEncoder.set(EndOfPrompt, 200018);
|
|
28264
|
+
return {
|
|
28265
|
+
tokenSplitRegex: O200K_TOKEN_SPLIT_REGEX,
|
|
28266
|
+
bytePairRankDecoder,
|
|
28267
|
+
specialTokensEncoder,
|
|
28268
|
+
chatFormatter: "harmony"
|
|
28269
|
+
};
|
|
28270
|
+
}
|
|
28271
|
+
var RESERVED_TOKEN_RANGE_START, RESERVED_TOKEN_RANGE_END, STATIC_SPECIAL_TOKEN_ENTRIES;
|
|
28272
|
+
var init_o200k_harmony = __esm({
|
|
28273
|
+
"node_modules/gpt-tokenizer/esm/encodingParams/o200k_harmony.js"() {
|
|
28274
|
+
init_specialTokens();
|
|
28275
|
+
init_constants2();
|
|
28276
|
+
RESERVED_TOKEN_RANGE_START = 200013;
|
|
28277
|
+
RESERVED_TOKEN_RANGE_END = 201088;
|
|
28278
|
+
STATIC_SPECIAL_TOKEN_ENTRIES = [
|
|
28279
|
+
[HarmonyStartOfText, 199998],
|
|
28280
|
+
[EndOfText, 199999],
|
|
28281
|
+
["<|reserved_200000|>", 2e5],
|
|
28282
|
+
["<|reserved_200001|>", 200001],
|
|
28283
|
+
[HarmonyReturn, 200002],
|
|
28284
|
+
[HarmonyConstrain, 200003],
|
|
28285
|
+
["<|reserved_200004|>", 200004],
|
|
28286
|
+
[HarmonyChannel, 200005],
|
|
28287
|
+
[HarmonyStart, 200006],
|
|
28288
|
+
[HarmonyEnd, 200007],
|
|
28289
|
+
[HarmonyMessage, 200008],
|
|
28290
|
+
["<|reserved_200009|>", 200009],
|
|
28291
|
+
["<|reserved_200010|>", 200010],
|
|
28292
|
+
["<|reserved_200011|>", 200011],
|
|
28293
|
+
[HarmonyCall, 200012]
|
|
28294
|
+
];
|
|
28064
28295
|
}
|
|
28065
28296
|
});
|
|
28066
28297
|
|
|
@@ -28134,6 +28365,8 @@ function getEncodingParams(encodingName, getMergeableRanks) {
|
|
|
28134
28365
|
return Cl100KBase(mergeableBytePairRanks);
|
|
28135
28366
|
case "o200k_base":
|
|
28136
28367
|
return O200KBase(mergeableBytePairRanks);
|
|
28368
|
+
case "o200k_harmony":
|
|
28369
|
+
return O200KHarmony(mergeableBytePairRanks);
|
|
28137
28370
|
default:
|
|
28138
28371
|
throw new Error(`Unknown encoding name: ${encodingName}`);
|
|
28139
28372
|
}
|
|
@@ -28142,6 +28375,7 @@ var init_modelParams = __esm({
|
|
|
28142
28375
|
"node_modules/gpt-tokenizer/esm/modelParams.js"() {
|
|
28143
28376
|
init_cl100k_base();
|
|
28144
28377
|
init_o200k_base2();
|
|
28378
|
+
init_o200k_harmony();
|
|
28145
28379
|
init_p50k_base();
|
|
28146
28380
|
init_p50k_edit();
|
|
28147
28381
|
init_r50k_base();
|
|
@@ -28154,6 +28388,7 @@ var init_GptEncoding = __esm({
|
|
|
28154
28388
|
"node_modules/gpt-tokenizer/esm/GptEncoding.js"() {
|
|
28155
28389
|
init_BytePairEncodingCore();
|
|
28156
28390
|
init_constants();
|
|
28391
|
+
init_functionCalling();
|
|
28157
28392
|
init_mapping();
|
|
28158
28393
|
init_modelParams();
|
|
28159
28394
|
init_specialTokens();
|
|
@@ -28172,8 +28407,10 @@ var init_GptEncoding = __esm({
|
|
|
28172
28407
|
specialTokensSet;
|
|
28173
28408
|
allSpecialTokenRegex;
|
|
28174
28409
|
defaultSpecialTokenConfig;
|
|
28410
|
+
chatFormatter;
|
|
28411
|
+
countChatCompletionTokens;
|
|
28175
28412
|
vocabularySize;
|
|
28176
|
-
constructor({ bytePairRankDecoder: mergeableBytePairRanks, specialTokensEncoder, expectedVocabularySize, modelName, modelSpec, ...rest }) {
|
|
28413
|
+
constructor({ bytePairRankDecoder: mergeableBytePairRanks, specialTokensEncoder, expectedVocabularySize, modelName, modelSpec, chatFormatter, ...rest }) {
|
|
28177
28414
|
this.specialTokensEncoder = specialTokensEncoder;
|
|
28178
28415
|
this.specialTokensSet = new Set(this.specialTokensEncoder.keys());
|
|
28179
28416
|
this.allSpecialTokenRegex = getSpecialTokenRegex(this.specialTokensSet);
|
|
@@ -28206,8 +28443,69 @@ var init_GptEncoding = __esm({
|
|
|
28206
28443
|
this.setMergeCacheSize = this.setMergeCacheSize.bind(this);
|
|
28207
28444
|
this.clearMergeCache = this.clearMergeCache.bind(this);
|
|
28208
28445
|
this.estimateCost = this.estimateCost.bind(this);
|
|
28446
|
+
if (modelSpec?.supported_features?.includes("function_calling")) {
|
|
28447
|
+
this.countChatCompletionTokens = this.countChatCompletionTokensInternal.bind(this);
|
|
28448
|
+
}
|
|
28209
28449
|
this.modelName = modelName;
|
|
28210
28450
|
this.modelSpec = modelSpec;
|
|
28451
|
+
this.chatFormatter = chatFormatter ?? "chatml";
|
|
28452
|
+
}
|
|
28453
|
+
*encodeHarmonyChatGenerator(chat, encodeOptions) {
|
|
28454
|
+
const harmonyStart = this.specialTokensEncoder.get(HarmonyStart);
|
|
28455
|
+
const harmonyMessage = this.specialTokensEncoder.get(HarmonyMessage);
|
|
28456
|
+
const harmonyEnd = this.specialTokensEncoder.get(HarmonyEnd);
|
|
28457
|
+
const harmonyReturn = this.specialTokensEncoder.get(HarmonyReturn);
|
|
28458
|
+
const harmonyCall = this.specialTokensEncoder.get(HarmonyCall);
|
|
28459
|
+
const harmonyChannel = this.specialTokensEncoder.get(HarmonyChannel);
|
|
28460
|
+
const harmonyConstrain = this.specialTokensEncoder.get(HarmonyConstrain);
|
|
28461
|
+
if (harmonyStart === void 0 || harmonyMessage === void 0 || harmonyEnd === void 0 || harmonyReturn === void 0 || harmonyCall === void 0 || harmonyChannel === void 0 || harmonyConstrain === void 0) {
|
|
28462
|
+
throw new Error("Harmony chat format requires dedicated special tokens.");
|
|
28463
|
+
}
|
|
28464
|
+
const encodeHeaderText = (text) => text.length > 0 ? this.encode(text) : [];
|
|
28465
|
+
const resolveTerminatorToken = (terminator) => {
|
|
28466
|
+
switch (terminator) {
|
|
28467
|
+
case "<|return|>":
|
|
28468
|
+
return harmonyReturn;
|
|
28469
|
+
case "<|call|>":
|
|
28470
|
+
return harmonyCall;
|
|
28471
|
+
// eslint-disable-next-line unicorn/no-useless-switch-case
|
|
28472
|
+
case "<|end|>":
|
|
28473
|
+
default:
|
|
28474
|
+
return harmonyEnd;
|
|
28475
|
+
}
|
|
28476
|
+
};
|
|
28477
|
+
for (const message of chat) {
|
|
28478
|
+
if (message.content === void 0) {
|
|
28479
|
+
throw new Error("Content must be defined for all messages.");
|
|
28480
|
+
}
|
|
28481
|
+
const roleOrName = message.name ?? message.role ?? "user";
|
|
28482
|
+
yield [harmonyStart];
|
|
28483
|
+
yield encodeHeaderText(roleOrName);
|
|
28484
|
+
const recipientInRole = message.recipient && (message.recipientPlacement === "role" || !message.channel);
|
|
28485
|
+
const recipientInChannel = message.recipient && !recipientInRole;
|
|
28486
|
+
if (recipientInRole) {
|
|
28487
|
+
yield encodeHeaderText(` to=${message.recipient}`);
|
|
28488
|
+
}
|
|
28489
|
+
if (message.channel) {
|
|
28490
|
+
yield [harmonyChannel];
|
|
28491
|
+
yield encodeHeaderText(message.channel);
|
|
28492
|
+
if (recipientInChannel) {
|
|
28493
|
+
yield encodeHeaderText(` to=${message.recipient}`);
|
|
28494
|
+
}
|
|
28495
|
+
}
|
|
28496
|
+
if (message.constraint) {
|
|
28497
|
+
yield [harmonyConstrain];
|
|
28498
|
+
yield encodeHeaderText(message.constraint);
|
|
28499
|
+
}
|
|
28500
|
+
yield [harmonyMessage];
|
|
28501
|
+
yield* this.encodeGenerator(message.content, encodeOptions);
|
|
28502
|
+
yield [resolveTerminatorToken(message.terminator)];
|
|
28503
|
+
}
|
|
28504
|
+
const assistantPrime = encodeOptions?.primeWithAssistantResponse ?? "assistant";
|
|
28505
|
+
if (assistantPrime.length > 0) {
|
|
28506
|
+
yield [harmonyStart];
|
|
28507
|
+
yield encodeHeaderText(assistantPrime);
|
|
28508
|
+
}
|
|
28211
28509
|
}
|
|
28212
28510
|
static getEncodingApi(encodingName, getMergeableRanks) {
|
|
28213
28511
|
const modelParams = getEncodingParams(encodingName, getMergeableRanks);
|
|
@@ -28280,9 +28578,16 @@ var init_GptEncoding = __esm({
|
|
|
28280
28578
|
throw new Error("Model name must be provided either during initialization or passed in to the method.");
|
|
28281
28579
|
}
|
|
28282
28580
|
const params = chatModelParams[model];
|
|
28581
|
+
if (!params) {
|
|
28582
|
+
throw new Error(`Model '${model}' does not support chat.`);
|
|
28583
|
+
}
|
|
28584
|
+
if (this.chatFormatter === "harmony") {
|
|
28585
|
+
yield* this.encodeHarmonyChatGenerator(chat, encodeOptions);
|
|
28586
|
+
return;
|
|
28587
|
+
}
|
|
28283
28588
|
const chatStartToken = this.specialTokensEncoder.get(ImStart);
|
|
28284
28589
|
const chatEndToken = this.specialTokensEncoder.get(ImEnd);
|
|
28285
|
-
if (
|
|
28590
|
+
if (chatStartToken === void 0 || chatEndToken === void 0) {
|
|
28286
28591
|
throw new Error(`Model '${model}' does not support chat.`);
|
|
28287
28592
|
}
|
|
28288
28593
|
const allowedSpecial = /* @__PURE__ */ new Set([ImSep]);
|
|
@@ -28305,8 +28610,11 @@ var init_GptEncoding = __esm({
|
|
|
28305
28610
|
yield [chatEndToken];
|
|
28306
28611
|
yield encodedMessageSeparator;
|
|
28307
28612
|
}
|
|
28308
|
-
|
|
28309
|
-
|
|
28613
|
+
const assistantPrime = encodeOptions?.primeWithAssistantResponse ?? "assistant";
|
|
28614
|
+
if (assistantPrime.length > 0) {
|
|
28615
|
+
yield [chatStartToken];
|
|
28616
|
+
yield* this.encodeGenerator(assistantPrime, encodeOptions);
|
|
28617
|
+
}
|
|
28310
28618
|
if (encodedRoleSeparator.length > 0) {
|
|
28311
28619
|
yield encodedRoleSeparator;
|
|
28312
28620
|
}
|
|
@@ -28364,6 +28672,15 @@ var init_GptEncoding = __esm({
|
|
|
28364
28672
|
}
|
|
28365
28673
|
return count;
|
|
28366
28674
|
}
|
|
28675
|
+
countStringTokens(text) {
|
|
28676
|
+
if (!text) {
|
|
28677
|
+
return 0;
|
|
28678
|
+
}
|
|
28679
|
+
return this.bytePairEncodingCoreProcessor.countNative(text);
|
|
28680
|
+
}
|
|
28681
|
+
countChatCompletionTokensInternal(request) {
|
|
28682
|
+
return computeChatCompletionTokenCount(request, (text) => this.countStringTokens(text));
|
|
28683
|
+
}
|
|
28367
28684
|
setMergeCacheSize(size) {
|
|
28368
28685
|
this.bytePairEncodingCoreProcessor.setMergeCacheSize(size);
|
|
28369
28686
|
}
|
|
@@ -29055,10 +29372,22 @@ function extractThinkingContent(xmlString) {
|
|
|
29055
29372
|
return thinkingMatch ? thinkingMatch[1].trim() : null;
|
|
29056
29373
|
}
|
|
29057
29374
|
function checkAttemptCompleteRecovery(cleanedXmlString, validTools = []) {
|
|
29058
|
-
const
|
|
29059
|
-
if (
|
|
29060
|
-
const
|
|
29061
|
-
const
|
|
29375
|
+
const openTagIndex = cleanedXmlString.indexOf("<attempt_completion>");
|
|
29376
|
+
if (openTagIndex !== -1) {
|
|
29377
|
+
const afterOpenTag = cleanedXmlString.substring(openTagIndex + "<attempt_completion>".length);
|
|
29378
|
+
const closeTagIndex = cleanedXmlString.lastIndexOf("</attempt_completion>");
|
|
29379
|
+
let content;
|
|
29380
|
+
let hasClosingTag = false;
|
|
29381
|
+
if (closeTagIndex !== -1 && closeTagIndex >= openTagIndex + "<attempt_completion>".length) {
|
|
29382
|
+
content = cleanedXmlString.substring(
|
|
29383
|
+
openTagIndex + "<attempt_completion>".length,
|
|
29384
|
+
closeTagIndex
|
|
29385
|
+
).trim();
|
|
29386
|
+
hasClosingTag = true;
|
|
29387
|
+
} else {
|
|
29388
|
+
content = afterOpenTag.trim();
|
|
29389
|
+
hasClosingTag = false;
|
|
29390
|
+
}
|
|
29062
29391
|
if (content) {
|
|
29063
29392
|
return {
|
|
29064
29393
|
toolName: "attempt_completion",
|
|
@@ -33375,7 +33704,15 @@ function parseXmlToolCall(xmlString, validTools = DEFAULT_VALID_TOOLS) {
|
|
|
33375
33704
|
if (openIndex === -1) {
|
|
33376
33705
|
continue;
|
|
33377
33706
|
}
|
|
33378
|
-
let closeIndex
|
|
33707
|
+
let closeIndex;
|
|
33708
|
+
if (toolName === "attempt_completion") {
|
|
33709
|
+
closeIndex = xmlString.lastIndexOf(closeTag);
|
|
33710
|
+
if (closeIndex !== -1 && closeIndex <= openIndex + openTag.length) {
|
|
33711
|
+
closeIndex = -1;
|
|
33712
|
+
}
|
|
33713
|
+
} else {
|
|
33714
|
+
closeIndex = xmlString.indexOf(closeTag, openIndex + openTag.length);
|
|
33715
|
+
}
|
|
33379
33716
|
let hasClosingTag = closeIndex !== -1;
|
|
33380
33717
|
if (closeIndex === -1) {
|
|
33381
33718
|
closeIndex = xmlString.length;
|
|
@@ -80064,6 +80401,170 @@ var init_FallbackManager = __esm({
|
|
|
80064
80401
|
}
|
|
80065
80402
|
});
|
|
80066
80403
|
|
|
80404
|
+
// src/agent/contextCompactor.js
|
|
80405
|
+
var contextCompactor_exports = {};
|
|
80406
|
+
__export(contextCompactor_exports, {
|
|
80407
|
+
calculateCompactionStats: () => calculateCompactionStats,
|
|
80408
|
+
compactMessages: () => compactMessages,
|
|
80409
|
+
handleContextLimitError: () => handleContextLimitError,
|
|
80410
|
+
identifyMessageSegments: () => identifyMessageSegments,
|
|
80411
|
+
isContextLimitError: () => isContextLimitError
|
|
80412
|
+
});
|
|
80413
|
+
function isContextLimitError(error2) {
|
|
80414
|
+
if (!error2) return false;
|
|
80415
|
+
const errorMessage = (typeof error2 === "string" ? error2 : error2?.message || "").toLowerCase();
|
|
80416
|
+
const errorString = error2.toString().toLowerCase();
|
|
80417
|
+
for (const pattern of CONTEXT_LIMIT_ERROR_PATTERNS) {
|
|
80418
|
+
const lowerPattern = pattern.toLowerCase();
|
|
80419
|
+
if (errorMessage.includes(lowerPattern) || errorString.includes(lowerPattern)) {
|
|
80420
|
+
return true;
|
|
80421
|
+
}
|
|
80422
|
+
}
|
|
80423
|
+
return false;
|
|
80424
|
+
}
|
|
80425
|
+
function identifyMessageSegments(messages) {
|
|
80426
|
+
const segments = [];
|
|
80427
|
+
let currentSegment = null;
|
|
80428
|
+
for (let i3 = 0; i3 < messages.length; i3++) {
|
|
80429
|
+
const msg = messages[i3];
|
|
80430
|
+
if (msg.role === "system") {
|
|
80431
|
+
continue;
|
|
80432
|
+
}
|
|
80433
|
+
if (msg.role === "user") {
|
|
80434
|
+
const content = typeof msg.content === "string" ? msg.content : "";
|
|
80435
|
+
const isToolResult = content.includes("<tool_result>");
|
|
80436
|
+
if (isToolResult && currentSegment) {
|
|
80437
|
+
currentSegment.finalIndex = i3;
|
|
80438
|
+
segments.push(currentSegment);
|
|
80439
|
+
currentSegment = null;
|
|
80440
|
+
} else {
|
|
80441
|
+
if (currentSegment) {
|
|
80442
|
+
segments.push(currentSegment);
|
|
80443
|
+
}
|
|
80444
|
+
currentSegment = {
|
|
80445
|
+
userIndex: i3,
|
|
80446
|
+
monologueIndices: [],
|
|
80447
|
+
finalIndex: null
|
|
80448
|
+
};
|
|
80449
|
+
}
|
|
80450
|
+
}
|
|
80451
|
+
if (msg.role === "assistant" && currentSegment) {
|
|
80452
|
+
const content = typeof msg.content === "string" ? msg.content : "";
|
|
80453
|
+
if (content.includes("<attempt_completion>") || content.includes("attempt_completion")) {
|
|
80454
|
+
currentSegment.monologueIndices.push(i3);
|
|
80455
|
+
currentSegment.finalIndex = i3;
|
|
80456
|
+
segments.push(currentSegment);
|
|
80457
|
+
currentSegment = null;
|
|
80458
|
+
} else {
|
|
80459
|
+
currentSegment.monologueIndices.push(i3);
|
|
80460
|
+
}
|
|
80461
|
+
}
|
|
80462
|
+
}
|
|
80463
|
+
if (currentSegment) {
|
|
80464
|
+
segments.push(currentSegment);
|
|
80465
|
+
}
|
|
80466
|
+
return segments;
|
|
80467
|
+
}
|
|
80468
|
+
function compactMessages(messages, options = {}) {
|
|
80469
|
+
const {
|
|
80470
|
+
keepLastSegment = true,
|
|
80471
|
+
minSegmentsToKeep = 1
|
|
80472
|
+
} = options;
|
|
80473
|
+
if (!messages || messages.length === 0) {
|
|
80474
|
+
return messages;
|
|
80475
|
+
}
|
|
80476
|
+
const segments = identifyMessageSegments(messages);
|
|
80477
|
+
if (segments.length === 0) {
|
|
80478
|
+
return messages;
|
|
80479
|
+
}
|
|
80480
|
+
const segmentsToPreserve = keepLastSegment ? Math.max(minSegmentsToKeep, 1) : minSegmentsToKeep;
|
|
80481
|
+
const compactableSegments = segments.slice(0, -segmentsToPreserve);
|
|
80482
|
+
const preservedSegments = segments.slice(-segmentsToPreserve);
|
|
80483
|
+
const indicesToKeep = /* @__PURE__ */ new Set();
|
|
80484
|
+
messages.forEach((msg, idx) => {
|
|
80485
|
+
if (msg.role === "system") {
|
|
80486
|
+
indicesToKeep.add(idx);
|
|
80487
|
+
}
|
|
80488
|
+
});
|
|
80489
|
+
compactableSegments.forEach((segment) => {
|
|
80490
|
+
indicesToKeep.add(segment.userIndex);
|
|
80491
|
+
if (segment.finalIndex !== null) {
|
|
80492
|
+
indicesToKeep.add(segment.finalIndex);
|
|
80493
|
+
}
|
|
80494
|
+
});
|
|
80495
|
+
preservedSegments.forEach((segment) => {
|
|
80496
|
+
indicesToKeep.add(segment.userIndex);
|
|
80497
|
+
segment.monologueIndices.forEach((idx) => indicesToKeep.add(idx));
|
|
80498
|
+
if (segment.finalIndex !== null) {
|
|
80499
|
+
indicesToKeep.add(segment.finalIndex);
|
|
80500
|
+
}
|
|
80501
|
+
});
|
|
80502
|
+
const compactedMessages = messages.filter((_2, idx) => indicesToKeep.has(idx));
|
|
80503
|
+
return compactedMessages;
|
|
80504
|
+
}
|
|
80505
|
+
function calculateCompactionStats(originalMessages, compactedMessages) {
|
|
80506
|
+
const originalCount = originalMessages.length;
|
|
80507
|
+
const compactedCount = compactedMessages.length;
|
|
80508
|
+
const removed = originalCount - compactedCount;
|
|
80509
|
+
const reductionPercent = originalCount > 0 ? (removed / originalCount * 100).toFixed(1) : 0;
|
|
80510
|
+
const estimateTokens = (msgs) => {
|
|
80511
|
+
return msgs.reduce((sum, msg) => {
|
|
80512
|
+
const content = typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content);
|
|
80513
|
+
return sum + Math.ceil(content.length / 4);
|
|
80514
|
+
}, 0);
|
|
80515
|
+
};
|
|
80516
|
+
const originalTokens = estimateTokens(originalMessages);
|
|
80517
|
+
const compactedTokens = estimateTokens(compactedMessages);
|
|
80518
|
+
const tokensSaved = originalTokens - compactedTokens;
|
|
80519
|
+
return {
|
|
80520
|
+
originalCount,
|
|
80521
|
+
compactedCount,
|
|
80522
|
+
removed,
|
|
80523
|
+
reductionPercent: parseFloat(reductionPercent),
|
|
80524
|
+
originalTokens,
|
|
80525
|
+
compactedTokens,
|
|
80526
|
+
tokensSaved
|
|
80527
|
+
};
|
|
80528
|
+
}
|
|
80529
|
+
function handleContextLimitError(error2, messages, options = {}) {
|
|
80530
|
+
if (!isContextLimitError(error2)) {
|
|
80531
|
+
return null;
|
|
80532
|
+
}
|
|
80533
|
+
const compactedMessages = compactMessages(messages, options);
|
|
80534
|
+
const stats = calculateCompactionStats(messages, compactedMessages);
|
|
80535
|
+
return {
|
|
80536
|
+
compacted: true,
|
|
80537
|
+
messages: compactedMessages,
|
|
80538
|
+
stats
|
|
80539
|
+
};
|
|
80540
|
+
}
|
|
80541
|
+
var CONTEXT_LIMIT_ERROR_PATTERNS;
|
|
80542
|
+
var init_contextCompactor = __esm({
|
|
80543
|
+
"src/agent/contextCompactor.js"() {
|
|
80544
|
+
"use strict";
|
|
80545
|
+
CONTEXT_LIMIT_ERROR_PATTERNS = [
|
|
80546
|
+
// Anthropic
|
|
80547
|
+
"context_length_exceeded",
|
|
80548
|
+
"prompt is too long",
|
|
80549
|
+
// OpenAI
|
|
80550
|
+
"maximum context length",
|
|
80551
|
+
"context length is",
|
|
80552
|
+
// Google/Gemini
|
|
80553
|
+
"input token count exceeds",
|
|
80554
|
+
"token limit exceeded",
|
|
80555
|
+
// Generic patterns
|
|
80556
|
+
"context window",
|
|
80557
|
+
"too many tokens",
|
|
80558
|
+
"token limit",
|
|
80559
|
+
"context limit",
|
|
80560
|
+
"exceed",
|
|
80561
|
+
// Catches "exceeds", "exceed maximum", etc.
|
|
80562
|
+
"over the limit",
|
|
80563
|
+
"maximum tokens"
|
|
80564
|
+
];
|
|
80565
|
+
}
|
|
80566
|
+
});
|
|
80567
|
+
|
|
80067
80568
|
// src/agent/ProbeAgent.js
|
|
80068
80569
|
var ProbeAgent_exports = {};
|
|
80069
80570
|
__export(ProbeAgent_exports, {
|
|
@@ -80098,6 +80599,7 @@ var init_ProbeAgent = __esm({
|
|
|
80098
80599
|
init_mcp();
|
|
80099
80600
|
init_RetryManager();
|
|
80100
80601
|
init_FallbackManager();
|
|
80602
|
+
init_contextCompactor();
|
|
80101
80603
|
import_dotenv.default.config();
|
|
80102
80604
|
MAX_TOOL_ITERATIONS = (() => {
|
|
80103
80605
|
const val = parseInt(process.env.MAX_TOOL_ITERATIONS || "30", 10);
|
|
@@ -80133,6 +80635,8 @@ var init_ProbeAgent = __esm({
|
|
|
80133
80635
|
* @param {Array} [options.mcpServers] - Deprecated, use mcpConfig instead
|
|
80134
80636
|
* @param {Object} [options.storageAdapter] - Custom storage adapter for history management
|
|
80135
80637
|
* @param {Object} [options.hooks] - Hook callbacks for events (e.g., {'tool:start': callback})
|
|
80638
|
+
* @param {Array<string>|null} [options.allowedTools] - List of allowed tool names. Use ['*'] for all tools (default), [] or null for no tools (raw AI mode), or specific tool names like ['search', 'query', 'extract']. Supports exclusion with '!' prefix (e.g., ['*', '!bash'])
|
|
80639
|
+
* @param {boolean} [options.disableTools=false] - Convenience flag to disable all tools (equivalent to allowedTools: []). Takes precedence over allowedTools if set.
|
|
80136
80640
|
* @param {Object} [options.retry] - Retry configuration
|
|
80137
80641
|
* @param {number} [options.retry.maxRetries=3] - Maximum retry attempts per provider
|
|
80138
80642
|
* @param {number} [options.retry.initialDelay=1000] - Initial delay in ms
|
|
@@ -80166,6 +80670,8 @@ var init_ProbeAgent = __esm({
|
|
|
80166
80670
|
this.maxIterations = options.maxIterations || null;
|
|
80167
80671
|
this.disableMermaidValidation = !!options.disableMermaidValidation;
|
|
80168
80672
|
this.disableJsonValidation = !!options.disableJsonValidation;
|
|
80673
|
+
const effectiveAllowedTools = options.disableTools ? [] : options.allowedTools;
|
|
80674
|
+
this.allowedTools = this._parseAllowedTools(effectiveAllowedTools);
|
|
80169
80675
|
this.storageAdapter = options.storageAdapter || new InMemoryStorageAdapter();
|
|
80170
80676
|
this.hooks = new HookManager();
|
|
80171
80677
|
if (options.hooks) {
|
|
@@ -80209,6 +80715,61 @@ var init_ProbeAgent = __esm({
|
|
|
80209
80715
|
this.fallbackManager = null;
|
|
80210
80716
|
this.initializeModel();
|
|
80211
80717
|
}
|
|
80718
|
+
/**
|
|
80719
|
+
* Parse allowedTools configuration
|
|
80720
|
+
* @param {Array<string>|null|undefined} allowedTools - Tool filtering configuration
|
|
80721
|
+
* @returns {Object} Parsed configuration with isEnabled method
|
|
80722
|
+
* @private
|
|
80723
|
+
*/
|
|
80724
|
+
_parseAllowedTools(allowedTools) {
|
|
80725
|
+
const matchesPattern2 = (toolName, pattern) => {
|
|
80726
|
+
if (!pattern.includes("*")) {
|
|
80727
|
+
return toolName === pattern;
|
|
80728
|
+
}
|
|
80729
|
+
const regexPattern = pattern.replace(/\*/g, ".*");
|
|
80730
|
+
return new RegExp(`^${regexPattern}$`).test(toolName);
|
|
80731
|
+
};
|
|
80732
|
+
if (!allowedTools || Array.isArray(allowedTools) && allowedTools.includes("*")) {
|
|
80733
|
+
const exclusions = Array.isArray(allowedTools) ? allowedTools.filter((t3) => t3.startsWith("!")).map((t3) => t3.slice(1)) : [];
|
|
80734
|
+
return {
|
|
80735
|
+
mode: "all",
|
|
80736
|
+
exclusions,
|
|
80737
|
+
isEnabled: (toolName) => !exclusions.some((pattern) => matchesPattern2(toolName, pattern))
|
|
80738
|
+
};
|
|
80739
|
+
}
|
|
80740
|
+
if (Array.isArray(allowedTools) && allowedTools.length === 0) {
|
|
80741
|
+
return {
|
|
80742
|
+
mode: "none",
|
|
80743
|
+
isEnabled: () => false
|
|
80744
|
+
};
|
|
80745
|
+
}
|
|
80746
|
+
const allowedPatterns = allowedTools.filter((t3) => !t3.startsWith("!"));
|
|
80747
|
+
return {
|
|
80748
|
+
mode: "whitelist",
|
|
80749
|
+
allowed: allowedPatterns,
|
|
80750
|
+
isEnabled: (toolName) => allowedPatterns.some((pattern) => matchesPattern2(toolName, pattern))
|
|
80751
|
+
};
|
|
80752
|
+
}
|
|
80753
|
+
/**
|
|
80754
|
+
* Check if an MCP tool is allowed based on allowedTools configuration
|
|
80755
|
+
* Uses mcp__ prefix convention (like Claude Code)
|
|
80756
|
+
* @param {string} toolName - The MCP tool name (without mcp__ prefix)
|
|
80757
|
+
* @returns {boolean} - Whether the tool is allowed
|
|
80758
|
+
* @private
|
|
80759
|
+
*/
|
|
80760
|
+
_isMcpToolAllowed(toolName) {
|
|
80761
|
+
const mcpToolName = `mcp__${toolName}`;
|
|
80762
|
+
return this.allowedTools.isEnabled(mcpToolName) || this.allowedTools.isEnabled(toolName);
|
|
80763
|
+
}
|
|
80764
|
+
/**
|
|
80765
|
+
* Filter MCP tools based on allowedTools configuration
|
|
80766
|
+
* @param {string[]} mcpToolNames - Array of MCP tool names
|
|
80767
|
+
* @returns {string[]} - Filtered array of allowed MCP tool names
|
|
80768
|
+
* @private
|
|
80769
|
+
*/
|
|
80770
|
+
_filterMcpTools(mcpToolNames) {
|
|
80771
|
+
return mcpToolNames.filter((toolName) => this._isMcpToolAllowed(toolName));
|
|
80772
|
+
}
|
|
80212
80773
|
/**
|
|
80213
80774
|
* Initialize the agent asynchronously (must be called after constructor)
|
|
80214
80775
|
* This method initializes MCP and merges MCP tools into the tool list, and loads history from storage
|
|
@@ -80235,7 +80796,11 @@ var init_ProbeAgent = __esm({
|
|
|
80235
80796
|
if (this.mcpBridge) {
|
|
80236
80797
|
const mcpTools = this.mcpBridge.mcpTools || {};
|
|
80237
80798
|
for (const [toolName, toolImpl] of Object.entries(mcpTools)) {
|
|
80238
|
-
this.
|
|
80799
|
+
if (this._isMcpToolAllowed(toolName)) {
|
|
80800
|
+
this.toolImplementations[toolName] = toolImpl;
|
|
80801
|
+
} else if (this.debug) {
|
|
80802
|
+
console.error(`[DEBUG] MCP tool '${toolName}' filtered out by allowedTools`);
|
|
80803
|
+
}
|
|
80239
80804
|
}
|
|
80240
80805
|
}
|
|
80241
80806
|
if (this.debug) {
|
|
@@ -80855,7 +81420,11 @@ var init_ProbeAgent = __esm({
|
|
|
80855
81420
|
if (this.mcpBridge) {
|
|
80856
81421
|
const mcpTools = this.mcpBridge.mcpTools || {};
|
|
80857
81422
|
for (const [toolName, toolImpl] of Object.entries(mcpTools)) {
|
|
80858
|
-
this.
|
|
81423
|
+
if (this._isMcpToolAllowed(toolName)) {
|
|
81424
|
+
this.toolImplementations[toolName] = toolImpl;
|
|
81425
|
+
} else if (this.debug) {
|
|
81426
|
+
console.error(`[DEBUG] MCP tool '${toolName}' filtered out by allowedTools`);
|
|
81427
|
+
}
|
|
80859
81428
|
}
|
|
80860
81429
|
}
|
|
80861
81430
|
} catch (error2) {
|
|
@@ -80865,27 +81434,49 @@ var init_ProbeAgent = __esm({
|
|
|
80865
81434
|
}
|
|
80866
81435
|
}
|
|
80867
81436
|
}
|
|
80868
|
-
let toolDefinitions =
|
|
80869
|
-
|
|
80870
|
-
|
|
80871
|
-
|
|
80872
|
-
${listFilesToolDefinition}
|
|
80873
|
-
${searchFilesToolDefinition}
|
|
80874
|
-
${attemptCompletionToolDefinition}
|
|
81437
|
+
let toolDefinitions = "";
|
|
81438
|
+
const isToolAllowed = (toolName) => this.allowedTools.isEnabled(toolName);
|
|
81439
|
+
if (isToolAllowed("search")) {
|
|
81440
|
+
toolDefinitions += `${searchToolDefinition}
|
|
80875
81441
|
`;
|
|
80876
|
-
|
|
81442
|
+
}
|
|
81443
|
+
if (isToolAllowed("query")) {
|
|
81444
|
+
toolDefinitions += `${queryToolDefinition}
|
|
81445
|
+
`;
|
|
81446
|
+
}
|
|
81447
|
+
if (isToolAllowed("extract")) {
|
|
81448
|
+
toolDefinitions += `${extractToolDefinition}
|
|
81449
|
+
`;
|
|
81450
|
+
}
|
|
81451
|
+
if (isToolAllowed("listFiles")) {
|
|
81452
|
+
toolDefinitions += `${listFilesToolDefinition}
|
|
81453
|
+
`;
|
|
81454
|
+
}
|
|
81455
|
+
if (isToolAllowed("searchFiles")) {
|
|
81456
|
+
toolDefinitions += `${searchFilesToolDefinition}
|
|
81457
|
+
`;
|
|
81458
|
+
}
|
|
81459
|
+
if (this.allowEdit && isToolAllowed("implement")) {
|
|
80877
81460
|
toolDefinitions += `${implementToolDefinition}
|
|
80878
81461
|
`;
|
|
81462
|
+
}
|
|
81463
|
+
if (this.allowEdit && isToolAllowed("edit")) {
|
|
80879
81464
|
toolDefinitions += `${editToolDefinition}
|
|
80880
81465
|
`;
|
|
81466
|
+
}
|
|
81467
|
+
if (this.allowEdit && isToolAllowed("create")) {
|
|
80881
81468
|
toolDefinitions += `${createToolDefinition}
|
|
80882
81469
|
`;
|
|
80883
81470
|
}
|
|
80884
|
-
if (this.enableBash) {
|
|
81471
|
+
if (this.enableBash && isToolAllowed("bash")) {
|
|
80885
81472
|
toolDefinitions += `${bashToolDefinition}
|
|
80886
81473
|
`;
|
|
80887
81474
|
}
|
|
80888
|
-
if (
|
|
81475
|
+
if (isToolAllowed("attempt_completion")) {
|
|
81476
|
+
toolDefinitions += `${attemptCompletionToolDefinition}
|
|
81477
|
+
`;
|
|
81478
|
+
}
|
|
81479
|
+
if (this.enableDelegate && isToolAllowed("delegate")) {
|
|
80889
81480
|
toolDefinitions += `${delegateToolDefinition}
|
|
80890
81481
|
`;
|
|
80891
81482
|
}
|
|
@@ -81047,11 +81638,14 @@ ${xmlToolGuidelines}
|
|
|
81047
81638
|
${toolDefinitions}
|
|
81048
81639
|
`;
|
|
81049
81640
|
if (this.mcpBridge && this.mcpBridge.getToolNames().length > 0) {
|
|
81050
|
-
|
|
81641
|
+
const allMcpTools = this.mcpBridge.getToolNames();
|
|
81642
|
+
const allowedMcpTools = this._filterMcpTools(allMcpTools);
|
|
81643
|
+
if (allowedMcpTools.length > 0) {
|
|
81644
|
+
systemMessage += `
|
|
81051
81645
|
## MCP Tools (JSON parameters in <params> tag)
|
|
81052
81646
|
`;
|
|
81053
|
-
|
|
81054
|
-
|
|
81647
|
+
systemMessage += this.mcpBridge.getXmlToolDefinitions(allowedMcpTools);
|
|
81648
|
+
systemMessage += `
|
|
81055
81649
|
|
|
81056
81650
|
For MCP tools, use JSON format within the params tag, e.g.:
|
|
81057
81651
|
<mcp_tool>
|
|
@@ -81060,6 +81654,7 @@ For MCP tools, use JSON format within the params tag, e.g.:
|
|
|
81060
81654
|
</params>
|
|
81061
81655
|
</mcp_tool>
|
|
81062
81656
|
`;
|
|
81657
|
+
}
|
|
81063
81658
|
}
|
|
81064
81659
|
const searchDirectory = this.allowedFolders.length > 0 ? this.allowedFolders[0] : process.cwd();
|
|
81065
81660
|
if (this.debug) {
|
|
@@ -81216,44 +81811,83 @@ You are working with a repository located at: ${searchDirectory}
|
|
|
81216
81811
|
}
|
|
81217
81812
|
}
|
|
81218
81813
|
let assistantResponseContent = "";
|
|
81219
|
-
|
|
81220
|
-
|
|
81221
|
-
|
|
81222
|
-
const
|
|
81223
|
-
|
|
81224
|
-
|
|
81225
|
-
|
|
81226
|
-
|
|
81227
|
-
|
|
81228
|
-
|
|
81229
|
-
|
|
81230
|
-
|
|
81231
|
-
|
|
81232
|
-
|
|
81814
|
+
let compactionAttempted = false;
|
|
81815
|
+
while (true) {
|
|
81816
|
+
try {
|
|
81817
|
+
const executeAIRequest = async () => {
|
|
81818
|
+
const messagesForAI = this.prepareMessagesWithImages(currentMessages);
|
|
81819
|
+
const result = await this.streamTextWithRetryAndFallback({
|
|
81820
|
+
model: this.provider(this.model),
|
|
81821
|
+
messages: messagesForAI,
|
|
81822
|
+
maxTokens: maxResponseTokens,
|
|
81823
|
+
temperature: 0.3
|
|
81824
|
+
});
|
|
81825
|
+
const usagePromise = result.usage;
|
|
81826
|
+
for await (const delta of result.textStream) {
|
|
81827
|
+
assistantResponseContent += delta;
|
|
81828
|
+
if (options.onStream) {
|
|
81829
|
+
options.onStream(delta);
|
|
81830
|
+
}
|
|
81831
|
+
}
|
|
81832
|
+
const usage = await usagePromise;
|
|
81833
|
+
if (usage) {
|
|
81834
|
+
this.tokenCounter.recordUsage(usage, result.experimental_providerMetadata);
|
|
81233
81835
|
}
|
|
81836
|
+
return result;
|
|
81837
|
+
};
|
|
81838
|
+
if (this.tracer) {
|
|
81839
|
+
await this.tracer.withSpan("ai.request", executeAIRequest, {
|
|
81840
|
+
"ai.model": this.model,
|
|
81841
|
+
"ai.provider": this.clientApiProvider || "auto",
|
|
81842
|
+
"iteration": currentIteration,
|
|
81843
|
+
"max_tokens": maxResponseTokens,
|
|
81844
|
+
"temperature": 0.3,
|
|
81845
|
+
"message_count": currentMessages.length
|
|
81846
|
+
});
|
|
81847
|
+
} else {
|
|
81848
|
+
await executeAIRequest();
|
|
81234
81849
|
}
|
|
81235
|
-
|
|
81236
|
-
|
|
81237
|
-
|
|
81850
|
+
break;
|
|
81851
|
+
} catch (error2) {
|
|
81852
|
+
if (!compactionAttempted && handleContextLimitError) {
|
|
81853
|
+
const compactionResult = handleContextLimitError(error2, currentMessages, {
|
|
81854
|
+
keepLastSegment: true,
|
|
81855
|
+
minSegmentsToKeep: 1
|
|
81856
|
+
});
|
|
81857
|
+
if (compactionResult) {
|
|
81858
|
+
const { messages: compactedMessages, stats } = compactionResult;
|
|
81859
|
+
if (stats.removed === 0) {
|
|
81860
|
+
console.error(`[ERROR] Context window exceeded but no messages can be compacted.`);
|
|
81861
|
+
console.error(`[ERROR] The conversation history is already minimal (${stats.originalCount} messages).`);
|
|
81862
|
+
finalResult = `Error: Context window limit exceeded and conversation cannot be compacted further. Consider starting a new session or reducing system message size.`;
|
|
81863
|
+
throw new Error(finalResult);
|
|
81864
|
+
}
|
|
81865
|
+
compactionAttempted = true;
|
|
81866
|
+
console.log(`[INFO] Context window limit exceeded. Compacting conversation...`);
|
|
81867
|
+
console.log(`[INFO] Removed ${stats.removed} messages (${stats.reductionPercent}% reduction)`);
|
|
81868
|
+
console.log(`[INFO] Estimated token savings: ${stats.tokensSaved} tokens`);
|
|
81869
|
+
if (this.debug) {
|
|
81870
|
+
console.log(`[DEBUG] Compaction stats:`, stats);
|
|
81871
|
+
console.log(`[DEBUG] Original message count: ${stats.originalCount}`);
|
|
81872
|
+
console.log(`[DEBUG] Compacted message count: ${stats.compactedCount}`);
|
|
81873
|
+
}
|
|
81874
|
+
currentMessages = [...compactedMessages];
|
|
81875
|
+
if (this.tracer) {
|
|
81876
|
+
this.tracer.addEvent("context.compacted", {
|
|
81877
|
+
"iteration": currentIteration,
|
|
81878
|
+
"original_count": stats.originalCount,
|
|
81879
|
+
"compacted_count": stats.compactedCount,
|
|
81880
|
+
"reduction_percent": stats.reductionPercent,
|
|
81881
|
+
"tokens_saved": stats.tokensSaved
|
|
81882
|
+
});
|
|
81883
|
+
}
|
|
81884
|
+
continue;
|
|
81885
|
+
}
|
|
81238
81886
|
}
|
|
81239
|
-
|
|
81240
|
-
|
|
81241
|
-
|
|
81242
|
-
await this.tracer.withSpan("ai.request", executeAIRequest, {
|
|
81243
|
-
"ai.model": this.model,
|
|
81244
|
-
"ai.provider": this.clientApiProvider || "auto",
|
|
81245
|
-
"iteration": currentIteration,
|
|
81246
|
-
"max_tokens": maxResponseTokens,
|
|
81247
|
-
"temperature": 0.3,
|
|
81248
|
-
"message_count": currentMessages.length
|
|
81249
|
-
});
|
|
81250
|
-
} else {
|
|
81251
|
-
await executeAIRequest();
|
|
81887
|
+
console.error(`Error during streamText (Iter ${currentIteration}):`, error2);
|
|
81888
|
+
finalResult = `Error: Failed to get response from AI model during iteration ${currentIteration}. ${error2.message}`;
|
|
81889
|
+
throw new Error(finalResult);
|
|
81252
81890
|
}
|
|
81253
|
-
} catch (error2) {
|
|
81254
|
-
console.error(`Error during streamText (Iter ${currentIteration}):`, error2);
|
|
81255
|
-
finalResult = `Error: Failed to get response from AI model during iteration ${currentIteration}. ${error2.message}`;
|
|
81256
|
-
throw new Error(finalResult);
|
|
81257
81891
|
}
|
|
81258
81892
|
if (this.debug && assistantResponseContent) {
|
|
81259
81893
|
const assistantPreview = createMessagePreview(assistantResponseContent);
|
|
@@ -81618,7 +82252,6 @@ Convert your previous response content into actual JSON data that follows this s
|
|
|
81618
82252
|
...options,
|
|
81619
82253
|
_schemaFormatted: true
|
|
81620
82254
|
});
|
|
81621
|
-
finalResult = cleanSchemaResponse(finalResult);
|
|
81622
82255
|
if (!this.disableMermaidValidation) {
|
|
81623
82256
|
try {
|
|
81624
82257
|
if (this.debug) {
|
|
@@ -81674,14 +82307,11 @@ Convert your previous response content into actual JSON data that follows this s
|
|
|
81674
82307
|
} else if (this.debug) {
|
|
81675
82308
|
console.log(`[DEBUG] Mermaid validation: Skipped due to disableMermaidValidation option`);
|
|
81676
82309
|
}
|
|
82310
|
+
finalResult = cleanSchemaResponse(finalResult);
|
|
81677
82311
|
if (isJsonSchema(options.schema)) {
|
|
81678
82312
|
if (this.debug) {
|
|
81679
82313
|
console.log(`[DEBUG] JSON validation: Starting validation process for schema response`);
|
|
81680
|
-
console.log(`[DEBUG] JSON validation:
|
|
81681
|
-
}
|
|
81682
|
-
finalResult = cleanSchemaResponse(finalResult);
|
|
81683
|
-
if (this.debug) {
|
|
81684
|
-
console.log(`[DEBUG] JSON validation: After cleaning, length: ${finalResult.length} chars`);
|
|
82314
|
+
console.log(`[DEBUG] JSON validation: Cleaned response length: ${finalResult.length} chars`);
|
|
81685
82315
|
}
|
|
81686
82316
|
if (this.tracer) {
|
|
81687
82317
|
this.tracer.recordJsonValidationEvent("started", {
|
|
@@ -81771,10 +82401,9 @@ Convert your previous response content into actual JSON data that follows this s
|
|
|
81771
82401
|
console.log("[DEBUG] Skipping schema formatting due to max iterations reached without completion");
|
|
81772
82402
|
} else if (completionAttempted && options.schema && !options._schemaFormatted && !options._skipValidation) {
|
|
81773
82403
|
try {
|
|
81774
|
-
finalResult = cleanSchemaResponse(finalResult);
|
|
81775
82404
|
if (!this.disableMermaidValidation) {
|
|
81776
82405
|
if (this.debug) {
|
|
81777
|
-
console.log(`[DEBUG] Mermaid validation: Validating attempt_completion result...`);
|
|
82406
|
+
console.log(`[DEBUG] Mermaid validation: Validating attempt_completion result BEFORE schema cleaning...`);
|
|
81778
82407
|
}
|
|
81779
82408
|
const mermaidValidation = await validateAndFixMermaidResponse(finalResult, {
|
|
81780
82409
|
debug: this.debug,
|
|
@@ -81797,6 +82426,7 @@ Convert your previous response content into actual JSON data that follows this s
|
|
|
81797
82426
|
} else if (this.debug) {
|
|
81798
82427
|
console.log(`[DEBUG] Mermaid validation: Skipped for attempt_completion result due to disableMermaidValidation option`);
|
|
81799
82428
|
}
|
|
82429
|
+
finalResult = cleanSchemaResponse(finalResult);
|
|
81800
82430
|
if (isJsonSchema(options.schema)) {
|
|
81801
82431
|
if (this.debug) {
|
|
81802
82432
|
console.log(`[DEBUG] JSON validation: Starting validation process for attempt_completion result`);
|
|
@@ -81970,6 +82600,56 @@ Convert your previous response content into actual JSON data that follows this s
|
|
|
81970
82600
|
console.log(`[DEBUG] Cleared conversation history and reset counters for session ${this.sessionId}`);
|
|
81971
82601
|
}
|
|
81972
82602
|
}
|
|
82603
|
+
/**
|
|
82604
|
+
* Manually compact conversation history
|
|
82605
|
+
* Removes intermediate monologues from older segments while preserving
|
|
82606
|
+
* user messages, final answers, and the most recent segment
|
|
82607
|
+
*
|
|
82608
|
+
* @param {Object} options - Compaction options
|
|
82609
|
+
* @param {boolean} [options.keepLastSegment=true] - Keep the most recent segment intact
|
|
82610
|
+
* @param {number} [options.minSegmentsToKeep=1] - Number of recent segments to preserve fully
|
|
82611
|
+
* @returns {Object} Compaction statistics
|
|
82612
|
+
*/
|
|
82613
|
+
async compactHistory(options = {}) {
|
|
82614
|
+
const { compactMessages: compactMessages2, calculateCompactionStats: calculateCompactionStats2 } = await Promise.resolve().then(() => (init_contextCompactor(), contextCompactor_exports));
|
|
82615
|
+
if (this.history.length === 0) {
|
|
82616
|
+
if (this.debug) {
|
|
82617
|
+
console.log(`[DEBUG] No history to compact for session ${this.sessionId}`);
|
|
82618
|
+
}
|
|
82619
|
+
return {
|
|
82620
|
+
originalCount: 0,
|
|
82621
|
+
compactedCount: 0,
|
|
82622
|
+
removed: 0,
|
|
82623
|
+
reductionPercent: 0,
|
|
82624
|
+
originalTokens: 0,
|
|
82625
|
+
compactedTokens: 0,
|
|
82626
|
+
tokensSaved: 0
|
|
82627
|
+
};
|
|
82628
|
+
}
|
|
82629
|
+
const compactedMessages = compactMessages2(this.history, options);
|
|
82630
|
+
const stats = calculateCompactionStats2(this.history, compactedMessages);
|
|
82631
|
+
this.history = compactedMessages;
|
|
82632
|
+
try {
|
|
82633
|
+
await this.storageAdapter.clearHistory(this.sessionId);
|
|
82634
|
+
for (const message of compactedMessages) {
|
|
82635
|
+
await this.storageAdapter.saveMessage(this.sessionId, message);
|
|
82636
|
+
}
|
|
82637
|
+
} catch (error2) {
|
|
82638
|
+
console.error(`[ERROR] Failed to save compacted messages to storage:`, error2);
|
|
82639
|
+
}
|
|
82640
|
+
console.log(`[INFO] Manually compacted conversation history`);
|
|
82641
|
+
console.log(`[INFO] Removed ${stats.removed} messages (${stats.reductionPercent}% reduction)`);
|
|
82642
|
+
console.log(`[INFO] Estimated token savings: ${stats.tokensSaved} tokens`);
|
|
82643
|
+
if (this.debug) {
|
|
82644
|
+
console.log(`[DEBUG] Compaction stats:`, stats);
|
|
82645
|
+
}
|
|
82646
|
+
await this.hooks.emit(HOOK_TYPES.STORAGE_SAVE, {
|
|
82647
|
+
sessionId: this.sessionId,
|
|
82648
|
+
compacted: true,
|
|
82649
|
+
stats
|
|
82650
|
+
});
|
|
82651
|
+
return stats;
|
|
82652
|
+
}
|
|
81973
82653
|
/**
|
|
81974
82654
|
* Clone this agent's session to create a new agent with shared conversation history
|
|
81975
82655
|
* @param {Object} options - Clone options
|
|
@@ -81992,6 +82672,14 @@ Convert your previous response content into actual JSON data that follows this s
|
|
|
81992
82672
|
if (stripInternalMessages) {
|
|
81993
82673
|
clonedHistory = this._stripInternalMessages(clonedHistory, keepSystemMessage);
|
|
81994
82674
|
}
|
|
82675
|
+
let allowedToolsArray = null;
|
|
82676
|
+
if (this.allowedTools.mode === "whitelist") {
|
|
82677
|
+
allowedToolsArray = [...this.allowedTools.allowed];
|
|
82678
|
+
} else if (this.allowedTools.mode === "none") {
|
|
82679
|
+
allowedToolsArray = [];
|
|
82680
|
+
} else if (this.allowedTools.mode === "all" && this.allowedTools.exclusions.length > 0) {
|
|
82681
|
+
allowedToolsArray = ["*", ...this.allowedTools.exclusions.map((t3) => "!" + t3)];
|
|
82682
|
+
}
|
|
81995
82683
|
const clonedAgent = new _ProbeAgent({
|
|
81996
82684
|
// Copy current agent's config
|
|
81997
82685
|
customPrompt: this.customPrompt,
|
|
@@ -82009,6 +82697,7 @@ Convert your previous response content into actual JSON data that follows this s
|
|
|
82009
82697
|
maxIterations: this.maxIterations,
|
|
82010
82698
|
disableMermaidValidation: this.disableMermaidValidation,
|
|
82011
82699
|
disableJsonValidation: this.disableJsonValidation,
|
|
82700
|
+
allowedTools: allowedToolsArray,
|
|
82012
82701
|
enableMcp: !!this.mcpBridge,
|
|
82013
82702
|
mcpConfig: this.mcpConfig,
|
|
82014
82703
|
enableBash: this.enableBash,
|