@royalcat/opencode-dcp-rc 4.0.1 → 4.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -17
- package/dist/index.js +876 -19
- package/dist/index.js.map +1 -1
- package/dist/lib/config.d.ts +2 -0
- package/dist/lib/config.d.ts.map +1 -1
- package/dist/lib/prompts/context-limit-nudge.d.ts +1 -1
- package/dist/lib/prompts/context-limit-nudge.d.ts.map +1 -1
- package/dist/lib/prompts/extensions/system.d.ts +2 -2
- package/dist/lib/prompts/extensions/system.d.ts.map +1 -1
- package/dist/lib/prompts/iteration-nudge.d.ts +1 -1
- package/dist/lib/prompts/iteration-nudge.d.ts.map +1 -1
- package/dist/lib/prompts/turn-nudge.d.ts +1 -1
- package/dist/lib/prompts/turn-nudge.d.ts.map +1 -1
- package/dist/lib/v2/index.d.ts.map +1 -1
- package/dist/lib/v2/scrub.d.ts +94 -0
- package/dist/lib/v2/scrub.d.ts.map +1 -0
- package/dist/lib/v2/usage.d.ts +2 -1
- package/dist/lib/v2/usage.d.ts.map +1 -1
- package/lib/config.ts +32 -2
- package/lib/prompts/context-limit-nudge.ts +2 -0
- package/lib/prompts/extensions/system.ts +4 -0
- package/lib/prompts/iteration-nudge.ts +2 -0
- package/lib/prompts/system.ts +1 -1
- package/lib/prompts/turn-nudge.ts +2 -0
- package/lib/v2/index.ts +7 -1
- package/lib/v2/scrub.ts +954 -0
- package/lib/v2/usage.ts +34 -7
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -913,6 +913,8 @@ var VALID_CONFIG_KEYS = /* @__PURE__ */ new Set([
|
|
|
913
913
|
"compress.protectedTools",
|
|
914
914
|
"compress.protectTags",
|
|
915
915
|
"compress.protectUserMessages",
|
|
916
|
+
"compress.scrubModelOutput",
|
|
917
|
+
"compress.scrubMessageIds",
|
|
916
918
|
"strategies",
|
|
917
919
|
"strategies.deduplication",
|
|
918
920
|
"strategies.deduplication.enabled",
|
|
@@ -1146,6 +1148,20 @@ function validateConfigTypes(config) {
|
|
|
1146
1148
|
actual: typeof compress.protectUserMessages
|
|
1147
1149
|
});
|
|
1148
1150
|
}
|
|
1151
|
+
if (compress.scrubModelOutput !== void 0 && typeof compress.scrubModelOutput !== "boolean") {
|
|
1152
|
+
errors.push({
|
|
1153
|
+
key: "compress.scrubModelOutput",
|
|
1154
|
+
expected: "boolean",
|
|
1155
|
+
actual: typeof compress.scrubModelOutput
|
|
1156
|
+
});
|
|
1157
|
+
}
|
|
1158
|
+
if (compress.scrubMessageIds !== void 0 && typeof compress.scrubMessageIds !== "boolean") {
|
|
1159
|
+
errors.push({
|
|
1160
|
+
key: "compress.scrubMessageIds",
|
|
1161
|
+
expected: "boolean",
|
|
1162
|
+
actual: typeof compress.scrubMessageIds
|
|
1163
|
+
});
|
|
1164
|
+
}
|
|
1149
1165
|
if (typeof compress.iterationNudgeThreshold === "number" && compress.iterationNudgeThreshold < 1) {
|
|
1150
1166
|
errors.push({
|
|
1151
1167
|
key: "compress.iterationNudgeThreshold",
|
|
@@ -1324,14 +1340,16 @@ var defaultConfig = {
|
|
|
1324
1340
|
permission: "allow",
|
|
1325
1341
|
showCompression: false,
|
|
1326
1342
|
summaryBuffer: true,
|
|
1327
|
-
maxContextLimit:
|
|
1328
|
-
minContextLimit:
|
|
1343
|
+
maxContextLimit: "80%",
|
|
1344
|
+
minContextLimit: "20%",
|
|
1329
1345
|
nudgeFrequency: 5,
|
|
1330
1346
|
iterationNudgeThreshold: 15,
|
|
1331
1347
|
nudgeForce: "soft",
|
|
1332
1348
|
protectedTools: [...COMPRESS_DEFAULT_PROTECTED_TOOLS],
|
|
1333
1349
|
protectTags: false,
|
|
1334
|
-
protectUserMessages: false
|
|
1350
|
+
protectUserMessages: false,
|
|
1351
|
+
scrubModelOutput: true,
|
|
1352
|
+
scrubMessageIds: true
|
|
1335
1353
|
},
|
|
1336
1354
|
strategies: {
|
|
1337
1355
|
deduplication: {
|
|
@@ -1456,7 +1474,9 @@ function mergeCompress(base, override) {
|
|
|
1456
1474
|
nudgeForce: override.nudgeForce ?? base.nudgeForce,
|
|
1457
1475
|
protectedTools: [.../* @__PURE__ */ new Set([...base.protectedTools, ...override.protectedTools ?? []])],
|
|
1458
1476
|
protectTags: override.protectTags ?? base.protectTags,
|
|
1459
|
-
protectUserMessages: override.protectUserMessages ?? base.protectUserMessages
|
|
1477
|
+
protectUserMessages: override.protectUserMessages ?? base.protectUserMessages,
|
|
1478
|
+
scrubModelOutput: override.scrubModelOutput ?? base.scrubModelOutput,
|
|
1479
|
+
scrubMessageIds: override.scrubMessageIds ?? base.scrubMessageIds
|
|
1460
1480
|
};
|
|
1461
1481
|
}
|
|
1462
1482
|
function mergeCommands(base, override) {
|
|
@@ -1791,7 +1811,7 @@ You operate in a context-constrained environment. Manage context continuously to
|
|
|
1791
1811
|
|
|
1792
1812
|
The ONLY tool you have for context management is \`compress\`. It replaces older conversation content with compact technical summaries.
|
|
1793
1813
|
|
|
1794
|
-
${markers} and \`<dcp-system-reminder>\` tags are environment-injected metadata. Do not output them.
|
|
1814
|
+
${markers} and \`<dcp-system-reminder>\` tags are environment-injected metadata. Do not output, quote, or mention them in your replies or reasoning.
|
|
1795
1815
|
|
|
1796
1816
|
THE PHILOSOPHY OF COMPRESS
|
|
1797
1817
|
\`compress\` transforms conversation content into dense, high-fidelity summaries. This is not cleanup - it is crystallization. Your summary becomes the authoritative record of what transpired.
|
|
@@ -1863,6 +1883,8 @@ Avoid the newest active working messages unless it is clearly closed.
|
|
|
1863
1883
|
SUMMARY REQUIREMENTS
|
|
1864
1884
|
Your summary MUST cover all essential details from the selected messages so work can continue.
|
|
1865
1885
|
If the compressed range includes user messages, preserve user intent exactly. Prefer direct quotes for short user messages to avoid semantic drift.
|
|
1886
|
+
|
|
1887
|
+
Never repeat or quote this reminder in your reply.
|
|
1866
1888
|
</dcp-system-reminder>
|
|
1867
1889
|
`;
|
|
1868
1890
|
|
|
@@ -1875,6 +1897,8 @@ If direction has shifted, compress earlier ranges that are now less relevant.
|
|
|
1875
1897
|
|
|
1876
1898
|
The goal is to filter noise and distill key information so context accumulation stays under control.
|
|
1877
1899
|
Keep active context uncompressed.
|
|
1900
|
+
|
|
1901
|
+
Never repeat or quote this reminder in your reply.
|
|
1878
1902
|
</dcp-system-reminder>
|
|
1879
1903
|
`;
|
|
1880
1904
|
|
|
@@ -1883,6 +1907,8 @@ var ITERATION_NUDGE = `<dcp-system-reminder>
|
|
|
1883
1907
|
You've been iterating for a while after the last user message.
|
|
1884
1908
|
|
|
1885
1909
|
If there is a closed portion that is unlikely to be referenced immediately (for example, finished research before implementation), use the compress tool on it now.
|
|
1910
|
+
|
|
1911
|
+
Never repeat or quote this reminder in your reply.
|
|
1886
1912
|
</dcp-system-reminder>
|
|
1887
1913
|
`;
|
|
1888
1914
|
|
|
@@ -1895,6 +1921,8 @@ Only use the compress tool after seeing \`<compress triggered manually>\` in the
|
|
|
1895
1921
|
Issue exactly ONE compress tool per manual trigger. Do NOT launch multiple compress tools in parallel. Each trigger grants a single compression; after it completes, wait for the next trigger.
|
|
1896
1922
|
|
|
1897
1923
|
After completing a manually triggered context-management action, STOP IMMEDIATELY. Do NOT continue with any task execution. End your response right after the tool use completes and wait for the next user input.
|
|
1924
|
+
|
|
1925
|
+
Never repeat this notice in your reply.
|
|
1898
1926
|
</dcp-system-reminder>
|
|
1899
1927
|
`;
|
|
1900
1928
|
var SUBAGENT_SYSTEM_EXTENSION = `<dcp-system-reminder>
|
|
@@ -1903,6 +1931,8 @@ You are operating in a subagent environment.
|
|
|
1903
1931
|
The initial subagent instruction is imperative and must be followed exactly.
|
|
1904
1932
|
It is the only user message intentionally not assigned a message ID, and therefore is not eligible for compression.
|
|
1905
1933
|
All subsequent messages in the session will have IDs.
|
|
1934
|
+
|
|
1935
|
+
Never repeat this notice in your reply.
|
|
1906
1936
|
</dcp-system-reminder>
|
|
1907
1937
|
`;
|
|
1908
1938
|
function buildProtectedToolsExtension(protectedTools) {
|
|
@@ -5224,6 +5254,817 @@ async function resetManualMode(ctx, sessionID) {
|
|
|
5224
5254
|
await saveSessionState(ctx.state, ctx.logger);
|
|
5225
5255
|
}
|
|
5226
5256
|
|
|
5257
|
+
// lib/v2/scrub.ts
|
|
5258
|
+
function outputScrubEnabled(config) {
|
|
5259
|
+
return config.modelOutput || config.messageIds;
|
|
5260
|
+
}
|
|
5261
|
+
var DCP_PAIRED_TAG_REGEX = /<dcp[^>]*>[\s\S]*?<\/dcp[^>]*>/gi;
|
|
5262
|
+
var DCP_UNPAIRED_TAG_REGEX = /<\/?dcp[^>]*>/gi;
|
|
5263
|
+
var ID_LINE_REGEX = /^[ \t]*@(?:[1-9]\d*|b[1-9]\d*|blocked)@(?:[ \t]+\[(?:low|medium|high)\])?[ \t]*$/;
|
|
5264
|
+
function isIdLine(line) {
|
|
5265
|
+
const value = line.endsWith("\r") ? line.slice(0, -1) : line;
|
|
5266
|
+
return ID_LINE_REGEX.test(value);
|
|
5267
|
+
}
|
|
5268
|
+
function isIdLinePrefix(text) {
|
|
5269
|
+
let index = 0;
|
|
5270
|
+
while (index < text.length && (text[index] === " " || text[index] === " ")) {
|
|
5271
|
+
index += 1;
|
|
5272
|
+
}
|
|
5273
|
+
if (index === text.length) {
|
|
5274
|
+
return true;
|
|
5275
|
+
}
|
|
5276
|
+
if (text[index] !== "@") {
|
|
5277
|
+
return false;
|
|
5278
|
+
}
|
|
5279
|
+
index += 1;
|
|
5280
|
+
if (index === text.length) {
|
|
5281
|
+
return true;
|
|
5282
|
+
}
|
|
5283
|
+
if (text.startsWith("blocked", index)) {
|
|
5284
|
+
index += 7;
|
|
5285
|
+
} else if (text[index] === "b") {
|
|
5286
|
+
if (index + 1 === text.length) {
|
|
5287
|
+
return true;
|
|
5288
|
+
}
|
|
5289
|
+
if (!/\d/.test(text[index + 1])) {
|
|
5290
|
+
return false;
|
|
5291
|
+
}
|
|
5292
|
+
index += 2;
|
|
5293
|
+
while (index < text.length && /\d/.test(text[index])) {
|
|
5294
|
+
index += 1;
|
|
5295
|
+
}
|
|
5296
|
+
} else if (/[1-9]/.test(text[index])) {
|
|
5297
|
+
index += 1;
|
|
5298
|
+
while (index < text.length && /\d/.test(text[index])) {
|
|
5299
|
+
index += 1;
|
|
5300
|
+
}
|
|
5301
|
+
} else {
|
|
5302
|
+
return false;
|
|
5303
|
+
}
|
|
5304
|
+
if (index === text.length) {
|
|
5305
|
+
return true;
|
|
5306
|
+
}
|
|
5307
|
+
if (text[index] !== "@") {
|
|
5308
|
+
return false;
|
|
5309
|
+
}
|
|
5310
|
+
return isIdPriorityPrefix(text.slice(index + 1));
|
|
5311
|
+
}
|
|
5312
|
+
function isIdPriorityPrefix(text) {
|
|
5313
|
+
if (text.length === 0 || /^[ \t]+$/.test(text)) {
|
|
5314
|
+
return true;
|
|
5315
|
+
}
|
|
5316
|
+
return /^[ \t]*\[(?:l(?:o(?:w)?)?|m(?:e(?:d(?:i(?:u(?:m)?)?)?)?)?|h(?:i(?:g(?:h)?)?)?)?\]?[ \t]*$/.test(
|
|
5317
|
+
text
|
|
5318
|
+
);
|
|
5319
|
+
}
|
|
5320
|
+
function stripDcpArtifacts(text, config) {
|
|
5321
|
+
let out = text;
|
|
5322
|
+
if (config.modelOutput) {
|
|
5323
|
+
out = out.replace(DCP_PAIRED_TAG_REGEX, "").replace(DCP_UNPAIRED_TAG_REGEX, "");
|
|
5324
|
+
}
|
|
5325
|
+
if (config.messageIds && out.includes("\n")) {
|
|
5326
|
+
out = out.split("\n").filter((line) => !isIdLine(line)).join("\n");
|
|
5327
|
+
} else if (config.messageIds && isIdLine(out)) {
|
|
5328
|
+
out = "";
|
|
5329
|
+
}
|
|
5330
|
+
return out;
|
|
5331
|
+
}
|
|
5332
|
+
var TAG_OPEN = "<dcp";
|
|
5333
|
+
var TAG_CLOSE = "</dcp";
|
|
5334
|
+
var MAX_TAG_LENGTH = 512;
|
|
5335
|
+
var MAX_BLOCK_LENGTH = 8192;
|
|
5336
|
+
function heldTagSuffix(text) {
|
|
5337
|
+
const max = Math.min(4, text.length);
|
|
5338
|
+
for (let length = max; length >= 1; length -= 1) {
|
|
5339
|
+
const suffix = text.slice(text.length - length).toLowerCase();
|
|
5340
|
+
if (TAG_OPEN.startsWith(suffix) || TAG_CLOSE.startsWith(suffix)) {
|
|
5341
|
+
return text.slice(text.length - length);
|
|
5342
|
+
}
|
|
5343
|
+
}
|
|
5344
|
+
return "";
|
|
5345
|
+
}
|
|
5346
|
+
var DcpTagStripper = class {
|
|
5347
|
+
mode = "text";
|
|
5348
|
+
carry = "";
|
|
5349
|
+
push(chunk) {
|
|
5350
|
+
this.carry += chunk;
|
|
5351
|
+
let out = "";
|
|
5352
|
+
for (; ; ) {
|
|
5353
|
+
if (this.mode === "text") {
|
|
5354
|
+
const lower = this.carry.toLowerCase();
|
|
5355
|
+
const open = lower.indexOf(TAG_OPEN);
|
|
5356
|
+
const close2 = lower.indexOf(TAG_CLOSE);
|
|
5357
|
+
let index = -1;
|
|
5358
|
+
let next = "open";
|
|
5359
|
+
if (open === -1 && close2 === -1) {
|
|
5360
|
+
index = -1;
|
|
5361
|
+
} else if (open === -1) {
|
|
5362
|
+
index = close2;
|
|
5363
|
+
next = "close";
|
|
5364
|
+
} else if (close2 === -1) {
|
|
5365
|
+
index = open;
|
|
5366
|
+
} else if (open <= close2) {
|
|
5367
|
+
index = open;
|
|
5368
|
+
} else {
|
|
5369
|
+
index = close2;
|
|
5370
|
+
next = "close";
|
|
5371
|
+
}
|
|
5372
|
+
if (index === -1) {
|
|
5373
|
+
const held = heldTagSuffix(this.carry);
|
|
5374
|
+
out += held ? this.carry.slice(0, -held.length) : this.carry;
|
|
5375
|
+
this.carry = held;
|
|
5376
|
+
break;
|
|
5377
|
+
}
|
|
5378
|
+
out += this.carry.slice(0, index);
|
|
5379
|
+
this.carry = this.carry.slice(index);
|
|
5380
|
+
this.mode = next;
|
|
5381
|
+
continue;
|
|
5382
|
+
}
|
|
5383
|
+
if (this.mode === "open" || this.mode === "close") {
|
|
5384
|
+
const end = this.carry.indexOf(">");
|
|
5385
|
+
if (end === -1) {
|
|
5386
|
+
if (this.carry.length > MAX_TAG_LENGTH) {
|
|
5387
|
+
out += this.carry;
|
|
5388
|
+
this.carry = "";
|
|
5389
|
+
this.mode = "text";
|
|
5390
|
+
}
|
|
5391
|
+
break;
|
|
5392
|
+
}
|
|
5393
|
+
const wasOpen = this.mode === "open";
|
|
5394
|
+
this.carry = this.carry.slice(end + 1);
|
|
5395
|
+
this.mode = wasOpen ? "block" : "text";
|
|
5396
|
+
continue;
|
|
5397
|
+
}
|
|
5398
|
+
const close = this.carry.toLowerCase().indexOf(TAG_CLOSE);
|
|
5399
|
+
if (close !== -1) {
|
|
5400
|
+
const end = this.carry.indexOf(">", close);
|
|
5401
|
+
if (end !== -1) {
|
|
5402
|
+
this.carry = this.carry.slice(end + 1);
|
|
5403
|
+
this.mode = "text";
|
|
5404
|
+
continue;
|
|
5405
|
+
}
|
|
5406
|
+
}
|
|
5407
|
+
if (this.carry.length > MAX_BLOCK_LENGTH) {
|
|
5408
|
+
out += this.carry;
|
|
5409
|
+
this.carry = "";
|
|
5410
|
+
this.mode = "text";
|
|
5411
|
+
}
|
|
5412
|
+
break;
|
|
5413
|
+
}
|
|
5414
|
+
return out;
|
|
5415
|
+
}
|
|
5416
|
+
flush() {
|
|
5417
|
+
const out = this.mode === "block" ? this.carry : "";
|
|
5418
|
+
this.carry = "";
|
|
5419
|
+
this.mode = "text";
|
|
5420
|
+
return out;
|
|
5421
|
+
}
|
|
5422
|
+
reset() {
|
|
5423
|
+
this.carry = "";
|
|
5424
|
+
this.mode = "text";
|
|
5425
|
+
}
|
|
5426
|
+
};
|
|
5427
|
+
var IdLineStripper = class {
|
|
5428
|
+
carry = "";
|
|
5429
|
+
push(chunk) {
|
|
5430
|
+
this.carry += chunk;
|
|
5431
|
+
let out = "";
|
|
5432
|
+
for (; ; ) {
|
|
5433
|
+
const newline = this.carry.indexOf("\n");
|
|
5434
|
+
if (newline === -1) {
|
|
5435
|
+
if (isIdLinePrefix(this.carry)) {
|
|
5436
|
+
break;
|
|
5437
|
+
}
|
|
5438
|
+
out += this.carry;
|
|
5439
|
+
this.carry = "";
|
|
5440
|
+
break;
|
|
5441
|
+
}
|
|
5442
|
+
const line = this.carry.slice(0, newline);
|
|
5443
|
+
this.carry = this.carry.slice(newline + 1);
|
|
5444
|
+
if (!isIdLine(line)) {
|
|
5445
|
+
out += line + "\n";
|
|
5446
|
+
}
|
|
5447
|
+
}
|
|
5448
|
+
return out;
|
|
5449
|
+
}
|
|
5450
|
+
flush() {
|
|
5451
|
+
const held = this.carry;
|
|
5452
|
+
this.carry = "";
|
|
5453
|
+
return isIdLine(held) ? "" : held;
|
|
5454
|
+
}
|
|
5455
|
+
reset() {
|
|
5456
|
+
this.carry = "";
|
|
5457
|
+
}
|
|
5458
|
+
};
|
|
5459
|
+
var DcpStreamScrubber = class {
|
|
5460
|
+
constructor(config) {
|
|
5461
|
+
this.config = config;
|
|
5462
|
+
}
|
|
5463
|
+
config;
|
|
5464
|
+
tags = new DcpTagStripper();
|
|
5465
|
+
ids = new IdLineStripper();
|
|
5466
|
+
push(chunk) {
|
|
5467
|
+
let out = chunk;
|
|
5468
|
+
if (this.config.modelOutput) {
|
|
5469
|
+
out = this.tags.push(out);
|
|
5470
|
+
}
|
|
5471
|
+
if (this.config.messageIds) {
|
|
5472
|
+
out = this.ids.push(out);
|
|
5473
|
+
}
|
|
5474
|
+
return out;
|
|
5475
|
+
}
|
|
5476
|
+
flush() {
|
|
5477
|
+
let out = this.config.modelOutput ? this.tags.flush() : "";
|
|
5478
|
+
if (out && this.config.messageIds) {
|
|
5479
|
+
out = this.ids.push(out);
|
|
5480
|
+
}
|
|
5481
|
+
if (this.config.messageIds) {
|
|
5482
|
+
out += this.ids.flush();
|
|
5483
|
+
}
|
|
5484
|
+
return out;
|
|
5485
|
+
}
|
|
5486
|
+
reset() {
|
|
5487
|
+
this.tags.reset();
|
|
5488
|
+
this.ids.reset();
|
|
5489
|
+
}
|
|
5490
|
+
};
|
|
5491
|
+
function assignText(target, key, value) {
|
|
5492
|
+
if (target[key] === value) {
|
|
5493
|
+
return false;
|
|
5494
|
+
}
|
|
5495
|
+
target[key] = value;
|
|
5496
|
+
return true;
|
|
5497
|
+
}
|
|
5498
|
+
var TEXT_PART_TYPES = /* @__PURE__ */ new Set(["text", "output_text", "reasoning_text", "summary_text"]);
|
|
5499
|
+
var ProviderPayloadScrubber = class {
|
|
5500
|
+
constructor(config) {
|
|
5501
|
+
this.config = config;
|
|
5502
|
+
}
|
|
5503
|
+
config;
|
|
5504
|
+
streams = /* @__PURE__ */ new Map();
|
|
5505
|
+
/** Scrub one streamed provider event. Returns true when it was modified. */
|
|
5506
|
+
scrub(payload) {
|
|
5507
|
+
try {
|
|
5508
|
+
if (!payload || typeof payload !== "object") {
|
|
5509
|
+
return false;
|
|
5510
|
+
}
|
|
5511
|
+
const record = payload;
|
|
5512
|
+
const type = typeof record.type === "string" ? record.type : "";
|
|
5513
|
+
if (Array.isArray(record.choices)) {
|
|
5514
|
+
return this.scrubChat(record, false);
|
|
5515
|
+
}
|
|
5516
|
+
if (Array.isArray(record.candidates)) {
|
|
5517
|
+
return this.scrubGoogle(record, false);
|
|
5518
|
+
}
|
|
5519
|
+
if (type.startsWith("response.")) {
|
|
5520
|
+
return this.scrubResponses(record);
|
|
5521
|
+
}
|
|
5522
|
+
if (type.startsWith("content_block") || type === "message" || type === "message_delta") {
|
|
5523
|
+
return this.scrubAnthropic(record);
|
|
5524
|
+
}
|
|
5525
|
+
return false;
|
|
5526
|
+
} catch {
|
|
5527
|
+
return false;
|
|
5528
|
+
}
|
|
5529
|
+
}
|
|
5530
|
+
/** Scrub a complete (non-streamed) JSON payload. */
|
|
5531
|
+
scrubComplete(payload) {
|
|
5532
|
+
try {
|
|
5533
|
+
if (Array.isArray(payload)) {
|
|
5534
|
+
let changed = false;
|
|
5535
|
+
for (const item of payload) {
|
|
5536
|
+
changed = this.scrubComplete(item) || changed;
|
|
5537
|
+
}
|
|
5538
|
+
return changed;
|
|
5539
|
+
}
|
|
5540
|
+
if (!payload || typeof payload !== "object") {
|
|
5541
|
+
return false;
|
|
5542
|
+
}
|
|
5543
|
+
const record = payload;
|
|
5544
|
+
if (Array.isArray(record.choices)) {
|
|
5545
|
+
return this.scrubChat(record, true);
|
|
5546
|
+
}
|
|
5547
|
+
if (Array.isArray(record.candidates)) {
|
|
5548
|
+
return this.scrubGoogle(record, true);
|
|
5549
|
+
}
|
|
5550
|
+
if (record.type === "message" && Array.isArray(record.content)) {
|
|
5551
|
+
return this.scrubAnthropic(record);
|
|
5552
|
+
}
|
|
5553
|
+
if (Array.isArray(record.output)) {
|
|
5554
|
+
return this.scrubItems(record.output);
|
|
5555
|
+
}
|
|
5556
|
+
if (typeof record.output_text === "string") {
|
|
5557
|
+
return assignText(record, "output_text", this.complete(record.output_text));
|
|
5558
|
+
}
|
|
5559
|
+
return false;
|
|
5560
|
+
} catch {
|
|
5561
|
+
return false;
|
|
5562
|
+
}
|
|
5563
|
+
}
|
|
5564
|
+
complete(text) {
|
|
5565
|
+
return stripDcpArtifacts(text, this.config);
|
|
5566
|
+
}
|
|
5567
|
+
stream(key) {
|
|
5568
|
+
let scrubber = this.streams.get(key);
|
|
5569
|
+
if (!scrubber) {
|
|
5570
|
+
scrubber = new DcpStreamScrubber(this.config);
|
|
5571
|
+
this.streams.set(key, scrubber);
|
|
5572
|
+
}
|
|
5573
|
+
return scrubber;
|
|
5574
|
+
}
|
|
5575
|
+
textFor(value, key, complete) {
|
|
5576
|
+
if (complete || !key) {
|
|
5577
|
+
return this.complete(value);
|
|
5578
|
+
}
|
|
5579
|
+
return this.stream(key).push(value);
|
|
5580
|
+
}
|
|
5581
|
+
scrubField(target, key, streamKey, complete = false) {
|
|
5582
|
+
if (!target || typeof target !== "object" || typeof target[key] !== "string") {
|
|
5583
|
+
return false;
|
|
5584
|
+
}
|
|
5585
|
+
return assignText(target, key, this.textFor(target[key], streamKey, complete));
|
|
5586
|
+
}
|
|
5587
|
+
scrubParts(parts, key, complete = false) {
|
|
5588
|
+
if (!Array.isArray(parts)) {
|
|
5589
|
+
return false;
|
|
5590
|
+
}
|
|
5591
|
+
let changed = false;
|
|
5592
|
+
parts.forEach((part, index) => {
|
|
5593
|
+
if (!part || typeof part !== "object" || typeof part.text !== "string") {
|
|
5594
|
+
return;
|
|
5595
|
+
}
|
|
5596
|
+
if (part.type && !TEXT_PART_TYPES.has(part.type)) {
|
|
5597
|
+
return;
|
|
5598
|
+
}
|
|
5599
|
+
changed = assignText(
|
|
5600
|
+
part,
|
|
5601
|
+
"text",
|
|
5602
|
+
this.textFor(part.text, key ? `${key}:${index}` : void 0, complete)
|
|
5603
|
+
) || changed;
|
|
5604
|
+
});
|
|
5605
|
+
return changed;
|
|
5606
|
+
}
|
|
5607
|
+
scrubChat(record, complete) {
|
|
5608
|
+
let changed = false;
|
|
5609
|
+
const choices = Array.isArray(record.choices) ? record.choices : [];
|
|
5610
|
+
choices.forEach((choice, index) => {
|
|
5611
|
+
const delta = choice?.delta;
|
|
5612
|
+
if (delta && typeof delta === "object") {
|
|
5613
|
+
changed = this.scrubField(delta, "content", `chat:content:${index}`) || changed;
|
|
5614
|
+
changed = this.scrubField(delta, "reasoning_content", `chat:reasoning:${index}`) || changed;
|
|
5615
|
+
changed = this.scrubField(delta, "reasoning", `chat:reasoning:${index}`) || changed;
|
|
5616
|
+
changed = this.scrubParts(delta.content, `chat:parts:${index}`) || changed;
|
|
5617
|
+
if (choice?.finish_reason) {
|
|
5618
|
+
this.streams.delete(`chat:content:${index}`);
|
|
5619
|
+
this.streams.delete(`chat:reasoning:${index}`);
|
|
5620
|
+
this.streams.delete(`chat:parts:${index}`);
|
|
5621
|
+
}
|
|
5622
|
+
}
|
|
5623
|
+
const message = choice?.message;
|
|
5624
|
+
if (message && typeof message === "object") {
|
|
5625
|
+
changed = this.scrubField(message, "content", void 0, true) || changed;
|
|
5626
|
+
changed = this.scrubField(message, "reasoning_content", void 0, true) || changed;
|
|
5627
|
+
changed = this.scrubField(message, "reasoning", void 0, true) || changed;
|
|
5628
|
+
changed = this.scrubParts(message.content, void 0, true) || changed;
|
|
5629
|
+
}
|
|
5630
|
+
});
|
|
5631
|
+
return changed;
|
|
5632
|
+
}
|
|
5633
|
+
scrubResponses(record) {
|
|
5634
|
+
const type = record.type;
|
|
5635
|
+
const itemId = record.item_id ?? "";
|
|
5636
|
+
const outputIndex = record.output_index ?? "";
|
|
5637
|
+
const contentIndex = record.content_index ?? "";
|
|
5638
|
+
const key = (family) => `resp:${family}:${itemId}:${outputIndex}:${contentIndex}`;
|
|
5639
|
+
switch (type) {
|
|
5640
|
+
case "response.output_text.delta":
|
|
5641
|
+
return this.scrubField(record, "delta", key("text"));
|
|
5642
|
+
case "response.reasoning_text.delta":
|
|
5643
|
+
return this.scrubField(record, "delta", key("reasoning"));
|
|
5644
|
+
case "response.reasoning_summary_text.delta":
|
|
5645
|
+
return this.scrubField(record, "delta", key("summary"));
|
|
5646
|
+
case "response.output_text.done":
|
|
5647
|
+
case "response.reasoning_text.done":
|
|
5648
|
+
case "response.reasoning_summary_text.done":
|
|
5649
|
+
return this.scrubField(record, "text", void 0, true);
|
|
5650
|
+
case "response.content_part.added":
|
|
5651
|
+
case "response.content_part.done":
|
|
5652
|
+
case "response.reasoning_summary_part.added":
|
|
5653
|
+
case "response.reasoning_summary_part.done":
|
|
5654
|
+
return this.scrubParts(record.part ? [record.part] : void 0, void 0, true);
|
|
5655
|
+
case "response.output_item.added":
|
|
5656
|
+
case "response.output_item.done":
|
|
5657
|
+
return this.scrubItems(record.item ? [record.item] : void 0);
|
|
5658
|
+
case "response.completed": {
|
|
5659
|
+
const changed = this.scrubResponseObject(record.response);
|
|
5660
|
+
this.streams.clear();
|
|
5661
|
+
return changed;
|
|
5662
|
+
}
|
|
5663
|
+
default:
|
|
5664
|
+
return false;
|
|
5665
|
+
}
|
|
5666
|
+
}
|
|
5667
|
+
scrubResponseObject(response) {
|
|
5668
|
+
if (!response || typeof response !== "object") {
|
|
5669
|
+
return false;
|
|
5670
|
+
}
|
|
5671
|
+
let changed = this.scrubItems(response.output);
|
|
5672
|
+
if (typeof response.output_text === "string") {
|
|
5673
|
+
changed = assignText(response, "output_text", this.complete(response.output_text)) || changed;
|
|
5674
|
+
}
|
|
5675
|
+
return changed;
|
|
5676
|
+
}
|
|
5677
|
+
scrubItems(items) {
|
|
5678
|
+
if (!Array.isArray(items)) {
|
|
5679
|
+
return false;
|
|
5680
|
+
}
|
|
5681
|
+
let changed = false;
|
|
5682
|
+
items.forEach((entry) => {
|
|
5683
|
+
if (!entry || typeof entry !== "object") {
|
|
5684
|
+
return;
|
|
5685
|
+
}
|
|
5686
|
+
changed = this.scrubParts(entry.content, void 0, true) || changed;
|
|
5687
|
+
changed = this.scrubParts(entry.summary, void 0, true) || changed;
|
|
5688
|
+
});
|
|
5689
|
+
return changed;
|
|
5690
|
+
}
|
|
5691
|
+
scrubAnthropic(record) {
|
|
5692
|
+
const type = record.type;
|
|
5693
|
+
if (type === "content_block_start") {
|
|
5694
|
+
const block = record.content_block;
|
|
5695
|
+
if (!block || typeof block !== "object") {
|
|
5696
|
+
return false;
|
|
5697
|
+
}
|
|
5698
|
+
if (block.type === "text") {
|
|
5699
|
+
return this.scrubField(block, "text", `anthropic:text:${record.index}`);
|
|
5700
|
+
}
|
|
5701
|
+
if (block.type === "thinking") {
|
|
5702
|
+
return this.scrubField(block, "thinking", `anthropic:thinking:${record.index}`);
|
|
5703
|
+
}
|
|
5704
|
+
return false;
|
|
5705
|
+
}
|
|
5706
|
+
if (type === "content_block_delta") {
|
|
5707
|
+
const delta = record.delta;
|
|
5708
|
+
if (!delta || typeof delta !== "object") {
|
|
5709
|
+
return false;
|
|
5710
|
+
}
|
|
5711
|
+
if (delta.type === "text_delta") {
|
|
5712
|
+
return this.scrubField(delta, "text", `anthropic:text:${record.index}`);
|
|
5713
|
+
}
|
|
5714
|
+
if (delta.type === "thinking_delta") {
|
|
5715
|
+
return this.scrubField(delta, "thinking", `anthropic:thinking:${record.index}`);
|
|
5716
|
+
}
|
|
5717
|
+
return false;
|
|
5718
|
+
}
|
|
5719
|
+
if (type === "content_block_stop") {
|
|
5720
|
+
this.streams.delete(`anthropic:text:${record.index}`);
|
|
5721
|
+
this.streams.delete(`anthropic:thinking:${record.index}`);
|
|
5722
|
+
return false;
|
|
5723
|
+
}
|
|
5724
|
+
if (type === "message" && Array.isArray(record.content)) {
|
|
5725
|
+
let changed = false;
|
|
5726
|
+
record.content.forEach((block) => {
|
|
5727
|
+
if (block?.type === "text") {
|
|
5728
|
+
changed = this.scrubField(block, "text", void 0, true) || changed;
|
|
5729
|
+
} else if (block?.type === "thinking") {
|
|
5730
|
+
changed = this.scrubField(block, "thinking", void 0, true) || changed;
|
|
5731
|
+
}
|
|
5732
|
+
});
|
|
5733
|
+
return changed;
|
|
5734
|
+
}
|
|
5735
|
+
return false;
|
|
5736
|
+
}
|
|
5737
|
+
scrubGoogle(record, complete) {
|
|
5738
|
+
const candidates = Array.isArray(record.candidates) ? record.candidates : [];
|
|
5739
|
+
let changed = false;
|
|
5740
|
+
candidates.forEach((candidate, candidateIndex) => {
|
|
5741
|
+
const parts = candidate?.content?.parts;
|
|
5742
|
+
if (Array.isArray(parts)) {
|
|
5743
|
+
parts.forEach((part, partIndex) => {
|
|
5744
|
+
if (!part || typeof part !== "object" || typeof part.text !== "string") {
|
|
5745
|
+
return;
|
|
5746
|
+
}
|
|
5747
|
+
changed = assignText(
|
|
5748
|
+
part,
|
|
5749
|
+
"text",
|
|
5750
|
+
this.textFor(
|
|
5751
|
+
part.text,
|
|
5752
|
+
complete ? void 0 : `google:${candidateIndex}:${partIndex}`,
|
|
5753
|
+
complete
|
|
5754
|
+
)
|
|
5755
|
+
) || changed;
|
|
5756
|
+
});
|
|
5757
|
+
}
|
|
5758
|
+
if (candidate?.finishReason) {
|
|
5759
|
+
this.streams.clear();
|
|
5760
|
+
}
|
|
5761
|
+
});
|
|
5762
|
+
return changed;
|
|
5763
|
+
}
|
|
5764
|
+
};
|
|
5765
|
+
function findBoundary(buffer) {
|
|
5766
|
+
const lf = buffer.indexOf("\n\n");
|
|
5767
|
+
const crlf = buffer.indexOf("\r\n\r\n");
|
|
5768
|
+
if (lf === -1 && crlf === -1) {
|
|
5769
|
+
return null;
|
|
5770
|
+
}
|
|
5771
|
+
if (crlf !== -1 && (lf === -1 || crlf <= lf)) {
|
|
5772
|
+
return { start: crlf, end: crlf + 4, separator: "\r\n\r\n" };
|
|
5773
|
+
}
|
|
5774
|
+
return { start: lf, end: lf + 2, separator: "\n\n" };
|
|
5775
|
+
}
|
|
5776
|
+
function scrubJsonBody(body, config, scrubber) {
|
|
5777
|
+
const trimmed = body.trim();
|
|
5778
|
+
if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) {
|
|
5779
|
+
return body;
|
|
5780
|
+
}
|
|
5781
|
+
try {
|
|
5782
|
+
const payload = JSON.parse(trimmed);
|
|
5783
|
+
const worker = scrubber ?? new ProviderPayloadScrubber(config);
|
|
5784
|
+
if (worker.scrubComplete(payload)) {
|
|
5785
|
+
return JSON.stringify(payload);
|
|
5786
|
+
}
|
|
5787
|
+
} catch {
|
|
5788
|
+
}
|
|
5789
|
+
return body;
|
|
5790
|
+
}
|
|
5791
|
+
function createSseScrubTransform(config) {
|
|
5792
|
+
const TransformStreamImpl = globalThis.TransformStream;
|
|
5793
|
+
const decoder = new TextDecoder();
|
|
5794
|
+
const encoder = new TextEncoder();
|
|
5795
|
+
const scrubber = new ProviderPayloadScrubber(config);
|
|
5796
|
+
let buffer = "";
|
|
5797
|
+
const processBlock = (block) => {
|
|
5798
|
+
try {
|
|
5799
|
+
const lines = block.split("\n");
|
|
5800
|
+
const dataLines = [];
|
|
5801
|
+
lines.forEach((raw, index) => {
|
|
5802
|
+
const line = raw.endsWith("\r") ? raw.slice(0, -1) : raw;
|
|
5803
|
+
const match = /^data:( ?)(.*)$/.exec(line);
|
|
5804
|
+
if (match) {
|
|
5805
|
+
dataLines.push({ index, space: match[1] ?? "", value: match[2] ?? "" });
|
|
5806
|
+
}
|
|
5807
|
+
});
|
|
5808
|
+
if (dataLines.length === 0) {
|
|
5809
|
+
return block;
|
|
5810
|
+
}
|
|
5811
|
+
const data = dataLines.map((entry) => entry.value).join("\n");
|
|
5812
|
+
const payloadText = data.trim();
|
|
5813
|
+
if (!payloadText || payloadText === "[DONE]") {
|
|
5814
|
+
return block;
|
|
5815
|
+
}
|
|
5816
|
+
let payload;
|
|
5817
|
+
try {
|
|
5818
|
+
payload = JSON.parse(data);
|
|
5819
|
+
} catch {
|
|
5820
|
+
return block;
|
|
5821
|
+
}
|
|
5822
|
+
if (!scrubber.scrub(payload)) {
|
|
5823
|
+
return block;
|
|
5824
|
+
}
|
|
5825
|
+
const first = dataLines[0];
|
|
5826
|
+
const drop = new Set(dataLines.slice(1).map((entry) => entry.index));
|
|
5827
|
+
const rebuilt = [];
|
|
5828
|
+
lines.forEach((raw, index) => {
|
|
5829
|
+
if (drop.has(index)) {
|
|
5830
|
+
return;
|
|
5831
|
+
}
|
|
5832
|
+
if (index === first.index) {
|
|
5833
|
+
rebuilt.push(`data:${first.space}${JSON.stringify(payload)}`);
|
|
5834
|
+
return;
|
|
5835
|
+
}
|
|
5836
|
+
rebuilt.push(raw);
|
|
5837
|
+
});
|
|
5838
|
+
return rebuilt.join("\n");
|
|
5839
|
+
} catch {
|
|
5840
|
+
return block;
|
|
5841
|
+
}
|
|
5842
|
+
};
|
|
5843
|
+
return new TransformStreamImpl({
|
|
5844
|
+
transform(chunk, controller) {
|
|
5845
|
+
let enqueued = false;
|
|
5846
|
+
try {
|
|
5847
|
+
buffer += decoder.decode(chunk, { stream: true });
|
|
5848
|
+
let boundary = findBoundary(buffer);
|
|
5849
|
+
while (boundary) {
|
|
5850
|
+
const block = buffer.slice(0, boundary.start);
|
|
5851
|
+
buffer = buffer.slice(boundary.end);
|
|
5852
|
+
controller.enqueue(encoder.encode(processBlock(block) + boundary.separator));
|
|
5853
|
+
enqueued = true;
|
|
5854
|
+
boundary = findBoundary(buffer);
|
|
5855
|
+
}
|
|
5856
|
+
} catch {
|
|
5857
|
+
if (!enqueued) {
|
|
5858
|
+
try {
|
|
5859
|
+
controller.enqueue(chunk);
|
|
5860
|
+
} catch {
|
|
5861
|
+
}
|
|
5862
|
+
}
|
|
5863
|
+
}
|
|
5864
|
+
},
|
|
5865
|
+
flush(controller) {
|
|
5866
|
+
try {
|
|
5867
|
+
buffer += decoder.decode();
|
|
5868
|
+
if (buffer) {
|
|
5869
|
+
controller.enqueue(encoder.encode(processBlock(buffer)));
|
|
5870
|
+
}
|
|
5871
|
+
buffer = "";
|
|
5872
|
+
} catch {
|
|
5873
|
+
}
|
|
5874
|
+
}
|
|
5875
|
+
});
|
|
5876
|
+
}
|
|
5877
|
+
function createJsonScrubTransform(config) {
|
|
5878
|
+
const TransformStreamImpl = globalThis.TransformStream;
|
|
5879
|
+
const decoder = new TextDecoder();
|
|
5880
|
+
const encoder = new TextEncoder();
|
|
5881
|
+
const scrubber = new ProviderPayloadScrubber(config);
|
|
5882
|
+
let buffer = "";
|
|
5883
|
+
return new TransformStreamImpl({
|
|
5884
|
+
transform(chunk, _controller) {
|
|
5885
|
+
try {
|
|
5886
|
+
buffer += decoder.decode(chunk, { stream: true });
|
|
5887
|
+
} catch {
|
|
5888
|
+
}
|
|
5889
|
+
},
|
|
5890
|
+
flush(controller) {
|
|
5891
|
+
try {
|
|
5892
|
+
buffer += decoder.decode();
|
|
5893
|
+
controller.enqueue(encoder.encode(scrubJsonBody(buffer, config, scrubber)));
|
|
5894
|
+
buffer = "";
|
|
5895
|
+
} catch {
|
|
5896
|
+
try {
|
|
5897
|
+
controller.enqueue(encoder.encode(buffer));
|
|
5898
|
+
buffer = "";
|
|
5899
|
+
} catch {
|
|
5900
|
+
}
|
|
5901
|
+
}
|
|
5902
|
+
}
|
|
5903
|
+
});
|
|
5904
|
+
}
|
|
5905
|
+
function wrapScrubResponse(response, config) {
|
|
5906
|
+
try {
|
|
5907
|
+
if (!response || typeof response !== "object" || !response.body) {
|
|
5908
|
+
return response;
|
|
5909
|
+
}
|
|
5910
|
+
const headers = new Headers(response.headers);
|
|
5911
|
+
const contentType = headers.get("content-type") ?? "";
|
|
5912
|
+
let body;
|
|
5913
|
+
if (/text\/event-stream/i.test(contentType)) {
|
|
5914
|
+
body = response.body.pipeThrough(createSseScrubTransform(config));
|
|
5915
|
+
} else if (/json/i.test(contentType)) {
|
|
5916
|
+
body = response.body.pipeThrough(createJsonScrubTransform(config));
|
|
5917
|
+
} else {
|
|
5918
|
+
return response;
|
|
5919
|
+
}
|
|
5920
|
+
headers.delete("content-length");
|
|
5921
|
+
return new Response(body, {
|
|
5922
|
+
status: response.status,
|
|
5923
|
+
statusText: response.statusText,
|
|
5924
|
+
headers
|
|
5925
|
+
});
|
|
5926
|
+
} catch {
|
|
5927
|
+
return response;
|
|
5928
|
+
}
|
|
5929
|
+
}
|
|
5930
|
+
async function installResponseScrubHook(ctx, config, logger) {
|
|
5931
|
+
if (!outputScrubEnabled(config)) {
|
|
5932
|
+
return;
|
|
5933
|
+
}
|
|
5934
|
+
const session2 = ctx?.session;
|
|
5935
|
+
if (!session2 || typeof session2.hook !== "function") {
|
|
5936
|
+
return;
|
|
5937
|
+
}
|
|
5938
|
+
try {
|
|
5939
|
+
await session2.hook("http.response", async (event) => {
|
|
5940
|
+
try {
|
|
5941
|
+
const response = event?.response;
|
|
5942
|
+
if (!response || typeof response !== "object") {
|
|
5943
|
+
return;
|
|
5944
|
+
}
|
|
5945
|
+
const wrapped = wrapScrubResponse(response, config);
|
|
5946
|
+
if (wrapped !== response) {
|
|
5947
|
+
event.response = wrapped;
|
|
5948
|
+
}
|
|
5949
|
+
} catch (error) {
|
|
5950
|
+
logger.debug("DCP output scrub failed", { error: error?.message });
|
|
5951
|
+
}
|
|
5952
|
+
});
|
|
5953
|
+
logger.debug("DCP output scrub hook installed");
|
|
5954
|
+
} catch (error) {
|
|
5955
|
+
logger.warn("DCP output scrub hook registration failed", { error: error?.message });
|
|
5956
|
+
}
|
|
5957
|
+
}
|
|
5958
|
+
function isTextDelta(part) {
|
|
5959
|
+
if (part?.type === "text-delta") {
|
|
5960
|
+
return { family: "text-delta", field: deltaField(part) };
|
|
5961
|
+
}
|
|
5962
|
+
if (part?.type === "reasoning-delta" || part?.type === "reasoning") {
|
|
5963
|
+
return { family: "reasoning-delta", field: deltaField(part) };
|
|
5964
|
+
}
|
|
5965
|
+
return void 0;
|
|
5966
|
+
}
|
|
5967
|
+
function deltaField(part) {
|
|
5968
|
+
if (typeof part.text === "string") {
|
|
5969
|
+
return "text";
|
|
5970
|
+
}
|
|
5971
|
+
if (typeof part.delta === "string") {
|
|
5972
|
+
return "delta";
|
|
5973
|
+
}
|
|
5974
|
+
return "textDelta";
|
|
5975
|
+
}
|
|
5976
|
+
var AisdkPartScrubber = class {
|
|
5977
|
+
constructor(config) {
|
|
5978
|
+
this.config = config;
|
|
5979
|
+
}
|
|
5980
|
+
config;
|
|
5981
|
+
streams = /* @__PURE__ */ new Map();
|
|
5982
|
+
process(part) {
|
|
5983
|
+
try {
|
|
5984
|
+
if (!part || typeof part !== "object") {
|
|
5985
|
+
return [part];
|
|
5986
|
+
}
|
|
5987
|
+
const delta = isTextDelta(part);
|
|
5988
|
+
if (delta) {
|
|
5989
|
+
const key = `${delta.family}:${part.id ?? ""}`;
|
|
5990
|
+
return [{ ...part, [delta.field]: this.stream(key).push(part[delta.field]) }];
|
|
5991
|
+
}
|
|
5992
|
+
if (part.type === "text-start" || part.type === "reasoning-start") {
|
|
5993
|
+
const family = part.type === "text-start" ? "text-delta" : "reasoning-delta";
|
|
5994
|
+
this.streams.delete(`${family}:${part.id ?? ""}`);
|
|
5995
|
+
return [part];
|
|
5996
|
+
}
|
|
5997
|
+
if (part.type === "text-end" || part.type === "reasoning-end") {
|
|
5998
|
+
const family = part.type === "text-end" ? "text-delta" : "reasoning-delta";
|
|
5999
|
+
const out = [];
|
|
6000
|
+
const leftover = this.take(`${family}:${part.id ?? ""}`);
|
|
6001
|
+
if (leftover) {
|
|
6002
|
+
out.push({ type: family, id: part.id, text: leftover });
|
|
6003
|
+
}
|
|
6004
|
+
const field = deltaField(part);
|
|
6005
|
+
if (typeof part[field] === "string") {
|
|
6006
|
+
out.push({ ...part, [field]: stripDcpArtifacts(part[field], this.config) });
|
|
6007
|
+
} else {
|
|
6008
|
+
out.push(part);
|
|
6009
|
+
}
|
|
6010
|
+
return out;
|
|
6011
|
+
}
|
|
6012
|
+
if (part.type === "finish") {
|
|
6013
|
+
return [...this.flushStreams(), part];
|
|
6014
|
+
}
|
|
6015
|
+
return [part];
|
|
6016
|
+
} catch {
|
|
6017
|
+
return [part];
|
|
6018
|
+
}
|
|
6019
|
+
}
|
|
6020
|
+
stream(key) {
|
|
6021
|
+
let scrubber = this.streams.get(key);
|
|
6022
|
+
if (!scrubber) {
|
|
6023
|
+
scrubber = new DcpStreamScrubber(this.config);
|
|
6024
|
+
this.streams.set(key, scrubber);
|
|
6025
|
+
}
|
|
6026
|
+
return scrubber;
|
|
6027
|
+
}
|
|
6028
|
+
take(key) {
|
|
6029
|
+
const scrubber = this.streams.get(key);
|
|
6030
|
+
if (!scrubber) {
|
|
6031
|
+
return "";
|
|
6032
|
+
}
|
|
6033
|
+
this.streams.delete(key);
|
|
6034
|
+
return scrubber.flush();
|
|
6035
|
+
}
|
|
6036
|
+
flushStreams() {
|
|
6037
|
+
const out = [];
|
|
6038
|
+
for (const [key, scrubber] of this.streams) {
|
|
6039
|
+
const leftover = scrubber.flush();
|
|
6040
|
+
if (!leftover) {
|
|
6041
|
+
continue;
|
|
6042
|
+
}
|
|
6043
|
+
const separator = key.indexOf(":");
|
|
6044
|
+
out.push({
|
|
6045
|
+
type: separator === -1 ? "text-delta" : key.slice(0, separator),
|
|
6046
|
+
id: separator === -1 ? "" : key.slice(separator + 1),
|
|
6047
|
+
text: leftover
|
|
6048
|
+
});
|
|
6049
|
+
}
|
|
6050
|
+
this.streams.clear();
|
|
6051
|
+
return out;
|
|
6052
|
+
}
|
|
6053
|
+
};
|
|
6054
|
+
function stripAisdkResultContent(result, config) {
|
|
6055
|
+
if (!result || !Array.isArray(result.content)) {
|
|
6056
|
+
return;
|
|
6057
|
+
}
|
|
6058
|
+
for (const part of result.content) {
|
|
6059
|
+
if (!part || typeof part !== "object") {
|
|
6060
|
+
continue;
|
|
6061
|
+
}
|
|
6062
|
+
if ((part.type === "text" || part.type === "reasoning") && typeof part.text === "string") {
|
|
6063
|
+
part.text = stripDcpArtifacts(part.text, config);
|
|
6064
|
+
}
|
|
6065
|
+
}
|
|
6066
|
+
}
|
|
6067
|
+
|
|
5227
6068
|
// lib/v2/usage.ts
|
|
5228
6069
|
function createCompressionUsageTracker() {
|
|
5229
6070
|
const pending = /* @__PURE__ */ new Map();
|
|
@@ -5299,35 +6140,37 @@ function collectTexts(value, out, depth = 0) {
|
|
|
5299
6140
|
}
|
|
5300
6141
|
}
|
|
5301
6142
|
}
|
|
5302
|
-
function wrapLanguageModel(model, tracker, logger) {
|
|
6143
|
+
function wrapLanguageModel(model, tracker, logger, scrub) {
|
|
5303
6144
|
return new Proxy(model, {
|
|
5304
6145
|
get(target, property, receiver) {
|
|
5305
6146
|
if (property === "doStream") {
|
|
5306
|
-
return (options) => trackedStream(target, options, tracker, logger);
|
|
6147
|
+
return (options) => trackedStream(target, options, tracker, logger, scrub);
|
|
5307
6148
|
}
|
|
5308
6149
|
if (property === "doGenerate") {
|
|
5309
|
-
return (options) => trackedGenerate(target, options, tracker, logger);
|
|
6150
|
+
return (options) => trackedGenerate(target, options, tracker, logger, scrub);
|
|
5310
6151
|
}
|
|
5311
6152
|
return Reflect.get(target, property, receiver);
|
|
5312
6153
|
}
|
|
5313
6154
|
});
|
|
5314
6155
|
}
|
|
5315
|
-
async function trackedStream(target, options, tracker, logger) {
|
|
6156
|
+
async function trackedStream(target, options, tracker, logger, scrub) {
|
|
5316
6157
|
const result = await target.doStream(options);
|
|
5317
6158
|
const callId = findSummaryCallId(options);
|
|
6159
|
+
const scrubEnabled = !!scrub && outputScrubEnabled(scrub);
|
|
5318
6160
|
logger.debug("Compression usage stream call", { callId: callId ?? null });
|
|
5319
6161
|
const TransformStreamImpl = globalThis.TransformStream;
|
|
5320
|
-
if (!callId || typeof result?.stream?.pipeThrough !== "function") {
|
|
6162
|
+
if (!callId && !scrubEnabled || typeof result?.stream?.pipeThrough !== "function") {
|
|
5321
6163
|
return result;
|
|
5322
6164
|
}
|
|
5323
6165
|
if (typeof TransformStreamImpl !== "function") {
|
|
5324
6166
|
return result;
|
|
5325
6167
|
}
|
|
6168
|
+
const partScrubber = scrubEnabled ? new AisdkPartScrubber(scrub) : void 0;
|
|
5326
6169
|
let recorded = false;
|
|
5327
6170
|
const transform = new TransformStreamImpl({
|
|
5328
6171
|
transform(part, controller) {
|
|
5329
6172
|
try {
|
|
5330
|
-
if (!recorded && part?.type === "finish" && part.usage) {
|
|
6173
|
+
if (callId && !recorded && part?.type === "finish" && part.usage) {
|
|
5331
6174
|
const usage = normalizeProviderUsage(part.usage);
|
|
5332
6175
|
if (usage) {
|
|
5333
6176
|
recorded = true;
|
|
@@ -5336,15 +6179,24 @@ async function trackedStream(target, options, tracker, logger) {
|
|
|
5336
6179
|
}
|
|
5337
6180
|
} catch {
|
|
5338
6181
|
}
|
|
5339
|
-
|
|
6182
|
+
const parts = partScrubber ? partScrubber.process(part) : [part];
|
|
6183
|
+
for (const next of parts) {
|
|
6184
|
+
controller.enqueue(next);
|
|
6185
|
+
}
|
|
5340
6186
|
}
|
|
5341
6187
|
});
|
|
5342
6188
|
return { ...result, stream: result.stream.pipeThrough(transform) };
|
|
5343
6189
|
}
|
|
5344
|
-
async function trackedGenerate(target, options, tracker, logger) {
|
|
6190
|
+
async function trackedGenerate(target, options, tracker, logger, scrub) {
|
|
5345
6191
|
const result = await target.doGenerate(options);
|
|
5346
6192
|
const callId = findSummaryCallId(options);
|
|
5347
6193
|
logger.debug("Compression usage generate call", { callId: callId ?? null });
|
|
6194
|
+
if (scrub && outputScrubEnabled(scrub)) {
|
|
6195
|
+
try {
|
|
6196
|
+
stripAisdkResultContent(result, scrub);
|
|
6197
|
+
} catch {
|
|
6198
|
+
}
|
|
6199
|
+
}
|
|
5348
6200
|
if (!callId) {
|
|
5349
6201
|
return result;
|
|
5350
6202
|
}
|
|
@@ -5354,7 +6206,7 @@ async function trackedGenerate(target, options, tracker, logger) {
|
|
|
5354
6206
|
}
|
|
5355
6207
|
return result;
|
|
5356
6208
|
}
|
|
5357
|
-
async function installCompressionUsageHook(ctx, tracker, logger) {
|
|
6209
|
+
async function installCompressionUsageHook(ctx, tracker, logger, scrub) {
|
|
5358
6210
|
const aisdk = ctx?.aisdk;
|
|
5359
6211
|
if (!aisdk || typeof aisdk.hook !== "function") {
|
|
5360
6212
|
logger.debug("aisdk hook unavailable; compression usage will be estimated");
|
|
@@ -5378,7 +6230,7 @@ async function installCompressionUsageHook(ctx, tracker, logger) {
|
|
|
5378
6230
|
if (!isModel(model)) {
|
|
5379
6231
|
return;
|
|
5380
6232
|
}
|
|
5381
|
-
event.language = wrapLanguageModel(model, tracker, logger);
|
|
6233
|
+
event.language = wrapLanguageModel(model, tracker, logger, scrub);
|
|
5382
6234
|
} catch (error) {
|
|
5383
6235
|
logger.debug("Compression usage hook could not wrap language model", {
|
|
5384
6236
|
error: error?.message
|
|
@@ -5740,8 +6592,8 @@ async function handleContextCommand(ctx) {
|
|
|
5740
6592
|
// lib/messages/utils.ts
|
|
5741
6593
|
import { createHash } from "crypto";
|
|
5742
6594
|
var SUMMARY_ID_HASH_LENGTH = 16;
|
|
5743
|
-
var
|
|
5744
|
-
var
|
|
6595
|
+
var DCP_PAIRED_TAG_REGEX2 = /<dcp[^>]*>[\s\S]*?<\/dcp[^>]*>/gi;
|
|
6596
|
+
var DCP_UNPAIRED_TAG_REGEX2 = /<\/?dcp[^>]*>/gi;
|
|
5745
6597
|
var INJECTED_MESSAGE_ID_SUFFIX_REGEX = /(?<=\n)<dcp-message-id[^>]*>m\d+<\/dcp-message-id>\s*$/;
|
|
5746
6598
|
var HALLUCINATED_PARAMETER_SUFFIX_REGEX = /(?<=\n)m\d+<\/parameter>\s*$/;
|
|
5747
6599
|
var COMPACT_TAG_REGEX = /@(?:[1-9]\d*|b[1-9]\d*|blocked)@(?:[ \t]+\[(?:low|medium|high)\])?/gi;
|
|
@@ -5865,7 +6717,7 @@ function buildToolIdList(state, messages) {
|
|
|
5865
6717
|
var stripHallucinationsFromString = (text, format2 = "xml") => {
|
|
5866
6718
|
if (format2 === "compact") text = text.replace(COMPACT_TAG_REGEX, "");
|
|
5867
6719
|
const withoutKnownSuffixes = text.replace(INJECTED_MESSAGE_ID_SUFFIX_REGEX, "").replace(HALLUCINATED_PARAMETER_SUFFIX_REGEX, "");
|
|
5868
|
-
return withoutKnownSuffixes.replace(
|
|
6720
|
+
return withoutKnownSuffixes.replace(DCP_PAIRED_TAG_REGEX2, "").replace(DCP_UNPAIRED_TAG_REGEX2, "");
|
|
5869
6721
|
};
|
|
5870
6722
|
var stripHallucinations = (messages, format2 = "xml") => {
|
|
5871
6723
|
for (const message of messages) {
|
|
@@ -7786,8 +8638,13 @@ async function setup(ctx) {
|
|
|
7786
8638
|
}
|
|
7787
8639
|
const logger = new Logger(config.debug);
|
|
7788
8640
|
const usageTracker = createCompressionUsageTracker();
|
|
7789
|
-
|
|
8641
|
+
const scrub = {
|
|
8642
|
+
modelOutput: config.compress.scrubModelOutput,
|
|
8643
|
+
messageIds: config.compress.scrubMessageIds
|
|
8644
|
+
};
|
|
8645
|
+
await installCompressionUsageHook(ctx, usageTracker, logger, scrub);
|
|
7790
8646
|
await installHttpUsageHook(ctx, usageTracker, logger);
|
|
8647
|
+
await installResponseScrubHook(ctx, scrub, logger);
|
|
7791
8648
|
const prompts = new PromptStore(
|
|
7792
8649
|
logger,
|
|
7793
8650
|
ctx.location.directory,
|