@remnic/bench 9.3.528 → 9.3.530
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1034 -142
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -3872,15 +3872,7 @@ var BENCHMARK_RESULT_SCHEMA = {
|
|
|
3872
3872
|
import { execFileSync } from "child_process";
|
|
3873
3873
|
import { createHash as createHash3 } from "crypto";
|
|
3874
3874
|
import { createReadStream } from "fs";
|
|
3875
|
-
import {
|
|
3876
|
-
lstat as lstat3,
|
|
3877
|
-
mkdir as mkdir3,
|
|
3878
|
-
readdir as readdir4,
|
|
3879
|
-
readlink,
|
|
3880
|
-
realpath as realpath3,
|
|
3881
|
-
stat as stat2,
|
|
3882
|
-
writeFile as writeFile3
|
|
3883
|
-
} from "fs/promises";
|
|
3875
|
+
import { lstat as lstat3, mkdir as mkdir3, readFile as readFile4, readdir as readdir4, readlink, realpath as realpath3, stat as stat2, writeFile as writeFile3 } from "fs/promises";
|
|
3884
3876
|
import os2 from "os";
|
|
3885
3877
|
import path4 from "path";
|
|
3886
3878
|
|
|
@@ -4573,6 +4565,94 @@ function resolveBenchmarkRunId() {
|
|
|
4573
4565
|
return generatedBenchmarkRunId;
|
|
4574
4566
|
}
|
|
4575
4567
|
|
|
4568
|
+
// src/security/url-secrets.ts
|
|
4569
|
+
function redactUrlSecrets(value, redactedValue, isSecretQueryKey) {
|
|
4570
|
+
return redactUrlQuerySecrets(redactUrlUserinfoSecrets(value, redactedValue), redactedValue, isSecretQueryKey);
|
|
4571
|
+
}
|
|
4572
|
+
function redactUrlUserinfoSecrets(value, redactedValue) {
|
|
4573
|
+
if (!value.includes("://")) return value;
|
|
4574
|
+
let redacted = "";
|
|
4575
|
+
let cursor = 0;
|
|
4576
|
+
let changed = false;
|
|
4577
|
+
for (let schemeEnd = value.indexOf("://"); schemeEnd !== -1; schemeEnd = value.indexOf("://", schemeEnd + 3)) {
|
|
4578
|
+
const schemeStart = findUrlSchemeStart(value, schemeEnd);
|
|
4579
|
+
if (schemeStart === -1) continue;
|
|
4580
|
+
const authorityStart = schemeEnd + 3;
|
|
4581
|
+
const authorityEnd = findUrlAuthorityEnd(value, authorityStart);
|
|
4582
|
+
const atIndex = value.lastIndexOf("@", authorityEnd - 1);
|
|
4583
|
+
if (atIndex < authorityStart) continue;
|
|
4584
|
+
redacted += value.slice(cursor, authorityStart);
|
|
4585
|
+
redacted += redactedValue;
|
|
4586
|
+
cursor = atIndex;
|
|
4587
|
+
changed = true;
|
|
4588
|
+
}
|
|
4589
|
+
return changed ? redacted + value.slice(cursor) : value;
|
|
4590
|
+
}
|
|
4591
|
+
function redactUrlQuerySecrets(value, redactedValue, isSecretQueryKey) {
|
|
4592
|
+
if (!/[?&;]/.test(value)) return value;
|
|
4593
|
+
let redacted = "";
|
|
4594
|
+
let cursor = 0;
|
|
4595
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
4596
|
+
if (value[index] !== "?" && value[index] !== "&" && value[index] !== ";") continue;
|
|
4597
|
+
const keyStart = index + 1;
|
|
4598
|
+
const equalsIndex = value.indexOf("=", keyStart);
|
|
4599
|
+
if (equalsIndex === -1) continue;
|
|
4600
|
+
const valueEnd = findQueryValueEnd(value, equalsIndex + 1);
|
|
4601
|
+
const key = value.slice(keyStart, equalsIndex);
|
|
4602
|
+
if (isSecretUrlQueryKey(key, isSecretQueryKey)) {
|
|
4603
|
+
if (redactedValue.length > 0 && value.startsWith(redactedValue, equalsIndex + 1)) continue;
|
|
4604
|
+
redacted += value.slice(cursor, equalsIndex + 1);
|
|
4605
|
+
redacted += redactedValue;
|
|
4606
|
+
cursor = valueEnd;
|
|
4607
|
+
index = valueEnd - 1;
|
|
4608
|
+
}
|
|
4609
|
+
}
|
|
4610
|
+
return cursor === 0 ? value : redacted + value.slice(cursor);
|
|
4611
|
+
}
|
|
4612
|
+
function findQueryValueEnd(value, startIndex) {
|
|
4613
|
+
let index = startIndex;
|
|
4614
|
+
while (index < value.length && !isQueryValueTerminator(value[index])) {
|
|
4615
|
+
index += 1;
|
|
4616
|
+
}
|
|
4617
|
+
return index;
|
|
4618
|
+
}
|
|
4619
|
+
function isQueryValueTerminator(char) {
|
|
4620
|
+
return char === "&" || char === ";" || char === "#" || char === '"' || char === "'" || char === "}" || char === "]" || isWhitespace(char);
|
|
4621
|
+
}
|
|
4622
|
+
function isSecretUrlQueryKey(key, isSecretQueryKey) {
|
|
4623
|
+
try {
|
|
4624
|
+
return isSecretQueryKey(decodeURIComponent(key.replace(/\+/g, " ")));
|
|
4625
|
+
} catch {
|
|
4626
|
+
return isSecretQueryKey(key);
|
|
4627
|
+
}
|
|
4628
|
+
}
|
|
4629
|
+
function findUrlSchemeStart(value, schemeEnd) {
|
|
4630
|
+
let cursor = schemeEnd - 1;
|
|
4631
|
+
while (cursor >= 0 && isUrlSchemeChar(value[cursor])) cursor -= 1;
|
|
4632
|
+
const schemeStart = cursor + 1;
|
|
4633
|
+
return schemeStart < schemeEnd && isAsciiLetter(value[schemeStart]) ? schemeStart : -1;
|
|
4634
|
+
}
|
|
4635
|
+
function isUrlSchemeChar(char) {
|
|
4636
|
+
return isAsciiAlnum(char) || char === "+" || char === "-" || char === ".";
|
|
4637
|
+
}
|
|
4638
|
+
function findUrlAuthorityEnd(value, authorityStart) {
|
|
4639
|
+
let cursor = authorityStart;
|
|
4640
|
+
while (cursor < value.length && !isUrlAuthorityTerminator(value[cursor])) cursor += 1;
|
|
4641
|
+
return cursor;
|
|
4642
|
+
}
|
|
4643
|
+
function isUrlAuthorityTerminator(char) {
|
|
4644
|
+
return char === "/" || char === "?" || char === "#" || isWhitespace(char) || char === '"' || char === "'";
|
|
4645
|
+
}
|
|
4646
|
+
function isAsciiLetter(char) {
|
|
4647
|
+
return char >= "A" && char <= "Z" || char >= "a" && char <= "z";
|
|
4648
|
+
}
|
|
4649
|
+
function isAsciiAlnum(char) {
|
|
4650
|
+
return isAsciiLetter(char) || char >= "0" && char <= "9";
|
|
4651
|
+
}
|
|
4652
|
+
function isWhitespace(char) {
|
|
4653
|
+
return char === " " || char === " " || char === "\n" || char === "\r";
|
|
4654
|
+
}
|
|
4655
|
+
|
|
4576
4656
|
// src/repro-manifest.ts
|
|
4577
4657
|
var BENCHMARK_REPRO_MANIFEST_FILENAME = "MANIFEST.json";
|
|
4578
4658
|
var BENCHMARK_REPRO_MANIFEST_SCHEMA_VERSION = 1;
|
|
@@ -4581,12 +4661,108 @@ var SECRET_ARG_FLAGS = /* @__PURE__ */ new Set([
|
|
|
4581
4661
|
"--system-api-key",
|
|
4582
4662
|
"--judge-api-key",
|
|
4583
4663
|
"--token",
|
|
4584
|
-
"--auth-token"
|
|
4664
|
+
"--auth-token",
|
|
4665
|
+
"-k",
|
|
4666
|
+
"-p",
|
|
4667
|
+
"-t"
|
|
4668
|
+
]);
|
|
4669
|
+
var ATTACHED_LONG_SECRET_ARG_FLAGS = [
|
|
4670
|
+
"--system-api-key",
|
|
4671
|
+
"--judge-api-key",
|
|
4672
|
+
"--auth-token",
|
|
4673
|
+
"--api-key",
|
|
4674
|
+
"--token"
|
|
4675
|
+
];
|
|
4676
|
+
var NON_SECRET_ARG_FLAGS = /* @__PURE__ */ new Set([
|
|
4677
|
+
"--ama-bench-cross-judge-base-url",
|
|
4678
|
+
"--base-url",
|
|
4679
|
+
"--config",
|
|
4680
|
+
"--dataset-dir",
|
|
4681
|
+
"--header",
|
|
4682
|
+
"--internal-base-url",
|
|
4683
|
+
"--judge-base-url",
|
|
4684
|
+
"--limit",
|
|
4685
|
+
"--max-tokens",
|
|
4686
|
+
"--mode",
|
|
4687
|
+
"--output-token-limit",
|
|
4688
|
+
"--provider-config",
|
|
4689
|
+
"--system-base-url"
|
|
4585
4690
|
]);
|
|
4586
|
-
var
|
|
4691
|
+
var BENCH_OPTION_BOUNDARY_FLAGS = /* @__PURE__ */ new Set([
|
|
4692
|
+
"--all",
|
|
4693
|
+
"--ama-bench-cross-judge-api-key",
|
|
4694
|
+
"--ama-bench-cross-judge-base-url",
|
|
4695
|
+
"--ama-bench-cross-judge-codex-reasoning-effort",
|
|
4696
|
+
"--ama-bench-cross-judge-model",
|
|
4697
|
+
"--ama-bench-cross-judge-provider",
|
|
4698
|
+
"--ama-bench-judge-protocol",
|
|
4699
|
+
"--base-url",
|
|
4700
|
+
"--baselines-dir",
|
|
4701
|
+
"--custom",
|
|
4702
|
+
"--dataset",
|
|
4703
|
+
"--dataset-dir",
|
|
4704
|
+
"--detail",
|
|
4705
|
+
"--disable-thinking",
|
|
4706
|
+
"--drain-timeout",
|
|
4707
|
+
"--dry-run",
|
|
4708
|
+
"--explain",
|
|
4709
|
+
"--fast-gateway-agent-id",
|
|
4710
|
+
"--format",
|
|
4711
|
+
"--gateway-agent-id",
|
|
4712
|
+
"--help",
|
|
4713
|
+
"--ingest-concurrency",
|
|
4714
|
+
"--internal-api-key",
|
|
4715
|
+
"--internal-base-url",
|
|
4716
|
+
"--internal-codex-reasoning-effort",
|
|
4717
|
+
"--internal-disable-thinking",
|
|
4718
|
+
"--internal-model",
|
|
4719
|
+
"--internal-provider",
|
|
4720
|
+
"--json",
|
|
4721
|
+
"--judge-api-key",
|
|
4722
|
+
"--judge-base-url",
|
|
4723
|
+
"--judge-codex-reasoning-effort",
|
|
4724
|
+
"--judge-model",
|
|
4725
|
+
"--judge-provider",
|
|
4726
|
+
"--limit",
|
|
4727
|
+
"--matrix",
|
|
4728
|
+
"--max-429-wait",
|
|
4729
|
+
"--model",
|
|
4730
|
+
"--model-source",
|
|
4731
|
+
"--name",
|
|
4732
|
+
"--openclaw-config",
|
|
4733
|
+
"--out",
|
|
4734
|
+
"--output",
|
|
4735
|
+
"--provider",
|
|
4736
|
+
"--quick",
|
|
4737
|
+
"--remnic-config",
|
|
4738
|
+
"--request-timeout",
|
|
4739
|
+
"--results-dir",
|
|
4740
|
+
"--resume",
|
|
4741
|
+
"--retry-failed",
|
|
4742
|
+
"--runtime-profile",
|
|
4743
|
+
"--seed",
|
|
4744
|
+
"--system-api-key",
|
|
4745
|
+
"--system-base-url",
|
|
4746
|
+
"--system-codex-reasoning-effort",
|
|
4747
|
+
"--system-model",
|
|
4748
|
+
"--system-provider",
|
|
4749
|
+
"--system-responder-context-budget-chars",
|
|
4750
|
+
"--system-responder-prompt-budget-chars",
|
|
4751
|
+
"--target",
|
|
4752
|
+
"--task-filter",
|
|
4753
|
+
"--threshold",
|
|
4754
|
+
"--trial-concurrency",
|
|
4755
|
+
"--trial-limit",
|
|
4756
|
+
"-h"
|
|
4757
|
+
]);
|
|
4758
|
+
var SECRET_KEY_PATTERN = /(^|[-_])(?:api[-_]?key|secret[-_]?access[-_]?key|secret[-_]?key|client[-_]?secret(?:[-_]?key)?|app[-_]?secret(?:[-_]?key)?|provider[-_]?secret(?:[-_]?key)?|access[-_]?key|private[-_]?key|secret|password|authorization|credential|access[-_]?token|auth[-_]?token|refresh[-_]?token|id[-_]?token|token)$/i;
|
|
4759
|
+
var REDACTED_ARG_VALUE = "[redacted]";
|
|
4587
4760
|
function sha256String(value) {
|
|
4588
4761
|
return createHash3("sha256").update(value).digest("hex");
|
|
4589
4762
|
}
|
|
4763
|
+
function sha256Buffer(value) {
|
|
4764
|
+
return createHash3("sha256").update(value).digest("hex");
|
|
4765
|
+
}
|
|
4590
4766
|
async function sha256File(filePath) {
|
|
4591
4767
|
const hash = createHash3("sha256");
|
|
4592
4768
|
await new Promise((resolve, reject) => {
|
|
@@ -4611,23 +4787,485 @@ function sanitizeArgv(argv) {
|
|
|
4611
4787
|
for (let index = 0; index < argv.length; index += 1) {
|
|
4612
4788
|
const arg = argv[index];
|
|
4613
4789
|
const isOptionFlag = arg.startsWith("-");
|
|
4614
|
-
const
|
|
4615
|
-
|
|
4616
|
-
|
|
4617
|
-
|
|
4790
|
+
const assignmentIndex = arg.indexOf("=");
|
|
4791
|
+
const flagName = assignmentIndex === -1 ? arg : arg.slice(0, assignmentIndex);
|
|
4792
|
+
const attachedShortSecretFlag = getAttachedShortSecretFlag(arg);
|
|
4793
|
+
const attachedLongSecret = getAttachedLongSecretArg(arg);
|
|
4794
|
+
if (attachedLongSecret) {
|
|
4795
|
+
sanitized.push(`${attachedLongSecret.flag}${attachedLongSecret.delimiter}${REDACTED_ARG_VALUE}`);
|
|
4796
|
+
const attachedSecretValue = arg.slice(
|
|
4797
|
+
attachedLongSecret.flag.length + (attachedLongSecret.delimiter === "=" ? 1 : 0)
|
|
4798
|
+
);
|
|
4799
|
+
if (shouldConsumeAuthSchemePair(attachedLongSecret.flag, attachedSecretValue)) {
|
|
4800
|
+
const consumedValues = countAuthSchemeContinuationTokens(attachedLongSecret.flag, argv, index + 1);
|
|
4801
|
+
sanitized.push(...redactedPlaceholders(consumedValues));
|
|
4802
|
+
index += consumedValues;
|
|
4803
|
+
}
|
|
4804
|
+
continue;
|
|
4805
|
+
}
|
|
4806
|
+
if (attachedShortSecretFlag) {
|
|
4807
|
+
sanitized.push(`${attachedShortSecretFlag}${REDACTED_ARG_VALUE}`);
|
|
4808
|
+
continue;
|
|
4809
|
+
}
|
|
4810
|
+
const secretDelimiterIndex = assignmentIndex === -1 ? findSecretConfigDelimiterIndex(arg) : -1;
|
|
4811
|
+
if (secretDelimiterIndex !== -1) {
|
|
4812
|
+
sanitized.push(sanitizeSecretDelimitedArg(arg, secretDelimiterIndex, ":"));
|
|
4813
|
+
if (arg.slice(secretDelimiterIndex + 1).trim().length === 0) {
|
|
4814
|
+
index += countBareSecretValueTokens(arg.slice(0, secretDelimiterIndex), argv, index + 1);
|
|
4815
|
+
}
|
|
4816
|
+
continue;
|
|
4817
|
+
}
|
|
4818
|
+
if (isOptionFlag && (SECRET_ARG_FLAGS.has(flagName) || isSecretOptionFlagName(flagName))) {
|
|
4819
|
+
if (assignmentIndex !== -1) {
|
|
4820
|
+
const assignedSecretValue = arg.slice(assignmentIndex + 1);
|
|
4821
|
+
sanitized.push(`${flagName}=${REDACTED_ARG_VALUE}`);
|
|
4822
|
+
if (shouldConsumeAuthSchemePair(flagName, assignedSecretValue)) {
|
|
4823
|
+
const consumedValues = countAuthSchemeContinuationTokens(flagName, argv, index + 1);
|
|
4824
|
+
sanitized.push(...redactedPlaceholders(consumedValues));
|
|
4825
|
+
index += consumedValues;
|
|
4826
|
+
}
|
|
4618
4827
|
} else {
|
|
4619
4828
|
sanitized.push(arg);
|
|
4620
|
-
if (index + 1 < argv.length) {
|
|
4621
|
-
|
|
4622
|
-
|
|
4829
|
+
if (index + 1 < argv.length && shouldConsumeSeparatedSecretFlagValue(argv[index + 1])) {
|
|
4830
|
+
const consumeAllValueTokens = isAuthorizationConfigKey(flagName) || (flagName === "--auth-token" || flagName === "--token") && isAuthSchemeToken(argv[index + 1]);
|
|
4831
|
+
let consumedValues = 1;
|
|
4832
|
+
while (consumeAllValueTokens && index + 1 + consumedValues < argv.length && !isOptionValueBoundaryFlag(argv[index + 1 + consumedValues])) {
|
|
4833
|
+
consumedValues += 1;
|
|
4834
|
+
}
|
|
4835
|
+
sanitized.push(...redactedPlaceholders(consumedValues));
|
|
4836
|
+
index += consumedValues;
|
|
4623
4837
|
}
|
|
4624
4838
|
}
|
|
4625
4839
|
continue;
|
|
4626
4840
|
}
|
|
4627
|
-
|
|
4841
|
+
if (assignmentIndex === -1 && isKnownNonSecretOptionFlag(arg)) {
|
|
4842
|
+
sanitized.push(arg);
|
|
4843
|
+
if (index + 1 < argv.length && !argv[index + 1].startsWith("-")) {
|
|
4844
|
+
const optionValue = sanitizeOptionValueSpan(arg, argv, index + 1);
|
|
4845
|
+
sanitized.push(...optionValue.values);
|
|
4846
|
+
index += optionValue.consumed;
|
|
4847
|
+
}
|
|
4848
|
+
continue;
|
|
4849
|
+
}
|
|
4850
|
+
if (assignmentIndex !== -1 && isOptionFlag && NON_SECRET_ARG_FLAGS.has(getOptionName(arg))) {
|
|
4851
|
+
const assignedValue = arg.slice(assignmentIndex + 1);
|
|
4852
|
+
const assignedOptionName = getOptionName(arg);
|
|
4853
|
+
if (assignedOptionName === "--header") {
|
|
4854
|
+
const headerDelimiterIndex = findSensitiveHeaderDelimiterIndex(assignedValue);
|
|
4855
|
+
if (headerDelimiterIndex !== -1) {
|
|
4856
|
+
let consumedValues = 0;
|
|
4857
|
+
while (index + 1 + consumedValues < argv.length && !isOptionValueBoundaryFlag(argv[index + 1 + consumedValues])) {
|
|
4858
|
+
consumedValues += 1;
|
|
4859
|
+
}
|
|
4860
|
+
sanitized.push(
|
|
4861
|
+
`${assignedOptionName}=${sanitizeSecretDelimitedArg(
|
|
4862
|
+
assignedValue,
|
|
4863
|
+
headerDelimiterIndex,
|
|
4864
|
+
assignedValue[headerDelimiterIndex]
|
|
4865
|
+
)}`
|
|
4866
|
+
);
|
|
4867
|
+
sanitized.push(...redactedPlaceholders(consumedValues));
|
|
4868
|
+
index += consumedValues;
|
|
4869
|
+
continue;
|
|
4870
|
+
}
|
|
4871
|
+
}
|
|
4872
|
+
const authSchemeAssignmentIndex = findAuthSchemeAssignmentIndex(assignedValue);
|
|
4873
|
+
if (authSchemeAssignmentIndex !== -1) {
|
|
4874
|
+
const authSchemeKey = assignedValue.slice(0, authSchemeAssignmentIndex);
|
|
4875
|
+
const consumedValues = countAuthSchemeContinuationTokens(authSchemeKey, argv, index + 1);
|
|
4876
|
+
if (consumedValues > 0) {
|
|
4877
|
+
sanitized.push(
|
|
4878
|
+
`${assignedOptionName}=${sanitizeAssignmentArg(assignedValue, authSchemeAssignmentIndex, false)}`
|
|
4879
|
+
);
|
|
4880
|
+
sanitized.push(...redactedPlaceholders(consumedValues));
|
|
4881
|
+
index += consumedValues;
|
|
4882
|
+
continue;
|
|
4883
|
+
}
|
|
4884
|
+
}
|
|
4885
|
+
const assignedValueIsSensitive = isSecretConfigKey(assignedValue) || assignedOptionName === "--header" && isSensitiveHeaderKey(assignedValue);
|
|
4886
|
+
if (assignedValueIsSensitive && index + 1 < argv.length && !isOptionValueBoundaryFlag(argv[index + 1])) {
|
|
4887
|
+
let consumedValues = 1;
|
|
4888
|
+
if (shouldConsumeAuthSchemePair(assignedValue, argv[index + 1]) && index + 2 < argv.length && !isOptionValueBoundaryFlag(argv[index + 2])) {
|
|
4889
|
+
consumedValues = 2;
|
|
4890
|
+
} else {
|
|
4891
|
+
const consumeAllValueTokens = assignedOptionName === "--header" || isAuthorizationConfigKey(assignedValue);
|
|
4892
|
+
while (index + 1 + consumedValues < argv.length && !isOptionValueBoundaryFlag(argv[index + 1 + consumedValues]) && consumeAllValueTokens) {
|
|
4893
|
+
consumedValues += 1;
|
|
4894
|
+
}
|
|
4895
|
+
}
|
|
4896
|
+
sanitized.push(arg);
|
|
4897
|
+
sanitized.push(...redactedPlaceholders(consumedValues));
|
|
4898
|
+
index += consumedValues;
|
|
4899
|
+
continue;
|
|
4900
|
+
}
|
|
4901
|
+
}
|
|
4902
|
+
if (assignmentIndex !== -1) {
|
|
4903
|
+
const authSchemeAssignmentIndex = findAuthSchemeAssignmentIndex(arg);
|
|
4904
|
+
if (authSchemeAssignmentIndex !== -1) {
|
|
4905
|
+
const authSchemeKey = arg.slice(0, authSchemeAssignmentIndex);
|
|
4906
|
+
const consumedValues = countAuthSchemeContinuationTokens(authSchemeKey, argv, index + 1);
|
|
4907
|
+
sanitized.push(sanitizeAssignmentArg(arg, assignmentIndex, isOptionFlag));
|
|
4908
|
+
sanitized.push(...redactedPlaceholders(consumedValues));
|
|
4909
|
+
index += consumedValues;
|
|
4910
|
+
continue;
|
|
4911
|
+
}
|
|
4912
|
+
sanitized.push(sanitizeAssignmentArg(arg, assignmentIndex, isOptionFlag));
|
|
4913
|
+
continue;
|
|
4914
|
+
}
|
|
4915
|
+
if (!isOptionFlag && isSecretConfigKey(arg)) {
|
|
4916
|
+
sanitized.push(arg);
|
|
4917
|
+
const consumedValues = countBareSecretValueTokens(arg, argv, index + 1);
|
|
4918
|
+
sanitized.push(...redactedPlaceholders(consumedValues));
|
|
4919
|
+
index += consumedValues;
|
|
4920
|
+
continue;
|
|
4921
|
+
}
|
|
4922
|
+
sanitized.push(sanitizeStructuredArg(arg));
|
|
4628
4923
|
}
|
|
4629
4924
|
return sanitized;
|
|
4630
4925
|
}
|
|
4926
|
+
function sanitizeOptionValueSpan(optionArg, argv, startIndex) {
|
|
4927
|
+
const value = argv[startIndex];
|
|
4928
|
+
const optionName = getOptionName(optionArg);
|
|
4929
|
+
if (optionName === "--header") {
|
|
4930
|
+
const headerDelimiterIndex = findSensitiveHeaderDelimiterIndex(value);
|
|
4931
|
+
if (headerDelimiterIndex !== -1) {
|
|
4932
|
+
let consumed = 1;
|
|
4933
|
+
while (startIndex + consumed < argv.length && !isOptionValueBoundaryFlag(argv[startIndex + consumed])) {
|
|
4934
|
+
consumed += 1;
|
|
4935
|
+
}
|
|
4936
|
+
return {
|
|
4937
|
+
values: [
|
|
4938
|
+
sanitizeSecretDelimitedArg(value, headerDelimiterIndex, value[headerDelimiterIndex]),
|
|
4939
|
+
...redactedPlaceholders(consumed - 1)
|
|
4940
|
+
],
|
|
4941
|
+
consumed
|
|
4942
|
+
};
|
|
4943
|
+
}
|
|
4944
|
+
if (isSensitiveHeaderKey(value) && startIndex + 1 < argv.length && !isOptionValueBoundaryFlag(argv[startIndex + 1])) {
|
|
4945
|
+
let consumed = 2;
|
|
4946
|
+
while (startIndex + consumed < argv.length && !isOptionValueBoundaryFlag(argv[startIndex + consumed])) {
|
|
4947
|
+
consumed += 1;
|
|
4948
|
+
}
|
|
4949
|
+
return { values: [value, ...redactedPlaceholders(consumed - 1)], consumed };
|
|
4950
|
+
}
|
|
4951
|
+
}
|
|
4952
|
+
const secretDelimiterIndex = findSecretConfigDelimiterIndex(value);
|
|
4953
|
+
if (secretDelimiterIndex !== -1 && value.slice(secretDelimiterIndex + 1).trim().length === 0) {
|
|
4954
|
+
const secretKey = value.slice(0, secretDelimiterIndex);
|
|
4955
|
+
let consumed = 1;
|
|
4956
|
+
if (isAuthorizationConfigKey(secretKey)) {
|
|
4957
|
+
while (startIndex + consumed < argv.length && !isOptionValueBoundaryFlag(argv[startIndex + consumed])) {
|
|
4958
|
+
consumed += 1;
|
|
4959
|
+
}
|
|
4960
|
+
} else if (startIndex + 1 < argv.length && shouldConsumeAuthSchemePair(secretKey, argv[startIndex + 1]) && startIndex + 2 < argv.length && !isOptionValueBoundaryFlag(argv[startIndex + 2])) {
|
|
4961
|
+
consumed = 3;
|
|
4962
|
+
} else if (startIndex + 1 < argv.length && !isOptionValueBoundaryFlag(argv[startIndex + 1])) {
|
|
4963
|
+
consumed = 2;
|
|
4964
|
+
}
|
|
4965
|
+
const continuationValues = consumed > 1 && (isAuthorizationConfigKey(secretKey) || consumed > 2 && shouldConsumeAuthSchemePair(secretKey, argv[startIndex + 1])) ? redactedPlaceholders(consumed - 1) : [];
|
|
4966
|
+
return { values: [sanitizeSecretDelimitedArg(value, secretDelimiterIndex, ":"), ...continuationValues], consumed };
|
|
4967
|
+
}
|
|
4968
|
+
if (isSecretConfigKey(value) && startIndex + 1 < argv.length && !isOptionValueBoundaryFlag(argv[startIndex + 1])) {
|
|
4969
|
+
let consumed = 2;
|
|
4970
|
+
if (shouldConsumeAuthSchemePair(value, argv[startIndex + 1]) && startIndex + 2 < argv.length && !isOptionValueBoundaryFlag(argv[startIndex + 2])) {
|
|
4971
|
+
consumed = 3;
|
|
4972
|
+
} else if (isAuthorizationConfigKey(value)) {
|
|
4973
|
+
while (startIndex + consumed < argv.length && !isOptionValueBoundaryFlag(argv[startIndex + consumed])) {
|
|
4974
|
+
consumed += 1;
|
|
4975
|
+
}
|
|
4976
|
+
}
|
|
4977
|
+
return { values: [value, ...redactedPlaceholders(consumed - 1)], consumed };
|
|
4978
|
+
}
|
|
4979
|
+
const assignmentIndex = value.indexOf("=");
|
|
4980
|
+
if (assignmentIndex !== -1 && findAuthSchemeAssignmentIndex(value) !== -1) {
|
|
4981
|
+
const authSchemeKey = value.slice(0, assignmentIndex);
|
|
4982
|
+
const consumedContinuations = countAuthSchemeContinuationTokens(authSchemeKey, argv, startIndex + 1);
|
|
4983
|
+
if (consumedContinuations > 0) {
|
|
4984
|
+
return {
|
|
4985
|
+
values: [sanitizeAssignmentArg(value, assignmentIndex, false), ...redactedPlaceholders(consumedContinuations)],
|
|
4986
|
+
consumed: 1 + consumedContinuations
|
|
4987
|
+
};
|
|
4988
|
+
}
|
|
4989
|
+
}
|
|
4990
|
+
return { values: [sanitizeOptionValueArg(value)], consumed: 1 };
|
|
4991
|
+
}
|
|
4992
|
+
function sanitizeOptionValueArg(arg) {
|
|
4993
|
+
const sanitizedStructuredValue = sanitizeStructuredArg(arg);
|
|
4994
|
+
if (sanitizedStructuredValue !== arg) return sanitizedStructuredValue;
|
|
4995
|
+
const sanitizedUrlValue = sanitizeUrlSecrets(arg);
|
|
4996
|
+
if (sanitizedUrlValue !== arg) return sanitizedUrlValue;
|
|
4997
|
+
const assignmentIndex = arg.indexOf("=");
|
|
4998
|
+
if (assignmentIndex !== -1) return sanitizeAssignmentArg(arg, assignmentIndex, false);
|
|
4999
|
+
const secretDelimiterIndex = findSecretConfigDelimiterIndex(arg);
|
|
5000
|
+
if (secretDelimiterIndex !== -1) return sanitizeSecretDelimitedArg(arg, secretDelimiterIndex, ":");
|
|
5001
|
+
return arg;
|
|
5002
|
+
}
|
|
5003
|
+
function sanitizeAssignmentArg(arg, assignmentIndex, isOptionFlag) {
|
|
5004
|
+
const key = arg.slice(0, assignmentIndex);
|
|
5005
|
+
const value = arg.slice(assignmentIndex + 1);
|
|
5006
|
+
if (isSensitiveHeaderKey(key)) return `${key}=${REDACTED_ARG_VALUE}`;
|
|
5007
|
+
const sanitizedStructuredValue = sanitizeStructuredArg(value);
|
|
5008
|
+
if (sanitizedStructuredValue !== value) return `${key}=${sanitizedStructuredValue}`;
|
|
5009
|
+
const sanitizedUrlValue = sanitizeUrlSecrets(value);
|
|
5010
|
+
if (sanitizedUrlValue !== value) return `${key}=${sanitizedUrlValue}`;
|
|
5011
|
+
const nestedHeaderDelimiterIndex = findSensitiveHeaderDelimiterIndex(value);
|
|
5012
|
+
if (nestedHeaderDelimiterIndex !== -1) {
|
|
5013
|
+
return `${key}=${sanitizeSecretDelimitedArg(value, nestedHeaderDelimiterIndex, value[nestedHeaderDelimiterIndex])}`;
|
|
5014
|
+
}
|
|
5015
|
+
const nestedSecretDelimiterIndex = findSecretConfigDelimiterIndex(value);
|
|
5016
|
+
if (nestedSecretDelimiterIndex !== -1) {
|
|
5017
|
+
const sanitizedValue2 = sanitizeSecretDelimitedArg(value, nestedSecretDelimiterIndex, ":");
|
|
5018
|
+
return sanitizedValue2 === value ? arg : `${key}=${sanitizedValue2}`;
|
|
5019
|
+
}
|
|
5020
|
+
if (!isOptionFlag) return arg;
|
|
5021
|
+
const nestedAssignmentIndex = value.indexOf("=");
|
|
5022
|
+
if (nestedAssignmentIndex === -1) return arg;
|
|
5023
|
+
const sanitizedValue = sanitizeAssignmentArg(value, nestedAssignmentIndex, false);
|
|
5024
|
+
return sanitizedValue === value ? arg : `${key}=${sanitizedValue}`;
|
|
5025
|
+
}
|
|
5026
|
+
function findSecretConfigDelimiterIndex(arg) {
|
|
5027
|
+
const trimmed = arg.trim();
|
|
5028
|
+
if (trimmed.startsWith("{") || trimmed.startsWith("[")) return -1;
|
|
5029
|
+
const delimiterIndex = arg.indexOf(":");
|
|
5030
|
+
if (delimiterIndex <= 0) return -1;
|
|
5031
|
+
const key = arg.slice(0, delimiterIndex);
|
|
5032
|
+
return isSensitiveHeaderKey(key) || isSensitiveHeaderKey(stripLeadingOptionPrefix(key)) ? delimiterIndex : -1;
|
|
5033
|
+
}
|
|
5034
|
+
function findSensitiveHeaderDelimiterIndex(arg) {
|
|
5035
|
+
const colonIndex = arg.indexOf(":");
|
|
5036
|
+
const equalsIndex = arg.indexOf("=");
|
|
5037
|
+
const delimiterIndex = colonIndex === -1 ? equalsIndex : equalsIndex === -1 ? colonIndex : Math.min(colonIndex, equalsIndex);
|
|
5038
|
+
if (delimiterIndex <= 0) return -1;
|
|
5039
|
+
return isSensitiveHeaderKey(arg.slice(0, delimiterIndex)) ? delimiterIndex : -1;
|
|
5040
|
+
}
|
|
5041
|
+
function sanitizeSecretDelimitedArg(arg, delimiterIndex, delimiter) {
|
|
5042
|
+
const key = arg.slice(0, delimiterIndex);
|
|
5043
|
+
const valueStart = skipLooseAssignmentWhitespace(arg, delimiterIndex + 1);
|
|
5044
|
+
const valueEnd = valueStart >= arg.length ? arg.length : findLooseAssignmentValueEnd(arg, valueStart, isRequestCookieHeaderKey(key));
|
|
5045
|
+
return `${arg.slice(0, delimiterIndex)}${delimiter}${REDACTED_ARG_VALUE}${arg.slice(valueEnd)}`;
|
|
5046
|
+
}
|
|
5047
|
+
function redactedPlaceholders(count) {
|
|
5048
|
+
return Array.from({ length: Math.max(0, count) }, () => REDACTED_ARG_VALUE);
|
|
5049
|
+
}
|
|
5050
|
+
function sanitizeStructuredArg(arg) {
|
|
5051
|
+
const sanitizedUrlArg = sanitizeUrlSecrets(arg);
|
|
5052
|
+
const trimmed = sanitizedUrlArg.trim();
|
|
5053
|
+
if (!(trimmed.startsWith("{") || trimmed.startsWith("["))) return sanitizeLooseSecretConfigText(sanitizedUrlArg);
|
|
5054
|
+
try {
|
|
5055
|
+
const parsed = JSON.parse(trimmed);
|
|
5056
|
+
const redacted = redactStructuredSecrets(parsed);
|
|
5057
|
+
if (!redacted.changed) return sanitizedUrlArg;
|
|
5058
|
+
return JSON.stringify(redacted.value);
|
|
5059
|
+
} catch {
|
|
5060
|
+
return sanitizeLooseSecretConfigText(sanitizedUrlArg);
|
|
5061
|
+
}
|
|
5062
|
+
}
|
|
5063
|
+
function sanitizeLooseSecretConfigText(value) {
|
|
5064
|
+
if (!value.includes(":") && !value.includes("=")) return value;
|
|
5065
|
+
let redacted = "";
|
|
5066
|
+
let cursor = 0;
|
|
5067
|
+
let changed = false;
|
|
5068
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
5069
|
+
const delimiter = value[index];
|
|
5070
|
+
if (delimiter !== ":" && delimiter !== "=") continue;
|
|
5071
|
+
const keySpan = findLooseAssignmentKey(value, index);
|
|
5072
|
+
if (!keySpan || !isSensitiveHeaderKey(keySpan.key)) continue;
|
|
5073
|
+
const valueStart = skipLooseAssignmentWhitespace(value, index + 1);
|
|
5074
|
+
if (valueStart >= value.length) continue;
|
|
5075
|
+
const valueEnd = findLooseAssignmentValueEnd(value, valueStart, isRequestCookieHeaderKey(keySpan.key));
|
|
5076
|
+
redacted += value.slice(cursor, valueStart);
|
|
5077
|
+
redacted += REDACTED_ARG_VALUE;
|
|
5078
|
+
cursor = valueEnd;
|
|
5079
|
+
changed = true;
|
|
5080
|
+
index = valueEnd - 1;
|
|
5081
|
+
}
|
|
5082
|
+
return changed ? redacted + value.slice(cursor) : value;
|
|
5083
|
+
}
|
|
5084
|
+
function findLooseAssignmentKey(value, delimiterIndex) {
|
|
5085
|
+
let keyEnd = delimiterIndex;
|
|
5086
|
+
while (keyEnd > 0 && isWhitespace2(value[keyEnd - 1])) keyEnd -= 1;
|
|
5087
|
+
if (keyEnd <= 0) return void 0;
|
|
5088
|
+
const quote = value[keyEnd - 1];
|
|
5089
|
+
if (quote === '"' || quote === "'") {
|
|
5090
|
+
const keyStart2 = value.lastIndexOf(quote, keyEnd - 2);
|
|
5091
|
+
if (keyStart2 === -1) return void 0;
|
|
5092
|
+
const key2 = value.slice(keyStart2 + 1, keyEnd - 1);
|
|
5093
|
+
return key2.length > 0 ? { key: key2 } : void 0;
|
|
5094
|
+
}
|
|
5095
|
+
let keyStart = keyEnd;
|
|
5096
|
+
while (keyStart > 0 && isLooseAssignmentKeyChar(value[keyStart - 1])) keyStart -= 1;
|
|
5097
|
+
const key = value.slice(keyStart, keyEnd);
|
|
5098
|
+
return key.length > 0 ? { key } : void 0;
|
|
5099
|
+
}
|
|
5100
|
+
function skipLooseAssignmentWhitespace(value, index) {
|
|
5101
|
+
let cursor = index;
|
|
5102
|
+
while (cursor < value.length && isWhitespace2(value[cursor])) cursor += 1;
|
|
5103
|
+
return cursor;
|
|
5104
|
+
}
|
|
5105
|
+
function findLooseAssignmentValueEnd(value, startIndex, consumeSemicolonPairs = false) {
|
|
5106
|
+
const quote = value[startIndex];
|
|
5107
|
+
if (quote === '"' || quote === "'") {
|
|
5108
|
+
let cursor2 = startIndex + 1;
|
|
5109
|
+
let escaped = false;
|
|
5110
|
+
while (cursor2 < value.length) {
|
|
5111
|
+
const char = value[cursor2];
|
|
5112
|
+
if (escaped) {
|
|
5113
|
+
escaped = false;
|
|
5114
|
+
} else if (char === "\\") {
|
|
5115
|
+
escaped = true;
|
|
5116
|
+
} else if (char === quote) {
|
|
5117
|
+
return cursor2 + 1;
|
|
5118
|
+
}
|
|
5119
|
+
cursor2 += 1;
|
|
5120
|
+
}
|
|
5121
|
+
return value.length;
|
|
5122
|
+
}
|
|
5123
|
+
let cursor = startIndex;
|
|
5124
|
+
while (cursor < value.length && !isLooseAssignmentValueTerminator(value[cursor], consumeSemicolonPairs)) cursor += 1;
|
|
5125
|
+
return cursor;
|
|
5126
|
+
}
|
|
5127
|
+
function isLooseAssignmentKeyChar(char) {
|
|
5128
|
+
return isAsciiAlnum2(char) || char === "_" || char === "-" || char === "." || char === "[" || char === "]";
|
|
5129
|
+
}
|
|
5130
|
+
function isLooseAssignmentValueTerminator(char, consumeSemicolonPairs) {
|
|
5131
|
+
return char === "," || char === "}" || char === "&" || !consumeSemicolonPairs && char === ";";
|
|
5132
|
+
}
|
|
5133
|
+
function isWhitespace2(char) {
|
|
5134
|
+
return char === " " || char === " " || char === "\n" || char === "\r";
|
|
5135
|
+
}
|
|
5136
|
+
function sanitizeUrlSecrets(value) {
|
|
5137
|
+
return redactUrlSecrets(value, REDACTED_ARG_VALUE, isSecretConfigKey);
|
|
5138
|
+
}
|
|
5139
|
+
function redactStructuredSecrets(value) {
|
|
5140
|
+
if (Array.isArray(value)) {
|
|
5141
|
+
let changed = false;
|
|
5142
|
+
const entries = value.map((entry) => {
|
|
5143
|
+
const redactedEntry = redactStructuredSecrets(entry);
|
|
5144
|
+
changed ||= redactedEntry.changed;
|
|
5145
|
+
return redactedEntry.value;
|
|
5146
|
+
});
|
|
5147
|
+
return { value: entries, changed };
|
|
5148
|
+
}
|
|
5149
|
+
if (value && typeof value === "object") {
|
|
5150
|
+
let changed = false;
|
|
5151
|
+
const entries = {};
|
|
5152
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
5153
|
+
if (isSensitiveHeaderKey(key)) {
|
|
5154
|
+
entries[key] = REDACTED_ARG_VALUE;
|
|
5155
|
+
changed = true;
|
|
5156
|
+
continue;
|
|
5157
|
+
}
|
|
5158
|
+
const redactedEntry = redactStructuredSecrets(entry);
|
|
5159
|
+
entries[key] = redactedEntry.value;
|
|
5160
|
+
changed ||= redactedEntry.changed;
|
|
5161
|
+
}
|
|
5162
|
+
return { value: entries, changed };
|
|
5163
|
+
}
|
|
5164
|
+
if (typeof value === "string") {
|
|
5165
|
+
let redactedValue = sanitizeUrlSecrets(value);
|
|
5166
|
+
const secretDelimiterIndex = findSecretConfigDelimiterIndex(redactedValue);
|
|
5167
|
+
if (secretDelimiterIndex !== -1) {
|
|
5168
|
+
redactedValue = sanitizeSecretDelimitedArg(redactedValue, secretDelimiterIndex, ":");
|
|
5169
|
+
}
|
|
5170
|
+
redactedValue = sanitizeLooseSecretConfigText(redactedValue);
|
|
5171
|
+
return { value: redactedValue, changed: redactedValue !== value };
|
|
5172
|
+
}
|
|
5173
|
+
return { value, changed: false };
|
|
5174
|
+
}
|
|
5175
|
+
function isSecretConfigKey(key) {
|
|
5176
|
+
return SECRET_KEY_PATTERN.test(normalizeSecretKey(key));
|
|
5177
|
+
}
|
|
5178
|
+
function isSecretOptionFlagName(flagName) {
|
|
5179
|
+
return isSecretConfigKey(flagName) || isSecretConfigKey(stripLeadingOptionPrefix(flagName));
|
|
5180
|
+
}
|
|
5181
|
+
function isAuthorizationConfigKey(key) {
|
|
5182
|
+
return /(^|[-_])authorization$/i.test(normalizeSecretKey(key));
|
|
5183
|
+
}
|
|
5184
|
+
function isSensitiveHeaderKey(key) {
|
|
5185
|
+
const normalized = normalizeSecretKey(key);
|
|
5186
|
+
return isSecretConfigKey(key) || /(^|[-_])(?:cookie|set[-_]?cookie|session)$/i.test(normalized);
|
|
5187
|
+
}
|
|
5188
|
+
function isRequestCookieHeaderKey(key) {
|
|
5189
|
+
const parts = normalizeSecretKey(key).toLowerCase().split(/[-_]+/).filter(Boolean);
|
|
5190
|
+
const lastPart = parts.at(-1);
|
|
5191
|
+
const previousPart = parts.at(-2);
|
|
5192
|
+
return lastPart === "cookie" && previousPart !== "set";
|
|
5193
|
+
}
|
|
5194
|
+
function isAuthSchemeToken(value) {
|
|
5195
|
+
return /^(?:bearer|basic|digest)$/i.test(value);
|
|
5196
|
+
}
|
|
5197
|
+
function isTokenConfigKey(key) {
|
|
5198
|
+
return /(^|[-_])(?:auth[-_]?token|access[-_]?token|refresh[-_]?token|id[-_]?token|token)$/i.test(
|
|
5199
|
+
normalizeSecretKey(key)
|
|
5200
|
+
);
|
|
5201
|
+
}
|
|
5202
|
+
function shouldConsumeAuthSchemePair(key, firstValue) {
|
|
5203
|
+
return (isTokenConfigKey(key) || isAuthorizationConfigKey(key)) && isAuthSchemeToken(firstValue);
|
|
5204
|
+
}
|
|
5205
|
+
function findAuthSchemeAssignmentIndex(arg) {
|
|
5206
|
+
const assignmentIndex = arg.indexOf("=");
|
|
5207
|
+
if (assignmentIndex === -1) return -1;
|
|
5208
|
+
return shouldConsumeAuthSchemePair(arg.slice(0, assignmentIndex), arg.slice(assignmentIndex + 1)) ? assignmentIndex : -1;
|
|
5209
|
+
}
|
|
5210
|
+
function countAuthSchemeContinuationTokens(key, argv, startIndex) {
|
|
5211
|
+
if (startIndex >= argv.length || isAuthSchemeContinuationBoundary(argv[startIndex])) return 0;
|
|
5212
|
+
let consumedValues = 1;
|
|
5213
|
+
while (isAuthorizationConfigKey(key) && startIndex + consumedValues < argv.length && !isAuthSchemeContinuationBoundary(argv[startIndex + consumedValues])) {
|
|
5214
|
+
consumedValues += 1;
|
|
5215
|
+
}
|
|
5216
|
+
return consumedValues;
|
|
5217
|
+
}
|
|
5218
|
+
function isAuthSchemeContinuationBoundary(arg) {
|
|
5219
|
+
return isOptionValueBoundaryFlag(arg) || findAuthSchemeAssignmentIndex(arg) !== -1;
|
|
5220
|
+
}
|
|
5221
|
+
function countBareSecretValueTokens(key, argv, startIndex) {
|
|
5222
|
+
if (startIndex >= argv.length || isOptionValueBoundaryFlag(argv[startIndex])) return 0;
|
|
5223
|
+
const consumeAllValueTokens = isAuthorizationConfigKey(key);
|
|
5224
|
+
let consumedValues = 1;
|
|
5225
|
+
if (!consumeAllValueTokens && shouldConsumeAuthSchemePair(key, argv[startIndex])) {
|
|
5226
|
+
return startIndex + 1 < argv.length && !isOptionValueBoundaryFlag(argv[startIndex + 1]) ? 2 : 1;
|
|
5227
|
+
}
|
|
5228
|
+
while (consumeAllValueTokens && startIndex + consumedValues < argv.length && !isOptionValueBoundaryFlag(argv[startIndex + consumedValues])) {
|
|
5229
|
+
consumedValues += 1;
|
|
5230
|
+
}
|
|
5231
|
+
return consumedValues;
|
|
5232
|
+
}
|
|
5233
|
+
function normalizeSecretKey(key) {
|
|
5234
|
+
let normalized = "";
|
|
5235
|
+
for (let index = 0; index < key.length; index += 1) {
|
|
5236
|
+
const char = key[index];
|
|
5237
|
+
if (char === "." || char === "[" || char === "]" || char === ":") {
|
|
5238
|
+
normalized += "-";
|
|
5239
|
+
continue;
|
|
5240
|
+
}
|
|
5241
|
+
if (isUppercaseAscii(char)) {
|
|
5242
|
+
const previous = key[index - 1];
|
|
5243
|
+
const next = key[index + 1];
|
|
5244
|
+
if (index > 0 && (previous !== void 0 && isLowercaseOrDigitAscii(previous) || previous !== void 0 && isUppercaseAscii(previous) && next !== void 0 && isLowercaseAscii(next))) {
|
|
5245
|
+
normalized += "-";
|
|
5246
|
+
}
|
|
5247
|
+
}
|
|
5248
|
+
normalized += char;
|
|
5249
|
+
}
|
|
5250
|
+
return normalized;
|
|
5251
|
+
}
|
|
5252
|
+
function stripLeadingOptionPrefix(key) {
|
|
5253
|
+
let index = 0;
|
|
5254
|
+
while (index < key.length && key[index] === "-") index += 1;
|
|
5255
|
+
return key.slice(index);
|
|
5256
|
+
}
|
|
5257
|
+
function isUppercaseAscii(char) {
|
|
5258
|
+
return char >= "A" && char <= "Z";
|
|
5259
|
+
}
|
|
5260
|
+
function isLowercaseAscii(char) {
|
|
5261
|
+
return char >= "a" && char <= "z";
|
|
5262
|
+
}
|
|
5263
|
+
function isLowercaseOrDigitAscii(char) {
|
|
5264
|
+
return isLowercaseAscii(char) || char >= "0" && char <= "9";
|
|
5265
|
+
}
|
|
5266
|
+
function isAsciiAlnum2(char) {
|
|
5267
|
+
return isUppercaseAscii(char) || isLowercaseAscii(char) || char >= "0" && char <= "9";
|
|
5268
|
+
}
|
|
4631
5269
|
function sanitizeEnvKeys(env, explicitKeys) {
|
|
4632
5270
|
const sourceKeys = explicitKeys ?? Object.keys(env ?? {});
|
|
4633
5271
|
return [...new Set(sourceKeys)].filter((key) => typeof key === "string" && key.length > 0).sort((left, right) => left.localeCompare(right));
|
|
@@ -4697,12 +5335,19 @@ async function scanDatasetFiles(root) {
|
|
|
4697
5335
|
const relativePath = path4.relative(root, entryPath).split(path4.sep).join("/");
|
|
4698
5336
|
if (entryStat.isSymbolicLink()) {
|
|
4699
5337
|
const target = await readlink(entryPath);
|
|
5338
|
+
const resolvedTarget = path4.resolve(directory, target);
|
|
5339
|
+
const realTarget = await realpath3(resolvedTarget);
|
|
5340
|
+
const targetRelativePath = path4.relative(root, realTarget);
|
|
5341
|
+
if (targetRelativePath.length === 0 || targetRelativePath === ".." || targetRelativePath.startsWith(`..${path4.sep}`) || path4.isAbsolute(targetRelativePath)) {
|
|
5342
|
+
throw new Error(`dataset symlink target must be inside ${root}: ${entryPath}`);
|
|
5343
|
+
}
|
|
5344
|
+
const manifestTarget = path4.isAbsolute(target) ? targetRelativePath.split(path4.sep).join("/") : target;
|
|
4700
5345
|
files.push({
|
|
4701
5346
|
path: relativePath,
|
|
4702
5347
|
kind: "symlink",
|
|
4703
|
-
sizeBytes: Buffer.byteLength(
|
|
4704
|
-
sha256: sha256String(
|
|
4705
|
-
target
|
|
5348
|
+
sizeBytes: Buffer.byteLength(manifestTarget, "utf8"),
|
|
5349
|
+
sha256: sha256String(manifestTarget),
|
|
5350
|
+
target: manifestTarget
|
|
4706
5351
|
});
|
|
4707
5352
|
continue;
|
|
4708
5353
|
}
|
|
@@ -4793,6 +5438,8 @@ async function buildDatasetManifest(benchmark, datasetDir) {
|
|
|
4793
5438
|
};
|
|
4794
5439
|
}
|
|
4795
5440
|
async function buildResultManifest(resultsDir, resultPath, result) {
|
|
5441
|
+
assertPathInsideRoot(resultsDir, resultPath, "result path");
|
|
5442
|
+
await assertRegularFileWithoutSymlinkComponents(resultPath, "result path");
|
|
4796
5443
|
const fileStats = await stat2(resultPath);
|
|
4797
5444
|
return {
|
|
4798
5445
|
path: path4.relative(resultsDir, resultPath).split(path4.sep).join("/"),
|
|
@@ -4810,11 +5457,64 @@ async function buildResultManifest(resultsDir, resultPath, result) {
|
|
|
4810
5457
|
}
|
|
4811
5458
|
async function resolveResultPaths(resultsDir, explicitPaths) {
|
|
4812
5459
|
if (explicitPaths !== void 0) {
|
|
4813
|
-
|
|
5460
|
+
const resolvedPaths = await Promise.all(
|
|
5461
|
+
explicitPaths.map(async (entry) => {
|
|
5462
|
+
const resultPath = path4.resolve(entry);
|
|
5463
|
+
assertPathInsideRoot(resultsDir, resultPath, "result path");
|
|
5464
|
+
await assertRegularFileWithoutSymlinkComponents(resultPath, "result path");
|
|
5465
|
+
return resultPath;
|
|
5466
|
+
})
|
|
5467
|
+
);
|
|
5468
|
+
return [...new Set(resolvedPaths)].sort((left, right) => left.localeCompare(right));
|
|
4814
5469
|
}
|
|
4815
5470
|
const summaries = await listBenchmarkResults(resultsDir);
|
|
4816
5471
|
return summaries.map((summary) => path4.resolve(summary.path));
|
|
4817
5472
|
}
|
|
5473
|
+
function assertPathInsideRoot(root, targetPath, label) {
|
|
5474
|
+
const resolvedRoot = path4.resolve(root);
|
|
5475
|
+
const resolvedTargetPath = path4.resolve(targetPath);
|
|
5476
|
+
const relativePath = path4.relative(resolvedRoot, resolvedTargetPath);
|
|
5477
|
+
if (relativePath.length === 0 || relativePath === ".." || relativePath.startsWith(`..${path4.sep}`) || path4.isAbsolute(relativePath)) {
|
|
5478
|
+
throw new Error(`${label} must be inside ${resolvedRoot}: ${resolvedTargetPath}`);
|
|
5479
|
+
}
|
|
5480
|
+
}
|
|
5481
|
+
async function assertRegularFileWithoutSymlinkComponents(targetPath, label) {
|
|
5482
|
+
const targetStat = await lstatPathWithoutSymlinkComponents(path4.resolve(targetPath));
|
|
5483
|
+
if (!targetStat?.isFile()) {
|
|
5484
|
+
throw new Error(`${label} must be a regular file without symlink components: ${path4.resolve(targetPath)}`);
|
|
5485
|
+
}
|
|
5486
|
+
}
|
|
5487
|
+
function isKnownNonSecretOptionFlag(arg) {
|
|
5488
|
+
if (!arg.startsWith("-")) return false;
|
|
5489
|
+
return NON_SECRET_ARG_FLAGS.has(getOptionName(arg));
|
|
5490
|
+
}
|
|
5491
|
+
function isOptionValueBoundaryFlag(arg) {
|
|
5492
|
+
if (!arg.startsWith("-")) return false;
|
|
5493
|
+
const optionName = getOptionName(arg);
|
|
5494
|
+
return BENCH_OPTION_BOUNDARY_FLAGS.has(optionName) || NON_SECRET_ARG_FLAGS.has(optionName) || SECRET_ARG_FLAGS.has(optionName) || arg.includes("=") && isSecretConfigKey(optionName);
|
|
5495
|
+
}
|
|
5496
|
+
function shouldConsumeSeparatedSecretFlagValue(arg) {
|
|
5497
|
+
return !isOptionValueBoundaryFlag(arg);
|
|
5498
|
+
}
|
|
5499
|
+
function getOptionName(arg) {
|
|
5500
|
+
return arg.includes("=") ? arg.slice(0, arg.indexOf("=")) : arg;
|
|
5501
|
+
}
|
|
5502
|
+
function getAttachedShortSecretFlag(arg) {
|
|
5503
|
+
if (arg.includes("=")) return void 0;
|
|
5504
|
+
for (const flag of ["-k", "-p", "-t"]) {
|
|
5505
|
+
if (arg.startsWith(flag) && arg.length > flag.length) return flag;
|
|
5506
|
+
}
|
|
5507
|
+
return void 0;
|
|
5508
|
+
}
|
|
5509
|
+
function getAttachedLongSecretArg(arg) {
|
|
5510
|
+
for (const flag of ATTACHED_LONG_SECRET_ARG_FLAGS) {
|
|
5511
|
+
if (arg.startsWith(`${flag}=`) && arg.length > flag.length + 1) {
|
|
5512
|
+
return { flag, delimiter: "=" };
|
|
5513
|
+
}
|
|
5514
|
+
if (arg.startsWith(flag) && arg.length > flag.length && arg[flag.length] !== ":") return { flag, delimiter: "" };
|
|
5515
|
+
}
|
|
5516
|
+
return void 0;
|
|
5517
|
+
}
|
|
4818
5518
|
async function buildConfigFileEntries(configFiles = []) {
|
|
4819
5519
|
const entries = [];
|
|
4820
5520
|
for (const configFile of configFiles) {
|
|
@@ -4827,11 +5527,14 @@ async function buildConfigFileEntries(configFiles = []) {
|
|
|
4827
5527
|
entries.push({ label: configFile.label, path: configFile.path, missing: true });
|
|
4828
5528
|
continue;
|
|
4829
5529
|
}
|
|
5530
|
+
const content = await readFile4(configFile.path);
|
|
5531
|
+
const sanitizedConfig = sanitizeConfigFileContent(content);
|
|
4830
5532
|
entries.push({
|
|
4831
5533
|
label: configFile.label,
|
|
4832
5534
|
path: configFile.path,
|
|
4833
5535
|
sizeBytes: fileStats.size,
|
|
4834
|
-
sha256:
|
|
5536
|
+
...sanitizedConfig.sha256 !== void 0 ? { sha256: sanitizedConfig.sha256 } : {},
|
|
5537
|
+
...sanitizedConfig.redacted ? { redacted: true } : {}
|
|
4835
5538
|
});
|
|
4836
5539
|
} catch {
|
|
4837
5540
|
entries.push({ label: configFile.label, path: configFile.path, missing: true });
|
|
@@ -4839,15 +5542,38 @@ async function buildConfigFileEntries(configFiles = []) {
|
|
|
4839
5542
|
}
|
|
4840
5543
|
return entries;
|
|
4841
5544
|
}
|
|
5545
|
+
function sanitizeConfigFileContent(content) {
|
|
5546
|
+
const text = content.toString("utf8");
|
|
5547
|
+
const trimmed = text.trim();
|
|
5548
|
+
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
|
5549
|
+
try {
|
|
5550
|
+
const parsed = JSON.parse(trimmed);
|
|
5551
|
+
const redacted = redactStructuredSecrets(parsed);
|
|
5552
|
+
if (redacted.changed) {
|
|
5553
|
+
return { sha256: sha256String(stableStringify(redacted.value)), redacted: true };
|
|
5554
|
+
}
|
|
5555
|
+
} catch {
|
|
5556
|
+
}
|
|
5557
|
+
}
|
|
5558
|
+
if (containsSecretConfigText(text)) {
|
|
5559
|
+
return { redacted: true };
|
|
5560
|
+
}
|
|
5561
|
+
return { sha256: sha256Buffer(content), redacted: false };
|
|
5562
|
+
}
|
|
5563
|
+
function containsSecretConfigText(text) {
|
|
5564
|
+
if (sanitizeUrlSecrets(text) !== text) return true;
|
|
5565
|
+
const assignmentPattern = /["']?([A-Za-z0-9_.:[\]-]{3,80})["']?\s*[:=]\s*("[^"]+"|'[^']+'|[^\s,;}]+)/g;
|
|
5566
|
+
for (const match of text.matchAll(assignmentPattern)) {
|
|
5567
|
+
const key = match[1];
|
|
5568
|
+
if (key && isSecretConfigKey(key)) return true;
|
|
5569
|
+
}
|
|
5570
|
+
return false;
|
|
5571
|
+
}
|
|
4842
5572
|
function collectQmdCollections(explicitCollections, results) {
|
|
4843
5573
|
const collections = new Set(explicitCollections ?? []);
|
|
4844
5574
|
for (const result of results) {
|
|
4845
5575
|
const config = result.config.remnicConfig ?? {};
|
|
4846
|
-
for (const key of [
|
|
4847
|
-
"qmdCollection",
|
|
4848
|
-
"qmdColdCollection",
|
|
4849
|
-
"conversationIndexQmdCollection"
|
|
4850
|
-
]) {
|
|
5576
|
+
for (const key of ["qmdCollection", "qmdColdCollection", "conversationIndexQmdCollection"]) {
|
|
4851
5577
|
const value = config[key];
|
|
4852
5578
|
if (typeof value === "string" && value.trim().length > 0) {
|
|
4853
5579
|
collections.add(value);
|
|
@@ -4873,9 +5599,7 @@ async function buildBenchmarkReproManifest(resultsDir, options = {}) {
|
|
|
4873
5599
|
const resultPaths = await resolveResultPaths(resolvedResultsDir, options.resultPaths);
|
|
4874
5600
|
const loadedResults = await Promise.all(resultPaths.map((resultPath) => loadBenchmarkResult(resultPath)));
|
|
4875
5601
|
const resultEntries = await Promise.all(
|
|
4876
|
-
resultPaths.map(
|
|
4877
|
-
(resultPath, index) => buildResultManifest(resolvedResultsDir, resultPath, loadedResults[index])
|
|
4878
|
-
)
|
|
5602
|
+
resultPaths.map((resultPath, index) => buildResultManifest(resolvedResultsDir, resultPath, loadedResults[index]))
|
|
4879
5603
|
);
|
|
4880
5604
|
const selectedBenchmarks = options.selectedBenchmarks ?? [...new Set(loadedResults.map((result) => result.meta.benchmark))].sort();
|
|
4881
5605
|
const selectedWorkItems = options.selectedWorkItems ?? loadedResults.map((result) => ({
|
|
@@ -4940,7 +5664,7 @@ async function writeBenchmarkReproManifest(resultsDir, options = {}) {
|
|
|
4940
5664
|
|
|
4941
5665
|
// src/published-artifact.ts
|
|
4942
5666
|
import { createHash as createHash4 } from "crypto";
|
|
4943
|
-
import { mkdir as mkdir4, readFile as
|
|
5667
|
+
import { mkdir as mkdir4, readFile as readFile5, writeFile as writeFile4 } from "fs/promises";
|
|
4944
5668
|
import path5 from "path";
|
|
4945
5669
|
var BENCHMARK_ARTIFACT_SCHEMA_VERSION = 1;
|
|
4946
5670
|
var PUBLISHED_BENCHMARK_ARTIFACT_IDS = Object.freeze([
|
|
@@ -5116,7 +5840,7 @@ function parseBenchmarkArtifact(raw) {
|
|
|
5116
5840
|
return parsed;
|
|
5117
5841
|
}
|
|
5118
5842
|
async function loadBenchmarkArtifact(filePath) {
|
|
5119
|
-
const raw = await
|
|
5843
|
+
const raw = await readFile5(filePath, "utf8");
|
|
5120
5844
|
const artifact = parseBenchmarkArtifact(raw);
|
|
5121
5845
|
return {
|
|
5122
5846
|
artifact,
|
|
@@ -5493,7 +6217,7 @@ function createAnthropicProvider(config) {
|
|
|
5493
6217
|
// src/providers/codex-cli.ts
|
|
5494
6218
|
import { spawn } from "child_process";
|
|
5495
6219
|
import { createHash as createHash5, randomUUID } from "crypto";
|
|
5496
|
-
import { mkdir as mkdir5, mkdtemp as mkdtemp2, readFile as
|
|
6220
|
+
import { mkdir as mkdir5, mkdtemp as mkdtemp2, readFile as readFile6, rm as rm2, writeFile as writeFile5 } from "fs/promises";
|
|
5497
6221
|
import os3 from "os";
|
|
5498
6222
|
import path6 from "path";
|
|
5499
6223
|
var DEFAULT_REASONING_EFFORT = "xhigh";
|
|
@@ -6340,7 +7064,7 @@ function signalExitCode(signal) {
|
|
|
6340
7064
|
}
|
|
6341
7065
|
async function readCodexOutput(outputPath, stdout) {
|
|
6342
7066
|
try {
|
|
6343
|
-
return await
|
|
7067
|
+
return await readFile6(outputPath, "utf8");
|
|
6344
7068
|
} catch {
|
|
6345
7069
|
return stdout;
|
|
6346
7070
|
}
|
|
@@ -6451,7 +7175,7 @@ function createCodexCliProvider(config, deps) {
|
|
|
6451
7175
|
|
|
6452
7176
|
// src/reporter.ts
|
|
6453
7177
|
import { execSync } from "child_process";
|
|
6454
|
-
import { mkdir as mkdir7, readFile as
|
|
7178
|
+
import { mkdir as mkdir7, readFile as readFile7, writeFile as writeFile7 } from "fs/promises";
|
|
6455
7179
|
import path9 from "path";
|
|
6456
7180
|
|
|
6457
7181
|
// src/filename-safety.ts
|
|
@@ -6485,10 +7209,7 @@ async function writeLeaderboardArtifactsForResult(result, outputDir) {
|
|
|
6485
7209
|
const leaderboardDir = resolveContainedPath(outputRoot, "leaderboard");
|
|
6486
7210
|
await mkdir6(leaderboardDir, { recursive: true });
|
|
6487
7211
|
const timestamp = sanitizeFilenameSegment(result.meta.timestamp.replace(/[:.]/g, "-"));
|
|
6488
|
-
const filePath = resolveContainedPath(
|
|
6489
|
-
leaderboardDir,
|
|
6490
|
-
`ama-bench-${timestamp}-answers.jsonl`
|
|
6491
|
-
);
|
|
7212
|
+
const filePath = resolveContainedPath(leaderboardDir, `ama-bench-${timestamp}-answers.jsonl`);
|
|
6492
7213
|
await writeFile6(filePath, serializeJsonl(rows), "utf8");
|
|
6493
7214
|
return [
|
|
6494
7215
|
{
|
|
@@ -6525,7 +7246,8 @@ function buildAmaBenchLeaderboardRows(result) {
|
|
|
6525
7246
|
}));
|
|
6526
7247
|
}
|
|
6527
7248
|
function serializeJsonl(rows) {
|
|
6528
|
-
return rows.map((row) => JSON.stringify(row)).join("\n")
|
|
7249
|
+
return `${rows.map((row) => JSON.stringify(row)).join("\n")}
|
|
7250
|
+
`;
|
|
6529
7251
|
}
|
|
6530
7252
|
function amaBenchEpisodeIdForTask(task) {
|
|
6531
7253
|
const raw = task.details?.episodeId ?? task.details?.episode_id;
|
|
@@ -6622,15 +7344,162 @@ function redactSecrets(value) {
|
|
|
6622
7344
|
if (Array.isArray(value)) {
|
|
6623
7345
|
return value.map((item) => redactSecrets(item));
|
|
6624
7346
|
}
|
|
7347
|
+
if (typeof value === "string") {
|
|
7348
|
+
return redactFreeformStringSecrets(value);
|
|
7349
|
+
}
|
|
6625
7350
|
if (!value || typeof value !== "object") {
|
|
6626
7351
|
return value;
|
|
6627
7352
|
}
|
|
6628
7353
|
const redacted = {};
|
|
6629
7354
|
for (const [key, nestedValue] of Object.entries(value)) {
|
|
6630
|
-
redacted[key] =
|
|
7355
|
+
redacted[key] = isSensitiveResultKey(key) ? REDACTED_SECRET : redactSecrets(nestedValue);
|
|
6631
7356
|
}
|
|
6632
7357
|
return redacted;
|
|
6633
7358
|
}
|
|
7359
|
+
function redactFreeformStringSecrets(value) {
|
|
7360
|
+
const structuredRedaction = redactStructuredStringSecrets(value);
|
|
7361
|
+
if (structuredRedaction !== void 0) return structuredRedaction;
|
|
7362
|
+
return redactSecretAssignments(redactAuthorizationSchemes(redactFreeformUrlSecrets(value)));
|
|
7363
|
+
}
|
|
7364
|
+
function redactStructuredStringSecrets(value) {
|
|
7365
|
+
const trimmed = value.trim();
|
|
7366
|
+
if (!(trimmed.startsWith("{") || trimmed.startsWith("["))) return void 0;
|
|
7367
|
+
try {
|
|
7368
|
+
const parsed = JSON.parse(trimmed);
|
|
7369
|
+
if (!parsed || typeof parsed !== "object") return void 0;
|
|
7370
|
+
const redacted = redactJsonValueSecrets(parsed);
|
|
7371
|
+
if (!redacted.changed) return value;
|
|
7372
|
+
return JSON.stringify(redacted.value);
|
|
7373
|
+
} catch {
|
|
7374
|
+
return void 0;
|
|
7375
|
+
}
|
|
7376
|
+
}
|
|
7377
|
+
function redactJsonValueSecrets(value) {
|
|
7378
|
+
if (Array.isArray(value)) {
|
|
7379
|
+
let changed2 = false;
|
|
7380
|
+
const entries = value.map((item) => {
|
|
7381
|
+
const redactedItem = redactJsonValueSecrets(item);
|
|
7382
|
+
changed2 ||= redactedItem.changed;
|
|
7383
|
+
return redactedItem.value;
|
|
7384
|
+
});
|
|
7385
|
+
return { value: entries, changed: changed2 };
|
|
7386
|
+
}
|
|
7387
|
+
if (typeof value === "string") {
|
|
7388
|
+
const redactedValue = redactSecretAssignments(redactAuthorizationSchemes(redactFreeformUrlSecrets(value)));
|
|
7389
|
+
return { value: redactedValue, changed: redactedValue !== value };
|
|
7390
|
+
}
|
|
7391
|
+
if (!value || typeof value !== "object") {
|
|
7392
|
+
return { value, changed: false };
|
|
7393
|
+
}
|
|
7394
|
+
const redacted = {};
|
|
7395
|
+
let changed = false;
|
|
7396
|
+
for (const [key, nestedValue] of Object.entries(value)) {
|
|
7397
|
+
if (isSensitiveResultKey(key)) {
|
|
7398
|
+
redacted[key] = REDACTED_SECRET;
|
|
7399
|
+
changed = true;
|
|
7400
|
+
continue;
|
|
7401
|
+
}
|
|
7402
|
+
const redactedNestedValue = redactJsonValueSecrets(nestedValue);
|
|
7403
|
+
redacted[key] = redactedNestedValue.value;
|
|
7404
|
+
changed ||= redactedNestedValue.changed;
|
|
7405
|
+
}
|
|
7406
|
+
return { value: redacted, changed };
|
|
7407
|
+
}
|
|
7408
|
+
function redactAuthorizationSchemes(value) {
|
|
7409
|
+
return value.replace(
|
|
7410
|
+
/\b(authorization)\b(\s*[:=]\s*)(bearer|basic|digest)(\s+)("[^"]+"|'[^']+'|[^\s,;}]+)/gi,
|
|
7411
|
+
(_match, key, separator, scheme, space, secret) => `${key}${separator}${scheme}${space}${redactSecretLiteral(secret)}`
|
|
7412
|
+
);
|
|
7413
|
+
}
|
|
7414
|
+
function redactSecretAssignments(value) {
|
|
7415
|
+
let redacted = "";
|
|
7416
|
+
let cursor = 0;
|
|
7417
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
7418
|
+
const parsed = parseSecretAssignment(value, index);
|
|
7419
|
+
if (!parsed) continue;
|
|
7420
|
+
if (!isSensitiveResultKey(parsed.key) || isAuthorizationAssignmentScheme(parsed.rawValue)) continue;
|
|
7421
|
+
const valueEnd = isCookieResultKey(parsed.key) ? findCookieAssignmentValueEnd(value, parsed.valueStart, parsed.valueEnd) : parsed.valueEnd;
|
|
7422
|
+
const rawValue = value.slice(parsed.valueStart, valueEnd);
|
|
7423
|
+
redacted += value.slice(cursor, parsed.valueStart);
|
|
7424
|
+
redacted += redactSecretLiteral(rawValue);
|
|
7425
|
+
cursor = valueEnd;
|
|
7426
|
+
index = valueEnd - 1;
|
|
7427
|
+
}
|
|
7428
|
+
return cursor === 0 ? value : redacted + value.slice(cursor);
|
|
7429
|
+
}
|
|
7430
|
+
function parseSecretAssignment(value, startIndex) {
|
|
7431
|
+
if (startIndex > 0 && isAssignmentKeyChar(value[startIndex - 1])) return void 0;
|
|
7432
|
+
let keyStart = startIndex;
|
|
7433
|
+
let keyEnd;
|
|
7434
|
+
const quote = value[startIndex];
|
|
7435
|
+
if (quote === '"' || quote === "'") {
|
|
7436
|
+
keyStart = startIndex + 1;
|
|
7437
|
+
keyEnd = value.indexOf(quote, keyStart);
|
|
7438
|
+
if (keyEnd === -1) return void 0;
|
|
7439
|
+
} else {
|
|
7440
|
+
if (!isAssignmentKeyChar(value[startIndex])) return void 0;
|
|
7441
|
+
keyEnd = keyStart;
|
|
7442
|
+
while (keyEnd < value.length && isAssignmentKeyChar(value[keyEnd]) && keyEnd - keyStart <= 80) {
|
|
7443
|
+
keyEnd += 1;
|
|
7444
|
+
}
|
|
7445
|
+
}
|
|
7446
|
+
const key = value.slice(keyStart, keyEnd);
|
|
7447
|
+
if (key.length < 3 || key.length > 80) return void 0;
|
|
7448
|
+
let separatorIndex = quote === '"' || quote === "'" ? keyEnd + 1 : keyEnd;
|
|
7449
|
+
while (separatorIndex < value.length && /\s/.test(value[separatorIndex])) separatorIndex += 1;
|
|
7450
|
+
if (value[separatorIndex] !== ":" && value[separatorIndex] !== "=") return void 0;
|
|
7451
|
+
let valueStart = separatorIndex + 1;
|
|
7452
|
+
while (valueStart < value.length && /\s/.test(value[valueStart])) valueStart += 1;
|
|
7453
|
+
if (valueStart >= value.length) return void 0;
|
|
7454
|
+
let valueEnd;
|
|
7455
|
+
const valueQuote = value[valueStart];
|
|
7456
|
+
if (valueQuote === '"' || valueQuote === "'") {
|
|
7457
|
+
const closingQuote = value.indexOf(valueQuote, valueStart + 1);
|
|
7458
|
+
if (closingQuote === -1) return void 0;
|
|
7459
|
+
valueEnd = closingQuote + 1;
|
|
7460
|
+
} else {
|
|
7461
|
+
valueEnd = valueStart;
|
|
7462
|
+
while (valueEnd < value.length && !isAssignmentValueTerminator(value[valueEnd])) valueEnd += 1;
|
|
7463
|
+
}
|
|
7464
|
+
return { key, valueStart, valueEnd, rawValue: value.slice(valueStart, valueEnd) };
|
|
7465
|
+
}
|
|
7466
|
+
function findCookieAssignmentValueEnd(value, valueStart, parsedValueEnd) {
|
|
7467
|
+
const quote = value[valueStart];
|
|
7468
|
+
if (quote === '"' || quote === "'") return parsedValueEnd;
|
|
7469
|
+
let cursor = valueStart;
|
|
7470
|
+
while (cursor < value.length && !isCookieAssignmentValueTerminator(value[cursor])) cursor += 1;
|
|
7471
|
+
return cursor;
|
|
7472
|
+
}
|
|
7473
|
+
function isAssignmentKeyChar(char) {
|
|
7474
|
+
return typeof char === "string" && /[A-Za-z0-9_.[\]-]/.test(char);
|
|
7475
|
+
}
|
|
7476
|
+
function isAssignmentValueTerminator(char) {
|
|
7477
|
+
return /\s/.test(char) || char === "," || char === ";" || char === "}" || char === "&" || char === "#";
|
|
7478
|
+
}
|
|
7479
|
+
function isCookieAssignmentValueTerminator(char) {
|
|
7480
|
+
return char === "\n" || char === "\r" || char === "," || char === "}" || char === "&" || char === "#";
|
|
7481
|
+
}
|
|
7482
|
+
function isAuthorizationAssignmentScheme(value) {
|
|
7483
|
+
return /^(?:bearer|basic|digest)$/i.test(value.replace(/^["']|["']$/g, ""));
|
|
7484
|
+
}
|
|
7485
|
+
function redactSecretLiteral(value) {
|
|
7486
|
+
const quote = value.length >= 2 && (value.startsWith('"') || value.startsWith("'")) ? value[0] : void 0;
|
|
7487
|
+
return quote && value.endsWith(quote) ? `${quote}${REDACTED_SECRET}${quote}` : REDACTED_SECRET;
|
|
7488
|
+
}
|
|
7489
|
+
function redactFreeformUrlSecrets(value) {
|
|
7490
|
+
return redactUrlSecrets(value, REDACTED_SECRET, isSensitiveResultKey);
|
|
7491
|
+
}
|
|
7492
|
+
function isSensitiveResultKey(key) {
|
|
7493
|
+
const normalizedKey = normalizeSensitiveResultKey(key).toLowerCase();
|
|
7494
|
+
if (normalizedKey === "source-session") return false;
|
|
7495
|
+
return isSecretKey(key) || normalizedKey === "cookie" || normalizedKey === "set-cookie" || normalizedKey === "session";
|
|
7496
|
+
}
|
|
7497
|
+
function isCookieResultKey(key) {
|
|
7498
|
+
return normalizeSensitiveResultKey(key).toLowerCase() === "cookie";
|
|
7499
|
+
}
|
|
7500
|
+
function normalizeSensitiveResultKey(key) {
|
|
7501
|
+
return key.replace(/([a-z0-9])([A-Z])/g, "$1-$2").replace(/[\[\]._:]+/g, "-");
|
|
7502
|
+
}
|
|
6634
7503
|
function sanitizeForJson(value) {
|
|
6635
7504
|
if (typeof value === "string") {
|
|
6636
7505
|
return replaceLoneSurrogates(value);
|
|
@@ -6674,48 +7543,39 @@ async function writeBenchmarkResult(result, outputDir) {
|
|
|
6674
7543
|
await mkdir7(outputRoot, { recursive: true });
|
|
6675
7544
|
const safeBenchmark = sanitizeFilenameSegment(result.meta.benchmark);
|
|
6676
7545
|
const safeRemnicVersion = sanitizeFilenameSegment(result.meta.remnicVersion);
|
|
6677
|
-
const timestamp = sanitizeFilenameSegment(
|
|
6678
|
-
|
|
6679
|
-
);
|
|
6680
|
-
const
|
|
6681
|
-
|
|
6682
|
-
|
|
7546
|
+
const timestamp = sanitizeFilenameSegment(result.meta.timestamp.replace(/[:.]/g, "-"));
|
|
7547
|
+
const filePath = resolveContainedPath(outputRoot, `${safeBenchmark}-v${safeRemnicVersion}-${timestamp}.json`);
|
|
7548
|
+
const publicBaseResult = sanitizeBenchmarkResultForJson(redactBenchmarkResultSecrets(result));
|
|
7549
|
+
const leaderboardArtifacts = await writeLeaderboardArtifactsForResult(publicBaseResult, outputRoot).catch(
|
|
7550
|
+
(error) => [
|
|
7551
|
+
{
|
|
7552
|
+
benchmark: publicBaseResult.meta.benchmark,
|
|
7553
|
+
path: "",
|
|
7554
|
+
format: "leaderboard-artifact-error",
|
|
7555
|
+
records: 0,
|
|
7556
|
+
error: error instanceof Error ? error.message : String(error)
|
|
7557
|
+
}
|
|
7558
|
+
]
|
|
6683
7559
|
);
|
|
6684
|
-
const leaderboardArtifacts = await writeLeaderboardArtifactsForResult(
|
|
6685
|
-
result,
|
|
6686
|
-
outputRoot
|
|
6687
|
-
).catch((error) => [
|
|
6688
|
-
{
|
|
6689
|
-
benchmark: result.meta.benchmark,
|
|
6690
|
-
path: "",
|
|
6691
|
-
format: "leaderboard-artifact-error",
|
|
6692
|
-
records: 0,
|
|
6693
|
-
error: error instanceof Error ? error.message : String(error)
|
|
6694
|
-
}
|
|
6695
|
-
]);
|
|
6696
7560
|
const resultWithArtifacts = {
|
|
6697
|
-
...
|
|
7561
|
+
...publicBaseResult,
|
|
6698
7562
|
config: {
|
|
6699
|
-
...
|
|
7563
|
+
...publicBaseResult.config,
|
|
6700
7564
|
benchmarkOptions: {
|
|
6701
|
-
...
|
|
7565
|
+
...publicBaseResult.config.benchmarkOptions ?? {},
|
|
6702
7566
|
leaderboardArtifacts
|
|
6703
7567
|
}
|
|
6704
7568
|
}
|
|
6705
7569
|
};
|
|
6706
|
-
const publicResult = sanitizeBenchmarkResultForJson(
|
|
6707
|
-
|
|
6708
|
-
|
|
6709
|
-
await writeFile7(filePath, JSON.stringify(publicResult, null, 2) + "\n");
|
|
7570
|
+
const publicResult = sanitizeBenchmarkResultForJson(redactBenchmarkResultSecrets(resultWithArtifacts));
|
|
7571
|
+
await writeFile7(filePath, `${JSON.stringify(publicResult, null, 2)}
|
|
7572
|
+
`);
|
|
6710
7573
|
return filePath;
|
|
6711
7574
|
}
|
|
6712
7575
|
async function getRemnicVersion() {
|
|
6713
7576
|
try {
|
|
6714
7577
|
const packageJson = JSON.parse(
|
|
6715
|
-
await
|
|
6716
|
-
path9.resolve(import.meta.dirname, "../../../package.json"),
|
|
6717
|
-
"utf8"
|
|
6718
|
-
)
|
|
7578
|
+
await readFile7(path9.resolve(import.meta.dirname, "../../../package.json"), "utf8")
|
|
6719
7579
|
);
|
|
6720
7580
|
return typeof packageJson.version === "string" ? packageJson.version : "unknown";
|
|
6721
7581
|
} catch {
|
|
@@ -7944,9 +8804,7 @@ function asStringArray(value) {
|
|
|
7944
8804
|
}
|
|
7945
8805
|
|
|
7946
8806
|
// src/responders.ts
|
|
7947
|
-
import {
|
|
7948
|
-
FallbackLlmClient
|
|
7949
|
-
} from "@remnic/core";
|
|
8807
|
+
import { FallbackLlmClient } from "@remnic/core";
|
|
7950
8808
|
var DEFAULT_RESPONDER_SYSTEM_PROMPT = [
|
|
7951
8809
|
"You answer benchmark questions using only the supplied Remnic memory context.",
|
|
7952
8810
|
"If the context does not contain enough information, say that the answer is unknown.",
|
|
@@ -7971,21 +8829,12 @@ var SCORE_CUE_REGEX = /\b(score|rated|rating|grade|graded|result|overall|final)\
|
|
|
7971
8829
|
var CONTEXT_COMPACTION_MARKER = "[...omitted unrelated recalled context...]";
|
|
7972
8830
|
var COMPACTED_CONTEXT_PREFIX = "[Remnic memory context compacted for the responder prompt; full recalled text is preserved in the benchmark artifact.]";
|
|
7973
8831
|
var TRAJECTORY_ANALYSIS_HEADING = "## Trajectory analysis";
|
|
7974
|
-
var TRAJECTORY_LABELS = Object.freeze([
|
|
7975
|
-
"action",
|
|
7976
|
-
"observation",
|
|
7977
|
-
"step",
|
|
7978
|
-
"turn"
|
|
7979
|
-
]);
|
|
8832
|
+
var TRAJECTORY_LABELS = Object.freeze(["action", "observation", "step", "turn"]);
|
|
7980
8833
|
function createResponderFromProvider(provider, options = {}) {
|
|
7981
8834
|
return {
|
|
7982
8835
|
async respond(question, recalledText, control) {
|
|
7983
8836
|
const responderQuestion = options.promptBudgetChars === void 0 ? question : compactResponderQuestion(question, options.promptBudgetChars);
|
|
7984
|
-
const responderContext = options.contextBudgetChars === void 0 ? recalledText : compactResponderContext(
|
|
7985
|
-
recalledText,
|
|
7986
|
-
question,
|
|
7987
|
-
options.contextBudgetChars
|
|
7988
|
-
);
|
|
8837
|
+
const responderContext = options.contextBudgetChars === void 0 ? recalledText : compactResponderContext(recalledText, question, options.contextBudgetChars);
|
|
7989
8838
|
const completion = await provider.complete(
|
|
7990
8839
|
[
|
|
7991
8840
|
`QUESTION: ${responderQuestion}`,
|
|
@@ -8036,10 +8885,7 @@ function compactResponderQuestion(question, maxChars) {
|
|
|
8036
8885
|
}
|
|
8037
8886
|
const separator = "\n\n";
|
|
8038
8887
|
const protocolBudget = maxChars - baseQuestion.length - separator.length;
|
|
8039
|
-
return `${baseQuestion}${separator}${headTailCompact(
|
|
8040
|
-
conciseProtocol,
|
|
8041
|
-
protocolBudget
|
|
8042
|
-
)}`;
|
|
8888
|
+
return `${baseQuestion}${separator}${headTailCompact(conciseProtocol, protocolBudget)}`;
|
|
8043
8889
|
}
|
|
8044
8890
|
function extractBenchmarkQuestionText(question) {
|
|
8045
8891
|
const strictIndex = question.indexOf("\n\nBenchmark answer protocol:");
|
|
@@ -8097,15 +8943,15 @@ function compactResponderContext(recalledText, question, maxChars) {
|
|
|
8097
8943
|
return recalledText;
|
|
8098
8944
|
}
|
|
8099
8945
|
const stepRefs = extractReferencedTrajectoryNumbers(question);
|
|
8100
|
-
const trajectoryAnalysis = extractContextSection(
|
|
8101
|
-
|
|
8102
|
-
|
|
8103
|
-
);
|
|
8104
|
-
const focusedTranscript = stepRefs.size > 0 ?
|
|
8105
|
-
|
|
8106
|
-
|
|
8107
|
-
|
|
8108
|
-
);
|
|
8946
|
+
const trajectoryAnalysis = extractContextSection(normalized, TRAJECTORY_ANALYSIS_HEADING);
|
|
8947
|
+
const preserveTrajectorySpan = shouldPreserveTrajectorySpan(question);
|
|
8948
|
+
const includeTrajectoryBoundary = shouldIncludeTrajectorySpanBoundary(question);
|
|
8949
|
+
const preserveTrajectoryInterval = shouldPreserveTrajectoryInterval(question);
|
|
8950
|
+
const focusedTranscript = stepRefs.size > 0 ? preserveTrajectorySpan ? buildTrajectorySpanContext(normalized, stepRefs, {
|
|
8951
|
+
includeBoundary: includeTrajectoryBoundary,
|
|
8952
|
+
intervalOnly: preserveTrajectoryInterval
|
|
8953
|
+
}) : buildTrajectoryFocusedContext(normalized, stepRefs) : "";
|
|
8954
|
+
const body = joinCompactedContextSections(trajectoryAnalysis, focusedTranscript);
|
|
8109
8955
|
const compactedBody = body.trim().length > 0 ? body.trim() : headTailCompact(normalized, maxChars);
|
|
8110
8956
|
const withPrefix = `${COMPACTED_CONTEXT_PREFIX}
|
|
8111
8957
|
${compactedBody}`;
|
|
@@ -8288,6 +9134,58 @@ function buildTrajectoryFocusedContext(recalledText, stepRefs) {
|
|
|
8288
9134
|
}
|
|
8289
9135
|
return rendered.join("\n");
|
|
8290
9136
|
}
|
|
9137
|
+
function buildTrajectorySpanContext(recalledText, stepRefs, options) {
|
|
9138
|
+
const lines = recalledText.split("\n");
|
|
9139
|
+
const include = /* @__PURE__ */ new Set();
|
|
9140
|
+
const minStep = Math.min(...stepRefs);
|
|
9141
|
+
const maxStep = Math.max(...stepRefs);
|
|
9142
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
9143
|
+
if (isContextHeading(lines[index])) {
|
|
9144
|
+
include.add(index);
|
|
9145
|
+
continue;
|
|
9146
|
+
}
|
|
9147
|
+
const trajectoryNumber = parseTrajectoryLineNumber(lines[index]);
|
|
9148
|
+
if (trajectoryNumber !== void 0 && isTrajectoryNumberInSpan(trajectoryNumber, minStep, maxStep, options)) {
|
|
9149
|
+
include.add(index);
|
|
9150
|
+
}
|
|
9151
|
+
}
|
|
9152
|
+
if (include.size === 0) {
|
|
9153
|
+
return "";
|
|
9154
|
+
}
|
|
9155
|
+
const rendered = [];
|
|
9156
|
+
let lastIncluded = -1;
|
|
9157
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
9158
|
+
if (!include.has(index)) {
|
|
9159
|
+
continue;
|
|
9160
|
+
}
|
|
9161
|
+
if (lastIncluded >= 0 && index > lastIncluded + 1 && rendered.at(-1) !== CONTEXT_COMPACTION_MARKER) {
|
|
9162
|
+
rendered.push(CONTEXT_COMPACTION_MARKER);
|
|
9163
|
+
}
|
|
9164
|
+
rendered.push(lines[index]);
|
|
9165
|
+
lastIncluded = index;
|
|
9166
|
+
}
|
|
9167
|
+
return rendered.join("\n");
|
|
9168
|
+
}
|
|
9169
|
+
function isTrajectoryNumberInSpan(trajectoryNumber, minStep, maxStep, options) {
|
|
9170
|
+
if (options.intervalOnly || minStep !== maxStep) {
|
|
9171
|
+
return trajectoryNumber >= minStep && (options.includeBoundary ? trajectoryNumber <= maxStep : trajectoryNumber < maxStep);
|
|
9172
|
+
}
|
|
9173
|
+
return options.includeBoundary ? trajectoryNumber <= maxStep : trajectoryNumber < maxStep;
|
|
9174
|
+
}
|
|
9175
|
+
function shouldPreserveTrajectorySpan(question) {
|
|
9176
|
+
const baseQuestion = extractBenchmarkQuestionText(question).toLowerCase();
|
|
9177
|
+
return /\b(?:before|until|through|thru|prior to|between|history|histories|range|count|counts|list|lists)\b|up to/.test(
|
|
9178
|
+
baseQuestion
|
|
9179
|
+
);
|
|
9180
|
+
}
|
|
9181
|
+
function shouldIncludeTrajectorySpanBoundary(question) {
|
|
9182
|
+
const baseQuestion = extractBenchmarkQuestionText(question).toLowerCase();
|
|
9183
|
+
return !/\b(?:before|prior to)\b/.test(baseQuestion);
|
|
9184
|
+
}
|
|
9185
|
+
function shouldPreserveTrajectoryInterval(question) {
|
|
9186
|
+
const baseQuestion = extractBenchmarkQuestionText(question).toLowerCase();
|
|
9187
|
+
return /\bbetween\b/.test(baseQuestion);
|
|
9188
|
+
}
|
|
8291
9189
|
function isContextHeading(line) {
|
|
8292
9190
|
const trimmed = line.trim();
|
|
8293
9191
|
return trimmed.startsWith("#") || trimmed === "REMNIC_MEMORY_CONTEXT:" || trimmed.startsWith("Memory context") || trimmed.startsWith("Relevant");
|
|
@@ -8326,9 +9224,9 @@ ${CONTEXT_COMPACTION_MARKER}
|
|
|
8326
9224
|
return `${text.slice(0, headChars).trimEnd()}${marker}${text.slice(text.length - tailChars).trimStart()}`;
|
|
8327
9225
|
}
|
|
8328
9226
|
function isWordBoundary(char) {
|
|
8329
|
-
return char === void 0 || !
|
|
9227
|
+
return char === void 0 || !isAsciiAlnum3(char);
|
|
8330
9228
|
}
|
|
8331
|
-
function
|
|
9229
|
+
function isAsciiAlnum3(char) {
|
|
8332
9230
|
const code = char.charCodeAt(0);
|
|
8333
9231
|
return code >= 48 && code <= 57 || code >= 65 && code <= 90 || code >= 97 && code <= 122;
|
|
8334
9232
|
}
|
|
@@ -8433,9 +9331,7 @@ function createAmaBenchRecommendedJudgeFromProvider(provider) {
|
|
|
8433
9331
|
}
|
|
8434
9332
|
function createProviderBackedAmaBenchRecommendedJudge(config, providerInstance) {
|
|
8435
9333
|
validateProviderConfig(config, "AMA-Bench recommended judge");
|
|
8436
|
-
return createAmaBenchRecommendedJudgeFromProvider(
|
|
8437
|
-
providerInstance ?? createProvider(config)
|
|
8438
|
-
);
|
|
9334
|
+
return createAmaBenchRecommendedJudgeFromProvider(providerInstance ?? createProvider(config));
|
|
8439
9335
|
}
|
|
8440
9336
|
function createStructuredJudgeFromProvider(provider) {
|
|
8441
9337
|
return {
|
|
@@ -8544,9 +9440,7 @@ function parseScalarJudgeScore(raw) {
|
|
|
8544
9440
|
return clampNormalizedScore(numerator);
|
|
8545
9441
|
}
|
|
8546
9442
|
}
|
|
8547
|
-
const fractionMatches = [
|
|
8548
|
-
...trimmed.matchAll(/(-?\d+(?:\.\d+)?)\s*\/\s*(-?\d+(?:\.\d+)?)/g)
|
|
8549
|
-
];
|
|
9443
|
+
const fractionMatches = [...trimmed.matchAll(/(-?\d+(?:\.\d+)?)\s*\/\s*(-?\d+(?:\.\d+)?)/g)];
|
|
8550
9444
|
for (const match of fractionMatches.reverse()) {
|
|
8551
9445
|
const numerator = Number.parseFloat(match[1]);
|
|
8552
9446
|
const denominator = Number.parseFloat(match[2]);
|
|
@@ -8561,9 +9455,7 @@ function parseScalarJudgeScore(raw) {
|
|
|
8561
9455
|
return clampNormalizedScore(percent / 100);
|
|
8562
9456
|
}
|
|
8563
9457
|
}
|
|
8564
|
-
const outOfMatches = [
|
|
8565
|
-
...trimmed.matchAll(/(-?\d+(?:\.\d+)?)\s+out\s+of\s+(-?\d+(?:\.\d+)?)/gi)
|
|
8566
|
-
];
|
|
9458
|
+
const outOfMatches = [...trimmed.matchAll(/(-?\d+(?:\.\d+)?)\s+out\s+of\s+(-?\d+(?:\.\d+)?)/gi)];
|
|
8567
9459
|
for (const match of outOfMatches.reverse()) {
|
|
8568
9460
|
const numerator = Number.parseFloat(match[1]);
|
|
8569
9461
|
const denominator = Number.parseFloat(match[2]);
|
|
@@ -8729,7 +9621,7 @@ function clampNormalizedScore(value) {
|
|
|
8729
9621
|
|
|
8730
9622
|
// src/runtime-profiles.ts
|
|
8731
9623
|
import path10 from "path";
|
|
8732
|
-
import { readFile as
|
|
9624
|
+
import { readFile as readFile8 } from "fs/promises";
|
|
8733
9625
|
import {
|
|
8734
9626
|
resolvePluginEntry,
|
|
8735
9627
|
setCodexCliFallbackRunnerForProcess
|
|
@@ -9313,7 +10205,7 @@ function deriveOpenclawRuntimeContext(configPath) {
|
|
|
9313
10205
|
};
|
|
9314
10206
|
}
|
|
9315
10207
|
async function loadJsonObject(filePath, label) {
|
|
9316
|
-
const raw = await
|
|
10208
|
+
const raw = await readFile8(filePath, "utf8");
|
|
9317
10209
|
let parsed;
|
|
9318
10210
|
try {
|
|
9319
10211
|
parsed = JSON.parse(raw);
|
|
@@ -9639,7 +10531,7 @@ import path31 from "path";
|
|
|
9639
10531
|
|
|
9640
10532
|
// src/benchmarks/published/ama-bench/runner.ts
|
|
9641
10533
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
9642
|
-
import { readFile as
|
|
10534
|
+
import { readFile as readFile9 } from "fs/promises";
|
|
9643
10535
|
import path11 from "path";
|
|
9644
10536
|
|
|
9645
10537
|
// src/benchmarks/published/ama-bench/fixture.ts
|
|
@@ -9851,15 +10743,15 @@ function normalizeTextForContainment(value) {
|
|
|
9851
10743
|
return trimTrailingSentencePunctuation(normalizeText(value).replace(/\s+/g, " "));
|
|
9852
10744
|
}
|
|
9853
10745
|
function isShortLexicalAnswer(value) {
|
|
9854
|
-
return value.length <= 3 && [...value].some((character) =>
|
|
10746
|
+
return value.length <= 3 && [...value].some((character) => isAsciiLetter2(character)) && [...value].every((character) => isAsciiAlphaNumeric(character));
|
|
9855
10747
|
}
|
|
9856
10748
|
function containsShortLexicalAnswer(value, expected) {
|
|
9857
10749
|
return value.split(/[^a-z0-9]+/).some((token) => token === expected);
|
|
9858
10750
|
}
|
|
9859
10751
|
function isAsciiAlphaNumeric(value) {
|
|
9860
|
-
return
|
|
10752
|
+
return isAsciiLetter2(value) || value >= "0" && value <= "9";
|
|
9861
10753
|
}
|
|
9862
|
-
function
|
|
10754
|
+
function isAsciiLetter2(value) {
|
|
9863
10755
|
return value >= "a" && value <= "z";
|
|
9864
10756
|
}
|
|
9865
10757
|
function trimTrailingSentencePunctuation(value) {
|
|
@@ -10217,7 +11109,7 @@ async function loadDataset(mode, datasetDir, limit) {
|
|
|
10217
11109
|
const filePath = path11.join(datasetDir, "open_end_qa_set.jsonl");
|
|
10218
11110
|
let raw;
|
|
10219
11111
|
try {
|
|
10220
|
-
raw = await
|
|
11112
|
+
raw = await readFile9(filePath, "utf8");
|
|
10221
11113
|
} catch (error) {
|
|
10222
11114
|
throw new Error(
|
|
10223
11115
|
`AMA-Bench dataset not found at ${filePath}: ${error instanceof Error ? error.message : String(error)}`
|
|
@@ -10509,7 +11401,7 @@ function isValidQaPairs(value) {
|
|
|
10509
11401
|
|
|
10510
11402
|
// src/benchmarks/published/amemgym/runner.ts
|
|
10511
11403
|
import { randomUUID as randomUUID3 } from "crypto";
|
|
10512
|
-
import { readFile as
|
|
11404
|
+
import { readFile as readFile10 } from "fs/promises";
|
|
10513
11405
|
import path12 from "path";
|
|
10514
11406
|
|
|
10515
11407
|
// src/benchmarks/published/amemgym/fixture.ts
|
|
@@ -11039,7 +11931,7 @@ async function loadDataset2(mode, datasetDir, limit) {
|
|
|
11039
11931
|
const datasetErrors = [];
|
|
11040
11932
|
for (const filename of DATASET_FILENAMES) {
|
|
11041
11933
|
try {
|
|
11042
|
-
const raw = await
|
|
11934
|
+
const raw = await readFile10(path12.join(datasetDir, filename), "utf8");
|
|
11043
11935
|
const parsed = parseDataset(raw, filename, normalizedLimit);
|
|
11044
11936
|
return ensureDatasetProfiles(parsed);
|
|
11045
11937
|
} catch (error) {
|
|
@@ -11213,7 +12105,7 @@ function normalizeRole(role) {
|
|
|
11213
12105
|
|
|
11214
12106
|
// src/benchmarks/published/memory-arena/runner.ts
|
|
11215
12107
|
import { randomUUID as randomUUID4 } from "crypto";
|
|
11216
|
-
import { readFile as
|
|
12108
|
+
import { readFile as readFile11, readdir as readdir5, stat as stat3 } from "fs/promises";
|
|
11217
12109
|
import path13 from "path";
|
|
11218
12110
|
import { expandTildePath as expandTildePath2 } from "@remnic/core";
|
|
11219
12111
|
|
|
@@ -11541,7 +12433,7 @@ async function loadDataset3(mode, datasetDir, limit) {
|
|
|
11541
12433
|
if (remainingLimit2 === 0) {
|
|
11542
12434
|
break;
|
|
11543
12435
|
}
|
|
11544
|
-
const raw = await
|
|
12436
|
+
const raw = await readFile11(path13.join(datasetDir, filename), "utf8");
|
|
11545
12437
|
const parsedTasks = [];
|
|
11546
12438
|
raw.split("\n").forEach((line, lineIndex) => {
|
|
11547
12439
|
if (line.trim().length === 0) {
|
|
@@ -11877,7 +12769,7 @@ async function loadMemoryArenaWebshopProductCatalog(datasetDir) {
|
|
|
11877
12769
|
`MemoryArena WebShop product sidecar is ${sourceStat.size} bytes; provide a compact JSON/JSONL sidecar smaller than ${MEMORY_ARENA_WEBSHOP_PRODUCTS_MAX_BYTES} bytes instead of the full WebShop catalog.`
|
|
11878
12770
|
);
|
|
11879
12771
|
}
|
|
11880
|
-
const raw = await
|
|
12772
|
+
const raw = await readFile11(sourcePath, "utf8");
|
|
11881
12773
|
const records = parseMemoryArenaWebshopSidecarRecords(raw, sourcePath);
|
|
11882
12774
|
const byAsin = /* @__PURE__ */ new Map();
|
|
11883
12775
|
for (const record of records) {
|
|
@@ -13326,7 +14218,7 @@ function scoreSubtaskSuccess(scores) {
|
|
|
13326
14218
|
import { collectTemporalLexicalCues } from "@remnic/core";
|
|
13327
14219
|
|
|
13328
14220
|
// src/benchmarks/published/dataset-loader.ts
|
|
13329
|
-
import { readFile as
|
|
14221
|
+
import { readFile as readFile12 } from "fs/promises";
|
|
13330
14222
|
import path14 from "path";
|
|
13331
14223
|
|
|
13332
14224
|
// src/benchmarks/published/longmemeval/fixture.ts
|
|
@@ -13433,7 +14325,7 @@ async function loadDataset4(options) {
|
|
|
13433
14325
|
const abs = path14.join(options.datasetDir, filename);
|
|
13434
14326
|
let raw;
|
|
13435
14327
|
try {
|
|
13436
|
-
raw = await
|
|
14328
|
+
raw = await readFile12(abs, "utf8");
|
|
13437
14329
|
} catch (error) {
|
|
13438
14330
|
errors.push(
|
|
13439
14331
|
`${filename}: ${error instanceof Error ? error.message : String(error)}`
|
|
@@ -16681,7 +17573,7 @@ var StructuredLiteralParser = class {
|
|
|
16681
17573
|
|
|
16682
17574
|
// src/benchmarks/published/personamem/runner.ts
|
|
16683
17575
|
import { createHash as createHash6, randomUUID as randomUUID7 } from "crypto";
|
|
16684
|
-
import { readFile as
|
|
17576
|
+
import { readFile as readFile13, realpath as realpath4 } from "fs/promises";
|
|
16685
17577
|
import path16 from "path";
|
|
16686
17578
|
|
|
16687
17579
|
// src/benchmarks/published/personamem/fixture.ts
|
|
@@ -16961,7 +17853,7 @@ async function loadDataset8(mode, datasetDir, limit) {
|
|
|
16961
17853
|
const datasetPath = path16.join(datasetDir, relativePath);
|
|
16962
17854
|
let raw;
|
|
16963
17855
|
try {
|
|
16964
|
-
raw = await
|
|
17856
|
+
raw = await readFile13(datasetPath, "utf8");
|
|
16965
17857
|
} catch (error) {
|
|
16966
17858
|
datasetErrors.push(
|
|
16967
17859
|
`${relativePath}: ${error instanceof Error ? error.message : String(error)}`
|
|
@@ -17019,7 +17911,7 @@ async function hydrateSample(row, datasetRoot) {
|
|
|
17019
17911
|
datasetRoot,
|
|
17020
17912
|
row.chat_history_32k_link
|
|
17021
17913
|
);
|
|
17022
|
-
const chatHistoryRaw = await
|
|
17914
|
+
const chatHistoryRaw = await readFile13(chatHistoryPath, "utf8");
|
|
17023
17915
|
const chatHistory = parseChatHistory(
|
|
17024
17916
|
chatHistoryRaw,
|
|
17025
17917
|
row.chat_history_32k_link
|
|
@@ -17485,7 +18377,7 @@ function applyLimit6(items, limit) {
|
|
|
17485
18377
|
|
|
17486
18378
|
// src/benchmarks/published/membench/runner.ts
|
|
17487
18379
|
import { randomUUID as randomUUID8 } from "crypto";
|
|
17488
|
-
import { readFile as
|
|
18380
|
+
import { readFile as readFile14, readdir as readdir7 } from "fs/promises";
|
|
17489
18381
|
import path17 from "path";
|
|
17490
18382
|
|
|
17491
18383
|
// src/benchmarks/published/membench/fixture.ts
|
|
@@ -17747,7 +18639,7 @@ async function loadDataset9(mode, datasetDir, limit) {
|
|
|
17747
18639
|
let remainingLimit = normalizedLimit;
|
|
17748
18640
|
for (const filename of filenames) {
|
|
17749
18641
|
try {
|
|
17750
|
-
const raw = await
|
|
18642
|
+
const raw = await readFile14(path17.join(datasetDir, filename), "utf8");
|
|
17751
18643
|
const parsed = filename.endsWith(".jsonl") ? parseJsonlDataset(raw, filename) : parseJsonDataset(raw, filename);
|
|
17752
18644
|
const limitedCases = remainingLimit === 0 ? [] : applyLimit7(parsed, remainingLimit);
|
|
17753
18645
|
if (limitedCases.length > 0) {
|
|
@@ -18614,7 +19506,7 @@ function isPlainObject2(value) {
|
|
|
18614
19506
|
|
|
18615
19507
|
// src/benchmarks/published/memoryagentbench/runner.ts
|
|
18616
19508
|
import { randomUUID as randomUUID9 } from "crypto";
|
|
18617
|
-
import { access, readFile as
|
|
19509
|
+
import { access, readFile as readFile15 } from "fs/promises";
|
|
18618
19510
|
import path18 from "path";
|
|
18619
19511
|
|
|
18620
19512
|
// src/benchmarks/published/memoryagentbench/fixture.ts
|
|
@@ -19637,7 +20529,7 @@ async function loadRecSysEntityMapping(datasetDir) {
|
|
|
19637
20529
|
}
|
|
19638
20530
|
let parsed;
|
|
19639
20531
|
try {
|
|
19640
|
-
parsed = JSON.parse(await
|
|
20532
|
+
parsed = JSON.parse(await readFile15(candidate, "utf8"));
|
|
19641
20533
|
} catch (error) {
|
|
19642
20534
|
console.error(
|
|
19643
20535
|
` [WARN] MemoryAgentBench ReDial entity mapping ${candidate} is invalid JSON; trying the next candidate: ${error instanceof Error ? error.message : String(error)}`
|
|
@@ -19800,7 +20692,7 @@ async function loadDataset10(mode, datasetDir, limit) {
|
|
|
19800
20692
|
return ensureDatasetItems(applyLimit8(MEMORY_AGENT_BENCH_SMOKE_FIXTURE, normalizedLimit));
|
|
19801
20693
|
}
|
|
19802
20694
|
async function readDatasetFile(filePath, filename) {
|
|
19803
|
-
const raw = await
|
|
20695
|
+
const raw = await readFile15(filePath, "utf8");
|
|
19804
20696
|
const parsed = filename.endsWith(".jsonl") ? parseJsonLines(raw, filename) : parseJsonArray(raw, filename);
|
|
19805
20697
|
return parsed.map(
|
|
19806
20698
|
(item, index) => parseMemoryAgentBenchItem(item, `${filename} item ${index + 1}`)
|
|
@@ -21415,7 +22307,7 @@ function parseNonNegativeInt(rawValue) {
|
|
|
21415
22307
|
import os6 from "os";
|
|
21416
22308
|
import path21 from "path";
|
|
21417
22309
|
import { randomUUID as randomUUID14 } from "crypto";
|
|
21418
|
-
import { mkdtemp as mkdtemp4, mkdir as mkdir8, readFile as
|
|
22310
|
+
import { mkdtemp as mkdtemp4, mkdir as mkdir8, readFile as readFile16, rm as rm4, writeFile as writeFile8 } from "fs/promises";
|
|
21419
22311
|
import {
|
|
21420
22312
|
createVersion,
|
|
21421
22313
|
diffVersions,
|
|
@@ -21582,7 +22474,7 @@ async function executeCase2(sample) {
|
|
|
21582
22474
|
await createVersion(pagePath, "modified content", "write", config, void 0, void 0, tmpDir);
|
|
21583
22475
|
await revertToVersion(pagePath, "1", config, void 0, tmpDir);
|
|
21584
22476
|
const history = await listVersions(pagePath, config, tmpDir);
|
|
21585
|
-
const pageContent = await
|
|
22477
|
+
const pageContent = await readFile16(pagePath, "utf-8");
|
|
21586
22478
|
const observed = await getVersion(pagePath, "3", config, tmpDir);
|
|
21587
22479
|
return {
|
|
21588
22480
|
versionIds: history.versions.map((version) => version.versionId),
|
|
@@ -21599,7 +22491,7 @@ async function executeCase2(sample) {
|
|
|
21599
22491
|
await createVersion(pagePath, content, "write", pruningConfig, void 0, void 0, tmpDir);
|
|
21600
22492
|
}
|
|
21601
22493
|
const history = await listVersions(pagePath, pruningConfig, tmpDir);
|
|
21602
|
-
const pageContent = await
|
|
22494
|
+
const pageContent = await readFile16(pagePath, "utf-8");
|
|
21603
22495
|
const prunedIds = [];
|
|
21604
22496
|
for (const versionId of ["1", "2"]) {
|
|
21605
22497
|
try {
|
|
@@ -21637,7 +22529,7 @@ async function executeCase2(sample) {
|
|
|
21637
22529
|
tmpDir
|
|
21638
22530
|
);
|
|
21639
22531
|
const history = await listVersions(pagePath, config, tmpDir);
|
|
21640
|
-
const pageContent = await
|
|
22532
|
+
const pageContent = await readFile16(pagePath, "utf-8");
|
|
21641
22533
|
const diff = await diffVersions(pagePath, "1", "2", config, tmpDir);
|
|
21642
22534
|
const observedLines = normalizeDiffChangedLines(diff);
|
|
21643
22535
|
return {
|
|
@@ -28679,7 +29571,7 @@ function getBenchmarkLowerIsBetter(benchmarkId) {
|
|
|
28679
29571
|
}
|
|
28680
29572
|
|
|
28681
29573
|
// src/integrity/sealed-qrels.ts
|
|
28682
|
-
import { readFile as
|
|
29574
|
+
import { readFile as readFile17 } from "fs/promises";
|
|
28683
29575
|
function isSealedQrelsArtifact(value) {
|
|
28684
29576
|
if (!value || typeof value !== "object") {
|
|
28685
29577
|
return false;
|
|
@@ -28749,7 +29641,7 @@ function parseSealedQrels(raw, options = {}) {
|
|
|
28749
29641
|
};
|
|
28750
29642
|
}
|
|
28751
29643
|
async function loadSealedQrels(filePath, options = {}) {
|
|
28752
|
-
const raw = await
|
|
29644
|
+
const raw = await readFile17(filePath, "utf8");
|
|
28753
29645
|
return parseSealedQrels(raw, options);
|
|
28754
29646
|
}
|
|
28755
29647
|
function serializeSealedQrels(artifact) {
|
|
@@ -28869,7 +29761,7 @@ function selectFixtureVariant(variants, seed) {
|
|
|
28869
29761
|
}
|
|
28870
29762
|
|
|
28871
29763
|
// src/benchmarks/custom/loader.ts
|
|
28872
|
-
import { readFile as
|
|
29764
|
+
import { readFile as readFile18 } from "fs/promises";
|
|
28873
29765
|
import { parse as parseYaml } from "yaml";
|
|
28874
29766
|
var CUSTOM_SCORING_VALUES = /* @__PURE__ */ new Set([
|
|
28875
29767
|
"exact_match",
|
|
@@ -28889,7 +29781,7 @@ function parseCustomBenchmark(source) {
|
|
|
28889
29781
|
async function loadCustomBenchmarkFile(filePath) {
|
|
28890
29782
|
let source;
|
|
28891
29783
|
try {
|
|
28892
|
-
source = await
|
|
29784
|
+
source = await readFile18(filePath, "utf8");
|
|
28893
29785
|
} catch (error) {
|
|
28894
29786
|
throw new Error(
|
|
28895
29787
|
`Failed to read custom benchmark file ${filePath}: ${formatError(error)}`
|
|
@@ -30036,7 +30928,7 @@ var chatFixture = {
|
|
|
30036
30928
|
};
|
|
30037
30929
|
|
|
30038
30930
|
// src/benchmarks/remnic/procedural-recall/ablation.ts
|
|
30039
|
-
import { mkdir as mkdir15, mkdtemp as mkdtemp11, rm as rm12, writeFile as writeFile14, readFile as
|
|
30931
|
+
import { mkdir as mkdir15, mkdtemp as mkdtemp11, rm as rm12, writeFile as writeFile14, readFile as readFile19 } from "fs/promises";
|
|
30040
30932
|
import os9 from "os";
|
|
30041
30933
|
import path33 from "path";
|
|
30042
30934
|
import {
|
|
@@ -30159,7 +31051,7 @@ async function runProceduralAblation(options) {
|
|
|
30159
31051
|
};
|
|
30160
31052
|
}
|
|
30161
31053
|
async function loadAblationFixture(fixturePath) {
|
|
30162
|
-
const raw = await
|
|
31054
|
+
const raw = await readFile19(fixturePath, "utf8");
|
|
30163
31055
|
let parsed;
|
|
30164
31056
|
try {
|
|
30165
31057
|
parsed = JSON.parse(raw);
|