axiom 0.34.0 → 0.34.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.cjs +26 -165
- package/dist/bin.cjs.map +1 -1
- package/dist/bin.js +5 -5
- package/dist/{chunk-ABD2PPUT.js → chunk-4G7GDQ6Q.js} +9 -150
- package/dist/chunk-4G7GDQ6Q.js.map +1 -0
- package/dist/{chunk-ZCDGDSZR.js → chunk-AAEGYMAU.js} +8 -8
- package/dist/chunk-AAEGYMAU.js.map +1 -0
- package/dist/{chunk-C5XDEFQ5.js → chunk-KX7Z2MF4.js} +18 -15
- package/dist/chunk-KX7Z2MF4.js.map +1 -0
- package/dist/config.cjs.map +1 -1
- package/dist/config.js +1 -1
- package/dist/evals.cjs +24 -66
- package/dist/evals.cjs.map +1 -1
- package/dist/evals.js +100 -6
- package/dist/evals.js.map +1 -1
- package/dist/index.cjs +7 -7
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1 -1
- package/package.json +1 -1
- package/dist/chunk-ABD2PPUT.js.map +0 -1
- package/dist/chunk-C5XDEFQ5.js.map +0 -1
- package/dist/chunk-ZCDGDSZR.js.map +0 -1
package/dist/bin.cjs
CHANGED
|
@@ -606,7 +606,7 @@ var import_api4 = require("@opentelemetry/api");
|
|
|
606
606
|
// package.json
|
|
607
607
|
var package_default = {
|
|
608
608
|
name: "axiom",
|
|
609
|
-
version: "0.34.
|
|
609
|
+
version: "0.34.1",
|
|
610
610
|
type: "module",
|
|
611
611
|
author: "Axiom, Inc.",
|
|
612
612
|
contributors: [
|
|
@@ -845,104 +845,6 @@ function withEvalContext(options = {}, fn) {
|
|
|
845
845
|
|
|
846
846
|
// src/evals/reporter.console-utils.ts
|
|
847
847
|
init_cjs_shims();
|
|
848
|
-
|
|
849
|
-
// src/util/deep-equal.ts
|
|
850
|
-
init_cjs_shims();
|
|
851
|
-
function deepEqual(data, other) {
|
|
852
|
-
if (data === other) {
|
|
853
|
-
return true;
|
|
854
|
-
}
|
|
855
|
-
if (Object.is(data, other)) {
|
|
856
|
-
return true;
|
|
857
|
-
}
|
|
858
|
-
if (typeof data !== "object" || typeof other !== "object") {
|
|
859
|
-
return false;
|
|
860
|
-
}
|
|
861
|
-
if (data === null || other === null) {
|
|
862
|
-
return false;
|
|
863
|
-
}
|
|
864
|
-
if (Object.getPrototypeOf(data) !== Object.getPrototypeOf(other)) {
|
|
865
|
-
return false;
|
|
866
|
-
}
|
|
867
|
-
if (Array.isArray(data)) {
|
|
868
|
-
return isDeepEqualArrays(data, other);
|
|
869
|
-
}
|
|
870
|
-
if (data instanceof Map) {
|
|
871
|
-
return isDeepEqualMaps(data, other);
|
|
872
|
-
}
|
|
873
|
-
if (data instanceof Set) {
|
|
874
|
-
return isDeepEqualSets(data, other);
|
|
875
|
-
}
|
|
876
|
-
if (data instanceof Date) {
|
|
877
|
-
return data.getTime() === other.getTime();
|
|
878
|
-
}
|
|
879
|
-
if (data instanceof RegExp) {
|
|
880
|
-
return data.toString() === other.toString();
|
|
881
|
-
}
|
|
882
|
-
if (Object.keys(data).length !== Object.keys(other).length) {
|
|
883
|
-
return false;
|
|
884
|
-
}
|
|
885
|
-
for (const [key, value] of Object.entries(data)) {
|
|
886
|
-
if (!(key in other)) {
|
|
887
|
-
return false;
|
|
888
|
-
}
|
|
889
|
-
if (!deepEqual(
|
|
890
|
-
value,
|
|
891
|
-
// @ts-expect-error [ts7053] - We already checked that `other` has `key`
|
|
892
|
-
other[key]
|
|
893
|
-
)) {
|
|
894
|
-
return false;
|
|
895
|
-
}
|
|
896
|
-
}
|
|
897
|
-
return true;
|
|
898
|
-
}
|
|
899
|
-
function isDeepEqualArrays(data, other) {
|
|
900
|
-
if (data.length !== other.length) {
|
|
901
|
-
return false;
|
|
902
|
-
}
|
|
903
|
-
for (const [index, item] of data.entries()) {
|
|
904
|
-
if (!deepEqual(item, other[index])) {
|
|
905
|
-
return false;
|
|
906
|
-
}
|
|
907
|
-
}
|
|
908
|
-
return true;
|
|
909
|
-
}
|
|
910
|
-
function isDeepEqualMaps(data, other) {
|
|
911
|
-
if (data.size !== other.size) {
|
|
912
|
-
return false;
|
|
913
|
-
}
|
|
914
|
-
for (const [key, value] of data.entries()) {
|
|
915
|
-
if (!other.has(key)) {
|
|
916
|
-
return false;
|
|
917
|
-
}
|
|
918
|
-
if (!deepEqual(value, other.get(key))) {
|
|
919
|
-
return false;
|
|
920
|
-
}
|
|
921
|
-
}
|
|
922
|
-
return true;
|
|
923
|
-
}
|
|
924
|
-
function isDeepEqualSets(data, other) {
|
|
925
|
-
if (data.size !== other.size) {
|
|
926
|
-
return false;
|
|
927
|
-
}
|
|
928
|
-
const otherCopy = [...other];
|
|
929
|
-
for (const dataItem of data) {
|
|
930
|
-
let isFound = false;
|
|
931
|
-
for (const [index, otherItem] of otherCopy.entries()) {
|
|
932
|
-
if (deepEqual(dataItem, otherItem)) {
|
|
933
|
-
isFound = true;
|
|
934
|
-
otherCopy.splice(index, 1);
|
|
935
|
-
break;
|
|
936
|
-
}
|
|
937
|
-
}
|
|
938
|
-
if (!isFound) {
|
|
939
|
-
return false;
|
|
940
|
-
}
|
|
941
|
-
}
|
|
942
|
-
return true;
|
|
943
|
-
}
|
|
944
|
-
|
|
945
|
-
// src/evals/reporter.console-utils.ts
|
|
946
848
|
function formatPercentage(value) {
|
|
947
849
|
if (!Number.isFinite(value)) {
|
|
948
850
|
return "N/A";
|
|
@@ -1129,47 +1031,6 @@ function printOrphanedBaselineCases(baseline, matchedIndices, logger = console.l
|
|
|
1129
1031
|
}
|
|
1130
1032
|
}
|
|
1131
1033
|
}
|
|
1132
|
-
function printConfigHeader(logger = console.log) {
|
|
1133
|
-
logger("");
|
|
1134
|
-
logger(" ", u.bgWhite(u.blackBright(" Config ")));
|
|
1135
|
-
}
|
|
1136
|
-
function printConfigEnd(configEnd, logger = console.log) {
|
|
1137
|
-
printConfigHeader(logger);
|
|
1138
|
-
maybePrintFlags(configEnd, logger);
|
|
1139
|
-
}
|
|
1140
|
-
function maybePrintFlags(configEnd, logger = console.log) {
|
|
1141
|
-
const defaults = configEnd?.flags ?? {};
|
|
1142
|
-
const overrides2 = configEnd?.overrides ?? {};
|
|
1143
|
-
const defaultKeys = Object.keys(defaults);
|
|
1144
|
-
const overrideKeys = Object.keys(overrides2);
|
|
1145
|
-
const allKeys = Array.from(/* @__PURE__ */ new Set([...defaultKeys, ...overrideKeys])).sort();
|
|
1146
|
-
if (allKeys.length === 0) {
|
|
1147
|
-
return;
|
|
1148
|
-
}
|
|
1149
|
-
for (const key of allKeys) {
|
|
1150
|
-
const hasDefault = key in defaults;
|
|
1151
|
-
const hasOverride = key in overrides2;
|
|
1152
|
-
if (hasDefault && hasOverride) {
|
|
1153
|
-
const defVal = defaults[key];
|
|
1154
|
-
const ovVal = overrides2[key];
|
|
1155
|
-
const changed = !deepEqual(ovVal, defVal);
|
|
1156
|
-
const ovText = truncate(stringify(ovVal), 80);
|
|
1157
|
-
const defText = truncate(stringify(defVal), 80);
|
|
1158
|
-
if (changed) {
|
|
1159
|
-
logger(" ", `${key}: ${ovText} ${u.dim(`(overridden by CLI, original: ${defText})`)}`);
|
|
1160
|
-
} else {
|
|
1161
|
-
logger(" ", `${key}: ${defText}`);
|
|
1162
|
-
}
|
|
1163
|
-
} else if (hasOverride) {
|
|
1164
|
-
const ovText = truncate(stringify(overrides2[key]), 80);
|
|
1165
|
-
logger(" ", `${key}: ${ovText} ${u.dim("(added by CLI)")}`);
|
|
1166
|
-
} else if (hasDefault) {
|
|
1167
|
-
const defText = truncate(stringify(defaults[key]), 80);
|
|
1168
|
-
logger(" ", `${key}: ${defText}`);
|
|
1169
|
-
}
|
|
1170
|
-
}
|
|
1171
|
-
logger("");
|
|
1172
|
-
}
|
|
1173
1034
|
var reporterDate = (d) => {
|
|
1174
1035
|
const date = d.toISOString().slice(0, 10);
|
|
1175
1036
|
const hours = d.getUTCHours().toString().padStart(2, "0");
|
|
@@ -1522,16 +1383,13 @@ var AxiomReporter = class {
|
|
|
1522
1383
|
registered: suite.registrationStatus?.status === "success",
|
|
1523
1384
|
error: suite.registrationStatus?.status === "failed" ? suite.registrationStatus.error : void 0
|
|
1524
1385
|
}));
|
|
1525
|
-
const
|
|
1386
|
+
const isDebug = process.env.AXIOM_DEBUG === "true";
|
|
1526
1387
|
printFinalReport({
|
|
1527
1388
|
suiteData: this._suiteData,
|
|
1528
1389
|
config: this._config,
|
|
1529
1390
|
registrationStatus,
|
|
1530
|
-
isDebug
|
|
1391
|
+
isDebug
|
|
1531
1392
|
});
|
|
1532
|
-
if (DEBUG && this._endOfRunConfigEnd) {
|
|
1533
|
-
printConfigEnd(this._endOfRunConfigEnd);
|
|
1534
|
-
}
|
|
1535
1393
|
}
|
|
1536
1394
|
};
|
|
1537
1395
|
|
|
@@ -1953,20 +1811,23 @@ function createPartialDefaults() {
|
|
|
1953
1811
|
}
|
|
1954
1812
|
function validateConfig(config) {
|
|
1955
1813
|
const errors = [];
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1814
|
+
const isDebug = process.env.AXIOM_DEBUG === "true";
|
|
1815
|
+
if (!isDebug) {
|
|
1816
|
+
if (!config.eval?.token) {
|
|
1817
|
+
errors.push(
|
|
1818
|
+
"eval.token is required (set in axiom.config.ts or AXIOM_TOKEN environment variable)"
|
|
1819
|
+
);
|
|
1820
|
+
}
|
|
1821
|
+
if (!config.eval?.dataset) {
|
|
1822
|
+
errors.push(
|
|
1823
|
+
"eval.dataset is required (set in axiom.config.ts or AXIOM_DATASET environment variable)"
|
|
1824
|
+
);
|
|
1825
|
+
}
|
|
1826
|
+
if (!config.eval?.url) {
|
|
1827
|
+
console.log(
|
|
1828
|
+
"eval.url was not specified. Defaulting to `https://api.axiom.co`. Please set it in axiom.config.ts or AXIOM_URL environment variable if you want to use a different endpoint."
|
|
1829
|
+
);
|
|
1830
|
+
}
|
|
1970
1831
|
}
|
|
1971
1832
|
const instrumentation = config.eval?.instrumentation;
|
|
1972
1833
|
if (instrumentation !== null && instrumentation !== void 0 && typeof instrumentation !== "function") {
|
|
@@ -2069,11 +1930,11 @@ function setupEvalProvider(connection) {
|
|
|
2069
1930
|
axiomProvider = new import_sdk_trace_node.NodeTracerProvider({
|
|
2070
1931
|
resource: (0, import_resources.resourceFromAttributes)({
|
|
2071
1932
|
["service.name"]: "axiom",
|
|
2072
|
-
["service.version"]: "0.34.
|
|
1933
|
+
["service.version"]: "0.34.1"
|
|
2073
1934
|
}),
|
|
2074
1935
|
spanProcessors: [processor]
|
|
2075
1936
|
});
|
|
2076
|
-
axiomTracer = axiomProvider.getTracer("axiom", "0.34.
|
|
1937
|
+
axiomTracer = axiomProvider.getTracer("axiom", "0.34.1");
|
|
2077
1938
|
}
|
|
2078
1939
|
async function initInstrumentation(config) {
|
|
2079
1940
|
if (initialized) {
|
|
@@ -2085,7 +1946,7 @@ async function initInstrumentation(config) {
|
|
|
2085
1946
|
}
|
|
2086
1947
|
initializationPromise = (async () => {
|
|
2087
1948
|
if (!config.enabled) {
|
|
2088
|
-
axiomTracer = import_api10.trace.getTracer("axiom", "0.34.
|
|
1949
|
+
axiomTracer = import_api10.trace.getTracer("axiom", "0.34.1");
|
|
2089
1950
|
initialized = true;
|
|
2090
1951
|
return;
|
|
2091
1952
|
}
|
|
@@ -2490,7 +2351,7 @@ var loadEvalCommand = (program2, flagOverrides = {}) => {
|
|
|
2490
2351
|
".",
|
|
2491
2352
|
"any *.eval.ts file in current directory"
|
|
2492
2353
|
)
|
|
2493
|
-
).option("-w, --watch true", "keep server running and watch for changes", false).option("-t, --token <TOKEN>", "axiom token", getDefaultToken).option("-d, --dataset <DATASET>", "axiom dataset name", process.env.AXIOM_DATASET).option("-u, --url <AXIOM URL>", "axiom url", getDefaultUrl).option("-o, --org-id <ORG ID>", "axiom organization id", getDefaultOrgId).option("-b, --baseline <BASELINE ID>", "id of baseline evaluation to compare against").option("--debug", "run locally without
|
|
2354
|
+
).option("-w, --watch true", "keep server running and watch for changes", false).option("-t, --token <TOKEN>", "axiom token", getDefaultToken).option("-d, --dataset <DATASET>", "axiom dataset name", process.env.AXIOM_DATASET).option("-u, --url <AXIOM URL>", "axiom url", getDefaultUrl).option("-o, --org-id <ORG ID>", "axiom organization id", getDefaultOrgId).option("-b, --baseline <BASELINE ID>", "id of baseline evaluation to compare against").option("--debug", "run locally without any network operations", false).option("--list", "list evaluations and test cases without running them", false).addOption(new import_commander.Option("-c, --console-url <URL>", "console url override").hideHelp()).action(async (target, options) => {
|
|
2494
2355
|
try {
|
|
2495
2356
|
if (options.debug) {
|
|
2496
2357
|
process.env.AXIOM_DEBUG = "true";
|
|
@@ -2935,7 +2796,7 @@ var import_commander2 = require("commander");
|
|
|
2935
2796
|
var loadVersionCommand = (program2) => {
|
|
2936
2797
|
return program2.addCommand(
|
|
2937
2798
|
new import_commander2.Command("version").description("cli version").action(() => {
|
|
2938
|
-
console.log("0.34.
|
|
2799
|
+
console.log("0.34.1");
|
|
2939
2800
|
})
|
|
2940
2801
|
);
|
|
2941
2802
|
};
|
|
@@ -2945,7 +2806,7 @@ var { loadEnvConfig } = import_env.default;
|
|
|
2945
2806
|
loadEnvConfig(process.cwd());
|
|
2946
2807
|
var { cleanedArgv, overrides } = extractOverrides(process.argv.slice(2));
|
|
2947
2808
|
var program = new import_commander3.Command();
|
|
2948
|
-
program.name("axiom").description("Axiom's CLI to manage your objects and run evals").version("0.34.
|
|
2809
|
+
program.name("axiom").description("Axiom's CLI to manage your objects and run evals").version("0.34.1");
|
|
2949
2810
|
program.hook("preAction", async (_, actionCommand) => {
|
|
2950
2811
|
const commandName = actionCommand.name();
|
|
2951
2812
|
const parentCommand = actionCommand.parent;
|