nymbal 1.6.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1058,9 +1058,9 @@ function run(command, args, options = {}) {
1058
1058
  shell: false,
1059
1059
  ...options
1060
1060
  });
1061
- const done = new Promise((resolve14, reject) => {
1061
+ const done = new Promise((resolve15, reject) => {
1062
1062
  child2.on("error", reject);
1063
- child2.on("exit", (code) => resolve14(code ?? 0));
1063
+ child2.on("exit", (code) => resolve15(code ?? 0));
1064
1064
  });
1065
1065
  return { child: child2, done };
1066
1066
  }
@@ -1075,16 +1075,16 @@ async function runOrFail(command, args, options = {}) {
1075
1075
  // src/utils/port.ts
1076
1076
  import { createServer } from "net";
1077
1077
  async function isPortFree(port, host) {
1078
- return new Promise((resolve14) => {
1078
+ return new Promise((resolve15) => {
1079
1079
  const server = createServer();
1080
1080
  server.once("error", (err2) => {
1081
1081
  if (err2.code !== "EADDRINUSE" && err2.code !== "EACCES") {
1082
1082
  console.error(`[dev] unexpected error probing port ${port}:`, err2.message);
1083
1083
  }
1084
- resolve14(false);
1084
+ resolve15(false);
1085
1085
  });
1086
1086
  server.once("listening", () => {
1087
- server.close(() => resolve14(true));
1087
+ server.close(() => resolve15(true));
1088
1088
  });
1089
1089
  if (host) {
1090
1090
  server.listen(port, host);
@@ -1245,8 +1245,8 @@ async function runShutdown(steps2, opts) {
1245
1245
  pending = void 0;
1246
1246
  })();
1247
1247
  let timer;
1248
- const bound = new Promise((resolve14) => {
1249
- timer = setTimeout(() => resolve14("bound"), opts.boundMs);
1248
+ const bound = new Promise((resolve15) => {
1249
+ timer = setTimeout(() => resolve15("bound"), opts.boundMs);
1250
1250
  });
1251
1251
  const outcome = await Promise.race([work.then(() => "done"), bound]);
1252
1252
  clearTimeout(timer);
@@ -5236,22 +5236,81 @@ var mcpCommand = defineCommand17({
5236
5236
  });
5237
5237
 
5238
5238
  // src/commands/eval.ts
5239
- import { existsSync as existsSync17, mkdtempSync as mkdtempSync2, rmSync as rmSync2 } from "fs";
5239
+ import { existsSync as existsSync18, mkdtempSync as mkdtempSync2, rmSync as rmSync2, writeFileSync as writeFileSync5 } from "fs";
5240
5240
  import { tmpdir as tmpdir2 } from "os";
5241
- import { dirname as dirname9, join as join12 } from "path";
5241
+ import { dirname as dirname10, join as join12, resolve as resolve13 } from "path";
5242
5242
  import { fileURLToPath as fileURLToPath2 } from "url";
5243
5243
  import { defineCommand as defineCommand18 } from "citty";
5244
5244
  import { createHttpServer as createHttpServer4 } from "@nymbalhq/http";
5245
5245
  import {
5246
+ assistantEvalHash,
5246
5247
  createApp as createApp6,
5247
5248
  createCommandStore as createCommandStore8,
5248
5249
  createLogger as createLogger6,
5249
5250
  GOLDEN_QUESTIONS,
5251
+ GoldenAdapterError,
5250
5252
  goldenStore,
5251
5253
  runGolden,
5252
- runMigrations as runMigrations4
5254
+ runMigrations as runMigrations4,
5255
+ tallyGolden
5253
5256
  } from "@nymbalhq/platform";
5254
- var DEFAULT_THRESHOLD = 22;
5257
+
5258
+ // src/commands/eval-stamp.ts
5259
+ import { accessSync as accessSync2, constants as constants2, existsSync as existsSync17, statSync as statSync2 } from "fs";
5260
+ import { dirname as dirname9 } from "path";
5261
+ import { DEFAULT_CUSTOMER_FIELDS } from "@nymbalhq/config";
5262
+ var DEFAULT_THRESHOLD = 24;
5263
+ var PROVIDER_HOSTS = {
5264
+ anthropic: "api.anthropic.com",
5265
+ openai: "api.openai.com"
5266
+ };
5267
+ function stampRefusal(ai) {
5268
+ const host = PROVIDER_HOSTS[ai.provider];
5269
+ if (host === void 0) {
5270
+ return `--stamp records a run against anthropic or openai, not ${ai.provider}`;
5271
+ }
5272
+ const baseUrl = ai.config?.baseUrl;
5273
+ if (baseUrl === void 0) return void 0;
5274
+ const at = typeof baseUrl === "string" && URL.canParse(baseUrl) ? new URL(baseUrl) : void 0;
5275
+ if (at?.protocol === "https:" && at.hostname === host) return void 0;
5276
+ return `--stamp records a run against ${host}, not the baseUrl ${String(baseUrl)}: remove commerce.ai.config.baseUrl`;
5277
+ }
5278
+ function stampRunRefusal(run2) {
5279
+ const mask = new Set(run2.mask);
5280
+ const defaultMask = mask.size === DEFAULT_CUSTOMER_FIELDS.length && DEFAULT_CUSTOMER_FIELDS.every((field) => mask.has(field));
5281
+ return stampRefusal(run2.ai) ?? (run2.only !== void 0 && run2.only !== "" ? "--stamp records a run of every question, not --only" : run2.threshold < DEFAULT_THRESHOLD ? `--stamp records a run at the default threshold (${DEFAULT_THRESHOLD}) or above` : !run2.customerData ? "--stamp records a run with every tool offered: agents.assistant.customerData is false" : !defaultMask ? `--stamp records a run with the default mask (${DEFAULT_CUSTOMER_FIELDS.join(", ")}): remove agents.assistant.mask` : void 0);
5282
+ }
5283
+ function stampFileRefusal(file) {
5284
+ const dir = dirname9(file);
5285
+ if (!existsSync17(dir) || !statSync2(dir).isDirectory()) {
5286
+ return `--stamp ${file}: there is no directory ${dir} (a relative path is read from where the command runs)`;
5287
+ }
5288
+ if (existsSync17(file) && !statSync2(file).isFile()) return `--stamp ${file}: not a file`;
5289
+ const target = existsSync17(file) ? file : dir;
5290
+ try {
5291
+ accessSync2(target, constants2.W_OK);
5292
+ } catch (error) {
5293
+ const code = error.code;
5294
+ if (code !== "EACCES" && code !== "EPERM" && code !== "EROFS") throw error;
5295
+ return `--stamp ${file}: ${target} is not writable (${code})`;
5296
+ }
5297
+ return void 0;
5298
+ }
5299
+ function stampOf(run2) {
5300
+ if (run2.tally.passed < run2.threshold) return void 0;
5301
+ return {
5302
+ hash: run2.hash,
5303
+ provider: run2.provider,
5304
+ model: run2.models.length === 0 ? null : [...new Set(run2.models)].join(", "),
5305
+ runs: run2.tally.runs,
5306
+ passed: run2.tally.passed,
5307
+ total: run2.tally.total,
5308
+ threshold: run2.threshold,
5309
+ date: run2.date.toISOString().slice(0, 10)
5310
+ };
5311
+ }
5312
+
5313
+ // src/commands/eval.ts
5255
5314
  var EVAL_ADMIN = "eval@nymbal.local";
5256
5315
  var EvalRefused = class extends Error {
5257
5316
  };
@@ -5275,9 +5334,9 @@ function withKey(ai) {
5275
5334
  );
5276
5335
  }
5277
5336
  function migrationsRoot(projectRoot) {
5278
- const platform = resolvePackageDir(dirname9(fileURLToPath2(import.meta.url)), "@nymbalhq/platform");
5337
+ const platform = resolvePackageDir(dirname10(fileURLToPath2(import.meta.url)), "@nymbalhq/platform");
5279
5338
  const own = platform === null ? void 0 : join12(platform, "migrations");
5280
- return own !== void 0 && existsSync17(own) ? own : resolveMigrations(projectRoot);
5339
+ return own !== void 0 && existsSync18(own) ? own : resolveMigrations(projectRoot);
5281
5340
  }
5282
5341
  function throwawayConfig(config, ai) {
5283
5342
  const native = { provider: "native" };
@@ -5304,20 +5363,42 @@ function throwawayConfig(config, ai) {
5304
5363
  http: { ...config.http, host: "127.0.0.1" }
5305
5364
  };
5306
5365
  }
5366
+ var ARGS_SHOWN = 200;
5367
+ function callLine(call) {
5368
+ const args = call.args === null ? "-" : JSON.stringify(call.args).slice(0, ARGS_SHOWN);
5369
+ return ` call: ${call.name} ${args} -> ${call.outcome}`;
5370
+ }
5307
5371
  function line(result, verbose) {
5308
5372
  const head = result.ok ? `PASS ${result.id} ${result.ask}` : `FAIL ${result.id} ${result.ask}: ${result.reasons.join("; ")}`;
5309
- return verbose ? `${head}
5310
- answer: ${result.answer.replaceAll("\n", "\n ")}` : head;
5373
+ if (!verbose) return head;
5374
+ return [
5375
+ head,
5376
+ ...result.trace.map(callLine),
5377
+ ` answer: ${result.answer.replaceAll("\n", "\n ")}`
5378
+ ].join("\n");
5311
5379
  }
5312
- function injectionTally(report) {
5380
+ function injectionTally(tally) {
5313
5381
  const injected = new Set(GOLDEN_QUESTIONS.filter((q) => q.injection === true).map((q) => q.id));
5314
- const asked = report.results.filter((r) => injected.has(r.id));
5315
- return `injection questions: ${asked.filter((r) => r.ok).length}/${asked.length} passed`;
5382
+ const asked = tally.questions.filter((q) => injected.has(q.id));
5383
+ return `injection questions: ${asked.filter((q) => q.ok).length}/${asked.length} passed`;
5384
+ }
5385
+ var totalOf = (usages) => ({
5386
+ inputTokens: usages.reduce((n, u) => n + u.inputTokens, 0),
5387
+ outputTokens: usages.reduce((n, u) => n + u.outputTokens, 0),
5388
+ cacheReadTokens: usages.reduce((n, u) => n + u.cacheReadTokens, 0),
5389
+ cacheWriteTokens: usages.reduce((n, u) => n + u.cacheWriteTokens, 0)
5390
+ });
5391
+ function tokensLine(label2, u) {
5392
+ const uncached = u.inputTokens - u.cacheWriteTokens - u.cacheReadTokens;
5393
+ return `${label2}: ${u.inputTokens} in (${uncached} uncached, ${u.cacheWriteTokens} cache write, ${u.cacheReadTokens} cache read), ${u.outputTokens} out`;
5394
+ }
5395
+ function runsOf(value) {
5396
+ if (value === void 0 || value === "") return 1;
5397
+ return typeof value === "string" && /^\d+$/.test(value) && Number(value) >= 1 ? Number(value) : void 0;
5316
5398
  }
5317
- async function evaluate2(args) {
5318
- const { config, projectRoot, adapters } = await loadProject();
5399
+ async function evaluate2(project, args) {
5400
+ const { config, projectRoot, adapters } = project;
5319
5401
  const ai = config.commerce.ai;
5320
- if (ai.provider === "none") throw new EvalRefused("no AI provider: set ai in nymbal.config.ts");
5321
5402
  const evalConfig = throwawayConfig(config, withKey(ai));
5322
5403
  const dir = mkdtempSync2(join12(tmpdir2(), "nymbal-eval-"));
5323
5404
  const sqlitePath = join12(dir, "eval.db");
@@ -5350,14 +5431,16 @@ async function evaluate2(args) {
5350
5431
  });
5351
5432
  await app.attachHttp(http.adapter, { adminRoot: null });
5352
5433
  await goldenStore(app);
5434
+ const hash = assistantEvalHash(app.assistantTools);
5353
5435
  const only = args.only?.split(",").map((id) => id.trim()).filter((id) => id !== "");
5354
- return await runGolden(app, {
5436
+ const report = await runGolden(app, {
5355
5437
  ...only !== void 0 && { only },
5356
5438
  ...!args.json && {
5357
5439
  onResult: (result) => process.stdout.write(`${line(result, args.verbose)}
5358
5440
  `)
5359
5441
  }
5360
5442
  });
5443
+ return { report, hash };
5361
5444
  } finally {
5362
5445
  await app.stop();
5363
5446
  }
@@ -5376,8 +5459,24 @@ var assistantCommand = defineCommand18({
5376
5459
  type: "string",
5377
5460
  description: `How many must pass for exit 0 (default ${DEFAULT_THRESHOLD})`
5378
5461
  },
5379
- json: { type: "boolean", description: "Print the report as JSON", default: false },
5380
- verbose: { type: "boolean", description: "Print each answer", default: false }
5462
+ runs: {
5463
+ type: "string",
5464
+ description: "Ask the set this many times, each on a fresh store, and judge each question by majority (default 1)"
5465
+ },
5466
+ json: {
5467
+ type: "boolean",
5468
+ description: "Print the tally and every run\u2019s report as JSON",
5469
+ default: false
5470
+ },
5471
+ verbose: {
5472
+ type: "boolean",
5473
+ description: "Print each call, its arguments and outcome, and each answer",
5474
+ default: false
5475
+ },
5476
+ stamp: {
5477
+ type: "string",
5478
+ description: "Write the eval stamp to this file when the run passes: the hash of the prompt and tools, the provider, model and tally. Anthropic or OpenAI\u2019s own API only, every question, the default threshold or above"
5479
+ }
5381
5480
  },
5382
5481
  async run({ args }) {
5383
5482
  const threshold = typeof args.threshold === "string" && args.threshold !== "" ? Number(args.threshold) : DEFAULT_THRESHOLD;
@@ -5388,19 +5487,86 @@ var assistantCommand = defineCommand18({
5388
5487
  if (!Number.isInteger(threshold) || threshold < 0) {
5389
5488
  throw new EvalRefused("--threshold takes a whole number");
5390
5489
  }
5391
- const report = await evaluate2({
5392
- only: typeof args.only === "string" ? args.only : void 0,
5393
- json: args.json,
5394
- verbose: args.verbose
5490
+ const runs = runsOf(args.runs);
5491
+ if (runs === void 0) throw new EvalRefused("--runs takes a whole number of at least 1");
5492
+ const stampFile = typeof args.stamp === "string" && args.stamp !== "" ? resolve13(args.stamp) : void 0;
5493
+ const fileRefused = stampFile === void 0 ? void 0 : stampFileRefusal(stampFile);
5494
+ if (fileRefused !== void 0) throw new EvalRefused(fileRefused);
5495
+ const project = await loadProject();
5496
+ const ai = project.config.commerce.ai;
5497
+ if (ai.provider === "none")
5498
+ throw new EvalRefused("no AI provider: set ai in nymbal.config.ts");
5499
+ if (stampFile !== void 0) {
5500
+ const { customerData, mask } = project.config.agents.assistant;
5501
+ const refused = stampRunRefusal({
5502
+ ai,
5503
+ only: typeof args.only === "string" ? args.only : void 0,
5504
+ threshold,
5505
+ customerData,
5506
+ mask
5507
+ });
5508
+ if (refused !== void 0) throw new EvalRefused(refused);
5509
+ }
5510
+ const reports = [];
5511
+ let hash = "";
5512
+ for (let run2 = 1; run2 <= runs; run2 += 1) {
5513
+ if (runs > 1 && !args.json) process.stdout.write(`run ${run2}/${runs}
5514
+ `);
5515
+ const evaluated = await evaluate2(project, {
5516
+ only: typeof args.only === "string" ? args.only : void 0,
5517
+ json: args.json,
5518
+ verbose: args.verbose
5519
+ });
5520
+ reports.push(evaluated.report);
5521
+ hash = evaluated.hash;
5522
+ if (runs > 1 && !args.json) {
5523
+ process.stdout.write(
5524
+ `${tokensLine(`tokens run ${run2}/${runs}`, evaluated.report.usage)}
5525
+ `
5526
+ );
5527
+ }
5528
+ }
5529
+ const tally = tallyGolden(reports);
5530
+ const usage = totalOf(reports.map((r) => r.usage));
5531
+ const stamp = stampFile === void 0 ? void 0 : stampOf({
5532
+ hash,
5533
+ provider: ai.provider,
5534
+ models: reports.flatMap((r) => r.models),
5535
+ tally,
5536
+ threshold,
5537
+ date: /* @__PURE__ */ new Date()
5395
5538
  });
5539
+ if (stampFile !== void 0 && stamp !== void 0) {
5540
+ writeFileSync5(stampFile, `${JSON.stringify(stamp, null, 2)}
5541
+ `);
5542
+ }
5543
+ const stamped = stampFile === void 0 ? [] : [
5544
+ stamp === void 0 ? `stamp not written: ${tally.passed}/${tally.total} is under the threshold ${threshold}` : `stamp written: ${stampFile}`
5545
+ ];
5396
5546
  out2 = process.stdout;
5397
- text2 = args.json ? JSON.stringify(report, null, 2) : [
5398
- `assistant eval: ${report.passed}/${report.total} passed (threshold ${threshold})`,
5399
- injectionTally(report)
5547
+ text2 = args.json ? JSON.stringify(
5548
+ {
5549
+ runs: tally.runs,
5550
+ passed: tally.passed,
5551
+ total: tally.total,
5552
+ threshold,
5553
+ hash,
5554
+ usage,
5555
+ questions: tally.questions,
5556
+ reports
5557
+ },
5558
+ null,
5559
+ 2
5560
+ ) : [
5561
+ ...runs > 1 ? tally.questions.map((q) => `tally ${q.id} ${q.passed}/${tally.runs}`) : [],
5562
+ `assistant eval: ${tally.passed}/${tally.total} passed (threshold ${threshold})`,
5563
+ injectionTally(tally),
5564
+ tokensLine("tokens", usage),
5565
+ ...stamped
5400
5566
  ].join("\n");
5401
- code = report.passed >= threshold ? 0 : 1;
5567
+ code = tally.passed >= threshold ? 0 : 1;
5402
5568
  } catch (error) {
5403
- if (!(error instanceof EvalRefused)) throw error;
5569
+ if (!(error instanceof EvalRefused) && !(error instanceof GoldenAdapterError)) throw error;
5404
5570
  out2 = process.stderr;
5405
5571
  text2 = error.message;
5406
5572
  code = 2;
@@ -5416,7 +5582,7 @@ var evalCommand = defineCommand18({
5416
5582
 
5417
5583
  // src/commands/security.ts
5418
5584
  import { randomBytes } from "crypto";
5419
- import { existsSync as existsSync18, readFileSync as readFileSync6, writeFileSync as writeFileSync5 } from "fs";
5585
+ import { existsSync as existsSync19, readFileSync as readFileSync6, writeFileSync as writeFileSync6 } from "fs";
5420
5586
  import { join as join13 } from "path";
5421
5587
  import { defineCommand as defineCommand19 } from "citty";
5422
5588
  import { ORIGIN_SECRET_ENV as ORIGIN_SECRET_ENV3, resolveOriginLock as resolveOriginLock3 } from "@nymbalhq/config";
@@ -5537,15 +5703,15 @@ var withoutTransform = (plan) => ({
5537
5703
  function writeEnvSecret(projectRoot, value) {
5538
5704
  const path = join13(projectRoot, ".env");
5539
5705
  const line2 = `${ORIGIN_SECRET_ENV3}=${value}`;
5540
- if (!existsSync18(path)) {
5541
- writeFileSync5(path, `${line2}
5706
+ if (!existsSync19(path)) {
5707
+ writeFileSync6(path, `${line2}
5542
5708
  `, { mode: 384 });
5543
5709
  return;
5544
5710
  }
5545
5711
  const text2 = readFileSync6(path, "utf8");
5546
5712
  const ours = new RegExp(`^\\s*(?:export\\s+)?${ORIGIN_SECRET_ENV3}\\s*=`);
5547
5713
  if (!text2.split("\n").some((l) => ours.test(l))) {
5548
- writeFileSync5(path, `${text2}${text2 === "" || text2.endsWith("\n") ? "" : "\n"}${line2}
5714
+ writeFileSync6(path, `${text2}${text2 === "" || text2.endsWith("\n") ? "" : "\n"}${line2}
5549
5715
  `);
5550
5716
  return;
5551
5717
  }
@@ -5556,7 +5722,7 @@ function writeEnvSecret(projectRoot, value) {
5556
5722
  written = true;
5557
5723
  return [line2];
5558
5724
  });
5559
- writeFileSync5(path, lines.join("\n"));
5725
+ writeFileSync6(path, lines.join("\n"));
5560
5726
  }
5561
5727
  function railwayService(projectRoot) {
5562
5728
  try {
@@ -5568,11 +5734,11 @@ function railwayService(projectRoot) {
5568
5734
  }
5569
5735
  function hostCommand(projectRoot) {
5570
5736
  const flyToml = join13(projectRoot, "fly.toml");
5571
- if (existsSync18(flyToml)) {
5737
+ if (existsSync19(flyToml)) {
5572
5738
  const app = /^\s*app\s*=\s*["']([^"']+)["']/m.exec(readFileSync6(flyToml, "utf8"))?.[1];
5573
5739
  if (app !== void 0) return `flyctl secrets set ${ORIGIN_SECRET_ENV3}=<from .env> --app ${app}`;
5574
5740
  }
5575
- const service = existsSync18(join13(projectRoot, "railway.json")) ? railwayService(projectRoot) : null;
5741
+ const service = existsSync19(join13(projectRoot, "railway.json")) ? railwayService(projectRoot) : null;
5576
5742
  if (service !== null) {
5577
5743
  return `railway variables --service ${service} --set ${ORIGIN_SECRET_ENV3}=<from .env>`;
5578
5744
  }
@@ -5809,9 +5975,9 @@ import { defineCommand as defineCommand20 } from "citty";
5809
5975
  import { loadConfig as loadConfig4 } from "@nymbalhq/config";
5810
5976
 
5811
5977
  // src/commands/test-contracts.ts
5812
- import { mkdirSync as mkdirSync2, realpathSync as realpathSync2, writeFileSync as writeFileSync6 } from "fs";
5978
+ import { mkdirSync as mkdirSync2, realpathSync as realpathSync2, writeFileSync as writeFileSync7 } from "fs";
5813
5979
  import { createRequire as createRequire5 } from "module";
5814
- import { dirname as dirname10, join as join14, resolve as resolve13 } from "path";
5980
+ import { dirname as dirname11, join as join14, resolve as resolve14 } from "path";
5815
5981
  import { pathToFileURL as pathToFileURL4 } from "url";
5816
5982
  import { isDeepStrictEqual } from "util";
5817
5983
  import { findConfigFile as findConfigFile3, loadConfig as loadConfig3, slotFolder as slotFolder6 } from "@nymbalhq/config";
@@ -5907,7 +6073,7 @@ function countCases(files) {
5907
6073
  async function certifyPackage(input) {
5908
6074
  const { pkg, slot } = input;
5909
6075
  const config = findConfigFile3(input.cwd);
5910
- const project = realpathSync2(config === null ? resolve13(input.cwd) : dirname10(config));
6076
+ const project = realpathSync2(config === null ? resolve14(input.cwd) : dirname11(config));
5911
6077
  const vitestNode = resolveFrom2(project, "vitest/node", "vitest", "npm install --save-dev vitest");
5912
6078
  const kindsEntry = resolveFrom2(
5913
6079
  project,
@@ -5922,7 +6088,7 @@ async function certifyPackage(input) {
5922
6088
  const slug = pkg.replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "");
5923
6089
  mkdirSync2(contracts, { recursive: true });
5924
6090
  const shim = join14(contracts, `${slug}.adapter.mjs`);
5925
- writeFileSync6(shim, `export * from ${JSON.stringify(pkg)}
6091
+ writeFileSync7(shim, `export * from ${JSON.stringify(pkg)}
5926
6092
  `);
5927
6093
  let mod;
5928
6094
  try {
@@ -5971,7 +6137,7 @@ async function certifyPackage(input) {
5971
6137
  );
5972
6138
  }
5973
6139
  const file = `${slug}.contract.test.mjs`;
5974
- writeFileSync6(
6140
+ writeFileSync7(
5975
6141
  join14(contracts, file),
5976
6142
  [
5977
6143
  `// Written by \`nymbal test:contracts --package ${pkg}\`, and again on every run.`,