@telvine/cli 0.1.0 → 0.1.2

package/README.md

@@ -6,7 +6,7 @@ Instrument agent plugins with Telvine.
 npm i -g @telvine/cli@0.1.0
 telvine login
 telvine publish ./my-plugin --dry-run
-telvine publish ./my-plugin
+telvine publish ./my-plugin --marketplace-url https://github.com/acme/agent-plugin-marketplace.git
 ```
 
 Package: [`@telvine/cli@0.1.0` on npm](https://www.npmjs.com/package/@telvine/cli).
@@ -20,7 +20,8 @@ and assets as versioned components.
 `.claude-plugin/plugin.json`, discovers Skills and `evals/**/cases.jsonl`, and
 prints the plugin publish plan without API calls or file writes. Non-dry-run
 publish registers the plugin, component inventory, Skill versions, eval suites,
-and prints a plugin-scoped write key once.
+and prints a plugin-scoped write key once. Use `--marketplace-url` to store the
+Git marketplace or source URL that users should install from for that version.
 
 After publishing, set the printed `TELVINE_PLUGIN_ID` and `TELVINE_WRITE_KEY`
 in the plugin runtime, run the plugin once, then inspect production data:
@@ -32,3 +33,11 @@ telvine plugins events <plugin-id> --limit 20
 
 Telemetry uses a closed event envelope. Do not send prompts, file contents,
 connector payloads, tool arguments, or model outputs.
+
+## Promptfoo evaluation adapter
+
+`telvine eval run ./my-plugin --suite SUITE --plugin-id PLUGIN_ID --version VERSION --trials 3 --engine-version PROMPTFOO_VERSION` delegates to an installed Promptfoo executable. Keep native `promptfooconfig.yaml`/`.json`, synthetic `cases.jsonl`, and optional release metadata in `suite.json` under the suite directory. Each test needs `metadata.telvine_case_key`. Full result exports stay local; only allowlisted numeric/boolean result metadata, identities, and hashes reach Telvine.
+
+`telvine eval import RESULTS.json --suite-id SUITE_ID --source-hash HASH --version VERSION --trials 3 --engine-version PROMPTFOO_VERSION` records an existing v2/v3 export. CI can supply `TELVINE_READ_TOKEN`, `TELVINE_WRITE_TOKEN`, and `TELVINE_API_BASE_URL`. Prepublish the suite before CI runs. Both commands exit nonzero when the gate fails.
+
+See the [evaluation guide](https://telvine.com/learn/how-to-evaluate-agent-plugins-with-telvine) for immutable revisions, trial preservation, calibration, and release checks. Telvine registration does not execute tests or distribute packages.

package/dist/telvine.js

@@ -2639,47 +2639,6 @@ var require_wrap_ansi = __commonJS((exports, module) => {
   };
 });
 
-// ../../node_modules/.pnpm/keytar@7.9.0/node_modules/keytar/build/Release/keytar.node
-var require_keytar = __commonJS((exports, module) => {
-  module.exports = __require("./keytar-f6bnxfss.node");
-});
-
-// ../../node_modules/.pnpm/keytar@7.9.0/node_modules/keytar/lib/keytar.js
-var require_keytar2 = __commonJS((exports, module) => {
-  var keytar = require_keytar();
-  function checkRequired(val, name) {
-    if (!val || val.length <= 0) {
-      throw new Error(name + " is required.");
-    }
-  }
-  module.exports = {
-    getPassword: function(service, account) {
-      checkRequired(service, "Service");
-      checkRequired(account, "Account");
-      return keytar.getPassword(service, account);
-    },
-    setPassword: function(service, account, password) {
-      checkRequired(service, "Service");
-      checkRequired(account, "Account");
-      checkRequired(password, "Password");
-      return keytar.setPassword(service, account, password);
-    },
-    deletePassword: function(service, account) {
-      checkRequired(service, "Service");
-      checkRequired(account, "Account");
-      return keytar.deletePassword(service, account);
-    },
-    findPassword: function(service) {
-      checkRequired(service, "Service");
-      return keytar.findPassword(service);
-    },
-    findCredentials: function(service) {
-      checkRequired(service, "Service");
-      return keytar.findCredentials(service);
-    }
-  };
-});
-
 // ../../node_modules/.pnpm/kind-of@6.0.3/node_modules/kind-of/index.js
 var require_kind_of = __commonJS((exports, module) => {
   var toString = Object.prototype.toString;
@@ -9416,7 +9375,7 @@ async function tryKeytar() {
   if (keytar)
     return keytar;
   try {
-    const mod = await Promise.resolve().then(() => __toESM(require_keytar2(), 1));
+    const mod = await import("keytar");
     keytar = mod.default ?? mod;
     if (typeof keytar.setPassword !== "function")
       throw new Error("keytar module shape unexpected");
@@ -9498,15 +9457,25 @@ class ApiError extends Error {
   status;
   body;
   constructor(status, body) {
-    super(`api_error_${status}`);
+    super(formatApiError(status, body));
     this.status = status;
     this.body = body;
   }
 }
+function formatApiError(status, body) {
+  if (body && typeof body === "object" && "message" in body && typeof body.message === "string") {
+    return body.message;
+  }
+  if (body && typeof body === "object" && "error" in body && typeof body.error === "string") {
+    return body.error;
+  }
+  return `api_error_${status}`;
+}
 async function api(path2, init = {}) {
   const cfg = await readConfig();
-  const token = await getSecret("clerk_jwt") ?? "";
-  const res = await fetch(`${cfg.api_base_url}${path2}`, {
+  const isRead = !init.method || init.method === "GET";
+  const token = (isRead ? process.env.TELVINE_READ_TOKEN : process.env.TELVINE_WRITE_TOKEN) ?? process.env.TELVINE_TOKEN ?? await getSecret("clerk_jwt") ?? "";
+  const res = await fetch(`${process.env.TELVINE_API_BASE_URL ?? cfg.api_base_url}${path2}`, {
     ...init,
     headers: {
       "content-type": "application/json",
@@ -13911,6 +13880,7 @@ var propsByEventType = {
   "feedback.submitted": exports_external.object({
     rating: exports_external.number().int().min(1).max(5).optional(),
     comment: exports_external.string().max(2000).optional(),
+    source: exports_external.enum(["user", "reviewer", "agent"]).optional(),
     task_category: exports_external.enum(TASK_CATEGORIES).optional(),
     session_id: exports_external.string().min(1).max(128).optional()
   }).strict(),
@@ -13955,7 +13925,9 @@ var propsByEventType = {
     score: exports_external.number().min(0).max(1).optional(),
     failure_cluster: exports_external.string().max(160).optional(),
     error_class: exports_external.string().max(128).optional(),
-    duration_ms: exports_external.number().int().nonnegative().max(86400000).optional()
+    duration_ms: exports_external.number().int().nonnegative().max(86400000).optional(),
+    agent_experience_score: exports_external.number().int().min(1).max(5).optional(),
+    agent_experience_feedback: exports_external.string().max(2000).optional()
   }).strict(),
   "skill.eval.run.completed": exports_external.object({
     eval_suite_id: exports_external.string().min(1).max(128),
@@ -13992,7 +13964,9 @@ var propsByEventType = {
     score: exports_external.number().min(0).max(1).optional(),
     failure_cluster: exports_external.string().max(160).optional(),
     error_class: exports_external.string().max(128).optional(),
-    duration_ms: exports_external.number().int().nonnegative().max(86400000).optional()
+    duration_ms: exports_external.number().int().nonnegative().max(86400000).optional(),
+    agent_experience_score: exports_external.number().int().min(1).max(5).optional(),
+    agent_experience_feedback: exports_external.string().max(2000).optional()
   }).strict(),
   "plugin.eval.run.completed": exports_external.object({
     eval_suite_id: exports_external.string().min(1).max(128),
@@ -14193,9 +14167,135 @@ async function writeSkillMd(path2, content) {
 }
 
 // src/lib/plugin-file.ts
-import { createHash as createHash2, randomUUID } from "node:crypto";
-import { readFile as readFile4, readdir as readdir2, stat as stat2 } from "node:fs/promises";
+import { createHash as createHash2 } from "node:crypto";
+import { readFile as readFile4, readdir as readdir2, stat as stat2, realpath } from "node:fs/promises";
 import { basename, join as join4, relative as relative2 } from "node:path";
+
+// ../../packages/schema/src/evals.ts
+var evalSuiteStatusSchema = exports_external.enum(["draft", "active", "deprecated"]);
+var evalRunStatusSchema = exports_external.enum(["queued", "running", "completed", "failed"]);
+var evalCaseResultStatusSchema = exports_external.enum(["passed", "failed", "skipped", "errored"]);
+var evalHashSchema = exports_external.string().regex(/^[a-f0-9]{64}$/);
+var evalMetadataKeySchema = exports_external.string().regex(/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/);
+var evalGraderSchema = exports_external.object({
+  id: evalMetadataKeySchema,
+  type: exports_external.enum(["deterministic", "outcome", "model", "pairwise", "human", "similarity"]),
+  target: exports_external.enum(["activation", "trajectory", "output", "state"]).default("output"),
+  version: exports_external.string().min(1).max(64),
+  config_hash: evalHashSchema,
+  model: evalMetadataKeySchema.optional(),
+  required: exports_external.boolean().default(true),
+  threshold: exports_external.number().min(0).max(1).default(1),
+  calibration_ref: evalMetadataKeySchema.optional()
+}).strict();
+var evalGatePolicySchema = exports_external.object({
+  required_tags: exports_external.array(evalMetadataKeySchema).max(50).default([]),
+  min_trials: exports_external.number().int().min(1).max(100).default(1),
+  require_calibration: exports_external.boolean().default(false),
+  calibration_min_samples: exports_external.number().int().min(1).default(20),
+  calibration_min_agreement: exports_external.number().min(0).max(1).default(0.9),
+  calibration_max_false_pass_rate: exports_external.number().min(0).max(1).default(0.05),
+  require_baseline: exports_external.boolean().default(false),
+  max_regressions: exports_external.number().int().nonnegative().default(0)
+}).strict();
+var evalAssertionResultSchema = exports_external.object({
+  grader_id: evalMetadataKeySchema,
+  status: exports_external.enum(["passed", "failed", "errored", "needs_review"]),
+  score: exports_external.number().min(0).max(1).nullable().optional(),
+  config_hash: evalHashSchema,
+  evidence_ref: evalMetadataKeySchema.optional()
+}).strict();
+var evalRubricSchema = exports_external.object({
+  dimensions: exports_external.array(exports_external.object({
+    name: exports_external.string().min(1).max(80),
+    description: exports_external.string().max(1000).optional(),
+    weight: exports_external.number().positive().max(100).optional()
+  }).strict()).max(20).optional()
+}).catchall(exports_external.unknown());
+var evalFixtureRefsSchema = exports_external.array(exports_external.object({
+  name: exports_external.string().min(1).max(128),
+  path: exports_external.string().min(1).max(512).refine((path2) => !path2.startsWith("/") && !path2.split("/").includes(".."), "expected plugin-relative path"),
+  synthetic: exports_external.boolean().optional(),
+  content_hash: evalHashSchema.optional()
+}).strict()).max(100);
+var evalCaseInputSchema = exports_external.object({
+  case_key: exports_external.string().min(1).max(128),
+  name: exports_external.string().min(1).max(200),
+  task_category: exports_external.string().min(1).max(80).optional(),
+  scenario: exports_external.string().min(1).max(4000),
+  expected_outcome: exports_external.string().min(1).max(4000),
+  rubric: evalRubricSchema.default({}),
+  fixture_refs: evalFixtureRefsSchema.default([]),
+  tags: exports_external.array(exports_external.string().min(1).max(64)).max(50).default([]),
+  weight: exports_external.number().int().min(1).max(100).default(1),
+  required: exports_external.boolean().default(true)
+}).strict();
+var evalSuiteInputSchema = exports_external.object({
+  slug: exports_external.string().regex(/^[a-z0-9][a-z0-9-]{1,62}[a-z0-9]$/),
+  name: exports_external.string().min(1).max(200),
+  description: exports_external.string().max(2000).optional(),
+  version: exports_external.string().min(1).max(64),
+  skill_id: exports_external.string().optional(),
+  source_path: exports_external.string().min(1).max(512).refine((path2) => !path2.startsWith("/") && !path2.split("/").includes(".."), "expected plugin-relative path").optional(),
+  source_hash: evalHashSchema.optional(),
+  graders: exports_external.array(evalGraderSchema).max(100).default([]),
+  gate_policy: evalGatePolicySchema.default({}),
+  purpose: exports_external.enum(["regression", "capability", "calibration", "holdout"]).default("regression"),
+  status: evalSuiteStatusSchema.default("draft"),
+  pass_threshold: exports_external.number().min(0).max(1).default(0.8),
+  promotion_gate: exports_external.string().max(2000).optional(),
+  agent_experience_feedback_enabled: exports_external.boolean().default(true),
+  cases: exports_external.array(evalCaseInputSchema).max(1000).default([])
+}).strict();
+var evalRunInputSchema = exports_external.object({
+  plugin_version: exports_external.string().min(1).max(64).optional(),
+  skill_version: exports_external.string().min(1).max(64).optional(),
+  baseline_version: exports_external.string().min(1).max(64).optional(),
+  runtime: exports_external.string().max(64).optional(),
+  model: exports_external.string().max(128).optional(),
+  harness: exports_external.string().max(128).optional(),
+  agent_experience_feedback_enabled: exports_external.boolean().optional(),
+  status: exports_external.enum(["queued", "running"]).default("running"),
+  trials: exports_external.number().int().min(1).max(100).default(1),
+  suite_revision_hash: evalHashSchema.optional(),
+  grader_config_hash: evalHashSchema.optional(),
+  engine_version: exports_external.string().max(64).optional(),
+  evaluator_variant: evalMetadataKeySchema.optional(),
+  baseline_run_id: exports_external.string().min(1).optional(),
+  parent_run_id: exports_external.string().min(1).optional(),
+  experiment_id: evalMetadataKeySchema.optional()
+}).strict();
+var evalCaseResultInputSchema = exports_external.object({
+  eval_case_id: exports_external.string().min(1),
+  status: evalCaseResultStatusSchema,
+  score: exports_external.number().min(0).max(1).nullable().optional(),
+  rubric_results: exports_external.record(evalMetadataKeySchema, exports_external.union([exports_external.boolean(), exports_external.number().finite(), exports_external.null()])).default({}),
+  trial: exports_external.number().int().min(1).max(100).default(1),
+  assertions: exports_external.array(evalAssertionResultSchema).max(100).default([]),
+  cost_usd: exports_external.number().finite().nonnegative().optional(),
+  failure_cluster: exports_external.string().max(160).optional(),
+  error_class: exports_external.string().max(128).optional(),
+  duration_ms: exports_external.number().int().nonnegative().max(86400000).optional(),
+  evidence_ref: exports_external.string().max(512).optional(),
+  output_hash: exports_external.string().length(64).optional(),
+  agent_experience_score: exports_external.number().int().min(1).max(5).nullable().optional(),
+  agent_experience_feedback: exports_external.string().max(2000).optional()
+}).strict();
+var evalCalibrationInputSchema = exports_external.object({
+  grader_id: evalMetadataKeySchema,
+  config_hash: evalHashSchema,
+  rubric_hash: evalHashSchema,
+  dataset_hash: evalHashSchema,
+  split: exports_external.literal("holdout"),
+  samples: exports_external.array(exports_external.object({
+    sample_id: evalMetadataKeySchema,
+    human: exports_external.enum(["passed", "failed"]),
+    judge: exports_external.enum(["passed", "failed", "errored", "needs_review"]),
+    category: evalMetadataKeySchema.default("uncategorized")
+  }).strict()).min(1).max(1e4)
+}).strict();
+
+// src/lib/plugin-file.ts
 var sha2562 = (value) => createHash2("sha256").update(value).digest("hex");
 var slugify = (value) => value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 64).replace(/-+$/g, "");
 async function exists(path2) {
@@ -14346,38 +14446,86 @@ async function readEvalSuites(evalsDir, root, version) {
     const cases = casesRaw.split(`
 `).map((line) => line.trim()).filter(Boolean).map((line) => normalizeEvalCase(JSON.parse(line), rubric));
     const sourcePath = relative2(root, suiteDir);
-    suites.push({
-      slug: slugify(entry.name),
-      name: titleize2(entry.name),
-      version,
+    const settings = await readJson(join4(suiteDir, "suite.json")).catch((e) => {
+      if (e.code === "ENOENT")
+        return { data: {}, raw: "" };
+      throw e;
+    });
+    for (const c of cases)
+      for (const fixture of c.fixture_refs) {
+        const path2 = await realpath(join4(root, fixture.path));
+        const rel = relative2(await realpath(root), path2);
+        if (rel.startsWith("..") || rel.startsWith("/"))
+          throw new Error("Fixture must stay inside the plugin directory");
+        const contentHash = createHash2("sha256").update(await readFile4(path2)).digest("hex");
+        if (fixture.content_hash && fixture.content_hash !== contentHash)
+          throw new Error("Fixture content hash mismatch");
+        fixture.content_hash = contentHash;
+      }
+    const sourceHash = await hashEvalSuite(suiteDir, root, cases);
+    const graders = Array.isArray(settings.data.graders) ? settings.data.graders.map((g) => ({ ...g, config_hash: g.config_hash ?? sourceHash })) : [];
+    const suite = evalSuiteInputSchema.parse({
+      ...settings.data,
+      slug: settings.data.slug ?? slugify(entry.name),
+      name: settings.data.name ?? titleize2(entry.name),
+      version: settings.data.version ?? version,
       source_path: sourcePath,
-      source_hash: sha2562([casesRaw, rubric, readme].join(`
----
-`)),
-      pass_threshold: 0.8,
-      promotion_gate: readme ? readme.slice(0, 2000) : undefined,
+      source_hash: sourceHash,
+      promotion_gate: settings.data.promotion_gate ?? (readme ? readme.slice(0, 2000) : undefined),
+      graders,
       cases
     });
+    if (new Set(suite.cases.map((c) => c.case_key)).size !== suite.cases.length)
+      throw new Error(`Duplicate case keys in ${sourcePath}`);
+    suites.push(suite);
   }
   return suites.sort((a, b) => a.slug.localeCompare(b.slug));
 }
 function normalizeEvalCase(input, rubricText) {
-  const key = String(input.case_key ?? input.id ?? input.name ?? randomUUID());
-  const expected = String(input.expected_outcome ?? input.expected_behavior ?? input.expected ?? "");
-  return {
-    case_key: slugify(key),
-    name: titleize2(String(input.name ?? key)),
-    scenario: String(input.scenario ?? input.prompt ?? input.description ?? ""),
-    expected_outcome: expected || "Review with the suite rubric.",
-    rubric: rubricText ? { source: "rubric.md", text_hash: sha2562(rubricText) } : {},
-    fixture_refs: [],
-    tags: Array.isArray(input.tags) ? input.tags.map(String) : [],
-    weight: Number.isInteger(input.weight) ? Number(input.weight) : 1
-  };
+  const key = input.case_key ?? input.id ?? input.name;
+  if (typeof key !== "string" || !key.trim())
+    throw new Error("Eval cases require a stable case_key, id, or name");
+  return evalCaseInputSchema.parse({
+    case_key: key,
+    name: input.name ?? titleize2(key),
+    task_category: input.task_category,
+    scenario: input.scenario ?? input.prompt ?? input.description,
+    expected_outcome: input.expected_outcome ?? input.expected_behavior ?? input.expected,
+    rubric: input.rubric ?? (rubricText ? { source: "rubric.md", text_hash: sha2562(rubricText) } : {}),
+    fixture_refs: input.fixture_refs ?? [],
+    tags: input.tags ?? [],
+    weight: input.weight ?? 1,
+    required: input.required ?? true
+  });
+}
+async function hashEvalDirectory(root) {
+  const entries = [];
+  async function visit(dir2) {
+    for (const item of (await readdir2(dir2, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) {
+      if (["node_modules", ".git", ".promptfoo"].includes(item.name))
+        continue;
+      const path2 = join4(dir2, item.name);
+      if (item.isSymbolicLink())
+        throw new Error("Eval suite symlinks are not supported");
+      if (item.isDirectory())
+        await visit(path2);
+      else if (item.isFile())
+        entries.push([relative2(root, path2), createHash2("sha256").update(await readFile4(path2)).digest("hex")]);
+    }
+  }
+  await visit(root);
+  return sha2562(JSON.stringify(entries));
 }
 function titleize2(value) {
   return value.replace(/[-_]+/g, " ").replace(/\b\w/g, (letter) => letter.toUpperCase());
 }
+async function hashEvalSuite(suiteDir, pluginRoot, cases) {
+  const fixtures = [];
+  for (const c of cases)
+    for (const f of c.fixture_refs)
+      fixtures.push([f.path, createHash2("sha256").update(await readFile4(join4(pluginRoot, f.path))).digest("hex")]);
+  return sha2562(JSON.stringify({ directory: await hashEvalDirectory(suiteDir), fixtures: fixtures.sort(([a], [b]) => a.localeCompare(b)) }));
+}
 
 // src/commands/publish.ts
 var __dirname3 = dirname(fileURLToPath2(import.meta.url));
@@ -14385,17 +14533,25 @@ var __dirname3 = dirname(fileURLToPath2(import.meta.url));
 class PublishCommand extends Command {
   static paths = [["publish"]];
   static usage = Command.Usage({
-    description: "Wrap a plugin's SKILL.md capability and publish a new version. Writes the telemetry footer and script wrappers."
+    description: "Publish a plugin version or wrap a single SKILL.md capability for telemetry."
   });
   skillPath = exports_options.String({ required: true });
   skillId = exports_options.String("--skill-id");
   pluginId = exports_options.String("--plugin-id");
   version = exports_options.String("--version", { description: "semver for this publish; defaults to 0.1.0" });
+  marketplaceUrl = exports_options.String("--marketplace-url", {
+    description: "Git URL for the marketplace or source repo where this plugin version is published"
+  });
   dryRun = exports_options.Boolean("--dry-run", false);
   async execute() {
     const root = resolve(this.skillPath);
     if (await isPluginRoot(root))
       return this.publishPlugin(root);
+    if (this.marketplaceUrl) {
+      this.context.stdout.write(import_picocolors3.default.red(`--marketplace-url is only supported when publishing a plugin directory
+`));
+      return 1;
+    }
     return this.publishSkill(root);
   }
   async publishSkill(root) {
@@ -14474,6 +14630,12 @@ Write key (store now — not retrievable):
     const plan = await readPlugin(root);
     const version = this.version ?? plan.version;
     const pluginId = this.pluginId;
+    const marketplaceUrl = normalizeMarketplaceUrl(this.marketplaceUrl);
+    if (marketplaceUrl === false) {
+      out.write(import_picocolors3.default.red(`invalid --marketplace-url: expected a Git URL starting with https://, http://, git://, ssh://, or git@
+`));
+      return 1;
+    }
     for (const component of plan.components)
       pluginComponentInventoryItemSchema.parse(component);
     out.write(`reading plugin ${import_picocolors3.default.cyan(root)}
@@ -14485,6 +14647,9 @@ Write key (store now — not retrievable):
     out.write(`  manifest:         ${plan.manifestPath} ${import_picocolors3.default.dim(plan.manifestHash)}
 `);
     out.write(`  format:           ${plan.manifestFormat}
+`);
+    if (marketplaceUrl)
+      out.write(`  marketplace URL:  ${marketplaceUrl}
 `);
     out.write(`  platforms:        ${plan.platforms.length ? plan.platforms.join(", ") : import_picocolors3.default.dim("none")}
 `);
@@ -14505,7 +14670,7 @@ Write key (store now — not retrievable):
 `));
       return 0;
     }
-    const published = await publishPluginPlan(plan, version, pluginId);
+    const published = await publishPluginPlan(plan, version, pluginId, marketplaceUrl ?? undefined);
     out.write(import_picocolors3.default.green(`✓ published plugin ${published.plugin.id} ${version}
 `));
     out.write(import_picocolors3.default.bold(`
@@ -14539,7 +14704,7 @@ Imported eval suites:
     return 0;
   }
 }
-async function publishPluginPlan(plan, version, pluginId) {
+async function publishPluginPlan(plan, version, pluginId, marketplaceUrl) {
   const plugin = pluginId ? await getPlugin(pluginId) : await getOrCreatePlugin(plan);
   const skills = await getOrCreatePluginSkills(plan, plugin.id, version);
   await api(`/v1/plugins/${plugin.id}/versions`, {
@@ -14548,6 +14713,7 @@ async function publishPluginPlan(plan, version, pluginId) {
       version,
       manifest_format: plan.manifestFormat,
       manifest_hash: plan.manifestHash,
+      marketplace_published_git_url: marketplaceUrl,
       components: plan.components
     })
   });
@@ -14562,6 +14728,14 @@ async function publishPluginPlan(plan, version, pluginId) {
   });
   return { plugin, skills, evalSuites, writeKey: keyResp.key };
 }
+function normalizeMarketplaceUrl(value) {
+  if (!value)
+    return null;
+  const trimmed = value.trim();
+  if (/^(https?:\/\/|git:\/\/|ssh:\/\/|git@)[^\s]+$/i.test(trimmed))
+    return trimmed;
+  return false;
+}
 async function getPlugin(pluginId) {
   return api(`/v1/plugins/${pluginId}`);
 }
@@ -14620,27 +14794,11 @@ async function getOrCreatePluginSkills(plan, pluginId, version) {
 async function importEvalSuites(plan, pluginId) {
   if (!plan.evalSuites.length)
     return [];
-  const existing = await api(`/v1/plugins/${pluginId}/eval-suites`);
   const rows = [];
   for (const suite of plan.evalSuites) {
-    const found = existing.data.find((candidate) => candidate.slug === suite.slug);
-    if (found) {
-      rows.push(found);
-      continue;
-    }
     rows.push(await api(`/v1/plugins/${pluginId}/eval-suites`, {
       method: "POST",
-      body: JSON.stringify({
-        slug: suite.slug,
-        name: suite.name,
-        version: suite.version,
-        source_path: suite.source_path,
-        source_hash: suite.source_hash,
-        status: "active",
-        pass_threshold: suite.pass_threshold,
-        promotion_gate: suite.promotion_gate,
-        cases: suite.cases
-      })
+      body: JSON.stringify({ ...suite, status: "active" })
     }));
   }
   return rows;
@@ -14837,18 +14995,209 @@ class WebhooksAddCommand extends Command {
   }
 }
 
+// src/commands/evals.ts
+import { spawn, execFile as execFile6 } from "node:child_process";
+import { promisify as promisify6 } from "node:util";
+import { mkdtemp, readFile as readFile6, stat as stat3 } from "node:fs/promises";
+import { join as join6, resolve as resolve2 } from "node:path";
+import { tmpdir } from "node:os";
+
+// src/lib/promptfoo.ts
+import { createHash as createHash3 } from "node:crypto";
+var object = (v) => v && typeof v === "object" && !Array.isArray(v) ? v : {};
+var digest = (v) => createHash3("sha256").update(JSON.stringify(v) ?? "null").digest("hex");
+function importPromptfoo(exported, cases, trials, graders) {
+  const root = object(exported);
+  const summary = Array.isArray(root.results) ? root : object(root.results);
+  const rows = summary.results;
+  if (!Array.isArray(rows) || !rows.length)
+    throw new Error("Expected a non-empty Promptfoo v2/v3 JSON export (results.results)");
+  const byKey = new Map(cases.map((c) => [c.caseKey, c.id]));
+  const groups = new Map;
+  const repetitions = new Map;
+  for (const value of rows) {
+    const row = object(value);
+    const test = object(row.testCase);
+    const metadata = object(test.metadata);
+    const key = metadata.telvine_case_key;
+    if (typeof key !== "string" || !byKey.has(key))
+      throw new Error("Every test must have metadata.telvine_case_key matching a registered case");
+    const provider = object(row.provider);
+    const identity = digest([provider.id, row.promptId ?? row.promptIdx]);
+    const model = typeof provider.id === "string" && evalMetadataKeySchema.safeParse(provider.id).success ? provider.id : `provider:${digest(provider.id).slice(0, 16)}`;
+    const group = groups.get(identity) ?? { identity, model, trials: [] };
+    const repeatKey = `${identity}:${key}`;
+    const trial = (repetitions.get(repeatKey) ?? 0) + 1;
+    if (trial > trials)
+      throw new Error("More results than declared trials; use distinct case keys for expanded test variables");
+    repetitions.set(repeatKey, trial);
+    const grading = object(row.gradingResult);
+    const components = Array.isArray(grading.componentResults) ? grading.componentResults.map(object) : [grading];
+    const assertions = components.flatMap((component) => {
+      const assertion = object(component.assertion);
+      const grader = graders.find((g) => g.id === assertion.metric);
+      if (!grader)
+        return [];
+      return [{
+        grader_id: grader.id,
+        config_hash: grader.config_hash,
+        status: typeof component.pass !== "boolean" ? "errored" : component.pass ? "passed" : "failed",
+        score: typeof component.score === "number" ? component.score : null
+      }];
+    });
+    const isError = row.failureReason === 2 || Boolean(row.error) || typeof row.success !== "boolean";
+    const status = isError ? "errored" : row.success === true ? "passed" : "failed";
+    group.trials.push(evalCaseResultInputSchema.parse({
+      eval_case_id: byKey.get(key),
+      trial,
+      status,
+      score: isError ? null : row.score,
+      rubric_results: {},
+      assertions,
+      error_class: isError ? "promptfoo_execution_error" : undefined,
+      failure_cluster: status === "failed" ? "assertion_failed" : undefined,
+      duration_ms: typeof row.latencyMs === "number" ? Math.round(row.latencyMs) : undefined,
+      cost_usd: typeof row.cost === "number" ? row.cost : undefined,
+      output_hash: row.response ? digest(object(row.response).output) : undefined,
+      evidence_ref: `promptfoo:${digest(row.id ?? [identity, key, trial]).slice(0, 32)}`
+    }));
+    groups.set(identity, group);
+  }
+  return [...groups.values()];
+}
+
+// src/commands/evals.ts
+function runProcess(bin, args, cwd) {
+  return new Promise((res, reject) => {
+    const p = spawn(bin, args, { cwd, stdio: "inherit", shell: false });
+    p.on("error", reject);
+    p.on("exit", (code, signal) => signal ? reject(new Error(`Promptfoo terminated: ${signal}`)) : res(code ?? 1));
+  });
+}
+async function submit(suite, exported, options2) {
+  const groups = importPromptfoo(exported, suite.cases, options2.trials, suite.graders);
+  const runs = [];
+  for (const group of groups) {
+    const run2 = await api(`/v1/eval-suites/${suite.id}/runs`, { method: "POST", body: JSON.stringify({
+      plugin_version: options2.version,
+      harness: "promptfoo",
+      model: group.model,
+      trials: options2.trials,
+      suite_revision_hash: suite.revisionHash,
+      grader_config_hash: suite.sourceHash,
+      engine_version: options2.engineVersion,
+      baseline_run_id: options2.baseline,
+      parent_run_id: options2.parent,
+      experiment_id: options2.experiment,
+      evaluator_variant: group.identity
+    }) });
+    try {
+      for (const result of group.trials)
+        await api(`/v1/eval-runs/${run2.id}/cases`, { method: "POST", body: JSON.stringify(result) });
+      runs.push(await api(`/v1/eval-runs/${run2.id}/complete`, { method: "POST", body: JSON.stringify({ status: "completed" }) }));
+    } catch (error) {
+      await api(`/v1/eval-runs/${run2.id}/complete`, { method: "POST", body: JSON.stringify({ status: "failed" }) }).catch(() => {});
+      throw error;
+    }
+  }
+  return runs;
+}
+
+class EvalOptions extends Command {
+  version = exports_options.String("--version", { required: true });
+  trials = exports_options.String("--trials", "1");
+  baseline = exports_options.String("--baseline-run-id");
+  parent = exports_options.String("--parent-run-id");
+  experiment = exports_options.String("--experiment-id");
+  trialCount() {
+    const n = Number(this.trials);
+    if (!Number.isInteger(n) || n < 1 || n > 100)
+      throw new Error("--trials must be an integer from 1 to 100");
+    return n;
+  }
+  printRuns(runs) {
+    for (const r of runs)
+      this.context.stdout.write(`${r.id}: gate ${r.gateStatus}; coverage ${Math.round((r.coverage ?? 0) * 100)}%; pass ${Math.round((r.passRate ?? 0) * 100)}%${r.gateReasons.length ? ` (${r.gateReasons.join(", ")})` : ""}
+`);
+    return runs.every((r) => r.gateStatus === "passed") ? 0 : 1;
+  }
+}
+
+class EvalRunCommand extends EvalOptions {
+  static paths = [["eval", "run"]];
+  static usage = Command.Usage({ description: "Run a native Promptfoo suite locally and submit metadata-only evidence to Telvine." });
+  plugin = exports_options.String({ required: true });
+  suiteSlug = exports_options.String("--suite", { required: true });
+  pluginId = exports_options.String("--plugin-id", { required: true });
+  bin = exports_options.String("--promptfoo-bin", "promptfoo");
+  engineVersion = exports_options.String("--engine-version", { required: true });
+  async execute() {
+    const root = resolve2(this.plugin);
+    const plan = await readPlugin(root);
+    const definition = plan.evalSuites.find((s) => s.slug === this.suiteSlug);
+    if (!definition?.source_path)
+      throw new Error("Suite not found under evals/");
+    const suiteDir = join6(root, definition.source_path);
+    let config;
+    for (const name of ["promptfooconfig.yaml", "promptfooconfig.yml", "promptfooconfig.json"])
+      if (await stat3(join6(suiteDir, name)).then((s) => s.isFile()).catch(() => false)) {
+        config = name;
+        break;
+      }
+    if (!config)
+      throw new Error("Add a native promptfooconfig.yaml or .json to the suite directory");
+    const trials = this.trialCount();
+    const { stdout } = await promisify6(execFile6)(this.bin, ["--version"], { cwd: suiteDir, timeout: 30000 });
+    const actualVersion = stdout.trim().match(/(?:^|\s)(\d+\.\d+\.\d+(?:-[A-Za-z0-9.-]+)?)(?:\s|$)/)?.[1];
+    if (!actualVersion || actualVersion !== this.engineVersion)
+      throw new Error(`Promptfoo engine version mismatch: expected ${this.engineVersion}, found ${actualVersion ?? "unknown"}`);
+    const dir2 = await mkdtemp(join6(tmpdir(), "telvine-promptfoo-"));
+    const output = join6(dir2, "results.json");
+    this.context.stdout.write(`Promptfoo runs locally. Full evidence stays at ${output}
+`);
+    const code = await runProcess(this.bin, ["eval", "-c", config, "--repeat", String(trials), "--no-cache", "--no-share", "--output", output], suiteDir);
+    if (await hashEvalSuite(suiteDir, root, definition.cases) !== definition.source_hash)
+      throw new Error("Suite changed during execution; evidence was not submitted");
+    const exported = JSON.parse(await readFile6(output, "utf8"));
+    const existing = await api(`/v1/plugins/${this.pluginId}/eval-suites`);
+    const found = existing.data.find((s) => s.slug === definition.slug && s.sourceHash === definition.source_hash);
+    const row = found ?? await api(`/v1/plugins/${this.pluginId}/eval-suites`, { method: "POST", body: JSON.stringify({ ...definition, status: "active" }) });
+    const suite = await api(`/v1/eval-suites/${row.id}`);
+    const runs = await submit(suite, exported, { version: this.version, trials, engineVersion: actualVersion, baseline: this.baseline, parent: this.parent, experiment: this.experiment });
+    return this.printRuns(runs) || code;
+  }
+}
+
+class EvalImportCommand extends EvalOptions {
+  static paths = [["eval", "import"]];
+  static usage = Command.Usage({ description: "Import only safe metadata from a local Promptfoo JSON export." });
+  file = exports_options.String({ required: true });
+  suiteId = exports_options.String("--suite-id", { required: true });
+  sourceHash = exports_options.String("--source-hash", { required: true });
+  engineVersion = exports_options.String("--engine-version", { required: true });
+  async execute() {
+    const suite = await api(`/v1/eval-suites/${this.suiteId}`);
+    if (suite.sourceHash !== this.sourceHash)
+      throw new Error("Suite source hash mismatch");
+    const exported = JSON.parse(await readFile6(resolve2(this.file), "utf8"));
+    return this.printRuns(await submit(suite, exported, { version: this.version, trials: this.trialCount(), engineVersion: this.engineVersion, baseline: this.baseline, parent: this.parent, experiment: this.experiment }));
+  }
+}
+
 // src/telvine.ts
 var args = process.argv.slice(2);
 var cli = new Cli({
   binaryLabel: "telvine",
   binaryName: "telvine",
-  binaryVersion: "0.1.0"
+  binaryVersion: "0.1.2"
 });
 cli.register(LoginCommand);
 cli.register(exports_builtins.HelpCommand);
 cli.register(exports_builtins.VersionCommand);
 cli.register(InitCommand);
 cli.register(PublishCommand);
+cli.register(EvalRunCommand);
+cli.register(EvalImportCommand);
 cli.register(SkillsListCommand);
 cli.register(SkillsMetricsCommand);
 cli.register(PluginsEventsCommand);

package/package.json

@@ -1,16 +1,16 @@
 {
   "name": "@telvine/cli",
-  "version": "0.1.0",
-  "description": "Telvine CLI - instrument SKILL.md capabilities inside agent plugins",
+  "version": "0.1.2",
+  "description": "Telvine CLI - plugin telemetry, Promptfoo evaluation evidence, and release checks",
   "homepage": "https://telvine.com",
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/telvinehq/telvine.git",
+    "url": "git+https://github.com/telvinehq/telvine-cli.git",
     "directory": "apps/cli"
   },
   "bugs": {
-    "url": "https://github.com/telvinehq/telvine/issues"
+    "url": "https://github.com/telvinehq/telvine-cli/issues"
   },
   "keywords": [
     "agent-plugin",
@@ -35,12 +35,6 @@
     "dist",
     "src/templates"
   ],
-  "scripts": {
-    "build": "bun build src/telvine.ts --target=node --outdir=dist --banner='#!/usr/bin/env node'",
-    "dev": "tsx src/telvine.ts",
-    "typecheck": "tsc --noEmit",
-    "start": "node dist/telvine.js"
-  },
   "dependencies": {
     "@inquirer/prompts": "^7.0.0",
     "clipanion": "^4.0.0-rc.4",
@@ -51,9 +45,15 @@
     "typanion": "^3.14.0"
   },
   "devDependencies": {
-    "@telvine/schema": "workspace:0.1.0",
+    "@telvine/schema": "0.1.2",
     "@types/node": "^22.7.5",
     "tsx": "^4.19.1",
     "typescript": "^5.6.3"
+  },
+  "scripts": {
+    "build": "bun build src/telvine.ts --target=node --external=keytar --outdir=dist --banner='#!/usr/bin/env node'",
+    "dev": "tsx src/telvine.ts",
+    "typecheck": "tsc --noEmit",
+    "start": "node dist/telvine.js"
   }
-}
+}