hivenet 0.6.0 → 0.7.1

package/dist/index.js

@@ -31,11 +31,11 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
   mod
 ));
 
-// ../../node_modules/.pnpm/tsup@8.5.1_jiti@2.7.0_postcss@8.5.26_typescript@5.9.3/node_modules/tsup/assets/esm_shims.js
+// ../../node_modules/.pnpm/tsup@8.5.1_jiti@2.7.0_postcss@8.5.30_typescript@5.9.3_yaml@2.9.1/node_modules/tsup/assets/esm_shims.js
 import path from "path";
 import { fileURLToPath } from "url";
 var init_esm_shims = __esm({
-  "../../node_modules/.pnpm/tsup@8.5.1_jiti@2.7.0_postcss@8.5.26_typescript@5.9.3/node_modules/tsup/assets/esm_shims.js"() {
+  "../../node_modules/.pnpm/tsup@8.5.1_jiti@2.7.0_postcss@8.5.30_typescript@5.9.3_yaml@2.9.1/node_modules/tsup/assets/esm_shims.js"() {
     "use strict";
   }
 });
@@ -15209,6 +15209,157 @@ function mediaTypeForFilename(filename) {
   const extension = /\.([^./\\]+)$/.exec(filename)?.[1]?.toLowerCase();
   return extension && MEDIA_TYPES[extension] || "application/octet-stream";
 }
+var evalCriterionIdSchema = external_exports.string().regex(
+  /^[a-z][a-z0-9_]{0,47}$/,
+  "Criterion ids are snake_case: a lowercase letter, then lowercase letters, digits or _ (at most 48)."
+);
+var weightSchema = external_exports.number().positive().max(100).optional();
+var evalJudgeCriterionSchema = external_exports.object({
+  id: evalCriterionIdSchema,
+  kind: external_exports.literal("judge"),
+  description: external_exports.string().trim().min(1).max(2e3).describe("The yes/no question the judge answers about the result."),
+  weight: weightSchema
+});
+var evalCommandCriterionSchema = external_exports.object({
+  id: evalCriterionIdSchema,
+  kind: external_exports.literal("command"),
+  description: external_exports.string().trim().min(1).max(500).describe("What the check proves, for people reading results."),
+  command: external_exports.string().trim().min(1).max(4e3),
+  contains: external_exports.string().min(1).max(1e3).optional().describe("Stdout must contain this text."),
+  regex: external_exports.string().min(1).max(500).optional().describe("Stdout must match this regex."),
+  timeout_sec: external_exports.number().int().min(1).max(600).optional(),
+  weight: weightSchema
+}).refine((criterion) => !(criterion.contains && criterion.regex), {
+  message: "A command criterion takes contains or regex, not both."
+});
+var evalCriterionSchema = external_exports.discriminatedUnion("kind", [
+  evalJudgeCriterionSchema,
+  evalCommandCriterionSchema
+]);
+var evalJudgeSchema = external_exports.object({
+  model: external_exports.string().trim().min(1).max(128).default("anthropic/claude-sonnet-5"),
+  /** Give the judge the agent's ATIF trajectory, e.g. to grade which tool or skill it used. */
+  reads_trajectory: external_exports.boolean().default(false),
+  /** Workspace files the judge reads, relative to the workdir. */
+  files: external_exports.array(external_exports.string().trim().min(1).max(256)).max(20).default([])
+});
+var ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]{0,63}$/;
+var SINGLE_LINE = /^[^\r\n]+$/;
+var evalMcpServerSchema = external_exports.object({
+  name: external_exports.string().regex(/^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/),
+  transport: external_exports.enum(["stdio", "sse", "streamable-http"]),
+  url: external_exports.url().max(2e3).optional(),
+  command: external_exports.string().trim().min(1).max(500).optional(),
+  args: external_exports.array(external_exports.string().max(500)).max(30).default([])
+});
+var evalEnvironmentSchema = external_exports.object({
+  /** The base image, e.g. `node:22-slim`. */
+  image: external_exports.string().trim().regex(/^[A-Za-z0-9][A-Za-z0-9._\-/:@]{0,255}$/, "image must be a Docker image reference."),
+  /** Dockerfile RUN lines, cached in the image: toolchains, not the product under test. */
+  build: external_exports.array(external_exports.string().trim().regex(SINGLE_LINE).max(2e3)).max(30).default([]),
+  /**
+   * Runs before every trial, so each one tests the current product. Harbor runs it as the
+   * healthcheck and retries it on failure, so it must be safe to run twice.
+   */
+  setup: external_exports.object({
+    command: external_exports.string().trim().min(1).max(8e3),
+    timeout_sec: external_exports.number().int().min(10).max(3600).default(600)
+  }).nullable().default(null),
+  workdir: external_exports.string().regex(/^\/[A-Za-z0-9._\-/]{0,255}$/).default("/app"),
+  network: external_exports.object({
+    mode: external_exports.enum(["public", "allowlist", "no-network"]).default("public"),
+    allowed_hosts: external_exports.array(external_exports.string().trim().min(1).max(253)).max(50).default([])
+  }).prefault({}),
+  /** Values reach setup and the agent; secret-like names take only `${VAR}` templates. */
+  env: external_exports.record(external_exports.string().regex(ENV_NAME), external_exports.string().max(2e3)).default({}),
+  mcp_servers: external_exports.array(evalMcpServerSchema).max(10).default([])
+});
+var evalAgentSchema = external_exports.object({
+  timeout_sec: external_exports.number().int().min(30).max(7200).default(900)
+});
+var evalVersionFieldsSchema = external_exports.object({
+  /** The task in the owner's words. Never the reporter's `actual` or `mistake`. */
+  instruction: external_exports.string().trim().min(1).max(2e4),
+  /** What "done" means, in one or two sentences. */
+  end_goal: external_exports.string().trim().min(1).max(2e3).optional(),
+  criteria: external_exports.array(evalCriterionSchema).min(1).max(30),
+  judge: evalJudgeSchema.optional(),
+  environment: evalEnvironmentSchema,
+  agent: evalAgentSchema.prefault({}),
+  /** A reference solution for Harbor's `oracle` agent, as a bash script. */
+  solution: external_exports.string().trim().min(1).max(2e4).optional(),
+  change_note: external_exports.string().trim().min(1).max(500).optional()
+});
+var evalVersionInputSchema = evalVersionFieldsSchema.refine(
+  (input) => new Set(input.criteria.map((c) => c.id)).size === input.criteria.length,
+  { message: "Criterion ids must be unique.", path: ["criteria"] }
+);
+var HARBOR_ID = external_exports.string().regex(/^[0-9a-f-]{36}$/i, "Harbor ids are UUIDs.");
+var boundedRecord = (max) => external_exports.record(external_exports.string().min(1).max(200), external_exports.number()).refine((record2) => Object.keys(record2).length <= max, `At most ${max} keys.`);
+var evalTrialScenarioSchema = external_exports.object({
+  /** Folders the job mounted (`harbor run --mounts`), by name: the path stays on the machine. */
+  mounts: external_exports.array(external_exports.string().min(1).max(200)).max(5).optional(),
+  /** `--skill` */
+  skills: external_exports.array(external_exports.string().min(1).max(200)).max(20).optional(),
+  /** `--mcp-config` */
+  mcp_servers: external_exports.array(external_exports.string().min(1).max(100)).max(20).optional(),
+  /** `--extra-instruction` text, or the file `--extra-instruction-path` named. */
+  instructions: external_exports.array(external_exports.string().min(1).max(500)).max(5).optional(),
+  /** `--ak` and `--ae`, without secret-looking names. */
+  agent_options: external_exports.record(external_exports.string().min(1).max(64), external_exports.string().max(200)).refine((record2) => Object.keys(record2).length <= 20, "At most 20 agent options.").optional()
+});
+var evalTrialCiSchema = external_exports.object({
+  provider: external_exports.literal("github"),
+  /** `owner/name` */
+  repository: external_exports.string().min(1).max(200),
+  /** The commit under test: a pull request's head, else the commit that triggered the run. */
+  commit: external_exports.string().regex(/^[0-9a-f]{7,64}$/i).optional(),
+  branch: external_exports.string().min(1).max(250).optional(),
+  pull_request: external_exports.number().int().positive().optional(),
+  /** What started the workflow: pull_request, push, schedule, workflow_dispatch… */
+  event: external_exports.string().min(1).max(50).optional(),
+  run_url: external_exports.url({ protocol: /^https?$/ }).max(500).optional()
+});
+var evalTrialUploadSchema = external_exports.object({
+  harbor_trial_id: HARBOR_ID,
+  harbor_job_id: HARBOR_ID,
+  harbor_job_name: external_exports.string().max(200).optional(),
+  /** Set on a regrade: the trial it derives from. */
+  source_harbor_trial_id: HARBOR_ID.optional(),
+  /** `[task].name` from the trial's result: `<project slug>/<eval id>` for Hivenet tasks. */
+  task_name: external_exports.string().min(1).max(300),
+  /** `task.digest` from the trial's lock.json. */
+  task_digest: external_exports.string().regex(/^sha256:[0-9a-f]{64}$/),
+  agent: external_exports.string().min(1).max(100),
+  agent_version: external_exports.string().max(100).optional(),
+  model: external_exports.string().max(200).optional(),
+  /** The reasoning effort it ran at (`harbor run --ak reasoning_effort=…`): part of the lane. */
+  effort: external_exports.string().min(1).max(40).optional(),
+  harbor_version: external_exports.string().max(50).optional(),
+  exception: external_exports.object({ type: external_exports.string().min(1).max(200), message: external_exports.string().max(2e3) }).optional(),
+  /** reward.json. Absent when the verifier wrote nothing: the trial wasn't graded. */
+  rewards: boundedRecord(32).optional(),
+  /** reward-details.json leaves, keyed by criterion id. */
+  criteria: boundedRecord(64).optional(),
+  metrics: external_exports.object({
+    input_tokens: external_exports.number().int().nonnegative(),
+    output_tokens: external_exports.number().int().nonnegative(),
+    cache_tokens: external_exports.number().int().nonnegative(),
+    cost_usd: external_exports.number().nonnegative(),
+    setup_sec: external_exports.number().nonnegative(),
+    agent_sec: external_exports.number().nonnegative(),
+    verifier_sec: external_exports.number().nonnegative()
+  }).partial().optional(),
+  started_at: external_exports.iso.datetime({ offset: true }).optional(),
+  finished_at: external_exports.iso.datetime({ offset: true }).optional(),
+  scenario: evalTrialScenarioSchema.optional(),
+  ci: evalTrialCiSchema.optional()
+});
+var MAX_TRIALS_PER_PUSH = 200;
+var MAX_TRAJECTORY_BYTES = 8 * 1024 * 1024;
+var evalResultsUploadSchema = external_exports.object({
+  trials: external_exports.array(evalTrialUploadSchema).min(1).max(MAX_TRIALS_PER_PUSH)
+});
 var PROTOCOL_VERSION = 1;
 var MAX_FEEDBACK_LENGTH = 5e3;
 var MAX_SUBJECT_LENGTH = 500;
@@ -15311,8 +15462,12 @@ var metadataSchema = external_exports.record(
   external_exports.string().min(1).max(40),
   external_exports.union([external_exports.string().max(200), external_exports.number().finite(), external_exports.boolean()])
 ).refine((value) => Object.keys(value).length <= 20, { message: "at most 20 metadata keys" });
+var threadSchema = external_exports.object({ id: external_exports.string().regex(THREAD_ID_PATTERN) });
+function newThreadId() {
+  return crypto.randomUUID().replaceAll("-", "");
+}
 var feedbackEventSchema = external_exports.object({
-  v: external_exports.literal(PROTOCOL_VERSION),
+  v: external_exports.literal(PROTOCOL_VERSION).default(PROTOCOL_VERSION),
   /** Tenant slug or domain this feedback is addressed to. Optional when the write key implies it. */
   to: external_exports.string().max(253).optional(),
   feedback: external_exports.string().min(1).max(MAX_FEEDBACK_LENGTH),
@@ -15320,7 +15475,7 @@ var feedbackEventSchema = external_exports.object({
   subject: external_exports.string().min(1).max(MAX_SUBJECT_LENGTH).optional(),
   sentiment: sentimentSchema.optional(),
   author: authorSchema.optional(),
-  thread: external_exports.object({ id: external_exports.string().regex(THREAD_ID_PATTERN) }),
+  thread: threadSchema.optional().transform((thread) => thread ?? { id: newThreadId() }),
   eval: evalReportSchema.optional(),
   /** Id of an owner/system question this submission answers (from a prior response's `ask`). */
   question: external_exports.string().max(64).optional(),
@@ -15333,8 +15488,8 @@ var feedbackEventSchema = external_exports.object({
   client: external_exports.object({
     name: external_exports.string().max(64),
     version: external_exports.string().max(32)
-  }),
-  consent: external_exports.object({ telemetry: external_exports.boolean() })
+  }).default({ name: "http", version: "0" }),
+  consent: external_exports.object({ telemetry: external_exports.boolean() }).default({ telemetry: false })
 });
 var knownIssueSchema = external_exports.object({
   title: external_exports.string().max(120),
@@ -15426,7 +15581,7 @@ function validateIngestResponse(input) {
   return result.success ? { ok: true, data: result.data } : { ok: false, errors: formatIssues(result.error) };
 }
 function feedbackEventJsonSchema() {
-  return external_exports.toJSONSchema(feedbackEventSchema);
+  return external_exports.toJSONSchema(feedbackEventSchema, { io: "input" });
 }
 function descriptorJsonSchema() {
   return external_exports.toJSONSchema(descriptorSchema);
@@ -16069,7 +16224,7 @@ function inferAuthor(context) {
 }
 var CONFIG_FILE_NAME = "feedback.config.json";
 var DESCRIPTOR_CACHE_TTL_MS = 60 * 60 * 1e3;
-var RESOLVE_TIMEOUT_MS = 5e3;
+var RESOLVE_TIMEOUT_MS = 1e4;
 function parseDsn(dsn) {
   let url2;
   try {
@@ -16185,7 +16340,17 @@ function destinationFromDescriptor(descriptor, to) {
   };
 }
 function registryUrl(env = process.env) {
-  return env[REGISTRY_ENV_VAR]?.trim() || DEFAULT_REGISTRY_URL;
+  const value = env[REGISTRY_ENV_VAR]?.trim() || DEFAULT_REGISTRY_URL;
+  try {
+    const url2 = new URL(value);
+    if (!["http:", "https:"].includes(url2.protocol) || url2.username || url2.password || url2.search || url2.hash || url2.pathname !== "/" || !/^[a-z0-9._:[\]-]+$/i.test(url2.hostname))
+      throw new Error("Not an origin");
+    return url2.origin;
+  } catch {
+    throw new FeedbackError(
+      "Registry must be an HTTP(S) origin, such as https://feedback.example.com, without credentials, a path, query or fragment."
+    );
+  }
 }
 async function resolveDestination(options = {}) {
   const env = options.env ?? process.env;
@@ -16194,6 +16359,19 @@ async function resolveDestination(options = {}) {
   if (options.dsn) {
     return parseDsn(options.dsn);
   }
+  if (options.registry !== void 0) {
+    const to2 = options.to?.trim();
+    if (!options.registry.trim() || !to2 || to2.includes(".")) {
+      throw new FeedbackError("Use --registry with --to <project-slug>, not a domain.");
+    }
+    const origin = registryUrl({ [REGISTRY_ENV_VAR]: options.registry });
+    return resolveViaDescriptor(
+      `${origin}/v1/registry/${encodeURIComponent(to2)}`,
+      to2,
+      fetchImpl,
+      cacheDir
+    );
+  }
   const envDsn = env[DSN_ENV_VAR]?.trim();
   if (envDsn) {
     return parseDsn(envDsn);
@@ -16256,23 +16434,24 @@ async function postEvent(options) {
     body = void 0;
   }
   const parsed = validateIngestResponse(body);
-  return {
-    kind: "accepted",
-    response: parsed.ok ? parsed.data : { id: "unknown", thread: options.event.thread.id }
-  };
+  return parsed.ok ? { kind: "accepted", response: parsed.data } : { kind: "unreachable", error: "Server returned an invalid delivery receipt." };
 }
 var OUTBOX_FILE = "outbox.jsonl";
-var MAX_OUTBOX_ENTRIES = 100;
+var MAX_FLUSH_ENTRIES = 100;
 var STALE_CLAIM_MS = 30 * 60 * 1e3;
 function outboxPath(env = process.env) {
   return path5.join(feedbackHomeDir(env), OUTBOX_FILE);
 }
 function appendToOutbox(entry, env = process.env) {
+  return appendEntriesToOutbox([entry], env);
+}
+function appendEntriesToOutbox(entries, env) {
+  if (entries.length === 0) return true;
   try {
     const file2 = outboxPath(env);
     mkdirSync2(path5.dirname(file2), { recursive: true });
-    appendFileSync(file2, `${JSON.stringify(entry)}
-`);
+    appendFileSync(file2, entries.map((entry) => `${JSON.stringify(entry)}
+`).join(""));
     return true;
   } catch {
     return false;
@@ -16314,24 +16493,30 @@ function claimQueue(env) {
 async function flushOutbox(options = {}) {
   const env = options.env ?? process.env;
   const claims = claimQueue(env);
-  const entries = claims.flatMap(readEntries).slice(-MAX_OUTBOX_ENTRIES);
+  const entries = claims.flatMap(readEntries);
+  const deadline = Date.now() + (options.maxDurationMs ?? Number.POSITIVE_INFINITY);
   const failures = [];
+  let attempted = 0;
   let delivered = 0;
-  for (const entry of entries) {
+  for (const entry of entries.slice(0, MAX_FLUSH_ENTRIES)) {
+    const timeLeft = deadline - Date.now();
+    if (timeLeft <= 0) break;
+    attempted++;
     const result = await postEvent({
       endpoint: entry.endpoint,
       ...entry.writeKey ? { writeKey: entry.writeKey } : {},
       event: entry.event,
-      timeoutMs: options.timeoutMs ?? 5e3,
+      timeoutMs: Math.max(1, Math.floor(Math.min(options.timeoutMs ?? 5e3, timeLeft))),
       ...options.fetchImpl ? { fetchImpl: options.fetchImpl } : {}
     });
-    if (result.kind === "accepted" || result.kind === "rejected") {
-      delivered += result.kind === "accepted" ? 1 : 0;
+    if (result.kind === "accepted") {
+      delivered++;
     } else {
       failures.push(entry);
     }
   }
-  if (failures.every((entry) => appendToOutbox(entry, env))) {
+  const remaining = entries.slice(attempted).concat(failures);
+  if (appendEntriesToOutbox(remaining, env)) {
     for (const claim of claims) {
       try {
         rmSync(claim, { force: true });
@@ -16339,16 +16524,22 @@ async function flushOutbox(options = {}) {
       }
     }
   }
-  return { delivered, remaining: failures.length };
+  return { delivered, remaining: remaining.length };
 }
 async function resolveApiOrigin(options) {
-  if (!options.to && !options.dsn) {
-    return new URL(DEFAULT_REGISTRY_URL).origin;
+  const env = options.env ?? process.env;
+  if (options.registry !== void 0 && !options.to && !options.dsn) {
+    if (!options.registry.trim()) throw new FeedbackError("Registry must not be empty.");
+    return registryUrl({ [REGISTRY_ENV_VAR]: options.registry });
+  }
+  if (!options.to && !options.dsn && !env[DSN_ENV_VAR]?.trim()) {
+    return registryUrl(env);
   }
   const destination = await resolveDestination({
     ...options.to !== void 0 ? { to: options.to } : {},
     ...options.dsn !== void 0 ? { dsn: options.dsn } : {},
-    ...options.env ? { env: options.env } : {},
+    ...options.registry !== void 0 ? { registry: options.registry } : {},
+    env,
     ...options.fetchImpl ? { fetchImpl: options.fetchImpl } : {}
   });
   return new URL(destination.endpoint).origin;
@@ -16357,12 +16548,12 @@ function ownerApi(options) {
   const origin = (options.apiOrigin ?? new URL(DEFAULT_REGISTRY_URL).origin).replace(/\/$/, "");
   const fetchImpl = options.fetchImpl ?? fetch;
   const timeoutMs = options.timeoutMs ?? 15e3;
-  async function request(path6, init) {
+  async function request(path6, init, timeout = timeoutMs) {
     let response;
     try {
       response = await fetchImpl(`${origin}${path6}`, {
         ...init,
-        signal: AbortSignal.timeout(timeoutMs),
+        signal: AbortSignal.timeout(timeout),
         headers: {
           authorization: `Bearer ${options.secretKey}`,
           ...init?.body ? { "content-type": "application/json" } : {},
@@ -16495,6 +16686,33 @@ function ownerApi(options) {
         body: JSON.stringify({ status, ...note !== void 0 ? { note } : {} })
       });
     },
+    /** Draft a runnable version of an eval (a person approves it in the dashboard). */
+    async draftEvalVersion(id, fields) {
+      return request(`/v1/evals/${encodeURIComponent(id)}/versions`, {
+        method: "POST",
+        body: JSON.stringify(fields)
+      });
+    },
+    /** Compiled Harbor tasks: every accepted eval's current version, or the given refs. */
+    async pullEvalTasks(refs = []) {
+      const query = refs.map((ref) => `ref=${encodeURIComponent(ref)}`).join("&");
+      return request(`/v1/evals/tasks${query ? `?${query}` : ""}`);
+    },
+    /** Harbor trials read from a job directory, with their grades. */
+    async pushEvalResults(trials) {
+      return request("/v1/evals/results", {
+        method: "POST",
+        body: JSON.stringify({ trials })
+      });
+    },
+    /** A stored trial's ATIF trajectory (`agent/trajectory.json`), as read from disk. */
+    async pushTrialTrajectory(harborTrialId, trajectory) {
+      return request(
+        `/v1/evals/trials/${encodeURIComponent(harborTrialId)}/trajectory`,
+        { method: "PUT", body: trajectory },
+        Math.max(timeoutMs, 6e4)
+      );
+    },
     /** Attach an external link (Slack message, Linear/GitHub issue, PR) to evt_… or iss_…. */
     async addLink(id, url2, label) {
       const path6 = id.startsWith("iss_") ? `/v1/issues/${id}` : `/v1/events/${id}`;
@@ -16534,6 +16752,7 @@ async function prepareFeedback(options) {
   const resolveOptions = {
     ...options.to !== void 0 ? { to: options.to } : {},
     ...options.dsn !== void 0 ? { dsn: options.dsn } : {},
+    ...options.registry !== void 0 ? { registry: options.registry } : {},
     cwd,
     env,
     ...options.fetchImpl ? { fetchImpl: options.fetchImpl } : {}
@@ -16702,6 +16921,7 @@ export {
   resolveThreadId,
   sendFeedback,
   sentimentSchema,
+  threadSchema,
   validateDescriptor,
   validateFeedbackEvent,
   validateIngestResponse