@osfactory/har 0.39.0 → 0.41.0

package/README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img src="logo.png" alt="HAR logo" width="120">
+  <img src="assets/logo.png" alt="HAR logo" width="120">
 </p>
 
 # HAR — Harnesses for coding agents
@@ -68,6 +68,19 @@ Everything beyond install and first commands lives at **[https://harproject.dev/
 
 See [CONTRIBUTING.md](./CONTRIBUTING.md) for local setup, the dogfood harness loop, and architecture. Coding agents working on this repo should start with [AGENT.md](./AGENT.md). Maintainer release process: [RELEASING.md](./RELEASING.md).
 
+## Sponsors
+
+HAR is sponsored by [Kerno](https://kerno.io) — runtime code and security review for coding agents.
+
+<p align="center">
+  <a href="https://kerno.io">
+    <picture>
+      <source media="(prefers-color-scheme: dark)" srcset="assets/kerno-logo.svg">
+      <img src="assets/kerno-logo-dark.svg" alt="Kerno" height="36">
+    </picture>
+  </a>
+</p>
+
 ## License
 
 Licensed under the [Apache License 2.0](./LICENSE).

package/dist/index.js

@@ -28533,7 +28533,9 @@ function readPortalCredentials() {
       token: parsed.token,
       workspace: parsed.workspace,
       email: parsed.email,
-      createdAt: parsed.createdAt ?? ""
+      createdAt: parsed.createdAt ?? "",
+      refreshToken: parsed.refreshToken,
+      expiresAt: parsed.expiresAt
     };
   } catch {
     return null;
@@ -28576,7 +28578,11 @@ function getPortalTarget() {
   }
   const stored = readPortalCredentials();
   if (stored) {
-    return { url: normalizeUrl(stored.portalUrl), token: stored.token };
+    return {
+      url: normalizeUrl(stored.portalUrl),
+      token: stored.token,
+      refreshToken: stored.refreshToken
+    };
   }
   if (process.env.HAR_CLOUD_API_URL && process.env.HAR_CLOUD_API_KEY) {
     return {
@@ -30982,6 +30988,25 @@ var init_codex = __esm({
 });
 
 // src/core/usage-harvest/index.ts
+function sessionToolKey(sessionKey, agentTool) {
+  return `${sessionKey}\0${agentTool}`;
+}
+function omitHarvestWhenOtelPresent(harvested, existing) {
+  const otelKeys = new Set(
+    existing.filter((row) => (row.sources ?? []).includes("otel")).map((row) => sessionToolKey(row.sessionKey, row.agentTool))
+  );
+  if (otelKeys.size === 0) return harvested;
+  return harvested.filter((row) => !otelKeys.has(sessionToolKey(row.sessionKey, row.agentTool)));
+}
+function omitHarvestEventsWhenOtelPresent(harvested, existing) {
+  const otelKeys = new Set(
+    existing.filter((event) => event.source === "otel").map((event) => sessionToolKey(event.sessionKey, event.agentTool))
+  );
+  if (otelKeys.size === 0) return harvested;
+  return harvested.filter(
+    (event) => !otelKeys.has(sessionToolKey(event.sessionKey, event.agentTool))
+  );
+}
 function harvestUsageForSlot(slot) {
   const out = [];
   const claude = harvestClaudeUsage(slot);
@@ -31043,6 +31068,27 @@ function writePortalWatermark(repoPath, portalUrl, lastSyncedAt) {
   }
   writeState(state);
 }
+function readRunsWatermarkEntry(repoPath, target) {
+  const canonical = canonicalizeControlRepoPath(repoPath);
+  const entry = readState().states.find(
+    (state) => state.repoPath === canonical && state.portalUrl === target
+  );
+  return entry ? { lastSyncedAt: entry.lastSyncedAt, repoId: entry.repoId } : null;
+}
+function writeRunsWatermark(repoPath, target, repoId, lastSyncedAt) {
+  const canonical = canonicalizeControlRepoPath(repoPath);
+  const state = readState();
+  const existing = state.states.find(
+    (entry) => entry.repoPath === canonical && entry.portalUrl === target
+  );
+  if (existing) {
+    existing.lastSyncedAt = lastSyncedAt;
+    existing.repoId = repoId;
+  } else {
+    state.states.push({ repoPath: canonical, portalUrl: target, lastSyncedAt, repoId });
+  }
+  writeState(state);
+}
 function selectSince(rows, since, getTimestamp) {
   let maxSyncedAt = null;
   const selected = [];
@@ -31250,6 +31296,7 @@ var init_portal_usage_merge = __esm({
 // src/core/control-sync.ts
 var control_sync_exports = {};
 __export(control_sync_exports, {
+  chunkBySerializedSize: () => chunkBySerializedSize,
   discoverHarRepos: () => discoverHarRepos,
   ensureRepoRegisteredWithControl: () => ensureRepoRegisteredWithControl,
   isControlApiReachable: () => isControlApiReachable,
@@ -31261,38 +31308,133 @@ __export(control_sync_exports, {
   syncRunWithControlAsync: () => syncRunWithControlAsync,
   waitForControlApi: () => waitForControlApi
 });
+function maxSyncBatchBytes() {
+  const raw = Number(process.env.HAR_SYNC_MAX_BATCH_BYTES);
+  return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_MAX_BATCH_BYTES;
+}
+function chunkBySerializedSize(items, maxBytes) {
+  const batches = [];
+  let current = [];
+  let currentBytes = 0;
+  for (const item of items) {
+    const itemBytes = Buffer.byteLength(JSON.stringify(item), "utf8");
+    if (current.length > 0 && currentBytes + itemBytes > maxBytes) {
+      batches.push(current);
+      current = [];
+      currentBytes = 0;
+    }
+    current.push(item);
+    currentBytes += itemBytes;
+  }
+  if (current.length > 0) batches.push(current);
+  return batches;
+}
+function runTimestamp(run2) {
+  return run2.finishedAt ?? run2.startedAt;
+}
+function rewindSince(since) {
+  if (!since) return null;
+  const raw = Number(process.env.HAR_SYNC_OVERLAP_MS);
+  const overlap = Number.isFinite(raw) && raw >= 0 ? raw : DEFAULT_SYNC_OVERLAP_MS;
+  return new Date(new Date(since).getTime() - overlap).toISOString();
+}
+function runsWatermarkTarget(target) {
+  return `runs:${target}`;
+}
+async function syncRunBatches(newRuns, repoPath, runsTarget, getRepoId, dryRun, send) {
+  const ordered = [...newRuns].sort((a2, b3) => runTimestamp(a2).localeCompare(runTimestamp(b3)));
+  const batches = chunkBySerializedSize(ordered, maxSyncBatchBytes());
+  const toSend = batches.length > 0 ? batches : [[]];
+  for (let i2 = 0; i2 < toSend.length; i2++) {
+    const batch = toSend[i2];
+    await send(batch, i2);
+    if (!dryRun && batch.length > 0) {
+      writeRunsWatermark(repoPath, runsTarget, getRepoId(), runTimestamp(batch[batch.length - 1]));
+    }
+  }
+}
+function describeFetchFailure(url, bodyBytes, err) {
+  const base = err instanceof Error ? err.message : String(err);
+  const cause = err.cause;
+  const code = cause?.code ? ` [${cause.code}]` : "";
+  const detail = cause?.message && cause.message !== base ? `: ${cause.message}` : "";
+  const size = bodyBytes > 0 ? ` (request body ${(bodyBytes / 1024 / 1024).toFixed(2)} MB)` : "";
+  return `${base}${code}${detail}${size} \u2014 POST ${url}`;
+}
+async function postForResponse(url, headers, payload) {
+  let lastErr;
+  for (let attempt = 1; attempt <= MAX_FETCH_ATTEMPTS; attempt++) {
+    try {
+      return await fetch(url, { method: "POST", headers, body: payload });
+    } catch (err) {
+      lastErr = err;
+      if (attempt < MAX_FETCH_ATTEMPTS) {
+        await new Promise((resolve47) => setTimeout(resolve47, attempt * 100));
+      }
+    }
+  }
+  throw new Error(describeFetchFailure(url, Buffer.byteLength(payload, "utf8"), lastErr));
+}
 async function postJson(url, body, dryRun) {
   if (dryRun) return null;
-  const response = await fetch(url, {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify(body)
-  });
+  const response = await postForResponse(
+    url,
+    { "Content-Type": "application/json" },
+    JSON.stringify(body)
+  );
   if (!response.ok) {
     const text = await response.text().catch(() => "");
     throw new Error(`Control API ${response.status}: ${text || response.statusText}`);
   }
   return await response.json();
 }
-async function postPortal(target, endpoint, body, dryRun) {
-  if (dryRun) return;
-  const response = await fetch(`${target.url}${endpoint}`, {
-    method: "POST",
-    headers: {
+function postPortalOnce(target, endpoint, body) {
+  return postForResponse(
+    `${target.url}${endpoint}`,
+    {
       "Content-Type": "application/json",
       Authorization: `Bearer ${target.token}`
     },
-    body: JSON.stringify(body)
-  });
+    JSON.stringify(body)
+  );
+}
+async function refreshPortalToken(target) {
+  if (!target.refreshToken) return false;
+  let response;
+  try {
+    response = await fetch(`${target.url}/api/cli/refresh`, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${target.refreshToken}` }
+    });
+  } catch {
+    return false;
+  }
+  if (!response.ok) return false;
+  const data = await response.json().catch(() => null);
+  if (!data?.token) return false;
+  target.token = data.token;
+  const stored = readPortalCredentials();
+  if (stored) {
+    writePortalCredentials({ ...stored, token: data.token, expiresAt: data.expiresAt });
+  }
+  return true;
+}
+async function postPortal(target, endpoint, body, dryRun) {
+  if (dryRun) return null;
+  let response = await postPortalOnce(target, endpoint, body);
+  if ((response.status === 401 || response.status === 403) && await refreshPortalToken(target)) {
+    response = await postPortalOnce(target, endpoint, body);
+  }
   if (!response.ok) {
     if (response.status === 401 || response.status === 403) {
       throw new Error(
-        `har-portal rejected the ingest token (HTTP ${response.status}) \u2014 check HAR_PORTAL_TOKEN.`
+        `har-portal rejected the ingest token (HTTP ${response.status}) \u2014 run \`har control login\` (or check HAR_PORTAL_TOKEN).`
       );
     }
     const text = await response.text().catch(() => "");
     throw new Error(`har-portal sync failed: HTTP ${response.status}${text ? ` \u2014 ${text}` : ""}`);
   }
+  return await response.json().catch(() => null);
 }
 function resolvePortalUserEmail() {
   return readPortalCredentials()?.email || void 0;
@@ -31348,7 +31490,9 @@ async function collectPortalTelemetry(repoPath, slots, controlApiUrl, since) {
       }))
     );
     const persisted = await fetchPersistedPortalTelemetry(repoPath, controlApiUrl, { since });
-    const usage2 = mergePortalUsage(liveUsage, persisted.usage).map((row) => {
+    const harvestedUsage = omitHarvestWhenOtelPresent(liveUsage, persisted.usage);
+    const harvestedEvents = omitHarvestEventsWhenOtelPresent(liveEvents, persisted.events);
+    const usage2 = mergePortalUsage(harvestedUsage, persisted.usage).map((row) => {
       const priced = enrichUsageWithPricing(row);
       const models = modelsFromBreakdown(
         priced.modelBreakdown
@@ -31359,7 +31503,7 @@ async function collectPortalTelemetry(repoPath, slots, controlApiUrl, since) {
         ...models.length > 0 ? { models } : {}
       };
     });
-    const events = dedupePortalEvents(liveEvents, persisted.events).map(
+    const events = dedupePortalEvents(harvestedEvents, persisted.events).map(
       (event) => dropNullFields({ ...event })
     );
     return { usage: usage2, events, maxSyncedAt: persisted.maxSyncedAt };
@@ -31387,7 +31531,6 @@ async function buildPortalPayload(repoPath, controlApiUrl, since) {
     path: repoPath,
     ...manifest ? { manifest } : {},
     ...stagesRegistry ? { stagesRegistry } : {},
-    runs,
     slots: status.slots,
     generatedAt: status.generatedAt,
     ...validations.length > 0 ? { validations } : {},
@@ -31396,7 +31539,7 @@ async function buildPortalPayload(repoPath, controlApiUrl, since) {
     ...validationBindings.length > 0 ? { validationBindings } : {},
     ...usage2.length > 0 ? { usage: usage2 } : {}
   };
-  return { syncBody, events, maxSyncedAt };
+  return { syncBody, runs, events, maxSyncedAt };
 }
 async function isControlApiReachable(apiUrl = getControlApiUrl()) {
   try {
@@ -31434,7 +31577,7 @@ async function syncReposWithControl(options) {
     try {
       await withTimeout(
         syncRepoWithControl({ repoPath, apiUrl, dryRun: options.dryRun, cloud: options.cloud, full: options.full }),
-        SYNC_TIMEOUT_MS,
+        EXPLICIT_SYNC_TIMEOUT_MS,
         "control sync"
       );
       synced++;
@@ -31497,7 +31640,7 @@ async function ensureRepoRegisteredWithControl(repoPath, apiUrl) {
 }
 async function syncRepoWithLocalControl(options) {
   const apiUrl = options.apiUrl ?? getControlApiUrl();
-  const { repoPath, dryRun } = options;
+  const { repoPath, dryRun, full } = options;
   const registerResult = await registerRepoWithControl({ ...options, repoPath });
   const repoId = registerResult?.id;
   if (!repoId && !dryRun) {
@@ -31508,11 +31651,11 @@ async function syncRepoWithLocalControl(options) {
     if (!existing) {
       return;
     }
-    await syncRepoRunsAndSlots(apiUrl, existing.id, repoPath, dryRun);
+    await syncRepoRunsAndSlots(apiUrl, existing.id, repoPath, dryRun, full);
     return;
   }
   if (repoId) {
-    await syncRepoRunsAndSlots(apiUrl, repoId, repoPath, dryRun);
+    await syncRepoRunsAndSlots(apiUrl, repoId, repoPath, dryRun, full);
   }
 }
 async function syncRepoWithPortal(options, controlApiUrl) {
@@ -31520,10 +31663,28 @@ async function syncRepoWithPortal(options, controlApiUrl) {
   if (!portal) return;
   const { repoPath, dryRun, full } = options;
   const since = full ? null : readPortalWatermark(repoPath, portal.url);
-  const { syncBody, events, maxSyncedAt } = await buildPortalPayload(repoPath, controlApiUrl, since);
-  await postPortal(portal, "/api/sync", syncBody, dryRun);
-  if (events.length > 0) {
-    await postPortal(portal, "/api/otel", { path: repoPath, events, spans: [] }, dryRun);
+  const runsTarget = runsWatermarkTarget(portal.url);
+  const stored = full ? null : readRunsWatermarkEntry(repoPath, runsTarget);
+  const runsSince = rewindSince(stored?.lastSyncedAt ?? null);
+  const { syncBody, runs, events, maxSyncedAt } = await buildPortalPayload(
+    repoPath,
+    controlApiUrl,
+    since
+  );
+  const { selected: newRuns } = selectSince(runs, runsSince, runTimestamp);
+  let repoId;
+  await syncRunBatches(newRuns, repoPath, runsTarget, () => repoId, dryRun, async (batch, i2) => {
+    const body = i2 === 0 ? { ...syncBody, runs: batch } : { path: repoPath, runs: batch };
+    const res = await postPortal(portal, "/api/sync", body, dryRun);
+    if (i2 === 0 && typeof res?.repositoryId === "string") repoId = res.repositoryId;
+  });
+  if (!full && !dryRun && stored?.repoId && repoId && stored.repoId !== repoId) {
+    await syncRunBatches(listRuns(repoPath), repoPath, runsTarget, () => repoId, dryRun, async (batch) => {
+      await postPortal(portal, "/api/sync", { path: repoPath, runs: batch }, dryRun);
+    });
+  }
+  for (const batch of chunkBySerializedSize(events, maxSyncBatchBytes())) {
+    await postPortal(portal, "/api/otel", { path: repoPath, events: batch, spans: [] }, dryRun);
   }
   if (!dryRun && maxSyncedAt) {
     writePortalWatermark(repoPath, portal.url, maxSyncedAt);
@@ -31564,10 +31725,15 @@ async function syncRepoWithControl(options) {
   await syncRepoWithLocalControl({ ...options, repoPath, apiUrl });
   await syncRepoWithPortal({ ...options, repoPath }, apiUrl);
 }
-async function syncRepoRunsAndSlots(apiUrl, repoId, repoPath, dryRun) {
+async function syncRepoRunsAndSlots(apiUrl, repoId, repoPath, dryRun, full) {
+  const runsTarget = runsWatermarkTarget(apiUrl);
+  const stored = full ? null : readRunsWatermarkEntry(repoPath, runsTarget);
+  const runsSince = rewindSince(stored?.repoId === repoId ? stored?.lastSyncedAt ?? null : null);
   const runs = listRuns(repoPath);
-  const runsBody = SyncRunsInputSchema.parse({ runs });
-  await postJson(`${apiUrl}/api/repos/${repoId}/runs`, runsBody, dryRun);
+  const { selected: newRuns } = selectSince(runs, runsSince, runTimestamp);
+  await syncRunBatches(newRuns, repoPath, runsTarget, () => repoId, dryRun, async (batch) => {
+    await postJson(`${apiUrl}/api/repos/${repoId}/runs`, SyncRunsInputSchema.parse({ runs: batch }), dryRun);
+  });
   const status = collectEnvironmentStatus(repoPath);
   const slotsBody = SyncSlotsInputSchema.parse({
     slots: status.slots,
@@ -31722,7 +31888,7 @@ async function syncRunWithControlAsync(repoPath, run2) {
 `);
   }
 }
-var path27, SYNC_TIMEOUT_MS;
+var path27, DEFAULT_MAX_BATCH_BYTES, DEFAULT_SYNC_OVERLAP_MS, MAX_FETCH_ATTEMPTS, SYNC_TIMEOUT_MS, EXPLICIT_SYNC_TIMEOUT_MS;
 var init_control_sync = __esm({
   "src/core/control-sync.ts"() {
     "use strict";
@@ -31746,7 +31912,11 @@ var init_control_sync = __esm({
     init_portal_usage_merge();
     init_schema2();
     init_portal_watermark();
+    DEFAULT_MAX_BATCH_BYTES = 4 * 1024 * 1024;
+    DEFAULT_SYNC_OVERLAP_MS = 6e4;
+    MAX_FETCH_ATTEMPTS = 3;
     SYNC_TIMEOUT_MS = 15e3;
+    EXPLICIT_SYNC_TIMEOUT_MS = 12e4;
   }
 });
 
@@ -81407,6 +81577,8 @@ async function loginViaBrowser(portalUrl, openUrl = openBrowser) {
       const returnedState = requestUrl.searchParams.get("state");
       const workspace = requestUrl.searchParams.get("workspace") ?? void 0;
       const email2 = requestUrl.searchParams.get("email") ?? void 0;
+      const refreshToken = requestUrl.searchParams.get("refreshToken") ?? void 0;
+      const expiresAt = requestUrl.searchParams.get("expiresAt") ?? void 0;
       if (returnedState !== state || !token) {
         res.writeHead(400, { "Content-Type": "text/html" });
         res.end("<h1>Login failed</h1><p>You can close this tab and try again.</p>");
@@ -81417,7 +81589,15 @@ async function loginViaBrowser(portalUrl, openUrl = openBrowser) {
       res.writeHead(200, { "Content-Type": "text/html" });
       res.end("<h1>Logged in</h1><p>You can close this tab and return to the terminal.</p>");
       cleanup();
-      resolve47({ portalUrl, token, workspace, email: email2, createdAt: (/* @__PURE__ */ new Date()).toISOString() });
+      resolve47({
+        portalUrl,
+        token,
+        workspace,
+        email: email2,
+        refreshToken,
+        expiresAt,
+        createdAt: (/* @__PURE__ */ new Date()).toISOString()
+      });
     });
     const cleanup = () => {
       clearTimeout(timer);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@osfactory/har",
-  "version": "0.39.0",
+  "version": "0.41.0",
   "description": "LLM-powered CLI for reproducible agent development environments",
   "keywords": [
     "har",