@osfactory/har 0.40.0 → 0.41.0

package/dist/index.js

@@ -31068,6 +31068,27 @@ function writePortalWatermark(repoPath, portalUrl, lastSyncedAt) {
   }
   writeState(state);
 }
+function readRunsWatermarkEntry(repoPath, target) {
+  const canonical = canonicalizeControlRepoPath(repoPath);
+  const entry = readState().states.find(
+    (state) => state.repoPath === canonical && state.portalUrl === target
+  );
+  return entry ? { lastSyncedAt: entry.lastSyncedAt, repoId: entry.repoId } : null;
+}
+function writeRunsWatermark(repoPath, target, repoId, lastSyncedAt) {
+  const canonical = canonicalizeControlRepoPath(repoPath);
+  const state = readState();
+  const existing = state.states.find(
+    (entry) => entry.repoPath === canonical && entry.portalUrl === target
+  );
+  if (existing) {
+    existing.lastSyncedAt = lastSyncedAt;
+    existing.repoId = repoId;
+  } else {
+    state.states.push({ repoPath: canonical, portalUrl: target, lastSyncedAt, repoId });
+  }
+  writeState(state);
+}
 function selectSince(rows, since, getTimestamp) {
   let maxSyncedAt = null;
   const selected = [];
@@ -31275,6 +31296,7 @@ var init_portal_usage_merge = __esm({
 // src/core/control-sync.ts
 var control_sync_exports = {};
 __export(control_sync_exports, {
+  chunkBySerializedSize: () => chunkBySerializedSize,
   discoverHarRepos: () => discoverHarRepos,
   ensureRepoRegisteredWithControl: () => ensureRepoRegisteredWithControl,
   isControlApiReachable: () => isControlApiReachable,
@@ -31286,13 +31308,80 @@ __export(control_sync_exports, {
   syncRunWithControlAsync: () => syncRunWithControlAsync,
   waitForControlApi: () => waitForControlApi
 });
+function maxSyncBatchBytes() {
+  const raw = Number(process.env.HAR_SYNC_MAX_BATCH_BYTES);
+  return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_MAX_BATCH_BYTES;
+}
+function chunkBySerializedSize(items, maxBytes) {
+  const batches = [];
+  let current = [];
+  let currentBytes = 0;
+  for (const item of items) {
+    const itemBytes = Buffer.byteLength(JSON.stringify(item), "utf8");
+    if (current.length > 0 && currentBytes + itemBytes > maxBytes) {
+      batches.push(current);
+      current = [];
+      currentBytes = 0;
+    }
+    current.push(item);
+    currentBytes += itemBytes;
+  }
+  if (current.length > 0) batches.push(current);
+  return batches;
+}
+function runTimestamp(run2) {
+  return run2.finishedAt ?? run2.startedAt;
+}
+function rewindSince(since) {
+  if (!since) return null;
+  const raw = Number(process.env.HAR_SYNC_OVERLAP_MS);
+  const overlap = Number.isFinite(raw) && raw >= 0 ? raw : DEFAULT_SYNC_OVERLAP_MS;
+  return new Date(new Date(since).getTime() - overlap).toISOString();
+}
+function runsWatermarkTarget(target) {
+  return `runs:${target}`;
+}
+async function syncRunBatches(newRuns, repoPath, runsTarget, getRepoId, dryRun, send) {
+  const ordered = [...newRuns].sort((a2, b3) => runTimestamp(a2).localeCompare(runTimestamp(b3)));
+  const batches = chunkBySerializedSize(ordered, maxSyncBatchBytes());
+  const toSend = batches.length > 0 ? batches : [[]];
+  for (let i2 = 0; i2 < toSend.length; i2++) {
+    const batch = toSend[i2];
+    await send(batch, i2);
+    if (!dryRun && batch.length > 0) {
+      writeRunsWatermark(repoPath, runsTarget, getRepoId(), runTimestamp(batch[batch.length - 1]));
+    }
+  }
+}
+function describeFetchFailure(url, bodyBytes, err) {
+  const base = err instanceof Error ? err.message : String(err);
+  const cause = err.cause;
+  const code = cause?.code ? ` [${cause.code}]` : "";
+  const detail = cause?.message && cause.message !== base ? `: ${cause.message}` : "";
+  const size = bodyBytes > 0 ? ` (request body ${(bodyBytes / 1024 / 1024).toFixed(2)} MB)` : "";
+  return `${base}${code}${detail}${size} \u2014 POST ${url}`;
+}
+async function postForResponse(url, headers, payload) {
+  let lastErr;
+  for (let attempt = 1; attempt <= MAX_FETCH_ATTEMPTS; attempt++) {
+    try {
+      return await fetch(url, { method: "POST", headers, body: payload });
+    } catch (err) {
+      lastErr = err;
+      if (attempt < MAX_FETCH_ATTEMPTS) {
+        await new Promise((resolve47) => setTimeout(resolve47, attempt * 100));
+      }
+    }
+  }
+  throw new Error(describeFetchFailure(url, Buffer.byteLength(payload, "utf8"), lastErr));
+}
 async function postJson(url, body, dryRun) {
   if (dryRun) return null;
-  const response = await fetch(url, {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify(body)
-  });
+  const response = await postForResponse(
+    url,
+    { "Content-Type": "application/json" },
+    JSON.stringify(body)
+  );
   if (!response.ok) {
     const text = await response.text().catch(() => "");
     throw new Error(`Control API ${response.status}: ${text || response.statusText}`);
@@ -31300,14 +31389,14 @@ async function postJson(url, body, dryRun) {
   return await response.json();
 }
 function postPortalOnce(target, endpoint, body) {
-  return fetch(`${target.url}${endpoint}`, {
-    method: "POST",
-    headers: {
+  return postForResponse(
+    `${target.url}${endpoint}`,
+    {
       "Content-Type": "application/json",
       Authorization: `Bearer ${target.token}`
     },
-    body: JSON.stringify(body)
-  });
+    JSON.stringify(body)
+  );
 }
 async function refreshPortalToken(target) {
   if (!target.refreshToken) return false;
@@ -31331,7 +31420,7 @@ async function refreshPortalToken(target) {
   return true;
 }
 async function postPortal(target, endpoint, body, dryRun) {
-  if (dryRun) return;
+  if (dryRun) return null;
   let response = await postPortalOnce(target, endpoint, body);
   if ((response.status === 401 || response.status === 403) && await refreshPortalToken(target)) {
     response = await postPortalOnce(target, endpoint, body);
@@ -31345,6 +31434,7 @@ async function postPortal(target, endpoint, body, dryRun) {
     const text = await response.text().catch(() => "");
     throw new Error(`har-portal sync failed: HTTP ${response.status}${text ? ` \u2014 ${text}` : ""}`);
   }
+  return await response.json().catch(() => null);
 }
 function resolvePortalUserEmail() {
   return readPortalCredentials()?.email || void 0;
@@ -31441,7 +31531,6 @@ async function buildPortalPayload(repoPath, controlApiUrl, since) {
     path: repoPath,
     ...manifest ? { manifest } : {},
     ...stagesRegistry ? { stagesRegistry } : {},
-    runs,
     slots: status.slots,
     generatedAt: status.generatedAt,
     ...validations.length > 0 ? { validations } : {},
@@ -31450,7 +31539,7 @@ async function buildPortalPayload(repoPath, controlApiUrl, since) {
     ...validationBindings.length > 0 ? { validationBindings } : {},
     ...usage2.length > 0 ? { usage: usage2 } : {}
   };
-  return { syncBody, events, maxSyncedAt };
+  return { syncBody, runs, events, maxSyncedAt };
 }
 async function isControlApiReachable(apiUrl = getControlApiUrl()) {
   try {
@@ -31488,7 +31577,7 @@ async function syncReposWithControl(options) {
     try {
       await withTimeout(
         syncRepoWithControl({ repoPath, apiUrl, dryRun: options.dryRun, cloud: options.cloud, full: options.full }),
-        SYNC_TIMEOUT_MS,
+        EXPLICIT_SYNC_TIMEOUT_MS,
         "control sync"
       );
       synced++;
@@ -31551,7 +31640,7 @@ async function ensureRepoRegisteredWithControl(repoPath, apiUrl) {
 }
 async function syncRepoWithLocalControl(options) {
   const apiUrl = options.apiUrl ?? getControlApiUrl();
-  const { repoPath, dryRun } = options;
+  const { repoPath, dryRun, full } = options;
   const registerResult = await registerRepoWithControl({ ...options, repoPath });
   const repoId = registerResult?.id;
   if (!repoId && !dryRun) {
@@ -31562,11 +31651,11 @@ async function syncRepoWithLocalControl(options) {
     if (!existing) {
       return;
     }
-    await syncRepoRunsAndSlots(apiUrl, existing.id, repoPath, dryRun);
+    await syncRepoRunsAndSlots(apiUrl, existing.id, repoPath, dryRun, full);
     return;
   }
   if (repoId) {
-    await syncRepoRunsAndSlots(apiUrl, repoId, repoPath, dryRun);
+    await syncRepoRunsAndSlots(apiUrl, repoId, repoPath, dryRun, full);
   }
 }
 async function syncRepoWithPortal(options, controlApiUrl) {
@@ -31574,10 +31663,28 @@ async function syncRepoWithPortal(options, controlApiUrl) {
   if (!portal) return;
   const { repoPath, dryRun, full } = options;
   const since = full ? null : readPortalWatermark(repoPath, portal.url);
-  const { syncBody, events, maxSyncedAt } = await buildPortalPayload(repoPath, controlApiUrl, since);
-  await postPortal(portal, "/api/sync", syncBody, dryRun);
-  if (events.length > 0) {
-    await postPortal(portal, "/api/otel", { path: repoPath, events, spans: [] }, dryRun);
+  const runsTarget = runsWatermarkTarget(portal.url);
+  const stored = full ? null : readRunsWatermarkEntry(repoPath, runsTarget);
+  const runsSince = rewindSince(stored?.lastSyncedAt ?? null);
+  const { syncBody, runs, events, maxSyncedAt } = await buildPortalPayload(
+    repoPath,
+    controlApiUrl,
+    since
+  );
+  const { selected: newRuns } = selectSince(runs, runsSince, runTimestamp);
+  let repoId;
+  await syncRunBatches(newRuns, repoPath, runsTarget, () => repoId, dryRun, async (batch, i2) => {
+    const body = i2 === 0 ? { ...syncBody, runs: batch } : { path: repoPath, runs: batch };
+    const res = await postPortal(portal, "/api/sync", body, dryRun);
+    if (i2 === 0 && typeof res?.repositoryId === "string") repoId = res.repositoryId;
+  });
+  if (!full && !dryRun && stored?.repoId && repoId && stored.repoId !== repoId) {
+    await syncRunBatches(listRuns(repoPath), repoPath, runsTarget, () => repoId, dryRun, async (batch) => {
+      await postPortal(portal, "/api/sync", { path: repoPath, runs: batch }, dryRun);
+    });
+  }
+  for (const batch of chunkBySerializedSize(events, maxSyncBatchBytes())) {
+    await postPortal(portal, "/api/otel", { path: repoPath, events: batch, spans: [] }, dryRun);
   }
   if (!dryRun && maxSyncedAt) {
     writePortalWatermark(repoPath, portal.url, maxSyncedAt);
@@ -31618,10 +31725,15 @@ async function syncRepoWithControl(options) {
   await syncRepoWithLocalControl({ ...options, repoPath, apiUrl });
   await syncRepoWithPortal({ ...options, repoPath }, apiUrl);
 }
-async function syncRepoRunsAndSlots(apiUrl, repoId, repoPath, dryRun) {
+async function syncRepoRunsAndSlots(apiUrl, repoId, repoPath, dryRun, full) {
+  const runsTarget = runsWatermarkTarget(apiUrl);
+  const stored = full ? null : readRunsWatermarkEntry(repoPath, runsTarget);
+  const runsSince = rewindSince(stored?.repoId === repoId ? stored?.lastSyncedAt ?? null : null);
   const runs = listRuns(repoPath);
-  const runsBody = SyncRunsInputSchema.parse({ runs });
-  await postJson(`${apiUrl}/api/repos/${repoId}/runs`, runsBody, dryRun);
+  const { selected: newRuns } = selectSince(runs, runsSince, runTimestamp);
+  await syncRunBatches(newRuns, repoPath, runsTarget, () => repoId, dryRun, async (batch) => {
+    await postJson(`${apiUrl}/api/repos/${repoId}/runs`, SyncRunsInputSchema.parse({ runs: batch }), dryRun);
+  });
   const status = collectEnvironmentStatus(repoPath);
   const slotsBody = SyncSlotsInputSchema.parse({
     slots: status.slots,
@@ -31776,7 +31888,7 @@ async function syncRunWithControlAsync(repoPath, run2) {
 `);
   }
 }
-var path27, SYNC_TIMEOUT_MS;
+var path27, DEFAULT_MAX_BATCH_BYTES, DEFAULT_SYNC_OVERLAP_MS, MAX_FETCH_ATTEMPTS, SYNC_TIMEOUT_MS, EXPLICIT_SYNC_TIMEOUT_MS;
 var init_control_sync = __esm({
   "src/core/control-sync.ts"() {
     "use strict";
@@ -31800,7 +31912,11 @@ var init_control_sync = __esm({
     init_portal_usage_merge();
     init_schema2();
     init_portal_watermark();
+    DEFAULT_MAX_BATCH_BYTES = 4 * 1024 * 1024;
+    DEFAULT_SYNC_OVERLAP_MS = 6e4;
+    MAX_FETCH_ATTEMPTS = 3;
     SYNC_TIMEOUT_MS = 15e3;
+    EXPLICIT_SYNC_TIMEOUT_MS = 12e4;
   }
 });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@osfactory/har",
-  "version": "0.40.0",
+  "version": "0.41.0",
   "description": "LLM-powered CLI for reproducible agent development environments",
   "keywords": [
     "har",