@cesarus777/opencode-cgp 0.4.0 → 0.4.2

package/dist/core.js

@@ -173,6 +173,11 @@ function reconcile(a, b) {
   };
 }
 function analyzeModelRequests(records, options) {
+  const accumulator = createModelAccumulator(options);
+  accumulator.append(records);
+  return accumulator.result();
+}
+function createModelAccumulator(options) {
   const maxRequests = options.maxRequests ?? 1e5;
   if (!Number.isSafeInteger(maxRequests) || maxRequests < 1 || maxRequests > 1e6)
     throw new Error("maxRequests must be an integer from 1 to 1000000");
@@ -183,7 +188,7 @@ function analyzeModelRequests(records, options) {
       throw new Error("Invalid analytics timestamp");
   if (options.from !== undefined && options.to !== undefined && options.from > options.to)
     throw new Error("--from must precede --to");
-  const coverage = {
+  const counters = {
     source: "retained-request-records",
     scannedRecords: 0,
     unscannedRecords: 0,
@@ -197,74 +202,98 @@ function analyzeModelRequests(records, options) {
     partial: false
   };
   const requests = new Map;
-  for (const record of records) {
-    coverage.scannedRecords++;
-    if (record.type === "gap") {
-      coverage.gapRecords++;
-      continue;
-    }
-    if (record.type !== "request")
-      continue;
-    const fact = requestFact(record);
-    if (!fact) {
-      coverage.missingIdentityRecords++;
-      continue;
-    }
-    fact.location = options.location;
-    const key = JSON.stringify([
-      fact.sessionId,
-      fact.requestId,
-      fact.startedAt
-    ]);
-    const previous = requests.get(key);
-    if (previous) {
-      coverage.duplicateRecords++;
-      const merged = reconcile(previous.fact, fact);
-      if (!merged)
-        previous.conflict = true;
-      else
-        previous.fact = merged;
-    } else {
-      if (requests.size === maxRequests) {
-        coverage.scannedRecords--;
-        coverage.unscannedRecords = records.length - coverage.scannedRecords;
+  let totalRecords = 0, stopped = false;
+  function append(records) {
+    totalRecords += records.length;
+    for (const record of records) {
+      if (stopped)
         break;
+      counters.scannedRecords++;
+      if (record.type === "gap") {
+        counters.gapRecords++;
+        continue;
+      }
+      if (record.type !== "request")
+        continue;
+      const fact = requestFact(record);
+      if (!fact) {
+        counters.missingIdentityRecords++;
+        continue;
+      }
+      fact.location = options.location;
+      const key = JSON.stringify([
+        fact.sessionId,
+        fact.requestId,
+        fact.startedAt
+      ]);
+      const previous = requests.get(key);
+      if (previous) {
+        counters.duplicateRecords++;
+        const merged = reconcile(previous.fact, fact);
+        if (!merged)
+          previous.conflict = true;
+        else
+          previous.fact = merged;
+      } else {
+        if (requests.size === maxRequests) {
+          counters.scannedRecords--;
+          stopped = true;
+          break;
+        }
+        requests.set(key, { fact, conflict: false });
       }
-      requests.set(key, { fact, conflict: false });
     }
   }
-  const accepted = [];
-  for (const { fact: r, conflict } of requests.values()) {
-    if (conflict) {
-      coverage.conflictingRequests++;
-      continue;
-    }
-    if (r.completedAt === undefined) {
-      coverage.incompleteRequests++;
-      continue;
-    }
-    const duration = r.completedAt - r.startedAt;
-    if (!nonnegative(r.completedAt) || !Number.isFinite(duration) || duration <= 0) {
-      coverage.invalidTimingRequests++;
-      continue;
+  function* steps() {
+    const coverage = {
+      ...counters,
+      unscannedRecords: totalRecords - counters.scannedRecords
+    };
+    const accepted = [];
+    for (const { fact: r, conflict } of requests.values()) {
+      yield;
+      if (conflict) {
+        coverage.conflictingRequests++;
+        continue;
+      }
+      if (r.completedAt === undefined) {
+        coverage.incompleteRequests++;
+        continue;
+      }
+      const duration = r.completedAt - r.startedAt;
+      if (!nonnegative(r.completedAt) || !Number.isFinite(duration) || duration <= 0) {
+        coverage.invalidTimingRequests++;
+        continue;
+      }
+      if (options.from !== undefined && r.completedAt < options.from || options.to !== undefined && r.completedAt >= options.to || options.model !== undefined && r.model !== options.model || options.agent !== undefined && r.agent !== options.agent)
+        continue;
+      accepted.push({ ...r, completedAt: r.completedAt });
     }
-    if (options.from !== undefined && r.completedAt < options.from || options.to !== undefined && r.completedAt >= options.to || options.model !== undefined && r.model !== options.model || options.agent !== undefined && r.agent !== options.agent)
-      continue;
-    accepted.push({ ...r, completedAt: r.completedAt });
+    const models = yield* summarizeModelRequestSteps(accepted);
+    coverage.partial = coverage.unscannedRecords > 0 || coverage.conflictingRequests > 0 || coverage.missingIdentityRecords > 0 || coverage.incompleteRequests > 0 || coverage.invalidTimingRequests > 0 || coverage.gapRecords > 0 || models.some((s) => s.partialRequests > 0 || s.legacyAttributionRequests > 0 || s.excludedTpsRequests > 0 || s.excludedTtftRequests > 0 || !s.model);
+    return {
+      schemaVersion: 1,
+      location: options.location,
+      models,
+      coverage,
+      requests: accepted
+    };
   }
-  const models = summarizeModelRequests(accepted);
-  coverage.partial = coverage.unscannedRecords > 0 || coverage.conflictingRequests > 0 || coverage.missingIdentityRecords > 0 || coverage.incompleteRequests > 0 || coverage.invalidTimingRequests > 0 || coverage.gapRecords > 0 || models.some((s) => s.partialRequests > 0 || s.legacyAttributionRequests > 0 || s.excludedTpsRequests > 0 || s.excludedTtftRequests > 0 || !s.model);
-  return {
-    schemaVersion: 1,
-    location: options.location,
-    models,
-    coverage,
-    requests: accepted
-  };
+  return { append, steps, result: () => finishSteps(steps()) };
+}
+function finishSteps(steps) {
+  let step = steps.next();
+  while (!step.done)
+    step = steps.next();
+  return step.value;
 }
 function summarizeModelRequests(requests) {
+  return finishSteps(summarizeModelRequestSteps(requests));
+}
+function* summarizeModelRequestSteps(requests) {
   const groups = new Map;
   for (const r of requests) {
+    yield;
     const duration = r.completedAt - r.startedAt;
     let group = groups.get(r.model);
     if (!group) {