@openpond/evals 0.11.20 → 0.12.0

package/dist/native-conversations/collector-source-cache.js

@@ -0,0 +1,41 @@
+import { contentHash } from "@openpond/harness";
+import { CollectorCoverageSessionSchema } from "../connected-evidence/collector-coverage-contracts.js";
+/** Reuse a settled source read only for the same revision and event-time window.
+ * Admission remains owned by the durable queue and boundary checkpoints. */
+export class CollectorSourceCache {
+    store;
+    constructor(store) {
+        this.store = store;
+        store.database.exec(`CREATE TABLE IF NOT EXISTS source_read_cache(
+      connection_id TEXT NOT NULL,session_key TEXT NOT NULL,selection_hash TEXT NOT NULL,value TEXT NOT NULL,
+      PRIMARY KEY(connection_id,session_key));`);
+    }
+    selection(connection, until, revision) {
+        return contentHash([connection.source, connection.projectId, connection.teamId,
+            connection.apiBaseUrl, connection.account ?? null, connection.since, until, revision]);
+    }
+    read(connection, until, sessionKey, revision) {
+        const row = this.store.database.prepare("SELECT value FROM source_read_cache WHERE connection_id=? AND session_key=? AND selection_hash=?").get(connection.id, sessionKey, this.selection(connection, until, revision));
+        return row ? CollectorCoverageSessionSchema.parse(JSON.parse(String(row.value))) : null;
+    }
+    retain(connection, until, sessionKey, revision, raw) {
+        const value = CollectorCoverageSessionSchema.parse(raw);
+        if (value.nativeRevisionHash === null || (value.state !== "admitted"
+            && !(value.state === "skipped" && value.reason === "no_eligible_boundaries")))
+            return;
+        this.store.database.prepare("INSERT INTO source_read_cache VALUES(?,?,?,?) ON CONFLICT(connection_id,session_key) DO UPDATE SET selection_hash=excluded.selection_hash,value=excluded.value").run(connection.id, sessionKey, this.selection(connection, until, revision), JSON.stringify(value));
+    }
+    /** Earlier collectors retained both scan revisions and settled coverage.
+     * Preserve that evidence when upgrading a partially completed import. */
+    seed(connection, until, prior) {
+        if (!prior || prior.connectionRevision !== connection.revision || prior.from !== connection.since
+            || prior.to !== until || prior.sourceInstanceId !== connection.source.instanceId || prior.machineId !== connection.source.machineId)
+            return;
+        const byKey = new Map(prior.sessions.map(session => [session.sourceKeyHash, session]));
+        for (const row of this.store.database.prepare("SELECT session_key,revision FROM source_scans WHERE connection_id=?").all(connection.id)) {
+            const key = String(row.session_key), value = byKey.get(contentHash(key));
+            if (value)
+                this.retain(connection, until, key, String(row.revision), value);
+        }
+    }
+}

package/dist/native-conversations/collector-store.js

@@ -4,12 +4,19 @@ import { join } from "node:path";
 import { COLLECTOR_DEFAULTS } from "./collector-contracts.js";
 import { collectorDestinationLinks } from "./collector-destinations.js";
 import { CollectorProgress } from "./collector-progress.js";
+import { CollectorCoverage } from "./collector-coverage.js";
+import { CollectorSnapshots, CollectorQueueFullError } from "./collector-snapshots.js";
+import { nextScheduledSync, validateCollectorSchedule } from "./collector-schedule.js";
 export class CollectorStore {
     database;
     progress;
+    snapshots;
+    coverage;
     constructor(database) {
         this.database = database;
         this.progress = new CollectorProgress(database);
+        this.snapshots = new CollectorSnapshots(database);
+        this.coverage = new CollectorCoverage(this);
     }
     static async open(directory) {
         await mkdir(directory, { recursive: true, mode: 0o700 });
@@ -63,6 +70,8 @@ export class CollectorStore {
             if (identityChanged)
                 connection = { ...connection, destinations: undefined };
             if (prior && (identityChanged || prior.since !== connection.since)) {
+                this.database.prepare("DELETE FROM settings WHERE key=?").run(`coverage:${connection.id}`);
+                this.database.prepare("DELETE FROM coverage_sessions WHERE connection_id=?").run(connection.id);
                 // Scope changes fence queued evidence too: do not upload a wider old
                 // selection under newly narrowed consent. Reacquire the approved scope.
                 this.progress.reset(connection.id);
@@ -76,6 +85,7 @@ export class CollectorStore {
                     this.database
                         .prepare(`DELETE FROM ${table} WHERE connection_id=?`)
                         .run(connection.id);
+                this.snapshots.collect();
                 if (identityChanged) {
                     this.database
                         .prepare("DELETE FROM admitted_boundaries WHERE connection_id=?")
@@ -134,16 +144,22 @@ export class CollectorStore {
             .prepare("SELECT revision FROM checkpoints WHERE connection_id=? AND session_key=? AND boundary_id=?")
             .get(connectionId, sessionKey, boundaryId)?.revision !== revision);
     }
-    enqueue(entry, revisions, progressSessionKey) {
-        const payload = JSON.stringify(entry), bytes = Buffer.byteLength(payload);
+    enqueue(entry, revisions, progressSessionKey, snapshot = this.snapshots.prepare(entry.files)) {
+        const { files: _files, ...metadata } = entry;
+        const payload = JSON.stringify(metadata), bytes = Buffer.byteLength(payload);
         this.database.exec("BEGIN IMMEDIATE");
         try {
+            if (this.database.prepare("SELECT 1 FROM pending WHERE id=?").get(entry.operationId)) {
+                this.database.exec("COMMIT");
+                return;
+            }
             const usage = this.database
                 .prepare("SELECT COUNT(*) AS count,COALESCE(SUM(bytes),0) AS bytes FROM pending")
                 .get();
             if (Number(usage.count) >= COLLECTOR_DEFAULTS.maxPending ||
-                Number(usage.bytes) + bytes > COLLECTOR_DEFAULTS.maxBytes)
-                throw new Error("Buffer full. Restore connectivity before acquiring more history.");
+                Number(usage.bytes) + this.snapshots.bytes() + bytes + (this.snapshots.exists(snapshot.hash) ? 0 : snapshot.bytes) > COLLECTOR_DEFAULTS.maxBytes)
+                throw new CollectorQueueFullError();
+            this.snapshots.retain(entry.operationId, snapshot);
             this.database
                 .prepare("INSERT OR IGNORE INTO pending(id,connection_id,payload,bytes,created_at) VALUES(?,?,?,?,?)")
                 .run(entry.operationId, entry.connectionId, payload, bytes, Date.now());
@@ -161,11 +177,11 @@ export class CollectorStore {
     }
     next(connectionId) {
         const row = this.database
-            .prepare("SELECT payload,attempts FROM pending WHERE connection_id=? AND retry_at <= ? ORDER BY created_at,id LIMIT 1")
-            .get(connectionId, Date.now());
+            .prepare("SELECT id,payload,attempts FROM pending WHERE connection_id=? ORDER BY created_at,id LIMIT 1")
+            .get(connectionId);
         return row
             ? {
-                entry: JSON.parse(String(row.payload)),
+                entry: { ...JSON.parse(String(row.payload)), files: this.snapshots.files(String(row.id)) },
                 attempts: Number(row.attempts),
             }
             : null;
@@ -177,6 +193,7 @@ export class CollectorStore {
                 .prepare("DELETE FROM pending WHERE id=?")
                 .run(entry.operationId);
             if (Number(removed.changes) > 0) {
+                this.snapshots.collect();
                 this.progress.acknowledge(entry.operationId);
                 this.set(`lastAdmission:${entry.connectionId}`, new Date().toISOString());
                 let admitted = 0;
@@ -200,6 +217,10 @@ export class CollectorStore {
             .prepare("UPDATE pending SET attempts=?,retry_at=? WHERE id=?")
             .run(attempts + 1, Date.now() + delay + Math.floor(Math.random() * Math.max(1, delay / 5)), id);
     }
+    schedule(id) {
+        const stored = this.setting(`schedule:${id}`);
+        return stored ? validateCollectorSchedule(JSON.parse(stored)) : null;
+    }
     status() {
         const heartbeatAt = this.setting("heartbeatAt"), pid = Number(this.setting("pid")) || null;
         let alive = false;
@@ -218,11 +239,19 @@ export class CollectorStore {
             desiredState: this.setting("desiredState") === "running" ? "running" : "stopped",
             pid,
             heartbeatAt,
+            timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
             connections: this.database
                 .prepare("SELECT id,config,admitted,error FROM connections ORDER BY id")
                 .all()
                 .map((row) => {
                 const config = JSON.parse(String(row.config));
+                const schedule = this.schedule(String(row.id));
+                const retainedRun = this.setting(`run:${row.id}`);
+                const run = retainedRun ? JSON.parse(retainedRun) : null;
+                if (run?.state === "running" && !alive) {
+                    run.state = "failed";
+                    run.error = "The import was interrupted before completion. Run Sync now to resume.";
+                }
                 return {
                     id: String(row.id),
                     state: config.state,
@@ -236,10 +265,16 @@ export class CollectorStore {
                     admitted: Number(row.admitted),
                     error: typeof row.error === "string" ? row.error : null,
                     backfill: this.progress.status(String(row.id)),
-                    pendingBytes: Number(this.database
+                    pendingBytes: this.snapshots.bytes(String(row.id)) + Number(this.database
                         .prepare("SELECT COALESCE(SUM(bytes),0) AS bytes FROM pending WHERE connection_id=?")
                         .get(String(row.id))?.bytes ?? 0),
                     lastAdmissionAt: this.setting(`lastAdmission:${row.id}`),
+                    since: config.since,
+                    schedule,
+                    nextRunAt: schedule && config.state === "active" ? nextScheduledSync(schedule) : null,
+                    lastSuccessfulSyncAt: this.setting(`lastSuccess:${row.id}`),
+                    run,
+                    coverage: this.coverage.summary(String(row.id)),
                     requestedSyncRevision: config.requestedSyncRevision ?? 0,
                     completedSyncRevision: config.completedSyncRevision ?? 0,
                     acknowledgedSyncRevision: Number(this.setting(`syncAcknowledged:${row.id}`)) || 0,