@openpond/evals 0.11.20 → 0.12.0

package/dist/javascript-verifier.js

@@ -1,6 +1,7 @@
 import { executeJavaScriptIsolate } from "./javascript-isolate.js";
 import { assertIsolatedVerifierRuntime, JavaScriptVerifierResultSchema } from "./javascript-verifier-contract.js";
 export { JavaScriptVerifierResultSchema } from "./javascript-verifier-contract.js";
+export { PREPARED_PROCESS_ISOLATION_SOURCE } from "./prepared-process-isolation.js";
 /** A fresh interpreter with no filesystem, network, timers or module loader.
  * Hosts should use the worker entrypoint for cancellation and responsiveness. */
 export async function executeJavaScriptVerifier(input) {

package/dist/native-conversations/codex-window-inventory.js

@@ -0,0 +1,96 @@
+import { createHash } from "node:crypto";
+import { open } from "node:fs/promises";
+import { CONNECTED_EVIDENCE_LIMITS } from "../connected-evidence/contracts.js";
+import { CodexMetadataChain } from "../connected-evidence/sources/codex-metadata.js";
+/** Streaming inventory may prove an irrelevant source without retaining it.
+ * It never truncates a selected conversation into an admissible snapshot. */
+export async function inspectCodexWindow(input) {
+    const handle = await open(input.path, "r");
+    try {
+        const initial = await handle.stat();
+        // This is an inventory I/O bound, independent of the smaller upload bound.
+        if (!initial.isFile() || initial.size > 512 * 1024 * 1024)
+            return null;
+        const digest = createHash("sha256");
+        const decoder = new TextDecoder("utf-8", { fatal: true });
+        const from = input.since === null ? -Infinity : Date.parse(input.since);
+        const to = Date.parse(input.until);
+        const metadata = new CodexMetadataChain();
+        let records = 0, bytes = 0;
+        let pending = [], pendingBytes = 0;
+        const inspect = (line) => {
+            if (++records > 1_000_000 || line.byteLength > CONNECTED_EVIDENCE_LIMITS.lineBytes)
+                return false;
+            let text;
+            try {
+                text = decoder.decode(line);
+            }
+            catch {
+                return false;
+            }
+            if (!text.trim())
+                return true;
+            let row;
+            try {
+                row = JSON.parse(text);
+            }
+            catch {
+                return false;
+            }
+            if (!row || typeof row !== "object" || Array.isArray(row))
+                return false;
+            const value = row;
+            const payload = value.payload && typeof value.payload === "object" && !Array.isArray(value.payload)
+                ? value.payload : {};
+            if (!metadata.accept(value) || (metadata.metadata && metadata.metadata.id !== input.sessionId))
+                return false;
+            // The Codex normalizer maps other native records to unknown events;
+            // they cannot create request boundaries. Missing metadata is still refused.
+            const request = value.type === "response_item" && payload.type === "message" && payload.role === "user"
+                || value.type === "event_msg" && payload.type === "user_message";
+            if (request) {
+                const timestamp = typeof value.timestamp === "string" ? Date.parse(value.timestamp) : NaN;
+                if (!Number.isFinite(timestamp) || timestamp >= from && timestamp < to)
+                    return false;
+            }
+            return true;
+        };
+        for await (const chunk of handle.createReadStream({ autoClose: false, highWaterMark: 256 * 1024 })) {
+            await input.check();
+            const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            bytes += buffer.byteLength;
+            if (bytes > 512 * 1024 * 1024)
+                return null;
+            digest.update(buffer);
+            let start = 0, end;
+            while ((end = buffer.indexOf(10, start)) !== -1) {
+                const part = buffer.subarray(start, end);
+                if (pendingBytes + part.byteLength > CONNECTED_EVIDENCE_LIMITS.lineBytes)
+                    return null;
+                const line = pending.length ? Buffer.concat([...pending, part], pendingBytes + part.byteLength) : part;
+                if (!inspect(line))
+                    return null;
+                pending = [];
+                pendingBytes = 0;
+                start = end + 1;
+            }
+            const tail = buffer.subarray(start);
+            if (tail.byteLength) {
+                pending.push(tail);
+                pendingBytes += tail.byteLength;
+            }
+            if (pendingBytes > CONNECTED_EVIDENCE_LIMITS.lineBytes)
+                return null;
+        }
+        if (pendingBytes && !inspect(Buffer.concat(pending, pendingBytes)))
+            return null;
+        const final = await handle.stat();
+        if (initial.dev !== final.dev || initial.ino !== final.ino || initial.size !== final.size
+            || initial.mtimeMs !== final.mtimeMs || initial.ctimeMs !== final.ctimeMs || bytes !== final.size)
+            throw new Error("Native source changed during window inventory.");
+        return metadata.metadata ? { sha256: digest.digest("hex") } : null;
+    }
+    finally {
+        await handle.close();
+    }
+}

package/dist/native-conversations/collector-acquisition.js

@@ -0,0 +1,149 @@
+import { contentHash } from "@openpond/harness";
+import { agentImportNormalizerRevision } from "../connected-evidence/imports.js";
+import { listSessions, readSession, inspectSessionWindow } from "./history.js";
+import { NativeSessionNotReadyError } from "./readiness.js";
+import { collectorBranchAnchor } from "./collector-branches.js";
+import { CollectorQueueFullError } from "./collector-snapshots.js";
+import { COLLECTOR_DEFAULTS } from "./collector-contracts.js";
+import { CollectorCoverage } from "./collector-coverage.js";
+import { CollectorSourceCache } from "./collector-source-cache.js";
+/** One pass over a frozen listing per page. Unchanged revisions are cheap; a
+ * failed source is tried again only in a later explicitly launched job. */
+export async function acquireCollectorSource(input) {
+    const { store, connection, check, drain, progress, until } = input;
+    const cache = new CollectorSourceCache(store);
+    let cursor, processed = 0, discovered = 0;
+    const failures = [];
+    do {
+        await check();
+        progress({ phase: "discovering" });
+        // File mtimes and native activity summaries are not event-time authority.
+        // Enumerate the source, then apply the consented window to native events.
+        const page = await listSessions(connection.source, { cursor, limit: 50 });
+        discovered += page.items.length;
+        if (discovered > 10_000)
+            throw new Error("Source inventory exceeds 10,000 sessions. Select a narrower import window.");
+        for (const native of page.items) {
+            const scanKey = JSON.stringify([native.path, native.nativeSessionId]);
+            store.coverage.record(connection.id, { sourceKeyHash: CollectorCoverage.sourceKey(scanKey),
+                nativeRevisionHash: native.storageRevision ? contentHash([native.storageRevision, agentImportNormalizerRevision(connection.source.source)]) : null,
+                state: "reading", reason: null, unknownTimeBoundaries: 0, normalized: [] });
+        }
+        progress({ discovered });
+        for (const native of page.items) {
+            await check();
+            // Do not parse another transcript while any retained admission is blocked.
+            await drain();
+            const queuedBytes = Number(store.database.prepare("SELECT COALESCE(SUM(bytes),0) AS n FROM pending").get()?.n) + store.snapshots.bytes();
+            if (queuedBytes >= COLLECTOR_DEFAULTS.maxBytes)
+                throw new CollectorQueueFullError();
+            progress({ phase: "reading" });
+            const scanKey = JSON.stringify([native.path, native.nativeSessionId]);
+            const scanRevision = native.storageRevision
+                ? contentHash([native.storageRevision, agentImportNormalizerRevision(connection.source.source)]) : null;
+            store.progress.select(connection.id, scanKey);
+            const cached = scanRevision ? cache.read(connection, until, scanKey, scanRevision) : null;
+            if (cached) {
+                store.coverage.record(connection.id, cached);
+                store.progress.read(connection.id, scanKey, cached.state === "admitted");
+                progress({ processed: ++processed });
+                continue;
+            }
+            const coverage = { sourceKeyHash: CollectorCoverage.sourceKey(scanKey), nativeRevisionHash: scanRevision,
+                state: "reading", reason: scanRevision ? null : "source_revision_unavailable", unknownTimeBoundaries: 0, normalized: [] };
+            try {
+                const outside = await inspectSessionWindow(connection.source, native, { since: connection.since, until, check });
+                if (outside) {
+                    coverage.nativeRevisionHash = contentHash(["codex-window-inventory-v1", outside.sha256]);
+                    coverage.state = "skipped";
+                    coverage.reason = "no_eligible_boundaries";
+                    store.coverage.record(connection.id, coverage);
+                    store.progress.read(connection.id, scanKey, false);
+                    if (scanRevision) {
+                        store.scanned(connection.id, scanKey, scanRevision);
+                        cache.retain(connection, until, scanKey, scanRevision, coverage);
+                    }
+                    continue;
+                }
+                const { files, preview, branchLeafId } = await readSession(connection.source, native, {
+                    branchAnchor: collectorBranchAnchor(store, connection.id, native.nativeSessionId),
+                });
+                coverage.nativeRevisionHash = contentHash([files, agentImportNormalizerRevision(connection.source.source)]);
+                await check();
+                const snapshot = store.snapshots.prepare(files);
+                let eligible = false;
+                for (const session of preview.sessions) {
+                    const key = JSON.stringify([connection.source.instanceId, session.sessionId, session.branchId]);
+                    store.progress.retained(connection.id, scanKey, key);
+                    const boundaries = session.boundaries.filter(boundary => {
+                        if (boundary.projection !== "turn" || boundary.terminal === "unknown")
+                            return false;
+                        const timestamp = session.events[boundary.start]?.occurredAt;
+                        if (!timestamp || !Number.isFinite(Date.parse(timestamp))) {
+                            coverage.unknownTimeBoundaries++;
+                            return false;
+                        }
+                        return Date.parse(timestamp) < Date.parse(until) && (!connection.since || Date.parse(timestamp) >= Date.parse(connection.since));
+                    });
+                    coverage.normalized.push({ sessionHash: session.contentHash, boundaries: boundaries.map(boundary => ({
+                            id: boundary.id, inputHash: boundary.inputHash, revisionHash: boundary.revisionHash, occurredAt: session.events[boundary.start].occurredAt,
+                        })).sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0) });
+                    store.coverage.record(connection.id, coverage);
+                    eligible ||= boundaries.length > 0;
+                    const changed = boundaries.filter(boundary => store.changed(connection.id, key, boundary.id, contentHash([boundary.inputHash, boundary.revisionHash])));
+                    for (let offset = 0; offset < changed.length; offset += COLLECTOR_DEFAULTS.batchCases) {
+                        await check();
+                        const selected = changed.slice(offset, offset + COLLECTOR_DEFAULTS.batchCases);
+                        const operationId = `sync-${contentHash([connection.id, key, selected.map(boundary => [boundary.id, boundary.inputHash, boundary.revisionHash])]).slice(0, 48)}`;
+                        store.enqueue({ operationId, connectionId: connection.id, sessionKey: key, contentHash: session.contentHash,
+                            files, boundaryIds: selected.map(boundary => boundary.id), ...(branchLeafId ? { branchLeafId } : {}) }, selected.map(boundary => ({ id: boundary.id, revision: contentHash([boundary.inputHash, boundary.revisionHash]) })), scanKey, snapshot);
+                    }
+                }
+                store.progress.read(connection.id, scanKey, eligible);
+                if (scanRevision)
+                    store.scanned(connection.id, scanKey, scanRevision);
+                await drain();
+                coverage.state = eligible ? "admitted" : "skipped";
+                coverage.reason = eligible ? null : "no_eligible_boundaries";
+                coverage.normalized.sort((a, b) => a.sessionHash < b.sessionHash ? -1 : a.sessionHash > b.sessionHash ? 1 : 0);
+                store.coverage.record(connection.id, coverage);
+                if (scanRevision)
+                    cache.retain(connection, until, scanKey, scanRevision, coverage);
+            }
+            catch (error) {
+                if (error instanceof NativeSessionNotReadyError) {
+                    coverage.state = "skipped";
+                    coverage.reason = "native_session_not_ready";
+                    store.coverage.record(connection.id, coverage);
+                    store.progress.read(connection.id, scanKey, false);
+                    if (scanRevision)
+                        store.scanned(connection.id, scanKey, scanRevision);
+                    continue;
+                }
+                // Capacity/lifecycle failures end this run instead of retrying all files.
+                if (error instanceof CollectorQueueFullError || error instanceof CollectorInterruptedError || error instanceof CollectorScopeChangedError)
+                    throw error;
+                const message = error instanceof Error ? error.message : "Unable to read session";
+                coverage.state = "failed";
+                coverage.reason = "source_read_failed";
+                store.coverage.record(connection.id, coverage);
+                store.progress.failed(connection.id, scanKey, message);
+                failures.push(`${native.nativeSessionId}: ${message}`);
+            }
+            finally {
+                progress({ processed: ++processed });
+            }
+            // Admit one source snapshot before moving on to the next session.
+            await drain();
+        }
+        cursor = page.nextCursor ?? undefined;
+    } while (cursor);
+    store.progress.discovered(connection.id);
+    store.coverage.finish(connection.id, true);
+    if (failures.length)
+        throw new Error(`${failures.length} session(s) need attention. ${failures[0]}`);
+}
+export class CollectorInterruptedError extends Error {
+}
+export class CollectorScopeChangedError extends Error {
+}

package/dist/native-conversations/collector-branches.js

@@ -61,6 +61,7 @@ export async function selectCollectorBranch(directory, connectionId, nativeSessi
             // Never admit previously queued sibling evidence after a changed choice.
             for (const table of ["source_scans", "checkpoints", "pending"])
                 store.database.prepare(`DELETE FROM ${table} WHERE connection_id=?`).run(connectionId);
+            store.snapshots.collect();
             const errors = new CollectorErrors(store);
             errors.set(connectionId, "source", errors.source(connectionId));
             store.progress.reset(connectionId);

package/dist/native-conversations/collector-calendar-gate.js

@@ -0,0 +1,19 @@
+import { importCalendarMatches, parseImportCron } from "./collector-cron.js";
+import { scheduleExpression } from "./collector-schedule.js";
+/** A tiny, dependency-free process for OS schedulers that cannot express cron.
+ * An unmatched calendar exits before importing the CLI or opening source files. */
+export function collectorCalendarGate(input) {
+    const calendars = input.schedules.map(schedule => parseImportCron(scheduleExpression(schedule)).calendar);
+    return `"use strict";
+const matches = ${importCalendarMatches.toString()};
+const calendars = ${JSON.stringify(calendars)};
+if (calendars.some(calendar => matches(calendar, new Date()))) {
+  const { spawn } = require("node:child_process");
+  const child = spawn(${JSON.stringify(input.executable)}, ${JSON.stringify(input.args)}, { stdio: "inherit", windowsHide: true });
+  process.on("SIGINT", () => child.kill("SIGINT"));
+  process.on("SIGTERM", () => child.kill("SIGTERM"));
+  child.on("error", error => { console.error(error.message); process.exitCode = 1; });
+  child.on("exit", (code, signal) => { process.exitCode = code ?? (signal === "SIGINT" ? 130 : 143); });
+}
+`;
+}

package/dist/native-conversations/collector-contracts.js

@@ -1,9 +1,10 @@
 export const COLLECTOR_DEFAULTS = {
-    reconcileMs: 30000,
     heartbeatMs: 10000,
     staleMs: 60000,
     maxPending: 10000,
-    maxBytes: 1024 * 1024 * 1024,
+    maxBytes: 128 * 1024 * 1024,
     batchCases: 20,
-    batchBytes: 1024 * 1024,
+    runMs: 60 * 60 * 1000,
+    requestMs: 60000,
+    attemptsPerRun: 3,
 };

package/dist/native-conversations/collector-coverage.js

@@ -0,0 +1,100 @@
+import { contentHash } from "@openpond/harness";
+import { CollectorCoverageManifestSchema, CollectorCoverageSessionSchema, summarizeCollectorCoverage, } from "../connected-evidence/collector-coverage-contracts.js";
+/** Retains the latest acquisition inventory independently of queue receipts.
+ * Native paths are hashed; this authority never stores transcript bytes. */
+export class CollectorCoverage {
+    store;
+    constructor(store) {
+        this.store = store;
+        store.database.exec(`CREATE TABLE IF NOT EXISTS coverage_sessions(
+      connection_id TEXT NOT NULL,source_key_hash TEXT NOT NULL,value TEXT NOT NULL,
+      PRIMARY KEY(connection_id,source_key_hash));`);
+    }
+    begin(connection, jobId, startedAt, to) {
+        const manifest = CollectorCoverageManifestSchema.parse({
+            schemaVersion: "openpond.collectorCoverageManifest.v1",
+            jobId,
+            connectionId: connection.id,
+            connectionRevision: connection.revision,
+            machineId: connection.source.machineId,
+            sourceInstanceId: connection.source.instanceId,
+            from: connection.since,
+            to,
+            startedAt,
+            finishedAt: null,
+            listingComplete: false,
+            pendingOperations: this.store.queued(connection.id),
+            sessions: [],
+        });
+        this.store.database.exec("BEGIN IMMEDIATE");
+        try {
+            this.store.database
+                .prepare("DELETE FROM coverage_sessions WHERE connection_id=?")
+                .run(connection.id);
+            this.store.set(`coverage:${connection.id}`, JSON.stringify(manifest));
+            this.store.database.exec("COMMIT");
+        }
+        catch (error) {
+            this.store.database.exec("ROLLBACK");
+            throw error;
+        }
+    }
+    record(connectionId, session) {
+        const value = CollectorCoverageSessionSchema.parse(session);
+        const serialized = JSON.stringify(value);
+        const prior = this.store.database
+            .prepare("SELECT COALESCE(SUM(length(value)),0) AS bytes FROM coverage_sessions WHERE connection_id=? AND source_key_hash!=?")
+            .get(connectionId, value.sourceKeyHash);
+        const boundaryUsage = this.store.database
+            .prepare("SELECT COALESCE(SUM(json_array_length(n.value,'$.boundaries')),0) AS n FROM coverage_sessions c,json_each(c.value,'$.normalized') n WHERE c.connection_id=? AND c.source_key_hash!=?")
+            .get(connectionId, value.sourceKeyHash);
+        if (Buffer.byteLength(serialized) + Number(prior?.bytes ?? 0) >
+            8 * 1024 * 1024 ||
+            value.normalized.reduce((sum, item) => sum + item.boundaries.length, 0) +
+                Number(boundaryUsage?.n ?? 0) >
+                20_000)
+            throw new Error("Source coverage exceeds its bounded inventory. Select a narrower import window.");
+        const count = Number(this.store.database
+            .prepare("SELECT COUNT(*) AS n FROM coverage_sessions WHERE connection_id=?")
+            .get(connectionId)?.n ?? 0);
+        const exists = this.store.database
+            .prepare("SELECT 1 FROM coverage_sessions WHERE connection_id=? AND source_key_hash=?")
+            .get(connectionId, value.sourceKeyHash);
+        if (!exists && count >= 10_000)
+            throw new Error("Source inventory exceeds 10,000 sessions. Select a narrower import window.");
+        this.store.database
+            .prepare("INSERT INTO coverage_sessions VALUES(?,?,?) ON CONFLICT(connection_id,source_key_hash) DO UPDATE SET value=excluded.value")
+            .run(connectionId, value.sourceKeyHash, serialized);
+    }
+    manifest(connectionId) {
+        const raw = this.store.setting(`coverage:${connectionId}`);
+        if (!raw)
+            return null;
+        const sessions = this.store.database
+            .prepare("SELECT value FROM coverage_sessions WHERE connection_id=? ORDER BY source_key_hash")
+            .all(connectionId)
+            .map((row) => CollectorCoverageSessionSchema.parse(JSON.parse(String(row.value))));
+        return CollectorCoverageManifestSchema.parse({
+            ...JSON.parse(raw),
+            pendingOperations: this.store.queued(connectionId),
+            sessions,
+        });
+    }
+    finish(connectionId, listingComplete) {
+        const manifest = this.manifest(connectionId);
+        if (!manifest)
+            throw new Error("Source coverage was not initialized.");
+        manifest.listingComplete ||= listingComplete;
+        manifest.finishedAt ??= new Date().toISOString();
+        const { sessions: _sessions, ...metadata } = CollectorCoverageManifestSchema.parse(manifest);
+        this.store.set(`coverage:${connectionId}`, JSON.stringify({ ...metadata, sessions: [] }));
+        return manifest;
+    }
+    summary(connectionId) {
+        const manifest = this.manifest(connectionId);
+        return manifest ? summarizeCollectorCoverage(manifest) : null;
+    }
+    static sourceKey(scanKey) {
+        return contentHash(scanKey);
+    }
+}

package/dist/native-conversations/collector-cron.js

@@ -0,0 +1,43 @@
+import { CronExpressionParser } from "cron-parser";
+/** Standard five-field cron: numbers, names, wildcards, lists, ranges and steps.
+ * Excludes seconds and Quartz/randomized extensions so runs stay deterministic. */
+export function parseImportCron(expression, now = new Date()) {
+    if (expression.length > 512)
+        throw new Error("Cron expressions must be at most 512 characters.");
+    const fields = expression.trim().split(/\s+/u);
+    if (fields.length !== 5)
+        throw new Error("Use five cron fields: minute hour day-of-month month day-of-week.");
+    const names = [[], [], [], ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"],
+        ["sun", "mon", "tue", "wed", "thu", "fri", "sat"]];
+    const numeric = fields.map((field, index) => field.replace(/[a-z]+/giu, name => {
+        const value = names[index].indexOf(name.toLowerCase());
+        if (value < 0)
+            throw new Error("Cron supports standard month/weekday names, numbers, *, lists, ranges and steps.");
+        return String(value + (index === 3 ? 1 : 0));
+    }));
+    if (numeric.some(field => !/^[\d*,/\-]+$/u.test(field)))
+        throw new Error("Cron supports numbers, *, lists, ranges and steps; second-level schedules are not supported.");
+    const normalized = numeric.join(" ");
+    const interval = CronExpressionParser.parse(normalized, {
+        currentDate: now, tz: Intl.DateTimeFormat().resolvedOptions().timeZone,
+    });
+    const cron = interval.fields;
+    if (cron.minute.values.length !== 1)
+        throw new Error("Imports support at most one scheduled run per hour. Choose a single cron minute, such as 0 or 30.");
+    const calendar = {
+        minute: [...cron.minute.values], hour: [...cron.hour.values], month: [...cron.month.values],
+        dayOfMonth: cron.dayOfMonth.values.map(Number), dayOfWeek: [...new Set(cron.dayOfWeek.values.map(value => Number(value) % 7))],
+        anyDayOfMonth: cron.dayOfMonth.isWildcard, anyDayOfWeek: cron.dayOfWeek.isWildcard,
+    };
+    return { expression: normalized, calendar, interval };
+}
+/** Deliberately self-contained: the same function is embedded in the tiny OS
+ * launcher, which has no package imports and never reads source transcripts. */
+export function importCalendarMatches(calendar, now) {
+    const dom = calendar.dayOfMonth.includes(now.getDate());
+    const dow = calendar.dayOfWeek.includes(now.getDay());
+    const day = (!calendar.anyDayOfMonth && !calendar.anyDayOfWeek && (dom || dow)) ||
+        (dom && calendar.anyDayOfWeek) || (calendar.anyDayOfMonth && !calendar.anyDayOfWeek && dow);
+    return day && calendar.month.includes(now.getMonth() + 1) &&
+        calendar.hour.includes(now.getHours()) && calendar.minute.includes(now.getMinutes());
+}

package/dist/native-conversations/collector-schedule.js

@@ -0,0 +1,61 @@
+import { importCalendarMatches, parseImportCron } from "./collector-cron.js";
+export const SYNC_WEEKDAYS = ["sun", "mon", "tue", "wed", "thu", "fri", "sat"];
+/** Calendar times follow the host timezone, including daylight-saving changes. */
+export function syncTime(value) {
+    if (!/^(?:[01]\d|2[0-3]):[0-5]\d$/u.test(value))
+        throw new Error("Use a sync time in 24-hour HH:MM format, such as 00:00.");
+    return value;
+}
+export function validateCollectorSchedule(schedule) {
+    if (schedule.frequency === "hourly")
+        return { frequency: schedule.frequency };
+    if (schedule.frequency === "cron") {
+        const parsed = parseImportCron(schedule.expression);
+        parsed.interval.next(); // Reject impossible calendars before installing an OS trigger.
+        return { frequency: "cron", expression: parsed.expression };
+    }
+    const time = syncTime(schedule.time);
+    if (schedule.frequency === "daily")
+        return { frequency: "daily", time };
+    if (schedule.frequency === "weekly" && Number.isInteger(schedule.day) && schedule.day >= 0 && schedule.day <= 6)
+        return { frequency: "weekly", time, day: schedule.day };
+    throw new Error("Choose a supported import frequency and, for weekly imports, a weekday from Sunday (0) to Saturday (6).");
+}
+export function scheduleExpression(schedule) {
+    if (schedule.frequency === "cron")
+        return schedule.expression;
+    if (schedule.frequency === "hourly")
+        return "0 * * * *";
+    const [hour, minute] = schedule.time.split(":").map(Number);
+    return `${minute} ${hour} * * ${schedule.frequency === "weekly" ? schedule.day : "*"}`;
+}
+export function scheduleDescription(schedule) {
+    if (schedule.frequency === "cron")
+        return `cron ${schedule.expression}`;
+    if (schedule.frequency === "hourly")
+        return "hourly at minute 00";
+    return schedule.frequency === "daily" ? `daily at ${schedule.time}` : `weekly on ${SYNC_WEEKDAYS[schedule.day]} at ${schedule.time}`;
+}
+export function localDay(now) {
+    return `${now.getFullYear()}-${String(now.getMonth() + 1).padStart(2, "0")}-${String(now.getDate()).padStart(2, "0")}`;
+}
+export function scheduledMinute(now) {
+    return `${localDay(now)}T${String(now.getHours()).padStart(2, "0")}:${String(now.getMinutes()).padStart(2, "0")}`;
+}
+export function nextScheduledSync(schedule, now = new Date()) {
+    const { interval, calendar } = parseImportCron(scheduleExpression(schedule), now);
+    // cron-parser can shift a missing spring-forward time to the next hour.
+    // Only an actual matching wall-clock minute is a valid scheduled run.
+    for (let attempt = 0; attempt < 10; attempt++) {
+        const next = interval.next().toDate();
+        if (importCalendarMatches(calendar, next))
+            return next.toISOString();
+    }
+    throw new Error("Unable to determine the next sync time.");
+}
+/** Reject delayed wake deliveries and duplicate launches within a scheduled minute. */
+export function scheduledSyncDue(schedule, lastMinute, now = new Date()) {
+    if (!schedule || lastMinute === scheduledMinute(now))
+        return false;
+    return importCalendarMatches(parseImportCron(scheduleExpression(schedule), now).calendar, now);
+}

package/dist/native-conversations/collector-service-files.js

@@ -0,0 +1,51 @@
+import { SYNC_WEEKDAYS, validateCollectorSchedule } from "./collector-schedule.js";
+import { collectorCalendarGate } from "./collector-calendar-gate.js";
+const WINDOWS_WEEKDAYS = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"];
+export const xml = (value) => value.replace(/&/gu, "&amp;").replace(/</gu, "&lt;").replace(/>/gu, "&gt;").replace(/"/gu, "&quot;");
+export const systemdQuote = (value) => `"${value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"').replace(/%/gu, "%%").replace(/\$/gu, () => "$$")}"`;
+export const windowsQuote = (value) => `"${value.replace(/(\\*)"/gu, '$1$1\\"').replace(/(\\+)$/u, "$1$1")}"`;
+export function collectorServiceFiles(input) {
+    const schedules = [...new Map(input.schedules.map(value => {
+            const schedule = validateCollectorSchedule(value);
+            return [JSON.stringify(schedule), schedule];
+        })).values()];
+    const command = [input.executable, input.gatePath];
+    const triggers = schedules.some(schedule => schedule.frequency === "cron")
+        ? [{ frequency: "calendar-tick" }] : schedules;
+    const cron = (schedule) => {
+        if (schedule.frequency === "calendar-tick")
+            return "*-*-* *:*:00";
+        if (schedule.frequency === "hourly")
+            return "*-*-* *:00:00";
+        if (schedule.frequency === "cron")
+            throw new Error("Cron needs the calendar gate.");
+        return `${schedule.frequency === "weekly" ? `${SYNC_WEEKDAYS[schedule.day]} ` : ""}*-*-* ${schedule.time}:00`;
+    };
+    const launchd = (schedule) => {
+        if (schedule.frequency === "calendar-tick")
+            return "<dict/>";
+        if (schedule.frequency === "hourly")
+            return "<dict><key>Minute</key><integer>0</integer></dict>";
+        if (schedule.frequency === "cron")
+            throw new Error("Cron needs the calendar gate.");
+        const [hour, minute] = schedule.time.split(":").map(Number);
+        return `<dict>${schedule.frequency === "weekly" ? `<key>Weekday</key><integer>${schedule.day}</integer>` : ""}<key>Hour</key><integer>${hour}</integer><key>Minute</key><integer>${minute}</integer></dict>`;
+    };
+    const windows = (schedule) => {
+        if (schedule.frequency === "calendar-tick" || schedule.frequency === "hourly")
+            return `<CalendarTrigger><Repetition><Interval>${schedule.frequency === "hourly" ? "PT1H" : "PT1M"}</Interval><Duration>P1D</Duration><StopAtDurationEnd>false</StopAtDurationEnd></Repetition><StartBoundary>2020-01-01T00:00:00</StartBoundary><Enabled>true</Enabled><ScheduleByDay><DaysInterval>1</DaysInterval></ScheduleByDay></CalendarTrigger>`;
+        if (schedule.frequency === "cron")
+            throw new Error("Cron needs the calendar gate.");
+        return `<CalendarTrigger><StartBoundary>2020-01-01T${schedule.time}:00</StartBoundary><Enabled>true</Enabled>${schedule.frequency === "daily" ? "<ScheduleByDay><DaysInterval>1</DaysInterval></ScheduleByDay>" : `<ScheduleByWeek><WeeksInterval>1</WeeksInterval><DaysOfWeek><${WINDOWS_WEEKDAYS[schedule.day]}/></DaysOfWeek></ScheduleByWeek>`}</CalendarTrigger>`;
+    };
+    return {
+        gate: collectorCalendarGate({ executable: input.executable, args: input.args, schedules }),
+        service: `[Unit]\nDescription=OpenPond conversation import job\nAfter=network-online.target\n[Service]\nType=oneshot\n${input.environment.map(([key, value]) => `Environment=${systemdQuote(`${key}=${value}`)}`).join("\n")}\nExecStart=${command.map(systemdQuote).join(" ")}\nTimeoutStartSec=65min\nTimeoutStopSec=15s\nRestart=no\nNice=10\nUMask=0077\nNoNewPrivileges=true\n`,
+        timer: `[Unit]\nDescription=OpenPond scheduled conversation imports\n[Timer]\n${triggers.map(schedule => `OnCalendar=${cron(schedule)}`).join("\n")}\nAccuracySec=1s\nRandomizedDelaySec=0\nPersistent=false\nWakeSystem=false\nUnit=${input.name}.service\n[Install]\nWantedBy=timers.target\n`,
+        plist: `<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"><plist version="1.0"><dict><key>Label</key><string>${input.name}</string><key>ProgramArguments</key><array>${command.map(value => `<string>${xml(value)}</string>`).join("")}</array><key>EnvironmentVariables</key><dict>${input.environment.map(([key, value]) => `<key>${xml(key)}</key><string>${xml(value)}</string>`).join("")}</dict><key>StartCalendarInterval</key><array>${triggers.map(launchd).join("")}</array><key>RunAtLoad</key><false/><key>KeepAlive</key><false/><key>Nice</key><integer>10</integer></dict></plist>`,
+        windowsTask(user, script) {
+            return `<?xml version="1.0"?><Task version="1.4" xmlns="http://schemas.microsoft.com/windows/2004/02/mit/task"><Triggers>${triggers.map(windows).join("")}</Triggers><Principals><Principal id="Owner"><UserId>${xml(user)}</UserId><LogonType>InteractiveToken</LogonType><RunLevel>LeastPrivilege</RunLevel></Principal></Principals><Settings><MultipleInstancesPolicy>IgnoreNew</MultipleInstancesPolicy><DisallowStartIfOnBatteries>false</DisallowStartIfOnBatteries><StopIfGoingOnBatteries>false</StopIfGoingOnBatteries><StartWhenAvailable>false</StartWhenAvailable><WakeToRun>false</WakeToRun><ExecutionTimeLimit>PT65M</ExecutionTimeLimit></Settings><Actions Context="Owner"><Exec><Command>powershell.exe</Command><Arguments>${xml(["-NoProfile", "-NonInteractive", "-File", script].map(windowsQuote).join(" "))}</Arguments></Exec></Actions></Task>`;
+        },
+        powershell: `${input.environment.map(([key, value]) => `$env:${key} = '${value.replace(/'/gu, "''")}'`).join("\n")}\n& ${command.map(value => `'${value.replace(/'/gu, "''")}'`).join(" ")}\nexit $LASTEXITCODE\n`,
+    };
+}

package/dist/native-conversations/collector-snapshots.js

@@ -0,0 +1,70 @@
+import { setImmediate } from "node:timers/promises";
+import { contentHash } from "@openpond/harness";
+export class CollectorQueueFullError extends Error {
+    constructor() { super("Import queue is full. Pending uploads must finish before more history can be read."); }
+}
+/** Batches reference an immutable source snapshot instead of duplicating the
+ * whole transcript once for every twenty selected turns. */
+export class CollectorSnapshots {
+    database;
+    constructor(database) {
+        this.database = database;
+        database.exec(`
+      CREATE TABLE IF NOT EXISTS collector_snapshots(hash TEXT PRIMARY KEY,payload TEXT NOT NULL,bytes INTEGER NOT NULL);
+      CREATE TABLE IF NOT EXISTS pending_snapshots(operation_id TEXT PRIMARY KEY,snapshot_hash TEXT NOT NULL);
+      CREATE INDEX IF NOT EXISTS pending_snapshots_hash ON pending_snapshots(snapshot_hash);
+    `);
+    }
+    prepare(files) {
+        const payload = JSON.stringify(files);
+        return { hash: contentHash(files), payload, bytes: Buffer.byteLength(payload) };
+    }
+    exists(hash) {
+        return !!this.database.prepare("SELECT 1 FROM collector_snapshots WHERE hash=?").get(hash);
+    }
+    retain(id, snapshot) {
+        this.database.prepare("INSERT OR IGNORE INTO collector_snapshots VALUES(?,?,?)").run(snapshot.hash, snapshot.payload, snapshot.bytes);
+        this.database.prepare("INSERT OR REPLACE INTO pending_snapshots VALUES(?,?)").run(id, snapshot.hash);
+    }
+    files(id) {
+        const row = this.database.prepare("SELECT payload FROM collector_snapshots JOIN pending_snapshots ON hash=snapshot_hash WHERE operation_id=?").get(id);
+        if (!row)
+            throw new Error("The pending import's source snapshot is missing.");
+        return JSON.parse(String(row.payload));
+    }
+    bytes(connectionId) {
+        return Number(connectionId
+            ? this.database.prepare("SELECT COALESCE(SUM(bytes),0) AS n FROM collector_snapshots WHERE hash IN (SELECT snapshot_hash FROM pending_snapshots JOIN pending ON operation_id=pending.id WHERE connection_id=?)").get(connectionId)?.n
+            : this.database.prepare("SELECT COALESCE(SUM(bytes),0) AS n FROM collector_snapshots").get()?.n);
+    }
+    collect() {
+        this.database.exec(`DELETE FROM pending_snapshots WHERE operation_id NOT IN (SELECT id FROM pending);
+      DELETE FROM collector_snapshots WHERE hash NOT IN (SELECT snapshot_hash FROM pending_snapshots);`);
+    }
+    /** Upgrade retained queues only in the owned worker, never in a status read.
+     * Each operation commits independently so interruption preserves all work. */
+    async migrate(cancelled) {
+        const ids = this.database.prepare("SELECT id FROM pending WHERE id NOT IN (SELECT operation_id FROM pending_snapshots)").all();
+        for (const row of ids) {
+            await setImmediate();
+            if (cancelled())
+                return;
+            const stored = this.database.prepare("SELECT payload FROM pending WHERE id=?").get(row.id);
+            if (!stored)
+                continue;
+            const { files, ...entry } = JSON.parse(String(stored.payload));
+            const snapshot = this.prepare(files), payload = JSON.stringify(entry);
+            this.database.exec("BEGIN IMMEDIATE");
+            try {
+                this.retain(String(row.id), snapshot);
+                this.database.prepare("UPDATE pending SET payload=?,bytes=? WHERE id=?").run(payload, Buffer.byteLength(payload), row.id);
+                this.database.exec("COMMIT");
+            }
+            catch (error) {
+                this.database.exec("ROLLBACK");
+                throw error;
+            }
+        }
+        this.collect();
+    }
+}