opencode-graphiti 0.0.0-development

package/script/src/services/batch-drain.js

@@ -0,0 +1,221 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BatchDrainService = void 0;
+const index_js_1 = require("../types/index.js");
+const logger_js_1 = require("./logger.js");
+const redis_events_js_1 = require("./redis-events.js");
+const render_utils_js_1 = require("./render-utils.js");
+class DrainClaimLostError extends Error {
+    constructor() {
+        super("Drain claim lease lost during batch processing");
+        this.name = "DrainClaimLostError";
+    }
+}
+const makeBatchKey = (entries) => `${entries[0]?.event.id ?? "empty"}:${entries.at(-1)?.event.id ?? "empty"}`;
+const shouldDrainEntry = (entry) => {
+    const text = (0, render_utils_js_1.sanitizeMemoryInput)((0, index_js_1.getSessionEventRecallText)(entry.event));
+    if (!text)
+        return false;
+    if ((0, render_utils_js_1.looksLikeToolTranscript)(text))
+        return false;
+    if ((0, render_utils_js_1.looksLikeOperationalChatter)(text))
+        return false;
+    if ((0, render_utils_js_1.looksTranscriptHeavy)(text))
+        return false;
+    if (entry.event.role === "assistant" && entry.event.category !== "discovery") {
+        return false;
+    }
+    if (entry.event.category === "message" && entry.event.role !== "user") {
+        return false;
+    }
+    return true;
+};
+class BatchDrainService {
+    constructor(redis, events, options) {
+        Object.defineProperty(this, "redis", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: redis
+        });
+        Object.defineProperty(this, "events", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: events
+        });
+        Object.defineProperty(this, "options", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: options
+        });
+    }
+    getClaimHeartbeatIntervalMs(lockTtlSeconds) {
+        const ttlMs = Math.max(1_000, Math.floor(lockTtlSeconds * 1000));
+        const defaultIntervalMs = Math.max(250, Math.floor(ttlMs / 3));
+        const configuredIntervalMs = this.options.claimHeartbeatIntervalMs;
+        const requestedIntervalMs = configuredIntervalMs ?? defaultIntervalMs;
+        const minSafeIntervalMs = 250;
+        const maxSafeIntervalMs = Math.max(250, Math.floor(ttlMs / 2));
+        if (requestedIntervalMs < minSafeIntervalMs) {
+            if (configuredIntervalMs !== undefined) {
+                logger_js_1.logger.warn("Clamped drain heartbeat interval to a safe minimum", {
+                    claimLockTtlSeconds: lockTtlSeconds,
+                    requestedHeartbeatIntervalMs: requestedIntervalMs,
+                    effectiveHeartbeatIntervalMs: minSafeIntervalMs,
+                    configuredHeartbeatIntervalMs: configuredIntervalMs,
+                });
+            }
+            return minSafeIntervalMs;
+        }
+        if (requestedIntervalMs <= maxSafeIntervalMs) {
+            return requestedIntervalMs;
+        }
+        logger_js_1.logger.warn("Clamped drain heartbeat interval to stay below claim TTL", {
+            claimLockTtlSeconds: lockTtlSeconds,
+            requestedHeartbeatIntervalMs: requestedIntervalMs,
+            effectiveHeartbeatIntervalMs: maxSafeIntervalMs,
+            configuredHeartbeatIntervalMs: configuredIntervalMs,
+        });
+        return maxSafeIntervalMs;
+    }
+    async getRetryState(groupId, batchKey) {
+        const raw = await this.redis.getString((0, redis_events_js_1.drainRetryKey)(groupId, batchKey));
+        if (!raw)
+            return null;
+        try {
+            return JSON.parse(raw);
+        }
+        catch {
+            return null;
+        }
+    }
+    async setRetryState(groupId, batchKey, state) {
+        await this.redis.setString((0, redis_events_js_1.drainRetryKey)(groupId, batchKey), JSON.stringify(state), 7 * 24 * 60 * 60);
+    }
+    async drainGroup(groupId, graphiti) {
+        const claimed = await this.events.getPendingBatch(groupId, this.options.batchSize, this.options.batchMaxBytes);
+        if (!claimed || claimed.entries.length === 0) {
+            return { status: "empty", drained: 0 };
+        }
+        const batch = claimed.entries;
+        const batchKey = makeBatchKey(batch);
+        const semanticBatch = batch.filter(shouldDrainEntry);
+        if (semanticBatch.length === 0) {
+            await this.events.markBatchSuccess(groupId, claimed.claimToken, batch);
+            await this.redis.deleteKey((0, redis_events_js_1.drainRetryKey)(groupId, batchKey));
+            return { status: "success", drained: 0 };
+        }
+        const retryState = await this.getRetryState(groupId, batchKey);
+        if (retryState && retryState.nextAttemptAt > Date.now()) {
+            await this.events.releaseClaim(groupId, claimed.claimToken);
+            return { status: "backoff", drained: 0 };
+        }
+        let lostClaim = false;
+        const refreshClaimHeartbeat = async () => {
+            try {
+                const refreshed = await this.events.refreshClaimLease(groupId, claimed.claimToken, claimed.lockTtlSeconds);
+                if (!refreshed)
+                    lostClaim = true;
+            }
+            catch {
+                lostClaim = true;
+            }
+        };
+        const confirmClaimOwnership = async () => {
+            if (lostClaim)
+                return false;
+            try {
+                const refreshed = await this.events.refreshClaimLease(groupId, claimed.claimToken, claimed.lockTtlSeconds);
+                if (!refreshed)
+                    lostClaim = true;
+            }
+            catch {
+                lostClaim = true;
+            }
+            return !lostClaim;
+        };
+        const assertClaimOwnership = async () => {
+            if (!await confirmClaimOwnership()) {
+                throw new DrainClaimLostError();
+            }
+        };
+        const heartbeatInterval = setInterval(() => {
+            void refreshClaimHeartbeat();
+        }, this.getClaimHeartbeatIntervalMs(claimed.lockTtlSeconds));
+        const usePerEntryCheckpointing = semanticBatch.length !== 0;
+        let checkpointedCount = 0;
+        try {
+            for (const entry of batch) {
+                await assertClaimOwnership();
+                if (shouldDrainEntry(entry)) {
+                    await graphiti.addMemory({
+                        name: `${entry.event.category}:${entry.event.id}`,
+                        episodeBody: (0, redis_events_js_1.buildDrainEpisodeBody)(entry),
+                        groupId,
+                        source: "text",
+                        sourceDescription: `session-event:${entry.event.category}`,
+                    });
+                }
+                if (usePerEntryCheckpointing) {
+                    await assertClaimOwnership();
+                    await this.events.markClaimEntrySuccess(groupId, claimed.claimToken, entry);
+                    checkpointedCount += 1;
+                }
+            }
+            clearInterval(heartbeatInterval);
+            await assertClaimOwnership();
+            await this.events.markBatchSuccess(groupId, claimed.claimToken, batch);
+            await this.redis.deleteKey((0, redis_events_js_1.drainRetryKey)(groupId, batchKey));
+            return { status: "success", drained: semanticBatch.length };
+        }
+        catch (err) {
+            const lostOwnership = err instanceof DrainClaimLostError;
+            if (lostOwnership) {
+                logger_js_1.logger.warn("Drain claim heartbeat lost ownership", {
+                    groupId,
+                    eventIds: batch.map((entry) => entry.event.id),
+                });
+            }
+            const attempts = (retryState?.attempts ?? 0) + 1;
+            const stillOwnClaim = await confirmClaimOwnership();
+            if (!stillOwnClaim) {
+                if (!lostOwnership) {
+                    logger_js_1.logger.warn("Drain claim heartbeat lost ownership", {
+                        groupId,
+                        eventIds: batch.map((entry) => entry.event.id),
+                    });
+                }
+                await this.redis.deleteKey((0, redis_events_js_1.drainRetryKey)(groupId, batchKey));
+                logger_js_1.logger.warn("Drain batch failed after claim loss; waiting for recovery", {
+                    groupId,
+                    err,
+                });
+                return { status: "retry", drained: 0 };
+            }
+            if (attempts >= this.options.drainRetryMax) {
+                const remainingEntries = batch.slice(checkpointedCount);
+                logger_js_1.logger.warn("Moving drain batch to dead-letter", {
+                    groupId,
+                    eventIds: remainingEntries.map((entry) => entry.event.id),
+                });
+                await this.events.moveBatchToDeadLetter(groupId, remainingEntries);
+                await this.events.markBatchSuccess(groupId, claimed.claimToken, batch);
+                await this.redis.deleteKey((0, redis_events_js_1.drainRetryKey)(groupId, batchKey));
+                return { status: "dead-letter", drained: remainingEntries.length };
+            }
+            await this.events.releaseClaim(groupId, claimed.claimToken);
+            await this.setRetryState(groupId, batchKey, {
+                attempts,
+                nextAttemptAt: Date.now() + 1_000 * (2 ** (attempts - 1)),
+            });
+            logger_js_1.logger.warn("Drain batch failed; will retry later", { groupId, err });
+            return { status: "retry", drained: 0 };
+        }
+        finally {
+            clearInterval(heartbeatInterval);
+        }
+    }
+}
+exports.BatchDrainService = BatchDrainService;

package/script/src/services/connection-manager.d.ts

@@ -0,0 +1,104 @@
+export type GraphitiConnectionState = "connecting" | "connected" | "offline" | "closing" | "stopped";
+type TimerHandle = ReturnType<typeof setTimeout> | number;
+export declare class GraphitiOfflineError extends Error {
+    readonly state: "offline" | "closing" | "stopped";
+    readonly kind = "offline";
+    constructor(state: "offline" | "closing" | "stopped", message?: string);
+}
+export declare class GraphitiQueueTimeoutError extends Error {
+    readonly kind = "queue-timeout";
+    constructor(message?: string);
+}
+export declare class GraphitiRequestTimeoutError extends Error {
+    readonly kind = "request-timeout";
+    constructor(message?: string);
+}
+export declare class GraphitiTransportError extends Error {
+    readonly kind = "transport-failure";
+    constructor(message?: string);
+}
+export declare class GraphitiSessionExpiredError extends Error {
+    readonly kind = "session-expired";
+    constructor(message?: string);
+}
+export declare function isGraphitiOfflineError(err: unknown): err is GraphitiOfflineError;
+export declare function isGraphitiTimeoutError(err: unknown): err is GraphitiQueueTimeoutError | GraphitiRequestTimeoutError;
+export type GraphitiToolRequest = {
+    name: string;
+    arguments?: Record<string, unknown>;
+};
+export interface GraphitiConnection {
+    connect(): Promise<void>;
+    close(): Promise<void>;
+    callTool(request: GraphitiToolRequest): Promise<unknown>;
+}
+export interface GraphitiToolCaller {
+    start(): void;
+    stop(): Promise<void>;
+    ready(timeoutMs?: number): Promise<boolean>;
+    callTool(name: string, args: Record<string, unknown>, deadlineMs?: number): Promise<unknown>;
+}
+type ConnectionFactory = (endpoint: string) => GraphitiConnection;
+type GraphitiConnectionManagerOptions = {
+    endpoint: string;
+    requestDeadlineMs?: number;
+    queueCapacity?: number;
+    startupTimeoutMs?: number;
+    reconnectInitialDelayMs?: number;
+    reconnectMaxDelayMs?: number;
+    reconnectMultiplier?: number;
+    reconnectJitter?: number;
+    connectionFactory?: ConnectionFactory;
+    random?: () => number;
+    now?: () => number;
+    setTimer?: (callback: () => void, delayMs: number) => TimerHandle;
+    clearTimer?: (timer: TimerHandle) => void;
+};
+export declare class GraphitiConnectionManager implements GraphitiToolCaller {
+    private readonly endpoint;
+    private readonly requestDeadlineMs;
+    private readonly queueCapacity;
+    private readonly startupTimeoutMs;
+    private readonly reconnectInitialDelayMs;
+    private readonly reconnectMaxDelayMs;
+    private readonly reconnectMultiplier;
+    private readonly reconnectJitter;
+    private readonly connectionFactory;
+    private readonly random;
+    private readonly now;
+    private readonly setTimerImpl;
+    private readonly clearTimerImpl;
+    private state;
+    private connection;
+    private connectPromise;
+    private reconnectTimer;
+    private pendingRequests;
+    private readyWaiters;
+    private reconnectDelayMs;
+    private started;
+    private flushingQueue;
+    private stopPromise;
+    constructor(options: GraphitiConnectionManagerOptions);
+    getState(): GraphitiConnectionState;
+    start(): void;
+    stop(): Promise<void>;
+    ready(timeoutMs?: number): Promise<boolean>;
+    callTool(name: string, args: Record<string, unknown>, deadlineMs?: number): Promise<unknown>;
+    reconnect(): Promise<boolean>;
+    private performReconnect;
+    private executeConnectedCall;
+    private executeConnectedCallWithinDeadline;
+    private getRemainingDeadlineMs;
+    private reconnectWithinDeadline;
+    private runWithRequestDeadline;
+    private enqueueRequest;
+    private flushPendingQueue;
+    private removePendingRequest;
+    private clearPendingTimer;
+    private rejectAllPending;
+    private scheduleReconnect;
+    private cancelReconnectTimer;
+    private resolveReadyWaiters;
+}
+export {};
+//# sourceMappingURL=connection-manager.d.ts.map

package/script/src/services/connection-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"connection-manager.d.ts","sourceRoot":"","sources":["../../../src/src/services/connection-manager.ts"],"names":[],"mappings":"AAKA,MAAM,MAAM,uBAAuB,GAC/B,YAAY,GACZ,WAAW,GACX,SAAS,GACT,SAAS,GACT,SAAS,CAAC;AAEd,KAAK,WAAW,GAAG,UAAU,CAAC,OAAO,UAAU,CAAC,GAAG,MAAM,CAAC;AAE1D,qBAAa,oBAAqB,SAAQ,KAAK;IAI3C,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,SAAS,GAAG,SAAS;IAHnD,QAAQ,CAAC,IAAI,aAAa;gBAGf,KAAK,EAAE,SAAS,GAAG,SAAS,GAAG,SAAS,EACjD,OAAO,CAAC,EAAE,MAAM;CAYnB;AAED,qBAAa,yBAA0B,SAAQ,KAAK;IAClD,QAAQ,CAAC,IAAI,mBAAmB;gBAG9B,OAAO,SAA4D;CAKtE;AAED,qBAAa,2BAA4B,SAAQ,KAAK;IACpD,QAAQ,CAAC,IAAI,qBAAqB;gBAEtB,OAAO,SAA+B;CAInD;AAED,qBAAa,sBAAuB,SAAQ,KAAK;IAC/C,QAAQ,CAAC,IAAI,uBAAuB;gBAExB,OAAO,SAA+B;CAInD;AAED,qBAAa,2BAA4B,SAAQ,KAAK;IACpD,QAAQ,CAAC,IAAI,qBAAqB;gBAEtB,OAAO,SAA6B;CAIjD;AAED,wBAAgB,sBAAsB,CACpC,GAAG,EAAE,OAAO,GACX,GAAG,IAAI,oBAAoB,CAE7B;AAED,wBAAgB,sBAAsB,CACpC,GAAG,EAAE,OAAO,GACX,GAAG,IAAI,yBAAyB,GAAG,2BAA2B,CAGhE;AAED,MAAM,MAAM,mBAAmB,GAAG;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACrC,CAAC;AAEF,MAAM,WAAW,kBAAkB;IACjC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACzB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAC1D;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,IAAI,IAAI,CAAC;IACd,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,KAAK,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,QAAQ,CACN,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7B,UAAU,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,OAAO,CAAC,CAAC;CACrB;AAWD,KAAK,iBAAiB,GAAG,CAAC,QAAQ,EAAE,MAAM,KAAK,kBAAkB,CAAC;AAWlE,KAAK,gCAAgC,GAAG;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;IACtB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,IAAI,EAAE,OAAO,EAAE,MAAM,KAAK,WAAW,CAAC;IAClE,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;CAC3C,CAAC;AAsEF,qBAAa,yBAA0B,YAAW,kBAAkB;IAClE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAS;IACjD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAoB;IACtD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAGZ;IACjB,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA+B;IAE9D,OAAO,CAAC,KAAK,CAAsC;IACnD,OAAO,CAAC,UAAU,CAAmC;IACrD,OAAO,CAAC,cAAc,CAAiC;IACvD,OAAO,CAAC,cAAc,CAA4B;IAClD,OAAO,CAAC,eAAe,CAAwB;IAC/C,OAAO,CAAC,YAAY,CAAuC;IAC3D,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,WAAW,CAA8B;gBAErC,OAAO,EAAE,gCAAgC;IAmBrD,QAAQ,IAAI,uBAAuB;IAInC,KAAK,IAAI,IAAI;IAcP,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAmCrB,KAAK,CAAC,SAAS,SAAwB,GAAG,OAAO,CAAC,OAAO,CAAC;IAwB1D,QAAQ,CACZ,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7B,UAAU,SAAyB,GAClC,OAAO,CAAC,OAAO,CAAC;IA0Bb,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC;YAerB,gBAAgB;YAoDhB,oBAAoB;YAYpB,kCAAkC;IAuEhD,OAAO,CAAC,sBAAsB;YAIhB,uBAAuB;IASrC,OAAO,CAAC,sBAAsB;IA6B9B,OAAO,CAAC,cAAc;YAyCR,iBAAiB;IAgC/B,OAAO,CAAC,oBAAoB;IAQ5B,OAAO,CAAC,iBAAiB;IAOzB,OAAO,CAAC,gBAAgB;IAUxB,OAAO,CAAC,iBAAiB;IA6BzB,OAAO,CAAC,oBAAoB;IAO5B,OAAO,CAAC,mBAAmB;CAO5B"}