@looopy-ai/core 1.0.1

package/dist/server/event-router.js

@@ -0,0 +1,91 @@
+import { isDebugEvent } from '../events';
+export class EventRouter {
+    subscribers = new Map();
+    subscribe(subscriber) {
+        const { contextId } = subscriber.config;
+        let contextSubscribers = this.subscribers.get(contextId);
+        if (!contextSubscribers) {
+            contextSubscribers = new Map();
+            this.subscribers.set(contextId, contextSubscribers);
+        }
+        contextSubscribers.set(subscriber.id, subscriber);
+    }
+    unsubscribe(subscriberId, contextId) {
+        const contextSubscribers = this.subscribers.get(contextId);
+        if (!contextSubscribers) {
+            return;
+        }
+        contextSubscribers.delete(subscriberId);
+        if (contextSubscribers.size === 0) {
+            this.subscribers.delete(contextId);
+        }
+    }
+    route(contextId, event, eventId) {
+        const contextSubscribers = this.subscribers.get(contextId);
+        if (!contextSubscribers || contextSubscribers.size === 0) {
+            return 0;
+        }
+        let sentCount = 0;
+        for (const subscriber of contextSubscribers.values()) {
+            if (!event.kind || this.shouldSendToSubscriber(subscriber, event)) {
+                try {
+                    subscriber.send(event, eventId);
+                    sentCount++;
+                }
+                catch (error) {
+                    console.error(`Failed to send event to subscriber ${subscriber.id}:`, error);
+                }
+            }
+        }
+        return sentCount;
+    }
+    shouldSendToSubscriber(subscriber, event) {
+        const { config } = subscriber;
+        if (config.taskId && event.taskId !== config.taskId) {
+            return false;
+        }
+        if (config.filterInternal !== false && isDebugEvent(event)) {
+            return false;
+        }
+        if (config.includeKinds && !config.includeKinds.includes(event.kind)) {
+            return false;
+        }
+        if (config.excludeKinds?.includes(event.kind)) {
+            return false;
+        }
+        if (config.filter && !config.filter(event)) {
+            return false;
+        }
+        return true;
+    }
+    getSubscriberCount(contextId) {
+        return this.subscribers.get(contextId)?.size ?? 0;
+    }
+    getActiveContexts() {
+        return Array.from(this.subscribers.keys());
+    }
+    getStats() {
+        const contexts = this.subscribers.size;
+        let totalSubscribers = 0;
+        for (const contextSubscribers of this.subscribers.values()) {
+            totalSubscribers += contextSubscribers.size;
+        }
+        return {
+            totalContexts: contexts,
+            totalSubscribers,
+            averageSubscribersPerContext: contexts > 0 ? totalSubscribers / contexts : 0,
+        };
+    }
+    clear() {
+        for (const contextSubscribers of this.subscribers.values()) {
+            for (const subscriber of contextSubscribers.values()) {
+                try {
+                    subscriber.close();
+                }
+                catch {
+                }
+            }
+        }
+        this.subscribers.clear();
+    }
+}

package/dist/server/index.d.ts

@@ -0,0 +1,3 @@
+export { type BufferedEvent, EventBuffer, type EventBufferConfig, } from './event-buffer';
+export { type EventFilter, EventRouter, type Subscriber, type SubscriptionConfig, } from './event-router';
+export { SSEConnection, type SSEConnectionConfig, type SSEResponse, SSEServer, type SSEServerConfig, } from './sse';

package/dist/server/index.js

@@ -0,0 +1,3 @@
+export { EventBuffer, } from './event-buffer';
+export { EventRouter, } from './event-router';
+export { SSEConnection, SSEServer, } from './sse';

package/dist/server/sse.d.ts

@@ -0,0 +1,60 @@
+import type { AnyEvent } from '../types/event';
+import { EventBuffer, type EventBufferConfig } from './event-buffer';
+import { EventRouter, type Subscriber, type SubscriptionConfig } from './event-router';
+export interface SSEResponse {
+    setHeader(name: string, value: string): void;
+    write(chunk: string): void;
+    end(): void;
+    writable?: boolean;
+    on?(event: 'close', listener: () => void): void;
+    once?(event: 'close', listener: () => void): void;
+    removeListener?(event: 'close', listener: () => void): void;
+}
+export interface SSEConnectionConfig {
+    subscription: SubscriptionConfig;
+    response: SSEResponse;
+    lastEventId?: string;
+    heartbeatInterval?: number;
+}
+export declare class SSEConnection implements Subscriber {
+    readonly id: string;
+    readonly config: SubscriptionConfig;
+    private response;
+    private heartbeatTimer?;
+    private heartbeatInterval;
+    private closed;
+    private lastEventId?;
+    constructor(id: string, connectionConfig: SSEConnectionConfig);
+    send(event: AnyEvent, eventId: string): void;
+    private sendHeartbeat;
+    private startHeartbeat;
+    private stopHeartbeat;
+    getLastEventId(): string | undefined;
+    close(): void;
+    isClosed(): boolean;
+}
+export interface SSEServerConfig {
+    eventBuffer?: EventBufferConfig;
+    enableBuffering?: boolean;
+    enableHeartbeat?: boolean;
+    heartbeatInterval?: number;
+}
+export declare class SSEServer {
+    private router;
+    private buffer;
+    private connectionCounter;
+    private enableBuffering;
+    private enableHeartbeat;
+    private heartbeatInterval;
+    constructor(config?: SSEServerConfig);
+    subscribe(response: SSEResponse, config: SubscriptionConfig, lastEventId?: string): SSEConnection;
+    emit(contextId: string, event: AnyEvent): number;
+    getSubscriberCount(contextId: string): number;
+    getActiveContexts(): string[];
+    getStats(): {
+        router: ReturnType<EventRouter['getStats']>;
+        buffer?: ReturnType<EventBuffer['getStats']>;
+    };
+    clearBuffer(contextId: string): void;
+    shutdown(): void;
+}

package/dist/server/sse.js

@@ -0,0 +1,159 @@
+import { EventBuffer } from './event-buffer';
+import { EventRouter } from './event-router';
+export class SSEConnection {
+    id;
+    config;
+    response;
+    heartbeatTimer;
+    heartbeatInterval;
+    closed = false;
+    lastEventId;
+    constructor(id, connectionConfig) {
+        this.id = id;
+        this.config = connectionConfig.subscription;
+        this.response = connectionConfig.response;
+        this.lastEventId = connectionConfig.lastEventId;
+        this.heartbeatInterval = connectionConfig.heartbeatInterval ?? 30000;
+        this.response.setHeader('Content-Type', 'text/event-stream');
+        this.response.setHeader('Cache-Control', 'no-cache');
+        this.response.setHeader('Connection', 'keep-alive');
+        this.response.setHeader('X-Accel-Buffering', 'no');
+        if (this.response.on) {
+            this.response.on('close', () => {
+                this.close();
+            });
+        }
+        this.startHeartbeat();
+    }
+    send(event, eventId) {
+        if (this.closed) {
+            return;
+        }
+        try {
+            const { kind, contextId: _contextId, ...data } = event;
+            this.response.write(`id: ${eventId}\n`);
+            this.response.write(`event: ${kind}\n`);
+            this.response.write(`data: ${JSON.stringify(data)}\n\n`);
+            this.lastEventId = eventId;
+        }
+        catch (error) {
+            console.error(`Failed to send event to ${this.id}:`, error);
+            this.close();
+        }
+    }
+    sendHeartbeat() {
+        if (this.closed) {
+            return;
+        }
+        try {
+            if (this.response.writable === false) {
+                this.close();
+                return;
+            }
+            this.response.write(': heartbeat\n\n');
+        }
+        catch (error) {
+            console.error(`Failed to send heartbeat to ${this.id}:`, error);
+            this.close();
+        }
+    }
+    startHeartbeat() {
+        if (this.heartbeatTimer) {
+            return;
+        }
+        this.heartbeatTimer = setInterval(() => {
+            this.sendHeartbeat();
+        }, this.heartbeatInterval);
+        this.heartbeatTimer.unref?.();
+    }
+    stopHeartbeat() {
+        if (this.heartbeatTimer) {
+            clearInterval(this.heartbeatTimer);
+            this.heartbeatTimer = undefined;
+        }
+    }
+    getLastEventId() {
+        return this.lastEventId;
+    }
+    close() {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        this.stopHeartbeat();
+        try {
+            this.response.end();
+        }
+        catch {
+        }
+    }
+    isClosed() {
+        return this.closed;
+    }
+}
+export class SSEServer {
+    router;
+    buffer;
+    connectionCounter = 0;
+    enableBuffering;
+    enableHeartbeat;
+    heartbeatInterval;
+    constructor(config = {}) {
+        this.router = new EventRouter();
+        this.enableBuffering = config.enableBuffering ?? true;
+        this.enableHeartbeat = config.enableHeartbeat ?? true;
+        this.heartbeatInterval = config.heartbeatInterval ?? 30000;
+        this.buffer = this.enableBuffering ? new EventBuffer(config.eventBuffer) : null;
+    }
+    subscribe(response, config, lastEventId) {
+        const connectionId = `conn-${++this.connectionCounter}`;
+        const connection = new SSEConnection(connectionId, {
+            subscription: config,
+            response,
+            lastEventId,
+            heartbeatInterval: this.enableHeartbeat ? this.heartbeatInterval : 0,
+        });
+        this.router.subscribe(connection);
+        if (lastEventId && this.buffer) {
+            const bufferedEvents = this.buffer.getEventsSince(config.contextId, lastEventId);
+            for (const buffered of bufferedEvents) {
+                connection.send(buffered.event, buffered.id);
+            }
+        }
+        const originalClose = connection.close.bind(connection);
+        connection.close = () => {
+            this.router.unsubscribe(connectionId, config.contextId);
+            originalClose();
+        };
+        return connection;
+    }
+    emit(contextId, event) {
+        let eventId = '';
+        if (this.buffer) {
+            eventId = this.buffer.add(contextId, event);
+        }
+        else {
+            eventId = `${contextId}-${Date.now()}`;
+        }
+        return this.router.route(contextId, event, eventId);
+    }
+    getSubscriberCount(contextId) {
+        return this.router.getSubscriberCount(contextId);
+    }
+    getActiveContexts() {
+        return this.router.getActiveContexts();
+    }
+    getStats() {
+        return {
+            router: this.router.getStats(),
+            buffer: this.buffer?.getStats(),
+        };
+    }
+    clearBuffer(contextId) {
+        this.buffer?.clear(contextId);
+    }
+    shutdown() {
+        this.router.clear();
+        this.buffer?.shutdown();
+    }
+}

package/dist/stores/artifacts/artifact-scheduler.d.ts

@@ -0,0 +1,50 @@
+import type { ArtifactStore, DatasetSchema, StoredArtifact } from '../../types/artifact';
+export declare class ArtifactScheduler implements ArtifactStore {
+    private store;
+    private queues;
+    private processing;
+    constructor(store: ArtifactStore);
+    createFileArtifact(params: {
+        artifactId: string;
+        taskId: string;
+        contextId: string;
+        name?: string;
+        description?: string;
+        mimeType?: string;
+        encoding?: 'utf-8' | 'base64';
+        override?: boolean;
+    }): Promise<string>;
+    appendFileChunk(contextId: string, artifactId: string, chunk: string, options?: {
+        isLastChunk?: boolean;
+        encoding?: 'utf-8' | 'base64';
+    }): Promise<void>;
+    getFileContent(contextId: string, artifactId: string): Promise<string>;
+    createDataArtifact(params: {
+        artifactId: string;
+        taskId: string;
+        contextId: string;
+        name?: string;
+        description?: string;
+        override?: boolean;
+    }): Promise<string>;
+    writeData(contextId: string, artifactId: string, data: Record<string, unknown>): Promise<void>;
+    getDataContent(contextId: string, artifactId: string): Promise<Record<string, unknown>>;
+    createDatasetArtifact(params: {
+        artifactId: string;
+        taskId: string;
+        contextId: string;
+        name?: string;
+        description?: string;
+        schema?: DatasetSchema;
+        override?: boolean;
+    }): Promise<string>;
+    appendDatasetBatch(contextId: string, artifactId: string, rows: Record<string, unknown>[], options?: {
+        isLastBatch?: boolean;
+    }): Promise<void>;
+    getDatasetRows(contextId: string, artifactId: string): Promise<Record<string, unknown>[]>;
+    getArtifact(contextId: string, artifactId: string): Promise<StoredArtifact | null>;
+    listArtifacts(contextId: string, taskId?: string): Promise<string[]>;
+    deleteArtifact(contextId: string, artifactId: string): Promise<void>;
+    private scheduleOperation;
+    private processQueue;
+}

package/dist/stores/artifacts/artifact-scheduler.js

@@ -0,0 +1,86 @@
+export class ArtifactScheduler {
+    store;
+    queues = new Map();
+    processing = new Map();
+    constructor(store) {
+        this.store = store;
+    }
+    async createFileArtifact(params) {
+        return this.scheduleOperation(params.artifactId, () => this.store.createFileArtifact(params));
+    }
+    async appendFileChunk(contextId, artifactId, chunk, options) {
+        return this.scheduleOperation(artifactId, () => this.store.appendFileChunk(contextId, artifactId, chunk, options));
+    }
+    async getFileContent(contextId, artifactId) {
+        return this.scheduleOperation(artifactId, () => this.store.getFileContent(contextId, artifactId));
+    }
+    async createDataArtifact(params) {
+        return this.scheduleOperation(params.artifactId, () => this.store.createDataArtifact(params));
+    }
+    async writeData(contextId, artifactId, data) {
+        return this.scheduleOperation(artifactId, () => this.store.writeData(contextId, artifactId, data));
+    }
+    async getDataContent(contextId, artifactId) {
+        return this.scheduleOperation(artifactId, () => this.store.getDataContent(contextId, artifactId));
+    }
+    async createDatasetArtifact(params) {
+        return this.scheduleOperation(params.artifactId, () => this.store.createDatasetArtifact(params));
+    }
+    async appendDatasetBatch(contextId, artifactId, rows, options) {
+        return this.scheduleOperation(artifactId, () => this.store.appendDatasetBatch(contextId, artifactId, rows, options));
+    }
+    async getDatasetRows(contextId, artifactId) {
+        return this.scheduleOperation(artifactId, () => this.store.getDatasetRows(contextId, artifactId));
+    }
+    async getArtifact(contextId, artifactId) {
+        return this.scheduleOperation(artifactId, () => this.store.getArtifact(contextId, artifactId));
+    }
+    async listArtifacts(contextId, taskId) {
+        return this.store.listArtifacts(contextId, taskId);
+    }
+    async deleteArtifact(contextId, artifactId) {
+        return this.scheduleOperation(artifactId, () => this.store.deleteArtifact(contextId, artifactId));
+    }
+    scheduleOperation(artifactId, operation) {
+        return new Promise((resolve, reject) => {
+            if (!this.queues.has(artifactId)) {
+                this.queues.set(artifactId, []);
+            }
+            const queue = this.queues.get(artifactId);
+            if (!queue) {
+                reject(new Error(`Failed to get queue for artifact ${artifactId}`));
+                return;
+            }
+            queue.push({ operation, resolve, reject });
+            if (!this.processing.get(artifactId)) {
+                this.processQueue(artifactId);
+            }
+        });
+    }
+    async processQueue(artifactId) {
+        this.processing.set(artifactId, true);
+        const queue = this.queues.get(artifactId);
+        if (!queue) {
+            this.processing.set(artifactId, false);
+            return;
+        }
+        while (queue.length > 0) {
+            const item = queue.shift();
+            if (!item) {
+                break;
+            }
+            try {
+                const result = await item.operation();
+                item.resolve(result);
+            }
+            catch (error) {
+                item.reject(error);
+            }
+        }
+        this.processing.set(artifactId, false);
+        if (queue.length === 0) {
+            this.queues.delete(artifactId);
+            this.processing.delete(artifactId);
+        }
+    }
+}

package/dist/stores/artifacts/index.d.ts

@@ -0,0 +1,3 @@
+export { ArtifactScheduler } from './artifact-scheduler';
+export { InternalEventArtifactStore, type InternalEventArtifactStoreConfig, type InternalEventEmitter, } from './internal-event-artifact-store';
+export { InMemoryArtifactStore } from './memory-artifact-store';

package/dist/stores/artifacts/index.js

@@ -0,0 +1,3 @@
+export { ArtifactScheduler } from './artifact-scheduler';
+export { InternalEventArtifactStore, } from './internal-event-artifact-store';
+export { InMemoryArtifactStore } from './memory-artifact-store';

package/dist/stores/artifacts/internal-event-artifact-store.d.ts

@@ -0,0 +1,54 @@
+import type { ArtifactStore, DatasetSchema, StoredArtifact } from '../../types/artifact';
+import type { AnyEvent } from '../../types/event';
+export interface InternalEventEmitter {
+    emit(event: AnyEvent): void;
+}
+export interface InternalEventArtifactStoreConfig {
+    delegate: ArtifactStore;
+    eventEmitter: InternalEventEmitter;
+    enableEvents?: boolean;
+}
+export declare class InternalEventArtifactStore implements ArtifactStore {
+    private delegate;
+    private eventEmitter;
+    private enableEvents;
+    constructor(config: InternalEventArtifactStoreConfig);
+    createFileArtifact(params: {
+        artifactId: string;
+        taskId: string;
+        contextId: string;
+        name?: string;
+        description?: string;
+        mimeType?: string;
+        encoding?: 'utf-8' | 'base64';
+    }): Promise<string>;
+    appendFileChunk(contextId: string, artifactId: string, chunk: string, options?: {
+        isLastChunk?: boolean;
+        encoding?: 'utf-8' | 'base64';
+    }): Promise<void>;
+    getFileContent(contextId: string, artifactId: string): Promise<string>;
+    createDataArtifact(params: {
+        artifactId: string;
+        taskId: string;
+        contextId: string;
+        name?: string;
+        description?: string;
+    }): Promise<string>;
+    writeData(contextId: string, artifactId: string, data: Record<string, unknown>): Promise<void>;
+    getDataContent(contextId: string, artifactId: string): Promise<Record<string, unknown>>;
+    createDatasetArtifact(params: {
+        artifactId: string;
+        taskId: string;
+        contextId: string;
+        name?: string;
+        description?: string;
+        schema?: DatasetSchema;
+    }): Promise<string>;
+    appendDatasetBatch(contextId: string, artifactId: string, rows: Record<string, unknown>[], options?: {
+        isLastBatch?: boolean;
+    }): Promise<void>;
+    getDatasetRows(contextId: string, artifactId: string): Promise<Record<string, unknown>[]>;
+    getArtifact(contextId: string, artifactId: string): Promise<StoredArtifact | null>;
+    listArtifacts(contextId: string, taskId?: string): Promise<string[]>;
+    deleteArtifact(contextId: string, artifactId: string): Promise<void>;
+}

package/dist/stores/artifacts/internal-event-artifact-store.js

@@ -0,0 +1,126 @@
+import { createDatasetWriteEvent, createDataWriteEvent, createFileWriteEvent, } from '../../events';
+export class InternalEventArtifactStore {
+    delegate;
+    eventEmitter;
+    enableEvents;
+    constructor(config) {
+        this.delegate = config.delegate;
+        this.eventEmitter = config.eventEmitter;
+        this.enableEvents = config.enableEvents ?? true;
+    }
+    async createFileArtifact(params) {
+        return this.delegate.createFileArtifact(params);
+    }
+    async appendFileChunk(contextId, artifactId, chunk, options) {
+        await this.delegate.appendFileChunk(contextId, artifactId, chunk, options);
+        if (this.enableEvents) {
+            const artifact = await this.delegate.getArtifact(contextId, artifactId);
+            if (artifact && artifact.type === 'file') {
+                const chunkIndex = artifact.chunks.length - 1;
+                const eventOptions = {
+                    contextId: artifact.contextId,
+                    taskId: artifact.taskId,
+                    artifactId,
+                    data: chunk,
+                    index: chunkIndex,
+                    complete: options?.isLastChunk ?? false,
+                    name: artifact.name,
+                    description: artifact.description,
+                    mimeType: artifact.mimeType,
+                    metadata: {
+                        encoding: options?.encoding || artifact.encoding,
+                        chunkSize: Buffer.byteLength(chunk, options?.encoding || artifact.encoding || 'utf-8'),
+                        totalChunks: artifact.chunks.length,
+                        totalSize: artifact.totalSize,
+                    },
+                };
+                this.eventEmitter.emit(createFileWriteEvent(eventOptions));
+            }
+        }
+    }
+    async getFileContent(contextId, artifactId) {
+        return this.delegate.getFileContent(contextId, artifactId);
+    }
+    async createDataArtifact(params) {
+        return this.delegate.createDataArtifact(params);
+    }
+    async writeData(contextId, artifactId, data) {
+        await this.delegate.writeData(contextId, artifactId, data);
+        if (this.enableEvents) {
+            const artifact = await this.delegate.getArtifact(contextId, artifactId);
+            if (artifact && artifact.type === 'data') {
+                const eventOptions = {
+                    contextId: artifact.contextId,
+                    taskId: artifact.taskId,
+                    artifactId,
+                    data,
+                    name: artifact.name,
+                    description: artifact.description,
+                    metadata: {
+                        version: artifact.version,
+                        dataSize: JSON.stringify(data).length,
+                    },
+                };
+                this.eventEmitter.emit(createDataWriteEvent(eventOptions));
+            }
+        }
+    }
+    async getDataContent(contextId, artifactId) {
+        return this.delegate.getDataContent(contextId, artifactId);
+    }
+    async createDatasetArtifact(params) {
+        return this.delegate.createDatasetArtifact(params);
+    }
+    async appendDatasetBatch(contextId, artifactId, rows, options) {
+        await this.delegate.appendDatasetBatch(contextId, artifactId, rows, options);
+        if (this.enableEvents) {
+            const artifact = await this.delegate.getArtifact(contextId, artifactId);
+            if (artifact && artifact.type === 'dataset') {
+                const batchIndex = artifact.totalChunks - 1;
+                const eventOptions = {
+                    contextId: artifact.contextId,
+                    taskId: artifact.taskId,
+                    artifactId,
+                    rows,
+                    index: batchIndex,
+                    complete: options?.isLastBatch ?? false,
+                    name: artifact.name,
+                    description: artifact.description,
+                    schema: artifact.schema
+                        ? {
+                            type: 'object',
+                            properties: Object.fromEntries(artifact.schema.columns.map((col) => [
+                                col.name,
+                                {
+                                    type: col.type,
+                                    description: col.description,
+                                },
+                            ])),
+                            required: artifact.schema.columns
+                                .filter((col) => !col.nullable)
+                                .map((col) => col.name),
+                        }
+                        : undefined,
+                    metadata: {
+                        batchSize: rows.length,
+                        totalRows: artifact.totalSize,
+                        totalBatches: artifact.totalChunks,
+                    },
+                };
+                this.eventEmitter.emit(createDatasetWriteEvent(eventOptions));
+            }
+        }
+    }
+    async getDatasetRows(contextId, artifactId) {
+        return this.delegate.getDatasetRows(contextId, artifactId);
+    }
+    async getArtifact(contextId, artifactId) {
+        return this.delegate.getArtifact(contextId, artifactId);
+    }
+    async listArtifacts(contextId, taskId) {
+        return this.delegate.listArtifacts(contextId, taskId);
+    }
+    async deleteArtifact(contextId, artifactId) {
+        return this.delegate.deleteArtifact(contextId, artifactId);
+    }
+}