@holon-run/agentinbox 0.1.0 → 0.1.1

package/dist/src/sources/remote.js

@@ -0,0 +1,362 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RemoteSourceRuntime = exports.RpcUxcRemoteSourceClient = void 0;
+const uxc_daemon_client_1 = require("@holon-run/uxc-daemon-client");
+const paths_1 = require("../paths");
+const util_1 = require("../util");
+const remote_profiles_1 = require("./remote_profiles");
+const REMOTE_SOURCE_TYPES = new Set([
+    "remote_source",
+    "github_repo",
+    "github_repo_ci",
+    "feishu_bot",
+]);
+const DEFAULT_SYNC_INTERVAL_MS = 2_000;
+const DEFAULT_BACKOFF_BASE_SECS = 2;
+const MAX_ERROR_BACKOFF_MULTIPLIER = 8;
+const MANAGED_SOURCE_NAMESPACE = "agentinbox";
+class RpcUxcRemoteSourceClient {
+    client;
+    constructor(client = new uxc_daemon_client_1.UxcDaemonClient({ env: process.env })) {
+        this.client = client;
+    }
+    sourceEnsure(args) {
+        return this.client.sourceEnsure({
+            namespace: args.namespace,
+            sourceKey: args.sourceKey,
+            spec: {
+                ...args.spec,
+                operation_id: args.spec.operation_id ?? null,
+                resource_uri: args.spec.resource_uri ?? null,
+                read_resource: args.spec.read_resource ?? false,
+                transport_hint: args.spec.transport_hint ?? null,
+                subprotocols: args.spec.subprotocols ?? [],
+                initial_text_frames: args.spec.initial_text_frames ?? [],
+                poll_config: args.spec.poll_config ?? null,
+                options: args.spec.options,
+            },
+        });
+    }
+    async sourceStop(namespace, sourceKey) {
+        await this.client.sourceStop(namespace, sourceKey);
+    }
+    async sourceDelete(namespace, sourceKey) {
+        await this.client.sourceDelete(namespace, sourceKey);
+    }
+    streamRead(args) {
+        return this.client.streamRead({
+            streamId: args.streamId,
+            afterOffset: args.afterOffset ?? 0,
+            limit: args.limit ?? 100,
+        });
+    }
+}
+exports.RpcUxcRemoteSourceClient = RpcUxcRemoteSourceClient;
+class RemoteSourceRuntime {
+    store;
+    appendSourceEvent;
+    profileRegistry;
+    client;
+    interval = null;
+    inFlight = new Set();
+    errorCounts = new Map();
+    nextRetryAt = new Map();
+    homeDir;
+    constructor(store, appendSourceEvent, options) {
+        this.store = store;
+        this.appendSourceEvent = appendSourceEvent;
+        this.profileRegistry = options?.profileRegistry ?? new remote_profiles_1.RemoteSourceProfileRegistry();
+        this.client = options?.client ?? new RpcUxcRemoteSourceClient();
+        this.homeDir = options?.homeDir ?? (0, paths_1.resolveAgentInboxHome)(process.env);
+    }
+    async ensureSource(source) {
+        if (!REMOTE_SOURCE_TYPES.has(source.sourceType)) {
+            return;
+        }
+        await this.syncSource(source.sourceId, true);
+    }
+    async validateSource(source) {
+        if (!REMOTE_SOURCE_TYPES.has(source.sourceType)) {
+            return;
+        }
+        const profile = await this.profileRegistry.resolve(source, this.homeDir);
+        const profileSource = profileInputSource(source);
+        profile.validateConfig(profileSource);
+        profile.buildManagedSourceSpec(profileSource);
+    }
+    async start() {
+        if (this.interval) {
+            return;
+        }
+        this.interval = setInterval(() => {
+            void this.syncAll();
+        }, DEFAULT_SYNC_INTERVAL_MS);
+        try {
+            await this.syncAll();
+        }
+        catch (error) {
+            console.warn("remote_source initial sync failed:", error);
+        }
+    }
+    async stop() {
+        if (this.interval) {
+            clearInterval(this.interval);
+            this.interval = null;
+        }
+    }
+    async pollSource(sourceId) {
+        const source = this.store.getSource(sourceId);
+        if (!source) {
+            throw new Error(`unknown source: ${sourceId}`);
+        }
+        if (source.status === "paused") {
+            return {
+                sourceId,
+                sourceType: source.sourceType,
+                appended: 0,
+                deduped: 0,
+                eventsRead: 0,
+                note: "source is paused; resume it before polling",
+            };
+        }
+        return this.syncSource(sourceId, true);
+    }
+    async pauseSource(sourceId) {
+        const source = this.store.getSource(sourceId);
+        if (!source || !REMOTE_SOURCE_TYPES.has(source.sourceType)) {
+            return;
+        }
+        const binding = managedBindingForSource(source);
+        await this.client.sourceStop(binding.namespace, binding.sourceKey);
+        this.errorCounts.delete(sourceId);
+        this.nextRetryAt.delete(sourceId);
+        const checkpoint = parseRemoteCheckpoint(source.checkpoint);
+        this.store.updateSourceRuntime(sourceId, {
+            status: "paused",
+            checkpoint: JSON.stringify({
+                ...checkpoint,
+                ...(checkpoint.managedSource ? {
+                    managedSource: {
+                        namespace: binding.namespace,
+                        sourceKey: binding.sourceKey,
+                        streamId: checkpoint.managedSource.streamId,
+                    },
+                } : {}),
+                lastError: null,
+            }),
+        });
+    }
+    async resumeSource(sourceId) {
+        const source = this.store.getSource(sourceId);
+        if (!source || !REMOTE_SOURCE_TYPES.has(source.sourceType)) {
+            return;
+        }
+        await this.syncSource(sourceId, true);
+    }
+    async removeSource(sourceId) {
+        const source = this.store.getSource(sourceId);
+        if (!source || !REMOTE_SOURCE_TYPES.has(source.sourceType)) {
+            return;
+        }
+        const binding = managedBindingForSource(source);
+        await this.client.sourceDelete(binding.namespace, binding.sourceKey);
+    }
+    status() {
+        return {
+            activeSourceIds: Array.from(this.inFlight.values()).sort(),
+            erroredSourceIds: Array.from(this.errorCounts.keys()).sort(),
+        };
+    }
+    async syncAll() {
+        const sources = this.store
+            .listSources()
+            .filter((source) => REMOTE_SOURCE_TYPES.has(source.sourceType) && source.status !== "paused");
+        for (const source of sources) {
+            try {
+                await this.syncSource(source.sourceId, false);
+            }
+            catch (error) {
+                console.warn(`remote source sync failed for ${source.sourceId}:`, error);
+            }
+        }
+    }
+    async syncSource(sourceId, force) {
+        if (this.inFlight.has(sourceId)) {
+            const source = this.store.getSource(sourceId);
+            return {
+                sourceId,
+                sourceType: source?.sourceType ?? "remote_source",
+                appended: 0,
+                deduped: 0,
+                eventsRead: 0,
+                note: "source sync already in flight",
+            };
+        }
+        this.inFlight.add(sourceId);
+        try {
+            const source = this.store.getSource(sourceId);
+            if (!source) {
+                throw new Error(`unknown source: ${sourceId}`);
+            }
+            if (!REMOTE_SOURCE_TYPES.has(source.sourceType)) {
+                return {
+                    sourceId,
+                    sourceType: source.sourceType,
+                    appended: 0,
+                    deduped: 0,
+                    eventsRead: 0,
+                    note: "source type is not hosted by remote runtime",
+                };
+            }
+            if (!force && source.status === "error") {
+                const retryAt = this.nextRetryAt.get(sourceId) ?? 0;
+                if (Date.now() < retryAt) {
+                    return {
+                        sourceId,
+                        sourceType: source.sourceType,
+                        appended: 0,
+                        deduped: 0,
+                        eventsRead: 0,
+                        note: "error backoff not elapsed",
+                    };
+                }
+            }
+            const profile = await this.profileRegistry.resolve(source, this.homeDir);
+            const profileSource = profileInputSource(source);
+            profile.validateConfig(profileSource);
+            const spec = profile.buildManagedSourceSpec(profileSource);
+            const binding = managedBindingForSource(source);
+            const ensured = await this.client.sourceEnsure({
+                namespace: binding.namespace,
+                sourceKey: binding.sourceKey,
+                spec,
+            });
+            const checkpoint = parseRemoteCheckpoint(source.checkpoint);
+            const afterOffset = checkpoint.streamCursor?.afterOffset ?? 0;
+            const batch = await this.client.streamRead({
+                streamId: ensured.stream_id,
+                afterOffset,
+                limit: 100,
+            });
+            let appended = 0;
+            let deduped = 0;
+            for (const event of batch.events) {
+                const rawPayload = asRecord(event.raw_payload);
+                if (Object.keys(rawPayload).length === 0) {
+                    continue;
+                }
+                const mapped = profile.mapRawEvent(rawPayload, profileSource);
+                if (!mapped) {
+                    continue;
+                }
+                const result = await this.appendSourceEvent({
+                    sourceId: source.sourceId,
+                    sourceNativeId: mapped.sourceNativeId,
+                    eventVariant: mapped.eventVariant,
+                    occurredAt: mapped.occurredAt,
+                    metadata: mapped.metadata,
+                    rawPayload: mapped.rawPayload,
+                    deliveryHandle: mapped.deliveryHandle ?? null,
+                });
+                appended += result.appended;
+                deduped += result.deduped;
+            }
+            const latestSource = this.store.getSource(sourceId);
+            if (!latestSource) {
+                throw new Error(`unknown source: ${sourceId}`);
+            }
+            this.store.updateSourceRuntime(sourceId, {
+                status: latestSource.status === "paused" ? "paused" : "active",
+                checkpoint: JSON.stringify({
+                    managedSource: {
+                        namespace: ensured.namespace,
+                        sourceKey: ensured.source_key,
+                        streamId: ensured.stream_id,
+                    },
+                    streamCursor: {
+                        afterOffset: batch.next_after_offset,
+                    },
+                    lastEventAt: (0, util_1.nowIso)(),
+                    lastError: null,
+                }),
+            });
+            this.errorCounts.delete(sourceId);
+            this.nextRetryAt.delete(sourceId);
+            return {
+                sourceId,
+                sourceType: source.sourceType,
+                appended,
+                deduped,
+                eventsRead: batch.events.length,
+                note: `stream=${ensured.stream_id} status=${ensured.status}`,
+            };
+        }
+        catch (error) {
+            const source = this.store.getSource(sourceId);
+            if (source) {
+                const checkpoint = parseRemoteCheckpoint(source.checkpoint);
+                const nextErrorCount = (this.errorCounts.get(sourceId) ?? 0) + 1;
+                this.errorCounts.set(sourceId, nextErrorCount);
+                this.nextRetryAt.set(sourceId, Date.now() + computeErrorBackoffMs(DEFAULT_BACKOFF_BASE_SECS, nextErrorCount));
+                this.store.updateSourceRuntime(sourceId, {
+                    status: source.status === "paused" ? "paused" : "error",
+                    checkpoint: JSON.stringify({
+                        ...checkpoint,
+                        lastError: error instanceof Error ? error.message : String(error),
+                    }),
+                });
+            }
+            throw error;
+        }
+        finally {
+            this.inFlight.delete(sourceId);
+        }
+    }
+}
+exports.RemoteSourceRuntime = RemoteSourceRuntime;
+function profileInputSource(source) {
+    if (source.sourceType !== "remote_source") {
+        return source;
+    }
+    const config = asRecord(source.config);
+    return {
+        ...source,
+        config: asRecord(config.profileConfig),
+    };
+}
+function defaultManagedBindingForSource(source) {
+    return {
+        namespace: MANAGED_SOURCE_NAMESPACE,
+        sourceKey: `${source.sourceType}:${source.sourceKey}`,
+    };
+}
+function managedBindingForSource(source) {
+    const checkpoint = parseRemoteCheckpoint(source.checkpoint);
+    const defaultBinding = defaultManagedBindingForSource(source);
+    return {
+        namespace: checkpoint.managedSource?.namespace ?? defaultBinding.namespace,
+        sourceKey: checkpoint.managedSource?.sourceKey ?? defaultBinding.sourceKey,
+    };
+}
+function parseRemoteCheckpoint(checkpoint) {
+    if (!checkpoint) {
+        return {};
+    }
+    try {
+        return JSON.parse(checkpoint);
+    }
+    catch {
+        return {};
+    }
+}
+function computeErrorBackoffMs(baseIntervalSecs, errorCount) {
+    const baseMs = Math.max(1, baseIntervalSecs) * 1000;
+    const multiplier = Math.min(2 ** Math.max(0, errorCount - 1), MAX_ERROR_BACKOFF_MULTIPLIER);
+    return baseMs * multiplier;
+}
+function asRecord(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        return {};
+    }
+    return value;
+}

package/dist/src/sources/remote_profiles.js

@@ -0,0 +1,254 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RemoteSourceProfileRegistry = void 0;
+exports.builtInProfileIdForSourceType = builtInProfileIdForSourceType;
+exports.profileConfigForSource = profileConfigForSource;
+const node_fs_1 = __importDefault(require("node:fs"));
+const node_path_1 = __importDefault(require("node:path"));
+const node_url_1 = require("node:url");
+const github_1 = require("./github");
+const github_ci_1 = require("./github_ci");
+const feishu_1 = require("./feishu");
+const REMOTE_USER_PROFILE_ROOT_DIR = "source-profiles";
+const BUILTIN_PROFILE_IDS = new Set(["builtin.github_repo", "builtin.github_repo_ci", "builtin.feishu_bot"]);
+class RemoteSourceProfileRegistry {
+    profileCache = new Map();
+    resolve(source, homeDir) {
+        if (source.sourceType === "github_repo") {
+            return Promise.resolve(GITHUB_REPO_PROFILE);
+        }
+        if (source.sourceType === "github_repo_ci") {
+            return Promise.resolve(GITHUB_REPO_CI_PROFILE);
+        }
+        if (source.sourceType === "feishu_bot") {
+            return Promise.resolve(FEISHU_BOT_PROFILE);
+        }
+        if (source.sourceType !== "remote_source") {
+            throw new Error(`unsupported source type for remote profile: ${source.sourceType}`);
+        }
+        return this.loadUserProfile(source, homeDir);
+    }
+    async loadUserProfile(source, homeDir) {
+        const config = source.config ?? {};
+        const profilePath = asNonEmptyString(config.profilePath);
+        if (!profilePath) {
+            throw new Error("remote_source requires config.profilePath");
+        }
+        const profileRoot = node_path_1.default.resolve(homeDir, REMOTE_USER_PROFILE_ROOT_DIR);
+        const resolvedPath = resolveAndValidateProfilePath(profileRoot, profilePath);
+        const cacheKey = `${resolvedPath}:${source.sourceKey}`;
+        const cached = this.profileCache.get(cacheKey);
+        if (cached) {
+            return cached;
+        }
+        if (!node_fs_1.default.existsSync(resolvedPath)) {
+            throw new Error(`remote_source profile not found: ${resolvedPath}`);
+        }
+        const imported = await dynamicImportModule((0, node_url_1.pathToFileURL)(resolvedPath).href);
+        const profile = (imported.default ?? imported);
+        if (!profile || typeof profile !== "object") {
+            throw new Error(`remote_source profile must export default object: ${resolvedPath}`);
+        }
+        validateProfileContract(profile, resolvedPath);
+        this.profileCache.set(cacheKey, profile);
+        return profile;
+    }
+}
+exports.RemoteSourceProfileRegistry = RemoteSourceProfileRegistry;
+async function dynamicImportModule(moduleUrl) {
+    const importer = new Function("url", "return import(url);");
+    return importer(moduleUrl);
+}
+function builtInProfileIdForSourceType(sourceType) {
+    if (sourceType === "github_repo") {
+        return "builtin.github_repo";
+    }
+    if (sourceType === "github_repo_ci") {
+        return "builtin.github_repo_ci";
+    }
+    if (sourceType === "feishu_bot") {
+        return "builtin.feishu_bot";
+    }
+    return null;
+}
+function validateProfileContract(profile, sourcePath) {
+    if (!profile.id || typeof profile.id !== "string") {
+        throw new Error(`remote_source profile id must be a non-empty string: ${sourcePath}`);
+    }
+    if (BUILTIN_PROFILE_IDS.has(profile.id)) {
+        throw new Error(`remote_source profile id is reserved: ${profile.id}`);
+    }
+    if (typeof profile.validateConfig !== "function") {
+        throw new Error(`remote_source profile missing validateConfig(): ${sourcePath}`);
+    }
+    if (typeof profile.buildManagedSourceSpec !== "function") {
+        throw new Error(`remote_source profile missing buildManagedSourceSpec(): ${sourcePath}`);
+    }
+    if (typeof profile.mapRawEvent !== "function") {
+        throw new Error(`remote_source profile missing mapRawEvent(): ${sourcePath}`);
+    }
+}
+function resolveAndValidateProfilePath(profileRoot, profilePath) {
+    const resolved = node_path_1.default.isAbsolute(profilePath)
+        ? node_path_1.default.resolve(profilePath)
+        : node_path_1.default.resolve(profileRoot, profilePath);
+    const normalizedRoot = ensureTrailingSep(node_path_1.default.resolve(profileRoot));
+    const normalizedResolved = node_path_1.default.resolve(resolved);
+    if (!normalizedResolved.startsWith(normalizedRoot) && normalizedResolved !== node_path_1.default.resolve(profileRoot)) {
+        throw new Error(`remote_source profilePath must stay under ${profileRoot}`);
+    }
+    return normalizedResolved;
+}
+function ensureTrailingSep(input) {
+    return input.endsWith(node_path_1.default.sep) ? input : `${input}${node_path_1.default.sep}`;
+}
+function asRecord(value) {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        return {};
+    }
+    return value;
+}
+function asNonEmptyString(value) {
+    if (typeof value !== "string") {
+        return null;
+    }
+    const trimmed = value.trim();
+    return trimmed.length > 0 ? trimmed : null;
+}
+const GITHUB_REPO_PROFILE = {
+    id: "builtin.github_repo",
+    validateConfig(source) {
+        (0, github_1.parseGithubSourceConfig)(source);
+    },
+    buildManagedSourceSpec(source) {
+        const config = (0, github_1.parseGithubSourceConfig)(source);
+        return {
+            endpoint: github_1.GITHUB_ENDPOINT,
+            operation_id: "get:/repos/{owner}/{repo}/events",
+            args: {
+                owner: config.owner,
+                repo: config.repo,
+                per_page: config.perPage ?? 10,
+            },
+            mode: "poll",
+            poll_config: {
+                interval_secs: config.pollIntervalSecs ?? 30,
+                extract_items_pointer: "",
+                checkpoint_strategy: {
+                    type: "item_key",
+                    item_key_pointer: "/id",
+                    seen_window: 1024,
+                },
+            },
+            options: { auth: config.uxcAuth },
+        };
+    },
+    mapRawEvent(rawPayload, source) {
+        const config = (0, github_1.parseGithubSourceConfig)(source);
+        const normalized = (0, github_1.normalizeGithubRepoEvent)(source, config, rawPayload);
+        if (!normalized) {
+            return null;
+        }
+        return {
+            sourceNativeId: normalized.sourceNativeId,
+            eventVariant: normalized.eventVariant,
+            metadata: normalized.metadata ?? {},
+            rawPayload: normalized.rawPayload ?? rawPayload,
+            occurredAt: normalized.occurredAt,
+            deliveryHandle: normalized.deliveryHandle,
+        };
+    },
+};
+const GITHUB_REPO_CI_PROFILE = {
+    id: "builtin.github_repo_ci",
+    validateConfig(source) {
+        (0, github_ci_1.parseGithubCiSourceConfig)(source);
+    },
+    buildManagedSourceSpec(source) {
+        const config = (0, github_ci_1.parseGithubCiSourceConfig)(source);
+        const args = {
+            owner: config.owner,
+            repo: config.repo,
+            per_page: config.perPage ?? github_ci_1.DEFAULT_GITHUB_CI_PER_PAGE,
+        };
+        if (config.eventFilter) {
+            args.event = config.eventFilter;
+        }
+        if (config.branch) {
+            args.branch = config.branch;
+        }
+        if (config.statusFilter) {
+            args.status = config.statusFilter;
+        }
+        return {
+            endpoint: github_ci_1.GITHUB_CI_ENDPOINT,
+            operation_id: "get:/repos/{owner}/{repo}/actions/runs",
+            args,
+            mode: "poll",
+            poll_config: {
+                interval_secs: config.pollIntervalSecs ?? github_ci_1.DEFAULT_GITHUB_CI_POLL_INTERVAL_SECS,
+                extract_items_pointer: "/workflow_runs",
+                checkpoint_strategy: {
+                    type: "content_hash",
+                    seen_window: 1024,
+                },
+            },
+            options: { auth: config.uxcAuth },
+        };
+    },
+    mapRawEvent(rawPayload, source) {
+        const config = (0, github_ci_1.parseGithubCiSourceConfig)(source);
+        const normalized = (0, github_ci_1.normalizeGithubWorkflowRunEvent)(source, config, rawPayload);
+        if (!normalized) {
+            return null;
+        }
+        return {
+            sourceNativeId: normalized.sourceNativeId,
+            eventVariant: normalized.eventVariant,
+            metadata: normalized.metadata ?? {},
+            rawPayload: normalized.rawPayload ?? rawPayload,
+            occurredAt: normalized.occurredAt,
+            deliveryHandle: normalized.deliveryHandle,
+        };
+    },
+};
+const FEISHU_BOT_PROFILE = {
+    id: "builtin.feishu_bot",
+    validateConfig(source) {
+        (0, feishu_1.parseFeishuSourceConfig)(source);
+    },
+    buildManagedSourceSpec(source) {
+        const config = (0, feishu_1.parseFeishuSourceConfig)(source);
+        return {
+            endpoint: config.endpoint ?? feishu_1.FEISHU_OPENAPI_ENDPOINT,
+            mode: "stream",
+            transport_hint: "feishu_long_connection",
+            options: { auth: config.uxcAuth },
+        };
+    },
+    mapRawEvent(rawPayload, source) {
+        const config = (0, feishu_1.parseFeishuSourceConfig)(source);
+        const normalized = (0, feishu_1.normalizeFeishuBotEvent)(source, config, rawPayload);
+        if (!normalized) {
+            return null;
+        }
+        return {
+            sourceNativeId: normalized.sourceNativeId,
+            eventVariant: normalized.eventVariant,
+            metadata: normalized.metadata ?? {},
+            rawPayload: normalized.rawPayload ?? rawPayload,
+            occurredAt: normalized.occurredAt,
+            deliveryHandle: normalized.deliveryHandle,
+        };
+    },
+};
+function profileConfigForSource(source) {
+    const config = asRecord(source.config);
+    if (source.sourceType === "remote_source") {
+        return asRecord(config.profileConfig);
+    }
+    return config;
+}