@relayfile/core 0.1.0

package/dist/webhooks.js

@@ -0,0 +1,493 @@
+/**
+ * Webhook envelope normalization and processing.
+ *
+ * Extract from workspace.ts:
+ *   - generic webhook ingestion
+ *   - envelope normalization
+ *   - deduplication by delivery_id
+ *   - coalescing within time window
+ *   - suppression for loop prevention
+ *   - stale event filtering
+ *   - file materialization from webhook data
+ *
+ * This module stays pure. It only operates over plain storage records and
+ * optional callbacks for suppression/staleness policy.
+ */
+import { normalizePath, DEFAULT_CONTENT_TYPE, MAX_FILE_BYTES, encodedSize } from "./files.js";
+export function ingestWebhook(storage, input, options = {}) {
+    const envelopeStorage = getWebhookStorage(storage);
+    const correlationId = input.correlationId?.trim() ?? "";
+    if (options.signatureVerifier && !options.signatureVerifier(input)) {
+        return {
+            status: "signature_invalid",
+            envelopeId: "",
+            correlationId,
+        };
+    }
+    if (!envelopeStorage) {
+        return {
+            status: "invalid_storage",
+            envelopeId: "",
+            correlationId,
+        };
+    }
+    const now = options.now ?? nowIso;
+    const normalized = normalizeEnvelope(input, now);
+    const provider = asOptionalString(normalized.provider) ?? "";
+    const payload = asRecord(normalized.payload);
+    const path = normalizeEnvelopePath({ payload });
+    const receivedAt = asOptionalString(normalized.receivedAt) ?? now();
+    const workspaceId = storage.getWorkspaceId();
+    const deliveryId = asOptionalString(normalized.deliveryId) ??
+        defaultDeliveryId(provider, receivedAt);
+    if (!provider || !path || !normalizeEnvelopeEventType(payload)) {
+        return {
+            status: "invalid_input",
+            envelopeId: "",
+            correlationId,
+        };
+    }
+    const existing = findEnvelopeByDelivery(envelopeStorage, workspaceId, provider, deliveryId);
+    if (existing) {
+        return {
+            status: "duplicate",
+            envelopeId: existing.envelopeId,
+            correlationId: existing.correlationId || correlationId,
+        };
+    }
+    const queued = envelopeStorage.listEnvelopes({
+        workspaceId,
+        provider,
+        status: "queued",
+        limit: 100,
+    }).items;
+    const coalesced = findCoalescedEnvelope(queued, payload, receivedAt, options.coalesceWindowMs ?? DEFAULT_COALESCE_WINDOW_MS);
+    if (coalesced) {
+        const updated = {
+            ...coalesced,
+            deliveryId: coalesced.deliveryId,
+            deliveryIds: mergeDeliveryIds(coalesced, deliveryId),
+            receivedAt,
+            headers: normalized.headers ?? coalesced.headers ?? {},
+            payload,
+            correlationId: correlationId || coalesced.correlationId,
+            status: "queued",
+            lastError: null,
+        };
+        envelopeStorage.putEnvelope(updated);
+        envelopeStorage.putEnvelopeDeliveryAlias?.(workspaceId, provider, deliveryId, updated.envelopeId);
+        return {
+            status: "queued",
+            envelopeId: updated.envelopeId,
+            correlationId: updated.correlationId,
+        };
+    }
+    const envelopeId = (options.generateEnvelopeId ?? defaultEnvelopeId)();
+    envelopeStorage.putEnvelope({
+        envelopeId,
+        workspaceId,
+        provider,
+        deliveryId,
+        deliveryIds: [deliveryId],
+        receivedAt,
+        headers: normalized.headers ?? {},
+        payload,
+        correlationId,
+        status: "queued",
+        attemptCount: 0,
+        lastError: null,
+    });
+    envelopeStorage.putEnvelopeDeliveryAlias?.(workspaceId, provider, deliveryId, envelopeId);
+    return {
+        status: "queued",
+        envelopeId,
+        correlationId,
+    };
+}
+export function normalizeEnvelope(input, now = nowIso) {
+    const provider = normalizeProvider(input.provider);
+    const eventType = normalizeEventType(input.eventType);
+    const path = normalizePath(input.path ?? "");
+    const correlationId = input.correlationId?.trim() ?? "";
+    const receivedAt = normalizeIsoDate(input.timestamp) ?? now();
+    const deliveryId = input.deliveryId?.trim();
+    if (!provider || !eventType || !input.path?.trim()) {
+        return {
+            provider,
+            deliveryId,
+            receivedAt,
+            headers: normalizeHeaderMap(input.headers),
+            payload: {},
+            correlationId,
+        };
+    }
+    return {
+        provider,
+        deliveryId,
+        receivedAt,
+        headers: normalizeHeaderMap(input.headers),
+        payload: {
+            provider,
+            event_type: eventType,
+            path,
+            timestamp: receivedAt,
+            data: asRecord(input.data),
+            delivery_id: deliveryId,
+        },
+        correlationId,
+    };
+}
+export function normalizeEnvelopeEvent(envelope) {
+    const payload = envelope.payload;
+    const eventType = normalizeEnvelopeEventType(payload);
+    if (!eventType) {
+        return null;
+    }
+    const path = normalizePath(asOptionalString(payload.path) ?? "/");
+    const data = asRecord(payload.data);
+    const body = Object.keys(data).length > 0 ? data : payload;
+    const timestamp = normalizeIsoDate(asOptionalString(payload.timestamp) ?? envelope.receivedAt) ??
+        nowIso();
+    const content = typeof body.content === "string" ? body.content : undefined;
+    const contentType = typeof body.contentType === "string"
+        ? body.contentType
+        : typeof body.content_type === "string"
+            ? body.content_type
+            : undefined;
+    const encoding = typeof body.encoding === "string" ? body.encoding : undefined;
+    const semantics = body.semantics && typeof body.semantics === "object" && !Array.isArray(body.semantics)
+        ? normalizeSemantics(body.semantics)
+        : undefined;
+    return {
+        type: eventType,
+        path,
+        timestamp,
+        content,
+        contentType,
+        encoding,
+        semantics,
+        data: body,
+    };
+}
+export function normalizeEnvelopePath(envelope) {
+    const path = asOptionalString(envelope.payload.path);
+    return path ? normalizePath(path) : null;
+}
+export function applyWebhookEnvelope(storage, envelope, options = {}) {
+    const event = normalizeEnvelopeEvent(envelope);
+    if (!event) {
+        return {
+            status: "ignored",
+            eventType: null,
+            path: normalizeEnvelopePath(envelope),
+            revision: null,
+        };
+    }
+    if (options.shouldSuppress?.(envelope, event)) {
+        const revision = appendSyncEvent(storage, "sync.suppressed", event.path, envelope.provider, envelope.correlationId, options.now?.() ?? event.timestamp);
+        return {
+            status: "suppressed",
+            eventType: "sync.suppressed",
+            path: event.path,
+            revision,
+        };
+    }
+    if (options.isStale?.(envelope, event)) {
+        const revision = appendSyncEvent(storage, "sync.stale", event.path, envelope.provider, envelope.correlationId, options.now?.() ?? event.timestamp);
+        return {
+            status: "stale",
+            eventType: "sync.stale",
+            path: event.path,
+            revision,
+        };
+    }
+    if (event.type === "file.created" || event.type === "file.updated") {
+        if (options.isPathWriteAllowed && !options.isPathWriteAllowed(event.path)) {
+            return {
+                status: "ignored",
+                eventType: event.type,
+                path: event.path,
+                revision: null,
+            };
+        }
+        const revision = storage.nextRevision();
+        const content = typeof event.content === "string"
+            ? event.content
+            : JSON.stringify(event.data ?? {});
+        const encoding = normalizeEncoding(event.encoding) ?? "utf-8";
+        if (encodedSize(content, encoding) > MAX_FILE_BYTES) {
+            return {
+                status: "rejected",
+                eventType: event.type,
+                path: event.path,
+                revision: null,
+                reason: "file_too_large",
+            };
+        }
+        // Strip permissions from webhook-provided semantics to prevent
+        // external webhooks from injecting or overwriting ACL rules.
+        const semantics = normalizeSemantics(event.semantics);
+        delete semantics.permissions;
+        storage.putFile({
+            path: event.path,
+            revision,
+            contentType: event.contentType?.trim() || DEFAULT_CONTENT_TYPE,
+            content,
+            encoding,
+            provider: envelope.provider,
+            lastEditedAt: event.timestamp,
+            semantics,
+        });
+        storage.appendEvent({
+            eventId: storage.nextEventId(),
+            type: event.type,
+            path: event.path,
+            revision,
+            origin: "provider_sync",
+            provider: envelope.provider,
+            correlationId: envelope.correlationId,
+            timestamp: event.timestamp,
+        });
+        return {
+            status: "processed",
+            eventType: event.type,
+            path: event.path,
+            revision,
+        };
+    }
+    if (event.type === "file.deleted") {
+        if (options.isPathWriteAllowed && !options.isPathWriteAllowed(event.path)) {
+            return {
+                status: "ignored",
+                eventType: event.type,
+                path: event.path,
+                revision: null,
+            };
+        }
+        storage.deleteFile(event.path);
+        const revision = appendSyncEvent(storage, "file.deleted", event.path, envelope.provider, envelope.correlationId, event.timestamp);
+        return {
+            status: "processed",
+            eventType: "file.deleted",
+            path: event.path,
+            revision,
+        };
+    }
+    const revision = appendSyncEvent(storage, event.type, event.path, envelope.provider, envelope.correlationId, event.timestamp);
+    return {
+        status: "processed",
+        eventType: event.type,
+        path: event.path,
+        revision,
+    };
+}
+const DEFAULT_COALESCE_WINDOW_MS = 3_000;
+const ENVELOPE_SCAN_PAGE_SIZE = 100;
+const MAX_ENVELOPE_SCAN_PAGES = 1000;
+const VALID_EVENT_TYPES = new Set([
+    "file.created",
+    "file.updated",
+    "file.deleted",
+    "dir.created",
+    "dir.deleted",
+    "sync.error",
+    "sync.ignored",
+    "sync.suppressed",
+    "sync.stale",
+    "writeback.failed",
+    "writeback.succeeded",
+]);
+function getWebhookStorage(storage) {
+    if (typeof storage.getEnvelopeByDelivery !== "function" ||
+        typeof storage.putEnvelope !== "function" ||
+        typeof storage.listEnvelopes !== "function") {
+        return null;
+    }
+    return storage;
+}
+function findEnvelopeByDelivery(storage, workspaceId, provider, deliveryId) {
+    const direct = storage.getEnvelopeByDelivery(workspaceId, provider, deliveryId);
+    if (direct) {
+        return direct;
+    }
+    let cursor;
+    let pagesScanned = 0;
+    for (;;) {
+        if (pagesScanned >= MAX_ENVELOPE_SCAN_PAGES) {
+            return null;
+        }
+        const page = storage.listEnvelopes({
+            workspaceId,
+            provider,
+            cursor,
+            limit: ENVELOPE_SCAN_PAGE_SIZE,
+        });
+        pagesScanned += 1;
+        for (const envelope of page.items) {
+            if (deliveryMatches(envelope, deliveryId)) {
+                return envelope;
+            }
+        }
+        if (!page.nextCursor || page.nextCursor === cursor) {
+            return null;
+        }
+        cursor = page.nextCursor;
+    }
+}
+function appendSyncEvent(storage, type, path, provider, correlationId, timestamp) {
+    const revision = storage.nextRevision();
+    storage.appendEvent({
+        eventId: storage.nextEventId(),
+        type,
+        path,
+        revision,
+        origin: "provider_sync",
+        provider,
+        correlationId,
+        timestamp,
+    });
+    return revision;
+}
+function findCoalescedEnvelope(envelopes, payload, receivedAt, windowMs) {
+    const key = coalesceObjectKey(payload);
+    if (!key) {
+        return null;
+    }
+    let match = null;
+    for (const envelope of envelopes) {
+        if (coalesceObjectKey(envelope.payload) !== key) {
+            continue;
+        }
+        if (!withinCoalesceWindow(envelope.receivedAt, receivedAt, windowMs)) {
+            continue;
+        }
+        if (!match || envelope.receivedAt > match.receivedAt) {
+            match = envelope;
+        }
+    }
+    return match;
+}
+function mergeDeliveryIds(envelope, deliveryId) {
+    return Array.from(new Set([envelope.deliveryId, ...(envelope.deliveryIds ?? []), deliveryId]));
+}
+function deliveryMatches(envelope, deliveryId) {
+    return envelope.deliveryId === deliveryId || envelope.deliveryIds?.includes(deliveryId) === true;
+}
+function coalesceObjectKey(payload) {
+    const data = asRecord(payload.data);
+    const providerObjectId = asOptionalString(data.providerObjectId) ??
+        asOptionalString(data.provider_object_id) ??
+        asOptionalString(data.objectId) ??
+        asOptionalString(payload.providerObjectId) ??
+        asOptionalString(payload.provider_object_id) ??
+        asOptionalString(payload.objectId);
+    if (providerObjectId) {
+        return `object:${providerObjectId}`;
+    }
+    const path = normalizeEnvelopePath({ payload });
+    return path && path !== "/" ? `path:${path}` : "";
+}
+function withinCoalesceWindow(existingReceivedAt, incomingReceivedAt, windowMs) {
+    if (windowMs <= 0) {
+        return false;
+    }
+    const existingTs = Date.parse(existingReceivedAt);
+    const incomingTs = Date.parse(incomingReceivedAt);
+    if (Number.isNaN(existingTs) || Number.isNaN(incomingTs)) {
+        return false;
+    }
+    if (incomingTs < existingTs) {
+        return false;
+    }
+    return incomingTs - existingTs <= windowMs;
+}
+function normalizeProvider(provider) {
+    return provider?.trim().toLowerCase() ?? "";
+}
+function normalizeEncoding(encoding) {
+    const value = encoding?.trim().toLowerCase() ?? "";
+    if (!value || value === "utf-8" || value === "utf8") {
+        return "utf-8";
+    }
+    if (value === "base64") {
+        return "base64";
+    }
+    return null;
+}
+function normalizeEventType(value) {
+    const type = value?.trim() ?? "";
+    return VALID_EVENT_TYPES.has(type) ? type : null;
+}
+function normalizeEnvelopeEventType(payload) {
+    return normalizeEventType(asOptionalString(payload.event_type) ??
+        asOptionalString(payload.eventType) ??
+        asOptionalString(payload.type) ??
+        undefined);
+}
+function normalizeIsoDate(value) {
+    if (!value?.trim()) {
+        return null;
+    }
+    const timestamp = Date.parse(value);
+    return Number.isNaN(timestamp) ? null : new Date(timestamp).toISOString();
+}
+function normalizeHeaderMap(headers) {
+    if (!headers) {
+        return {};
+    }
+    return Object.entries(headers).reduce((acc, [key, value]) => {
+        const normalizedKey = key.trim();
+        if (normalizedKey) {
+            acc[normalizedKey] = String(value);
+        }
+        return acc;
+    }, {});
+}
+function normalizeSemantics(input) {
+    return {
+        properties: normalizeProperties(input?.properties),
+        relations: normalizeStringArray(input?.relations),
+        permissions: normalizeStringArray(input?.permissions),
+        comments: normalizeStringArray(input?.comments),
+    };
+}
+function normalizeProperties(input) {
+    if (!input) {
+        return undefined;
+    }
+    const out = {};
+    for (const [key, value] of Object.entries(input)) {
+        const normalizedKey = key.trim();
+        if (!normalizedKey) {
+            continue;
+        }
+        out[normalizedKey] = String(value).trim();
+    }
+    return Object.keys(out).length > 0 ? out : undefined;
+}
+function normalizeStringArray(values) {
+    if (!values || values.length === 0) {
+        return undefined;
+    }
+    const normalized = Array.from(new Set(values.map((value) => value.trim()).filter(Boolean)));
+    normalized.sort((left, right) => left.localeCompare(right));
+    return normalized.length > 0 ? normalized : undefined;
+}
+function asOptionalString(value) {
+    return typeof value === "string" && value.trim() ? value.trim() : undefined;
+}
+function asRecord(value) {
+    return value && typeof value === "object" && !Array.isArray(value)
+        ? value
+        : {};
+}
+function defaultEnvelopeId() {
+    return `env_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 10)}`;
+}
+function defaultDeliveryId(provider, receivedAt) {
+    const safeProvider = provider || "webhook";
+    return `dlv_${safeProvider}_${Date.parse(receivedAt) || Date.now()}`;
+}
+function nowIso() {
+    return new Date().toISOString();
+}

package/dist/writeback.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Writeback lifecycle.
+ *
+ * Extract from workspace.ts:
+ *   - pending writeback listing (filtered by op status)
+ *   - writeback acknowledgment
+ *   - dead-letter handling
+ *   - retry logic
+ */
+import type { StorageAdapter, WritebackItem } from "./storage.js";
+export declare const MAX_WRITEBACK_ATTEMPTS = 3;
+export declare const WRITEBACK_RETRY_DELAY_MS = 30000;
+export interface DispatchWritebackCallbacks {
+    send(item: WritebackItem): void;
+    onRetryScheduled?(item: WritebackItem, nextAttemptAt: string): void;
+    now?(): string;
+}
+export declare function getPendingWritebacks(storage: StorageAdapter): WritebackItem[];
+export declare function acknowledgeWriteback(storage: StorageAdapter, itemId: string, success: boolean, errorMsg?: string, correlationId?: string, now?: () => string): {
+    status: string;
+    id: string;
+    success: boolean;
+} | null;
+export declare function dispatchWriteback(storage: StorageAdapter, opId: string, callbacks: DispatchWritebackCallbacks): boolean;

package/dist/writeback.js

@@ -0,0 +1,148 @@
+/**
+ * Writeback lifecycle.
+ *
+ * Extract from workspace.ts:
+ *   - pending writeback listing (filtered by op status)
+ *   - writeback acknowledgment
+ *   - dead-letter handling
+ *   - retry logic
+ */
+import { createEvent } from "./events.js";
+export const MAX_WRITEBACK_ATTEMPTS = 3;
+export const WRITEBACK_RETRY_DELAY_MS = 30_000;
+export function getPendingWritebacks(storage) {
+    const items = [];
+    let cursor;
+    do {
+        const page = storage.listOperations({
+            status: "pending",
+            cursor,
+            limit: 1000,
+        });
+        items.push(...page.items
+            .filter((op) => op.status === "pending")
+            .map((op) => ({
+            id: op.opId,
+            workspaceId: storage.getWorkspaceId(),
+            path: normalizePath(op.path),
+            revision: op.revision,
+            correlationId: op.correlationId || "",
+        })));
+        cursor = page.nextCursor ?? undefined;
+    } while (cursor);
+    return items;
+}
+export function acknowledgeWriteback(storage, itemId, success, errorMsg, correlationId, now = defaultNow) {
+    const op = storage.getOperation(itemId.trim());
+    if (!op) {
+        return null;
+    }
+    const timestamp = now();
+    const effectiveCorrelationId = correlationId ?? op.correlationId;
+    storage.putOperation({
+        ...op,
+        status: success ? "succeeded" : "failed",
+        nextAttemptAt: null,
+        lastError: success ? null : normalizeError(errorMsg),
+        correlationId: effectiveCorrelationId,
+    });
+    createEvent(storage, {
+        type: success ? "writeback.succeeded" : "writeback.failed",
+        path: op.path,
+        revision: op.revision,
+        origin: "system",
+        provider: op.provider,
+        correlationId: effectiveCorrelationId,
+        timestamp,
+    });
+    return { status: "acknowledged", id: op.opId, success };
+}
+export function dispatchWriteback(storage, opId, callbacks) {
+    const op = storage.getOperation(opId.trim());
+    if (!op) {
+        return false;
+    }
+    if (op.status !== "pending" && op.status !== "running") {
+        return true;
+    }
+    const now = callbacks.now ?? defaultNow;
+    const runningOp = {
+        ...op,
+        status: "running",
+        nextAttemptAt: null,
+        lastError: null,
+    };
+    storage.putOperation(runningOp);
+    const item = toWritebackItem(storage, runningOp);
+    try {
+        callbacks.send(item);
+        storage.putOperation({ ...runningOp, status: "dispatched" });
+        return true;
+    }
+    catch (error) {
+        const lastError = error instanceof Error ? error.message : "queue send failed";
+        const attemptCount = (op.attemptCount ?? 0) + 1;
+        if (attemptCount >= MAX_WRITEBACK_ATTEMPTS) {
+            const timestamp = now();
+            const deadLetteredOp = {
+                ...op,
+                status: "dead_lettered",
+                attemptCount,
+                nextAttemptAt: null,
+                lastError,
+            };
+            storage.putOperation(deadLetteredOp);
+            createEvent(storage, {
+                type: "writeback.failed",
+                path: op.path,
+                revision: op.revision,
+                origin: "system",
+                provider: op.provider,
+                correlationId: op.correlationId,
+                timestamp,
+            });
+            return false;
+        }
+        const nextAttemptAt = addDelay(now(), WRITEBACK_RETRY_DELAY_MS);
+        const pendingOp = {
+            ...op,
+            status: "pending",
+            attemptCount,
+            nextAttemptAt,
+            lastError,
+        };
+        storage.putOperation(pendingOp);
+        callbacks.onRetryScheduled?.(item, nextAttemptAt);
+        return false;
+    }
+}
+function normalizePath(path) {
+    const trimmed = path.trim();
+    if (!trimmed) {
+        return "/";
+    }
+    const prefixed = trimmed.startsWith("/") ? trimmed : `/${trimmed}`;
+    return prefixed.length > 1 ? prefixed.replace(/\/+$/, "") : "/";
+}
+function normalizeError(errorMsg) {
+    return errorMsg?.trim() || "provider reported failure";
+}
+function addDelay(isoTimestamp, delayMs) {
+    const baseMs = Date.parse(isoTimestamp);
+    if (Number.isNaN(baseMs)) {
+        throw new Error("invalid timestamp");
+    }
+    return new Date(baseMs + delayMs).toISOString();
+}
+function defaultNow() {
+    return new Date().toISOString();
+}
+function toWritebackItem(storage, item) {
+    return {
+        id: item.opId,
+        workspaceId: storage.getWorkspaceId(),
+        path: item.path,
+        revision: item.revision,
+        correlationId: item.correlationId,
+    };
+}

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@relayfile/core",
+  "version": "0.1.0",
+  "description": "Shared business logic for relayfile — file operations, ACL, queries, events, and writeback lifecycle",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "type": "module",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "prepublishOnly": "npm run build"
+  },
+  "devDependencies": {
+    "typescript": "^5.7.3",
+    "vitest": "^3.0.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/AgentWorkforce/relayfile",
+    "directory": "packages/core"
+  },
+  "license": "MIT",
+  "keywords": [
+    "relayfile",
+    "filesystem",
+    "agent",
+    "collaboration",
+    "core"
+  ],
+  "engines": {
+    "node": ">=18"
+  }
+}