@floomhq/floom-mcp-sync 1.0.6 → 1.0.8

package/dist/auto-sync.js

@@ -6,7 +6,7 @@ import { getJsonWithEtag, FloomApiError } from "./lib/api.js";
 import { sha256 } from "./lib/hash.js";
 import { assertValidSlug } from "./lib/slug.js";
 import { skillsDir, skillTargetPath } from "./lib/paths.js";
-import { ensureSyncManifestDir, manifestKey, markSynced, readSyncManifest, unmarkSynced, writeSyncManifest } from "./lib/manifest.js";
+import { ensureSyncManifestDir, manifestKey, markSynced, readSyncManifest, unmarkSynced, withSyncLock, writeSyncManifest } from "./lib/manifest.js";
 // Module-level cache: ETag from the last successful (non-304) response, plus
 // the last time we logged a heartbeat. Survives across setInterval ticks
 // inside a single MCP server process.
@@ -16,7 +16,13 @@ let lastAuthWarningAt = 0;
 const HEARTBEAT_MS = 10 * 60 * 1000; // 10 minutes
 const AUTH_WARNING_MS = 10 * 60 * 1000;
 const MANIFEST_SEGMENT_RE = /^[A-Za-z0-9._-]{1,128}$/;
+const PACKAGE_FILE_SEGMENT_RE = /^[A-Za-z0-9._-]{1,128}$/;
+const SUPPORT_DIRS = new Set(["references", "examples", "scripts", "assets"]);
 const FD_PATH_ROOT = "/proc/self/fd";
+const PACKAGE_FILE_LIMIT = 100;
+const PACKAGE_TOTAL_BYTES_LIMIT = 1_000_000;
+const PACKAGE_FILE_BYTES_LIMIT = 500_000;
+const BASE64_RE = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/;
 async function localState(path) {
     try {
         const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
@@ -25,7 +31,7 @@ async function localState(path) {
             if (!stat.isFile()) {
                 return { kind: "conflict", reason: "path is blocked by an existing local file or directory" };
             }
-            return { kind: "file", hash: sha256(await handle.readFile("utf8")) };
+            return { kind: "file", hash: sha256(await handle.readFile()) };
         }
         finally {
             await handle.close();
@@ -75,6 +81,141 @@ function validateSyncSkillShape(skill) {
         typeof candidate.library_slug !== "string") {
         throw new Error("Invalid sync response");
     }
+    if (candidate.files !== undefined && !Array.isArray(candidate.files)) {
+        throw new Error("Invalid sync response");
+    }
+    if (candidate.package_files !== undefined && !Array.isArray(candidate.package_files)) {
+        throw new Error("Invalid sync response");
+    }
+}
+function packageFilesForSkill(skill, skillFileTarget) {
+    const packageRoot = dirname(skillFileTarget);
+    if ((skill.package_files ?? skill.files ?? []).length > PACKAGE_FILE_LIMIT) {
+        throw new Error("Skill package response has too many files");
+    }
+    const out = [{
+            relativePath: "SKILL.md",
+            target: skillFileTarget,
+            content: Buffer.from(skill.body_md, "utf8"),
+            hash: sha256(skill.body_md),
+        }];
+    const seen = new Set(["SKILL.md"]);
+    let totalBytes = Buffer.byteLength(skill.body_md, "utf8");
+    for (const file of skill.package_files ?? skill.files ?? []) {
+        const relativePath = normalizePackageFilePath(file.path);
+        if (relativePath === "SKILL.md")
+            continue;
+        if (seen.has(relativePath))
+            throw new Error(`Duplicate package file: ${relativePath}`);
+        seen.add(relativePath);
+        const content = packageFileContent(file);
+        if (content.length > PACKAGE_FILE_BYTES_LIMIT)
+            throw new Error(`Package file is too large: ${file.path}`);
+        totalBytes += content.length;
+        if (totalBytes > PACKAGE_TOTAL_BYTES_LIMIT)
+            throw new Error("Skill package response is too large");
+        out.push({
+            relativePath,
+            target: join(packageRoot, ...relativePath.split("/")),
+            content,
+            hash: sha256(content),
+        });
+    }
+    return out;
+}
+function packageFileContent(file) {
+    if (typeof file.content_base64 === "string") {
+        if (file.encoding !== undefined && file.encoding !== "base64") {
+            throw new Error(`Invalid package file encoding: ${file.path}`);
+        }
+        if (file.content_base64.length % 4 !== 0 || !BASE64_RE.test(file.content_base64)) {
+            throw new Error(`Invalid package file base64: ${file.path}`);
+        }
+        const content = Buffer.from(file.content_base64, "base64");
+        if (typeof file.size_bytes === "number" && file.size_bytes !== content.length) {
+            throw new Error(`Package file size mismatch: ${file.path}`);
+        }
+        const expectedHash = file.sha256 ?? file.content_hash;
+        if (typeof file.sha256 !== "string") {
+            throw new Error(`Package file missing sha256: ${file.path}`);
+        }
+        if (typeof expectedHash === "string" && expectedHash !== sha256(content)) {
+            throw new Error(`Package file hash mismatch: ${file.path}`);
+        }
+        return content;
+    }
+    const content = file.content ?? file.body ?? file.body_md ?? file.text;
+    if (typeof content !== "string")
+        throw new Error(`Invalid package file: ${file.path}`);
+    const bytes = Buffer.from(content, "utf8");
+    if (typeof file.content_hash === "string" && file.content_hash !== sha256(bytes)) {
+        throw new Error(`Package file hash mismatch: ${file.path}`);
+    }
+    return bytes;
+}
+function normalizePackageFilePath(path) {
+    if (typeof path !== "string" || path.length === 0 || path.length > 512) {
+        throw new Error("Invalid package file path");
+    }
+    if (isAbsolute(path) || path.includes("\\"))
+        throw new Error("Invalid package file path");
+    const segments = path.split("/").filter(Boolean);
+    if (segments.length === 1 && segments[0] === "SKILL.md")
+        return "SKILL.md";
+    const first = segments[0];
+    if (segments.length < 2 || first === undefined || !SUPPORT_DIRS.has(first)) {
+        throw new Error("Invalid package file path");
+    }
+    if (segments.some((segment) => segment === "." || segment === ".." || !PACKAGE_FILE_SEGMENT_RE.test(segment))) {
+        throw new Error("Invalid package file path");
+    }
+    return segments.join("/");
+}
+async function planPackageSync(root, files, manifest) {
+    let missing = 0;
+    let unchanged = 0;
+    let firstMissingTarget = null;
+    for (const file of files) {
+        const targetKey = manifestKey(root, file.target);
+        const tracked = manifest.files[targetKey];
+        try {
+            await assertSafeExistingParentDirectory(root, file.target);
+        }
+        catch (err) {
+            const code = err.code;
+            if (code === "ELOOP")
+                return { kind: "conflict", target: file.target, reason: "path contains a symbolic link" };
+            if (code === "ENOTDIR" || code === "EISDIR")
+                return { kind: "conflict", target: file.target, reason: "path is blocked by an existing local file or directory" };
+            if (code === "EEXIST" || code === "ENOENT")
+                return { kind: "conflict", target: file.target, reason: err instanceof Error ? err.message : "local file changed during Floom sync" };
+            throw err;
+        }
+        const state = await localState(file.target);
+        if (state.kind === "conflict")
+            return { kind: "conflict", target: file.target, reason: state.reason };
+        if (state.kind === "missing") {
+            firstMissingTarget ??= file.target;
+            missing += 1;
+            continue;
+        }
+        if (!tracked)
+            return { kind: "conflict", target: file.target, reason: "existing file is not tracked by Floom sync" };
+        if (state.hash !== tracked.hash)
+            return { kind: "conflict", target: file.target, reason: "local file changed since the last Floom sync" };
+        if (state.hash !== file.hash)
+            return { kind: "conflict", target: file.target, reason: "remote skill changed; move or delete the local file to accept the Floom version" };
+        unchanged += 1;
+    }
+    if (unchanged === files.length)
+        return { kind: "unchanged" };
+    if (missing === files.length)
+        return { kind: "write" };
+    return {
+        kind: "conflict",
+        target: firstMissingTarget ?? files[0]?.target ?? root,
+        reason: "local package is only partially installed",
+    };
 }
 async function manifestHasMissingTrackedFile(manifest, root) {
     for (const key of Object.keys(manifest.files)) {
@@ -99,219 +240,192 @@ export async function autoSync(log = (message) => process.stderr.write(`${messag
         return { synced: 0, unchanged: 0, updated: 0, conflicts: 0 };
     }
     await ensureSyncManifestDir();
-    const manifest = await readSyncManifest();
-    const root = skillsDir();
-    const apiUrl = apiUrlFromConfig(cfg);
-    let response;
-    try {
-        response = await getJsonWithEtag(`${apiUrl}/api/v1/me/skills`, cfg.accessToken, cachedEtag);
-    }
-    catch (err) {
-        if (err instanceof FloomApiError && err.status === 401) {
-            maybeAuthWarning(log, "[floom] sign-in expired; skipping sync (run `npx -y @floomhq/floom login` again)");
-            return { synced: 0, unchanged: 0, updated: 0, conflicts: 0 };
+    return await withSyncLock(async () => {
+        const manifest = await readSyncManifest();
+        const root = skillsDir();
+        const apiUrl = apiUrlFromConfig(cfg);
+        let response;
+        try {
+            response = await getJsonWithEtag(`${apiUrl}/api/v1/me/skills`, cfg.accessToken, cachedEtag);
         }
-        throw err;
-    }
-    if (response.status === 304) {
-        if (await manifestHasMissingTrackedFile(manifest, root)) {
-            response = await getJsonWithEtag(`${apiUrl}/api/v1/me/skills`, cfg.accessToken, null);
+        catch (err) {
+            if (err instanceof FloomApiError && err.status === 401) {
+                maybeAuthWarning(log, "[floom] sign-in expired; skipping sync (run `npx -y @floomhq/floom login` again)");
+                return { synced: 0, unchanged: 0, updated: 0, conflicts: 0 };
+            }
+            throw err;
         }
-        else {
+        if (response.status === 304) {
+            if (await manifestHasMissingTrackedFile(manifest, root)) {
+                response = await getJsonWithEtag(`${apiUrl}/api/v1/me/skills`, cfg.accessToken, null);
+            }
+            else {
+                maybeHeartbeat(log);
+                return { synced: 0, unchanged: 0, updated: 0, conflicts: 0 };
+            }
+        }
+        if (response.status === 304) {
             maybeHeartbeat(log);
             return { synced: 0, unchanged: 0, updated: 0, conflicts: 0 };
         }
-    }
-    if (response.status === 304) {
-        maybeHeartbeat(log);
-        return { synced: 0, unchanged: 0, updated: 0, conflicts: 0 };
-    }
-    if (response.etag)
-        cachedEtag = response.etag;
-    const payload = response.body;
-    if (!payload) {
-        return { synced: 0, unchanged: 0, updated: 0, conflicts: 0 };
-    }
-    await mkdir(skillsDir(), { recursive: true, mode: 0o700 });
-    if (!Array.isArray(payload.skills)) {
-        throw new Error("Invalid sync response");
-    }
-    for (const skill of payload.skills)
-        validateSyncSkillShape(skill);
-    // Version 1 preview syncs owned published skills only.
-    const buckets = [
-        { skills: payload.skills, defaultLib: null },
-    ];
-    let unchanged = 0;
-    let updated = 0;
-    let conflicts = 0;
-    let total = 0;
-    const writtenPaths = new Set();
-    const activeTargetKeys = new Set();
-    const pruneBlockedSlugs = new Set();
-    const noteConflict = (target, reason) => {
-        conflicts += 1;
-        log(`[floom] skipped local conflict: ${manifestKey(root, target)} (${reason})`);
-    };
-    const noteKeyConflict = (key, reason) => {
-        conflicts += 1;
-        log(`[floom] skipped local conflict: ${key} (${reason})`);
-    };
-    const noteRemoteConflict = (slug, reason) => {
-        conflicts += 1;
-        const label = typeof slug === "string" && slug ? slug : "<invalid>";
-        log(`[floom] skipped remote conflict: ${label} (${reason})`);
-    };
-    for (const bucket of buckets) {
-        const skills = bucket.skills ?? [];
-        for (const skill of skills) {
-            let target;
-            try {
-                assertValidSlug(skill.slug);
-                target = skillTargetPath({
-                    slug: skill.slug,
-                    folder: skill.folder ?? null,
-                    librarySlug: skill.library_slug ?? bucket.defaultLib,
-                });
-            }
-            catch (err) {
-                if (typeof skill.slug === "string")
-                    pruneBlockedSlugs.add(skill.slug);
-                noteRemoteConflict(skill.slug, err instanceof Error ? err.message : "invalid remote metadata");
-                continue;
-            }
-            if (writtenPaths.has(target))
-                continue;
-            writtenPaths.add(target);
-            total += 1;
-            const remoteHash = sha256(skill.body_md);
-            const targetKey = manifestKey(root, target);
-            activeTargetKeys.add(targetKey);
-            const tracked = manifest.files[targetKey];
-            try {
-                await assertSafeExistingParentDirectory(root, target);
-            }
-            catch (err) {
-                const code = err.code;
-                if (code === "ELOOP") {
-                    noteConflict(target, "path contains a symbolic link");
+        if (response.etag)
+            cachedEtag = response.etag;
+        const payload = response.body;
+        if (!payload) {
+            return { synced: 0, unchanged: 0, updated: 0, conflicts: 0 };
+        }
+        await mkdir(skillsDir(), { recursive: true, mode: 0o700 });
+        if (!Array.isArray(payload.skills)) {
+            throw new Error("Invalid sync response");
+        }
+        for (const skill of payload.skills)
+            validateSyncSkillShape(skill);
+        // Version 1 preview syncs owned published skills only.
+        const buckets = [
+            { skills: payload.skills, defaultLib: null },
+        ];
+        let unchanged = 0;
+        let updated = 0;
+        let conflicts = 0;
+        let total = 0;
+        const writtenPaths = new Set();
+        const activeTargetKeys = new Set();
+        const pruneBlockedSlugs = new Set();
+        const noteConflict = (target, reason) => {
+            conflicts += 1;
+            log(`[floom] skipped local conflict: ${manifestKey(root, target)} (${reason})`);
+        };
+        const noteKeyConflict = (key, reason) => {
+            conflicts += 1;
+            log(`[floom] skipped local conflict: ${key} (${reason})`);
+        };
+        const noteRemoteConflict = (slug, reason) => {
+            conflicts += 1;
+            const label = typeof slug === "string" && slug ? slug : "<invalid>";
+            log(`[floom] skipped remote conflict: ${label} (${reason})`);
+        };
+        for (const bucket of buckets) {
+            const skills = bucket.skills ?? [];
+            for (const skill of skills) {
+                let target;
+                let packageFiles;
+                try {
+                    assertValidSlug(skill.slug);
+                    target = skillTargetPath({
+                        slug: skill.slug,
+                        folder: skill.folder ?? null,
+                        librarySlug: skill.library_slug ?? bucket.defaultLib,
+                    });
+                    packageFiles = packageFilesForSkill(skill, target);
+                }
+                catch (err) {
+                    if (typeof skill.slug === "string")
+                        pruneBlockedSlugs.add(skill.slug);
+                    noteRemoteConflict(skill.slug, err instanceof Error ? err.message : "invalid remote metadata");
                     continue;
                 }
-                if (code === "ENOTDIR" || code === "EISDIR") {
-                    noteConflict(target, "path is blocked by an existing local file or directory");
+                if (writtenPaths.has(target))
+                    continue;
+                writtenPaths.add(target);
+                total += 1;
+                for (const file of packageFiles)
+                    activeTargetKeys.add(manifestKey(root, file.target));
+                const plan = await planPackageSync(root, packageFiles, manifest);
+                if (plan.kind === "conflict") {
+                    noteConflict(plan.target, plan.reason);
                     continue;
                 }
-                if (code === "EEXIST" || code === "ENOENT") {
-                    noteConflict(target, err instanceof Error ? err.message : "local file changed during Floom sync");
+                if (plan.kind === "unchanged") {
+                    unchanged += 1;
                     continue;
                 }
-                throw err;
-            }
-            const state = await localState(target);
-            if (state.kind === "conflict") {
-                noteConflict(target, state.reason);
-                continue;
-            }
-            if (state.kind === "file" && !tracked) {
-                noteConflict(target, "existing file is not tracked by Floom sync");
-                continue;
-            }
-            if (state.kind === "file" && state.hash !== tracked?.hash) {
-                noteConflict(target, "local file changed since the last Floom sync");
-                continue;
-            }
-            if (state.kind === "file" && state.hash === remoteHash) {
-                unchanged += 1;
-                continue;
-            }
-            if (state.kind === "file") {
-                noteConflict(target, "remote skill changed; move or delete the local file to accept the Floom version");
-                continue;
-            }
-            try {
-                await writeSyncedFile(target, skill.body_md);
+                try {
+                    for (const file of packageFiles)
+                        await writeSyncedFile(file.target, file.content);
+                }
+                catch (err) {
+                    const code = err.code;
+                    if (code === "ELOOP") {
+                        noteConflict(target, "path contains a symbolic link");
+                        continue;
+                    }
+                    if (code === "ENOTDIR" || code === "EISDIR" || code === "EEXIST") {
+                        noteConflict(target, "path is blocked by an existing local file or directory");
+                        continue;
+                    }
+                    throw err;
+                }
+                for (const file of packageFiles)
+                    markSynced(manifest, manifestKey(root, file.target), skill.slug, file.hash);
+                await writeSyncManifest(manifest);
+                updated += 1;
             }
-            catch (err) {
-                const code = err.code;
-                if (code === "ELOOP") {
-                    noteConflict(target, "path contains a symbolic link");
+        }
+        if (payload.full_sync === true) {
+            for (const [key, entry] of Object.entries(manifest.files)) {
+                if (activeTargetKeys.has(key))
+                    continue;
+                if (pruneBlockedSlugs.has(entry.slug)) {
+                    noteKeyConflict(key, "remote metadata is invalid for this skill");
                     continue;
                 }
-                if (code === "ENOTDIR" || code === "EISDIR") {
-                    noteConflict(target, "path is blocked by an existing local file or directory");
+                let target;
+                try {
+                    target = targetFromManifestKey(root, key);
+                    await assertSafeExistingParentDirectory(root, target);
+                }
+                catch (err) {
+                    const code = err.code;
+                    if (code === "ELOOP") {
+                        noteKeyConflict(key, "path contains a symbolic link");
+                        continue;
+                    }
+                    if (code === "ENOTDIR" || code === "EISDIR") {
+                        noteKeyConflict(key, "path is blocked by an existing local file or directory");
+                        continue;
+                    }
+                    noteKeyConflict(key, "invalid manifest target path");
                     continue;
                 }
-                throw err;
-            }
-            markSynced(manifest, targetKey, skill.slug, remoteHash);
-            await writeSyncManifest(manifest);
-            updated += 1;
-        }
-    }
-    if (payload.full_sync === true) {
-        for (const [key, entry] of Object.entries(manifest.files)) {
-            if (activeTargetKeys.has(key))
-                continue;
-            if (pruneBlockedSlugs.has(entry.slug)) {
-                noteKeyConflict(key, "remote metadata is invalid for this skill");
-                continue;
-            }
-            let target;
-            try {
-                target = targetFromManifestKey(root, key);
-                await assertSafeExistingParentDirectory(root, target);
-            }
-            catch (err) {
-                const code = err.code;
-                if (code === "ELOOP") {
-                    noteKeyConflict(key, "path contains a symbolic link");
+                const state = await localState(target);
+                if (state.kind === "missing") {
+                    unmarkSynced(manifest, key);
+                    await writeSyncManifest(manifest);
                     continue;
                 }
-                if (code === "ENOTDIR" || code === "EISDIR") {
-                    noteKeyConflict(key, "path is blocked by an existing local file or directory");
+                if (state.kind === "conflict") {
+                    noteConflict(target, state.reason);
+                    continue;
+                }
+                if (state.hash !== entry.hash) {
+                    noteConflict(target, "local file changed since the last Floom sync");
                     continue;
                 }
-                noteKeyConflict(key, "invalid manifest target path");
-                continue;
-            }
-            const state = await localState(target);
-            if (state.kind === "missing") {
                 unmarkSynced(manifest, key);
                 await writeSyncManifest(manifest);
-                continue;
-            }
-            if (state.kind === "conflict") {
-                noteConflict(target, state.reason);
-                continue;
-            }
-            if (state.hash !== entry.hash) {
-                noteConflict(target, "local file changed since the last Floom sync");
-                continue;
             }
-            unmarkSynced(manifest, key);
-            await writeSyncManifest(manifest);
         }
-    }
-    // Only log when there's actual movement (skills updated) OR heartbeat is due.
-    // Steady-state polling stays quiet so it doesn't pollute MCP stderr.
-    if (updated > 0) {
-        const conflictNote = conflicts > 0 ? `, ${conflicts} conflicts skipped` : "";
-        log(`[floom] synced ${total} skills (${unchanged} unchanged, ${updated} updated${conflictNote})`);
-        lastHeartbeatAt = Date.now();
+        // Only log when there's actual movement (skills updated) OR heartbeat is due.
+        // Steady-state polling stays quiet so it doesn't pollute MCP stderr.
         if (updated > 0) {
-            // Activation telemetry counts syncs that write new content. Best-effort;
-            // never blocks or throws.
-            void emitSyncCompleted(apiUrl, cfg.accessToken, { total, updated, unchanged }).catch(() => { });
+            const conflictNote = conflicts > 0 ? `, ${conflicts} conflicts skipped` : "";
+            log(`[floom] synced ${total} skills (${unchanged} unchanged, ${updated} updated${conflictNote})`);
+            lastHeartbeatAt = Date.now();
+            if (updated > 0) {
+                // Activation telemetry counts syncs that write new content. Best-effort;
+                // never blocks or throws.
+                void emitSyncCompleted(apiUrl, cfg.accessToken, { total, updated, unchanged }).catch(() => { });
+            }
         }
-    }
-    else if (conflicts > 0) {
-        log(`[floom] synced ${total} skills (${unchanged} unchanged, ${updated} updated, ${conflicts} conflicts skipped)`);
-        lastHeartbeatAt = Date.now();
-    }
-    else {
-        maybeHeartbeat(log, () => `[floom] heartbeat: ${total} skills tracked, all up-to-date`);
-    }
-    return { synced: total, unchanged, updated, conflicts };
+        else if (conflicts > 0) {
+            log(`[floom] synced ${total} skills (${unchanged} unchanged, ${updated} updated, ${conflicts} conflicts skipped)`);
+            lastHeartbeatAt = Date.now();
+        }
+        else {
+            maybeHeartbeat(log, () => `[floom] heartbeat: ${total} skills tracked, all up-to-date`);
+        }
+        return { synced: total, unchanged, updated, conflicts };
+    });
 }
 async function emitSyncCompleted(apiUrl, token, props) {
     try {
@@ -360,8 +474,7 @@ function childCreatePath(parent, fallbackParent, name) {
         return `${FD_PATH_ROOT}/${parent.fd}/${name}`;
     return join(resolve(fallbackParent), name);
 }
-async function writeAll(handle, body) {
-    const buffer = Buffer.from(body, "utf8");
+async function writeAll(handle, buffer) {
     let offset = 0;
     while (offset < buffer.length) {
         const result = await handle.write(buffer, offset, buffer.length - offset, offset);