@xopcai/xopc 0.0.283 → 0.0.285

package/dist/src/imports/files.js

@@ -0,0 +1,77 @@
+import { ImportError } from "./types.js";
+import { isAbsolute, join, relative } from "node:path";
+import { closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, readdirSync, realpathSync } from "node:fs";
+import { createHash } from "node:crypto";
+//#region src/imports/files.ts
+const IMPORT_LIMITS = {
+	file: 15 * 1024 * 1024,
+	total: 100 * 1024 * 1024,
+	count: 5e3,
+	depth: 16
+};
+function digest(value) {
+	return createHash("sha256").update(JSON.stringify(value)).digest("hex");
+}
+function fileHash(files) {
+	return digest([...files].sort((a, b) => a.path.localeCompare(b.path, "en")));
+}
+function inside(root, path) {
+	const rel = relative(root, path);
+	return rel === "" || !isAbsolute(rel) && rel !== ".." && !rel.startsWith("../") && !rel.startsWith("..\\");
+}
+function safePath(path) {
+	return !!path && !path.includes("\\") && !path.includes("\0") && !path.startsWith("/") && path.split("/").every((p) => !!p && p !== "." && p !== ".." && !/[<>:"|?*]/.test(p) && !/[. ]$/.test(p) && !/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(p));
+}
+function readRegularFile(path, limit = IMPORT_LIMITS.file) {
+	if (!lstatSync(path).isFile()) throw new ImportError("unsafe_file", "Only regular files are supported");
+	const fd = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+	try {
+		const before = fstatSync(fd);
+		if (!before.isFile() || before.size > limit) throw new ImportError("limit_exceeded", "File exceeds import limit", 413);
+		const data = readFileSync(fd);
+		const after = fstatSync(fd);
+		if (data.length > limit || before.size !== after.size || before.mtimeMs !== after.mtimeMs) throw new ImportError("source_changed", "Source changed while reading", 409);
+		return data;
+	} finally {
+		closeSync(fd);
+	}
+}
+function readTree(rootPath) {
+	const root = realpathSync(rootPath);
+	const files = [];
+	let total = 0;
+	const names = /* @__PURE__ */ new Set();
+	function walk(dir, prefix, depth) {
+		if (depth > IMPORT_LIMITS.depth) throw new ImportError("limit_exceeded", "Directory is too deep", 413);
+		for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+			if ([
+				".git",
+				".DS_Store",
+				"__MACOSX"
+			].includes(entry.name)) continue;
+			const rel = prefix + entry.name;
+			if (!safePath(rel)) throw new ImportError("unsafe_file", "Unsupported file name");
+			const key = rel.normalize("NFC").toLowerCase();
+			if (names.has(key)) throw new ImportError("conflict", "File names collide on the target filesystem", 409);
+			names.add(key);
+			if (names.size > IMPORT_LIMITS.count) throw new ImportError("limit_exceeded", "Too many files", 413);
+			const path = join(dir, entry.name);
+			if (entry.isSymbolicLink() || !inside(root, realpathSync(path))) throw new ImportError("unsafe_file", "Symbolic links must be replaced with local files before importing");
+			if (entry.isDirectory()) walk(path, rel + "/", depth + 1);
+			else {
+				const data = readRegularFile(path);
+				total += data.length;
+				if (total > IMPORT_LIMITS.total) throw new ImportError("limit_exceeded", "Import is too large", 413);
+				files.push({
+					path: rel,
+					data: data.toString("base64"),
+					executable: (lstatSync(path).mode & 73) !== 0
+				});
+			}
+		}
+	}
+	walk(root, "", 0);
+	return files;
+}
+//#endregion
+export { IMPORT_LIMITS, digest, fileHash, inside, readRegularFile, readTree, safePath };

package/dist/src/imports/importService.d.ts

@@ -0,0 +1,50 @@
+import { type ScanInput } from './scanner.js';
+import { type ImportAction, type ImportFile, type ImportJob, type ImportScan, type ImportTarget } from './types.js';
+export interface ImportServiceOptions {
+    stateDir: string;
+    owner: string;
+    validateTarget?: (target: ImportTarget) => void;
+    isConnected?: (candidate: ImportScan['candidates'][number], workspaceRoot?: string) => boolean;
+    reservedNames?: (target: ImportTarget) => string[];
+    isWorkspaceTrusted?: (root: string) => boolean;
+    refreshSkills?: () => void | Promise<void>;
+}
+export declare class ImportService {
+    private readonly options;
+    private readonly repo;
+    private readonly root;
+    constructor(options: ImportServiceOptions);
+    scan(input: ScanInput): Promise<ImportScan>;
+    getScan(id: string): ImportScan;
+    discardUnusedScan(id: string): void;
+    storeScan(scan: ImportScan, workspaceRoot?: string): ImportScan;
+    plan(scanId: string, target: ImportTarget, choices: Array<Pick<ImportAction, 'candidateId' | 'operation'> & {
+        name?: string;
+    }>): import("./types.js").ImportPlan;
+    describeSkills(scan: ImportScan, target: ImportTarget): {
+        candidate: import("./types.js").ImportCandidate;
+        status: import("./types.js").InventoryStatus;
+        reason: string;
+        targetName: string;
+    }[];
+    readSnapshot(scanId: string, candidateId: string): ImportFile[];
+    jobByKey(key: string): ImportJob;
+    validateTarget(target: ImportTarget): void;
+    reservedNames(target: ImportTarget): string[];
+    isWorkspaceTrusted(root: string): boolean;
+    getPlan(id: string): import("./types.js").ImportPlan;
+    apply(planId: string, key: string): ImportJob;
+    private renameSkill;
+    getJob(id: string): ImportJob;
+    recoverAndClean(): Promise<void>;
+    history(): ImportJob[];
+    activate(id: string, candidateIds: string[]): Promise<ImportJob>;
+    private publish;
+    rollback(id: string, candidateIds: string[]): Promise<ImportJob>;
+    private trashPath;
+    private recover;
+    private targetHash;
+    private assertLive;
+    private checkSelection;
+    private lock;
+}

package/dist/src/imports/importService.js

@@ -0,0 +1,414 @@
+import { createLogger } from "../utils/logger/index.js";
+import { init_logger } from "../utils/logger.js";
+import { init_write_file_atomic, writeTextAtomicSync } from "../infra/write-file-atomic.js";
+import { prepareManagedSkillTempDir } from "../agent/skills/managed-store.js";
+import { ImportError } from "./types.js";
+import { fileHash, readTree } from "./files.js";
+import { parseSkill } from "./compatibility.js";
+import { ImportRepository } from "../storage/sqlite/import-repository.js";
+import { canonicalTarget, createImportPlan, targetInventory, targetRevision } from "./planner.js";
+import { scanLocal } from "./scanner.js";
+import { dirname, join } from "node:path";
+import { existsSync, lstatSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { randomUUID } from "node:crypto";
+import yaml from "js-yaml";
+import lockfile from "proper-lockfile";
+//#region src/imports/importService.ts
+init_write_file_atomic();
+init_logger();
+const log = createLogger("ImportService");
+const RETENTION_MS = 720 * 60 * 60 * 1e3;
+var ImportService = class {
+	repo;
+	root;
+	constructor(options) {
+		this.options = options;
+		this.repo = new ImportRepository(options.owner);
+		this.root = join(canonicalTarget(options.stateDir), "imports");
+		mkdirSync(this.root, {
+			recursive: true,
+			mode: 448
+		});
+	}
+	async scan(input) {
+		return this.storeScan(await scanLocal(input), input.projectRoot);
+	}
+	getScan(id) {
+		return this.repo.get("scan", id);
+	}
+	discardUnusedScan(id) {
+		this.lock(() => {
+			if (this.repo.jobs(-1).some((job) => job.plan.scanId === id)) return;
+			rmSync(join(this.root, id), {
+				recursive: true,
+				force: true
+			});
+			this.repo.deleteScan(id);
+		});
+	}
+	storeScan(scan, workspaceRoot) {
+		const dir = join(this.root, scan.id);
+		mkdirSync(dir, { mode: 448 });
+		for (const item of scan.candidates) if (item.kind === "skill" && item.compatibility === "compatible") writeTextAtomicSync(join(dir, `${item.id}.json`), JSON.stringify(item.files));
+		const manifest = {
+			...scan,
+			candidates: scan.candidates.map((c) => ({
+				...c,
+				connected: this.options.isConnected?.(c, workspaceRoot) ?? false,
+				files: c.files.map((f) => ({
+					...f,
+					data: ""
+				}))
+			}))
+		};
+		this.repo.save("scan", manifest);
+		return manifest;
+	}
+	plan(scanId, target, choices) {
+		this.options.validateTarget?.(target);
+		const plan = createImportPlan(this.getScan(scanId), target, choices, this.options.reservedNames?.(target));
+		this.repo.save("plan", plan);
+		return plan;
+	}
+	describeSkills(scan, target) {
+		const inventory = targetInventory(target.root);
+		const names = new Set([...this.options.reservedNames?.(target) ?? [], ...inventory.flatMap((i) => [i.directory, i.name])].map((n) => n.toLowerCase()));
+		const previous = /* @__PURE__ */ new Map();
+		for (const job of this.repo.jobs(-1)) {
+			if (job.plan.target.root !== canonicalTarget(target.root)) continue;
+			const origin = this.getScan(job.plan.scanId);
+			for (const item of job.items.filter((i) => i.status === "active")) {
+				const candidate = origin.candidates.find((c) => c.id === item.action.candidateId);
+				if (candidate?.source === scan.source && !previous.has(candidate.location)) previous.set(candidate.location, item.action.name);
+			}
+		}
+		return scan.candidates.filter((c) => c.kind === "skill").map((candidate) => {
+			let status = "ready";
+			let reason;
+			let targetName = candidate.name;
+			if (candidate.connected) status = "existing";
+			else if (candidate.compatibility !== "compatible") {
+				status = "blocked";
+				reason = candidate.findings.join("; ");
+			} else {
+				const prior = previous.get(candidate.location);
+				const priorTarget = prior && inventory.find((i) => i.directory === prior);
+				const files = this.readSnapshot(scan.id, candidate.id);
+				if (priorTarget) {
+					targetName = priorTarget.directory;
+					status = fileHash(this.renameSkill(files, targetName)) === priorTarget.hash ? "existing" : "blocked";
+					if (status === "blocked") reason = "Previously imported content has changed; existing edits will be kept.";
+				} else if (inventory.some((i) => i.name.toLowerCase() === candidate.name.toLowerCase() && i.hash === candidate.hash)) status = "existing";
+				else if (names.has(targetName.toLowerCase())) {
+					targetName = `${candidate.source}-${candidate.name}`.slice(0, 63);
+					if (inventory.some((i) => i.name === targetName && i.hash === fileHash(this.renameSkill(files, targetName)))) status = "existing";
+					else if (names.has(targetName.toLowerCase())) {
+						status = "blocked";
+						reason = "Both the original and product-prefixed names are in use.";
+					} else status = "conflict";
+				}
+			}
+			if (status === "ready" || status === "conflict") names.add(targetName.toLowerCase());
+			return {
+				candidate,
+				status,
+				reason,
+				targetName
+			};
+		});
+	}
+	readSnapshot(scanId, candidateId) {
+		const candidate = this.getScan(scanId).candidates.find((c) => c.id === candidateId);
+		if (!candidate || candidate.kind !== "skill" || candidate.compatibility !== "compatible") throw new ImportError("not_found", "Snapshot not found", 404);
+		const files = JSON.parse(readFileSync(join(this.root, scanId, `${candidateId}.json`), "utf8"));
+		if (fileHash(files) !== candidate.hash) throw new ImportError("source_changed", "Snapshot changed", 409);
+		return files;
+	}
+	jobByKey(key) {
+		return this.repo.byKey(key);
+	}
+	validateTarget(target) {
+		this.options.validateTarget?.(target);
+	}
+	reservedNames(target) {
+		return this.options.reservedNames?.(target) ?? [];
+	}
+	isWorkspaceTrusted(root) {
+		return this.options.isWorkspaceTrusted?.(root) ?? false;
+	}
+	getPlan(id) {
+		return this.repo.get("plan", id);
+	}
+	apply(planId, key) {
+		if (!key || key.length > 128) throw new ImportError("invalid_key", "An idempotency key is required");
+		return this.lock(() => {
+			const previous = this.repo.byKey(key);
+			if (previous) {
+				if (previous.plan.id !== planId) throw new ImportError("conflict", "Idempotency key belongs to a different plan", 409);
+				return previous;
+			}
+			const plan = this.getPlan(planId);
+			this.options.validateTarget?.(plan.target);
+			if (Date.now() > plan.expiresAt || canonicalTarget(plan.target.root) !== plan.target.root || targetRevision(plan.target.root) !== plan.targetRevision) throw new ImportError("plan_stale", "Target changed or plan expired; preview again", 409);
+			const job = {
+				id: randomUUID(),
+				plan,
+				createdAt: Date.now(),
+				expiresAt: Date.now() + RETENTION_MS,
+				items: plan.actions.map((action) => ({
+					action,
+					status: action.operation === "skip" ? "skipped" : "staged"
+				}))
+			};
+			const dir = join(this.root, job.id);
+			mkdirSync(dir, { mode: 448 });
+			const scan = this.getScan(plan.scanId);
+			try {
+				for (const item of job.items) {
+					if (item.status === "skipped") continue;
+					const candidate = scan.candidates.find((c) => c.id === item.action.candidateId);
+					if (candidate.kind !== "skill") continue;
+					const files = JSON.parse(readFileSync(join(this.root, scan.id, `${candidate.id}.json`), "utf8"));
+					if (fileHash(files) !== candidate.hash) throw new ImportError("source_changed", "Snapshot changed; scan again", 409);
+					const rewritten = this.renameSkill(files, item.action.name);
+					item.afterHash = fileHash(rewritten);
+					writeTextAtomicSync(join(dir, `${candidate.id}.json`), JSON.stringify(rewritten));
+				}
+				this.repo.save("job", job, key);
+			} catch (error) {
+				rmSync(dir, {
+					recursive: true,
+					force: true
+				});
+				throw error;
+			}
+			log.info({
+				jobId: job.id,
+				count: job.items.length
+			}, "Import staged");
+			return job;
+		});
+	}
+	renameSkill(files, name) {
+		return files.map((file) => {
+			if (file.path !== "SKILL.md") return file;
+			const { metadata, body } = parseSkill(Buffer.from(file.data, "base64").toString("utf8"));
+			if (metadata.name === name) return file;
+			return {
+				...file,
+				data: Buffer.from(`---\n${yaml.dump({
+					...metadata,
+					name
+				})}---\n${body}`).toString("base64")
+			};
+		});
+	}
+	getJob(id) {
+		return this.lock(() => this.recover(this.repo.get("job", id)));
+	}
+	async recoverAndClean() {
+		const release = await lockfile.lock(this.root, {
+			realpath: false,
+			retries: {
+				retries: 15,
+				minTimeout: 250,
+				maxTimeout: 1e3
+			}
+		});
+		try {
+			for (const saved of this.repo.jobs(-1)) {
+				const job = this.recover(saved);
+				if (Date.now() <= job.expiresAt || canonicalTarget(job.plan.target.root) !== job.plan.target.root) continue;
+				for (const item of job.items) if (item.backupPath && existsSync(item.backupPath) && this.targetHash(item.backupPath) === item.action.beforeHash) rmSync(item.backupPath, {
+					recursive: true,
+					force: true
+				});
+				rmSync(join(this.root, job.id), {
+					recursive: true,
+					force: true
+				});
+			}
+			for (const scan of this.repo.scansBefore(Date.now() - RETENTION_MS - 1440 * 60 * 1e3)) rmSync(join(this.root, scan.id), {
+				recursive: true,
+				force: true
+			});
+		} finally {
+			await release();
+		}
+	}
+	history() {
+		return this.lock(() => this.repo.jobs().map((job) => this.recover(job)));
+	}
+	async activate(id, candidateIds) {
+		const job = this.lock(() => {
+			const current = this.recover(this.repo.get("job", id));
+			this.assertLive(current);
+			this.options.validateTarget?.(current.plan.target);
+			const scan = this.getScan(current.plan.scanId);
+			this.checkSelection(current, candidateIds);
+			for (const candidateId of candidateIds) {
+				const item = current.items.find((i) => i.action.candidateId === candidateId);
+				if (item.status === "active") continue;
+				if (!["staged", "failed"].includes(item.status)) throw new ImportError("invalid_state", "Item cannot be activated");
+				if (scan.candidates.find((c) => c.id === candidateId).kind !== "skill") throw new ImportError("needs_setup", "Review the MCP draft in connection settings before configuring it");
+				try {
+					this.publish(current, item);
+				} catch (error) {
+					if (item.status !== "publishing") item.status = "failed";
+					item.error = error instanceof ImportError ? error.message : "Publishing failed; retry after checking target permissions";
+				}
+				this.repo.save("job", current);
+			}
+			return current;
+		});
+		await this.options.refreshSkills?.();
+		return job;
+	}
+	publish(job, item) {
+		const root = job.plan.target.root;
+		if (canonicalTarget(root) !== root) throw new ImportError("unsafe_target", "Target location changed", 409);
+		const dest = join(root, item.action.name);
+		if (this.targetHash(dest) !== item.action.beforeHash) throw new ImportError("plan_stale", "Target changed after preview", 409);
+		const others = this.options.reservedNames?.(job.plan.target) ?? [];
+		if (item.action.operation !== "replace" && others.some((name) => name.toLowerCase() === item.action.name.toLowerCase())) throw new ImportError("conflict", "Skill name is now in use", 409);
+		const files = JSON.parse(readFileSync(join(this.root, job.id, `${item.action.candidateId}.json`), "utf8"));
+		if (fileHash(files) !== item.afterHash) throw new ImportError("source_changed", "Staged files changed", 409);
+		const { tempDir } = prepareManagedSkillTempDir(item.action.name, root);
+		try {
+			for (const file of files) {
+				const path = join(tempDir, file.path);
+				mkdirSync(dirname(path), {
+					recursive: true,
+					mode: 448
+				});
+				writeFileSync(path, Buffer.from(file.data, "base64"), {
+					mode: file.executable ? 448 : 384,
+					flag: "wx"
+				});
+			}
+			if (fileHash(readTree(tempDir)) !== item.afterHash) throw new ImportError("source_changed", "Staged verification failed", 409);
+			item.backupPath = join(root, `.tmp-import-backup-${job.id}-${item.action.candidateId}`);
+			item.status = "publishing";
+			delete item.error;
+			this.repo.save("job", job);
+			if (existsSync(dest)) renameSync(dest, item.backupPath);
+			renameSync(tempDir, dest);
+			item.status = "active";
+		} catch (error) {
+			if (item.backupPath && existsSync(item.backupPath) && !existsSync(dest)) renameSync(item.backupPath, dest);
+			throw error;
+		} finally {
+			rmSync(tempDir, {
+				recursive: true,
+				force: true
+			});
+		}
+	}
+	async rollback(id, candidateIds) {
+		const job = this.lock(() => {
+			const current = this.recover(this.repo.get("job", id));
+			this.assertLive(current);
+			this.checkSelection(current, candidateIds);
+			for (const candidateId of candidateIds) {
+				const item = current.items.find((i) => i.action.candidateId === candidateId);
+				if (["rolled_back", "skipped"].includes(item.status)) continue;
+				if (item.status === "staged" || item.status === "failed") {
+					item.status = "rolled_back";
+					this.repo.save("job", current);
+					continue;
+				}
+				const dest = join(current.plan.target.root, item.action.name);
+				if (canonicalTarget(current.plan.target.root) !== current.plan.target.root || this.targetHash(dest) !== item.afterHash) {
+					item.error = "Target changed after import; rollback would overwrite your edits";
+					this.repo.save("job", current);
+					continue;
+				}
+				if (item.action.beforeHash !== null && (!item.backupPath || this.targetHash(item.backupPath) !== item.action.beforeHash)) {
+					item.error = "Backup is missing or changed; manual review required";
+					this.repo.save("job", current);
+					continue;
+				}
+				item.status = "rolling_back";
+				this.repo.save("job", current);
+				const trash = this.trashPath(current, item);
+				renameSync(dest, trash);
+				if (item.backupPath && existsSync(item.backupPath)) {
+					if (this.targetHash(item.backupPath) !== item.action.beforeHash) throw new ImportError("rollback_conflict", "Backup changed; manual review required", 409);
+					renameSync(item.backupPath, dest);
+				}
+				item.status = "rolled_back";
+				delete item.error;
+				this.repo.save("job", current);
+				rmSync(trash, {
+					recursive: true,
+					force: true
+				});
+			}
+			return current;
+		});
+		await this.options.refreshSkills?.();
+		return job;
+	}
+	trashPath(job, item) {
+		return join(job.plan.target.root, `.tmp-import-undo-${job.id}-${item.action.candidateId}`);
+	}
+	recover(job) {
+		for (const item of job.items) {
+			if (!["publishing", "rolling_back"].includes(item.status)) continue;
+			const root = job.plan.target.root;
+			if (canonicalTarget(root) !== root) throw new ImportError("unsafe_target", "Target location changed", 409);
+			const dest = join(root, item.action.name);
+			const hash = this.targetHash(dest);
+			if (item.status === "publishing") if (hash === item.afterHash) item.status = "active";
+			else if (hash === item.action.beforeHash) item.status = "staged";
+			else if (!hash && item.action.beforeHash !== null && item.backupPath && this.targetHash(item.backupPath) === item.action.beforeHash) {
+				renameSync(item.backupPath, dest);
+				item.status = "staged";
+			} else {
+				item.status = "failed";
+				item.error = "Publication interrupted and target changed; manual review required";
+			}
+			else {
+				const trash = this.trashPath(job, item);
+				if (!hash && item.action.beforeHash !== null && item.backupPath && this.targetHash(item.backupPath) === item.action.beforeHash) renameSync(item.backupPath, dest);
+				else if (hash === item.afterHash && !existsSync(trash)) {
+					item.status = "active";
+					this.repo.save("job", job);
+					continue;
+				} else if (hash !== item.action.beforeHash) {
+					item.error = "Rollback interrupted; target needs review";
+					this.repo.save("job", job);
+					continue;
+				}
+				item.status = "rolled_back";
+				rmSync(trash, {
+					recursive: true,
+					force: true
+				});
+			}
+			this.repo.save("job", job);
+		}
+		return job;
+	}
+	targetHash(path) {
+		if (!existsSync(path)) return null;
+		if (canonicalTarget(path) !== path || lstatSync(path).isSymbolicLink()) throw new ImportError("unsafe_target", "Target is a symbolic link", 409);
+		return fileHash(readTree(path));
+	}
+	assertLive(job) {
+		if (Date.now() > job.expiresAt) throw new ImportError("expired", "Import recovery period has expired", 409);
+	}
+	checkSelection(job, ids) {
+		if (!ids.length || new Set(ids).size !== ids.length || ids.some((id) => !job.items.some((i) => i.action.candidateId === id))) throw new ImportError("invalid_selection", "Select valid import items");
+	}
+	lock(run) {
+		const release = lockfile.lockSync(this.root, { realpath: false });
+		try {
+			return run();
+		} finally {
+			release();
+		}
+	}
+};
+//#endregion
+export { ImportService };

package/dist/src/imports/inventory.d.ts

@@ -0,0 +1,11 @@
+import type { ImportService } from './importService.js';
+import { type ImportSource, type ImportInventory, type StoredInventory, type ImportCandidate } from './types.js';
+export declare function contextKey(candidate: ImportCandidate): string;
+export declare function existingContext(candidate: ImportCandidate, projectId?: string): import("../knowledge-memory/domain.js").KnowledgeItem;
+export declare function publicInventory(inventory: StoredInventory): ImportInventory;
+export declare function getLiveInventory(owner: string, id: string): StoredInventory;
+export declare function scanProduct(service: ImportService, source: ImportSource, owner?: string): Promise<ImportInventory>;
+export declare function previewInventoryItem(service: ImportService, owner: string, inventoryId: string, itemId: string): {
+    text: string;
+    truncated: boolean;
+};