@cocreate/file 1.19.6 → 1.21.0

package/CHANGELOG.md

@@ -1,3 +1,25 @@
+# [1.21.0](https://github.com/CoCreate-app/CoCreate-file/compare/v1.20.0...v1.21.0) (2025-11-16)
+
+
+### Features
+
+* enhance file processing by adding support for include/exclude filters and implementing dynamic data retrieval for various file types ([d8a2185](https://github.com/CoCreate-app/CoCreate-file/commit/d8a2185ce6f9554aa929f285e24526a281bc0b96))
+
+# [1.20.0](https://github.com/CoCreate-app/CoCreate-file/compare/v1.19.6...v1.20.0) (2025-10-11)
+
+
+### Bug Fixes
+
+* add error handling for translation service in file function ([262dc66](https://github.com/CoCreate-app/CoCreate-file/commit/262dc66d57c13a8948ef1afa863413c203f965bc))
+* update import statement and modify exit behavior in server function ([6e72fde](https://github.com/CoCreate-app/CoCreate-file/commit/6e72fdec3867c031e342b3faef598b661f43a620))
+
+
+### Features
+
+* add options parameter to file function for enhanced translation support ([7fac13d](https://github.com/CoCreate-app/CoCreate-file/commit/7fac13d10b8abd40584d5abc6b89d110f84db288))
+* enhance file handling by supporting base64 encoding for binary sources and improving file reading logic ([192337c](https://github.com/CoCreate-app/CoCreate-file/commit/192337c3a44d412f8a8d2c6bb1172451661eebf5))
+* enhance file processing with support for base64 and binary types, and add variable processing functionality ([eabfddf](https://github.com/CoCreate-app/CoCreate-file/commit/eabfddf2e1c36e7abd330369c1f28f90290cc1b8))
+
 ## [1.19.6](https://github.com/CoCreate-app/CoCreate-file/compare/v1.19.5...v1.19.6) (2025-09-07)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cocreate/file",
-  "version": "1.19.6",
+  "version": "1.21.0",
   "description": "A versatile, configurable headless file uploader supporting local and server operations. Accessible via a JavaScript API and HTML5 attributes, it provides seamless file reading, writing, and uploading with fallbacks to the standard HTML5 file input API. Ideal for developers needing robust file management in headless environments.",
   "keywords": [
     "file-uploader",

package/src/client.js

@@ -28,7 +28,7 @@ import Elements from "@cocreate/elements";
 import Actions from "@cocreate/actions";
 import { render } from "@cocreate/render";
 import { queryElements } from "@cocreate/utils";
-import "@cocreate/element-prototype";
+import { setValue } from "@cocreate/element-prototype";
 
 const inputs = new Map();
 const Files = new Map();

package/src/server.js

@@ -4,6 +4,7 @@ const fs = require("fs");
 const realpathAsync = fs.promises.realpath;
 
 const path = require("path");
+const { pathToFileURL } = require("url");
 const mimeTypes = {
 	".aac": "audio/aac",
 	".abw": "application/x-abiword",
@@ -83,7 +84,12 @@ const mimeTypes = {
 	".7z": "application/x-7z-compressed"
 };
 
-module.exports = async function file(CoCreateConfig, configPath, match) {
+module.exports = async function file(
+	CoCreateConfig,
+	configPath,
+	match,
+	options
+) {
 	let directories = CoCreateConfig.directories;
 	let sources = CoCreateConfig.sources;
 	let configDirectoryPath = path.dirname(configPath);
@@ -169,46 +175,60 @@ module.exports = async function file(CoCreateConfig, configPath, match) {
 	async function runDirectories() {
 		for (const directory of directories) {
 			const entry = directory.entry;
-			const exclude = directory.exclude || [];
-			await runFiles(directory, entry, exclude);
+			await runFiles(directory, entry);
 		}
 		return;
 	}
 
-	async function runFiles(directory, entry, exclude, Path, directoryName) {
+	async function runFiles(directory, entry, Path, directoryName) {
 		const entryPath = path.resolve(configDirectoryPath, entry);
 		let files = fs.readdirSync(entryPath);
-
+		let exclude = directory.exclude || [];
+		let include = directory.include || [];
 		for (let file of files) {
-			let skip = false;
-			for (let i = 0; i < exclude.length; i++) {
-				if (file.includes(exclude[i])) {
-					skip = true;
-					break;
-				}
-			}
-			if (skip) continue;
+			const filePath = path.resolve(entryPath, file);
 
 			let isDirectory;
 			let isSymlink = fs
-				.lstatSync(`${entryPath}/${file}`)
+				.lstatSync(filePath)
 				.isSymbolicLink();
 			if (isSymlink) {
-				let symlinkPath = await realpathAsync(`${entryPath}/${file}`);
+				let symlinkPath = await realpathAsync(filePath);
 				isDirectory =
 					fs.existsSync(symlinkPath) &&
 					fs.lstatSync(symlinkPath).isDirectory();
 			} else
 				isDirectory =
-					fs.existsSync(`${entryPath}/${file}`) &&
-					fs.lstatSync(`${entryPath}/${file}`).isDirectory();
+					fs.existsSync(filePath) &&
+					fs.lstatSync(filePath).isDirectory();
+
+			let skip = false;
+			for (let i = 0; i < exclude.length; i++) {
+				if (filePath.includes(exclude[i])) {
+					skip = true;
+					break;
+				}
+			}
+	
+			for (let i = 0; i < include.length; i++) {
+				if (filePath.includes(include[i])) {
+					skip = false;
+					break;
+				} else if (isDirectory) {
+					skip = "directory";
+					break;
+				} else {
+					skip = true;
+				}	
+			}
+
+			if (skip === true) continue;
 
 			let name = file;
 			let source = "";
 
 			for (let i = 0; i < match.length; i++) {
 				skip = true;
-				const filePath = path.resolve(entryPath, file);
 				if (filePath.startsWith(match[i])) {
 					skip = false;
 					break;
@@ -232,7 +252,7 @@ module.exports = async function file(CoCreateConfig, configPath, match) {
 				} else directoryName = "/";
 			}
 
-			if (exclude && exclude.includes(directoryName)) continue;
+			// if (exclude && exclude.includes(directoryName)) continue;
 
 			if (!Path) {
 				if (directoryName === "/") Path = directoryName;
@@ -243,76 +263,136 @@ module.exports = async function file(CoCreateConfig, configPath, match) {
 			if (Path === "/") pathname = Path + name;
 			else pathname = Path + "/" + name;
 
-			if (isDirectory) mimeType = "text/directory";
-			else
+			if (isDirectory) {
+				mimeType = "text/directory";
+			} else {
 				source = await getSource(
 					`${entryPath}/${file}`,
 					mimeType,
 					isSymlink
 				);
+			}
 
 			let values = {
 				"{{name}}": name || "",
-				"{{source}}": source || "",
+				"{{source}}": Buffer.isBuffer(source) ? `data:${mimeType};base64,${source.toString('base64')}` : source || "",
 				"{{directory}}": directoryName || "",
 				"{{path}}": Path || "",
 				"{{pathname}}": pathname,
-				"{{content-type}}": mimeType || ""
+				"{{content-type}}": mimeType || "",
 			};
 
-			let object = { ...directory.object };
-			if (!object.name) object.name = "{{name}}";
-			if (!object.src) object.src = "{{source}}";
-			if (!object.directory) object.directory = "{{directory}}";
-			if (!object.path) object.path = "{{path}}";
-			if (!object.pathname) object.pathname = "{{pathname}}";
-			if (!object["content-type"])
-				object["content-type"] = "{{content-type}}";
-			if (
-				!object.public &&
-				object.public != false &&
-				object.public != "false"
-			)
-				object.public = "true";
-
-			let newObject = {
-				array: directory.array || "files",
-				object
+			let data = {
+				array: directory.array || "files"
 			};
 
-			if (directory.storage) newObject.storage = directory.storage;
-			if (directory.database) newObject.database = directory.database;
-			if (directory.array) newObject.array = directory.array || "files";
-
-			for (const key of Object.keys(directory.object)) {
-				if (typeof directory.object[key] == "string") {
-					let variables = directory.object[key].match(
-						/{{([A-Za-z0-9_.,\[\]\-\/ ]*)}}/g
+			let isData = false;
+			if ( typeof directory.$data === "string") {
+				if (isDirectory) {
+					skip = "directory";
+				} else {
+					isData = true;
+					data = 	await getData(
+						`${entryPath}/${file}`,
+						mimeType,
+						isSymlink
+					);	
+					if (!data) continue
+				}
+			} else if ( typeof directory.object === "string") {
+				if (isDirectory) {
+					skip = "directory";
+				} else {
+					isData = true;
+					data.object = await getData(
+						`${entryPath}/${file}`,
+						mimeType,
+						isSymlink
 					);
-					if (variables) {
-						for (let variable of variables) {
-							newObject.object[key] = newObject.object[
-								key
-							].replace(variable, values[variable]);
-						}
-					}
+					if (!data.object) continue
 				}
-			}
-
-			if (skip !== "directory") {
-				if (!newObject.object._id)
-					newObject.$filter = {
+			} else if (typeof directory.object === "object" && directory.object !== null) {
+				data.array = directory.array || "files";
+				let object = { ...directory.object };
+				if (!object.name) object.name = "{{name}}";
+				if (!object.src) object.src = "{{source}}";
+				if (!object.directory) object.directory = "{{directory}}";
+				if (!object.path) object.path = "{{path}}";
+				if (!object.pathname) object.pathname = "{{pathname}}";
+				if (!object["content-type"])
+					object["content-type"] = "{{content-type}}";
+				if (
+					!object.public &&
+					object.public != false &&
+					object.public != "false"
+				)
+					object.public = "true";
+
+				data.object = object;
+
+				if (!data.object._id) {
+					data.$filter = {
 						query: {
 							pathname
 						}
 					};
+				}
 
-				response = await runStore(newObject);
-				console.log(
-					`Uploaded: ${entryPath}/${file}`,
-					`To: ${pathname}`
-				);
+				if (
+					options.translate &&
+					mimeType === "text/html" &&
+					Array.isArray(directory.languages) &&
+					!object.translations
+				) {
+					try {
+						// Call your AI translation service
+						const translations = await options.translate(
+							Buffer.isBuffer(source) ? source.toString('utf-8') : source,
+							directory.languages
+						);
+						data.object.translations = translations;
+					} catch (err) {
+						console.error("Translation error:", err);
+						// Continue without translations
+					}
+				}
+
+				if (directory.storage) data.storage = directory.storage;
+				if (directory.database) data.database = directory.database;
+				if (directory.array) data.array = directory.array || "files";
+
+				for (const key of Object.keys(directory.object)) {
+					if (typeof directory.object[key] == "string") {
+						let variables = directory.object[key].match(
+							/{{([A-Za-z0-9_.,\[\]\-\/ ]*)}}/g
+						);
+						if (variables) {
+							for (let variable of variables) {
+								let replacement = values[variable];
+								if (key === 'src' && variable === '{{source}}' && Buffer.isBuffer(source)) {
+									replacement = `data:${mimeType};base64,${source.toString('base64')}`;
+								}
+								data.object[key] = data.object[
+									key
+								].replace(variable, replacement);
+							}
+						}
+					}
+				}
+			}
 
+			if (skip !== "directory") {
+				response = await runStore(data);
+				if (isData) {
+					console.log(
+					`Saved: ${entryPath}/${file}`
+					);
+				} else {
+					console.log(
+						`Uploaded: ${entryPath}/${file}`,
+						`To: ${pathname}`
+					);
+				}
 				if (response.error) errorLog.push(response.error);
 			}
 
@@ -321,151 +401,195 @@ module.exports = async function file(CoCreateConfig, configPath, match) {
 				if (entry.endsWith("/")) newEntry = entry + name;
 				else newEntry = entry + "/" + name;
 
-				await runFiles(directory, newEntry, exclude, pathname, name);
+				await runFiles(directory, newEntry, pathname, name);
 			}
 		}
 		// if (errorLog.length)
 		//     console.log(...errorLog)
 	}
 
-	async function getSource(path, mimeType, isSymlink) {
-		let readType = "utf8";
-		if (mimeType === "image/svg+xml") {
-			readType = "utf8";
-		} else if (/^(image|audio|video)\/[-+.\w]+/.test(mimeType)) {
-			readType = "base64";
+	async function getSource(filePath, mimeType, isSymlink) {
+		// 1. UPDATED: Includes standard font types and uses simpler matching
+		const base64MimeTypes = /^(image|audio|video|font\/(woff2?|ttf|otf|eot)|application\/vnd\.ms-fontobject|application\/x-font-.*|application\/octet-stream)/;
+
+		// We only care if it needs to be Base64-encoded for a Data URI.
+		const needsBase64 = base64MimeTypes.test(mimeType);
+
+		let resolvedPath = filePath;
+		if (isSymlink) {
+			// Use promises for realpath
+			resolvedPath = await realpathAsync(filePath);
 		}
 
-		if (isSymlink) path = await realpathAsync(path);
+		// 2. READ: Always read the file as a raw Buffer (omitting encoding)
+		// This gives us the raw bytes, which is the safest start for any file.
+		let fileBuffer;
+		try {
+			fileBuffer = await fs.promises.readFile(resolvedPath);
+		} catch (error) {
+			console.error(`Error reading file: ${resolvedPath}`, error);
+			return ""; // Return empty string or handle error as appropriate
+		}
 
-		let binary = fs.readFileSync(path);
-		let content = new Buffer.from(binary).toString(readType);
+		if (needsBase64) {
+			// 3. RETURN BUFFER: Return the raw buffer for binary files.
+			return fileBuffer;
+		} else {
+			// 4. HANDLE TEXT/OTHER:
+			// For files not intended for Base64, convert the Buffer to a string using 'utf8'.
+			return fileBuffer.toString('utf8');
+		}
+	}
+
+	async function getData(filePath, mimeType, isSymlink) {
+		let resolvedPath = filePath;
+		if (isSymlink) {
+			resolvedPath = await realpathAsync(filePath);
+		}
+
+		try {
+			const fileMimeType = mimeTypes[path.extname(resolvedPath)] || "text/plain";
+			if (fileMimeType === "application/json") {
+				// Parse JSON files
+				return JSON.parse(fs.readFileSync(resolvedPath, "utf8"));
+			} else if (
+				fileMimeType === "application/javascript" ||
+				fileMimeType === "text/javascript"
+			) {
+				// Try CommonJS require first (fast path)
+				try {
+					// clear require cache to ensure fresh load
+					delete require.cache[require.resolve(resolvedPath)];
+					return require(resolvedPath);
+				} catch (err) {
+					// If require fails due to ESM syntax (export / import), fall back to dynamic import
+					const isESMSyntaxError =
+						err instanceof SyntaxError ||
+						/Unexpected token 'export'/.test(err.message) ||
+						/Cannot use import statement outside a module/.test(err.message) ||
+						/Unexpected token 'import'/.test(err.message);
+
+					if (isESMSyntaxError) {
+						try {
+							const module = await import(pathToFileURL(resolvedPath).href);
+							// return default export when present otherwise return full module
+							return module && module.default ? module.default : module;
+						} catch (impErr) {
+							console.error(`Failed to dynamic-import module: ${resolvedPath}`, impErr);
+							throw impErr;
+						}
+					}
+					// rethrow original require error if it's not an ESM issue
+					throw err;
+				}
+			} else {
+				return fs.readFileSync(resolvedPath, "utf8");
+			}
+		} catch (error) {
+			console.error(`Failed to process file: ${resolvedPath}`, error);
+			return "";
+		}
 
-		return content;
 	}
 
 	/**
 	 * Store files by config sources
 	 **/
 	async function runSources() {
-		let updatedSources = [];
+		let newConfig = require(configPath);
 
 		for (let i = 0; i < sources.length; i++) {
-			const { array, object } = sources[i];
-
-			let source = { ...sources[i] };
-			let keys = new Map();
-			let response = {};
-			let isMatch = false;
-
-			try {
-				if (array) {
-					if (!object) object = {};
-					else
-						for (const key of Object.keys(object)) {
-							if (typeof object[key] != "string") continue;
-
-							let variables = object[key].match(
-								/{{([A-Za-z0-9_.,\[\]\-\/ ]*)}}/g
-							);
-							if (variables) {
-								let originalValue = object[key];
-								keys.set(key, originalValue);
-								let value = "";
-								for (let variable of variables) {
-									let entry = /{{\s*([\w\W]+)\s*}}/g.exec(
-										variable
-									);
-									entry = entry[1].trim();
-									if (entry) {
-										if (!fs.existsSync(entry)) continue;
-
-										if (!isMatch) {
-											const filePath = path.resolve(
-												configDirectoryPath,
-												entry
-											);
-											for (
-												let i = 0;
-												i < match.length;
-												i++
-											) {
-												if (
-													filePath.startsWith(
-														match[i]
-													)
-												) {
-													console.log(
-														"Source saved",
-														sources[i]
-													);
-													isMatch = true;
-													break;
-												}
-											}
-										}
-
-										let read_type = "utf8";
-										const fileExtension =
-											path.extname(entry);
-										let mime_type =
-											mimeTypes[fileExtension] ||
-											"text/html";
-
-										if (
-											/^(image|audio|video)\/[-+.\w]+/.test(
-												mime_type
-											)
-										) {
-											read_type = "base64";
-										}
-
-										let binary = fs.readFileSync(entry);
-										let content = new Buffer.from(
-											binary
-										).toString(read_type);
-										if (content) value += content;
-										// object[key] = object[key].replace(variable, content);
-									}
-								}
-								object[key] = value;
-							}
+			let data = sources[i];
+
+			// Handle string values
+			if (typeof data === "string") {
+				let {value, filePath } = await processVariables(data);
+				let response = await runStore(value);
+				if (response && response.object && response.object[0]) {
+					updateFilePath(filePath, response); // Call the new function to update the file path
+				}
+			} else if (data.array && data.object) {
+				if (typeof data.object === "string") {
+					let {value, filePath } = await processVariables(data.object);
+					if (value) {
+						let response = await runStore(value);
+						if (response && response.object && response.object[0]) {
+							updateFilePath(filePath, response.object);
 						}
-
-					let data = { array, object };
-					if (!object._id && object.pathname)
-						data.$filter = {
-							query: {
-								$or: [{ pathname: object.pathname }]
-							}
-						};
-
-					if (match.length && isMatch)
-						response = await runStore(data);
+					}
+				} else if (typeof data.object === "object" && data.object !== null) {
+					for (const key in data) {
+						let {value } = await processVariables(data[key]);
+						if (data) {
+							data.object[key] = value;
+						}
+					}
+					let response = await runStore(data);
+					if (response && response.object && response.object[0] && response.object[0]._id) {
+						newConfig.sources[i].object._id = response.object[0]._id;
+					}
 				}
-			} catch (err) {
-				console.log(err);
-				process.exit();
 			}
 
-			if (
-				response.object &&
-				response.object[0] &&
-				response.object[0]._id
-			) {
-				source.object._id = response.object[0]._id;
-			}
+		}
 
-			for (const [key, value] of keys) {
-				source.object[key] = value;
-			}
+		return newConfig;
+	}
+
+	async function processVariables(value) {
+		let variableMatch = /{{\s*([\w\W]+)\s*}}/g.exec(value);
+		if (!variableMatch) return { value, filePath: null };
+
+		let entry = variableMatch[1].trim();
+		if (!fs.existsSync(entry)) return { value, filePath: null };
 
-			updatedSources.push(source);
+		const filePath = path.resolve(configDirectoryPath, entry);
+
+		// Check if the file path matches any of the provided match patterns
+		let isMatched = match.some((pattern) => filePath.startsWith(pattern));
+		if (!isMatched) return { value, filePath: null };
+
+		// Read the file as is
+		let content;
+		try {
+			const fileMimeType = mimeTypes[path.extname(entry)] || "text/plain";
+
+			if (fileMimeType === "application/json") {
+				// Parse JSON files
+				content = JSON.parse(fs.readFileSync(filePath, "utf8"));
+			} else if (fileMimeType === "application/javascript" || fileMimeType === "text/javascript") {
+				// For JavaScript files, require the file to execute exports
+				content = require(filePath);
+			} else {
+				// For plain strings, read as UTF-8 without conversion
+				content = fs.readFileSync(filePath, "utf8");
+			}
+		} catch (error) {
+			console.error(`Failed to process file: ${filePath}`, error);
+			return { value, filePath: null };
 		}
 
-		return updatedSources;
+		return { value: content, filePath };
+	}
+
+	/**
+	 * Updates the file at the given file path with the provided data.
+	 * The data is saved as a JSON string.
+	 * 
+	 * @param {string} filePath - The path of the file to update.
+	 * @param {object} data - The data to write to the file.
+	 */
+	function updateFilePath(filePath, data) {
+		try {
+			const jsonData = JSON.stringify(data, null, 4); // Format JSON with indentation
+			fs.writeFileSync(filePath, jsonData, "utf8");
+			console.log(`File updated successfully at: ${filePath}`);
+		} catch (error) {
+			console.error(`Failed to update file at: ${filePath}`, error);
+		}
 	}
 
+
 	async function runStore(data) {
 		try {
 			let response;
@@ -493,42 +617,22 @@ module.exports = async function file(CoCreateConfig, configPath, match) {
 	}
 
 	async function run() {
-		if (directories) await runDirectories();
+		if (directories) {
+			await runDirectories();
+		}
 
 		if (sources && sources.length) {
-			let sources = await runSources();
-			let newConfig = { ...CoCreateConfig };
-			if (directories && directories.length)
-				newConfig.directories = directories;
-
-			newConfig.sources = sources;
-
-			if (newConfig.repositories)
-				newConfig.repositories.forEach((obj) => {
-					for (const key in obj) {
-						if (!["path", "repo", "exclude"].includes(key)) {
-							delete obj[key];
-						}
-					}
-				});
-
-			delete newConfig.url;
-			delete newConfig.broadcast;
-
+			let newConfig = await runSources();
 			fs.writeFileSync(
 				configPath,
 				`module.exports = ${JSON.stringify(newConfig, null, 4)};`
 			);
 		}
-
-		if (!match.length) {
-			console.log("upload complete!");
-
-			setTimeout(function () {
-				process.exit();
-			}, 2000);
-		}
 	}
 
 	await run();
+	// Only exit if not in watch mode
+	if (!process.argv.includes("--watch") && !process.argv.includes("-w")) {
+		process.exit();
+	}
 };