lucy-cli 0.10.0 → 1.0.0

package/README.md

@@ -30,32 +30,42 @@ Lucy-CLI is designed to streamline the setup and management of TypeScript within
 
 4. **Git Submodule Support**  
    - Includes support for git submodules, providing full type support within submodules and enabling the use of libraries and types across projects.
-
-5. **Configurable Setup**  
+   - To add a submodule, include the following in the `lucy.json` file:
+
+      ``` json
+      "modules": {
+        "<repoName>": {
+          "url": "String",
+          "branch": "String"
+        }
+      }
+      ```
+
+1. **Configurable Setup**  
    - After initialization, Lucy-CLI creates a `lucy-config.json` configuration file where you can modify settings, add dev packages, specify Wix packages, and configure git submodules.
 
-6. **Execute render functions**  
+2. **Execute render functions**  
    - Lucy-CLI can execute render functions located in the backend template folder, allowing you to test render functions locally.
 
-7. **compile sccs files**  
+3. **compile sccs files**  
    - Lucy-CLI can compile scss files to css files.
    - It compiles styles/global.scss file to global.css.
 
-8. **Wix NPM Package Installation**
+4. **Wix NPM Package Installation**
    - Lucy-CLI can install Wix npm packages from the `lucy.json` file in the project directory.
 
-9. **Teting with Vitest**
+5. **Teting with Vitest**
    - Lucy-CLI can run tests with Vitest.
    - It runs tests located backend folder with the file name ending with `.spec.ts`.
    - it creates a code coverage report in the coverage folder in the lib folders and typescript folders.
    - Vitest is looking for mokes in typescript folder and lib folder.
      - You can add additional mock folders in vitest.config.ts file.
 
-10. **Linting with ESLint**
+6.  **Linting with ESLint**
     - Lucy-CLI can lint the code with ESLint.
     - It uses the ESLint configuration in the `.eslintrc.cjs` file in the project directory.  
 
-11. **Add git version during production build**
+7.  **Add git version during production build**
       - Lucy-CLI can add the git version to the production build.
       - It adds the git version to the `public/constant/env.ts` file in the public folder under the key gitTag.
 

package/dist/gulp/checks.js

@@ -1,5 +1,5 @@
 import * as fs from 'fs';
-import glob from 'glob';
+import { glob } from 'glob';
 import * as path from 'path';
 import gulp from 'gulp';
 import ts from 'gulp-typescript';
@@ -23,52 +23,21 @@ function extractMatchFromFile(filePath, pattern) {
         });
     });
 }
-/**
- * Reads files in a folder
- * @param {string} folderPath Folder path
- * @param {string} pattern Pattern to match
- * @param {string} globPattern Glob pattern
- */
-function readFilesInFolder(folderPath, pattern, globPattern) {
-    return new Promise((resolve, reject) => {
-        glob(path.join(folderPath, globPattern), (err, files) => {
-            if (err) {
-                reject(err);
-                return;
-            }
-            const filenameList = [];
-            /**
-             * Traverse files
-             * @param {number} index Index
-             */
-            function traverseFiles(index) {
-                if (index === files.length) {
-                    resolve(filenameList);
-                    return;
-                }
-                const file = files[index];
-                if (pattern) {
-                    if (!file)
-                        return;
-                    extractMatchFromFile(file, pattern)
-                        .then((capturedGroup) => {
-                        if (capturedGroup) {
-                            filenameList.push(capturedGroup);
-                        }
-                        traverseFiles(index + 1);
-                    })
-                        .catch(reject);
-                }
-                if (!pattern) {
-                    if (!file)
-                        return;
-                    filenameList.push(path.basename(file));
-                    traverseFiles(index + 1);
-                }
+async function readFilesInFolder(folderPath, pattern, globPattern) {
+    const files = await glob(path.join(folderPath, globPattern));
+    const filenameList = [];
+    for (const file of files) {
+        if (pattern) {
+            const capturedGroup = await extractMatchFromFile(file, pattern);
+            if (capturedGroup) {
+                filenameList.push(capturedGroup);
             }
-            traverseFiles(0);
-        });
-    });
+        }
+        else {
+            filenameList.push(path.basename(file));
+        }
+    }
+    return filenameList;
 }
 export async function checkPages(fail, force) {
     console.log("🐕" + green.underline.bold(' => Checking pages...'));

package/dist/gulp/types.js

@@ -149,9 +149,9 @@ export function updateWixTypes(options) {
 }
 export function addTypes(options, done) {
     const { replaceOptions } = options;
-    const processPages = gulp.src(['./.wix/types/wix-code-types/dist/types/page/$w.d.ts'])
-        .pipe(replace('declare namespace \\$w {', ' declare namespace $w{\nconst api: $w.Api;\n', replaceOptions))
-        .pipe(gulp.dest('./.wix/types/wix-code-types/dist/types/page/'));
+    // const processPages = gulp.src(['./.wix/types/wix-code-types/dist/types/page/$w.d.ts'])
+    // 	.pipe(replace('declare namespace \\$w {', ' declare namespace $w{\nconst api: $w.Api;\n', replaceOptions))
+    // 	.pipe(gulp.dest('./.wix/types/wix-code-types/dist/types/page/'));
     const exportTypes = gulp.src(['./.wix/types/wix-code-types/dist/types/common/*.d.ts', '!./.wix/types/wix-code-types/dist/types/common/$w.d.ts'])
         .pipe(replace('interface ', 'export interface ', replaceOptions))
         .pipe(replace('enum ', 'export enum ', replaceOptions))
@@ -163,10 +163,12 @@ export function addTypes(options, done) {
         .pipe(replace('type ', 'export type ', replaceOptions))
         .pipe(gulp.dest('./.wix/types/wix-code-types/dist/types/beta/common/'));
     const processCommon = gulp.src(['./.wix/types/wix-code-types/dist/types/common/$w.d.ts'])
-        .pipe(insert.prepend("import { FrontendAPI } from '../../../../../../typescript/public/models/frontendApi.model';\nimport '@total-typescript/ts-reset';\n"))
-        .pipe(replace('namespace \\$w {', 'declare namespace $w{\ntype Api = FrontendAPI;\n', replaceOptions))
+        .pipe(insert.prepend("import '@total-typescript/ts-reset';\n"))
+        // .pipe(replace('namespace \\$w {', 'declare namespace $w{\ntype Api = FrontendAPI;\n', replaceOptions))
         .pipe(gulp.dest('./.wix/types/wix-code-types/dist/types/common/'));
-    return merge(processPages, processCommon, exportTypesBeta, exportTypes)
+    return merge(
+    // processPages,
+    processCommon, exportTypesBeta, exportTypes)
         .on('error', function (e) {
         console.log("💩" + red.underline.bold(' => Updating WIX failed!'));
         console.log("💩" + red.underline.bold(` => Error: ${orange(e.message)}`));

package/dist/helpers.d.ts

@@ -1,6 +1,6 @@
-import { ModuleSettings, ProjectSettings } from '.';
+import { LucySettings, ModuleSettings, ProjectSettings } from '.';
 export declare function installPackages(wixPackages: Record<string, string>, devPackages: Record<string, string>, cwd: string, locked: boolean): Promise<void>;
-export declare function gitInit(cwd: string, modules: Record<string, string>): Promise<void>;
+export declare function gitInit(cwd: string, modules: LucySettings['modules']): Promise<void>;
 export declare function runGulp(moduleSettings: ModuleSettings, projectSettings: ProjectSettings, task: string): Promise<void>;
 /**
  * Clean up and run a command before exiting the process.
@@ -11,3 +11,9 @@ export declare function cleanupWatchers(): void;
  * @param {string} processPattern - The substring to match (e.g., "wix:dev" or "@wix/cli/bin/wix.cjs").
  */
 export declare function killAllProcesses(processPattern: string): void;
+export interface VeloSyncConfig {
+    siteUrl: string;
+    secret: string;
+}
+export declare function saveConfig(config: VeloSyncConfig, file: string): Promise<void>;
+export declare function readConfig(file: string): Promise<VeloSyncConfig>;

package/dist/helpers.js

@@ -6,6 +6,7 @@ import path from 'path';
 import { fileURLToPath } from 'url';
 import { exec } from 'child_process';
 import os from 'os';
+import fs from 'fs';
 import { blue, green, orange, red, yellow, magenta } from './index.js';
 export async function installPackages(wixPackages, devPackages, cwd, locked) {
     if (locked)
@@ -43,11 +44,19 @@ export async function installPackages(wixPackages, devPackages, cwd, locked) {
     }
 }
 export async function gitInit(cwd, modules) {
-    const git = simpleGit({ baseDir: cwd });
-    for (const [name, url] of Object.entries(modules)) {
+    for (const [name, repo] of Object.entries(modules)) {
         console.log(chalk.green.underline.bold(`Cloning ${name}`));
+        const git = simpleGit({ baseDir: cwd });
         try {
-            await git.submoduleAdd(url, name);
+            const repoPath = path.resolve(cwd, name);
+            if (!fs.existsSync(repoPath)) {
+                await git.submoduleAdd(repo.url, name);
+            }
+            if (fs.existsSync(repoPath)) {
+                console.log(`🐕 ${blue.underline(' => Module already cloned!')}`);
+            }
+            const localGit = simpleGit({ baseDir: `${cwd}/${name}` });
+            await localGit.checkout(repo.branch);
         }
         catch (err) {
             console.log((`💩 ${red.underline.bold("=> Command failed =>")} ${orange(err)}`));
@@ -152,3 +161,10 @@ export function killAllProcesses(processPattern) {
         });
     });
 }
+export async function saveConfig(config, file) {
+    await fs.promises.writeFile(file, JSON.stringify(config));
+}
+export async function readConfig(file) {
+    let content = await fs.promises.readFile(file, 'utf-8');
+    return JSON.parse(content);
+}

package/dist/index.d.ts

@@ -1,7 +1,10 @@
 #!/usr/bin/env node --no-warnings
 export type LucySettings = {
     modules: {
-        [llibName: string]: string;
+        [llibName: string]: {
+            url: string;
+            branch: string;
+        };
     };
     wixSettings: {
         compilerOptions: {
@@ -33,6 +36,7 @@ export type ModuleSettings = {
     settings: LucySettings;
     lockVersion: boolean;
     force: boolean;
+    veloConfigName: string;
 };
 export type ProjectSettings = {
     modules?: Record<string, string>;

package/dist/index.js

@@ -78,12 +78,18 @@ async function main() {
         packageJsonPath: join(process.cwd(), 'package.json'),
         force: false,
         lockVersion: false,
+        veloConfigName: 'config.json'
     };
     let projectSettings = {};
     if (moduleSettings.args.includes('version') || moduleSettings.args.includes('-v')) {
         console.log("🐾" + blue.bold(` => ${projectPackageJSON.version}`));
         return;
     }
+    // Run velo sync
+    if (moduleSettings.args.includes('velo-sync')) {
+        await sync(moduleSettings, projectSettings);
+        return;
+    }
     if (moduleSettings.args.includes('help') || moduleSettings.args.includes('-h')) {
         console.log("🦮 " + green.underline.bold(' => Lucy CLI Help'));
         console.log("Usage: lucy-cli <command> [options]");
@@ -92,7 +98,7 @@ async function main() {
         console.log("🦮 " + magenta.bold('dev') + "                : Starts the development environment. This includes setting up any required services for local development.");
         console.log("🦮 " + magenta.bold('build-prod') + "         : Builds the project in production mode, optimizing files for deployment.");
         console.log("🦮 " + magenta.bold('prepare') + "            : Prepares the project by installing packages & initializing git modules, configured in lucy.json");
-        console.log("🦮 " + magenta.bold('sync') + "               : Synchronizes the database (not Implemented)");
+        console.log("🦮 " + magenta.bold('velo-sync') + "          : Synchronizes wix collections(velo-sync -h for help)");
         console.log("🦮 " + magenta.bold('install') + "            : Installs all Wix npm packages listed in the 'lucy.json' file in the project directory.");
         console.log("🦮 " + magenta.bold('fix') + "                : Runs a fix command to resolve common issues in development or production settings.");
         console.log("🦮 " + magenta.bold('docs') + "               : Generates documentation for the project.");
@@ -215,10 +221,6 @@ async function main() {
         await installPackages(projectSettings.lucySettings.wixPackages, projectSettings.lucySettings.devPackages, moduleSettings.targetFolder, moduleSettings.lockVersion);
         return;
     }
-    if (moduleSettings.args.includes('sync')) {
-        sync(moduleSettings, projectSettings);
-        return;
-    }
     if (moduleSettings.args.includes('dev')) {
         runGulp(moduleSettings, projectSettings, 'dev');
         return;

package/dist/settings.json

@@ -17,7 +17,11 @@
         "@total-typescript/ts-reset": "^0.6.1",
         "@types/node": "^22.9.1",
         "@types/nodemailer": "^6.4.17",
+        "typedoc-github-theme": "^0.2.0",
         "@types/react": "^19.0.0",
+        "@vitest/ui": "^2.1.8",
+        "eslint-import-resolver-typescript": "^3.7.0",
+        "@vitest/coverage-v8": "^2.1.8",
         "@typescript-eslint/eslint-plugin": "^8.15.0",
         "@typescript-eslint/parser": "^8.15.0",
         "@typescript-eslint/utils": "^8.15.0",

package/dist/sync.d.ts

@@ -1,2 +1,2 @@
-import { ModuleSettings, ProjectSettings } from ".";
-export declare function sync(moduleSettings: ModuleSettings, projectSettings: ProjectSettings): void;
+import { ModuleSettings, ProjectSettings } from "./index.js";
+export declare function sync(moduleSettings: ModuleSettings, projectSettings: ProjectSettings): Promise<void>;

package/dist/sync.js

@@ -1,5 +1,87 @@
-export function sync(moduleSettings, projectSettings) {
+import chalk from "chalk";
+import { green, orange, red } from "./index.js";
+import readline from 'node:readline';
+import veloAPI from 'velo-sync/dist/velo/velo-api.js';
+import syncTask from 'velo-sync/dist/tasks/sync-task.js';
+import { readConfig, saveConfig } from "./helpers.js";
+import optimist from 'optimist';
+import migrateFileCache from 'velo-sync/dist/tasks/migrate-files-cache-task.js';
+function printUsage() {
+    console.log('Usage:  ');
+    console.log('');
+    console.log('Commands:');
+    console.log('  init          generates a config file for the import / export / sync process');
+    console.log('  is-alive      tests the config and the connection to the site');
+    console.log('  sync          runs the sync process');
+    console.log('  import        runs an import process');
+    console.log('  migrate       migrate existing nedb cache to sqlite cache (.upload-cache.db => .upload-cache.sqlite.db)');
+}
+function syncOrImportTask(importOnly) {
+    let argv = optimist
+        .usage(`Usage: $0 ${importOnly ? 'import' : 'sync'} -f <scv filename> -c <collection>`)
+        .demand('f')
+        .alias('f', 'filename')
+        .describe('f', 'csv filename to import')
+        .demand('c')
+        .describe('c', 'the name of the collection to import into')
+        .alias('c', 'collection')
+        .demand('s')
+        .describe('s', 'schema file describing the fields of the collection')
+        .alias('s', 'schema')
+        .describe('dry', 'dry-run that does not upload any data or files, and does not remove or update anything on the site')
+        .alias('dry', 'dryrun')
+        .parse(process.argv.slice(3));
+    let filename = argv.filename;
+    let collection = argv.collection;
+    let schema = argv.schema;
+    let dryrun = argv.dryrun;
+    //@ts-ignore
+    syncTask.default(filename, collection, schema, importOnly, dryrun);
+}
+export async function sync(moduleSettings, projectSettings) {
+    if (moduleSettings.args.includes('-h') || moduleSettings.args.includes('help'))
+        return printUsage();
+    if (moduleSettings.args.includes('init')) {
+        const rl = readline.createInterface({
+            input: process.stdin,
+            output: process.stdout,
+            terminal: true
+        });
+        async function askQuestion(query) {
+            return new Promise((resolve) => rl.question(query, (answer) => resolve(answer)));
+        }
+        console.log(chalk.yellow('hello to velo-sync init'));
+        let siteUrl = await askQuestion('what is the url of the site homepage? ');
+        let secret = await askQuestion('what is the velo-sync secret? ');
+        rl.close();
+        let config = { siteUrl, secret };
+        await saveConfig(config, moduleSettings.veloConfigName);
+        return console.log(chalk.green("🐕" + 'config saved!'));
+    }
+    if (moduleSettings.args.includes('is-alive')) {
+        try {
+            let config = await readConfig(moduleSettings.veloConfigName);
+            console.log("🐕" + green(` => checking if the API for site ${chalk.greenBright(config.siteUrl)} is alive...`));
+            await veloAPI.isAlive(config);
+            return console.log(chalk.green("🐕" + `API of site ${chalk.greenBright(config.siteUrl)} is working and alive!!!`));
+        }
+        catch (e) {
+            if (e instanceof Error) {
+                return console.log((`💩 ${red.underline.bold("=> Failed to check endpoint")} ${orange(e.message)}`));
+            }
+        }
+    }
     if (moduleSettings.args.includes('sync')) {
+        return syncOrImportTask(false);
+    }
+    if (moduleSettings.args.includes('import')) {
+        return syncOrImportTask(true);
+    }
+    if (moduleSettings.args.includes('export')) {
+        return console.log((`💩 ${red.underline.bold("=> Not implemented")}`));
+    }
+    if (moduleSettings.args.includes('migrate')) {
+        //@ts-ignore
+        migrateFileCache.default();
     }
-    console.log('Hello sync');
 }

package/files/eslint.config.mjs

@@ -22,6 +22,11 @@ export default tseslint.config(
 			'named-import-spacing': namedImportSpacing,
 			jsdoc,
 		},
+		settings: {
+			'import/resolver': {
+				typescript: {}
+			}
+		},
 		languageOptions: {
 			parser: tseslint.parser,
 			parserOptions: {
@@ -37,6 +42,12 @@ export default tseslint.config(
 			},
 		},
 		rules: {
+			'no-restricted-imports': [
+				'error',
+				{
+					'patterns': ['*/backend/*', '*/**/public/*']
+				}
+			],
 			'no-restricted-syntax': [
 				'error',
 				{

package/files/typedoc.json

@@ -1,4 +1,3 @@
-/** @type { import('typedoc').TypeDocOptionMap & import('typedoc-plugin-merge-modules').Config } */
 {
   "entryPoints": [
     "*/**/*.ts",
@@ -7,22 +6,20 @@
   "entryPointStrategy": "expand",
   "out": "docs",
   "plugin": [
-    "typedoc-github-theme",
+    "typedoc-github-theme"
   ],
   "tsconfig": "local.tsconfig.json",
   "excludeExternals": true,
   "externalPattern": [
     "**/node_modules/**",
     "**/.wix/**",
-    ".wix/**",
+    ".wix/**"
   ],
   "exclude": [
     "./.wix/**/*",
     ".wix/**/*",
     "cypress/**",
-    "**/__mocks__/**",
+    "**/__mocks__/**"
   ],
-  "name": "wix-integraleakademie",
-  // "mergeModulesRenameDefaults": false, // NEW option of TypeDoc added by this plugin
-  // "mergeModulesMergeMode": "project", // NEW option of TypeDoc added by this plugin
+  "name": "Wix-lucy"
 }

package/files/typescript/backend/http-functions.ts

@@ -0,0 +1,65 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import { batchCheckUpdateState, clearStale, getImageUploadUrl, insertItemBatch, isAlive, saveItemBatch } from 'backend/lib/http-functions/sync2';
+import { WixHttpFunctionRequest } from 'wix-http-functions';
+
+/**------------------------------------------------------------------------
+ **                            Velo-Sync endpoints
+ * This function are for the velo-sync functionality
+ * https://www.npmjs.com/package/velo-sync
+ *------------------------------------------------------------------------**/
+/**
+ * Handle the isAlive request
+ * @param request The request object
+ * @returns The response object
+ */
+export async function post_isAlive(request: WixHttpFunctionRequest) {
+	return await isAlive(request);
+}
+
+/**
+ * Handle the insertItemBatch request
+ * @param request The request object
+ * @returns The response object
+ */
+export async function post_insertItemBatch(request: WixHttpFunctionRequest) {
+
+	return await insertItemBatch(request);
+}
+
+/**
+ * Handle the saveItemBatch request
+ * @param request The request object
+ * @returns The response object
+ */
+export async function post_saveItemBatch(request: WixHttpFunctionRequest) {
+	return await saveItemBatch(request);
+}
+
+/**
+ * Handle the clearStale request
+ * @param request The request object
+ * @returns The response object
+ */
+export async function post_clearStale(request: WixHttpFunctionRequest) {
+	return await clearStale(request);
+}
+
+/**
+ * Handle the batchCheckUpdateState request
+ * @param request The request object
+ * @returns The response object
+ */
+export async function post_batchCheckUpdateState(request: WixHttpFunctionRequest) {
+	return await batchCheckUpdateState(request);
+}
+
+/**
+ * Handle the getImageUploadUrl request
+ * @param request The request object
+ * @returns The response object
+ */
+export async function post_getImageUploadUrl(request: WixHttpFunctionRequest) {
+	return await getImageUploadUrl(request);
+}
+/*--------------- END OF SECTION --------------*/
+

package/files/typescript/backend/lib/http-functions/sync.ts

@@ -0,0 +1,270 @@
+/* eslint-disable @typescript-eslint/no-unsafe-call */
+/* eslint-disable @typescript-eslint/no-unsafe-argument */
+/* eslint-disable @typescript-eslint/no-unsafe-return */
+/* eslint-disable @typescript-eslint/naming-convention */
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable @typescript-eslint/no-unsafe-assignment */
+
+import crypto from 'crypto';
+import wixData from 'wix-data';
+import { forbidden, ok, serverError, WixHttpFunctionRequest, WixHttpFunctionResponse } from 'wix-http-functions';
+import { mediaManager } from 'wix-media-backend';
+import wixSecretsBackend from 'wix-secrets-backend';
+
+/**----------------------------------------------
+ * *                   INFO
+ *   URL to call this HTTP function from your published site looks like:
+ *   Premium site - https://mysite.com/_functions/example/multiply?leftOperand=3&rightOperand=4
+ *   Free site - https://username.wixsite.com/mysite/_functions/example/multiply?leftOperand=3&rightOperand=4
+ * 
+ *   URL to test this HTTP function from your saved site looks like:
+ *   Premium site - https://mysite.com/_functions-dev/example/multiply?leftOperand=3&rightOperand=4
+ *   Free site - https://username.wixsite.com/mysite/_functions-dev/example/multiply?leftOperand=3&rightOperand=4
+ *---------------------------------------------**/
+
+//! WARNING: The following code is taken out of velo-sync package.
+//! WARNING: This serves as example and is not tested.
+
+class ForbiddenError extends Error {
+	constructor(message: string) {
+		super(message);
+		this.type = FORBIDDEN;
+	}
+	type: string;
+}
+
+const FORBIDDEN = 'forbidden';
+
+/**
+ * Validates the request and parses the payload
+ * @param request - the request object
+ * @returns the parsed payload
+ */
+export async function validateAndParseRequest(request: WixHttpFunctionRequest) {
+	const payload = await request.body.text();
+	const payloadJson = JSON.parse(payload, dateReviver) as any;
+	const secret = await wixSecretsBackend.getSecret('velo-sync');
+	const hmac = crypto.createHmac('sha256', secret);
+	hmac.update(JSON.stringify(payloadJson.data, dateReplacer));
+	const digest = hmac.digest('hex');
+	if (digest !== payloadJson.signature){
+		const forbiddenError = new ForbiddenError('invalid signature check');
+		forbiddenError.type = FORBIDDEN;
+		throw forbiddenError;
+	}
+	
+	return payloadJson.data;
+}
+
+/**
+ * Logs the request and handles the response
+ * @param name - the name of the request
+ * @param handler - the handler function
+ * @returns the response object
+ */
+export async function logRequest(name:string, handler: () => Promise<any>) {
+	console.log(name, 'start');
+	const start = new Date().getTime();
+	try {
+		const response = await handler();
+		const now = new Date().getTime();
+		console.log(name, 'completed ok, time:', now - start);
+		
+		return ok({ body: response });
+	}
+	catch (e: any){
+		const now = new Date().getTime();
+		if (e.type === FORBIDDEN){
+			console.log(name, 'forbidden:', e.message, ', time:', now - start);
+			
+			return forbidden({ body: e.message });
+		}
+		else {
+			console.log(name, 'failed with error:', e.message, ', time:', now - start);
+			
+			return serverError({ body: e.message });
+		}
+	}
+}
+
+/**
+ * Checks if the service is alive
+ * @param request - the request object
+ * @returns the response object
+ */
+export async function isAlive(request: WixHttpFunctionRequest): Promise<WixHttpFunctionResponse> {
+	return await logRequest('isAlive', async () => {
+		const data = await validateAndParseRequest(request);
+		if (data.isAlive === '?')
+		{return 'ok';}
+		else
+		{throw new Error('protocol error - the isAlive API expects isAlive member in the data payload');}
+	});
+}
+
+/**
+ * Inserts a batch of items
+ * @param request - the request object
+ * @returns the response object
+ */
+export async function insertItemBatch(request: WixHttpFunctionRequest): Promise<WixHttpFunctionResponse> {
+	return await logRequest('insertItemBatch', async () => {
+		const data = await validateAndParseRequest(request);
+		const itemsToInsert = data.items;
+		const collection = data.collection;
+		
+		return await wixData.bulkInsert(collection, itemsToInsert, { suppressAuth: true });
+	});
+}
+
+/**
+ * Saves a batch of items
+ * @param request - the request object
+ * @returns the response object
+ */
+export async function saveItemBatch(request: WixHttpFunctionRequest): Promise<WixHttpFunctionResponse> {
+	return await logRequest('saveItemBatch', async () => {
+		const data = await validateAndParseRequest(request);
+		const items = data.items;
+		const collection = data.collection;
+		
+		return await wixData.bulkSave(collection, items, { suppressAuth: true });
+	});
+}
+
+/**
+ * Clears stale items
+ * @param request - the request object
+ * @returns the response object
+ */
+export async function clearStale(request: WixHttpFunctionRequest): Promise<WixHttpFunctionResponse> {
+	return await logRequest('clearStale', async () => {
+		const data = await validateAndParseRequest(request);
+		const collection = data.collection;
+
+		const date = new Date();
+		date.setDate(date.getDate() - 3);
+
+		const res = await wixData.query(collection)
+			.lt('_updatedDate', date)
+			.find({ suppressAuth: true });
+		console.log(`clearStale - found ${res.totalCount} items to remove, current page ${res.length}`);
+		const itemsToDelete = res.items;
+		const ids = itemsToDelete.map((_: any )=> _._id);
+		const removeResult = await wixData.bulkRemove(collection, ids, { suppressAuth: true });
+
+		return { itemsRemoved: removeResult.removed, staleItems: res.totalCount - removeResult.removed, errors: removeResult.errors };
+	});
+}
+
+/**
+ * Batch checks the update state of items
+ * @param request - the request object
+ * @returns the response object
+ */
+export async function batchCheckUpdateState(request: WixHttpFunctionRequest): Promise<WixHttpFunctionResponse> {
+	return await logRequest('batchCheckUpdateState', async () => {
+		const data = await validateAndParseRequest(request);
+
+		const collection = data.collection;
+		const items = data.items;
+		const dryrun = data.dryrun;
+
+		const queries = items.map((item: any) => wixData.query(collection).eq('_id', item._id));
+
+		const query = queries.reduce((accuQuery: any, query: any) => (accuQuery)?accuQuery.or(query): query);
+		const result:any[] = [];
+		const itemsToUpdate: any[] = [];
+		const res = await query.find({ suppressAuth: true });
+		items.forEach((item: any) => {
+			const foundItem = res.items.find((_: any) => _._id === item._id);
+			if (foundItem && foundItem._hash === item._hash){
+				itemsToUpdate.push(foundItem);
+				result.push({ status: 'ok', _id: item._id });
+			}
+			else if (foundItem){
+				result.push({ status: 'need-update', _id: item._id });
+			}
+			else {
+				result.push({ status: 'not-found', _id: item._id });
+			}
+		});
+		if (!dryrun)
+		{await wixData.bulkUpdate(collection, itemsToUpdate, { suppressAuth: true });}
+		
+		return JSON.stringify(result);
+	});
+}
+
+/**
+ * Gets an image upload URL
+ * @param request - the request object
+ * @returns the response object
+ */
+export async function getImageUploadUrl(request: WixHttpFunctionRequest): Promise<WixHttpFunctionResponse> {
+	return await logRequest('getImageUploadUrl', async () => {
+		const data = await validateAndParseRequest(request);
+
+		const mimeType = data.mimeTypes;
+		const _id = data._id;
+		const fieldName = data.fieldName;
+		const collection = data.collection;
+		const mediaType = data.mediaType;
+
+		const uploadUrlObj = await mediaManager.getUploadUrl('/synced-images',
+			{
+				'mediaOptions': {
+					mimeType,
+					mediaType
+				},
+				'metadataOptions': {
+					'isPrivate': false,
+					'isVisitorUpload': false,
+					'context': {
+						_id,
+						fieldName,
+						collection
+					}
+				}
+			});
+		
+		return uploadUrlObj;
+	});
+}
+
+const dateRegex = /^Date\((\d+)\)$/;
+/**
+ *
+ * @param key
+ * @param value
+ */
+ 
+/**
+ * Reviver function for JSON.parse that converts string representations of Date objects to Date objects
+ * @param key The key of the value
+ * @param value The value to convert
+ * @returns The converted Date object
+ */
+export function dateReviver(key: string, value: string) {
+	const match = dateRegex.exec(value);
+	if (match){
+		return new Date(Number(match[1]));
+	}
+	
+	return value;
+}
+
+/**
+ * Replacer function for JSON.stringify that converts Date objects to a string representation
+ * @param key The key of the value
+ * @param value The value to convert
+ * @returns The converted value
+ */
+export function dateReplacer(this: any, key: string, value: string) {
+	const v = this[key];
+	if (v instanceof Date)
+	{return 'Date('+v.getTime()+')';}
+	else
+	{return value;}
+}

package/files/typescript/tsconfig.json

@@ -5,6 +5,7 @@
     "compilerOptions": {
         "outDir": "../src",
         "rootDir": ".",
+        "baseUrl": "./",
         "target": "ES2020",
         "module": "ES2020",
         "moduleResolution": "Node",

package/package.json

@@ -1,7 +1,7 @@
 {
   "type": "module",
   "name": "lucy-cli",
-  "version": "0.10.0",
+  "version": "1.0.0",
   "description": "Lucy Framework for WIX Studio Editor",
   "main": ".dist/index.js",
   "scripts": {
@@ -61,6 +61,7 @@
     "jest": "^29.7.0",
     "merge-stream": "^2.0.0",
     "merge2": "^1.4.1",
+    "optimist": "^0.6.1",
     "prettier": "^3.0.3",
     "sass": "^1.65.1",
     "simple-git": "^3.20.0",
@@ -84,6 +85,7 @@
     "@types/jest": "^29.5.3",
     "@types/merge-stream": "2.0.0",
     "@types/merge2": "^1.4.4",
+    "@types/optimist": "^0.0.33",
     "@typescript-eslint/eslint-plugin": "8.14.0",
     "@typescript-eslint/parser": "8.14.0",
     "@typescript-eslint/utils": "8.14.0",
@@ -92,6 +94,8 @@
     "eslint-plugin-jsdoc": "50.5.0",
     "eslint-plugin-named-import-spacing": "^1.0.3",
     "eslint-plugin-simple-import-sort": "12.1.1",
+    "i": "^0.3.7",
+    "npm": "^11.0.0",
     "ts-node": "^10.9.1"
   }
 }

package/src/gulp/checks.ts

@@ -1,5 +1,5 @@
 import * as fs from 'fs';
-import glob from 'glob';
+import { glob } from 'glob';
 import * as path from 'path';
 import gulp from 'gulp';
 import ts from 'gulp-typescript';
@@ -27,54 +27,23 @@ function extractMatchFromFile(filePath: string, pattern: string) {
 	});
 }
 
-/**
- * Reads files in a folder
- * @param {string} folderPath Folder path
- * @param {string} pattern Pattern to match
- * @param {string} globPattern Glob pattern
- */
-function readFilesInFolder(folderPath: string, pattern: string | null, globPattern: string,) {
-	return new Promise((resolve, reject) => {
-		glob(path.join(folderPath, globPattern), (err: unknown, files: string[]) => {
-			if (err){
-				reject(err);
+async function readFilesInFolder(folderPath: string, pattern: string | null, globPattern: string): Promise<Object[]> {
+    const files = await glob(path.join(folderPath, globPattern));
+    const filenameList: Object[] = [];
 
-				return;
-			}  
-			const filenameList: Object[] = [];
-			/**
-			 * Traverse files
-			 * @param {number} index Index
-			 */
-			function traverseFiles(index: number) {
-				if (index === files.length){
-					resolve(filenameList);
-					
-					return;
-				}
-				const file = files[index];
-				if(pattern){
-					if(!file) return
-					extractMatchFromFile(file, pattern)
-						.then((capturedGroup) => {
-							if (capturedGroup){
-								filenameList.push(capturedGroup);
-							}
-							traverseFiles(index + 1);
-						})
-						.catch(reject);
-				}
-				if(!pattern){
-					if(!file) return
-					filenameList.push(path.basename(file));
-					traverseFiles(index + 1);
-				}
-			}
-			traverseFiles(0);
-		});
-	});
-}
+    for (const file of files) {
+        if (pattern) {
+            const capturedGroup = await extractMatchFromFile(file, pattern);
+            if (capturedGroup) {
+                filenameList.push(capturedGroup);
+            }
+        } else {
+            filenameList.push(path.basename(file));
+        }
+    }
 
+    return filenameList;
+}
 export async function checkPages(fail: boolean, force: boolean) {
 	console.log("🐕" + green.underline.bold(' => Checking pages...'));
 	return new Promise<void>(async (resolve, reject) => {

package/src/gulp/types.ts

@@ -159,9 +159,9 @@ export function updateWixTypes(options: TaskOptions) {
 
 export function addTypes(options: TaskOptions, done: gulp.TaskFunctionCallback): NodeJS.ReadWriteStream {
 	const { replaceOptions } = options;
-	const processPages = gulp.src(['./.wix/types/wix-code-types/dist/types/page/$w.d.ts'])
-		.pipe(replace('declare namespace \\$w {', ' declare namespace $w{\nconst api: $w.Api;\n', replaceOptions))
-		.pipe(gulp.dest('./.wix/types/wix-code-types/dist/types/page/'));
+	// const processPages = gulp.src(['./.wix/types/wix-code-types/dist/types/page/$w.d.ts'])
+	// 	.pipe(replace('declare namespace \\$w {', ' declare namespace $w{\nconst api: $w.Api;\n', replaceOptions))
+	// 	.pipe(gulp.dest('./.wix/types/wix-code-types/dist/types/page/'));
 
 	const exportTypes = gulp.src(['./.wix/types/wix-code-types/dist/types/common/*.d.ts', '!./.wix/types/wix-code-types/dist/types/common/$w.d.ts'])
 		.pipe(replace('interface ', 'export interface ', replaceOptions))
@@ -176,12 +176,12 @@ export function addTypes(options: TaskOptions, done: gulp.TaskFunctionCallback):
 		.pipe(gulp.dest('./.wix/types/wix-code-types/dist/types/beta/common/'));
 
 	const processCommon = gulp.src(['./.wix/types/wix-code-types/dist/types/common/$w.d.ts'])
-		.pipe(insert.prepend("import { FrontendAPI } from '../../../../../../typescript/public/models/frontendApi.model';\nimport '@total-typescript/ts-reset';\n"))
-		.pipe(replace('namespace \\$w {', 'declare namespace $w{\ntype Api = FrontendAPI;\n', replaceOptions))
+		.pipe(insert.prepend("import '@total-typescript/ts-reset';\n"))
+		// .pipe(replace('namespace \\$w {', 'declare namespace $w{\ntype Api = FrontendAPI;\n', replaceOptions))
 		.pipe(gulp.dest('./.wix/types/wix-code-types/dist/types/common/'));
 
 	return merge(
-		processPages,
+		// processPages,
 		processCommon,
 		exportTypesBeta,
 		exportTypes,

package/src/helpers.ts

@@ -4,11 +4,13 @@ import { spawnSync } from 'child_process';
 // https://www.sergevandenoever.nl/run-gulp4-tasks-programatically-from-node/
 import path from 'path';
 import { fileURLToPath } from 'url';
-import { ModuleSettings, ProjectSettings } from '.';
+import { LucySettings, ModuleSettings, ProjectSettings } from '.';
 import { exec } from 'child_process';
 import os from 'os';
+import fs from 'fs';
 
 import { blue, green, orange, red, yellow, magenta } from './index.js';
+import { fsync } from 'fs';
 
 export async function installPackages(wixPackages: Record<string, string>, devPackages: Record<string, string>,  cwd: string, locked: boolean ) {
 	if (locked) console.log("🐕" + blue.underline(` => Installing & version locked packages!`));
@@ -47,12 +49,23 @@ export async function installPackages(wixPackages: Record<string, string>, devPa
 	}																
 }
 
-export async function gitInit(cwd: string, modules: Record<string, string>) {
-	const git = simpleGit({ baseDir: cwd });
-	for (const [name, url] of Object.entries(modules)) {
+export async function gitInit(cwd: string, modules: LucySettings['modules']) {
+	for (const [name, repo] of Object.entries(modules)) {
 		console.log(chalk.green.underline.bold(`Cloning ${name}`));
+        const git = simpleGit({ baseDir: cwd });
+
 		try {
-			await git.submoduleAdd(url, name)
+            const repoPath = path.resolve(cwd, name);
+            if (!fs.existsSync(repoPath)) {
+                await git.submoduleAdd(repo.url, name)
+            }
+
+            if (fs.existsSync(repoPath)) {
+                console.log(`🐕 ${blue.underline(' => Module already cloned!')}`);
+            }
+
+            const localGit = simpleGit({ baseDir: `${cwd}/${name}` });
+            await localGit.checkout(repo.branch);
 		} catch (err) {
 			console.log((`💩 ${red.underline.bold("=> Command failed =>")} ${orange(err)}`));
 		} finally {	
@@ -162,4 +175,16 @@ export function killAllProcesses(processPattern: string) {
             }
         });
     });
+}
+
+export interface VeloSyncConfig {
+    siteUrl: string;
+    secret: string;
+}
+export async function saveConfig(config:VeloSyncConfig, file: string) {
+    await fs.promises.writeFile(file, JSON.stringify(config));
+}
+export async function readConfig(file: string): Promise<VeloSyncConfig> {
+    let content = await fs.promises.readFile(file, 'utf-8');
+    return JSON.parse(content);
 }

package/src/index.ts

@@ -17,7 +17,10 @@ import { spawnSync } from 'child_process';
 
 export type LucySettings = {
 	modules: {
-		[llibName: string]: string;
+		[llibName: string]: {
+			url: string;
+			branch: string;
+		};
 	};
 	wixSettings: {
 		compilerOptions: {
@@ -50,6 +53,7 @@ export type ModuleSettings = {
 	settings: LucySettings;
 	lockVersion: boolean;
 	force: boolean;
+	veloConfigName: string;
 }
 
 export type ProjectSettings = {
@@ -133,6 +137,7 @@ async function main(): Promise<void> {
 		packageJsonPath: join(process.cwd(), 'package.json'),
 		force: false,
 		lockVersion: false,
+		veloConfigName: 'config.json'
 	}
 
 	let projectSettings: ProjectSettings = {}; 
@@ -142,6 +147,12 @@ async function main(): Promise<void> {
         
 		return;
 	}
+	// Run velo sync
+	if(moduleSettings.args.includes('velo-sync')){
+		await sync(moduleSettings, projectSettings);
+
+		return;
+	}
 
 	if(moduleSettings.args.includes('help') || moduleSettings.args.includes('-h')){
 		console.log("🦮 " + green.underline.bold(' => Lucy CLI Help'));
@@ -151,7 +162,7 @@ async function main(): Promise<void> {
 		console.log("🦮 " + magenta.bold('dev') + "                : Starts the development environment. This includes setting up any required services for local development.");
 		console.log("🦮 " + magenta.bold('build-prod') + "         : Builds the project in production mode, optimizing files for deployment.");
 		console.log("🦮 " + magenta.bold('prepare') + "            : Prepares the project by installing packages & initializing git modules, configured in lucy.json");
-		console.log("🦮 " + magenta.bold('sync') + "               : Synchronizes the database (not Implemented)");
+		console.log("🦮 " + magenta.bold('velo-sync') + "          : Synchronizes wix collections(velo-sync -h for help)");
 		console.log("🦮 " + magenta.bold('install') + "            : Installs all Wix npm packages listed in the 'lucy.json' file in the project directory.");
 		console.log("🦮 " + magenta.bold('fix') + "                : Runs a fix command to resolve common issues in development or production settings.");
 		console.log("🦮 " + magenta.bold('docs') + "               : Generates documentation for the project.");
@@ -220,6 +231,7 @@ async function main(): Promise<void> {
 
 	console.log("🐕" + magenta.underline(' => Lucy CLI => RUNNING: ' + orange('Press Ctrl+C to stop.')));
 	// INFO: Run commands
+
 	if(moduleSettings.args.includes('init')){
 		if(projectSettings.lucySettings?.initialized && !moduleSettings.force) {
 			console.log((`💩 ${red.underline.bold("=> This project is already initialized =>")} ${orange(moduleSettings.targetFolder)}`));
@@ -286,12 +298,7 @@ async function main(): Promise<void> {
 
 		return;
 	}
-	
-	if(moduleSettings.args.includes('sync')){
-		sync(moduleSettings, projectSettings);
 
-		return;
-	}
 
 	if(moduleSettings.args.includes('dev')){
 		runGulp(moduleSettings, projectSettings, 'dev');

package/src/settings.json

@@ -20,6 +20,7 @@
     "typedoc-github-theme": "^0.2.0",
     "@types/react": "^19.0.0",
     "@vitest/ui": "^2.1.8",
+    "eslint-import-resolver-typescript": "^3.7.0",
     "@vitest/coverage-v8": "^2.1.8",
     "@typescript-eslint/eslint-plugin": "^8.15.0",
     "@typescript-eslint/parser": "^8.15.0",

package/src/sync.ts

@@ -1,10 +1,97 @@
-import { modifierNames } from "chalk";
-import { ModuleSettings, ProjectSettings } from ".";
+import chalk, { modifierNames } from "chalk";
+import { blue, green, ModuleSettings, orange, ProjectSettings, red } from "./index.js";
+import readline from 'node:readline';
+import {createDataSync, LoggingStatistics, LoggerRejectsReporter} from 'velo-sync';
+import veloAPI from 'velo-sync/dist/velo/velo-api.js';
+import syncTask from 'velo-sync/dist/tasks/sync-task.js';
+import { readConfig, saveConfig, VeloSyncConfig } from "./helpers.js";
+import optimist from 'optimist';
+import migrateFileCache from 'velo-sync/dist/tasks/migrate-files-cache-task.js';
+function printUsage() {
+    console.log('Usage:  ');
+    console.log('');
+    console.log('Commands:');
+    console.log('  init          generates a config file for the import / export / sync process');
+    console.log('  is-alive      tests the config and the connection to the site');
+    console.log('  sync          runs the sync process');
+    console.log('  import        runs an import process');
+    console.log('  migrate       migrate existing nedb cache to sqlite cache (.upload-cache.db => .upload-cache.sqlite.db)');
+}
 
-export function sync(moduleSettings: ModuleSettings, projectSettings: ProjectSettings) {
+function syncOrImportTask(importOnly: boolean) {
+    let argv = optimist
+        .usage(`Usage: $0 ${importOnly ? 'import' : 'sync'} -f <scv filename> -c <collection>`)
+        .demand('f')
+        .alias('f', 'filename')
+        .describe('f', 'csv filename to import')
+        .demand('c')
+        .describe('c', 'the name of the collection to import into')
+        .alias('c', 'collection')
+        .demand('s')
+        .describe('s', 'schema file describing the fields of the collection')
+        .alias('s', 'schema')
+        .describe('dry', 'dry-run that does not upload any data or files, and does not remove or update anything on the site')
+        .alias('dry', 'dryrun')
+        .parse(process.argv.slice(3));
+    let filename = argv.filename;
+    let collection = argv.collection;
+    let schema = argv.schema;
+    let dryrun = argv.dryrun;
+    //@ts-ignore
+    syncTask.default(filename, collection, schema, importOnly, dryrun);
+}
 
-    if(moduleSettings.args.includes('sync')){
-		
+export async function sync(moduleSettings: ModuleSettings, projectSettings: ProjectSettings) {
+
+    if(moduleSettings.args.includes('-h') || moduleSettings.args.includes('help')) return printUsage();
+    if(moduleSettings.args.includes('init')) {
+        const rl = readline.createInterface({
+            input: process.stdin,
+            output: process.stdout,
+            terminal: true
+        });
+
+        async function askQuestion(query: string):Promise<string> {
+            return new Promise((resolve) => rl.question(query, (answer) => resolve(answer)));
+        }
+
+        console.log(chalk.yellow('hello to velo-sync init'));
+        let siteUrl = await askQuestion('what is the url of the site homepage? ');
+        let secret = await askQuestion('what is the velo-sync secret? ');
+        rl.close();
+        let config: VeloSyncConfig = { siteUrl, secret };
+        await saveConfig(config, moduleSettings.veloConfigName);
+        return console.log(chalk.green("🐕" + 'config saved!'));
 	}
-    console.log('Hello sync');
+
+    if(moduleSettings.args.includes('is-alive')) {
+        try {
+            let config = await readConfig(moduleSettings.veloConfigName);
+            console.log("🐕" + green(` => checking if the API for site ${chalk.greenBright(config.siteUrl)} is alive...`));
+            await veloAPI.isAlive(config);
+            return console.log(chalk.green("🐕" + `API of site ${chalk.greenBright(config.siteUrl)} is working and alive!!!`));
+        }
+        catch (e) {
+            if(e instanceof Error)  {
+                return console.log((`💩 ${red.underline.bold("=> Failed to check endpoint")} ${orange(e.message)}`));
+            }
+        }
+    }
+    if(moduleSettings.args.includes('sync')) {
+        return syncOrImportTask(false);
+    }
+    
+    if(moduleSettings.args.includes('import')) {
+        return syncOrImportTask(true);
+    }
+
+    if(moduleSettings.args.includes('export')) {
+        return console.log((`💩 ${red.underline.bold("=> Not implemented")}`));
+    }
+    
+    if(moduleSettings.args.includes('migrate')) {
+        //@ts-ignore
+        migrateFileCache.default();
+    }
+
 }

package/src/types.d.ts

@@ -5,4 +5,4 @@ declare module 'gulp-string-replace';
 declare module 'gulp-wait';
 declare module 'gulp-jest';
 declare module 'gulp-flatmap';
-declare module 'gulp-swc';
+declare module 'gulp-swc';