@caweb/cli 1.3.0 → 1.3.2

package/README.md

@@ -11,6 +11,7 @@ The following WordPress packages are used:
   - <strong>For Debian-Based Linux distributions:</strong> <code>docker-compose</code> may need to be installed with: <code>sudo apt-get install docker-compose</code>.
   - <strong>For Windows users:</strong> [WSL should be set to version 2 for Windows Docker Desktop compatibility](https://docs.docker.com/desktop/windows/wsl/).
 - git is installed.
+- php is installed.
 
 ## Additional Features
 - Downloads and configures the [CAWeb Theme](https://github.com/CAWebPublishing/CAWeb)
@@ -20,13 +21,17 @@ The following WordPress packages are used:
   - phpMyAdmin development site starts at http://localhost:8080  
   - phpMyAdmin test site started at http://localhost:9090
 - Uses CAWebPublishing [External Project Template Configuration](https://developer.wordpress.org/block-editor/reference-guides/packages/packages-create-block/packages-create-block-external-template/) when creating Gutenberg Blocks, see configurations [here](https://github.com/CAWebPublishing/cli/lib/template)
-- Adds config.yml to both cli containers 
+- Allows for syncing of WordPress instance via Rest API and CLI, requires ssh. 
+- Adds config.yml to both cli containers, alias are added for each ssh connection.  
+<b>Example config.yml file</b>
 <pre>
-path: /var/www/html
-apache_modules:
-  - mod_rewrite
+    path: /var/www/html
+    apache_modules:
+      - mod_rewrite
+    @staging:
+        ssh: wpcli@staging.wp-cli.org
 </pre>
-.
+
 
 ## Command Reference
 ### `caweb start`  

package/commands/env/start.js

@@ -20,15 +20,17 @@ const CONFIG_CACHE_KEY = 'config_checksum';
 /**
  * Internal dependencies
  */
-import { appPath } from '../../lib/helpers.js';
-import wpEnvConfig from '../../configs/wp-env.js';
-import dockerConfig from '../../configs/docker-compose.js';
-
 import { 
+	appPath,
+	projectPath,
 	configureCAWeb,
 	downloadSources,
-	configureWordPress
- } from '../../lib/wordpress/index.js';
+	configureWordPress,
+	runCmd
+} from '../../lib/index.js';
+import wpEnvConfig from '../../configs/wp-env.js';
+import dockerConfig from '../../configs/docker-compose.js';
+
 
 /**
  * Starts the development server.
@@ -145,6 +147,17 @@ export default async function start({
 			} ),
 		] );
 
+		// Create an Application Password for the user.
+		/*
+		const devAppPwd = await runCLICmd(
+				'wp',
+					[
+						`user application-password create 1 caweb`,
+						'--porcelain'
+					]
+				);
+		*/
+		
 	}
 
 	// Start phpMyAdmin Service.

package/commands/sync.js

@@ -3,339 +3,21 @@
  */
 import path from 'path';
 import fs from 'fs';
-import axios from 'axios';
-import terminalLink from  'terminal-link';
 import loadConfig from '@wordpress/env/lib/config/load-config.js';
+import axios from 'axios';
 
 /**
  * Internal dependencies
  */
 import {
 	appPath,
-    projectPath,
-    runCmd
+    getTaxonomies,
+    createTaxonomies
 } from '../lib/index.js';
 
-const endpoint = '/wp-json/wp/v2';
-const requestDelay = 1000; // we wait 1 second between requests
-
-const sleep = (waitTimeInMs) => new Promise(resolve => setTimeout(resolve, waitTimeInMs));
-
-function processData( data ){
-    /**
-     * If wpautop is enabled and data contains Divi shortcodes, 
-     * WordPress will automatically add an opening/closing p tag that needs to be removed.
-     * @link https://developer.wordpress.org/reference/functions/wpautop/
-    */
-    if( data.includes('et_pb_section') && 
-        data.startsWith('<p>') &&
-        data.endsWith('</p>\n')
-    ){
-        data = data.substring(3, data.length - 5)
-    }
-    return data
-        .replace(/(&#8221;)|(&#8243;)|(&#8220;)/g, '') // replace double encoding
-        .replace(/(\s{2,})/g, ' ') // replace misc spacing // &#8220;
-
-}
-
-async function getTaxonomies( request, tax = 'pages', limit = 5000 ){
-    const fields = request.fields ? request.fields : [
-        'id',
-        'type',
-        'alt_text',
-        'caption', 
-        'slug',
-        'title',
-        'source_url',
-        'mime_type',
-        'content',
-        'date',
-        'date_gmt',
-        'status',
-        'featured_media',
-        'comment_status',
-        'ping_status',
-        'template',
-        'format',
-        'categories',
-        'tags',
-        'meta'
-    ];
-
-    let urlParams = [
-        'per_page=100',
-        `_fields=${ fields.join(',')}`,
-        'order=asc'
-    ]
-
-    // if no request is added default to GET
-    if( ! request.method ){
-        request.method = 'GET'
-    }
-
-    // if parent argument 
-    if( request.parent ){
-        urlParams.push( `parent=` + ( Array === typeof request.parent ? request.parent.join(',') : request.parent ))
-    }
-
-    // if include argument 
-    if( request.include ){
-        urlParams.push( `include=` + ( Array === typeof request.include ? request.include.join(',') : request.include ))
-    }
-
-    let collection = [];
-
-    for( let i = 1; i < limit / 100; i++ ){
-        // make requests for each page
-        urlParams.push(`page=${i}`)
-
-        let results = await axios.request(
-            {
-                ...request,
-                url: `${request.url}${endpoint}/${tax}?` + urlParams.join('&')
-            }
-        ).then(
-            (res) => { 
-                return res.data
-        }).catch( 
-            (error) => {return false}
-        );
-
-
-        // if no more results stop making requests
-        if( ! results ){
-            break;
-        }
-        
-        // add results to collection
-        collection = collection.concat( results );
-
-        // lets wait requestDelay between requests.
-        await sleep( requestDelay );
-    }
-    
-    return collection;
-}
-
-async function createTaxonomies( taxData, request, tax = 'pages', spinner ){
-    // if no request is added default to POST
-    if( ! request.method ){
-        request.method = 'POST'
-    }
-
-    let collection = [];
-
-
-    for( let obj of taxData ){
-        // endpoint url.
-        let url = `${request.url}${endpoint}/${tax}`;
-        let existingID = false;
-        
-        // in order to maintain ID's, we have to use the cli, the REST API doesn't allow passing the id.
-        if( obj.id ){
-            // first we check if the ID exist
-            let idExists = await axios.request(
-                {   
-                    ...request, 
-                    method: 'GET',
-                    url: `${url}/${ obj.id }`
-                } 
-                )
-                .then((res) => { return res.data.id })
-                .catch(error => {return false;})
-
-            // for menus if ID doesn't exist we also check for matching slug
-            // if a menu with a matching slug exists we return that id instead.
-            if( 'menus' === tax && ! idExists ){
-                idExists = await axios.request(
-                    {   
-                        ...request, 
-                        method: 'GET',
-                        url: `${url}/?slug=${ obj.slug }`
-                    } 
-                    )
-                    .then((res) => { return res.data[0].id })
-                    .catch(error => {return false;})
-
-            }
-
-            /**
-             * if the ID doesn't exist we save it so we can update via CLI later on.
-             * if it does exist update endpoint url so we update the existing item.
-             */
-            if( ! idExists ){
-                existingID = obj.id;
-            }else{
-                url = `${url}/${ obj.id }`;
-            }
-
-            // id has to be deleted.        
-            delete obj.id;
-        }
-
-        // process properties
-        if( obj.content ){
-            obj.content = processData(obj.content.rendered);
-        }
-        
-        if( obj.title ){
-            obj.title = processData(obj.title.rendered);
-        }
-
-        if( obj.caption ){
-            obj.caption = processData(obj.caption.rendered);
-        }
-
-        // make WordPress REST API request.
-        let results = await axios.request({
-                ...request,
-                data: 'media' === tax ? createMediaItem(obj) : obj,
-                url
-            })
-            .then( async (res) => { return res.data; }, axiosErrorHandler )
-
-            
-        /**
-         * if the obj had an existing ID that didn't exist
-         */
-        if( existingID ){
-            
-            let post_tbl = `wp_posts`;
-            let post_meta_tbl = `wp_postmeta`;
-            let term_taxonomy_tbl = `wp_term_taxonomy`;
-            
-            const cmd = [
-                path.join(projectPath, 'bin', 'wp-cli.phar'),
-                `--ssh=${request.ssh}`,
-                '--skip-themes',
-                '--skip-column-names',
-                `--url=${new URL(request.url).origin}`
-            ];
-
-            // if taxonomy is page/post/media.
-            if( ['pages', 'posts', 'media'].includes(tax) ){
+const errors = [];
 
 
-                /**
-                 * Since the REST API doesn't detected trashed posts/pages.
-                 * We have to do a drop before we can do an update.
-                 * Since trashed posts/pages stay in the database
-                 */
-                // drop post
-                await runCmd(
-                    'php',
-                    [
-                        ...cmd,
-                        `db query 'DELETE FROM ${post_tbl} WHERE ID=${existingID}'`,
-                    ]
-                )
-
-                // drop post meta.
-                await runCmd(
-                    'php',
-                    [
-                        ...cmd,
-                        `db query 'DELETE FROM ${post_meta_tbl} WHERE post_id=${existingID}'`,
-                    ]
-                )
-
-                // now we can update the new ID with the existing ID.
-                await runCmd(
-                    'php',
-                    [
-                        ...cmd,
-                        `db query 'UPDATE ${post_tbl} SET ID=${existingID}  WHERE ID=${results.id}'`,
-                    ]
-                )
-                // update post meta.
-                await runCmd(
-                    'php',
-                    [
-                        ...cmd,
-                        `db query 'UPDATE ${post_meta_tbl} SET post_id=${existingID}  WHERE post_id=${results.id}'`,
-                    ]
-                )
-
-            // if taxonomy is menu.
-            }else if( 'menus' === tax ){
-                // update the term taxonomy table.
-                await runCmd(
-                    'php',
-                    [
-                        ...cmd,
-                        `db query 'UPDATE ${term_taxonomy_tbl} SET term_taxonomy_id=${existingID},term_id=${existingID} WHERE term_id=${results.id}'`,
-                    ]
-                )
-
-            }
-
-            // update the API results ID, back to the existing ID from earlier.
-            results.id = existingID;
-        }
-
-        // add results to collection
-        collection = collection.concat( results );
-
-        // We wait requestDelay seconds between requests.
-        await sleep( requestDelay );
-        
-    }
-
-
-}
-
-function createMediaItem( media ){
-    
-    let fd = new FormData();
-    let featureMediaFile = new File(
-        [
-            media.data
-        ], 
-        media.source_url.split('/').pop(), 
-        {
-            type: media.mime_type
-        }
-    );
-    fd.append('file', featureMediaFile );
-    fd.append('title', media.title);
-    fd.append('caption', media.caption);
-    fd.append('alt_text', media.alt_text);
-    fd.append('date', media.date);
-
-    return fd;
-}
-
-function axiosErrorHandler( error ){
-
-    if (error.response) {
-        // The request was made and the server responded with a status code
-        // that falls out of the range of 2xx
-        let { data, status } = error.response;
-
-        switch( status ){
-            case 401: // Invalid Credentials
-                data.message +=  `\nPlease check your ${terminalLink('Application Password', 'https://make.wordpress.org/core/2020/11/05/application-passwords-integration-guide/')}`;
-            case 404: // Not Found
-                //spinner.fail( `${data.message}` )
-                //console.log('Error', `${data.message}`);
-                //console.log('Error', `${data}`);
-                //process.exit(1)
-            default:
-                console.log('Error', error.response);
-                break;
-        }
-    } else if (error.request) {
-        // The request was made but no response was received
-        // `error.request` is an instance of XMLHttpRequest in the browser and an instance of
-        // http.ClientRequest in node.js
-        console.log(error.request);
-    } else {
-        // Something happened in setting up the request that triggered an Error
-        console.log('Error', error.message);
-    }
-}
-
 /**
  * Sync Environments.
  * 
@@ -344,12 +26,14 @@ function axiosErrorHandler( error ){
  * @param {boolean} options.from   Remote Site URL with current changes.
  * @param {boolean} options.to   Destination Site URL that should be synced.
  * @param {boolean} options.debug   True if debug mode is enabled.
+ * @param {Array} options.tax   Taxonomy that should be synced.
  */
 export default async function sync({
 	spinner,
 	debug,
 	from,
-    to
+    to,
+    tax
 } ) {
 
     const localFile = path.join(appPath, 'caweb.json');
@@ -409,90 +93,185 @@ export default async function sync({
         }
     }
 
+    let pages = [];
+    let posts = [];
+    let attachedMedia = [];
+    let featuredMedia = [];
+    let menus = [];
+    let menuNavItems = [];
+
+    // get all settings 
+    // site settings are always synced
+    spinner.text = `Getting Site Settings from ${from.url}`;
+    let settings = await getTaxonomies({
+        ...fromOptions,
+        fields: [
+            'show_on_front',
+            'page_on_front'
+        ],
+    }, 'settings')
+    
+    /**
+     * If taxonomy is undefined then we sync everything, otherwise we only sync what's being requested.
+     * First we collect everything we need from the base, then we send the information to the target.
+     * If media is requested then we need to download posts/pages to get any attached/featured media.
+     */
+    // collect pages
+    if( undefined === tax || tax.includes('pages') || tax.includes('media')){
+        // get all pages/posts
+        spinner.text = `Gathering all pages from ${from.url}`;
+        pages = await getTaxonomies(fromOptions);
+    }
 
-    // get all pages/posts
-    spinner.text = `Gathering all pages from ${from.url}`;
-    let pages = await getTaxonomies(fromOptions);
+    // collect posts
+    if( undefined === tax || tax.includes('posts')|| tax.includes('media')){
+        // get all pages/posts
+        spinner.text = `Gathering all posts from ${from.url}`;
+        posts = await getTaxonomies(fromOptions, 'posts');
+    }
 
-    // data display
-    //spinner.info();
+    // collect media
+    if( undefined === tax || tax.includes('media')){
+       
+        spinner.text = `Collecting all attached/featured images from ${from.url}`;
+        /**
+         * Media Library Handling
+         * 1) attached media items by default are only possible on pages.
+         * 2) featured media by default are only possible on posts.
+         */
+        const mediaFields = [
+            'id', 
+            'source_url', 
+            'title', 
+            'caption', 
+            'alt_text', 
+            'date',
+            'mime_type'
+        ];
+        attachedMedia = await getTaxonomies({ 
+                ...fromOptions, 
+                fields: mediaFields,
+                parent: pages.map((p) => { return p.id })
+            }, 
+            'media'
+        );
+
+        featuredMedia = await getTaxonomies({ 
+                ...fromOptions, 
+                fields: mediaFields,
+                include: posts.map((p) => { if(p.featured_media){return p.featured_media } })
+            }, 
+            'media'
+        );
+        
+        // before we can upload media files.
+        for( let mediaObj of [].concat( attachedMedia, featuredMedia ) ){
+            // generate a blob from the media object source_url.
+            const mediaBlob = await axios.request( 
+                {
+                    ...fromOptions,
+                    url: mediaObj.source_url,
+                    responseType: 'arraybuffer'
+                } 
+            )
+            .then( (img) => {  return new Blob([img.data]) })
+            .catch( (error) => {
+                errors.push(`${mediaObj.source_url} could not be downloaded.`)
+                return false;
+            });
+            
+            if( mediaBlob ){
+                mediaObj.data = mediaBlob;
+            }
+        }
+    }
 
-    let posts = await getTaxonomies(fromOptions, 'posts');
+    // collect menu and nav items.
+    if( undefined === tax || tax.includes('menus')){
+        spinner.text = `Collecting assigned navigation menus from ${from.url}`;
+        // get all menus and navigation links
+        menus = await getTaxonomies({
+            ...fromOptions,
+            fields: [
+                'id',
+                'description',
+                'name',
+                'slug',
+                'meta',
+                'locations'
+            ]
+            }, 'menus');
+    
+        menuNavItems = await getTaxonomies( 
+            {
+                ...fromOptions,
+                fields: [
+                    'id',
+                    'title',
+                    'url',
+                    'status',
+                    'attr_title',
+                    'description',
+                    'type',
+                    'type_label',
+                    'object',
+                    'object_id',
+                    'parent',
+                    'menu_order',
+                    'target',
+                    'classes',
+                    'xfn',
+                    'meta',
+                    'menus'
+                ],
+                menus: menus.map((menu) => { return menu.id; })
+            }, 
+            'menu-items'
+            )
 
-    // create all pages/posts
-    spinner.text = `Creating all pages to ${to.url}`;
-    await createTaxonomies( pages, toOptions, 'pages', spinner );
-    await createTaxonomies( posts, toOptions, 'posts', spinner );
+    }
 
     /**
-     * Media Library Handling
-     * 1) attached media items by default are only possible on pages.
-     * 2) featured media by default are only possible on posts.
+     * Now we have all the data we can begin to create the taxonomies on the target.
+     * 
+     * Creation order is important otherwise missing dependencies will cause errors to trigger.
+     * We create media first, then pages, then posts, then menus and navigation links.
      */
-    const mediaFields = [
-        'id', 
-        'source_url', 
-        'title', 
-        'caption', 
-        'alt_text', 
-        'date',
-        'mime_type'
-    ];
 
-    let attachedMedia = await getTaxonomies({ 
-            ...fromOptions, 
-            fields: mediaFields,
-            parent: pages.map((p) => { return p.id })
-        }, 
-        'media'
-    );
-
-    let featuredMedia = await getTaxonomies({ 
-            ...fromOptions, 
-            fields: mediaFields,
-            include: posts.map((p) => { if(p.featured_media){return p.featured_media } })
-        }, 
-        'media'
-    );
+    // create media attachments
+    if( undefined === tax || tax.includes('media')){
+        spinner.text = `Uploading media files to ${to.url}`;
+        await createTaxonomies(
+            [].concat( attachedMedia, featuredMedia ).filter((img) => { return img.data }), 
+            toOptions, 
+            'media', 
+            spinner
+            )
+    }
 
-    // before we can upload media files.
-    for( let mediaObj of [].concat( attachedMedia, featuredMedia ) ){
-        // generate a blob from the media object source_url.
-        const mediaBlob = await axios.request( 
-            {
-                ...fromOptions,
-                url: mediaObj.source_url,
-                responseType: 'arraybuffer'
-            } 
-        ).then( (img) => {  return new Blob([img.data]) });
-        
-        mediaObj.data = mediaBlob;
+     // create pages
+    if( undefined === tax || tax.includes('pages')){
+        spinner.text = `Creating all pages to ${to.url}`;
+        await createTaxonomies( pages, toOptions, 'pages', spinner );
     }
 
-    // create media attachments
-    spinner.text = `Uploading media files to ${to.url}`;
-    //await createTaxonomies([].concat( attachedMedia, featuredMedia ), toOptions, 'media', spinner)
+     // create posts
+    if( undefined === tax || tax.includes('posts')){
+        spinner.text = `Creating all posts to ${to.url}`;
+        await createTaxonomies( posts, toOptions, 'posts', spinner );
+    }
 
-    // get all menus
-    let menus = await getTaxonomies({
-        ...fromOptions,
-        fields: [
-            'id',
-            'description',
-            'name',
-            'slug',
-            'meta',
-            'locations'
-        ]
-    }, 'menus').then((navs) =>{
-        // filter out any menus not assigned to a location.
-        return navs.filter((menu) => { 
-            return menu.locations.length 
-        });
-    })
+    // create menus and navigation links
+    if( undefined === tax || tax.includes('menus')){
+        spinner.text = `Reconstructing navigation menus to ${to.url}`;
+        await createTaxonomies(menus, toOptions, 'menus', spinner);
+        await createTaxonomies(menuNavItems, toOptions, 'menu-items', spinner);
+    }
+    
 
-    // create menus
-    await createTaxonomies(menus, toOptions, 'menus', spinner);
+    // update settings
+    await createTaxonomies(settings, toOptions, 'settings', spinner);
 
+    spinner.text = `Sync from ${from.url} to ${to.url} completed successfully.`
 
 };

package/commands/test.js

@@ -2,24 +2,22 @@
  * External dependencies
  */
 import path from 'path';
-import axios from 'axios';
 import fs from 'fs';
 import loadConfig from '@wordpress/env/lib/config/load-config.js';
-import cp from 'child_process';
+import os from 'os';
 
 //var WP = require('wp-cli');
 /**
  * Internal dependencies
  */
 import { runCLICmds, runCmd, projectPath } from '../lib/index.js';
+import { CAWEB_OPTIONS, DIVI_OPTIONS } from '../lib/wordpress/index.js';
 /*
 import generateOverridesMD from '../admin.js';
-import { CAWEB_OPTIONS, DIVI_OPTIONS } from '../options.js';
 */
 /**
  * Promisified dependencies
  */
-import os from 'os';
 
 /**
  * Test code.
@@ -36,65 +34,21 @@ export default async function test({
 } ) {
 	
 	spinner.text = "Testing Code Functionality";
-		const config = await loadConfig(path.resolve('.'));
-		
-		process.env.WP_CLI_CONFIG_PATH  = path.join(config.workDirectoryPath, 'config.yml');
-	 // update post meta.
-	 let id = 35;
-	 let newId = 30
-	 let tbl = `wp_term_taxonomy`;
+	const config = await loadConfig(path.resolve('.'));
+	const { workDirectoryPath } = config;	
+	process.env.WP_CLI_CONFIG_PATH  = path.join(config.workDirectoryPath, 'config.yml');
 
-		let result = await runCmd(
+	let cawebOptNames = CAWEB_OPTIONS;
+
+	let result = await runCmd(
                     'php',
                     [
-                        path.join(projectPath, 'bin', 'wp-cli.phar'),
-						'--ssh=docker:' + path.basename(config.workDirectoryPath) + '-cli-1',
-                        `db query 'UPDATE ${tbl} SET term_taxonomy_id=${newId},term_id=${newId} WHERE term_id=${id}'`,
+						//'--info'
                     ]
                 )
 
-				console.log( result );
-		/*
-		let q = `UPDATE wp_posts SET id=1 WHERE id=10 `;
-		let u = 'http://danny.com/wp-json/wp/v2/pages/2827';
-		let result = await axios.request({
-			url: u,
-			method: 'POST',
-			data: {
-				title: 'Audio',
-				meta: {
-					_et_pb_use_builder: 'on'
-				}
-			},
-			headers: {
-				Authorization: 'Basic ' + Buffer.from(`admin:9RUG nPuo u581 cY2A uSJd FE2P`).toString('base64')
-			}
-		})
-		.then( async (res) => { return res.data; } )
-		.catch( async (err) => { return err; } )
-
-		*/
-		/*
-		let result = await runCmd(
-			'php', 
-			[
-				//'-r',
-				path.join(projectPath, 'bin', 'wp-cli.phar'),
-				'--ssh=' + `docker:${path.basename(config.workDirectoryPath)}-cli-1`,
-				//'@dev',
-				//`db query '${q}'`,
-				'post list',
-				//'--info',
-				'--skip-themes',
-				//`--url=${new URL(u).origin}`
-				// '--format=json'
-				//'--skip-column-names',
-				//'post list path=/nas/content/live/cawebdev/ --http=https://dev.sites.ca.gov --url=https://dev.sites.ca.gov',
-			], 
-			config
-		)*/
+			
+	console.log( 'out', result.stdout.toString() );
+	console.log( 'err', result.stderr.toString() );
 		
-		//console.log( 'Result', result.toString() );
-		//console.log( 'Output', result.stdout.toString() );
-		//console.log( 'Error', result.stderr.toString() );
 };

package/configs/webpack.config.js

@@ -85,7 +85,7 @@ let webpackConfig = {
   devtool: false,
   output: {
     ...baseConfig.output,
-    publicPath: `/`,
+    publicPath: `./`,
     clean: false
   },
   plugins: [

package/lib/cli.js

@@ -259,6 +259,10 @@ export default function cli() {
 		.description('Sync changes from one destination to another.')
 		.argument('<from>', 'Target Site URL with current changes.')
 		.argument('<to>', 'Destination Site URL that should be synced.')
+		.addOption(new Option(
+			'-t,--tax [tax...]',
+			'Taxonomy that should be synced. Omitting this option will sync the full site.'
+		).choices(['pages', 'posts', 'media', 'menus']))
 		.allowUnknownOption(true)
 		.action( withSpinner(env.sync) )
 		
@@ -269,7 +273,6 @@ export default function cli() {
 		//.addArgument(envArg)
 		.allowUnknownOption(true)
 		.action(withSpinner(env.test))
-		
 	
 	addWPEnvCommands();
 

package/lib/helpers.js

@@ -30,7 +30,7 @@ const appPath = path.resolve( projectPath, '../../../' );
  * 
  * @returns {Promise}
  */
-async function runCmd(cmd, args,opts = { stdio: 'inherit' }){
+async function runCmd(cmd, args,opts = { stdio: ['inherit', 'pipe'] }){
 	// fix various commands.
 	switch (cmd) {
 		case 'npm':

package/lib/index.js

@@ -14,6 +14,20 @@ import {
     appPath
 } from './helpers.js';
 
+import {
+    activateCAWeb,
+    configureCAWeb,
+    downloadSources,
+	configureDivi,
+	isDiviThemeActive,
+	configureWordPress,
+	isMultisite,
+	convertToMultisite,
+	generateHTAccess,
+	getTaxonomies,
+    createTaxonomies
+} from './wordpress/index.js';
+
 export {
     currentPath,
     projectPath,
@@ -24,5 +38,16 @@ export {
     wpYellow,
     withSpinner,
 	runCmd,
-    runCLICmds
+    runCLICmds,
+    activateCAWeb,
+    configureCAWeb,
+    downloadSources,
+	configureDivi,
+	isDiviThemeActive,
+	configureWordPress,
+	isMultisite,
+	convertToMultisite,
+	generateHTAccess,
+	getTaxonomies,
+    createTaxonomies
 }

package/lib/spinner.js

@@ -1,5 +1,9 @@
+/**
+ * External dependencies
+ */
 import chalk from 'chalk';
 import ora from 'ora';
+import terminalLink from  'terminal-link';
 
 // Colors.
 const boldWhite = chalk.bold.white;
@@ -48,7 +52,29 @@ const withSpinner =
 				process.exit( 0 );
 			},
 			( error ) => {
-				if( error ){
+
+				// Axios Error
+				if( error &&
+					'object' === typeof error &&
+					'response' in error &&
+					'request' in error 
+				){
+
+					let { data, status } = error.response;
+					let msg = data.message || 'An unknown error occurred.';
+
+					if( 401 === status ){
+						msg +=  `\nPlease check your ${terminalLink('Application Password', 'https://make.wordpress.org/core/2020/11/05/application-passwords-integration-guide/')}`;
+					} else if( 403 === status ){
+						msg =  'Forbidden Request: A potentially unsafe operation has been detected in your request to this site';
+					}
+
+					spinner.fail( msg )
+					process.exit( 1 );
+					
+
+
+				}else if( error ){
 					// Error is an unknown error. That means there was a bug in our code.
 					spinner.fail(
 						typeof error === 'string' ? error : error.message
@@ -56,10 +82,13 @@ const withSpinner =
 					// Disable reason: Using console.error() means we get a stack trace.
 					console.error( error );
 					process.exit( 1 );
+
 				}else{
 					spinner.fail( 'An unknown error occurred.' );
 					process.exit( 1 );
+
 				}
+
 			}
 		);
 	};

package/lib/wordpress/api.js

@@ -0,0 +1,303 @@
+/**
+ * External dependencies
+ */
+import path from 'path';
+import axios from 'axios';
+import terminalLink from  'terminal-link';
+
+
+/**
+ * Internal dependencies
+ */
+import {
+    projectPath,
+    runCmd
+} from '../index.js';
+
+const endpoint = '/wp-json/wp/v2';
+const requestDelay = 1000; // we wait 1 second between requests
+
+
+const sleep = (waitTimeInMs) => new Promise(resolve => setTimeout(resolve, waitTimeInMs));
+
+function processData( data ){
+    /**
+     * If wpautop is enabled and data contains Divi shortcodes, 
+     * WordPress will automatically add an opening/closing p tag that needs to be removed.
+     * @link https://developer.wordpress.org/reference/functions/wpautop/
+    */
+    if( data.includes('et_pb_section') && 
+        data.startsWith('<p>') &&
+        data.endsWith('</p>\n')
+    ){
+        data = data.substring(3, data.length - 5)
+    }
+    return data
+        .replace(/(&#8221;)|(&#8243;)|(&#8220;)/g, '') // replace double encoding
+        .replace(/(\s{2,})/g, ' ') // replace misc spacing // &#8220;
+
+}
+
+function processUrlParam( param ){
+    return ( Array === typeof param ? param.join(',') : param )
+}
+
+/**
+ * Make Requests to WordPress Rest API and return different taxonomy results.
+ * 
+ * 
+ * @async
+ * @param {Object} request Data to pass to Axios request.
+ * @param {string} [tax='pages'] Taxonomy of the request made. Default pages, available choices: pages,posts,menus,menu-items,settings,media
+ * @returns {unknown}
+ */
+async function getTaxonomies( request, tax = 'pages' ){
+    const fields = request.fields ? request.fields : [
+        'id',
+        'type',
+        'alt_text',
+        'caption', 
+        'slug',
+        'title',
+        'source_url',
+        'mime_type',
+        'content',
+        'date',
+        'date_gmt',
+        'status',
+        'featured_media',
+        'comment_status',
+        'ping_status',
+        'template',
+        'format',
+        'categories',
+        'tags',
+        'meta',
+    ];
+
+    let urlParams = [
+        'order=asc'
+    ]
+
+
+    // if fields are passed
+    if( fields.length && 'all' !== fields ){
+        urlParams.push( `_fields=${ fields.join(',')}` );
+    }
+    
+
+    // if no request is added default to GET
+    if( ! request.method ){
+        request.method = 'GET'
+    }
+
+    // if parent argument 
+    if( request.parent ){
+        urlParams.push( `parent=` + processUrlParam( request.parent ))
+    }
+
+    // if include argument 
+    if( request.include ){
+        urlParams.push( `include=` + processUrlParam( request.include ))
+    }
+
+    // if menus argument 
+    if( request.menus ){
+        urlParams.push( 'menus=' + processUrlParam( request.menus ) );
+    }
+
+    // if per page argument.
+    urlParams.push( 'per_page=' + request.per_page ? request.per_page : '100' )
+
+
+    // if posts_per_page argument.
+    urlParams.push( 'posts_per_page=' + request.posts_per_page ? request.posts_per_page : '10' )
+
+
+    let collection = [];
+    let results = false;
+    
+    /**
+     * Results are capped to 100 max per page
+     * in order to return all results, we make the same requests and pass the page parameter.
+     * @see https://developer.wordpress.org/rest-api/using-the-rest-api/pagination/
+     * @type {number}
+    */
+    let page = 1;
+
+    do{
+        
+        results = await axios.request(
+            {
+                ...request,
+                url: `${request.url}${endpoint}/${tax}?` + urlParams.join('&') + `&page=${page}`
+            }
+        ).then(
+            (res) => { 
+                // if no data or empty data return false.
+                if( ! res.data || ( 'object' === typeof res.data && ! Object.entries(res.data).length ) ){
+                    return false;
+                }
+
+                // add results to collection
+                collection = collection.concat( res.data );
+                return true
+        }).catch( 
+            (error) => {
+                return false
+            }
+        );
+
+        // settings taxonomies will always return results regardless of the page parameter.
+        // so we return false after the first request
+        if( 'settings' === tax ){
+            break;
+        }
+
+        page++;
+    }while( results )
+   
+    
+    return collection;
+}
+
+async function createTaxonomies( taxData, request, tax = 'pages', spinner ){
+    // if no request is added default to POST
+    if( ! request.method ){
+        request.method = 'POST'
+    }
+
+    let collection = [];
+
+    for( let obj of taxData ){
+        // endpoint url.
+        let url = `${request.url}${endpoint}/${tax}`;
+        let existingID = false;
+
+        // The REST API doesn't allow passing the id, in order to maintain ID's we make an additional request to our plugin endpoint.
+        if( obj.id ){
+            // first we check if the ID exist
+            let idExists = await axios.request(
+                {   
+                    ...request, 
+                    method: 'GET',
+                    url: `${url}/${ obj.id }`
+                } 
+                )
+                .then((res) => { return res.data.id })
+                .catch(error => {return false;})
+
+            // for menus if ID doesn't exist we also check for matching slug
+            // if a menu with a matching slug exists we return that id instead.
+            if( 'menus' === tax && ! idExists ){
+                idExists = await axios.request(
+                    {   
+                        ...request, 
+                        method: 'GET',
+                        url: `${url}/?slug=${ obj.slug }`
+                    } 
+                    )
+                    .then((res) => { return res.data[0].id })
+                    .catch(error => {return false;})
+
+            }
+
+            /**
+             * if the ID doesn't exist we save it so we can make a request to our plugin endpoint.
+             * if it does exist update endpoint url so we update the existing item.
+             */
+            if( ! idExists ){
+                existingID = obj.id;
+            }else{
+                url = `${url}/${ obj.id }`;
+            }
+
+            // id has to be deleted.        
+            delete obj.id;
+        }
+
+        // process properties
+        if( obj.content && obj.content.rendered ){
+            obj.content = processData(obj.content.rendered);
+        }
+        
+        if( obj.title && obj.title.rendered ){
+            obj.title = processData(obj.title.rendered);
+        }
+
+        if( obj.caption && obj.caption.rendered ){
+            obj.caption = processData(obj.caption.rendered);
+        }
+
+        // make WordPress REST API request.
+        let results = await axios.request({
+                ...request,
+                timeout: 7500,
+                data: 'media' === tax ? createMediaItem(obj) : obj,
+                url
+            })
+            .then( async (res) => { return res.data; } )
+
+            
+        /**
+         * if the obj had an existing ID that didn't exist we make a request to our plugin endpoint to update the IDs.
+         */
+        if( existingID ){
+
+            await axios.request({
+                ...request,
+                url: `${request.url}/wp-json/caweb-dev/v1/sync`,
+                timeout: 7500,
+                data: {
+                    id: results.id,
+                    newId: existingID,
+                    tax,
+                    locations: 'menus' === tax ? obj.locations : []
+                }
+            })
+            .then( async (res) => { return res.data; } )
+            
+            // update the API results ID, back to the existing ID from earlier.
+            results.id = existingID;
+        }
+
+        // add results to collection
+        collection = collection.concat( results );
+
+        let p = tax.charAt(0).toUpperCase() + tax.substring(1, tax.length - ( tax.endsWith('s') ? 1 : 0 ) );
+
+        if( undefined !== results && results.id ){
+            spinner.info(`${p}: ${results.id} was created.`);
+        }
+        
+    }
+
+
+}
+
+function createMediaItem( media ){
+    
+    let fd = new FormData();
+    let featureMediaFile = new File(
+        [
+            media.data
+        ], 
+        media.source_url.split('/').pop(), 
+        {
+            type: media.mime_type
+        }
+    );
+    fd.append('file', featureMediaFile );
+    fd.append('title', media.title);
+    fd.append('caption', media.caption);
+    fd.append('alt_text', media.alt_text);
+    fd.append('date', media.date);
+
+    return fd;
+}
+
+
+export {
+    getTaxonomies,
+    createTaxonomies
+}

package/lib/wordpress/index.js

@@ -2,6 +2,8 @@ import { activateCAWeb, configureCAWeb } from "./caweb.js";
 import { downloadSources } from "./download-sources.js";
 import { configureDivi, isDiviThemeActive } from "./divi.js";
 import { configureWordPress, isMultisite, convertToMultisite, generateHTAccess } from "./wordpress.js";
+import { getTaxonomies, createTaxonomies }  from "./api.js";
+import { CAWEB_OPTIONS, DIVI_OPTIONS } from './options.js';
 
 export {
     activateCAWeb,
@@ -12,5 +14,9 @@ export {
 	configureWordPress,
 	isMultisite,
 	convertToMultisite,
-	generateHTAccess
+	generateHTAccess,
+	getTaxonomies,
+    createTaxonomies,
+	CAWEB_OPTIONS,
+	DIVI_OPTIONS
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@caweb/cli",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "description": "CAWebPublishing Command Line Interface.",
   "exports": "./lib/env.js",
   "type": "module",
@@ -57,8 +57,8 @@
     }
   },
   "dependencies": {
-    "@wordpress/env": "^9.2.0",
-    "@wordpress/scripts": "^27.1.0",
+    "@wordpress/env": "^9.4.0",
+    "@wordpress/scripts": "^27.3.0",
     "accessibility-checker": "^3.1.67",
     "autoprefixer": "^10.4.17",
     "axios": "^1.6.7",
@@ -66,17 +66,17 @@
     "commander": "^12.0.0",
     "cross-spawn": "^7.0.3",
     "css-loader": "^6.10.0",
-    "docker-compose": "^0.24.3",
+    "docker-compose": "^0.24.6",
     "handlebars-loader": "^1.7.3",
-    "html-to-json-parser": "^2.0.0",
+    "html-to-json-parser": "^2.0.1",
     "html-webpack-plugin": "^5.6.0",
     "mini-css-extract-plugin": "^2.8.0",
     "ora": "^8.0.1",
     "postcss-loader": "^8.1.0",
     "resolve-bin": "^1.0.1",
-    "sass-loader": "^14.1.0",
+    "sass-loader": "^14.1.1",
     "terminal-link": "^3.0.0",
     "url": "^0.11.3",
-    "webpack": "^5.90.1"
+    "webpack": "^5.90.3"
   }
 }