@caweb/cli 1.3.3 → 1.3.4

package/README.md

@@ -18,7 +18,7 @@
   - phpMyAdmin test site started at http://localhost:9090
 - Uses CAWebPublishing [External Project Template Configuration](https://developer.wordpress.org/block-editor/reference-guides/packages/packages-create-block/packages-create-block-external-template/) when creating Gutenberg Blocks, see configurations [here](https://github.com/CAWebPublishing/cli/lib/template)
 - Allows for syncing of WordPress instance via Rest API, to maintain ID's please ensure [CAWebPublishing Development Toolbox](https://github.com/CAWebPublishing/caweb-dev/) plugin is installed. 
-- Adds config.yml to both cli containers, alias are added for each ssh connection.  
+- Adds config.yml to both cli containers.  
 <b>Example config.yml file</b>
 <pre>
     path: /var/www/html

package/commands/sync.js

@@ -14,26 +14,65 @@ import {
     getTaxonomies,
     createTaxonomies
 } from '../lib/index.js';
+import { get } from 'http';
 
 const errors = [];
 
+/**
+ * Return all parent items for a given set of ids.
+ *
+ * @async
+ * @param {Array} ids
+ * @param {*} request
+ * @param {string} [tax='pages']
+ * @returns {unknown}
+ */
+async function getParentItems( objects, request, tax = 'pages' ){
+    let parentItemsObjects = [];
+    let objectParentIds = objects.map((obj) => { return obj.parent; });
+
+    while( objectParentIds.length > 0 ){
+
+        // if we have parent ids, we have to collect any parent items.
+        let parentItems = await getTaxonomies({
+            ...request, 
+            orderby: 'parent',
+            embed: true,
+            include: objectParentIds
+        }, tax);
+
+        // if we have parent items, we have to add to the items array.
+        if( parentItems ){
+            parentItemsObjects = parentItemsObjects.concat( parentItems );
+        }
+
+        // if the parent items have parent ids, we have to save those for the next run.
+        objectParentIds = parentItems.map((obj) => { return obj.parent; }).filter((id) => { return id !== 0; })   
+    }
+
+    return objects.concat( parentItemsObjects ).sort( (a,b) => a.parent - b.parent );
+}
 
 /**
  * Sync Environments.
  * 
+ * @see https://developer.wordpress.org/rest-api/reference/
+ * 
  * @param {Object}  options
  * @param {Object}  options.spinner A CLI spinner which indicates progress.
  * @param {boolean} options.from   Remote Site URL with current changes.
  * @param {boolean} options.to   Destination Site URL that should be synced.
  * @param {boolean} options.debug   True if debug mode is enabled.
  * @param {Array} options.tax   Taxonomy that should be synced.
+ * @param {Array} options.include   Include specific IDs only.
  */
 export default async function sync({
 	spinner,
 	debug,
 	from,
     to,
-    tax
+    tax,
+    include
 } ) {
 
     const localFile = path.join(appPath, 'caweb.json');
@@ -91,104 +130,165 @@ export default async function sync({
         }
     }
 
+    /**
+     * Sync Process 
+     * If taxonomy is undefined then we don't sync that section.
+     * 
+     * 1) Site Settings are always synced.
+     * 2) We always collect all the media.
+     * 3) We only collect pages if the taxonomy is undefined or it's set to pages.
+     * 4) We only collect posts if the taxonomy is undefined or it's set to posts.
+     * 5) We only collect menus if the taxonomy is undefined or it's set to menus.
+     *    - We also collect menu items if menus are collected.
+     */
+    let settings = [];
+    let media = [];
     let pages = [];
     let posts = [];
-    let attachedMedia = [];
-    let featuredMedia = [];
     let menus = [];
     let menuNavItems = [];
 
-    spinner.text = `Getting Site Settings from ${from.url}`;
-    /**
-     * Site settings are always synced
-     * 
-     * @see https://developer.wordpress.org/rest-api/reference/settings/
-     */
-    let settings = await getTaxonomies({
-        ...fromOptions,
-        fields: [
-            'show_on_front',
-            'page_on_front',
-            'posts_per_page'
-        ],
-    }, 'settings')
-    
-    /**
-     * If taxonomy is undefined then we sync everything, otherwise we only sync what's being requested.
-     * First we collect everything we need from the base, then we send the information to the target.
-     * If media is requested then we need to download posts/pages to get any attached/featured media.
-     */
-    // collect pages
-    if( undefined === tax || tax.includes('pages') || tax.includes('media')){
-        // get all pages/posts
-        spinner.text = `Gathering all pages from ${from.url}`;
-        pages = await getTaxonomies(fromOptions);
-    }
+    // Media Library.
+    spinner.text = `Collecting Media Library ${from.url}`;
+    let mediaLibrary = await getTaxonomies({ 
+            ...fromOptions, 
+            fields: [
+                'id', 
+                'source_url', 
+                'title', 
+                'caption', 
+                'alt_text', 
+                'date',
+                'mime_type',
+                'post',
+                'media_details'
+            ],
+            include: tax.includes('media') && include ? include.join(',') : null
+        }, 
+        'media'
+    );
 
-    // collect posts
-    if( undefined === tax || tax.includes('posts')|| tax.includes('media')){
-        // get all pages/posts
-        spinner.text = `Gathering all posts from ${from.url}`;
-        posts = await getTaxonomies(fromOptions, 'posts');
+    // Site Settings.
+    if( undefined === tax || tax.includes('settings') ){
+        spinner.text = `Collecting Site Settings from ${from.url}`;
+        settings = await getTaxonomies({
+            ...fromOptions,
+            fields: [
+                'show_on_front',
+                'page_on_front',
+                'posts_per_page'
+            ],
+        }, 'settings')
+        
     }
 
-    // collect media
-    if( undefined === tax || tax.includes('media')){
-       
-        spinner.text = `Collecting all attached/featured images from ${from.url}`;
-        /**
-         * Media Library Handling
-         * 1) attached media items by default are only possible on pages.
-         * 2) featured media by default are only possible on posts.
-         */
-        const mediaFields = [
-            'id', 
-            'source_url', 
-            'title', 
-            'caption', 
-            'alt_text', 
-            'date',
-            'mime_type'
-        ];
-        attachedMedia = await getTaxonomies({ 
+    // Pages.
+    if( undefined === tax || tax.includes('pages') ){
+        // get all pages/posts
+        spinner.text = `Collecting pages from ${from.url}`;
+        pages = await getTaxonomies({
                 ...fromOptions, 
-                fields: mediaFields,
-                parent: pages.map((p) => { return p.id })
+                orderby: 'parent',
+                embed: true,
+                include: tax && include ? include.join(',') : null
             }, 
-            'media'
+            'pages'
         );
 
-        featuredMedia = await getTaxonomies({ 
+        // we only do this if specific ids were requested.
+        if( include ){
+            // if we have parent ids, we have to collect any parent items.
+            pages = await getParentItems( 
+                pages, 
+                fromOptions, 
+                'pages'
+            ) 
+        }
+    }
+
+    // Posts.
+    if( undefined === tax || tax.includes('posts') ){
+        // get all pages/posts
+        spinner.text = `Collecting all posts from ${from.url}`;
+        posts = await getTaxonomies({
                 ...fromOptions, 
-                fields: mediaFields,
-                include: posts.map((p) => { if(p.featured_media){return p.featured_media } })
+                orderby: 'parent',
+                include: tax && include ? include.join(',') : null
             }, 
-            'media'
+            'posts'
         );
-        
-        // before we can upload media files.
-        for( let mediaObj of [].concat( attachedMedia, featuredMedia ) ){
-            // generate a blob from the media object source_url.
-            const mediaBlob = await axios.request( 
-                {
-                    ...fromOptions,
-                    url: mediaObj.source_url,
-                    responseType: 'arraybuffer'
-                } 
-            )
-            .then( (img) => {  return new Blob([img.data]) })
-            .catch( (error) => {
-                errors.push(`${mediaObj.source_url} could not be downloaded.`)
-                return false;
-            });
-            
-            if( mediaBlob ){
-                mediaObj.data = mediaBlob;
+
+        // we only do this if specific ids were requested.
+        if( include ){
+            // if we have parent ids, we have to collect any parent items.
+            posts = await getParentItems( 
+                posts, 
+                fromOptions, 
+                'posts'
+            ) 
+        }
+    }
+
+    /**
+     * Media Library Handling
+     * 
+     * We iterate thru the media library to identify which media should be synced.
+     * 1) attached media items by default are only possible on pages.
+     * 2) featured media by default are only possible on posts.
+     * 3) save any linked media in the content of pages and posts.
+     */
+    for( let m of mediaLibrary ){
+        // check if the media is attached to a page.
+        // if tax is undefined, we collect all media attached to posts and pages.
+        // if tax is defined, we only collect media attached to posts and pages if the id match.
+        if( ( ! tax && m.post ) ||
+             ( tax && include.includes(m.id.toString()) ) ){
+            media.push( m );
+
+            // we don't have to check any further.
+            continue;
+        }
+
+        for( let p of [].concat( pages, posts ) ){
+            // if the media is featured on a post or linked in the content.
+            if( p.featured_media === m.id ||
+                p.content.rendered.match( new RegExp(`src=”(${m.source_url}.*)”`, 'g') )){
+                media.push( m );
             }
         }
     }
+    
+    // filter any duplicate media.
+    media = media.filter((m, index, self) => { return index === self.findIndex((t) => { return t.id === m.id; })} );
+    
+    // before we can upload media files we have to generate the media blob data.
+    for( let m of media ){
+        const mediaBlob = await axios.request( 
+            {
+                ...fromOptions,
+                url: m.source_url,
+                responseType: 'arraybuffer'
+            } 
+        )
+        .then( (img) => {  return new Blob([img.data]) })
+        .catch( (error) => {
+            errors.push(`${m.source_url} could not be downloaded.`)
+            return false;
+        });
+        
+        if( mediaBlob ){
+            m.data = mediaBlob;
+        }
+    }
+    
+    // this has to be done after we have the media data.
+    // Lets replace the url references in the content of the pages and posts.
+    for( let p of [].concat( pages, posts ) ){
+        p.content.rendered = p.content.rendered.replace( new RegExp(from.url, 'g'), to.url );
+    }
 
-    // collect menu and nav items.
+    
+    // Menu and Nav Items.
     if( undefined === tax || tax.includes('menus')){
         spinner.text = `Collecting assigned navigation menus from ${from.url}`;
         // get all menus and navigation links
@@ -201,7 +301,8 @@ export default async function sync({
                 'slug',
                 'meta',
                 'locations'
-            ]
+            ],
+            include: tax && include ? include.join(',') : null
             }, 'menus');
     
         menuNavItems = await getTaxonomies( 
@@ -226,6 +327,7 @@ export default async function sync({
                     'meta',
                     'menus'
                 ],
+                include: tax && include ? include.join(',') : null,
                 menus: menus.map((menu) => { return menu.id; })
             }, 
             'menu-items'
@@ -236,43 +338,50 @@ export default async function sync({
     /**
      * Now we have all the data we can begin to create the taxonomies on the target.
      * 
-     * Creation order is important otherwise missing dependencies will cause errors to trigger.
-     * We create media first, then pages, then posts, then menus and navigation links.
+     * Import Order is important otherwise missing dependencies will cause errors to trigger
+     * 1) Media
+     * 2) Pages
+     * 3) Posts
+     * 4) Menus & Navigation Links
+     * 5) Settings
      */
 
-    // create media attachments
-    if( undefined === tax || tax.includes('media')){
+    // Media.
+    if( media ){
         spinner.text = `Uploading media files to ${to.url}`;
         await createTaxonomies(
-            [].concat( attachedMedia, featuredMedia ).filter((img) => { return img.data }), 
+            media.filter((img) => { return img.data }), 
             toOptions, 
             'media', 
             spinner
             )
     }
 
-     // create pages
-    if( undefined === tax || tax.includes('pages')){
+     // Pages.
+    if( pages ){
         spinner.text = `Creating all pages to ${to.url}`;
         await createTaxonomies( pages, toOptions, 'pages', spinner );
     }
 
-     // create posts
-    if( undefined === tax || tax.includes('posts')){
+     // Posts.
+    if( posts ){
         spinner.text = `Creating all posts to ${to.url}`;
         await createTaxonomies( posts, toOptions, 'posts', spinner );
     }
 
-    // create menus and navigation links
-    if( undefined === tax || tax.includes('menus')){
+    // Menus and Navigation Links.
+    if( menus ){
         spinner.text = `Reconstructing navigation menus to ${to.url}`;
         await createTaxonomies(menus, toOptions, 'menus', spinner);
         await createTaxonomies(menuNavItems, toOptions, 'menu-items', spinner);
     }
     
 
-    // update settings
-    await createTaxonomies(settings, toOptions, 'settings', spinner);
+    // Settings.
+    if( settings ){
+        spinner.text = `Updating site settings to ${to.url}`;
+        await createTaxonomies(settings, toOptions, 'settings', spinner);
+    }
 
     spinner.text = `Sync from ${from.url} to ${to.url} completed successfully.`
 

package/configs/docker-compose.js

@@ -24,21 +24,6 @@ async function generateCLIConfig(workDirectoryPath){
         apache_modules: ['mod_rewrite']
     };
     
-    let { homedir } = os.userInfo();
-	
-    // add any ssh hosts to the config
-    if( fs.existsSync( path.join( homedir, '.ssh', 'config' )) ){
-			
-		let ssh_hosts = fs.readFileSync(path.join( homedir, '.ssh', 'config' )).toString().match(/Host\s+(.*)[^\n\r]/g);
-
-		ssh_hosts.forEach((host) => {
-			let s = host.replace(/Host\s+/, '');
-
-			yml[`@${s}`] = { ssh: s};
-		})
-	}
-
-
     fs.writeFileSync(
         path.join(workDirectoryPath, 'config.yml'),
         yaml.dump(yml));

package/lib/cli.js

@@ -262,7 +262,11 @@ export default function cli() {
 		.addOption(new Option(
 			'-t,--tax [tax...]',
 			'Taxonomy that should be synced. Omitting this option will sync the full site.'
-		).choices(['pages', 'posts', 'media', 'menus']))
+		).choices(['pages', 'posts', 'media', 'menus', 'settings']))
+		.addOption(new Option(
+			'-i,--include [include...]',
+			'IDs to of taxonomies to include. Omitting this option will sync all items for given taxonomy.'
+		))
 		.allowUnknownOption(true)
 		.action( withSpinner(env.sync) )
 		

package/lib/wordpress/api.js

@@ -81,19 +81,19 @@ async function getTaxonomies( request, tax = 'pages' ){
         'categories',
         'tags',
         'meta',
+        'parent',
+        '_links'
     ];
 
     let urlParams = [
-        'order=asc'
+        'order=' + ( request.order ? processUrlParam(request.order) : 'asc' )
     ]
 
-
     // if fields are passed
     if( fields.length && 'all' !== fields ){
         urlParams.push( `_fields=${ fields.join(',')}` );
     }
     
-
     // if no request is added default to GET
     if( ! request.method ){
         request.method = 'GET'
@@ -114,6 +114,16 @@ async function getTaxonomies( request, tax = 'pages' ){
         urlParams.push( 'menus=' + processUrlParam( request.menus ) );
     }
 
+    // if taxonomies is not menus, we add the orderby parameter.
+    if( 'menus' !== tax  ){
+        urlParams.push( `orderby=` + (request.orderby ? processUrlParam(request.orderby) : 'id') );
+    }
+
+    // if embed is set to true, we add the _embed parameter.
+    if( request.embed ){
+        urlParams.push( '_embed' );
+    }
+
     /**
      * The per_page parameter is capped at 100.
      * 
@@ -122,7 +132,6 @@ async function getTaxonomies( request, tax = 'pages' ){
     request.per_page = request.per_page && request.per_page <= 100 ? request.per_page : 100;
     urlParams.push( 'per_page=' + request.per_page  );
 
-
     let collection = [];
     let results = false;
     
@@ -181,8 +190,26 @@ async function createTaxonomies( taxData, request, tax = 'pages', spinner ){
     for( let obj of taxData ){
         // endpoint url.
         let url = `${request.url}${endpoint}/${tax}`;
-        let existingID = false;
+        let existingID = obj.id ? obj.id : false;
+
+        // process object properties.
+        for( let prop in obj ){
+            // we process the rendered property and delete the rendered property.
+            if( 'object' === typeof obj[prop] && null !== obj[prop] && obj[prop].hasOwnProperty('rendered') ){
+                    obj[prop] = processData(obj[prop].rendered);
+            }
 
+            // if obj has a parent and it's 0, we delete it.
+            if( 'parent' === prop && 0 === obj[prop] ){
+                delete obj[prop];
+            }
+
+            // disallowed props.
+            if( ['_links', '_embedded'].includes(prop) ){
+                delete obj[prop];
+            }
+        }
+       
         // The REST API doesn't allow passing the id, in order to maintain ID's we make an additional request to our plugin endpoint.
         if( obj.id ){
             // first we check if the ID exist
@@ -196,9 +223,11 @@ async function createTaxonomies( taxData, request, tax = 'pages', spinner ){
                 .then((res) => { return res.data.id })
                 .catch(error => {return false;})
 
-            // for menus if ID doesn't exist we also check for matching slug
-            // if a menu with a matching slug exists we return that id instead.
-            if( 'menus' === tax && ! idExists ){
+            /**
+             * Menus Only
+             * Since we can't have duplicated slugs, if the ID doesn't exist we also check for matching slugs.
+             */
+            if( 'menus' === tax && false === idExists ){
                 idExists = await axios.request(
                     {   
                         ...request, 
@@ -210,33 +239,19 @@ async function createTaxonomies( taxData, request, tax = 'pages', spinner ){
                     .catch(error => {return false;})
 
             }
-            /**
-             * if the ID doesn't exist we save it so we can make a request to our plugin endpoint.
-             * if it does exist update endpoint url so we update the existing item.
-             */
-            if( ! idExists ){
-                existingID = obj.id;
-            }else{
-                url = `${url}/${ obj.id }`;
+
+            // if the ID exists we append it to the url.
+            if( idExists ){
+                url = `${url}/${ idExists }`;
+                
+                // we store the existing ID for later use.
+                existingID = idExists;
             }
 
             // id has to be deleted.        
             delete obj.id;
         }
 
-        // process properties
-        if( obj.content && obj.content.rendered ){
-            obj.content = processData(obj.content.rendered);
-        }
-        
-        if( obj.title && obj.title.rendered ){
-            obj.title = processData(obj.title.rendered);
-        }
-
-        if( obj.caption && obj.caption.rendered ){
-            obj.caption = processData(obj.caption.rendered);
-        }
-
         // make WordPress REST API request.
         let results = await axios.request({
                 ...request,
@@ -245,25 +260,45 @@ async function createTaxonomies( taxData, request, tax = 'pages', spinner ){
             })
             .then( async (res) => { return res.data; } )
 
+            
         /**
-         * if the obj had an existing ID that didn't exist we make a request to our plugin endpoint to update the IDs.
+         * if the obj had an existing ID we make a request to our plugin endpoint to update the IDs.
          */
         if( existingID ){
+            let extraArgs = {};
+
+            // if media 
+            if( 'media' === tax ){
+                // get original source domain
+                let sourceDomain = obj.source_url.substring(0, obj.source_url.indexOf('/wp-content/uploads'));
+                // get expected guid, replace source domain with the new domain.
+                let expectedGuid = obj.source_url.replace(sourceDomain, request.url);
+
+                extraArgs = {
+                    guid: results.guid.rendered,
+                    newGuid: expectedGuid,
+                    // if media and source_url is different from the guid, we update the guid.
+                    media_details: results.media_details,
+                }
+            }
 
             await axios.request({
                 ...request,
                 url: `${request.url}${syncEndpoint}`,
                 data: {
+                    ...extraArgs,
                     id: results.id,
                     newId: existingID,
                     tax,
-                    locations: 'menus' === tax ? obj.locations : []
+                    locations: 'menus' === tax ? obj.locations : [],
+
                 }
             })
             .then( async (res) => { return res.data; } )
             
             // update the API results ID, back to the existing ID from earlier.
-            results.id = existingID;
+           results.id = existingID;
+
         }
 
         // add results to collection
@@ -280,8 +315,13 @@ async function createTaxonomies( taxData, request, tax = 'pages', spinner ){
 
 }
 
+/**
+ * Create a FormData from the Media Item Object
+ *
+ * @param {Object} media Media Object.
+ * @returns {FormData}
+ */
 function createMediaItem( media ){
-    
     let fd = new FormData();
     let featureMediaFile = new File(
         [
@@ -293,10 +333,12 @@ function createMediaItem( media ){
         }
     );
     fd.append('file', featureMediaFile );
+
     fd.append('title', media.title);
-    fd.append('caption', media.caption);
+    fd.append('caption', media.caption );
     fd.append('alt_text', media.alt_text);
     fd.append('date', media.date);
+    fd.append('media_details', media.media_details);
 
     return fd;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@caweb/cli",
-  "version": "1.3.3",
+  "version": "1.3.4",
   "description": "CAWebPublishing Command Line Interface.",
   "exports": "./lib/env.js",
   "type": "module",