@miso.ai/server-wordpress 0.6.3-beta.0 → 0.6.3-beta.10

package/cli/entities.js

@@ -1,3 +1,5 @@
+import { Transform } from 'stream';
+import split2 from 'split2';
 import { stream, parseDuration } from '@miso.ai/server-commons';
 import { WordPressClient } from '../src/index.js';
 import { normalizeOptions, normalizeTransform, parseDate } from './utils.js';
@@ -34,6 +36,11 @@ export function buildForEntities(yargs) {
       alias: 'include',
       describe: 'Specify post ids'
     })
+    .option('fields', {
+      describe: 'Specify which record fields are retrieved',
+      type: 'array',
+      coerce: yargs.coerceToArray,
+    })
     .option('resolve', {
       alias: 'r',
       describe: 'Attach resolved entities (author, catagories) linked with the subjects',
@@ -60,9 +67,23 @@ function build(yargs) {
     });
 }
 
-async function run({ count, terms, update, name, ...options }) {
+async function run({ subcmd, count, terms, update, name, ...options }) {
   options = normalizeOptions(options);
   const client = new WordPressClient(options);
+  switch (subcmd) {
+    case 'ids':
+      await runIds(client, name, { update, ...options });
+      return;
+    case 'count':
+      await runCount(client, name, options);
+      return;
+    case 'absence':
+      await runPresence(client, name, { present: false });
+      return;
+    case 'presence':
+      await runPresence(client, name, { present: true });
+      return;
+  }
   if (count) {
     await runCount(client, name, options);
   } else if (terms) {
@@ -95,46 +116,91 @@ export async function runGet(client, name, { transform, ...options }) {
   );
 }
 
-export async function runUpdate(client, name, update, {
+export async function runIds(client, name, { update, transform, resolve, fields, ...options }) {
+  if (update) {
+    await stream.pipeline(
+      await buildUpdateStream(client, name, update, { ...options, fields: ['id', 'modified_gmt'] }),
+      new Transform({
+        objectMode: true,
+        transform({ id }, _, callback) {
+          callback(null, id);
+        },
+      }),
+      new stream.OutputStream(),
+    );
+  } else {
+    await stream.pipeline(
+      await client.entities(name).ids(options),
+      new stream.OutputStream(),
+    );
+  }
+}
+
+export async function runUpdate(client, name, update, options) {
+  await stream.pipeline(
+    await buildUpdateStream(client, name, update, options),
+    new stream.OutputStream(),
+  );
+}
+
+export async function runPresence(client, name, options) {
+  await stream.pipeline(
+    process.stdin,
+    split2(),
+    client.entities(name).presence(options),
+    new stream.OutputStream({
+      objectMode: false,
+    }),
+  );
+}
+
+async function buildUpdateStream(client, name, update, {
   date, after, before, orderBy, order, // strip off date filters and order criteria
   transform,
   ...options
 }) {
+  // TODO: move the logic into client itself
   transform = await normalizeTransform(transform);
   const now = Date.now();
   update = parseDuration(update);
   const threshold = now - update;
   const entities = client.entities(name);
-  await stream.pipelineToStdout(
-    stream.concat(
-      ...await Promise.all([
-        // get recent published
-        entities.stream({
-          ...options,
-          transform,
-          after: threshold,
-        }),
-        // get recent modified, excluding ones already fetched
-        entities.stream({
-          ...options,
-          transform,
-          orderBy: 'modified',
-          before: threshold,
-          pageSize: 20,
-          strategy: {
-            highWatermark: 100,
-            eagerLoad: true,
-            terminate: entity => parseDate(entity.modified_gmt) < threshold,
-          },
-        })
-      ])
-    ),
-    stream.stringify(),
+  return stream.concat(
+    ...await Promise.all([
+      // get recent published
+      entities.stream({
+        ...options,
+        transform,
+        after: threshold,
+      }),
+      // get recent modified, excluding ones already fetched
+      entities.stream({
+        ...options,
+        transform,
+        orderBy: 'modified',
+        modifiedAfter: threshold,
+        before: threshold,
+      }),
+      /*
+      entities.stream({
+        ...options,
+        transform,
+        orderBy: 'modified',
+        before: threshold,
+        pageSize: 20,
+        strategy: {
+          highWatermark: 100,
+          eagerLoad: true,
+          terminate: entity => parseDate(entity.modified_gmt) < threshold,
+        },
+      })
+      */
+    ])
   );
 }
 
 export default {
-  command: ['$0 <name>'],
+  command: ['$0 <name> [subcmd]'],
   desc: 'List entities from WordPress REST API',
   builder: build,
   handler: run,

package/cli/index.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { yargs } from '@miso.ai/server-commons';
 import version from '../src/version.js';
-import profile from './profile.js';
+import { profile, init } from './profile.js';
 import taxonomies from './taxonomies.js';
 import entities from './entities.js';
 
@@ -16,11 +16,15 @@ yargs.build(yargs => {
       alias: 'p',
       describe: 'Site profile file location',
     })
+    .option('auth', {
+      describe: 'Authentication string',
+    })
     .option('debug', {
       type: 'boolean',
       default: false,
     })
     .hide('debug')
+    .command(init)
     .command(profile)
     .command(taxonomies)
     .command(entities)

package/cli/profile.js

@@ -11,7 +11,7 @@ function build(yargs) {
     });
 }
 
-async function run({ generate, ...options }) {
+async function runProfile({ generate, ...options }) {
   if (generate) {
     await runGenerate(options);
   } else {
@@ -38,9 +38,19 @@ async function runView(options) {
   console.log(JSON.stringify(client.profile, undefined, 2));
 }
 
-export default {
+async function runInit(options) {
+  await runGenerate(options);
+}
+
+export const profile = {
   command: 'profile',
   desc: 'WordPress site profile management',
   builder: build,
-  handler: run,
+  handler: runProfile,
+};
+
+export const init = {
+  command: 'init <site>',
+  desc: 'Initialize WordPress site profile',
+  handler: runInit,
 };

package/package.json

@@ -17,9 +17,9 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.3-beta.0",
+    "@miso.ai/server-commons": "0.6.3-beta.10",
     "axios": "^0.27.2",
     "axios-retry": "^3.3.1"
   },
-  "version": "0.6.3-beta.0"
+  "version": "0.6.3-beta.10"
 }

package/src/client.js

@@ -70,7 +70,7 @@ export default class WordPressClient {
 
 }
 
-const SITE_PROFILE_PROPS = ['site', 'utcOffset'];
+const SITE_PROFILE_PROPS = ['site', 'utcOffset', 'resources'];
 
 class SiteProfile {
 

package/src/entities/entity-index.js

@@ -2,7 +2,7 @@ import { asArray, Resolution } from '@miso.ai/server-commons';
 
 export default class EntityIndex {
 
-  constructor(entities, { process, value } = {}) {
+  constructor(entities, { process, value, fields } = {}) {
     this._entities = entities;
     if (process) {
       this._process = process;
@@ -10,6 +10,7 @@ export default class EntityIndex {
     if (value) {
       this._value = (en => en && value(en)); // null-safe
     }
+    this._fields = fields;
     this.name = entities.name;
     this._index = new Map();
     this._notFound = new Set();
@@ -66,7 +67,7 @@ export default class EntityIndex {
     if (idsToFetch.length > 0) {
       (async () => {
         const idsFetchSet = new Set(idsToFetch);
-        const stream = await this._entities.stream({ ids: idsToFetch });
+        const stream = await this._entities.stream({ ids: idsToFetch, fields: this._fields });
         for await (const entity of stream) {
            const { id } = entity;
            this._index.set(id, this._process(entity));

package/src/entities/index.js

@@ -1,6 +1,8 @@
+import { Transform } from 'stream';
 import { asArray, stream } from '@miso.ai/server-commons';
 import EntityIndex from './entity-index.js';
 import EntityTransformStream from './transform.js';
+import EntityPresenceStream from './presence.js';
 import defaultTransform from './transform-default.js';
 import legacyTransform from './transform-legacy.js';
 
@@ -14,6 +16,7 @@ export default class Entities {
   }
   
   async stream({ resolve = false, transform, ...options } = {}) {
+    // TODO: when after/before is used and fields are specified, we need to retain only fields that user wants
     if (!resolve && !transform) {
       return this._client._helpers.stream(this.name, options);
     }
@@ -24,12 +27,17 @@ export default class Entities {
     // we need taxonomy fetched so we know whether it's hierarchical
     const taxonomies = await client._helpers.findAssociatedTaxonomies(this.name);
 
+    // TODO: omit specific indicies by config
     // prepare entity indicies
+    const { resources = {} } = client._profile || {};
+    const ignored = new Set(resources.ignore || []);
+
     const indicies = [
       client.users.index,
       client.media.index,
       ...taxonomies.map(({ rest_base }) => client.entities(rest_base).index),
-    ];
+    ].filter(index => !ignored.has(index.name));
+
     await Promise.all(indicies.map(index => index.ready()));
     for (const index of indicies) {
       if (index.hierarchical) {
@@ -53,6 +61,21 @@ export default class Entities {
       .pipe(transformStream);
   }
 
+  async ids(options = {}) {
+    const { before, after, u } = options;
+    const fields = ['id'];
+    if (before || after) {
+      fields.push('modified_gmt');
+    }
+    return (await this._client._helpers.stream(this.name, { ...options, fields }))
+      .pipe(new Transform({
+        objectMode: true,
+        transform({ id }, _, callback) {
+          callback(null, id);
+        },
+      }));
+  }
+
   async getAll(options) {
     return stream.collect(await this.stream(options));
   }
@@ -65,6 +88,10 @@ export default class Entities {
     return this._client._helpers.terms(this.name, options);
   }
 
+  presence(options) {
+    return new EntityPresenceStream(this._client, this.name, options);
+  }
+
   get index() {
     return this._index;
   }

package/src/entities/presence.js

@@ -0,0 +1,105 @@
+import { Transform } from 'stream';
+
+export default class EntityPresenceStream extends Transform {
+
+  constructor(client, name, {
+    present = true,
+    fetchSize = 20,
+    preserveOrder = true,
+  } = {}) {
+    super();
+    this._client = client;
+    this._name = name;
+    this._options = {
+      present,
+      fetchSize,
+      preserveOrder,
+    }
+    this._inputs = [];
+    this._pendingSet = new Set();
+    this._requests = [];
+    this._map = new Map();
+    this._done = false;
+  }
+
+  async _transform(id, _, next) {
+    id = `${id}`; // buffer -> string
+    if (id) {
+      this._inputs.push(id);
+      this._outputAll();
+      this._requestAll();
+    }
+    next();
+  }
+
+  _flush(done) {
+    this._done = done;
+    this._outputAll();
+    if (this._inputs.length > 0) {
+      this._requestAll(true);
+    }
+  }
+
+  _outputAll() {
+    // TODO: implement when preserveOrder = false
+    let i = 0;
+    for (const len = this._inputs.length; i < len; i++) {
+      const id = this._inputs[i];
+      const entry = this._map.get(id);
+      if (!entry || entry.value === undefined) {
+        break;
+      }
+      if (this._options.present === entry.value) {
+        this.push(id);
+      }
+    }
+    if (i > 0) {
+      this._inputs = this._inputs.slice(i);
+    }
+    if (this._done && this._inputs.length === 0) {
+      this._done();
+    }
+  }
+
+  _requestAll(flush = false) {
+    for (const id of this._inputs) {
+      this._fetchAll();
+      if (!this._map.has(id)) {
+        this._map.set(id, { status: 'pending' });
+        this._pendingSet.add(id);
+      }
+    }
+    this._fetchAll(flush);
+  }
+
+  async _fetchAll(flush = false) {
+    if (!flush && this._pendingSet.size < this._options.fetchSize) {
+      return;
+    }
+    const ids = Array.from(this._pendingSet);
+    for (const id of ids) {
+      this._map.get(id).status = 'fetching';
+    }
+    this._pendingSet = new Set();
+
+    const presences = await this._fetch(ids);
+
+    for (const id of ids) {
+      const entry = this._map.get(id);
+      entry.status = 'ready';
+      entry.value = presences.has(id);
+    }
+    this._outputAll();
+  }
+
+  async _fetch(ids) {
+    const url = await this._client._helpers.url.build(this._name, { include: ids, fields: ['id'] });
+    const { data } = await this._client._helpers.axios.get(url);
+    const presences = new Set();
+    for (const { id } of data) {
+      presences.add(`${id}`);
+    }
+    return presences;
+  }
+
+}

package/src/entities/transform-default.js

@@ -14,14 +14,14 @@ export default function transform({
   modified_gmt,
   guid: {
     rendered: guid,
-  },
+  } = {},
   slug,
   title: {
     rendered: title,
-  },
+  } = {},
   content: {
     rendered: html,
-  },
+  } = {},
   link: url,
   status,
   sticky,
@@ -42,6 +42,7 @@ export default function transform({
     product_id,
     type,
     created_at,
+    published_at: created_at,
     updated_at,
     title,
     cover_image,

package/src/helpers.js

@@ -1,21 +1,49 @@
+import axios from 'axios';
+import axiosRetry from 'axios-retry';
 import { asNumber, splitObj, stream } from '@miso.ai/server-commons';
-import axios from './axios.js';
 import DataSource from './source/index.js';
+import version from './version.js';
 
 const MS_PER_HOUR = 1000 * 60 * 60;
 
 const STREAM_OPTIONS = ['offset', 'limit', 'strategy', 'filter', 'transform', 'onLoad'];
 
+function createAxios(client) {
+  const { auth } = client._options || {};
+  const headers = {
+    'User-Agent': `MisoWordPressTool/${version}`,
+  };
+  if (auth) {
+    if (typeof auth === 'object' && auth.username && auth.password) {
+      auth = `${auth.username}:${auth.password}`;
+    }
+    if (typeof auth !== 'string') {
+      throw new TypeError(`Invalid auth: must me a string or an object.`);
+    }
+    headers['Authorization'] = 'Basic ' + Buffer.from(auth).toString('base64');
+  }
+  const instance = axios.create({
+    headers,
+  });
+  axiosRetry(instance, { retries: 5, retryDelay: count => count * 300 });
+  return instance;
+}
+
 export default class Helpers {
 
   constructor(client) {
     this._start = Date.now();
     this._client = client;
+    this._axios = createAxios(client);
     this.url = new Url(this);
     this._samples = {};
     this.debug = this.debug.bind(this);
   }
 
+  get axios() {
+    return this._axios;
+  }
+
   async stream(resource, options) {
     const [streamOptions, sourceOptions] = splitObj(options, STREAM_OPTIONS);
     const source = new DataSource(this, resource, sourceOptions);
@@ -32,7 +60,7 @@ export default class Helpers {
 
   async _fetchSample(resource) {
     const url = await this.url.build(resource, { page: 0, pageSize: 1 });
-    const { data, headers } = await axios.get(url);
+    const { data, headers } = await this.axios.get(url);
     if (!data.length) {
       throw new Error(`No record of ${resource} avaliable`);
     }
@@ -71,7 +99,7 @@ export default class Helpers {
 
   async _fetchTaxonomies() {
     const url = await this.url.build('taxonomies');
-    const { data } = await axios.get(url);
+    const { data } = await this.axios.get(url);
     this.debug(`Fetched taxonomies.`);
     return Object.values(data);
   }
@@ -82,7 +110,7 @@ export default class Helpers {
 
   async count(resource, { offset: _, ...options } = {}) {
     const url = await this.url.build(resource, { ...options, page: 0, pageSize: 1 });
-    const { headers } = await axios.get(url);
+    const { headers } = await this.axios.get(url);
     return asNumber(headers['x-wp-total']);
   }
 
@@ -92,7 +120,7 @@ export default class Helpers {
 
   async countUrl(url) {
     url = await this.url.append(url, { page: 0, pageSize: 1 });
-    const { headers } = await axios.get(url);
+    const { headers } = await this.axios.get(url);
     return asNumber(headers['x-wp-total']);
   }
 
@@ -134,16 +162,19 @@ class Url {
   // modifiedAfter, modifiedBefore is supported since WordPress 5.7
   // https://make.wordpress.org/core/2021/02/23/rest-api-changes-in-wordpress-5-7/
   async append(url, options = {}) {
-    const { after, before, order, orderBy, page, pageSize, offset, include, exclude } = options;
+    const { after, before, modifiedAfter, modifiedBefore, order, orderBy, page, pageSize, offset, include, exclude } = options;
+    let { fields } = options;
     const params = [];
 
     // TODO: support single id
 
     // The date is compared against site's local time, not UTC, so we have to work on timezone offset
-    if (has(after) || has(before)) {
+    if (has(after) || has(before) || has(modifiedAfter) || has(modifiedBefore)) {
       const utcOffset = await this._helpers.utcOffsetInMs();
       has(after) && params.push(`after=${toISOString(after, utcOffset)}`);
       has(before) && params.push(`before=${toISOString(before, utcOffset)}`);
+      has(modifiedAfter) && params.push(`modified_after=${toISOString(modifiedAfter, utcOffset)}`);
+      has(modifiedBefore) && params.push(`modified_before=${toISOString(modifiedBefore, utcOffset)}`);
     }
 
     has(order) && params.push(`order=${order}`);
@@ -153,6 +184,12 @@ class Url {
     has(offset) && params.push(`offset=${offset}`);
     has(include) && include.length && params.push(`include=${joinIds(include)}`);
     has(exclude) && exclude.length && params.push(`exclude=${joinIds(exclude)}`);
+    if (has(fields) && fields.length) {
+      if (has(before) && !fields.includes('modified_gmt')) {
+        fields = [...fields, 'modified_gmt'];
+      }
+      params.push(`_fields=${fields.join(',')}`);
+    }
 
     const head = params.length === 0 ? '' : url.indexOf('?') < 0 ? '?' : '&';
     return `${url}${head}${params.join('&')}`;

package/src/source/base.js

@@ -1,5 +1,3 @@
-import axios from '../axios.js';
-
 export default class WordPressDataSource {
 
   constructor(helpers, resource, options = {}) {
@@ -32,14 +30,17 @@ export default class WordPressDataSource {
     this._debug(`[WordPressDataSource] request ${url}`);
     const response =  await this._axiosGet(url);
     this._debug(`[WordPressDataSource] response ${response.status} ${url}`);
-    return this._process(response);
+    return this._process(response, { url });
   }
 
-  _process({ status, data }) {
+  _process({ status, data }, { url }) {
     if (status >= 400 && status < 500 && data.code === 'rest_post_invalid_page_number') {
       // out of bound, so there is no more data
       return { data: [], terminate: true };
     }
+    if (!Array.isArray(data)) {
+      throw new Error(`Unexpected response from WordPress API for ${url}. Expected an array of objects: ${data}`);
+    }
     if (!this._options.preserveLinks) {
       data = data.map(this._helpers.removeLinks);
     }
@@ -58,7 +59,7 @@ export default class WordPressDataSource {
 
   async _axiosGet(url) {
     try {
-      return await axios.get(url);
+      return await this._helpers.axios.get(url);
     } catch(error) {
       if (error.response) {
         return error.response;

package/src/source/paged.js

@@ -50,8 +50,8 @@ export default class PagedWordPressDataSource extends WordPressDataSource {
     return total;
   }
 
-  _process({ status, data, headers }) {
-    const result = super._process({ status, data, headers });
+  _process({ status, data, headers }, meta) {
+    const result = super._process({ status, data, headers }, meta);
     const total = asNumber(headers['x-wp-total']);
     if (total !== undefined) {
       result.total = total;

package/src/version.js

@@ -1 +1 @@
-export default '0.6.3-beta.0';
+export default '0.6.3-beta.10';

package/src/axios.js

@@ -1,6 +0,0 @@
-import axios from 'axios';
-import axiosRetry from 'axios-retry';
-
-axiosRetry(axios, { retries: 5, retryDelay: count => count * 300 });
-
-export default axios;