@miso.ai/server-wordpress 0.6.3-beta.1 → 0.6.3-beta.11

package/cli/entities.js

@@ -1,7 +1,8 @@
 import { Transform } from 'stream';
+import split2 from 'split2';
 import { stream, parseDuration } from '@miso.ai/server-commons';
 import { WordPressClient } from '../src/index.js';
-import { normalizeOptions, normalizeTransform, parseDate } from './utils.js';
+import { normalizeOptions, normalizeTransform } from './utils.js';
 
 export function buildForEntities(yargs) {
   // TODO: make them mutually exclusive
@@ -33,7 +34,7 @@ export function buildForEntities(yargs) {
     })
     .option('ids', {
       alias: 'include',
-      describe: 'Specify post ids'
+      describe: 'Specify post ids',
     })
     .option('fields', {
       describe: 'Specify which record fields are retrieved',
@@ -76,6 +77,12 @@ async function run({ subcmd, count, terms, update, name, ...options }) {
     case 'count':
       await runCount(client, name, options);
       return;
+    case 'absence':
+      await runPresence(client, name, { present: false });
+      return;
+    case 'presence':
+      await runPresence(client, name, { present: true });
+      return;
   }
   if (count) {
     await runCount(client, name, options);
@@ -136,6 +143,17 @@ export async function runUpdate(client, name, update, options) {
   );
 }
 
+export async function runPresence(client, name, options) {
+  await stream.pipeline(
+    process.stdin,
+    split2(),
+    client.entities(name).presence(options),
+    new stream.OutputStream({
+      objectMode: false,
+    }),
+  );
+}
+
 async function buildUpdateStream(client, name, update, {
   date, after, before, orderBy, order, // strip off date filters and order criteria
   transform,
@@ -156,6 +174,14 @@ async function buildUpdateStream(client, name, update, {
         after: threshold,
       }),
       // get recent modified, excluding ones already fetched
+      entities.stream({
+        ...options,
+        transform,
+        orderBy: 'modified',
+        modifiedAfter: threshold,
+        before: threshold,
+      }),
+      /*
       entities.stream({
         ...options,
         transform,
@@ -168,6 +194,7 @@ async function buildUpdateStream(client, name, update, {
           terminate: entity => parseDate(entity.modified_gmt) < threshold,
         },
       })
+      */
     ])
   );
 }

package/cli/index.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { yargs } from '@miso.ai/server-commons';
 import version from '../src/version.js';
-import profile from './profile.js';
+import { profile, init } from './profile.js';
 import taxonomies from './taxonomies.js';
 import entities from './entities.js';
 
@@ -16,11 +16,15 @@ yargs.build(yargs => {
       alias: 'p',
       describe: 'Site profile file location',
     })
+    .option('auth', {
+      describe: 'Authentication string',
+    })
     .option('debug', {
       type: 'boolean',
       default: false,
     })
     .hide('debug')
+    .command(init)
     .command(profile)
     .command(taxonomies)
     .command(entities)

package/cli/profile.js

@@ -11,7 +11,7 @@ function build(yargs) {
     });
 }
 
-async function run({ generate, ...options }) {
+async function runProfile({ generate, ...options }) {
   if (generate) {
     await runGenerate(options);
   } else {
@@ -38,9 +38,19 @@ async function runView(options) {
   console.log(JSON.stringify(client.profile, undefined, 2));
 }
 
-export default {
+async function runInit(options) {
+  await runGenerate(options);
+}
+
+export const profile = {
   command: 'profile',
   desc: 'WordPress site profile management',
   builder: build,
-  handler: run,
+  handler: runProfile,
+};
+
+export const init = {
+  command: 'init <site>',
+  desc: 'Initialize WordPress site profile',
+  handler: runInit,
 };

package/cli/utils.js

@@ -3,8 +3,9 @@ import { startOfDate, endOfDate } from '@miso.ai/server-commons';
 
 const PWD = process.env.PWD;
 
-export function normalizeOptions({ date, after, before, ids, ...options }) {
+export function normalizeOptions({ date, after, before, ids, include, ...options }) {
   [after, before] = [startOfDate(date || after), endOfDate(date || before)];
+  // TODO: rely on yargs to coerce to array
   ids = ids ? `${ids}`.split(',').map(s => s.trim()) : ids;
   return { ...options, after, before, ids };
 }

package/package.json

@@ -17,9 +17,9 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.3-beta.1",
-    "axios": "^0.27.2",
+    "@miso.ai/server-commons": "0.6.3-beta.11",
+    "axios": "^1.6.2",
     "axios-retry": "^3.3.1"
   },
-  "version": "0.6.3-beta.1"
+  "version": "0.6.3-beta.11"
 }

package/src/client.js

@@ -70,7 +70,7 @@ export default class WordPressClient {
 
 }
 
-const SITE_PROFILE_PROPS = ['site', 'utcOffset'];
+const SITE_PROFILE_PROPS = ['site', 'utcOffset', 'resources'];
 
 class SiteProfile {
 

package/src/entities/entity-index.js

@@ -2,7 +2,7 @@ import { asArray, Resolution } from '@miso.ai/server-commons';
 
 export default class EntityIndex {
 
-  constructor(entities, { process, value } = {}) {
+  constructor(entities, { process, value, fields } = {}) {
     this._entities = entities;
     if (process) {
       this._process = process;
@@ -10,6 +10,7 @@ export default class EntityIndex {
     if (value) {
       this._value = (en => en && value(en)); // null-safe
     }
+    this._fields = fields;
     this.name = entities.name;
     this._index = new Map();
     this._notFound = new Set();
@@ -66,7 +67,7 @@ export default class EntityIndex {
     if (idsToFetch.length > 0) {
       (async () => {
         const idsFetchSet = new Set(idsToFetch);
-        const stream = await this._entities.stream({ ids: idsToFetch });
+        const stream = await this._entities.stream({ ids: idsToFetch, fields: this._fields });
         for await (const entity of stream) {
            const { id } = entity;
            this._index.set(id, this._process(entity));

package/src/entities/index.js

@@ -2,6 +2,7 @@ import { Transform } from 'stream';
 import { asArray, stream } from '@miso.ai/server-commons';
 import EntityIndex from './entity-index.js';
 import EntityTransformStream from './transform.js';
+import EntityPresenceStream from './presence.js';
 import defaultTransform from './transform-default.js';
 import legacyTransform from './transform-legacy.js';
 
@@ -26,12 +27,17 @@ export default class Entities {
     // we need taxonomy fetched so we know whether it's hierarchical
     const taxonomies = await client._helpers.findAssociatedTaxonomies(this.name);
 
+    // TODO: omit specific indicies by config
     // prepare entity indicies
+    const { resources = {} } = client._profile || {};
+    const ignored = new Set(resources.ignore || []);
+
     const indicies = [
       client.users.index,
       client.media.index,
       ...taxonomies.map(({ rest_base }) => client.entities(rest_base).index),
-    ];
+    ].filter(index => !ignored.has(index.name));
+
     await Promise.all(indicies.map(index => index.ready()));
     for (const index of indicies) {
       if (index.hierarchical) {
@@ -82,6 +88,10 @@ export default class Entities {
     return this._client._helpers.terms(this.name, options);
   }
 
+  presence(options) {
+    return new EntityPresenceStream(this._client, this.name, options);
+  }
+
   get index() {
     return this._index;
   }

package/src/entities/presence.js

@@ -0,0 +1,105 @@
+import { Transform } from 'stream';
+
+export default class EntityPresenceStream extends Transform {
+
+  constructor(client, name, {
+    present = true,
+    fetchSize = 20,
+    preserveOrder = true,
+  } = {}) {
+    super();
+    this._client = client;
+    this._name = name;
+    this._options = {
+      present,
+      fetchSize,
+      preserveOrder,
+    }
+    this._inputs = [];
+    this._pendingSet = new Set();
+    this._requests = [];
+    this._map = new Map();
+    this._done = false;
+  }
+
+  async _transform(id, _, next) {
+    id = `${id}`; // buffer -> string
+    if (id) {
+      this._inputs.push(id);
+      this._outputAll();
+      this._requestAll();
+    }
+    next();
+  }
+
+  _flush(done) {
+    this._done = done;
+    this._outputAll();
+    if (this._inputs.length > 0) {
+      this._requestAll(true);
+    }
+  }
+
+  _outputAll() {
+    // TODO: implement when preserveOrder = false
+    let i = 0;
+    for (const len = this._inputs.length; i < len; i++) {
+      const id = this._inputs[i];
+      const entry = this._map.get(id);
+      if (!entry || entry.value === undefined) {
+        break;
+      }
+      if (this._options.present === entry.value) {
+        this.push(id);
+      }
+    }
+    if (i > 0) {
+      this._inputs = this._inputs.slice(i);
+    }
+    if (this._done && this._inputs.length === 0) {
+      this._done();
+    }
+  }
+
+  _requestAll(flush = false) {
+    for (const id of this._inputs) {
+      this._fetchAll();
+      if (!this._map.has(id)) {
+        this._map.set(id, { status: 'pending' });
+        this._pendingSet.add(id);
+      }
+    }
+    this._fetchAll(flush);
+  }
+
+  async _fetchAll(flush = false) {
+    if (!flush && this._pendingSet.size < this._options.fetchSize) {
+      return;
+    }
+    const ids = Array.from(this._pendingSet);
+    for (const id of ids) {
+      this._map.get(id).status = 'fetching';
+    }
+    this._pendingSet = new Set();
+
+    const presences = await this._fetch(ids);
+
+    for (const id of ids) {
+      const entry = this._map.get(id);
+      entry.status = 'ready';
+      entry.value = presences.has(id);
+    }
+    this._outputAll();
+  }
+
+  async _fetch(ids) {
+    const url = await this._client._helpers.url.build(this._name, { include: ids, fields: ['id'] });
+    const { data } = await this._client._helpers.axios.get(url);
+    const presences = new Set();
+    for (const { id } of data) {
+      presences.add(`${id}`);
+    }
+    return presences;
+  }
+
+}

package/src/entities/transform-default.js

@@ -14,14 +14,14 @@ export default function transform({
   modified_gmt,
   guid: {
     rendered: guid,
-  },
+  } = {},
   slug,
   title: {
     rendered: title,
-  },
+  } = {},
   content: {
     rendered: html,
-  },
+  } = {},
   link: url,
   status,
   sticky,
@@ -42,6 +42,7 @@ export default function transform({
     product_id,
     type,
     created_at,
+    published_at: created_at,
     updated_at,
     title,
     cover_image,

package/src/helpers.js

@@ -1,21 +1,49 @@
+import axios from 'axios';
+import axiosRetry from 'axios-retry';
 import { asNumber, splitObj, stream } from '@miso.ai/server-commons';
-import axios from './axios.js';
 import DataSource from './source/index.js';
+import version from './version.js';
 
 const MS_PER_HOUR = 1000 * 60 * 60;
 
 const STREAM_OPTIONS = ['offset', 'limit', 'strategy', 'filter', 'transform', 'onLoad'];
 
+function createAxios(client) {
+  const { auth } = client._options || {};
+  const headers = {
+    'User-Agent': `MisoBot/${version}`,
+  };
+  if (auth) {
+    if (typeof auth === 'object' && auth.username && auth.password) {
+      auth = `${auth.username}:${auth.password}`;
+    }
+    if (typeof auth !== 'string') {
+      throw new TypeError(`Invalid auth: must me a string or an object.`);
+    }
+    headers['Authorization'] = 'Basic ' + Buffer.from(auth).toString('base64');
+  }
+  const instance = axios.create({
+    headers,
+  });
+  axiosRetry(instance, { retries: 5, retryDelay: count => count * 300 });
+  return instance;
+}
+
 export default class Helpers {
 
   constructor(client) {
     this._start = Date.now();
     this._client = client;
+    this._axios = createAxios(client);
     this.url = new Url(this);
     this._samples = {};
     this.debug = this.debug.bind(this);
   }
 
+  get axios() {
+    return this._axios;
+  }
+
   async stream(resource, options) {
     const [streamOptions, sourceOptions] = splitObj(options, STREAM_OPTIONS);
     const source = new DataSource(this, resource, sourceOptions);
@@ -32,7 +60,7 @@ export default class Helpers {
 
   async _fetchSample(resource) {
     const url = await this.url.build(resource, { page: 0, pageSize: 1 });
-    const { data, headers } = await axios.get(url);
+    const { data, headers } = await this.axios.get(url);
     if (!data.length) {
       throw new Error(`No record of ${resource} avaliable`);
     }
@@ -71,7 +99,7 @@ export default class Helpers {
 
   async _fetchTaxonomies() {
     const url = await this.url.build('taxonomies');
-    const { data } = await axios.get(url);
+    const { data } = await this.axios.get(url);
     this.debug(`Fetched taxonomies.`);
     return Object.values(data);
   }
@@ -82,7 +110,7 @@ export default class Helpers {
 
   async count(resource, { offset: _, ...options } = {}) {
     const url = await this.url.build(resource, { ...options, page: 0, pageSize: 1 });
-    const { headers } = await axios.get(url);
+    const { headers } = await this.axios.get(url);
     return asNumber(headers['x-wp-total']);
   }
 
@@ -92,7 +120,7 @@ export default class Helpers {
 
   async countUrl(url) {
     url = await this.url.append(url, { page: 0, pageSize: 1 });
-    const { headers } = await axios.get(url);
+    const { headers } = await this.axios.get(url);
     return asNumber(headers['x-wp-total']);
   }
 
@@ -134,17 +162,19 @@ class Url {
   // modifiedAfter, modifiedBefore is supported since WordPress 5.7
   // https://make.wordpress.org/core/2021/02/23/rest-api-changes-in-wordpress-5-7/
   async append(url, options = {}) {
-    const { after, before, order, orderBy, page, pageSize, offset, include, exclude } = options;
+    const { after, before, modifiedAfter, modifiedBefore, order, orderBy, page, pageSize, offset, include, exclude } = options;
     let { fields } = options;
     const params = [];
 
     // TODO: support single id
 
     // The date is compared against site's local time, not UTC, so we have to work on timezone offset
-    if (has(after) || has(before)) {
+    if (has(after) || has(before) || has(modifiedAfter) || has(modifiedBefore)) {
       const utcOffset = await this._helpers.utcOffsetInMs();
       has(after) && params.push(`after=${toISOString(after, utcOffset)}`);
       has(before) && params.push(`before=${toISOString(before, utcOffset)}`);
+      has(modifiedAfter) && params.push(`modified_after=${toISOString(modifiedAfter, utcOffset)}`);
+      has(modifiedBefore) && params.push(`modified_before=${toISOString(modifiedBefore, utcOffset)}`);
     }
 
     has(order) && params.push(`order=${order}`);

package/src/source/base.js

@@ -1,5 +1,3 @@
-import axios from '../axios.js';
-
 export default class WordPressDataSource {
 
   constructor(helpers, resource, options = {}) {
@@ -32,14 +30,17 @@ export default class WordPressDataSource {
     this._debug(`[WordPressDataSource] request ${url}`);
     const response =  await this._axiosGet(url);
     this._debug(`[WordPressDataSource] response ${response.status} ${url}`);
-    return this._process(response);
+    return this._process(response, { url });
   }
 
-  _process({ status, data }) {
+  _process({ status, data }, { url }) {
     if (status >= 400 && status < 500 && data.code === 'rest_post_invalid_page_number') {
       // out of bound, so there is no more data
       return { data: [], terminate: true };
     }
+    if (!Array.isArray(data)) {
+      throw new Error(`Unexpected response from WordPress API for ${url}. Expected an array of objects: ${data}`);
+    }
     if (!this._options.preserveLinks) {
       data = data.map(this._helpers.removeLinks);
     }
@@ -52,13 +53,13 @@ export default class WordPressDataSource {
 
   async _buildBaseUrl() {
     // exclude parameters meant to be dealt with state
-    const { page, ...options } = this._options;
+    const { page, ids, ...options } = this._options;
     return this._helpers.url.build(this._resource, options);
   }
 
   async _axiosGet(url) {
     try {
-      return await axios.get(url);
+      return await this._helpers.axios.get(url);
     } catch(error) {
       if (error.response) {
         return error.response;

package/src/source/paged.js

@@ -50,8 +50,8 @@ export default class PagedWordPressDataSource extends WordPressDataSource {
     return total;
   }
 
-  _process({ status, data, headers }) {
-    const result = super._process({ status, data, headers });
+  _process({ status, data, headers }, meta) {
+    const result = super._process({ status, data, headers }, meta);
     const total = asNumber(headers['x-wp-total']);
     if (total !== undefined) {
       result.total = total;

package/src/version.js

@@ -1 +1 @@
-export default '0.6.3-beta.1';
+export default '0.6.3-beta.11';

package/src/axios.js

@@ -1,6 +0,0 @@
-import axios from 'axios';
-import axiosRetry from 'axios-retry';
-
-axiosRetry(axios, { retries: 5, retryDelay: count => count * 300 });
-
-export default axios;