@miso.ai/server-wordpress 0.6.0-beta.1

package/cli/entities.js

@@ -0,0 +1,141 @@
+import { stream, parseDuration } from '@miso.ai/server-commons';
+import { WordPressClient } from '../src/index.js';
+import { normalizeOptions, normalizeTransform, parseDate } from './utils.js';
+
+export function buildForEntities(yargs) {
+  // TODO: make them mutually exclusive
+  return yargs
+    .option('terms', {
+      describe: 'Display terms associated with this type of resource',
+      type: 'boolean',
+    })
+    .option('count', {
+      alias: 'c',
+      describe: 'Return the total number of records',
+      type: 'boolean',
+    })
+    .option('date', {
+      alias: 'd',
+      describe: 'Only include records in this year/month/day',
+    })
+    .option('after', {
+      alias: 'a',
+      describe: 'Only include records after this time',
+    })
+    .option('before', {
+      alias: 'b',
+      describe: 'Only include records before this time',
+    })
+    .option('update', {
+      alias: 'u',
+      describe: 'Only include records modified in given duration (3h, 2d, etc.)',
+    })
+    .option('ids', {
+      alias: 'include',
+      describe: 'Specify post ids'
+    })
+    .option('resolve', {
+      alias: 'r',
+      describe: 'Attach resolved entities (author, catagories) linked with the subjects',
+      type: 'boolean',
+    })
+    .option('transform', {
+      alias: 't',
+      describe: 'Apply transform function to the entities',
+    });
+    /*
+    .option('limit', {
+      alias: 'n',
+      describe: 'Limit the amount of records',
+      type: 'number',
+    })
+    */
+}
+
+function build(yargs) {
+  return buildForEntities(yargs)
+    .positional('name', {
+      describe: 'Resource type',
+      type: 'string',
+    });
+}
+
+async function run({ count, terms, update, name, ...options }) {
+  options = normalizeOptions(options);
+  const client = new WordPressClient(options);
+  if (count) {
+    await runCount(client, name, options);
+  } else if (terms) {
+    await runTerms(client, name, options);
+  } else if (update) {
+    await runUpdate(client, name, update, options);
+  } else {
+    await runGet(client, name, options);
+  }
+}
+
+export async function runCount(client, name, options) {
+  console.log(await client.entities(name).count(options));
+}
+
+export async function runTerms(client, name, options) {
+  const terms = await client.entities(name).terms(options);
+  for (const term of terms) {
+    console.log(JSON.stringify(term));
+  }
+}
+
+export async function runGet(client, name, { transform, ...options }) {
+  await stream.pipelineToStdout(
+    await client.entities(name).stream({
+      ...options,
+      transform: await normalizeTransform(transform),
+    }),
+    stream.stringify(),
+  );
+}
+
+export async function runUpdate(client, name, update, {
+  date, after, before, orderBy, order, // strip off date filters and order criteria
+  transform,
+  ...options
+}) {
+  transform = await normalizeTransform(transform);
+  const now = Date.now();
+  update = parseDuration(update);
+  const threshold = now - update;
+  const entities = client.entities(name);
+  await stream.pipelineToStdout(
+    stream.concat(
+      ...await Promise.all([
+        // get recent published
+        entities.stream({
+          ...options,
+          transform,
+          after: threshold,
+        }),
+        // get recent modified, excluding ones already fetched
+        entities.stream({
+          ...options,
+          transform,
+          orderBy: 'modified',
+          before: threshold,
+          pageSize: 20,
+          strategy: {
+            highWatermark: 100,
+            eagerLoad: true,
+            terminate: entity => parseDate(entity.modified_gmt) < threshold,
+          },
+        })
+      ])
+    ),
+    stream.stringify(),
+  );
+}
+
+export default {
+  command: ['$0 <name>'],
+  desc: 'List entities from WordPress REST API',
+  builder: build,
+  handler: run,
+};

package/cli/index.js

@@ -0,0 +1,28 @@
+#!/usr/bin/env node
+import { yargs } from '@miso.ai/server-commons';
+import version from '../src/version.js';
+import profile from './profile.js';
+import taxonomies from './taxonomies.js';
+import entities from './entities.js';
+
+yargs.build(yargs => {
+  yargs
+    .env('MISO_WORDPRESS')
+    .option('site', {
+      alias: 's',
+      describe: 'the WordPress site',
+    })
+    .option('profile', {
+      alias: 'p',
+      describe: 'Site profile file location',
+    })
+    .option('debug', {
+      type: 'boolean',
+      default: false,
+    })
+    .hide('debug')
+    .command(profile)
+    .command(taxonomies)
+    .command(entities)
+    .version(version);
+});

package/cli/profile.js

@@ -0,0 +1,46 @@
+import { resolve } from 'path';
+import { fileExists } from '@miso.ai/server-commons';
+import { WordPressClient } from '../src/index.js';
+
+function build(yargs) {
+  return yargs
+    .option('generate', {
+      alias: 'g',
+      describe: 'Generate profile',
+      type: 'boolean',
+    });
+}
+
+async function run({ generate, ...options }) {
+  if (generate) {
+    await runGenerate(options);
+  } else {
+    await runView(options);
+  }
+}
+
+async function runGenerate(options) {
+  // it's OK if profile file doesn't exist in generate mode
+  // but we need to leave it out from the options
+  const { profile } = options;
+  if (profile && !await fileExists(profile)) {
+    delete options.profile;
+  }
+  const client = new WordPressClient(options);
+  await client.generateProfile();
+  await client.saveProfile(profile);
+  const location = resolve(process.env.PWD, profile || WordPressClient.DEFAULT_PROFILE_LOCATION);
+  console.log(`WordPress profile generated at: ${location}`);
+}
+
+async function runView(options) {
+  const client = new WordPressClient(options);
+  console.log(JSON.stringify(client.profile, undefined, 2));
+}
+
+export default {
+  command: 'profile',
+  desc: 'WordPress site profile management',
+  builder: build,
+  handler: run,
+};

package/cli/taxonomies.js

@@ -0,0 +1,21 @@
+import { WordPressClient } from '../src/index.js';
+
+function build(yargs) {
+  return yargs;
+}
+
+async function run(options) {
+  const client = new WordPressClient(options);
+  const taxonomies = await client.taxonomies({ noCache: true });
+  for (const taxonomy of taxonomies) {
+    console.log(JSON.stringify(taxonomy));
+  }
+}
+
+export default {
+  command: 'taxonomies',
+  aliases: ['tax'],
+  desc: 'List taxonomies from WordPress REST API',
+  builder: build,
+  handler: run,
+};

package/cli/utils.js

@@ -0,0 +1,24 @@
+import { join } from 'path';
+import { startOfDate, endOfDate } from '@miso.ai/server-commons';
+
+const PWD = process.env.PWD;
+
+export function normalizeOptions({ date, after, before, ids, ...options }) {
+  [after, before] = [startOfDate(date || after), endOfDate(date || before)];
+  ids = ids ? `${ids}`.split(',').map(s => s.trim()) : ids;
+  return { ...options, after, before, ids };
+}
+
+export async function normalizeTransform(transform) {
+  if (typeof transform === 'string') {
+    if (transform === 'default' || transform === 'legacy') {
+      return transform;
+    }
+    return (await import(join(PWD, transform))).default;
+  }
+  return !!transform;
+}
+
+export function parseDate(value) {
+  return Date.parse(`${value}Z`);
+}

package/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@miso.ai/server-wordpress",
+  "description": "Miso WordPress API data tools",
+  "type": "module",
+  "main": "src/index.js",
+  "bin": {
+    "miso-wordpress": "cli/index.js",
+    "miso-wp": "cli/index.js"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {},
+  "repository": "MisoAI/miso-server-js-sdk",
+  "license": "MIT",
+  "contributors": [
+    "simonpai <simon.pai@askmiso.com>"
+  ],
+  "dependencies": {
+    "@miso.ai/server-commons": "0.6.0-beta.1",
+    "axios": "^0.27.2",
+    "axios-retry": "^3.3.1"
+  },
+  "version": "0.6.0-beta.1"
+}

package/src/axios.js

@@ -0,0 +1,6 @@
+import axios from 'axios';
+import axiosRetry from 'axios-retry';
+
+axiosRetry(axios, { retries: 5, retryDelay: count => count * 300 });
+
+export default axios;

package/src/client.js

@@ -0,0 +1,97 @@
+import { defineValues, copyValues, trimObj, loadConfigSync, saveConfig, splitObj, fileExistsSync } from '@miso.ai/server-commons';
+import { constants } from 'fs';
+import Helpers from './helpers.js';
+import Media from './media.js';
+import Posts from './posts/index.js';
+import Entities from './entities/index.js';
+
+const DEFAULT_PROFILE = './wordpress.json';
+
+export default class WordPressClient {
+
+  static DEFAULT_PROFILE_LOCATION = DEFAULT_PROFILE;
+
+  constructor({ profile, ...options } = {}) {
+    if (!profile && fileExistsSync(DEFAULT_PROFILE, constants.R_OK)) {
+      profile = DEFAULT_PROFILE;
+    }
+    if (profile) {
+      options = { ...loadConfigSync(profile), ...options };
+    }
+    if (!options.site) {
+      throw new Error(`Require 'site' property in either options or profile.`);
+    }
+    defineValues(this, { site: options.site });
+    let profileObj;
+    [profileObj, options] = splitObj(options, SITE_PROFILE_PROPS);
+    this._options = options;
+    this._profile = new SiteProfile(profileObj);
+
+    this._helpers = new Helpers(this);
+    this._entities = {
+      media: new Media(this),
+      posts: new Posts(this),
+    };
+  }
+
+  async generateProfile() {
+    this._profile.clear();
+    // write UTC offset
+    await this._helpers.utcOffsetInMs();
+  }
+
+  async saveProfile(file = DEFAULT_PROFILE) {
+    await saveConfig(file, this._profile.export());
+  }
+
+  get profile() {
+    return this._profile.export();
+  }
+
+  taxonomies(options) {
+    return this._helpers.taxonomies(options);
+  }
+
+  get posts() {
+    return this.entities('posts');
+  }
+
+  get users() {
+    return this.entities('users');
+  }
+
+  get media() {
+    return this.entities('media');
+  }
+
+  entities(name) {
+    return this._entities[name] || (this._entities[name] = new Entities(this, name));
+  }
+
+}
+
+const SITE_PROFILE_PROPS = ['site', 'utcOffset'];
+
+class SiteProfile {
+
+  constructor(profile) {
+    profile && this.load(profile);
+  }
+
+  clear() {
+    for (const key in this) {
+      if (key !== 'site') {
+        delete this[key];
+      }
+    }
+  }
+
+  load(profile) {
+    copyValues(this, profile, SITE_PROFILE_PROPS);
+  }
+
+  export() {
+    return trimObj(copyValues({}, this, SITE_PROFILE_PROPS));
+  }
+
+}

package/src/entities/entity-index.js

@@ -0,0 +1,160 @@
+import { asArray, Resolution } from '@miso.ai/server-commons';
+
+export default class EntityIndex {
+
+  constructor(entities, { process, value } = {}) {
+    this._entities = entities;
+    if (process) {
+      this._process = process;
+    }
+    if (value) {
+      this._value = (en => en && value(en)); // null-safe
+    }
+    this.name = entities.name;
+    this._index = new Map();
+    this._notFound = new Set();
+    this._fetching = new Map();
+  }
+
+  async ready() {
+    if (!this._ready) {
+      this._ready = this._build();
+    }
+    return this._ready;
+  }
+
+  async _dataReady() {
+    await this.ready();
+    if (this.hierarchical) {
+      return this._allFetched || (this._allFetched = this._fetchAll());
+    }
+  }
+
+  async _fetchAll() {
+    const records = await this._entities.getAll();
+    for (const record of records) {
+      this._index.set(record.id, record);
+    }
+    if (this.hierarchical) {
+      shimFullPath(records, this._index);
+    }
+  }
+
+  async _build() {
+    const taxonomy = this._taxonomy = await this._entities._taxonomy();
+    this.hierarchical = !!(taxonomy && taxonomy.hierarchical);
+  }
+
+  async fetch(ids) {
+    if (this.hierarchical) {
+      return; // already all fetched
+    }
+    ids = asArray(ids);
+
+    const promises = []
+    const idsToFetch = [];
+    for (const id of ids) {
+      if (this._index.has(id) || this._notFound.has(id)) {
+        continue;
+      }
+      if (!this._fetching.has(id)) {
+        this._fetching.set(id, new Resolution());
+        idsToFetch.push(id);
+      }
+      promises.push(this._fetching.get(id).promise);
+    }
+    if (idsToFetch.length > 0) {
+      (async () => {
+        const idsFetchSet = new Set(idsToFetch);
+        const stream = await this._entities.stream({ ids: idsToFetch });
+        for await (const entity of stream) {
+           const { id } = entity;
+           this._index.set(id, this._process(entity));
+           idsFetchSet.delete(id);
+           this._resolveFetch(id);
+        }
+        // handle unavailable ones
+        for (const id of idsFetchSet) {
+          this._notFound.add(id);
+          this._resolveFetch(id);
+        }
+      })();
+    }
+    return Promise.all(promises);
+  }
+
+  _resolveFetch(id) {
+    this._fetching.get(id).resolve();
+    this._fetching.delete(id);
+  }
+
+  async get(id) {
+    await this._dataReady();
+    await this.fetch([id]);
+    return this._index.get(id);
+  }
+
+  async getAll(ids) {
+    await this._dataReady();
+    await this.fetch(ids);
+    return ids.map(id => this._index.get(id));
+  }
+
+  async getValue(id) {
+    if (id === undefined) {
+      return undefined;
+    }
+    return this._value(await this.get(id));
+  }
+
+  async getValues(ids = []) {
+    if (ids.length === 0) {
+      return [];
+    }
+    const entities = await this.getAll(ids);
+    return entities.map(en => this._value(en)).filter(v => v);
+  }
+
+  _process(entity) {
+    return entity;
+  }
+
+  _value(entity) {
+    return entity && (this.hierarchical ? entity.fullPath.names : entity.name);
+  }
+
+  async patch(post, propName) {
+    propName = propName || this.name;
+    const { [propName]: ids } = post;
+    const value = await (Array.isArray(ids) ? this.getValues(ids) : this.getValue(ids));
+    return value ? { [propName]: value } : undefined;
+  }
+
+}
+
+function shimFullPath(entities, cache) {
+  // DP to compute full path
+  function fullPath(entity) {
+    if (!entity.fullPath) {
+      const { parent, id, name } = entity;
+      if (parent) {
+        const { ids, names } = fullPath(cache.get(parent));
+        entity.fullPath = {
+          ids: [...ids, id],
+          names: [...names, name],
+        };
+      } else {
+        entity.fullPath = {
+          ids: [id],
+          names: [name],
+        };
+      }
+    }
+    return entity.fullPath;
+  }
+
+  return entities.map(c => {
+    fullPath(c);
+    return c;
+  });
+}

package/src/entities/index.js

@@ -0,0 +1,95 @@
+import { asArray, stream } from '@miso.ai/server-commons';
+import EntityIndex from './entity-index.js';
+import EntityTransformStream from './transform.js';
+import defaultTransform from './transform-default.js';
+import legacyTransform from './transform-legacy.js';
+
+export default class Entities {
+
+  constructor(client, name) {
+    this._client = client;
+    this.name = name;
+    this._index = this._createIndex();
+    Object.freeze(this);
+  }
+  
+  async stream({ resolve = false, transform, ...options } = {}) {
+    if (!resolve && !transform) {
+      return this._client._helpers.stream(this.name, options);
+    }
+    transform = getTransformFn(transform);
+
+    const client = this._client;
+
+    // we need taxonomy fetched so we know whether it's hierarchical
+    const taxonomies = await client._helpers.findAssociatedTaxonomies(this.name);
+
+    // prepare entity indicies
+    const indicies = [
+      client.users.index,
+      client.media.index,
+      ...taxonomies.map(({ rest_base }) => client.entities(rest_base).index),
+    ];
+    await Promise.all(indicies.map(index => index.ready()));
+    for (const index of indicies) {
+      if (index.hierarchical) {
+        index._dataReady(); // kick off fetch all, but don't wait
+      }
+    }
+
+    // onLoad function, which let us fill index caches more efficiently
+    const onLoad = records => {
+      for (const index of indicies) {
+        if (!index.hierarchical) {
+          index.fetch(aggregateIds(records, EntityTransformStream.getPropName(index.name)));
+        }
+      }
+    };
+
+    // transform stream
+    const transformStream = new EntityTransformStream(this._client, indicies, { transform });
+
+    return (await this._client._helpers.stream(this.name, { ...options, onLoad }))
+      .pipe(transformStream);
+  }
+
+  async getAll(options) {
+    return stream.collect(await this.stream(options));
+  }
+
+  async count(options) {
+    return this._client._helpers.count(this.name, options);
+  }
+
+  async terms(options) {
+    return this._client._helpers.terms(this.name, options);
+  }
+
+  get index() {
+    return this._index;
+  }
+
+  _createIndex() {
+    return new EntityIndex(this);
+  }
+
+  async _taxonomy(options) {
+    return await this._client._helpers.findTaxonomyByResourceName(this.name, options);
+  }
+
+}
+
+function aggregateIds(records, propName) {
+  return Array.from(records.reduce((idSet, record) => {
+    for (const id of asArray(record[propName])) {
+      idSet.add(id);
+    }
+    return idSet;
+  }, new Set()));
+}
+
+function getTransformFn(transform) {
+  return typeof transform === 'function' ? post => transform(post, { defaultTransform }) :
+    (transform === true || transform === 'default') ? defaultTransform :
+    transform === 'legacy' ? legacyTransform : undefined;
+}

package/src/entities/transform-default.js

@@ -0,0 +1,63 @@
+import { trimObj, asArray } from '@miso.ai/server-commons';
+
+export default function transform({
+  _linked: {
+    author,
+    featured_media,
+    categories,
+    tags,
+    //..._linked
+  } = {},
+  id,
+  type,
+  date_gmt,
+  modified_gmt,
+  guid: {
+    rendered: guid,
+  },
+  slug,
+  title: {
+    rendered: title,
+  },
+  content: {
+    rendered: html,
+  },
+  link: url,
+  status,
+  sticky,
+  comment_status,
+  ping_status,
+}) {
+  const product_id = `${id}`;
+  if (!product_id) {
+    throw new Error(`ID is absent.`);
+  }
+  const created_at = date_gmt && `${date_gmt}Z`;
+  const updated_at = modified_gmt && `${modified_gmt}Z`;
+
+  const authors = asArray(author);
+  const cover_image = featured_media && encodeURI(featured_media);
+
+  return trimObj({
+    product_id,
+    type,
+    created_at,
+    updated_at,
+    title,
+    cover_image,
+    html,
+    url,
+    authors,
+    categories,
+    tags,
+    custom_attributes: trimObj({
+      guid,
+      slug,
+      status,
+      sticky,
+      comment_status,
+      ping_status,
+      //..._linked,
+    }),
+  });
+}

package/src/entities/transform-legacy.js

@@ -0,0 +1,63 @@
+import { trimObj, asArray } from '@miso.ai/server-commons';
+
+export default function transform({
+  _linked = {},
+  id,
+  type,
+  date_gmt,
+  modified_gmt,
+  title: {
+    rendered: title,
+  },
+  content: {
+    rendered: html,
+  },
+  author: authorId,
+  categories: categoryIds,
+  tags_names: tags,
+  link: url,
+  better_featured_image,
+  status,
+  sticky,
+  comment_status,
+  ping_status,
+  format,
+  yoast,
+}) {
+  const product_id = `${id}`;
+  if (!product_id) {
+    throw new Error(`ID is absent.`);
+  }
+  const created_at = date_gmt && `${date_gmt}Z`;
+  const updated_at = modified_gmt && `${modified_gmt}Z`;
+  const authors = asArray(_linked.author);
+  const categories = _linked.categories;
+
+  // TODO: ad-hoc
+  const description = yoast && yoast.metadesc;
+  const cover_image = better_featured_image && better_featured_image.source_url && encodeURI(better_featured_image.source_url);
+
+  return trimObj({
+    product_id,
+    type,
+    created_at,
+    updated_at,
+    title,
+    description,
+    html,
+    cover_image,
+    url,
+    authors,
+    categories,
+    tags,
+    custom_attributes: trimObj({
+      status,
+      sticky,
+      comment_status,
+      ping_status,
+      format,
+      authorId,
+      categoryIds,
+    }),
+  });
+}

package/src/entities/transform.js

@@ -0,0 +1,57 @@
+import { Transform } from 'stream';
+
+function applyPatch(post, patch) {
+  if (!patch) {
+    return post;
+  }
+  return {
+    ...post,
+    _linked: {
+      ...post._linked,
+      ...patch,
+    },
+  };
+}
+
+const PROP_NAME_OVERRIDES = {
+  users: 'author',
+  media: 'featured_media',
+};
+
+export default class EntityTransformStream extends Transform {
+
+  static getPropName(resource) {
+    return PROP_NAME_OVERRIDES[resource] || resource;
+  }
+
+  constructor(client, indicies, { transform } = {}) {
+    super({ objectMode: true });
+    this._client = client;
+    this._indicies = indicies;
+    this._transformFn = transform;
+  }
+
+  async _transform(post, _, next) {
+    try {
+      post = await this._resolveLinkedEntities(post);
+      if (this._transformFn) {
+        post = await this._transformFn(post);
+      }
+      next(undefined, post);
+    } catch (error) {
+      next(error);
+    }
+  }
+
+  async _resolveLinkedEntities(post) {
+    const patches = await Promise.all(
+      this._indicies.map(index => 
+        index.patch(post, EntityTransformStream.getPropName(index.name)))
+    );
+    for (const patch of patches) {
+      post = applyPatch(post, patch);
+    }
+    return post;
+  }
+
+}

package/src/helpers.js

@@ -0,0 +1,173 @@
+import { asNumber, splitObj, stream } from '@miso.ai/server-commons';
+import axios from './axios.js';
+import DataSource from './source/index.js';
+
+const MS_PER_HOUR = 1000 * 60 * 60;
+
+const STREAM_OPTIONS = ['offset', 'limit', 'strategy', 'filter', 'transform', 'onLoad'];
+
+export default class Helpers {
+
+  constructor(client) {
+    this._start = Date.now();
+    this._client = client;
+    this.url = new Url(this);
+    this._samples = {};
+    this.debug = this.debug.bind(this);
+  }
+
+  async stream(resource, options) {
+    const [streamOptions, sourceOptions] = splitObj(options, STREAM_OPTIONS);
+    const source = new DataSource(this, resource, sourceOptions);
+    return new stream.BufferedReadStream(source, { ...streamOptions, debug: this.debug });
+  }
+
+  async sample(resource, { noCache = false } = {}) {
+    if (noCache || !this._samples[resource]) {
+      // don't await, save the promise
+      this._samples[resource] = this._fetchSample(resource);
+    }
+    return this._samples[resource];
+  }
+
+  async _fetchSample(resource) {
+    const url = await this.url.build(resource, { page: 0, pageSize: 1 });
+    const { data, headers } = await axios.get(url);
+    if (!data.length) {
+      throw new Error(`No record of ${resource} avaliable`);
+    }
+    this.debug(`Fetched ${resource} sample, total = ${asNumber(headers['x-wp-total'])}`);
+
+    return {
+      data: data[0],
+      headers,
+      terms: this.extractTerms(data[0]),
+    };
+  }
+
+  async findTaxonomyByResourceName(name, options) {
+    const taxonomies = await this.taxonomies(options);
+    for (const taxonomy of taxonomies) {
+      if (taxonomy.rest_base === name) {
+        return taxonomy;
+      }
+    }
+    return undefined;
+  }
+
+  async findAssociatedTaxonomies(type, options) {
+    // TODO: try using terms()
+    type = type === 'posts' ? 'post' : type;
+    return (await this.taxonomies(options)).filter(taxonomy => taxonomy.types.includes(type));
+  }
+
+  async taxonomies({ noCache = false } = {}) {
+    if (noCache || !this._taxonomies) {
+      // don't await, save the promise
+      this._taxonomies = this._fetchTaxonomies();
+    }
+    return this._taxonomies;
+  }
+
+  async _fetchTaxonomies() {
+    const url = await this.url.build('taxonomies');
+    const { data } = await axios.get(url);
+    this.debug(`Fetched taxonomies.`);
+    return Object.values(data);
+  }
+
+  extractTerms(data) {
+    return data._links['wp:term'] || [];
+  }
+
+  async count(resource, { offset: _, ...options } = {}) {
+    const url = await this.url.build(resource, { ...options, page: 0, pageSize: 1 });
+    const { headers } = await axios.get(url);
+    return asNumber(headers['x-wp-total']);
+  }
+
+  async terms(resource, { noCache = false } = {}) {
+    return (await this.sample(resource, { noCache })).terms;
+  }
+
+  async countUrl(url) {
+    url = await this.url.append(url, { page: 0, pageSize: 1 });
+    const { headers } = await axios.get(url);
+    return asNumber(headers['x-wp-total']);
+  }
+
+  /**
+   * Return the UTC offset in milliseconds for this site.
+   */
+  async utcOffsetInMs() {
+    const profile = this._client._profile;
+    if (profile.utcOffset === undefined) {
+      // we may call this multiple times but it's ok
+      const { data: post } = await this.sample('posts');
+      profile.utcOffset = (Date.parse(`${post.date}Z`) - Date.parse(`${post.date_gmt}Z`)) / MS_PER_HOUR;
+      this.debug(`Got utcOffset: ${profile.utcOffset}`);
+    }
+    return profile.utcOffset * MS_PER_HOUR;
+  }
+
+  removeLinks({ _links: _, ...record }) {
+    return record;
+  }
+
+  debug(v) {
+    const elapsed = (Date.now() - this._start) / 1000;
+    this._client._options.debug && console.error(`[${elapsed}]`, v);
+  }
+
+}
+
+class Url {
+
+  constructor(helpers) {
+    this._helpers = helpers;
+  }
+
+  async build(resource, options) {
+    return this.append(`https://${this._helpers._client.site}/wp-json/wp/v2/${resource}`, options);
+  }
+
+  // modifiedAfter, modifiedBefore is supported since WordPress 5.7
+  // https://make.wordpress.org/core/2021/02/23/rest-api-changes-in-wordpress-5-7/
+  async append(url, options = {}) {
+    const { after, before, order, orderBy, page, pageSize, offset, include, exclude } = options;
+    const params = [];
+
+    // TODO: support single id
+
+    // The date is compared against site's local time, not UTC, so we have to work on timezone offset
+    if (has(after) || has(before)) {
+      const utcOffset = await this._helpers.utcOffsetInMs();
+      has(after) && params.push(`after=${toISOString(after, utcOffset)}`);
+      has(before) && params.push(`before=${toISOString(before, utcOffset)}`);
+    }
+
+    has(order) && params.push(`order=${order}`);
+    has(orderBy) && params.push(`orderby=${orderBy}`);
+    has(page) && params.push(`page=${page + 1}`); // 0-based to 1-based
+    has(pageSize) && params.push(`per_page=${pageSize}`);
+    has(offset) && params.push(`offset=${offset}`);
+    has(include) && include.length && params.push(`include=${joinIds(include)}`);
+    has(exclude) && exclude.length && params.push(`exclude=${joinIds(exclude)}`);
+
+    const head = params.length === 0 ? '' : url.indexOf('?') < 0 ? '?' : '&';
+    return `${url}${head}${params.join('&')}`;
+  }
+
+}
+
+function joinIds(ids) {
+  return ids.map(id => encodeURIComponent(`${id}`)).join(',');
+}
+
+function has(value) {
+  return value !== undefined;
+}
+
+function toISOString(ts, utcOffset) {
+  return new Date(ts + utcOffset).toISOString();
+}

package/src/index.js

@@ -0,0 +1 @@
+export { default as WordPressClient } from './client.js';

package/src/media.js

@@ -0,0 +1,26 @@
+import Entities from './entities/index.js';
+import EntityIndex from './entities/entity-index.js';
+
+const RESOURCE_NAME = 'media';
+
+export default class Media extends Entities {
+
+  constructor(client) {
+    super(client, RESOURCE_NAME);
+  }
+  
+  _createIndex() {
+    return new EntityIndex(this, {
+      process,
+      value: en => en.source_url,
+    });
+  }
+
+}
+
+/*
+ * We don't keep track of everything, for media entity is generally too large.
+ */
+function process({ id, date, date_gmt, alt_text, media_details: { width, height, filesize }, source_url }) {
+  return { id, date, date_gmt, alt_text, width, height, filesize, source_url };
+}

package/src/posts/index.js

@@ -0,0 +1,21 @@
+import Entities from '../entities/index.js';
+
+const RESOURCE_NAME = 'posts';
+
+// TODO: classify post-like entities
+
+export default class Posts extends Entities {
+
+  constructor(client) {
+    super(client, RESOURCE_NAME);
+  }
+  
+  async getAll() {
+    throw new Error(`Getting all posts is not supported.`);
+  }
+
+  async index() {
+    throw new Error(`Indexing posts is not supported.`);
+  }
+
+}

package/src/source/base.js

@@ -0,0 +1,74 @@
+import axios from '../axios.js';
+
+export default class WordPressDataSource {
+
+  constructor(helpers, resource, options = {}) {
+    this._helpers = helpers;
+    this._resource = resource;
+    this._options = options;
+    this._debug(`[WordPressDataSource] constructed for resource: ${resource}.`);
+  }
+
+  request() {
+    throw new Error(`Unimplemented`);
+  }
+
+  async url(request) {
+    const baseUrl = await this.baseUrl();
+    return this._url(baseUrl, request);
+  }
+
+  async baseUrl() {
+    return this._baseUrl || (this._baseUrl = this._buildBaseUrl());
+  }
+
+  async total() {
+    throw new Error(`Unimplemented`);
+  }
+
+  async get(request) {
+    this._debug(`[WordPressDataSource] get ${JSON.stringify(request)}`);
+    const url = await this.url(request);
+    this._debug(`[WordPressDataSource] request ${url}`);
+    const response =  await this._axiosGet(url);
+    this._debug(`[WordPressDataSource] response ${response.status} ${url}`);
+    return this._process(response);
+  }
+
+  _process({ status, data }) {
+    if (status >= 400 && status < 500 && data.code === 'rest_post_invalid_page_number') {
+      // out of bound, so there is no more data
+      return { data: [], terminate: true };
+    }
+    if (!this._options.preserveLinks) {
+      data = data.map(this._helpers.removeLinks);
+    }
+    return { data };
+  }
+
+  async _url(baseUrl, request) {
+    throw new Error(`Unimplemented`);
+  }
+
+  async _buildBaseUrl() {
+    // exclude parameters meant to be dealt with state
+    const { page, ...options } = this._options;
+    return this._helpers.url.build(this._resource, options);
+  }
+
+  async _axiosGet(url) {
+    try {
+      return await axios.get(url);
+    } catch(error) {
+      if (error.response) {
+        return error.response;
+      }
+      throw error;
+    }
+  }
+
+  _debug(...args) {
+    this._helpers.debug(...args);
+  }
+
+}

package/src/source/ids.js

@@ -0,0 +1,37 @@
+import WordPressDataSource from './base.js';
+
+const PAGE_SIZE = 100;
+
+export default class IdsWordPressDataSource extends WordPressDataSource {
+
+  constructor(helpers, resource, { ids, page: _, limit, ...options } = {}) {
+    super(helpers, resource, options);
+    if (!Array.isArray(ids)) {
+      throw new Error(`ids must be an array: ${ids}`);
+    }
+    this._pageSize = options.pageSize = PAGE_SIZE;
+    this._ids = ids;
+    this._page = 0;
+  }
+
+  request() {
+    const pageSize = this._pageSize;
+    const page = this._page++;
+    const start = pageSize * page;
+    const end = start + pageSize;
+    const ids = this._ids.slice(start, end);
+    const records = ids.length;
+    const exhaust = end >= this._ids.length;
+    return exhaust ? { ids, records, exhaust } : { ids, records };
+  }
+
+  async _url(baseUrl, { ids }) {
+    const head = baseUrl.indexOf('?') < 0 ? '?' : '&';
+    return `${baseUrl}${head}include=${joinIds(ids)}`;
+  }
+
+}
+
+function joinIds(ids) {
+  return ids.map(id => encodeURIComponent(`${id}`)).join(',');
+}

package/src/source/index.js

@@ -0,0 +1,7 @@
+import PagedWordPressDataSource from './paged.js';
+import IdsWordPressDataSource from './ids.js';
+
+export default function source(helpers, resource, options) {
+  const DataSource = options.ids ? IdsWordPressDataSource : PagedWordPressDataSource;
+  return new DataSource(helpers, resource, options);
+}

package/src/source/paged.js

@@ -0,0 +1,65 @@
+import { asNumber } from '@miso.ai/server-commons';
+import WordPressDataSource from './base.js';
+
+const MAX_PAGE_SIZE = 100;
+
+export default class PagedWordPressDataSource extends WordPressDataSource {
+
+  constructor(helpers, resource, { page: _, limit, ...options } = {}) {
+    super(helpers, resource, options);
+    const { pageSize = MAX_PAGE_SIZE } = options;
+    if (pageSize <= 0) {
+      throw new Error(`Page size must be positive: ${pageSize}`);
+    }
+    if (pageSize > MAX_PAGE_SIZE) {
+      throw new Error(`Page size cannot be greater than ${MAX_PAGE_SIZE}: ${pageSize}`);
+    }
+    // TODO: limit
+    this._pageSize = options.pageSize = pageSize;
+    this._page = 0;
+  }
+
+  async init() {
+    await this.total();
+  }
+
+  request() {
+    const page = this._page++;
+    const records = this._pageSize;
+    const total = this._totalValue;
+    // if we know total, we know when the data is exhausted
+    const exhaust = total !== undefined && ((page + 1) * this._pageSize > total + 10); // 10 for a buffer
+    return exhaust ? { records, page, exhaust } : { records, page };
+  }
+
+  async total() {
+    return this._totalPromise || (this._totalPromise = this._fetchTotal());
+  }
+
+  async _url(baseUrl, { page }) {
+    const head = baseUrl.indexOf('?') < 0 ? '?' : '&';
+    return `${baseUrl}${head}page=${page + 1}`;
+  }
+
+  async _fetchTotal() {
+    const baseUrl = await this.baseUrl();
+    this._debug(`[PagedWordPressDataSource] fetch total for ${baseUrl}`);
+    const total = await this._helpers.countUrl(baseUrl);
+    this._debug(`[PagedWordPressDataSource] fetch total for ${baseUrl} = ${total}`);
+    this._totalValue = total;
+    return total;
+  }
+
+  _process({ status, data, headers }) {
+    const result = super._process({ status, data, headers });
+    const total = asNumber(headers['x-wp-total']);
+    if (total !== undefined) {
+      result.total = total;
+    }
+    if (data.length < this._pageSize) {
+      result.terminate = true;
+    }
+    return result;
+  }
+
+}

package/src/version.js

@@ -0,0 +1 @@
+export default '0.6.0-beta.1';