@miso.ai/server-wordpress 0.6.3-beta.2 → 0.6.3-beta.20

package/cli/download.js

@@ -0,0 +1,116 @@
+import { createWriteStream } from 'fs';
+import { access, mkdir } from 'fs/promises';
+import { createGzip } from 'zlib';
+import { startOfDate, endOfDate, stream } from '@miso.ai/server-commons';
+import { WordPressClient } from '../src/index.js';
+import { buildForEntities } from './utils.js';
+
+function build(yargs) {
+  return buildForEntities(yargs);
+}
+
+async function run({
+  destination = './data',
+  batchSize = 30000,
+  ...options
+} = {}) {
+  const client = new WordPressClient(options);
+
+  // TODO: respect --after and --before
+  const [firstPostYear, lastPostYear] = await client.posts.yearRange();
+
+  // divide into batches
+  const batches = [];
+  let endYear, sum = 0;
+  for (let year = lastPostYear; year >= firstPostYear; year--) {
+    if (endYear === undefined) {
+      endYear = year;
+    }
+    const after = startOfDate(year);
+    const before = endOfDate(year);
+    const count = await client.posts.count({ ...options, after, before });
+    sum += count;
+    if (sum >= batchSize) {
+      batches.push({ start: year, end: endYear, records: sum });
+      endYear = undefined;
+      sum = 0;
+    }
+  }
+  if (sum > 0) {
+    batches.push({ start: firstPostYear, end: endYear, records: sum });
+  }
+  const batchCount = batches.length;
+  if (batchCount === 0) {
+    console.log('No posts found.');
+    return;
+  }
+  // merge last two batches if last batch is too small
+  if (batches[batchCount - 1].records < batchSize * 0.2) {
+    const last = batches.pop();
+    batches[batchCount - 2].end = last.end;
+    batches[batchCount - 2].records += last.records;
+  }
+  console.log(`Divide into ${batchCount} batches:`);
+  for (const { start, end, records } of batches) {
+    console.log(`- ${start} -> ${end} (${records} records)`);
+  }
+
+  // mkdir -p
+  try {
+    await access(destination);
+  } catch (err) {
+    if (err.code !== 'ENOENT') {
+      throw err;
+    }
+    await mkdir(destination, { recursive: true });
+    console.log(`Created directory ${destination}`);
+  }
+
+  // download
+  let index = 0;
+  for (const batch of batches) {
+    const { start, end, records } = batch;
+    const after = startOfDate(start);
+    const before = endOfDate(end);
+    const filename = `${options.transform ? 'miso' : 'wp'}-posts.${start}-${end}.jsonl.gz`;
+
+    console.log(`[${index + 1} / ${batchCount}] Downloading ${filename}`);
+
+    const startTime = Date.now();
+    const sourceStream = await client.posts.stream({ ...options, after, before });
+
+    await stream.pipeline(
+      sourceStream,
+      stream.stringify(),
+      createGzip(),
+      createWriteStream(`${destination}/${filename}`),
+    );
+
+    const elapsed = Date.now() - startTime;
+    console.log(`[${index + 1} / ${batchCount}] Downloaded ${filename} (${records} records in ${formatDuration(elapsed)})`);
+    index++;
+  }
+
+  console.log('Done.');
+}
+
+function formatDuration(duration) {
+  const seconds = Math.floor(duration / 1000);
+  const minutes = Math.floor(seconds / 60);
+  const hours = Math.floor(minutes / 60);
+  if (hours > 0) {
+    return `${hours}h${minutes % 60}m${seconds % 60}s`;
+  }
+  if (minutes > 0) {
+    return `${minutes}m${seconds % 60}s`;
+  }
+  return `${seconds}s`;
+}
+
+export default {
+  command: 'download',
+  aliases: ['down'],
+  desc: 'Download all posts and save as files.',
+  builder: build,
+  handler: run,
+};

package/cli/entities.js

@@ -1,62 +1,9 @@
 import { Transform } from 'stream';
+import  { pipeline } from 'stream/promises';
+import split2 from 'split2';
 import { stream, parseDuration } from '@miso.ai/server-commons';
 import { WordPressClient } from '../src/index.js';
-import { normalizeOptions, normalizeTransform, parseDate } from './utils.js';
-
-export function buildForEntities(yargs) {
-  // TODO: make them mutually exclusive
-  return yargs
-    .option('terms', {
-      describe: 'Display terms associated with this type of resource',
-      type: 'boolean',
-    })
-    .option('count', {
-      alias: 'c',
-      describe: 'Return the total number of records',
-      type: 'boolean',
-    })
-    .option('date', {
-      alias: 'd',
-      describe: 'Only include records in this year/month/day',
-    })
-    .option('after', {
-      alias: 'a',
-      describe: 'Only include records after this time',
-    })
-    .option('before', {
-      alias: 'b',
-      describe: 'Only include records before this time',
-    })
-    .option('update', {
-      alias: 'u',
-      describe: 'Only include records modified in given duration (3h, 2d, etc.)',
-    })
-    .option('ids', {
-      alias: 'include',
-      describe: 'Specify post ids'
-    })
-    .option('fields', {
-      describe: 'Specify which record fields are retrieved',
-      type: 'array',
-      coerce: yargs.coerceToArray,
-    })
-    .option('resolve', {
-      alias: 'r',
-      describe: 'Attach resolved entities (author, catagories) linked with the subjects',
-      type: 'boolean',
-    })
-    .option('transform', {
-      alias: 't',
-      describe: 'Apply transform function to the entities',
-    });
-    /*
-    .option('limit', {
-      alias: 'n',
-      describe: 'Limit the amount of records',
-      type: 'number',
-    })
-    */
-}
+import { normalizeOptions, buildForEntities } from './utils.js';
 
 function build(yargs) {
   return buildForEntities(yargs)
@@ -76,6 +23,12 @@ async function run({ subcmd, count, terms, update, name, ...options }) {
     case 'count':
       await runCount(client, name, options);
       return;
+    case 'absence':
+      await runPresence(client, name, { present: false });
+      return;
+    case 'presence':
+      await runPresence(client, name, { present: true });
+      return;
   }
   if (count) {
     await runCount(client, name, options);
@@ -99,20 +52,17 @@ export async function runTerms(client, name, options) {
   }
 }
 
-export async function runGet(client, name, { transform, ...options }) {
-  await stream.pipelineToStdout(
-    await client.entities(name).stream({
-      ...options,
-      transform: await normalizeTransform(transform),
-    }),
-    stream.stringify(),
+export async function runGet(client, name, options) {
+  await pipeline(
+    await client.entities(name).stream(options),
+    new stream.OutputStream(),
   );
 }
 
 export async function runIds(client, name, { update, transform, resolve, fields, ...options }) {
   if (update) {
     await stream.pipeline(
-      await buildUpdateStream(client, name, update, { ...options, fields: ['id', 'modified_gmt'] }),
+      await buildUpdateStream(client, name, update, { ...options, fields: ['id'] }),
       new Transform({
         objectMode: true,
         transform({ id }, _, callback) {
@@ -136,13 +86,23 @@ export async function runUpdate(client, name, update, options) {
   );
 }
 
+export async function runPresence(client, name, options) {
+  await stream.pipeline(
+    process.stdin,
+    split2(),
+    client.entities(name).presence(options),
+    new stream.OutputStream({
+      objectMode: false,
+    }),
+  );
+}
+
 async function buildUpdateStream(client, name, update, {
   date, after, before, orderBy, order, // strip off date filters and order criteria
   transform,
   ...options
 }) {
   // TODO: move the logic into client itself
-  transform = await normalizeTransform(transform);
   const now = Date.now();
   update = parseDuration(update);
   const threshold = now - update;
@@ -156,6 +116,14 @@ async function buildUpdateStream(client, name, update, {
         after: threshold,
       }),
       // get recent modified, excluding ones already fetched
+      entities.stream({
+        ...options,
+        transform,
+        orderBy: 'modified',
+        modifiedAfter: threshold,
+        before: threshold,
+      }),
+      /*
       entities.stream({
         ...options,
         transform,
@@ -168,6 +136,7 @@ async function buildUpdateStream(client, name, update, {
           terminate: entity => parseDate(entity.modified_gmt) < threshold,
         },
       })
+      */
     ])
   );
 }

package/cli/index.js

@@ -4,6 +4,8 @@ import version from '../src/version.js';
 import { profile, init } from './profile.js';
 import taxonomies from './taxonomies.js';
 import entities from './entities.js';
+import summarize from './summarize.js';
+import download from './download.js';
 
 yargs.build(yargs => {
   yargs
@@ -16,6 +18,9 @@ yargs.build(yargs => {
       alias: 'p',
       describe: 'Site profile file location',
     })
+    .option('auth', {
+      describe: 'Authentication string',
+    })
     .option('debug', {
       type: 'boolean',
       default: false,
@@ -23,6 +28,8 @@ yargs.build(yargs => {
     .hide('debug')
     .command(init)
     .command(profile)
+    .command(summarize)
+    .command(download)
     .command(taxonomies)
     .command(entities)
     .version(version);

package/cli/summarize.js

@@ -0,0 +1,57 @@
+import { startOfDate, endOfDate, getYear } from '@miso.ai/server-commons';
+import { WordPressClient } from '../src/index.js';
+
+function build(yargs) {
+  return yargs;
+}
+
+async function run({ ...options } = {}) {
+  const client = new WordPressClient(options);
+  const [total, [firstPostDate, lastPostDate]] = await Promise.all([
+    client.posts.count(options),
+    client.posts.dateRange(),
+  ]);
+  const totalStrLength = `${total}`.length;
+  console.log();
+  console.log(`Total posts: ${total}`);
+  console.log(`First post at: ${firstPostDate}`);
+  console.log(`Last post at: ${lastPostDate}`);
+
+  // drill down by year
+  console.log();
+  const bar = `| ---- | ${'-'.repeat(totalStrLength)} |`;
+  console.log(bar);
+  console.log(`| Year | ${'Posts'.padStart(totalStrLength)} |`);
+  console.log(bar);
+  for (let year = getYear(firstPostDate), lastYear = getYear(lastPostDate); year <= lastYear; year++) {
+    const after = startOfDate(year);
+    const before = endOfDate(year);
+    const count = await client.posts.count({ ...options, after, before });
+    console.log(`| ${year} | ${`${count}`.padStart(totalStrLength)} |`);
+  }
+  console.log(bar);
+}
+
+function printTable(arr) {
+  arr = arr.map((row) => row.map(str));
+  const colWidths = arr[0].map((_, i) => Math.max(...arr.map((row) => (row[i] || '').length)));
+  for (const row of arr) {
+    console.log(row.map((v, i) => rightPad(v, colWidths[i])).join(' '));
+  }
+}
+
+function str(value) {
+  return value === undefined ? '--' : `${value}`;
+}
+
+function rightPad(str = '', length) {
+  return str.padEnd(length);
+}
+
+export default {
+  command: 'summarize',
+  aliases: ['sum'],
+  desc: 'Print out a summary of the WordPress site',
+  builder: build,
+  handler: run,
+};

package/cli/utils.js

@@ -1,24 +1,68 @@
-import { join } from 'path';
 import { startOfDate, endOfDate } from '@miso.ai/server-commons';
 
-const PWD = process.env.PWD;
-
-export function normalizeOptions({ date, after, before, ids, ...options }) {
+export function normalizeOptions({ date, after, before, ids, include, ...options }) {
+  // TODO: should be able to turn this off, as it's covered by helper
   [after, before] = [startOfDate(date || after), endOfDate(date || before)];
+  // TODO: rely on yargs to coerce to array
   ids = ids ? `${ids}`.split(',').map(s => s.trim()) : ids;
   return { ...options, after, before, ids };
 }
 
-export async function normalizeTransform(transform) {
-  if (typeof transform === 'string') {
-    if (transform === 'default' || transform === 'legacy') {
-      return transform;
-    }
-    return (await import(join(PWD, transform))).default;
-  }
-  return !!transform;
-}
-
+/*
 export function parseDate(value) {
   return Date.parse(`${value}Z`);
 }
+*/
+
+export function buildForEntities(yargs) {
+  // TODO: make them mutually exclusive
+  return yargs
+    .option('terms', {
+      describe: 'Display terms associated with this type of resource',
+      type: 'boolean',
+    })
+    .option('count', {
+      alias: 'c',
+      describe: 'Return the total number of records',
+      type: 'boolean',
+    })
+    .option('date', {
+      alias: 'd',
+      describe: 'Only include records in this year/month/day',
+    })
+    .option('after', {
+      alias: 'a',
+      describe: 'Only include records after this time',
+    })
+    .option('before', {
+      alias: 'b',
+      describe: 'Only include records before this time',
+    })
+    .option('update', {
+      alias: 'u',
+      describe: 'Only include records modified in given duration (3h, 2d, etc.)',
+    })
+    .option('ids', {
+      alias: 'include',
+      describe: 'Specify post ids',
+    })
+    .option('fields', {
+      describe: 'Specify which record fields are retrieved',
+      type: 'array',
+      coerce: yargs.coerceToArray,
+    })
+    .option('resolve', {
+      alias: 'r',
+      describe: 'Attach resolved entities (author, catagories) linked with the subjects',
+      type: 'boolean',
+    })
+    .option('transform', {
+      alias: 't',
+      describe: 'Apply transform function to the entities',
+    })
+    .option('limit', {
+      alias: 'n',
+      describe: 'Limit the amount of records',
+      type: 'number',
+    });
+}

package/package.json

@@ -17,9 +17,9 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.3-beta.2",
-    "axios": "^0.27.2",
+    "@miso.ai/server-commons": "0.6.3-beta.20",
+    "axios": "^1.6.2",
     "axios-retry": "^3.3.1"
   },
-  "version": "0.6.3-beta.2"
+  "version": "0.6.3-beta.20"
 }

package/src/client.js

@@ -70,7 +70,7 @@ export default class WordPressClient {
 
 }
 
-const SITE_PROFILE_PROPS = ['site', 'utcOffset'];
+const SITE_PROFILE_PROPS = ['site', 'utcOffset', 'resources', 'defaults'];
 
 class SiteProfile {
 

package/src/entities/entity-index.js

@@ -2,7 +2,7 @@ import { asArray, Resolution } from '@miso.ai/server-commons';
 
 export default class EntityIndex {
 
-  constructor(entities, { process, value } = {}) {
+  constructor(entities, { process, value, fields } = {}) {
     this._entities = entities;
     if (process) {
       this._process = process;
@@ -10,6 +10,7 @@ export default class EntityIndex {
     if (value) {
       this._value = (en => en && value(en)); // null-safe
     }
+    this._fields = fields;
     this.name = entities.name;
     this._index = new Map();
     this._notFound = new Set();
@@ -49,7 +50,7 @@ export default class EntityIndex {
     if (this.hierarchical) {
       return; // already all fetched
     }
-    ids = asArray(ids);
+    ids = asArray(ids).filter(id => id); // discard 0, null, undefined
 
     const promises = []
     const idsToFetch = [];
@@ -66,7 +67,7 @@ export default class EntityIndex {
     if (idsToFetch.length > 0) {
       (async () => {
         const idsFetchSet = new Set(idsToFetch);
-        const stream = await this._entities.stream({ ids: idsToFetch });
+        const stream = await this._entities.stream({ ids: idsToFetch, fields: this._fields });
         for await (const entity of stream) {
            const { id } = entity;
            this._index.set(id, this._process(entity));
@@ -84,8 +85,11 @@ export default class EntityIndex {
   }
 
   _resolveFetch(id) {
-    this._fetching.get(id).resolve();
-    this._fetching.delete(id);
+    const res = this._fetching.get(id);
+    if (res) {
+      res.resolve();
+      this._fetching.delete(id);
+    }
   }
 
   async get(id) {
@@ -95,13 +99,14 @@ export default class EntityIndex {
   }
 
   async getAll(ids) {
+    ids = ids.filter(id => id); // discard 0, null, undefined
     await this._dataReady();
     await this.fetch(ids);
     return ids.map(id => this._index.get(id));
   }
 
   async getValue(id) {
-    if (id === undefined) {
+    if (!id) { // 0, null, undefined
       return undefined;
     }
     return this._value(await this.get(id));

package/src/entities/index.js

@@ -1,7 +1,9 @@
+import { join } from 'path';
 import { Transform } from 'stream';
-import { asArray, stream } from '@miso.ai/server-commons';
+import { asArray, stream, getYear } from '@miso.ai/server-commons';
 import EntityIndex from './entity-index.js';
 import EntityTransformStream from './transform.js';
+import EntityPresenceStream from './presence.js';
 import defaultTransform from './transform-default.js';
 import legacyTransform from './transform-legacy.js';
 
@@ -19,19 +21,23 @@ export default class Entities {
     if (!resolve && !transform) {
       return this._client._helpers.stream(this.name, options);
     }
-    transform = getTransformFn(transform);
-
     const client = this._client;
+    transform = await getTransformFn(client, this.name, transform);
 
     // we need taxonomy fetched so we know whether it's hierarchical
     const taxonomies = await client._helpers.findAssociatedTaxonomies(this.name);
 
+    // TODO: omit specific indicies by config
     // prepare entity indicies
+    const { resources = {} } = client._profile || {};
+    const ignored = new Set(resources.ignore || []);
+
     const indicies = [
       client.users.index,
       client.media.index,
       ...taxonomies.map(({ rest_base }) => client.entities(rest_base).index),
-    ];
+    ].filter(index => !ignored.has(index.name));
+
     await Promise.all(indicies.map(index => index.ready()));
     for (const index of indicies) {
       if (index.hierarchical) {
@@ -56,12 +62,7 @@ export default class Entities {
   }
 
   async ids(options = {}) {
-    const { before, after, u } = options;
-    const fields = ['id'];
-    if (before || after) {
-      fields.push('modified_gmt');
-    }
-    return (await this._client._helpers.stream(this.name, { ...options, fields }))
+    return (await this._client._helpers.stream(this.name, { ...options, fields: ['id'] }))
       .pipe(new Transform({
         objectMode: true,
         transform({ id }, _, callback) {
@@ -71,6 +72,10 @@ export default class Entities {
   }
 
   async getAll(options) {
+    return this.all(options);
+  }
+
+  async all(options) {
     return stream.collect(await this.stream(options));
   }
 
@@ -82,6 +87,22 @@ export default class Entities {
     return this._client._helpers.terms(this.name, options);
   }
 
+  presence(options) {
+    return new EntityPresenceStream(this._client, this.name, options);
+  }
+
+  async dateRange() {
+    // TODO: options?
+    return Promise.all([
+      getPostDate(this._client, 'asc'),
+      getPostDate(this._client, 'desc'),
+    ]);
+  }
+
+  async yearRange() {
+    return (await this.dateRange()).map(getYear);
+  }
+
   get index() {
     return this._index;
   }
@@ -96,6 +117,30 @@ export default class Entities {
 
 }
 
+async function getTransformFn(client, name, transform) {
+  switch (transform) {
+    case 'default':
+      return defaultTransform;
+    case 'legacy':
+      return legacyTransform;
+  }
+  if (transform === true) {
+    const { defaults } = client._profile || {};
+    if (!defaults || !defaults.transform || !defaults.transform[name]) {
+      return defaultTransform;
+    }
+    transform = defaults.transform[name];
+  }
+  if (typeof transform === 'string') {
+    // try as file path
+    transform = (await import(join(process.env.PWD, transform))).default;
+  }
+  if (typeof transform === 'function') {
+    return post => transform(post, { defaultTransform });
+  }
+  return undefined;
+}
+
 function aggregateIds(records, propName) {
   return Array.from(records.reduce((idSet, record) => {
     for (const id of asArray(record[propName])) {
@@ -105,8 +150,6 @@ function aggregateIds(records, propName) {
   }, new Set()));
 }
 
-function getTransformFn(transform) {
-  return typeof transform === 'function' ? post => transform(post, { defaultTransform }) :
-    (transform === true || transform === 'default') ? defaultTransform :
-    transform === 'legacy' ? legacyTransform : undefined;
+async function getPostDate(client, order, options = {}) {
+  return (await client.posts.getAll({ ...options, limit: 1, order, fields: ['date_gmt'] }))[0].date_gmt;
 }

package/src/entities/presence.js

@@ -0,0 +1,105 @@
+import { Transform } from 'stream';
+
+export default class EntityPresenceStream extends Transform {
+
+  constructor(client, name, {
+    present = true,
+    fetchSize = 20,
+    preserveOrder = true,
+  } = {}) {
+    super();
+    this._client = client;
+    this._name = name;
+    this._options = {
+      present,
+      fetchSize,
+      preserveOrder,
+    }
+    this._inputs = [];
+    this._pendingSet = new Set();
+    this._requests = [];
+    this._map = new Map();
+    this._done = false;
+  }
+
+  _transform(id, _, next) {
+    id = `${id}`; // buffer -> string
+    if (id) {
+      this._inputs.push(id);
+      this._outputAll();
+      this._requestAll();
+    }
+    next();
+  }
+
+  _flush(done) {
+    this._done = done;
+    this._outputAll();
+    if (this._inputs.length > 0) {
+      this._requestAll(true);
+    }
+  }
+
+  _outputAll() {
+    // TODO: implement when preserveOrder = false
+    let i = 0;
+    for (const len = this._inputs.length; i < len; i++) {
+      const id = this._inputs[i];
+      const entry = this._map.get(id);
+      if (!entry || entry.value === undefined) {
+        break;
+      }
+      if (this._options.present === entry.value) {
+        this.push(id);
+      }
+    }
+    if (i > 0) {
+      this._inputs = this._inputs.slice(i);
+    }
+    if (this._done && this._inputs.length === 0) {
+      this._done();
+    }
+  }
+
+  _requestAll(flush = false) {
+    for (const id of this._inputs) {
+      this._fetchAll();
+      if (!this._map.has(id)) {
+        this._map.set(id, { status: 'pending' });
+        this._pendingSet.add(id);
+      }
+    }
+    this._fetchAll(flush);
+  }
+
+  async _fetchAll(flush = false) {
+    if (!flush && this._pendingSet.size < this._options.fetchSize) {
+      return;
+    }
+    const ids = Array.from(this._pendingSet);
+    for (const id of ids) {
+      this._map.get(id).status = 'fetching';
+    }
+    this._pendingSet = new Set();
+
+    const presences = await this._fetch(ids);
+
+    for (const id of ids) {
+      const entry = this._map.get(id);
+      entry.status = 'ready';
+      entry.value = presences.has(id);
+    }
+    this._outputAll();
+  }
+
+  async _fetch(ids) {
+    const url = await this._client._helpers.url.build(this._name, { include: ids, fields: ['id'] });
+    const { data } = await this._client._helpers.axios.get(url);
+    const presences = new Set();
+    for (const { id } of data) {
+      presences.add(`${id}`);
+    }
+    return presences;
+  }
+
+}

package/src/entities/transform-default.js

@@ -14,14 +14,14 @@ export default function transform({
   modified_gmt,
   guid: {
     rendered: guid,
-  },
+  } = {},
   slug,
   title: {
     rendered: title,
-  },
+  } = {},
   content: {
     rendered: html,
-  },
+  } = {},
   link: url,
   status,
   sticky,
@@ -42,6 +42,7 @@ export default function transform({
     product_id,
     type,
     created_at,
+    published_at: created_at,
     updated_at,
     title,
     cover_image,

package/src/helpers.js

@@ -1,24 +1,52 @@
-import { asNumber, splitObj, stream } from '@miso.ai/server-commons';
-import axios from './axios.js';
+import axios from 'axios';
+import axiosRetry from 'axios-retry';
+import { asNumber, splitObj, stream, startOfDate, endOfDate } from '@miso.ai/server-commons';
 import DataSource from './source/index.js';
+import version from './version.js';
 
 const MS_PER_HOUR = 1000 * 60 * 60;
 
-const STREAM_OPTIONS = ['offset', 'limit', 'strategy', 'filter', 'transform', 'onLoad'];
+const STREAM_OPTIONS = ['offset', 'strategy', 'filter', 'transform', 'onLoad'];
+
+function createAxios(client) {
+  const { auth } = client._options || {};
+  const headers = {
+    'User-Agent': `MisoBot/${version}`,
+  };
+  if (auth) {
+    if (typeof auth === 'object' && auth.username && auth.password) {
+      auth = `${auth.username}:${auth.password}`;
+    }
+    if (typeof auth !== 'string') {
+      throw new TypeError(`Invalid auth: must me a string or an object.`);
+    }
+    headers['Authorization'] = 'Basic ' + Buffer.from(auth).toString('base64');
+  }
+  const instance = axios.create({
+    headers,
+  });
+  axiosRetry(instance, { retries: 5, retryDelay: count => count * 300 });
+  return instance;
+}
 
 export default class Helpers {
 
   constructor(client) {
     this._start = Date.now();
     this._client = client;
+    this._axios = createAxios(client);
     this.url = new Url(this);
     this._samples = {};
     this.debug = this.debug.bind(this);
   }
 
-  async stream(resource, options) {
+  get axios() {
+    return this._axios;
+  }
+
+  async stream(resource, options = {}) {
     const [streamOptions, sourceOptions] = splitObj(options, STREAM_OPTIONS);
-    const source = new DataSource(this, resource, sourceOptions);
+    const source = options.source || new DataSource(this, resource, sourceOptions);
     return new stream.BufferedReadStream(source, { ...streamOptions, debug: this.debug });
   }
 
@@ -32,7 +60,7 @@ export default class Helpers {
 
   async _fetchSample(resource) {
     const url = await this.url.build(resource, { page: 0, pageSize: 1 });
-    const { data, headers } = await axios.get(url);
+    const { data, headers } = await this.axios.get(url);
     if (!data.length) {
       throw new Error(`No record of ${resource} avaliable`);
     }
@@ -71,7 +99,7 @@ export default class Helpers {
 
   async _fetchTaxonomies() {
     const url = await this.url.build('taxonomies');
-    const { data } = await axios.get(url);
+    const { data } = await this.axios.get(url);
     this.debug(`Fetched taxonomies.`);
     return Object.values(data);
   }
@@ -82,7 +110,7 @@ export default class Helpers {
 
   async count(resource, { offset: _, ...options } = {}) {
     const url = await this.url.build(resource, { ...options, page: 0, pageSize: 1 });
-    const { headers } = await axios.get(url);
+    const { headers } = await this.axios.get(url);
     return asNumber(headers['x-wp-total']);
   }
 
@@ -92,7 +120,7 @@ export default class Helpers {
 
   async countUrl(url) {
     url = await this.url.append(url, { page: 0, pageSize: 1 });
-    const { headers } = await axios.get(url);
+    const { headers } = await this.axios.get(url);
     return asNumber(headers['x-wp-total']);
   }
 
@@ -133,18 +161,34 @@ class Url {
 
   // modifiedAfter, modifiedBefore is supported since WordPress 5.7
   // https://make.wordpress.org/core/2021/02/23/rest-api-changes-in-wordpress-5-7/
-  async append(url, options = {}) {
-    const { after, before, order, orderBy, page, pageSize, offset, include, exclude } = options;
-    let { fields } = options;
+  async append(url, {
+    date,
+    after,
+    before,
+    modifiedAfter,
+    modifiedBefore,
+    order,
+    orderBy,
+    page,
+    pageSize,
+    offset,
+    include,
+    exclude,
+    fields,
+  } = {}) {
     const params = [];
 
     // TODO: support single id
 
+    [after, before] = [startOfDate(date || after), endOfDate(date || before)];
+
     // The date is compared against site's local time, not UTC, so we have to work on timezone offset
-    if (has(after) || has(before)) {
+    if (has(after) || has(before) || has(modifiedAfter) || has(modifiedBefore)) {
       const utcOffset = await this._helpers.utcOffsetInMs();
       has(after) && params.push(`after=${toISOString(after, utcOffset)}`);
       has(before) && params.push(`before=${toISOString(before, utcOffset)}`);
+      has(modifiedAfter) && params.push(`modified_after=${toISOString(modifiedAfter, utcOffset)}`);
+      has(modifiedBefore) && params.push(`modified_before=${toISOString(modifiedBefore, utcOffset)}`);
     }
 
     has(order) && params.push(`order=${order}`);
@@ -155,6 +199,7 @@ class Url {
     has(include) && include.length && params.push(`include=${joinIds(include)}`);
     has(exclude) && exclude.length && params.push(`exclude=${joinIds(exclude)}`);
     if (has(fields) && fields.length) {
+      // TODO: is this unused?
       if (has(before) && !fields.includes('modified_gmt')) {
         fields = [...fields, 'modified_gmt'];
       }

package/src/posts/index.js

@@ -10,8 +10,12 @@ export default class Posts extends Entities {
     super(client, RESOURCE_NAME);
   }
   
-  async getAll() {
-    throw new Error(`Getting all posts is not supported.`);
+  async getAll(options = {}) {
+    if (!options.ids && !options.limit) {
+      // TODO: should be more tolerant
+      throw new Error(`Getting all posts is not supported.`);
+    }
+    return super.getAll(options);
   }
 
   async index() {

package/src/source/base.js

@@ -1,5 +1,3 @@
-import axios from '../axios.js';
-
 export default class WordPressDataSource {
 
   constructor(helpers, resource, options = {}) {
@@ -32,14 +30,21 @@ export default class WordPressDataSource {
     this._debug(`[WordPressDataSource] request ${url}`);
     const response =  await this._axiosGet(url);
     this._debug(`[WordPressDataSource] response ${response.status} ${url}`);
-    return this._process(response);
+    return this._process(response, { request, url });
   }
 
-  _process({ status, data }) {
+  _process({ status, data }, { request, url }) {
     if (status >= 400 && status < 500 && data.code === 'rest_post_invalid_page_number') {
       // out of bound, so there is no more data
       return { data: [], terminate: true };
     }
+    if (!Array.isArray(data)) {
+      throw new Error(`Unexpected response from WordPress API for ${url}. Expected an array of objects: ${data}`);
+    }
+    const { records } = request;
+    if (records) {
+      data = data.slice(0, records);
+    }
     if (!this._options.preserveLinks) {
       data = data.map(this._helpers.removeLinks);
     }
@@ -52,13 +57,13 @@ export default class WordPressDataSource {
 
   async _buildBaseUrl() {
     // exclude parameters meant to be dealt with state
-    const { page, ...options } = this._options;
+    const { page, ids, ...options } = this._options;
     return this._helpers.url.build(this._resource, options);
   }
 
   async _axiosGet(url) {
     try {
-      return await axios.get(url);
+      return await this._helpers.axios.get(url);
     } catch(error) {
       if (error.response) {
         return error.response;

package/src/source/paged.js

@@ -14,7 +14,7 @@ export default class PagedWordPressDataSource extends WordPressDataSource {
     if (pageSize > MAX_PAGE_SIZE) {
       throw new Error(`Page size cannot be greater than ${MAX_PAGE_SIZE}: ${pageSize}`);
     }
-    // TODO: limit
+    this._limit = limit;
     this._pageSize = options.pageSize = pageSize;
     this._page = 0;
   }
@@ -25,10 +25,13 @@ export default class PagedWordPressDataSource extends WordPressDataSource {
 
   request() {
     const page = this._page++;
-    const records = this._pageSize;
-    const total = this._totalValue;
+    let records = this._pageSize;
+    const limit = combineLimit(this._totalValue, this._limit);
     // if we know total, we know when the data is exhausted
-    const exhaust = total !== undefined && ((page + 1) * this._pageSize > total + 10); // 10 for a buffer
+    const exhaust = limit !== undefined && ((page + 1) * this._pageSize > limit);
+    if (exhaust && this._limit !== undefined) {
+      records = this._limit - (page * this._pageSize);
+    }
     return exhaust ? { records, page, exhaust } : { records, page };
   }
 
@@ -36,9 +39,18 @@ export default class PagedWordPressDataSource extends WordPressDataSource {
     return this._totalPromise || (this._totalPromise = this._fetchTotal());
   }
 
-  async _url(baseUrl, { page }) {
+  async _url(baseUrl, { records, page }) {
     const head = baseUrl.indexOf('?') < 0 ? '?' : '&';
-    return `${baseUrl}${head}page=${page + 1}`;
+    let url = `${baseUrl}${head}page=${page + 1}`;
+    // optimize: if limit < page size we can save much bandwidth
+    if (page === 0 && records < this._pageSize) {
+      if (url.indexOf('per_page=') > -1) {
+        url = url.replace(/per_page=\d+/, `per_page=${records}`);
+      } else {
+        url += `&per_page=${records}`;
+      }
+    }
+    return url;
   }
 
   async _fetchTotal() {
@@ -50,8 +62,8 @@ export default class PagedWordPressDataSource extends WordPressDataSource {
     return total;
   }
 
-  _process({ status, data, headers }) {
-    const result = super._process({ status, data, headers });
+  _process({ status, data, headers }, meta) {
+    const result = super._process({ status, data, headers }, meta);
     const total = asNumber(headers['x-wp-total']);
     if (total !== undefined) {
       result.total = total;
@@ -63,3 +75,9 @@ export default class PagedWordPressDataSource extends WordPressDataSource {
   }
 
 }
+
+const TOTAL_BUFFER = 10;
+
+function combineLimit(total, limit) {
+  return total === undefined ? limit : limit === undefined ? total + TOTAL_BUFFER : Math.min(total + TOTAL_BUFFER, limit);
+}

package/src/version.js

@@ -1 +1 @@
-export default '0.6.3-beta.2';
+export default '0.6.3-beta.20';

package/src/axios.js

@@ -1,8 +0,0 @@
-import axios from 'axios';
-import axiosRetry from 'axios-retry';
-
-// TODO: create an instance
-
-axiosRetry(axios, { retries: 5, retryDelay: count => count * 300 });
-
-export default axios;