@miso.ai/server-wordpress 0.6.3-beta.11 → 0.6.3-beta.13

package/cli/download.js

@@ -0,0 +1,120 @@
+import { createWriteStream } from 'fs';
+import { access, mkdir } from 'fs/promises';
+import { createGzip } from 'zlib';
+import { startOfDate, endOfDate, stream } from '@miso.ai/server-commons';
+import { WordPressClient } from '../src/index.js';
+import { getFirstPostDate, getLastPostDate, getYear, buildForEntities } from './utils.js';
+
+function build(yargs) {
+  return buildForEntities(yargs);
+}
+
+async function run({
+  destination = './data',
+  batchSize = 30000,
+  ...options
+} = {}) {
+  const client = new WordPressClient(options);
+
+  const [firstPostDate, lastPostDate] = await Promise.all([
+    getFirstPostDate(client),
+    getLastPostDate(client),
+  ]);
+  const firstPostYear = getYear(firstPostDate);
+  const lastPostYear = getYear(lastPostDate);
+
+  // divide into batches
+  const batches = [];
+  let endYear, sum = 0;
+  for (let year = lastPostYear; year >= firstPostYear; year--) {
+    if (endYear === undefined) {
+      endYear = year;
+    }
+    const after = startOfDate(year);
+    const before = endOfDate(year);
+    const count = await client.posts.count({ ...options, after, before });
+    sum += count;
+    if (sum >= batchSize) {
+      batches.push({ start: year, end: endYear, records: sum });
+      endYear = undefined;
+      sum = 0;
+    }
+  }
+  if (sum > 0) {
+    batches.push({ start: firstPostYear, end: endYear, records: sum });
+  }
+  const batchCount = batches.length;
+  if (batchCount === 0) {
+    console.log('No posts found.');
+    return;
+  }
+  // merge last two batches if last batch is too small
+  if (batches[batchCount - 1].records < batchSize * 0.2) {
+    const last = batches.pop();
+    batches[batchCount - 2].end = last.end;
+    batches[batchCount - 2].records += last.records;
+  }
+  console.log(`Divide into ${batchCount} batches:`);
+  for (const { start, end, records } of batches) {
+    console.log(`- ${start} -> ${end} (${records} records)`);
+  }
+
+  // mkdir -p
+  try {
+    await access(destination);
+  } catch (err) {
+    if (err.code !== 'ENOENT') {
+      throw err;
+    }
+    await mkdir(destination, { recursive: true });
+    console.log(`Created directory ${destination}`);
+  }
+
+  // download
+  let index = 0;
+  for (const batch of batches) {
+    const { start, end, records } = batch;
+    const after = startOfDate(start);
+    const before = endOfDate(end);
+    const filename = `${options.transform ? 'miso' : 'wp'}-posts.${start}-${end}.jsonl.gz`;
+
+    console.log(`[${index + 1} / ${batchCount}] Downloading ${filename}`);
+
+    const startTime = Date.now();
+    const sourceStream = await client.posts.stream({ ...options, after, before });
+
+    await stream.pipeline(
+      sourceStream,
+      stream.stringify(),
+      createGzip(),
+      createWriteStream(`${destination}/${filename}`),
+    );
+
+    const elapsed = Date.now() - startTime;
+    console.log(`[${index + 1} / ${batchCount}] Downloaded ${filename} (${records} records in ${formatDuration(elapsed)})`);
+    index++;
+  }
+
+  console.log('Done.');
+}
+
+function formatDuration(duration) {
+  const seconds = Math.floor(duration / 1000);
+  const minutes = Math.floor(seconds / 60);
+  const hours = Math.floor(minutes / 60);
+  if (hours > 0) {
+    return `${hours}h${minutes % 60}m${seconds % 60}s`;
+  }
+  if (minutes > 0) {
+    return `${minutes}m${seconds % 60}s`;
+  }
+  return `${seconds}s`;
+}
+
+export default {
+  command: 'download',
+  aliases: ['down'],
+  desc: 'Download all posts and save as files.',
+  builder: build,
+  handler: run,
+};

package/cli/entities.js

@@ -2,62 +2,7 @@ import { Transform } from 'stream';
 import split2 from 'split2';
 import { stream, parseDuration } from '@miso.ai/server-commons';
 import { WordPressClient } from '../src/index.js';
-import { normalizeOptions, normalizeTransform } from './utils.js';
-
-export function buildForEntities(yargs) {
-  // TODO: make them mutually exclusive
-  return yargs
-    .option('terms', {
-      describe: 'Display terms associated with this type of resource',
-      type: 'boolean',
-    })
-    .option('count', {
-      alias: 'c',
-      describe: 'Return the total number of records',
-      type: 'boolean',
-    })
-    .option('date', {
-      alias: 'd',
-      describe: 'Only include records in this year/month/day',
-    })
-    .option('after', {
-      alias: 'a',
-      describe: 'Only include records after this time',
-    })
-    .option('before', {
-      alias: 'b',
-      describe: 'Only include records before this time',
-    })
-    .option('update', {
-      alias: 'u',
-      describe: 'Only include records modified in given duration (3h, 2d, etc.)',
-    })
-    .option('ids', {
-      alias: 'include',
-      describe: 'Specify post ids',
-    })
-    .option('fields', {
-      describe: 'Specify which record fields are retrieved',
-      type: 'array',
-      coerce: yargs.coerceToArray,
-    })
-    .option('resolve', {
-      alias: 'r',
-      describe: 'Attach resolved entities (author, catagories) linked with the subjects',
-      type: 'boolean',
-    })
-    .option('transform', {
-      alias: 't',
-      describe: 'Apply transform function to the entities',
-    });
-    /*
-    .option('limit', {
-      alias: 'n',
-      describe: 'Limit the amount of records',
-      type: 'number',
-    })
-    */
-}
+import { normalizeOptions, buildForEntities } from './utils.js';
 
 function build(yargs) {
   return buildForEntities(yargs)
@@ -110,7 +55,7 @@ export async function runGet(client, name, { transform, ...options }) {
   await stream.pipelineToStdout(
     await client.entities(name).stream({
       ...options,
-      transform: await normalizeTransform(transform),
+      transform,
     }),
     stream.stringify(),
   );
@@ -160,7 +105,6 @@ async function buildUpdateStream(client, name, update, {
   ...options
 }) {
   // TODO: move the logic into client itself
-  transform = await normalizeTransform(transform);
   const now = Date.now();
   update = parseDuration(update);
   const threshold = now - update;

package/cli/index.js

@@ -4,6 +4,8 @@ import version from '../src/version.js';
 import { profile, init } from './profile.js';
 import taxonomies from './taxonomies.js';
 import entities from './entities.js';
+import summarize from './summarize.js';
+import download from './download.js';
 
 yargs.build(yargs => {
   yargs
@@ -26,6 +28,8 @@ yargs.build(yargs => {
     .hide('debug')
     .command(init)
     .command(profile)
+    .command(summarize)
+    .command(download)
     .command(taxonomies)
     .command(entities)
     .version(version);

package/cli/summarize.js

@@ -0,0 +1,59 @@
+import { startOfDate, endOfDate } from '@miso.ai/server-commons';
+import { WordPressClient } from '../src/index.js';
+import { getFirstPostDate, getLastPostDate, getYear } from './utils.js';
+
+function build(yargs) {
+  return yargs;
+}
+
+async function run({ ...options } = {}) {
+  const client = new WordPressClient(options);
+  const [total, firstPostDate, lastPostDate] = await Promise.all([
+    client.posts.count(options),
+    getFirstPostDate(client),
+    getLastPostDate(client),
+  ]);
+  const totalStrLength = `${total}`.length;
+  console.log();
+  console.log(`Total posts: ${total}`);
+  console.log(`First post at: ${firstPostDate}`);
+  console.log(`Last post at: ${lastPostDate}`);
+
+  // drill down by year
+  console.log();
+  const bar = `| ---- | ${'-'.repeat(totalStrLength)} |`;
+  console.log(bar);
+  console.log(`| Year | ${'Posts'.padStart(totalStrLength)} |`);
+  console.log(bar);
+  for (let year = getYear(firstPostDate), lastYear = getYear(lastPostDate); year <= lastYear; year++) {
+    const after = startOfDate(year);
+    const before = endOfDate(year);
+    const count = await client.posts.count({ ...options, after, before });
+    console.log(`| ${year} | ${`${count}`.padStart(totalStrLength)} |`);
+  }
+  console.log(bar);
+}
+
+function printTable(arr) {
+  arr = arr.map((row) => row.map(str));
+  const colWidths = arr[0].map((_, i) => Math.max(...arr.map((row) => (row[i] || '').length)));
+  for (const row of arr) {
+    console.log(row.map((v, i) => rightPad(v, colWidths[i])).join(' '));
+  }
+}
+
+function str(value) {
+  return value === undefined ? '--' : `${value}`;
+}
+
+function rightPad(str = '', length) {
+  return str.padEnd(length);
+}
+
+export default {
+  command: 'summarize',
+  aliases: ['sum'],
+  desc: 'Print out a summary of the WordPress site',
+  builder: build,
+  handler: run,
+};

package/cli/utils.js

@@ -1,8 +1,5 @@
-import { join } from 'path';
 import { startOfDate, endOfDate } from '@miso.ai/server-commons';
 
-const PWD = process.env.PWD;
-
 export function normalizeOptions({ date, after, before, ids, include, ...options }) {
   [after, before] = [startOfDate(date || after), endOfDate(date || before)];
   // TODO: rely on yargs to coerce to array
@@ -10,16 +7,75 @@ export function normalizeOptions({ date, after, before, ids, include, ...options
   return { ...options, after, before, ids };
 }
 
-export async function normalizeTransform(transform) {
-  if (typeof transform === 'string') {
-    if (transform === 'default' || transform === 'legacy') {
-      return transform;
-    }
-    return (await import(join(PWD, transform))).default;
-  }
-  return !!transform;
-}
-
 export function parseDate(value) {
   return Date.parse(`${value}Z`);
 }
+
+export async function getFirstPostDate(client, options) {
+  return getPostDate(client, 'asc', options);
+}
+
+export async function getLastPostDate(client, options) {
+  return getPostDate(client, 'desc', options);
+}
+
+async function getPostDate(client, order, options = {}) {
+  return (await client.posts.getAll({ ...options, limit: 1, order, fields: ['date_gmt'] }))[0].date_gmt;
+}
+
+export function getYear(dateStr) {
+  return new Date(dateStr).getFullYear();
+}
+
+export function buildForEntities(yargs) {
+  // TODO: make them mutually exclusive
+  return yargs
+    .option('terms', {
+      describe: 'Display terms associated with this type of resource',
+      type: 'boolean',
+    })
+    .option('count', {
+      alias: 'c',
+      describe: 'Return the total number of records',
+      type: 'boolean',
+    })
+    .option('date', {
+      alias: 'd',
+      describe: 'Only include records in this year/month/day',
+    })
+    .option('after', {
+      alias: 'a',
+      describe: 'Only include records after this time',
+    })
+    .option('before', {
+      alias: 'b',
+      describe: 'Only include records before this time',
+    })
+    .option('update', {
+      alias: 'u',
+      describe: 'Only include records modified in given duration (3h, 2d, etc.)',
+    })
+    .option('ids', {
+      alias: 'include',
+      describe: 'Specify post ids',
+    })
+    .option('fields', {
+      describe: 'Specify which record fields are retrieved',
+      type: 'array',
+      coerce: yargs.coerceToArray,
+    })
+    .option('resolve', {
+      alias: 'r',
+      describe: 'Attach resolved entities (author, catagories) linked with the subjects',
+      type: 'boolean',
+    })
+    .option('transform', {
+      alias: 't',
+      describe: 'Apply transform function to the entities',
+    })
+    .option('limit', {
+      alias: 'n',
+      describe: 'Limit the amount of records',
+      type: 'number',
+    });
+}

package/package.json

@@ -17,9 +17,9 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.3-beta.11",
+    "@miso.ai/server-commons": "0.6.3-beta.13",
     "axios": "^1.6.2",
     "axios-retry": "^3.3.1"
   },
-  "version": "0.6.3-beta.11"
+  "version": "0.6.3-beta.13"
 }

package/src/client.js

@@ -70,7 +70,7 @@ export default class WordPressClient {
 
 }
 
-const SITE_PROFILE_PROPS = ['site', 'utcOffset', 'resources'];
+const SITE_PROFILE_PROPS = ['site', 'utcOffset', 'resources', 'defaults'];
 
 class SiteProfile {
 

package/src/entities/entity-index.js

@@ -50,7 +50,7 @@ export default class EntityIndex {
     if (this.hierarchical) {
       return; // already all fetched
     }
-    ids = asArray(ids);
+    ids = asArray(ids).filter(id => id); // discard 0, null, undefined
 
     const promises = []
     const idsToFetch = [];
@@ -85,8 +85,11 @@ export default class EntityIndex {
   }
 
   _resolveFetch(id) {
-    this._fetching.get(id).resolve();
-    this._fetching.delete(id);
+    const res = this._fetching.get(id);
+    if (res) {
+      res.resolve();
+      this._fetching.delete(id);
+    }
   }
 
   async get(id) {
@@ -96,13 +99,14 @@ export default class EntityIndex {
   }
 
   async getAll(ids) {
+    ids = ids.filter(id => id); // discard 0, null, undefined
     await this._dataReady();
     await this.fetch(ids);
     return ids.map(id => this._index.get(id));
   }
 
   async getValue(id) {
-    if (id === undefined) {
+    if (!id) { // 0, null, undefined
       return undefined;
     }
     return this._value(await this.get(id));

package/src/entities/index.js

@@ -1,3 +1,4 @@
+import { join } from 'path';
 import { Transform } from 'stream';
 import { asArray, stream } from '@miso.ai/server-commons';
 import EntityIndex from './entity-index.js';
@@ -20,9 +21,8 @@ export default class Entities {
     if (!resolve && !transform) {
       return this._client._helpers.stream(this.name, options);
     }
-    transform = getTransformFn(transform);
-
     const client = this._client;
+    transform = await getTransformFn(client, this.name, transform);
 
     // we need taxonomy fetched so we know whether it's hierarchical
     const taxonomies = await client._helpers.findAssociatedTaxonomies(this.name);
@@ -115,8 +115,26 @@ function aggregateIds(records, propName) {
   }, new Set()));
 }
 
-function getTransformFn(transform) {
-  return typeof transform === 'function' ? post => transform(post, { defaultTransform }) :
-    (transform === true || transform === 'default') ? defaultTransform :
-    transform === 'legacy' ? legacyTransform : undefined;
+async function getTransformFn(client, name, transform) {
+  switch (transform) {
+    case 'default':
+      return defaultTransform;
+    case 'legacy':
+      return legacyTransform;
+  }
+  if (transform === true) {
+    const { defaults } = client._profile || {};
+    if (!defaults || !defaults.transform || !defaults.transform[name]) {
+      return defaultTransform;
+    }
+    transform = defaults.transform[name];
+  }
+  if (typeof transform === 'string') {
+    // try as file path
+    transform = (await import(join(process.env.PWD, transform))).default;
+  }
+  if (typeof transform === 'function') {
+    return post => transform(post, { defaultTransform });
+  }
+  return undefined;
 }

package/src/helpers.js

@@ -6,7 +6,7 @@ import version from './version.js';
 
 const MS_PER_HOUR = 1000 * 60 * 60;
 
-const STREAM_OPTIONS = ['offset', 'limit', 'strategy', 'filter', 'transform', 'onLoad'];
+const STREAM_OPTIONS = ['offset', 'strategy', 'filter', 'transform', 'onLoad'];
 
 function createAxios(client) {
   const { auth } = client._options || {};

package/src/posts/index.js

@@ -10,8 +10,12 @@ export default class Posts extends Entities {
     super(client, RESOURCE_NAME);
   }
   
-  async getAll() {
-    throw new Error(`Getting all posts is not supported.`);
+  async getAll(options = {}) {
+    if (!options.ids && !options.limit) {
+      // TODO: should be more tolerant
+      throw new Error(`Getting all posts is not supported.`);
+    }
+    return super.getAll(options);
   }
 
   async index() {

package/src/source/base.js

@@ -30,10 +30,10 @@ export default class WordPressDataSource {
     this._debug(`[WordPressDataSource] request ${url}`);
     const response =  await this._axiosGet(url);
     this._debug(`[WordPressDataSource] response ${response.status} ${url}`);
-    return this._process(response, { url });
+    return this._process(response, { request, url });
   }
 
-  _process({ status, data }, { url }) {
+  _process({ status, data }, { request, url }) {
     if (status >= 400 && status < 500 && data.code === 'rest_post_invalid_page_number') {
       // out of bound, so there is no more data
       return { data: [], terminate: true };
@@ -41,6 +41,10 @@ export default class WordPressDataSource {
     if (!Array.isArray(data)) {
       throw new Error(`Unexpected response from WordPress API for ${url}. Expected an array of objects: ${data}`);
     }
+    const { records } = request;
+    if (records) {
+      data = data.slice(0, records);
+    }
     if (!this._options.preserveLinks) {
       data = data.map(this._helpers.removeLinks);
     }

package/src/source/paged.js

@@ -14,7 +14,7 @@ export default class PagedWordPressDataSource extends WordPressDataSource {
     if (pageSize > MAX_PAGE_SIZE) {
       throw new Error(`Page size cannot be greater than ${MAX_PAGE_SIZE}: ${pageSize}`);
     }
-    // TODO: limit
+    this._limit = limit;
     this._pageSize = options.pageSize = pageSize;
     this._page = 0;
   }
@@ -25,10 +25,13 @@ export default class PagedWordPressDataSource extends WordPressDataSource {
 
   request() {
     const page = this._page++;
-    const records = this._pageSize;
-    const total = this._totalValue;
+    let records = this._pageSize;
+    const limit = combineLimit(this._totalValue, this._limit);
     // if we know total, we know when the data is exhausted
-    const exhaust = total !== undefined && ((page + 1) * this._pageSize > total + 10); // 10 for a buffer
+    const exhaust = limit !== undefined && ((page + 1) * this._pageSize > limit);
+    if (exhaust && this._limit !== undefined) {
+      records = this._limit - (page * this._pageSize);
+    }
     return exhaust ? { records, page, exhaust } : { records, page };
   }
 
@@ -36,9 +39,18 @@ export default class PagedWordPressDataSource extends WordPressDataSource {
     return this._totalPromise || (this._totalPromise = this._fetchTotal());
   }
 
-  async _url(baseUrl, { page }) {
+  async _url(baseUrl, { records, page }) {
     const head = baseUrl.indexOf('?') < 0 ? '?' : '&';
-    return `${baseUrl}${head}page=${page + 1}`;
+    let url = `${baseUrl}${head}page=${page + 1}`;
+    // optimize: if limit < page size we can save much bandwidth
+    if (page === 0 && records < this._pageSize) {
+      if (url.indexOf('per_page=') > -1) {
+        url = url.replace(/per_page=\d+/, `per_page=${records}`);
+      } else {
+        url += `&per_page=${records}`;
+      }
+    }
+    return url;
   }
 
   async _fetchTotal() {
@@ -63,3 +75,9 @@ export default class PagedWordPressDataSource extends WordPressDataSource {
   }
 
 }
+
+const TOTAL_BUFFER = 10;
+
+function combineLimit(total, limit) {
+  return total === undefined ? limit : limit === undefined ? total + TOTAL_BUFFER : Math.min(total + TOTAL_BUFFER, limit);
+}

package/src/version.js

@@ -1 +1 @@
-export default '0.6.3-beta.11';
+export default '0.6.3-beta.13';