@miso.ai/server-wordpress 0.6.4 → 0.6.5-beta.0

package/README.md

@@ -0,0 +1,163 @@
+# Miso Node.js SDK (WordPress)
+
+## Setup
+
+1. Install Node.js.
+
+2. Install the package locally:
+
+```bash
+npm i @miso.ai/server-wordpress
+```
+
+Or, install the package globally:
+
+```bash
+npm i -g @miso.ai/server-wordpress
+```
+
+Or, use `npx` to run the commands, which will guide the package installation.
+
+3. Initialize a project profile
+
+```bash
+miso-wp init "b2c-contenthub.com"
+```
+
+which will generate a file `wordpress.json` in the current directory, with content like this:
+
+```json
+{
+  "site": "b2c-contenthub.com",
+  "utcOffset": 0
+}
+```
+
+## Usage
+
+### Help message
+
+```bash
+miso-wp --help
+```
+
+### Get
+
+This command emits infinite stream of posts from the WordPress site.
+
+```bash
+miso-wp posts
+```
+
+You can specify a limit of posts to emit, or pipe to a `head`:
+
+```bash
+miso-wp posts -n 10 # this will be more efficient
+miso-wp posts | head -10
+```
+
+### Get, by ids
+
+Get posts by ids:
+
+```bash
+miso-wp posts --ids 10001
+miso-wp posts --ids 10001,10002
+```
+
+### Get, with a time range
+
+You can specify a time range:
+
+```bash
+# The range is always inclusive
+miso-wp posts --after 2025-01-01 --before 2025-01-31
+miso-wp posts -a 2025-01-01 -b 2025-01-31
+
+miso-wp posts -a 2025-01 -b 2025-02
+miso-wp posts -a 2024 -b 2025
+
+miso-wp posts --date 2025-01-15
+miso-wp posts -d 2025-01-15
+
+miso-wp posts -d 2025-01
+miso-wp posts -d 2025
+```
+
+Get posts that are either created or updated in a duration:
+
+```bash
+miso-wp posts -u 3h # in last 3 hours
+miso-wp posts -u 4d # in last 4 days
+miso-wp posts -u 5w # in last 5 weeks
+```
+
+### Count the items
+
+All filters above can be applied:
+
+```bash
+miso-wp posts -c
+miso-wp posts -d 2025 -c
+miso-wp posts -d 2025-01 -c
+```
+
+### Get, with linked resources resolved
+
+There are many linked resources in a post, such as author, categories, tags, etc. In the default mode, those resources are represented by their numberic ids. You can resolve them by:
+
+```bash
+miso-wp posts --resolve
+```
+
+The resolved content will be put in a `_linked` field under the post object.
+
+### Get, restricting fields retrieved
+
+```bash
+miso-wp posts --fields id,title
+```
+
+### Get, with extra parameters
+
+```bash
+miso-wp posts --param "order=asc"
+```
+
+See [WordPress JSON API reference](https://developer.wordpress.org/rest-api/reference/posts/#definition).
+
+### Get, with records transformed
+
+Tranform the posts to Miso product records:
+
+```bash
+miso-wp posts --transform
+```
+
+You can customize the transformation:
+
+```bash
+miso-wp posts --transform 'path-to/your-transform.js'
+```
+
+```js
+export default function(post, { defaultTransform }) {
+  const record = defaultTransform(post);
+  // do something with the record
+  return record;
+}
+```
+
+### List all types of entities
+
+```bash
+miso-wp taxonomies
+```
+
+### Getting other types of items
+
+Simply replace `posts` with other types of items. The term should match the `rest_base` property in the response of the `taxonomies` command.
+
+```
+miso-wp categories
+```

package/cli/index.js

@@ -6,6 +6,7 @@ import taxonomies from './taxonomies.js';
 import entities from './entities.js';
 import summarize from './summarize.js';
 import download from './download.js';
+import xml from './xml.js';
 
 yargs.build(yargs => {
   yargs
@@ -28,6 +29,7 @@ yargs.build(yargs => {
     .hide('debug')
     .command(init)
     .command(profile)
+    .command(xml)
     .command(summarize)
     .command(download)
     .command(taxonomies)

package/cli/utils.js

@@ -51,6 +51,11 @@ export function buildForEntities(yargs) {
       type: 'array',
       coerce: yargs.coerceToArray,
     })
+    .option('params', {
+      describe: 'Specify additional query parameters in the form of key=value',
+      type: 'array',
+      coerce: yargs.coerceToArray,
+    })
     .option('resolve', {
       alias: 'r',
       describe: 'Attach resolved entities (author, catagories) linked with the subjects',

package/cli/xml.js

@@ -0,0 +1,61 @@
+import { join } from 'path';
+import { pipeline } from 'stream/promises';
+import { stream } from '@miso.ai/server-commons';
+import { getEntityTransformFunction, EntityTransformStream } from '../src/entities/index.js';
+import { XmlParser } from '../src/index.js';
+
+function build(yargs) {
+  return yargs
+    .option('transform', {
+      alias: 't',
+      describe: 'Transform function',
+    })
+    .option('parser', {
+      alias: 'p',
+      describe: 'Parser function',
+    })
+    .option('surpress-errors', {
+      alias: 's',
+      type: 'boolean',
+      default: false,
+      describe: 'Surpress errors',
+    });
+}
+
+async function run({ file, parser, transform, ...options } = {}) {
+  parser = await getParser(parser);
+  await pipeline(
+    file ? stream.fileReadStream(file) : process.stdin,
+    new stream.XmlParseStream(parser, options),
+    ...(await getTransformStreams(transform)),
+    new stream.OutputStream(),
+  );
+}
+
+export default {
+  command: 'xml [file]',
+  desc: 'Read from XML file',
+  builder: build,
+  handler: run,
+};
+
+async function getParser(file) {
+  if (!file || file === 'default') {
+    return new XmlParser();
+  }
+  try {
+    const cls = (await import(join(process.env.PWD, file))).default;
+    return new cls();
+  } catch (e) {
+    throw new Error(`Failed to load parser from ${file}: ${e.message}`);
+  }
+}
+
+async function getTransformStreams(transform) {
+  if (!transform) {
+    return [];
+  }
+  return [
+    new EntityTransformStream([], { transform: await getEntityTransformFunction(transform) }),
+  ];
+}

package/package.json

@@ -17,11 +17,12 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.4",
+    "@miso.ai/server-commons": "0.6.5-beta.0",
     "axios": "^1.6.2",
     "axios-retry": "^3.3.1",
     "dotenv": "^16.4.5",
+    "saxes": "^6.0.0",
     "split2": "^4.2.0"
   },
-  "version": "0.6.4"
+  "version": "0.6.5-beta.0"
 }

package/src/client.js

@@ -3,7 +3,7 @@ import { constants } from 'fs';
 import Helpers from './helpers.js';
 import Media from './media.js';
 import Posts from './posts/index.js';
-import Entities from './entities/index.js';
+import { Entities } from './entities/index.js';
 
 const DEFAULT_PROFILE = './wordpress.json';
 

package/src/entities/index.js

@@ -7,7 +7,9 @@ import EntityPresenceStream from './presence.js';
 import defaultTransform from './transform-default.js';
 import legacyTransform from './transform-legacy.js';
 
-export default class Entities {
+export { EntityTransformStream };
+
+export class Entities {
 
   constructor(client, name) {
     this._client = client;
@@ -22,7 +24,7 @@ export default class Entities {
       return this._client._helpers.stream(this.name, options);
     }
     const client = this._client;
-    transform = await getTransformFn(client, this.name, transform);
+    transform = await _getTransformFn(client, this.name, transform);
 
     // we need taxonomy fetched so we know whether it's hierarchical
     const taxonomies = await client._helpers.findAssociatedTaxonomies(this.name);
@@ -55,7 +57,7 @@ export default class Entities {
     };
 
     // transform stream
-    const transformStream = new EntityTransformStream(this._client, indicies, { transform });
+    const transformStream = new EntityTransformStream(indicies, { transform });
 
     return (await this._client._helpers.stream(this.name, { ...options, onLoad }))
       .pipe(transformStream);
@@ -117,7 +119,11 @@ export default class Entities {
 
 }
 
-async function getTransformFn(client, name, transform) {
+export async function getEntityTransformFunction(transform) {
+  return _getTransformFn(undefined, undefined, transform === true ? 'default' : transform);
+}
+
+async function _getTransformFn(client, name, transform) {
   switch (transform) {
     case 'default':
       return defaultTransform;
@@ -151,5 +157,5 @@ function aggregateIds(records, propName) {
 }
 
 async function getPostDate(client, order, options = {}) {
-  return (await client.posts.getAll({ ...options, limit: 1, order, fields: ['date_gmt'] }))[0].date_gmt;
+  return (await client.posts.getAll({ ...options, after: 1, limit: 1, order, fields: ['date_gmt'] }))[0].date_gmt;
 }

package/src/entities/transform.js

@@ -24,9 +24,8 @@ export default class EntityTransformStream extends Transform {
     return PROP_NAME_OVERRIDES[resource] || resource;
   }
 
-  constructor(client, indicies, { transform } = {}) {
+  constructor(indicies, { transform } = {}) {
     super({ objectMode: true });
-    this._client = client;
     this._indicies = indicies;
     this._transformFn = transform;
   }

package/src/helpers.js

@@ -175,6 +175,7 @@ class Url {
     include,
     exclude,
     fields,
+    params: extraParams,
   } = {}) {
     const params = [];
 
@@ -198,6 +199,12 @@ class Url {
     has(offset) && params.push(`offset=${offset}`);
     has(include) && include.length && params.push(`include=${joinIds(include)}`);
     has(exclude) && exclude.length && params.push(`exclude=${joinIds(exclude)}`);
+    if (has(extraParams)) {
+      for (const str of extraParams) {
+        const [key, value] = str.split('=', 2);
+        params.push(`${encodeURIComponent(key)}=${encodeURIComponent(value)}`);
+      }
+    }
     if (has(fields) && fields.length) {
       // TODO: is this unused?
       if (has(before) && !fields.includes('modified_gmt')) {

package/src/index.js

@@ -1 +1,2 @@
 export { default as WordPressClient } from './client.js';
+export * from './xml.js';

package/src/media.js

@@ -1,4 +1,4 @@
-import Entities from './entities/index.js';
+import { Entities } from './entities/index.js';
 import EntityIndex from './entities/entity-index.js';
 
 const RESOURCE_NAME = 'media';

package/src/posts/index.js

@@ -1,4 +1,4 @@
-import Entities from '../entities/index.js';
+import { Entities } from '../entities/index.js';
 
 const RESOURCE_NAME = 'posts';
 

package/src/source/base.js

@@ -28,7 +28,7 @@ export default class WordPressDataSource {
     this._debug(`[WordPressDataSource] get ${JSON.stringify(request)}`);
     const url = await this.url(request);
     this._debug(`[WordPressDataSource] request ${url}`);
-    const response =  await this._axiosGet(url);
+    const response = await this._axiosGet(url);
     this._debug(`[WordPressDataSource] response ${response.status} ${url}`);
     return this._process(response, { request, url });
   }

package/src/version.js

@@ -1 +1 @@
-export default '0.6.4';
+export default '0.6.5-beta.0';

package/src/xml.js

@@ -0,0 +1,81 @@
+export class XmlParser {
+
+  constructor({ ...options } = {}) {
+    this._options = options;
+    this._currentNode = undefined;
+    this._currentText = '';
+    this._currentPostMeta = {};
+  }
+
+  _onOpenTag({ name }) {
+    if (name === 'item') {
+      this._currentNode = { categories: [], tags: [], postmeta: {} };
+    } else if (this._currentNode) {
+      this._currentNode[name] = '';
+    }
+  }
+
+  onCloseTag({ name }, { push }) {
+    if (this._currentNode) {
+      switch (name) {
+        case 'item':
+          push(nodeToPost(this._currentNode));
+          this._currentNode = undefined;
+          break;
+        case 'category':
+          this._currentNode.categories.push(this._currentText);
+          break;
+        case 'wp:meta_key':
+          this._currentPostMetaKey = this._currentText;
+          break;
+        case 'wp:meta_value':
+          this._currentNode.postmeta[this._currentPostMetaKey] = this._currentText;
+          this._currentPostMetaKey = undefined;
+          break;
+        default:
+          if (name.startsWith('wp:tag')) {
+            this._currentNode.tags.push(this._currentText);
+          } else {
+            this._currentNode[name] = this._currentText;
+          }
+      }
+    }
+    this._currentText = '';
+  }
+
+  onText(text) {
+    this._currentText += text.trim();
+  }
+
+}
+
+function nodeToPost(node) {
+  return {
+    id: node['wp:post_id'],
+    date: node['wp:post_date'],
+    date_gmt: node['wp:post_date_gmt'],
+    modified: node['wp:post_modified'],
+    modified_gmt: node['wp:post_modified_gmt'],
+    slug: node['wp:post_name'],
+    status: node['wp:status'],
+    title: node['title'],
+    link: node['link'],
+    guid: {
+      rendered: node['guid'],
+    },
+    content: {
+      rendered: node['content:encoded'],
+    },
+    excerpt: {
+      rendered: node['excerpt:encoded'],
+    },
+    author: node['dc:creator'],
+    featured_media: node['wp:attachment_url'] || null,
+    comment_status: node['wp:comment_status'],
+    ping_status: node['wp:ping_status'],
+    categories: node.categories,
+    tags: node.tags,
+    type: node['wp:post_type'],
+    meta: node.postmeta,
+  };
+}