@miso.ai/server-feed 0.6.3-beta.10

package/cli/index.js

@@ -0,0 +1,40 @@
+#!/usr/bin/env node
+import { yargs, stream } from '@miso.ai/server-commons';
+import { version, feedStreams } from '../src/index.js';
+
+yargs.build(yargs => {
+  yargs
+    .env('MISO_FEED')
+    .option('after', {
+      alias: 'a',
+      describe: 'Only include records after this time',
+    })
+    .option('update', {
+      alias: 'u',
+      describe: 'Only include records modified in given duration (3h, 2d, etc.)',
+    })
+    .option('transform', {
+      alias: 't',
+      type: 'boolean',
+      default: false,
+    })
+    .option('debug', {
+      type: 'boolean',
+      default: false,
+    })
+    .hide('debug')
+    .command({
+      command: '*',
+      description: 'Parse items from feed content',
+      handler: run,
+    })
+    .version(version);
+});
+
+async function run(options) {
+  await stream.pipeline(
+    process.stdin,
+    ...feedStreams(options),
+    new stream.OutputStream(),
+  );
+}

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@miso.ai/server-feed",
+  "description": "Miso RSS/Atom feed data tools",
+  "type": "module",
+  "main": "src/index.js",
+  "bin": {
+    "miso-feed": "cli/index.js"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {},
+  "repository": "MisoAI/miso-server-js-sdk",
+  "license": "MIT",
+  "contributors": [
+    "simonpai <simon.pai@askmiso.com>"
+  ],
+  "dependencies": {
+    "@miso.ai/server-commons": "0.6.3-beta.10",
+    "feedparser": "^2.2.10",
+    "node-fetch": "^3.3.2"
+  },
+  "version": "0.6.3-beta.10"
+}

package/src/index.js

@@ -0,0 +1,2 @@
+export * from './stream/index.js';
+export { default as version } from './version.js';

package/src/stream/date-filter.js

@@ -0,0 +1,25 @@
+import { Transform } from 'stream';
+
+export default class DateFilterStream extends Transform {
+
+  constructor(threshold) {
+    super({ objectMode: true });
+    this._threshold = threshold;
+  }
+
+  async _transform(record, _) {
+    try {
+      const timestamp = Date.parse(record.date);
+      if (timestamp < this._threshold) {
+        //this.end();
+        return;
+      }
+    } catch (err) {
+      console.error(err);
+      //this.end();
+      return;
+    }
+    this.push(record);
+  }
+
+}

package/src/stream/feed.js

@@ -0,0 +1,18 @@
+import { parseDuration, startOfDate } from '@miso.ai/server-commons';
+import FeedParser from 'feedparser';
+import DateFilterStream from './date-filter.js';
+import ArticleTransformStream from './transform.js';
+
+export default function feedStreams({ parse, after, update, transform } = {}) {
+  const threshold = update ? (Date.now() - parseDuration(update)) : startOfDate(after);
+  const streams = [
+    new FeedParser(parse),
+  ];
+  if (threshold) {
+    streams.push(new DateFilterStream(threshold));
+  }
+  if (transform) {
+    streams.push(new ArticleTransformStream());
+  }
+  return streams;
+}

package/src/stream/index.js

@@ -0,0 +1,3 @@
+export { default as feedStreams } from './feed.js';
+export { default as ArticleTransformStream } from './transform.js';
+export { default as DateFilterStream } from './date-filter.js';

package/src/stream/transform.js

@@ -0,0 +1,61 @@
+import { trimObj } from '@miso.ai/server-commons';
+import { Transform } from 'stream';
+
+export default class ArticleTransformStream extends Transform {
+
+  constructor({ after } = {}) {
+    super({ objectMode: true });
+    this._after = after;
+  }
+
+  async _transform({
+    title,
+    description,
+    summary,
+    date,
+    pubdate,
+    link,
+    origlink,
+    guid,
+    image,
+    author,
+    categories,
+  }, _) {
+    if (this._after !== undefined) {
+      try {
+        const timestamp = Date.parse(date);
+        if (timestamp < this._after) {
+          this.end();
+          return;
+        }
+      } catch (err) {
+        console.error(err);
+        this.end();
+        return;
+      }
+    }
+    const authors = author ? [author] : [];
+    title = title || undefined;
+    categories = categories && categories.map(c => [c]) || [];
+    const url = origlink || link || undefined;
+    const cover_image = image && image.url || undefined;
+    summary = summary || undefined;
+    this.push(trimObj({
+      product_id: guid,
+      type: 'article',
+      title,
+      updated_at: date || undefined,
+      published_at: pubdate || undefined,
+      created_at: pubdate || undefined,
+      categories,
+      url,
+      cover_image,
+      authors,
+      html: description || undefined,
+      custom_attributes: trimObj({
+        summary,
+      }),
+    }));
+  }
+
+}

package/src/version.js

@@ -0,0 +1 @@
+export default '0.6.3-beta.10';