@miso.ai/server-feed 0.6.3-beta.2 → 0.6.3-beta.4

package/cli/index.js

@@ -1,7 +1,6 @@
 #!/usr/bin/env node
 import { yargs, stream } from '@miso.ai/server-commons';
-import version from '../src/version.js';
-import { stream as createFeedStream } from '../src/index.js';
+import { version, feedStream } from '../src/index.js';
 
 yargs.build(yargs => {
   yargs
@@ -10,9 +9,21 @@ yargs.build(yargs => {
       type: 'string',
     })
     .option('auth', {
-      alias: 'u',
       type: 'string',
     })
+    .option('after', {
+      alias: 'a',
+      describe: 'Only include records after this time',
+    })
+    .option('update', {
+      alias: 'u',
+      describe: 'Only include records modified in given duration (3h, 2d, etc.)',
+    })
+    .option('transform', {
+      alias: 't',
+      type: 'boolean',
+      default: false,
+    })
     .option('debug', {
       type: 'boolean',
       default: false,
@@ -26,9 +37,9 @@ yargs.build(yargs => {
     .version(version);
 });
 
-async function run({ url, auth } = {}) {
+async function run({ url, auth, after, update, transform } = {}) {
   await stream.pipeline(
-    await createFeedStream(url, { fetch: { auth } }),
+    await feedStream(url, { fetch: { auth }, after, update, transform }),
     new stream.OutputStream(),
   );
 }

package/package.json

@@ -16,9 +16,9 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.3-beta.2",
+    "@miso.ai/server-commons": "0.6.3-beta.4",
     "feedparser": "^2.2.10",
     "node-fetch": "^3.3.2"
   },
-  "version": "0.6.3-beta.2"
+  "version": "0.6.3-beta.4"
 }

package/src/index.js

@@ -1 +1,2 @@
-export { default as stream } from './stream.js';
+export * from './stream/index.js';
+export { default as version } from './version.js';

package/src/stream/date-filter.js

@@ -0,0 +1,25 @@
+import { Transform } from 'stream';
+
+export default class DateFilterStream extends Transform {
+
+  constructor(threshold) {
+    super({ objectMode: true });
+    this._threshold = threshold;
+  }
+
+  async _transform(record, _) {
+    try {
+      const timestamp = Date.parse(record.date);
+      if (timestamp < this._threshold) {
+        this.end();
+        return;
+      }
+    } catch (err) {
+      console.error(err);
+      this.end();
+      return;
+    }
+    this.push(record);
+  }
+
+}

package/src/stream/feed.js

@@ -0,0 +1,16 @@
+import { parseDuration, startOfDate } from '@miso.ai/server-commons';
+import rawFeedStream from './raw.js';
+import DateFilterStream from './date-filter.js';
+import ArticleTransformStream from './transform.js';
+
+export default async function feedStream(url, { fetch, parse, after, update, transform } = {}) {
+  let stream = await rawFeedStream(url, { fetch, parse });
+  const threshold = update ? (Date.now() - parseDuration(update)) : startOfDate(after);
+  if (threshold) {
+    stream = stream.pipe(new DateFilterStream(threshold));
+  }
+  if (transform) {
+    stream = stream.pipe(new ArticleTransformStream());
+  }
+  return stream;
+}

package/src/stream/index.js

@@ -0,0 +1,4 @@
+export { default as feedStream } from './feed.js';
+export { default as rawFeedStream } from './raw.js';
+export { default as ArticleTransformStream } from './transform.js';
+export { default as DateFilterStream } from './date-filter.js';

package/src/{stream.js → stream/raw.js}

@@ -2,7 +2,7 @@ import { Buffer } from 'buffer';
 import fetch, { Headers } from 'node-fetch';
 import FeedParser from 'feedparser';
 
-export default async function stream(url, { fetch: fetchOptions, parse: parseOptions } = {}) {
+export default async function rawFeedStream(url, { fetch: fetchOptions, parse: parseOptions } = {}) {
   const { status, body } = await fetch(url, buildFetchOptions(fetchOptions));
   if (status !== 200) {
     throw new Error(`Failed to fetch ${url}: ${status}`);

package/src/stream/transform.js

@@ -0,0 +1,61 @@
+import { trimObj } from '@miso.ai/server-commons';
+import { Transform } from 'stream';
+
+export default class ArticleTransformStream extends Transform {
+
+  constructor({ after } = {}) {
+    super({ objectMode: true });
+    this._after = after;
+  }
+
+  async _transform({
+    title,
+    description,
+    summary,
+    date,
+    pubdate,
+    link,
+    origlink,
+    guid,
+    image,
+    author,
+    categories,
+  }, _) {
+    if (this._after !== undefined) {
+      try {
+        const timestamp = Date.parse(date);
+        if (timestamp < this._after) {
+          this.end();
+          return;
+        }
+      } catch (err) {
+        console.error(err);
+        this.end();
+        return;
+      }
+    }
+    const authors = author ? [author] : [];
+    title = title || undefined;
+    categories = categories && categories.map(c => [c]) || [];
+    const url = origlink || link || undefined;
+    const cover_image = image && image.url || undefined;
+    summary = summary || undefined;
+    this.push(trimObj({
+      product_id: guid,
+      type: 'article',
+      title,
+      updated_at: date || undefined,
+      published_at: pubdate || undefined,
+      created_at: pubdate || undefined,
+      categories,
+      url,
+      cover_image,
+      authors,
+      html: description || undefined,
+      custom_attributes: trimObj({
+        summary,
+      }),
+    }));
+  }
+
+}

package/src/version.js

@@ -1 +1 @@
-export default '0.6.3-beta.2';
+export default '0.6.3-beta.4';