@miso.ai/server-feed 0.6.3-beta.2 → 0.6.3-beta.20

package/cli/index.js

@@ -1,17 +1,22 @@
 #!/usr/bin/env node
 import { yargs, stream } from '@miso.ai/server-commons';
-import version from '../src/version.js';
-import { stream as createFeedStream } from '../src/index.js';
+import { version, feedStreams } from '../src/index.js';
 
 yargs.build(yargs => {
   yargs
     .env('MISO_FEED')
-    .option('url', {
-      type: 'string',
+    .option('after', {
+      alias: 'a',
+      describe: 'Only include records after this time',
     })
-    .option('auth', {
+    .option('update', {
       alias: 'u',
-      type: 'string',
+      describe: 'Only include records modified in given duration (3h, 2d, etc.)',
+    })
+    .option('transform', {
+      alias: 't',
+      type: 'boolean',
+      default: false,
     })
     .option('debug', {
       type: 'boolean',
@@ -19,16 +24,17 @@ yargs.build(yargs => {
     })
     .hide('debug')
     .command({
-      command: '* [url]',
-      description: 'Read items from feed',
+      command: '*',
+      description: 'Parse items from feed content',
       handler: run,
     })
     .version(version);
 });
 
-async function run({ url, auth } = {}) {
+async function run(options) {
   await stream.pipeline(
-    await createFeedStream(url, { fetch: { auth } }),
+    process.stdin,
+    ...feedStreams(options),
     new stream.OutputStream(),
   );
 }

package/package.json

@@ -16,9 +16,9 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.3-beta.2",
+    "@miso.ai/server-commons": "0.6.3-beta.20",
     "feedparser": "^2.2.10",
     "node-fetch": "^3.3.2"
   },
-  "version": "0.6.3-beta.2"
+  "version": "0.6.3-beta.20"
 }

package/src/index.js

@@ -1 +1,2 @@
-export { default as stream } from './stream.js';
+export * from './stream/index.js';
+export { default as version } from './version.js';

package/src/stream/date-filter.js

@@ -0,0 +1,22 @@
+import { Transform } from 'stream';
+
+export default class DateFilterStream extends Transform {
+
+  constructor(threshold) {
+    super({ objectMode: true });
+    this._threshold = threshold;
+  }
+
+  _transform(record, _, next) {
+    try {
+      const timestamp = Date.parse(record.date);
+      if (timestamp >= this._threshold) {
+        this.push(record);
+      }
+    } catch (err) {
+      console.error(err);
+    }
+    next();
+  }
+
+}

package/src/stream/feed.js

@@ -0,0 +1,23 @@
+import { parseDuration, startOfDate } from '@miso.ai/server-commons';
+import FeedParser from 'feedparser';
+import DateFilterStream from './date-filter.js';
+import ArticleTransformStream from './transform.js';
+import IdentityStream from './identity.js';
+
+export default function feedStreams({ parse, after, update, transform } = {}) {
+  const threshold = update ? (Date.now() - parseDuration(update)) : startOfDate(after);
+  const streams = [
+    new FeedParser(parse),
+  ];
+  if (threshold) {
+    streams.push(new DateFilterStream(threshold));
+  }
+  if (transform) {
+    streams.push(new ArticleTransformStream());
+  }
+  if (streams.length === 1) {
+    // because Duplex stream is not async iterable
+    streams.push(new IdentityStream());
+  }
+  return streams;
+}

package/src/stream/identity.js

@@ -0,0 +1,13 @@
+import { Transform } from 'stream';
+
+export default class IdentityStream extends Transform {
+
+  constructor() {
+    super({ objectMode: true });
+  }
+
+  _transform(record, _, next) {
+    next(undefined, record);
+  }
+
+}

package/src/stream/index.js

@@ -0,0 +1,3 @@
+export { default as feedStreams } from './feed.js';
+export { default as ArticleTransformStream } from './transform.js';
+export { default as DateFilterStream } from './date-filter.js';

package/src/stream/transform.js

@@ -0,0 +1,62 @@
+import { trimObj } from '@miso.ai/server-commons';
+import { Transform } from 'stream';
+
+export default class ArticleTransformStream extends Transform {
+
+  constructor({ after } = {}) {
+    super({ objectMode: true });
+    this._after = after;
+  }
+
+  _transform({
+    title,
+    description,
+    summary,
+    date,
+    pubdate,
+    link,
+    origlink,
+    guid,
+    image,
+    author,
+    categories,
+  }, _, next) {
+    if (this._after !== undefined) {
+      try {
+        const timestamp = Date.parse(date);
+        if (timestamp < this._after) {
+          this.end();
+          return;
+        }
+      } catch (err) {
+        console.error(err);
+        this.end();
+        return;
+      }
+    }
+    const authors = author ? [author] : [];
+    title = title || undefined;
+    categories = categories && categories.map(c => [c]) || [];
+    const url = origlink || link || undefined;
+    const cover_image = image && image.url || undefined;
+    summary = summary || undefined;
+    this.push(trimObj({
+      product_id: guid,
+      type: 'article',
+      title,
+      updated_at: date || undefined,
+      published_at: pubdate || undefined,
+      created_at: pubdate || undefined,
+      categories,
+      url,
+      cover_image,
+      authors,
+      html: description || undefined,
+      custom_attributes: trimObj({
+        summary,
+      }),
+    }));
+    next();
+  }
+
+}

package/src/version.js

@@ -1 +1 @@
-export default '0.6.3-beta.2';
+export default '0.6.3-beta.20';

package/src/stream.js

@@ -1,28 +0,0 @@
-import { Buffer } from 'buffer';
-import fetch, { Headers } from 'node-fetch';
-import FeedParser from 'feedparser';
-
-export default async function stream(url, { fetch: fetchOptions, parse: parseOptions } = {}) {
-  const { status, body } = await fetch(url, buildFetchOptions(fetchOptions));
-  if (status !== 200) {
-    throw new Error(`Failed to fetch ${url}: ${status}`);
-  }
-  return body.pipe(new FeedParser(parseOptions));
-}
-
-function buildFetchOptions({ headers, auth, ...options } = {}) {
-  headers = new Headers(headers);
-  if (auth) {
-    if (typeof auth === 'object' && auth.username && auth.password) {
-      auth = `${auth.username}:${auth.password}`;
-    }
-    if (typeof auth !== 'string') {
-      throw new TypeError(`Invalid auth: must me a string or an object.`);
-    }
-    headers.set('Authorization', 'Basic ' + Buffer.from(auth).toString('base64'));
-  }
-  return {
-    ...options,
-    headers,
-  };
-}