@miso.ai/server-feed 0.6.3-beta.6 → 0.6.3-beta.8

package/cli/index.js

@@ -1,16 +1,10 @@
 #!/usr/bin/env node
 import { yargs, stream } from '@miso.ai/server-commons';
-import { version, feedStream } from '../src/index.js';
+import { version, feedStreams } from '../src/index.js';
 
 yargs.build(yargs => {
   yargs
     .env('MISO_FEED')
-    .option('url', {
-      type: 'string',
-    })
-    .option('auth', {
-      type: 'string',
-    })
     .option('after', {
       alias: 'a',
       describe: 'Only include records after this time',
@@ -30,16 +24,17 @@ yargs.build(yargs => {
     })
     .hide('debug')
     .command({
-      command: '* [url]',
-      description: 'Read items from feed',
+      command: '*',
+      description: 'Parse items from feed content',
       handler: run,
     })
     .version(version);
 });
 
-async function run({ url, auth, after, update, transform } = {}) {
+async function run(options) {
   await stream.pipeline(
-    await feedStream(url, { fetch: { auth }, after, update, transform }),
+    process.stdin,
+    ...feedStreams(options),
     new stream.OutputStream(),
   );
 }

package/package.json

@@ -16,9 +16,9 @@
     "simonpai <simon.pai@askmiso.com>"
   ],
   "dependencies": {
-    "@miso.ai/server-commons": "0.6.3-beta.6",
+    "@miso.ai/server-commons": "0.6.3-beta.8",
     "feedparser": "^2.2.10",
     "node-fetch": "^3.3.2"
   },
-  "version": "0.6.3-beta.6"
+  "version": "0.6.3-beta.8"
 }

package/src/stream/feed.js

@@ -1,16 +1,18 @@
 import { parseDuration, startOfDate } from '@miso.ai/server-commons';
-import rawFeedStream from './raw.js';
+import FeedParser from 'feedparser';
 import DateFilterStream from './date-filter.js';
 import ArticleTransformStream from './transform.js';
 
-export default async function feedStream(url, { fetch, parse, after, update, transform } = {}) {
-  let stream = await rawFeedStream(url, { fetch, parse });
+export default function feedStreams({ parse, after, update, transform } = {}) {
   const threshold = update ? (Date.now() - parseDuration(update)) : startOfDate(after);
+  const streams = [
+    new FeedParser(parse),
+  ];
   if (threshold) {
-    stream = stream.pipe(new DateFilterStream(threshold));
+    streams.push(new DateFilterStream(threshold));
   }
   if (transform) {
-    stream = stream.pipe(new ArticleTransformStream());
+    streams.push(new ArticleTransformStream());
   }
-  return stream;
+  return streams;
 }

package/src/stream/index.js

@@ -1,4 +1,3 @@
-export { default as feedStream } from './feed.js';
-export { default as rawFeedStream } from './raw.js';
+export { default as feedStreams } from './feed.js';
 export { default as ArticleTransformStream } from './transform.js';
 export { default as DateFilterStream } from './date-filter.js';

package/src/version.js

@@ -1 +1 @@
-export default '0.6.3-beta.6';
+export default '0.6.3-beta.8';

package/src/stream/raw.js

@@ -1,28 +0,0 @@
-import { Buffer } from 'buffer';
-import fetch, { Headers } from 'node-fetch';
-import FeedParser from 'feedparser';
-
-export default async function rawFeedStream(url, { fetch: fetchOptions, parse: parseOptions } = {}) {
-  const { status, body } = await fetch(url, buildFetchOptions(fetchOptions));
-  if (status !== 200) {
-    throw new Error(`Failed to fetch ${url}: ${status}`);
-  }
-  return body.pipe(new FeedParser(parseOptions));
-}
-
-function buildFetchOptions({ headers, auth, ...options } = {}) {
-  headers = new Headers(headers);
-  if (auth) {
-    if (typeof auth === 'object' && auth.username && auth.password) {
-      auth = `${auth.username}:${auth.password}`;
-    }
-    if (typeof auth !== 'string') {
-      throw new TypeError(`Invalid auth: must me a string or an object.`);
-    }
-    headers.set('Authorization', 'Basic ' + Buffer.from(auth).toString('base64'));
-  }
-  return {
-    ...options,
-    headers,
-  };
-}