@neurowire/cli 0.3.0 → 0.5.0

package/dist/index.js

@@ -1,23 +1,103 @@
 #!/usr/bin/env node
 
 // src/index.ts
-import { readFileSync, writeFileSync } from "fs";
+import { existsSync, readFileSync, writeFileSync } from "fs";
 import { parseArgs } from "util";
 import {
   FORMATS,
   MeshSchema,
+  entryKey,
+  filterEntries,
   isFormat,
+  newEntries,
   parseDuration,
   resolveWindow,
   selectEntries,
-  serialize,
+  serialize as serialize2,
   validateNwf
 } from "@neurowire/core";
-import { FeedTemplateSchema, fetchFeed, fetchMesh } from "@neurowire/ingest";
+import {
+  FeedTemplateSchema,
+  fetchDocument,
+  fetchFeed,
+  fetchMesh,
+  proposeTemplate
+} from "@neurowire/ingest";
 import { registerAllTaps } from "@neurowire/taps";
-var VERSION = "0.3.0";
+
+// src/sinks.ts
+import { serialize } from "@neurowire/core";
+function sinkKind(url) {
+  if (url.includes("slack.com")) return "slack";
+  if (url.includes("discord.com") || url.includes("discordapp.com")) return "discord";
+  return "webhook";
+}
+function buildText(feedTitle, entries, max = 10) {
+  const lines = [`${feedTitle}: ${entries.length} new`];
+  for (const entry of entries.slice(0, max)) {
+    lines.push(`\u2022 ${entry.title} - ${entry.link}`);
+  }
+  if (entries.length > max) {
+    lines.push(`\u2026and ${entries.length - max} more`);
+  }
+  return lines.join("\n");
+}
+function buildSlackBody(feedTitle, entries) {
+  return { text: buildText(feedTitle, entries) };
+}
+function buildDiscordBody(feedTitle, entries) {
+  const content = buildText(feedTitle, entries);
+  return { content: content.length > 2e3 ? content.slice(0, 2e3) : content };
+}
+function buildWebhookBody(feed) {
+  return serialize(feed, "json");
+}
+var hostOf = (url) => {
+  try {
+    return new URL(url).host;
+  } catch {
+    return url;
+  }
+};
+async function deliver(url, feed) {
+  const kind = sinkKind(url);
+  let body;
+  let contentType;
+  if (kind === "slack") {
+    body = JSON.stringify(buildSlackBody(feed.title, feed.entries));
+    contentType = "application/json";
+  } else if (kind === "discord") {
+    body = JSON.stringify(buildDiscordBody(feed.title, feed.entries));
+    contentType = "application/json";
+  } else {
+    body = buildWebhookBody(feed);
+    contentType = "application/feed+json";
+  }
+  try {
+    const res = await fetch(url, {
+      method: "POST",
+      headers: { "content-type": contentType },
+      body
+    });
+    if (!res.ok) {
+      process.stderr.write(`[sink] ${hostOf(url)} failed: ${res.status} ${res.statusText}
+`);
+      return false;
+    }
+    return true;
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : String(error);
+    process.stderr.write(`[sink] ${hostOf(url)} failed: ${reason}
+`);
+    return false;
+  }
+}
+
+// src/index.ts
+var VERSION = "0.5.0";
 var SORT_KEYS = ["date", "title", "source"];
 var SORT_ORDERS = ["asc", "desc"];
+var FILTER_FIELDS = ["title", "summary", "source", "author", "tag"];
 var HELP = `Neurowire ${VERSION} - turn any blog or feed into Atom and friends.
 
 Usage:
@@ -35,6 +115,10 @@ Options:
   -v, --version          Show the version.
 
 Shape the output (applied before --format):
+      --filter <f:p>     Keep entries where field f matches pattern p. Repeatable.
+      --exclude <f:p>    Drop entries where field f matches pattern p. Repeatable.
+                         Pattern is a substring by default, or /regex/ for a regex.
+                         Fields: title, summary, source, author, tag.
       --sort <key>       Sort by date, title, or source.
       --order <dir>      asc or desc (default: newest-first for date, A-Z otherwise).
   -n, --limit <n>        Keep at most n entries. Handy for integrations: --limit 10.
@@ -44,8 +128,19 @@ Shape the output (applied before --format):
       --this-week        Keep entries since Monday midnight UTC.
       --between <a>..<b> Keep entries between two dates, e.g. 2026-01-01..2026-02-01.
 
+Watch a feed or mesh and emit only new entries:
+  -w, --watch            Long-poll on an interval, printing only entries not seen yet.
+      --interval <age>   Poll interval, e.g. 30m, 6h, 1d (default: 5m).
+      --state <file>     JSON file of seen entry keys, so restarts skip old items.
+
+Deliver to sinks (push entries to a destination):
+      --sink <url>       POST entries to a destination. Repeatable. Slack, Discord,
+                         or a generic webhook, auto-detected by URL. With --watch,
+                         only the new entries are delivered each tick.
+
 Commands:
   validate <file-or-url> Check that an nwf document is well-formed (exits non-zero if not).
+  tap doctor <url>       Propose a FeedTemplate (tap) for a feed-less page.
 
 A mesh bundles many sources into one feed:
   { "name": "AI News", "sources": [{ "name": "...", "url": "..." }] }
@@ -59,7 +154,11 @@ Examples:
   neurowire https://example.com/feed.xml --format atom > feed.xml
   neurowire --mesh ai-news.json --format json --limit 10
   neurowire --mesh ai-news.json --since 24h --sort date --format atom
+  neurowire --mesh ai-news.json --filter tag:release --exclude title:sponsored --format json
+  neurowire --mesh ai-news.json --watch --interval 15m --format json
+  neurowire --mesh ai-news.json --watch --sink https://hooks.slack.com/services/...
   neurowire validate feed.nwf
+  neurowire tap doctor https://example.com/blog > ~/.config/neurowire/taps/example.com.json
 `;
 var useColor = Boolean(process.stdout.isTTY) && !process.env.NO_COLOR;
 var paint = (code) => (s) => useColor ? `\x1B[${code}m${s}\x1B[0m` : s;
@@ -115,6 +214,82 @@ async function runValidate(input) {
     process.exitCode = 1;
   }
 }
+async function runTapDoctor(url) {
+  if (!url) {
+    process.stderr.write("error: tap doctor needs a url\n\nUsage: neurowire tap doctor <url>\n");
+    process.exitCode = 1;
+    return;
+  }
+  const doc = await fetchDocument(url);
+  const proposal = proposeTemplate(doc.body, doc.url);
+  if (!proposal) {
+    process.stderr.write(`${red("error")}: could not propose a template for ${url}
+`);
+    process.exitCode = 1;
+    return;
+  }
+  process.stdout.write(`${JSON.stringify(proposal.template, null, 2)}
+`);
+  process.stderr.write(
+    `${green("\u2713")} matched ${proposal.matched} entr${proposal.matched === 1 ? "y" : "ies"}
+`
+  );
+  for (const title of proposal.sampleTitles) {
+    process.stderr.write(`  ${dim("\xB7")} ${title}
+`);
+  }
+  const host = proposal.template.host ?? "host";
+  process.stderr.write(
+    dim(`# save this as ~/.config/neurowire/taps/${host}.json or pass with --taps
+`)
+  );
+}
+function parseFilterRule(value) {
+  const colon = value.indexOf(":");
+  const field = colon === -1 ? value : value.slice(0, colon);
+  if (!FILTER_FIELDS.includes(field)) return void 0;
+  let pattern = colon === -1 ? "" : value.slice(colon + 1);
+  let regex = false;
+  if (pattern.length >= 2 && pattern.startsWith("/") && pattern.endsWith("/")) {
+    pattern = pattern.slice(1, -1);
+    regex = true;
+  }
+  return { field, pattern, regex };
+}
+function applyFilters(feed, values) {
+  const fail = (raw) => {
+    process.stderr.write(
+      `error: bad filter "${raw}". Use field:pattern with one of: ${FILTER_FIELDS.join(", ")}
+`
+    );
+    process.exitCode = 1;
+    return void 0;
+  };
+  const collect = (raw) => {
+    const rules = [];
+    for (const value of raw) {
+      const rule = parseFilterRule(value);
+      if (!rule) return fail(value);
+      rules.push(rule);
+    }
+    return rules;
+  };
+  const filterRaw = values.filter ?? [];
+  const excludeRaw = values.exclude ?? [];
+  if (filterRaw.length === 0 && excludeRaw.length === 0) return feed;
+  const include = collect(filterRaw);
+  if (!include) return void 0;
+  const exclude = collect(excludeRaw);
+  if (!exclude) return void 0;
+  const spec = { include, exclude };
+  const before = feed.entries.length;
+  const filtered = filterEntries(feed, spec);
+  if (filtered.entries.length !== before) {
+    process.stderr.write(`Filtered to ${filtered.entries.length} of ${before} entries
+`);
+  }
+  return filtered;
+}
 function refineFeed(feed, values) {
   const fail = (message) => {
     process.stderr.write(`error: ${message}
@@ -200,6 +375,80 @@ function renderTerminal(feed) {
   process.stdout.write(`${out.join("\n")}
 `);
 }
+async function loadFeed(values, positionals) {
+  if (typeof values.mesh === "string") {
+    const mesh = MeshSchema.parse(JSON.parse(readFileSync(values.mesh, "utf8")));
+    return fetchMesh(mesh);
+  }
+  const url = positionals[0];
+  if (!url) {
+    process.stderr.write("error: missing <url> (or use --mesh <file>)\n\n");
+    process.stderr.write(HELP);
+    process.exitCode = 1;
+    return void 0;
+  }
+  let template;
+  if (typeof values.template === "string") {
+    template = FeedTemplateSchema.parse(JSON.parse(readFileSync(values.template, "utf8")));
+  }
+  return fetchFeed(url, { template });
+}
+function emitFeed(feed, values) {
+  if (typeof values.format === "string") {
+    if (!isFormat(values.format)) {
+      process.stderr.write(
+        `error: unknown format "${values.format}". Use one of: ${FORMATS.join(", ")}
+`
+      );
+      process.exitCode = 1;
+      return false;
+    }
+    process.stdout.write(serialize2(feed, values.format));
+    return true;
+  }
+  renderTerminal(feed);
+  return true;
+}
+async function deliverToSinks(feed, values) {
+  if (feed.entries.length === 0) return;
+  const sinks = values.sink ?? [];
+  for (const url of sinks) {
+    await deliver(url, feed);
+  }
+}
+function loadSeenState(path) {
+  if (!existsSync(path)) return [];
+  return JSON.parse(readFileSync(path, "utf8"));
+}
+async function runWatch(values, positionals) {
+  const intervalMs = parseDuration(values.interval ?? "5m");
+  if (intervalMs === void 0) {
+    process.stderr.write(
+      `error: invalid --interval "${values.interval}" (use e.g. 30m, 6h, 1d)
+`
+    );
+    process.exitCode = 1;
+    return;
+  }
+  const statePath = values.state;
+  const seen = new Set(statePath ? loadSeenState(statePath) : []);
+  for (; ; ) {
+    const feed = await loadFeed(values, positionals);
+    if (!feed) return;
+    const filtered = applyFilters(feed, values);
+    if (!filtered) return;
+    const refined = refineFeed(filtered, values);
+    if (!refined) return;
+    const fresh = newEntries(refined, seen);
+    if (fresh.length > 0) emitFeed({ ...refined, entries: fresh }, values);
+    await deliverToSinks({ ...refined, entries: fresh }, values);
+    for (const entry of fresh) seen.add(entryKey(entry));
+    if (statePath) writeFileSync(statePath, JSON.stringify([...seen]));
+    process.stderr.write(`[watch] ${fresh.length} new (${seen.size} seen)
+`);
+    await new Promise((resolve) => setTimeout(resolve, intervalMs));
+  }
+}
 async function main() {
   const argv = process.argv.slice(2);
   const args = argv[0] === "--" ? argv.slice(1) : argv;
@@ -212,6 +461,8 @@ async function main() {
       template: { type: "string", short: "t" },
       mesh: { type: "string", short: "m" },
       taps: { type: "string", multiple: true },
+      filter: { type: "string", multiple: true },
+      exclude: { type: "string", multiple: true },
       sort: { type: "string" },
       order: { type: "string" },
       limit: { type: "string", short: "n" },
@@ -220,6 +471,10 @@ async function main() {
       today: { type: "boolean" },
       "this-week": { type: "boolean" },
       between: { type: "string" },
+      watch: { type: "boolean", short: "w" },
+      interval: { type: "string" },
+      state: { type: "string" },
+      sink: { type: "string", multiple: true },
       help: { type: "boolean", short: "h" },
       version: { type: "boolean", short: "v" }
     }
@@ -237,30 +492,27 @@ async function main() {
     await runValidate(positionals[1]);
     return;
   }
+  if (positionals[0] === "tap" && positionals[1] === "doctor") {
+    await runTapDoctor(positionals[2]);
+    return;
+  }
+  if (positionals[0] === "doctor") {
+    await runTapDoctor(positionals[1]);
+    return;
+  }
   const { user } = registerAllTaps(values.taps ?? []);
   if (user.length) process.stderr.write(`Loaded ${user.length} custom tap(s)
 `);
-  let feed;
-  if (values.mesh) {
-    const mesh = MeshSchema.parse(JSON.parse(readFileSync(values.mesh, "utf8")));
-    feed = await fetchMesh(mesh);
-  } else {
-    const url = positionals[0];
-    if (!url) {
-      process.stderr.write("error: missing <url> (or use --mesh <file>)\n\n");
-      process.stderr.write(HELP);
-      process.exitCode = 1;
-      return;
-    }
-    let template;
-    if (values.template) {
-      template = FeedTemplateSchema.parse(JSON.parse(readFileSync(values.template, "utf8")));
-    }
-    feed = await fetchFeed(url, { template });
+  if (values.watch) {
+    await runWatch(values, positionals);
+    return;
   }
-  const refined = refineFeed(feed, values);
-  if (!refined) return;
-  feed = refined;
+  const loaded = await loadFeed(values, positionals);
+  if (!loaded) return;
+  const filtered = applyFilters(loaded, values);
+  if (!filtered) return;
+  const feed = refineFeed(filtered, values);
+  if (!feed) return;
   if (values.format) {
     if (!isFormat(values.format)) {
       process.stderr.write(
@@ -270,7 +522,7 @@ async function main() {
       process.exitCode = 1;
       return;
     }
-    const output = serialize(feed, values.format);
+    const output = serialize2(feed, values.format);
     if (values.out) {
       writeFileSync(values.out, output);
       process.stderr.write(`Wrote ${feed.entries.length} entries to ${values.out}
@@ -278,9 +530,11 @@ async function main() {
     } else {
       process.stdout.write(output);
     }
+    await deliverToSinks(feed, values);
     return;
   }
   renderTerminal(feed);
+  await deliverToSinks(feed, values);
 }
 main().catch((error) => {
   process.stderr.write(`error: ${error instanceof Error ? error.message : String(error)}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@neurowire/cli",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "Neurowire command-line tool: turn any blog, feed, or mesh into Atom, JSON Feed, Markdown, or nwf.",
   "type": "module",
   "bin": {
@@ -39,9 +39,9 @@
     "access": "public"
   },
   "dependencies": {
-    "@neurowire/ingest": "0.2.0",
-    "@neurowire/core": "0.3.0",
-    "@neurowire/taps": "0.2.0"
+    "@neurowire/ingest": "0.4.0",
+    "@neurowire/taps": "0.2.0",
+    "@neurowire/core": "0.5.0"
   },
   "devDependencies": {
     "@types/node": "^22.10.5",