@neurowire/cli 0.1.0 → 0.4.0

package/dist/index.js

@@ -1,18 +1,33 @@
 #!/usr/bin/env node
 
 // src/index.ts
-import { readFileSync, writeFileSync } from "fs";
+import { existsSync, readFileSync, writeFileSync } from "fs";
 import { parseArgs } from "util";
 import {
   FORMATS,
   MeshSchema,
+  entryKey,
+  filterEntries,
   isFormat,
+  newEntries,
+  parseDuration,
+  resolveWindow,
+  selectEntries,
   serialize,
   validateNwf
 } from "@neurowire/core";
-import { FeedTemplateSchema, fetchFeed, fetchMesh } from "@neurowire/ingest";
+import {
+  FeedTemplateSchema,
+  fetchDocument,
+  fetchFeed,
+  fetchMesh,
+  proposeTemplate
+} from "@neurowire/ingest";
 import { registerAllTaps } from "@neurowire/taps";
-var VERSION = "0.1.0";
+var VERSION = "0.4.0";
+var SORT_KEYS = ["date", "title", "source"];
+var SORT_ORDERS = ["asc", "desc"];
+var FILTER_FIELDS = ["title", "summary", "source", "author", "tag"];
 var HELP = `Neurowire ${VERSION} - turn any blog or feed into Atom and friends.
 
 Usage:
@@ -29,8 +44,28 @@ Options:
   -h, --help             Show this help.
   -v, --version          Show the version.
 
+Shape the output (applied before --format):
+      --filter <f:p>     Keep entries where field f matches pattern p. Repeatable.
+      --exclude <f:p>    Drop entries where field f matches pattern p. Repeatable.
+                         Pattern is a substring by default, or /regex/ for a regex.
+                         Fields: title, summary, source, author, tag.
+      --sort <key>       Sort by date, title, or source.
+      --order <dir>      asc or desc (default: newest-first for date, A-Z otherwise).
+  -n, --limit <n>        Keep at most n entries. Handy for integrations: --limit 10.
+      --since <age>      Keep entries within this window, e.g. 24h, 90m, 7d.
+      --max-age <age>    Drop entries older than this (same window as --since).
+      --today            Keep entries since midnight UTC today.
+      --this-week        Keep entries since Monday midnight UTC.
+      --between <a>..<b> Keep entries between two dates, e.g. 2026-01-01..2026-02-01.
+
+Watch a feed or mesh and emit only new entries:
+  -w, --watch            Long-poll on an interval, printing only entries not seen yet.
+      --interval <age>   Poll interval, e.g. 30m, 6h, 1d (default: 5m).
+      --state <file>     JSON file of seen entry keys, so restarts skip old items.
+
 Commands:
   validate <file-or-url> Check that an nwf document is well-formed (exits non-zero if not).
+  tap doctor <url>       Propose a FeedTemplate (tap) for a feed-less page.
 
 A mesh bundles many sources into one feed:
   { "name": "AI News", "sources": [{ "name": "...", "url": "..." }] }
@@ -42,8 +77,12 @@ Taps teach Neurowire to read sites with no RSS/Atom feed. Add your own with
 Examples:
   neurowire https://example.com/blog
   neurowire https://example.com/feed.xml --format atom > feed.xml
-  neurowire --mesh ai-news.json --format atom
+  neurowire --mesh ai-news.json --format json --limit 10
+  neurowire --mesh ai-news.json --since 24h --sort date --format atom
+  neurowire --mesh ai-news.json --filter tag:release --exclude title:sponsored --format json
+  neurowire --mesh ai-news.json --watch --interval 15m --format json
   neurowire validate feed.nwf
+  neurowire tap doctor https://example.com/blog > ~/.config/neurowire/taps/example.com.json
 `;
 var useColor = Boolean(process.stdout.isTTY) && !process.env.NO_COLOR;
 var paint = (code) => (s) => useColor ? `\x1B[${code}m${s}\x1B[0m` : s;
@@ -99,6 +138,145 @@ async function runValidate(input) {
     process.exitCode = 1;
   }
 }
+async function runTapDoctor(url) {
+  if (!url) {
+    process.stderr.write("error: tap doctor needs a url\n\nUsage: neurowire tap doctor <url>\n");
+    process.exitCode = 1;
+    return;
+  }
+  const doc = await fetchDocument(url);
+  const proposal = proposeTemplate(doc.body, doc.url);
+  if (!proposal) {
+    process.stderr.write(`${red("error")}: could not propose a template for ${url}
+`);
+    process.exitCode = 1;
+    return;
+  }
+  process.stdout.write(`${JSON.stringify(proposal.template, null, 2)}
+`);
+  process.stderr.write(
+    `${green("\u2713")} matched ${proposal.matched} entr${proposal.matched === 1 ? "y" : "ies"}
+`
+  );
+  for (const title of proposal.sampleTitles) {
+    process.stderr.write(`  ${dim("\xB7")} ${title}
+`);
+  }
+  const host = proposal.template.host ?? "host";
+  process.stderr.write(
+    dim(`# save this as ~/.config/neurowire/taps/${host}.json or pass with --taps
+`)
+  );
+}
+function parseFilterRule(value) {
+  const colon = value.indexOf(":");
+  const field = colon === -1 ? value : value.slice(0, colon);
+  if (!FILTER_FIELDS.includes(field)) return void 0;
+  let pattern = colon === -1 ? "" : value.slice(colon + 1);
+  let regex = false;
+  if (pattern.length >= 2 && pattern.startsWith("/") && pattern.endsWith("/")) {
+    pattern = pattern.slice(1, -1);
+    regex = true;
+  }
+  return { field, pattern, regex };
+}
+function applyFilters(feed, values) {
+  const fail = (raw) => {
+    process.stderr.write(
+      `error: bad filter "${raw}". Use field:pattern with one of: ${FILTER_FIELDS.join(", ")}
+`
+    );
+    process.exitCode = 1;
+    return void 0;
+  };
+  const collect = (raw) => {
+    const rules = [];
+    for (const value of raw) {
+      const rule = parseFilterRule(value);
+      if (!rule) return fail(value);
+      rules.push(rule);
+    }
+    return rules;
+  };
+  const filterRaw = values.filter ?? [];
+  const excludeRaw = values.exclude ?? [];
+  if (filterRaw.length === 0 && excludeRaw.length === 0) return feed;
+  const include = collect(filterRaw);
+  if (!include) return void 0;
+  const exclude = collect(excludeRaw);
+  if (!exclude) return void 0;
+  const spec = { include, exclude };
+  const before = feed.entries.length;
+  const filtered = filterEntries(feed, spec);
+  if (filtered.entries.length !== before) {
+    process.stderr.write(`Filtered to ${filtered.entries.length} of ${before} entries
+`);
+  }
+  return filtered;
+}
+function refineFeed(feed, values) {
+  const fail = (message) => {
+    process.stderr.write(`error: ${message}
+`);
+    process.exitCode = 1;
+    return void 0;
+  };
+  const sort = values.sort;
+  if (sort !== void 0 && !SORT_KEYS.includes(sort)) {
+    return fail(`unknown --sort "${sort}". Use one of: ${SORT_KEYS.join(", ")}`);
+  }
+  const order = values.order;
+  if (order !== void 0 && !SORT_ORDERS.includes(order)) {
+    return fail(`unknown --order "${order}". Use asc or desc`);
+  }
+  let limit;
+  if (values.limit !== void 0) {
+    limit = Number(values.limit);
+    if (!Number.isInteger(limit) || limit < 0) {
+      return fail(`--limit must be a non-negative integer (got "${values.limit}")`);
+    }
+  }
+  const spec = {};
+  if (typeof values.since === "string") spec.since = values.since;
+  if (typeof values["max-age"] === "string") spec.maxAge = values["max-age"];
+  if (values.today) spec.today = true;
+  if (values["this-week"]) spec.thisWeek = true;
+  if (typeof values.between === "string") {
+    const parts = values.between.split("..");
+    if (parts.length !== 2 || !parts[0] || !parts[1]) {
+      return fail("--between expects <start>..<end>, e.g. 2026-01-01..2026-02-01");
+    }
+    spec.between = [parts[0], parts[1]];
+  }
+  for (const [flag, value] of [
+    ["--since", spec.since],
+    ["--max-age", spec.maxAge]
+  ]) {
+    if (value !== void 0 && parseDuration(value) === void 0) {
+      return fail(`invalid ${flag} "${value}" (use e.g. 24h, 90m, 7d)`);
+    }
+  }
+  if (spec.between) {
+    const [a, b] = spec.between;
+    if (Number.isNaN(Date.parse(a)) || Number.isNaN(Date.parse(b))) {
+      return fail("--between needs two parseable dates, e.g. 2026-01-01..2026-02-01");
+    }
+  }
+  const window = resolveWindow(spec, Date.now());
+  const opts = {
+    ...window,
+    sort,
+    order,
+    limit
+  };
+  const before = feed.entries.length;
+  const refined = selectEntries(feed, opts);
+  if (refined.entries.length !== before) {
+    process.stderr.write(`Refined to ${refined.entries.length} of ${before} entries
+`);
+  }
+  return refined;
+}
 function renderTerminal(feed) {
   const out = [bold(cyan(feed.title))];
   const sub = [feed.home, `${feed.entries.length} entries`, `updated ${day(feed.updated)}`].filter(
@@ -121,6 +299,72 @@ function renderTerminal(feed) {
   process.stdout.write(`${out.join("\n")}
 `);
 }
+async function loadFeed(values, positionals) {
+  if (typeof values.mesh === "string") {
+    const mesh = MeshSchema.parse(JSON.parse(readFileSync(values.mesh, "utf8")));
+    return fetchMesh(mesh);
+  }
+  const url = positionals[0];
+  if (!url) {
+    process.stderr.write("error: missing <url> (or use --mesh <file>)\n\n");
+    process.stderr.write(HELP);
+    process.exitCode = 1;
+    return void 0;
+  }
+  let template;
+  if (typeof values.template === "string") {
+    template = FeedTemplateSchema.parse(JSON.parse(readFileSync(values.template, "utf8")));
+  }
+  return fetchFeed(url, { template });
+}
+function emitFeed(feed, values) {
+  if (typeof values.format === "string") {
+    if (!isFormat(values.format)) {
+      process.stderr.write(
+        `error: unknown format "${values.format}". Use one of: ${FORMATS.join(", ")}
+`
+      );
+      process.exitCode = 1;
+      return false;
+    }
+    process.stdout.write(serialize(feed, values.format));
+    return true;
+  }
+  renderTerminal(feed);
+  return true;
+}
+function loadSeenState(path) {
+  if (!existsSync(path)) return [];
+  return JSON.parse(readFileSync(path, "utf8"));
+}
+async function runWatch(values, positionals) {
+  const intervalMs = parseDuration(values.interval ?? "5m");
+  if (intervalMs === void 0) {
+    process.stderr.write(
+      `error: invalid --interval "${values.interval}" (use e.g. 30m, 6h, 1d)
+`
+    );
+    process.exitCode = 1;
+    return;
+  }
+  const statePath = values.state;
+  const seen = new Set(statePath ? loadSeenState(statePath) : []);
+  for (; ; ) {
+    const feed = await loadFeed(values, positionals);
+    if (!feed) return;
+    const filtered = applyFilters(feed, values);
+    if (!filtered) return;
+    const refined = refineFeed(filtered, values);
+    if (!refined) return;
+    const fresh = newEntries(refined, seen);
+    if (fresh.length > 0) emitFeed({ ...refined, entries: fresh }, values);
+    for (const entry of fresh) seen.add(entryKey(entry));
+    if (statePath) writeFileSync(statePath, JSON.stringify([...seen]));
+    process.stderr.write(`[watch] ${fresh.length} new (${seen.size} seen)
+`);
+    await new Promise((resolve) => setTimeout(resolve, intervalMs));
+  }
+}
 async function main() {
   const argv = process.argv.slice(2);
   const args = argv[0] === "--" ? argv.slice(1) : argv;
@@ -133,6 +377,19 @@ async function main() {
       template: { type: "string", short: "t" },
       mesh: { type: "string", short: "m" },
       taps: { type: "string", multiple: true },
+      filter: { type: "string", multiple: true },
+      exclude: { type: "string", multiple: true },
+      sort: { type: "string" },
+      order: { type: "string" },
+      limit: { type: "string", short: "n" },
+      since: { type: "string" },
+      "max-age": { type: "string" },
+      today: { type: "boolean" },
+      "this-week": { type: "boolean" },
+      between: { type: "string" },
+      watch: { type: "boolean", short: "w" },
+      interval: { type: "string" },
+      state: { type: "string" },
       help: { type: "boolean", short: "h" },
       version: { type: "boolean", short: "v" }
     }
@@ -150,27 +407,27 @@ async function main() {
     await runValidate(positionals[1]);
     return;
   }
+  if (positionals[0] === "tap" && positionals[1] === "doctor") {
+    await runTapDoctor(positionals[2]);
+    return;
+  }
+  if (positionals[0] === "doctor") {
+    await runTapDoctor(positionals[1]);
+    return;
+  }
   const { user } = registerAllTaps(values.taps ?? []);
   if (user.length) process.stderr.write(`Loaded ${user.length} custom tap(s)
 `);
-  let feed;
-  if (values.mesh) {
-    const mesh = MeshSchema.parse(JSON.parse(readFileSync(values.mesh, "utf8")));
-    feed = await fetchMesh(mesh);
-  } else {
-    const url = positionals[0];
-    if (!url) {
-      process.stderr.write("error: missing <url> (or use --mesh <file>)\n\n");
-      process.stderr.write(HELP);
-      process.exitCode = 1;
-      return;
-    }
-    let template;
-    if (values.template) {
-      template = FeedTemplateSchema.parse(JSON.parse(readFileSync(values.template, "utf8")));
-    }
-    feed = await fetchFeed(url, { template });
+  if (values.watch) {
+    await runWatch(values, positionals);
+    return;
   }
+  const loaded = await loadFeed(values, positionals);
+  if (!loaded) return;
+  const filtered = applyFilters(loaded, values);
+  if (!filtered) return;
+  const feed = refineFeed(filtered, values);
+  if (!feed) return;
   if (values.format) {
     if (!isFormat(values.format)) {
       process.stderr.write(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@neurowire/cli",
-  "version": "0.1.0",
+  "version": "0.4.0",
   "description": "Neurowire command-line tool: turn any blog, feed, or mesh into Atom, JSON Feed, Markdown, or nwf.",
   "type": "module",
   "bin": {
@@ -33,15 +33,15 @@
     "url": "https://github.com/starside-io/neurowire/issues"
   },
   "engines": {
-    "node": ">=20"
+    "node": ">=24"
   },
   "publishConfig": {
     "access": "public"
   },
   "dependencies": {
-    "@neurowire/ingest": "0.1.0",
-    "@neurowire/core": "0.1.0",
-    "@neurowire/taps": "0.1.0"
+    "@neurowire/core": "0.4.0",
+    "@neurowire/ingest": "0.3.0",
+    "@neurowire/taps": "0.2.0"
   },
   "devDependencies": {
     "@types/node": "^22.10.5",